device_memory.cpp

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#include "device_memory.h"
 
#include <entropic/types/logging.h>
 
#include <ggml-backend.h>
 
namespace {
auto logger = entropic::log::get("inference.device_memory");
}  // namespace
 
namespace entropic {
 
uint64_t query_device_free_vram_bytes() {
    ggml_backend_dev_t device =
        ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU);
    if (device == nullptr) {
        logger->info("[residency] no GPU device registered — "
                     "VRAM budget unknown, admission gate disabled");
        return 0;
    }
    size_t free_bytes = 0;
    size_t total_bytes = 0;
    ggml_backend_dev_memory(device, &free_bytes, &total_bytes);
    logger->info("[residency] device '{}' reports {} MiB free of {} MiB total",
                 ggml_backend_dev_name(device),
                 free_bytes / (1024 * 1024), total_bytes / (1024 * 1024));
    return static_cast<uint64_t>(free_bytes);
}
 
}  // namespace entropic