device_memory.cpp

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#include "device_memory.h"
 
#include <entropic/types/logging.h>
 
#include <ggml-backend.h>
 
#include "partial_offload.h"   // kMemlockUnlimited
 
#include <sys/resource.h>
 
#include <atomic>
 
namespace {
auto logger = entropic::log::get("inference.device_memory");
 
std::atomic<uint64_t> s_query_count{0};
}  // namespace
 
namespace entropic {
 
uint64_t query_device_free_vram_bytes() {
    s_query_count.fetch_add(1, std::memory_order_relaxed);
    ggml_backend_dev_t device =
        ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU);
    if (device == nullptr) {
        logger->info("[residency] no GPU device registered — "
                     "VRAM budget unknown, admission gate disabled");
        return 0;
    }
    size_t free_bytes = 0;
    size_t total_bytes = 0;
    ggml_backend_dev_memory(device, &free_bytes, &total_bytes);
    logger->info("[residency] device '{}' reports {} MiB free of {} MiB total",
                 ggml_backend_dev_name(device),
                 free_bytes / (1024 * 1024), total_bytes / (1024 * 1024));
    return static_cast<uint64_t>(free_bytes);
}
 
uint64_t host_memlock_limit_bytes() {
    struct rlimit rl {};
    if (getrlimit(RLIMIT_MEMLOCK, &rl) != 0
        || rl.rlim_cur == RLIM_INFINITY) {
        return kMemlockUnlimited;
    }
    return static_cast<uint64_t>(rl.rlim_cur);
}
 
uint64_t device_memory_query_count() {
    return s_query_count.load(std::memory_order_relaxed);
}
 
}  // namespace entropic