backend.cpp

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#include <entropic/inference/backend.h>
#include <entropic/types/logging.h>
 
#include <cmath>
#include <cstdlib>
#include <stdexcept>
#include <string>
 
namespace entropic {
 
namespace {
 
auto logger = entropic::log::get("inference.backend");
 
const char* state_name(ModelState s) {
    static constexpr const char* names[] = {"COLD", "WARM", "ACTIVE"};
    int idx = static_cast<int>(s);
    return (idx >= 0 && idx <= 2) ? names[idx] : "UNKNOWN";
}
 
} // anonymous namespace
 
// ── Lifecycle ──────────────────────────────────────────────
 
bool InferenceBackend::load(const ModelConfig& config) {
    std::lock_guard<std::mutex> lock(transition_mutex_);
 
    if (state() != ModelState::COLD) {
        logger->info("[VRAM] load() no-op: already {}", state_name(state()));
        return true;
    }
 
    // Hook: ON_MODEL_LOAD — can cancel (v1.9.1)
    bool cancelled = fire_model_load_hook(config);
    if (cancelled) {
        return false;
    }
 
    logger->info("[VRAM] Loading: {}", config.path.string());
    auto start = entropic::log::now();
 
    config_ = config;
    bool ok = do_load(config);
    if (!ok) {
        logger->error("[VRAM] Load failed: {}", last_error_);
    } else {
        state_.store(ModelState::WARM, std::memory_order_release);
        logger->info("[VRAM] Warm in {:.2f}ms", entropic::log::elapsed_ms(start, entropic::log::now()));
    }
    return ok;
}
 
bool InferenceBackend::activate() {
    std::lock_guard<std::mutex> lock(transition_mutex_);
    return promote_to_active();
}
 
bool InferenceBackend::promote_to_active() {
    if (state() == ModelState::ACTIVE) {
        logger->info("[VRAM] activate() no-op: already ACTIVE");
        return true;
    }
    if (state() != ModelState::WARM) {
        logger->error("[VRAM] activate() failed: not WARM ({})", state_name(state()));
        return false;
    }
 
    logger->info("[VRAM] Activating");
    auto start = entropic::log::now();
    bool ok = do_activate();
    if (!ok) {
        logger->error("[VRAM] Activate failed: {}", last_error_);
    } else {
        state_.store(ModelState::ACTIVE, std::memory_order_release);
        logger->info("[VRAM] Active in {:.2f}ms", entropic::log::elapsed_ms(start, entropic::log::now()));
    }
    return ok;
}
 
void InferenceBackend::deactivate() {
    std::lock_guard<std::mutex> lock(transition_mutex_);
 
    if (state() != ModelState::ACTIVE) {
        logger->info("[VRAM] deactivate() no-op: {}", state_name(state()));
        return;
    }
 
    logger->info("[VRAM] Deactivating");
    auto start = entropic::log::now();
 
    do_deactivate();
    state_.store(ModelState::WARM, std::memory_order_release);
 
    logger->info("[VRAM] Deactivated in {:.2f}ms", entropic::log::elapsed_ms(start, entropic::log::now()));
}
 
void InferenceBackend::unload() {
    std::lock_guard<std::mutex> lock(transition_mutex_);
 
    // Hook: ON_MODEL_UNLOAD — informational (v1.9.1)
    if (hooks_.fire_info != nullptr) {
        std::string json = "{\"state\":\""
            + std::string(state_name(state())) + "\"}";
        hooks_.fire_info(hooks_.registry,
            ENTROPIC_HOOK_ON_MODEL_UNLOAD, json.c_str());
    }
 
    logger->info("[VRAM] Unloading from {}", state_name(state()));
 
    do_unload();
    state_.store(ModelState::COLD, std::memory_order_release);
 
    logger->info("[VRAM] Unloaded");
}
 
bool InferenceBackend::load_and_activate(const ModelConfig& config) {
    std::lock_guard<std::mutex> lock(transition_mutex_);
 
    if (state() != ModelState::COLD) { return promote_to_active(); }
 
    // Hook: ON_MODEL_LOAD — can cancel (v1.9.1). Same contract as load().
    if (fire_model_load_hook(config)) { return false; }
 
    logger->info("[VRAM] Loading into target residency: {} (gpu_layers={})",
                 config.path.string(), config.gpu_layers);
    auto start = entropic::log::now();
 
    config_ = config;
    bool ok = do_load_active(config);
    if (!ok) {
        logger->error("[VRAM] Load+activate failed: {}", last_error_);
    } else {
        state_.store(ModelState::ACTIVE, std::memory_order_release);
        logger->info("[VRAM] Active in {:.2f}ms (one file read)",
                     entropic::log::elapsed_ms(start, entropic::log::now()));
    }
    return ok;
}
 
// ── Generation ─────────────────────────────────────────────
 
GenerationResult InferenceBackend::generate(
    const std::vector<Message>& messages,
    const GenerationParams& params)
{
    if (!is_active()) {
        GenerationResult err;
        err.error_code = ENTROPIC_ERROR_INVALID_STATE;
        err.error_message = "generate() requires ACTIVE state";
        err.finish_reason = "error";
        logger->error("{}", err.error_message);
        return err;
    }
 
    auto start = entropic::log::now();
    auto result = do_generate(messages, params);
    result.generation_time_ms = entropic::log::elapsed_ms(start, entropic::log::now());
    return result;
}
 
GenerationResult InferenceBackend::generate(
    const std::vector<Message>& messages,
    const GenerationParams& params,
    std::atomic<bool>& cancel)
{
    if (!is_active()) {
        GenerationResult err;
        err.error_code = ENTROPIC_ERROR_INVALID_STATE;
        err.error_message = "generate() requires ACTIVE state";
        err.finish_reason = "error";
        logger->error("{}", err.error_message);
        return err;
    }
 
    auto start = entropic::log::now();
    auto result = do_generate(messages, params, cancel);
    result.generation_time_ms = entropic::log::elapsed_ms(start, entropic::log::now());
    return result;
}
 
std::vector<GenerationResult> InferenceBackend::generate_batch(
    const std::vector<std::vector<Message>>& requests,
    const std::vector<GenerationParams>& params,
    std::atomic<bool>& cancel)
{
    if (!is_active()) {
        GenerationResult err;
        err.error_code = ENTROPIC_ERROR_INVALID_STATE;
        err.error_message = "generate_batch() requires ACTIVE state";
        err.finish_reason = "error";
        logger->error("{}", err.error_message);
        return {err};
    }
    auto start = entropic::log::now();
    auto results = do_generate_batch(requests, params, cancel);
    double ms = entropic::log::elapsed_ms(start, entropic::log::now());
    for (auto& r : results) { r.total_ms = ms; }
    return results;
}
 
GenerationResult InferenceBackend::generate_streaming(
    const std::vector<Message>& messages,
    const GenerationParams& params,
    std::function<void(std::string_view token)> on_token,
    std::atomic<bool>& cancel)
{
    if (!is_active()) {
        GenerationResult err;
        err.error_code = ENTROPIC_ERROR_INVALID_STATE;
        err.error_message = "generate_streaming() requires ACTIVE state";
        err.finish_reason = "error";
        logger->error("{}", err.error_message);
        return err;
    }
 
    auto start = entropic::log::now();
    auto result = do_generate_streaming(messages, params, on_token, cancel);
    result.generation_time_ms = entropic::log::elapsed_ms(start, entropic::log::now());
    return result;
}
 
GenerationResult InferenceBackend::generate_speculative(
    const std::vector<Message>& messages,
    const GenerationParams& params,
    std::function<void(std::string_view token)> on_token,
    std::atomic<bool>& cancel)
{
    if (!is_active()) {
        GenerationResult err;
        err.error_code = ENTROPIC_ERROR_INVALID_STATE;
        err.error_message =
            "generate_speculative() requires ACTIVE state";
        err.finish_reason = "error";
        logger->error("{}", err.error_message);
        return err;
    }
    auto start = entropic::log::now();
    auto result = do_generate_speculative(
        messages, params, std::move(on_token), cancel);
    result.generation_time_ms =
        entropic::log::elapsed_ms(start, entropic::log::now());
    return result;
}
 
GenerationResult InferenceBackend::do_generate_speculative(
    const std::vector<Message>& /*messages*/,
    const GenerationParams& /*params*/,
    std::function<void(std::string_view)> /*on_token*/,
    std::atomic<bool>& /*cancel*/)
{
    GenerationResult result;
    result.error_code = ENTROPIC_ERROR_NOT_SUPPORTED;
    result.error_message =
        "speculative decoding not implemented for this backend";
    result.finish_reason = "error";
    return result;
}
 
GenerationResult InferenceBackend::complete(
    const std::string& prompt,
    const GenerationParams& params)
{
    if (!is_active()) {
        GenerationResult err;
        err.error_code = ENTROPIC_ERROR_INVALID_STATE;
        err.error_message = "complete() requires ACTIVE state";
        err.finish_reason = "error";
        logger->error("{}", err.error_message);
        return err;
    }
 
    auto start = entropic::log::now();
    auto result = do_complete(prompt, params);
    result.generation_time_ms = entropic::log::elapsed_ms(start, entropic::log::now());
    return result;
}
 
// ── Evaluation (v1.9.10) ───────────────────────────────────
 
LogprobResult InferenceBackend::evaluate_logprobs(
    const int32_t* tokens,
    int n_tokens)
{
    if (!is_active()) {
        logger->error("evaluate_logprobs: model not ACTIVE (state={})",
                      state_name(state()));
        throw std::runtime_error("Model must be ACTIVE for evaluation");
    }
 
    if (n_tokens < 2) {
        logger->error("evaluate_logprobs: need >= 2 tokens, got {}",
                      n_tokens);
        throw std::runtime_error(
            "Need at least 2 tokens for logprob evaluation");
    }
 
    std::lock_guard<std::mutex> lock(eval_mutex_);
 
    logger->info("evaluate_logprobs: {} tokens, first=[{},{},{}...]",
                 n_tokens, tokens[0],
                 n_tokens > 1 ? tokens[1] : 0,
                 n_tokens > 2 ? tokens[2] : 0);
    auto start = entropic::log::now();
 
    LogprobResult result = do_evaluate_logprobs(tokens, n_tokens);
 
    result.total_logprob = 0.0f;
    for (float lp : result.logprobs) {
        result.total_logprob += lp;
    }
    float mean_lp = result.total_logprob /
        static_cast<float>(result.n_logprobs);
    result.perplexity = std::exp(-mean_lp);
 
    auto ms = entropic::log::elapsed_ms(start, entropic::log::now());
    logger->info("evaluate_logprobs: perplexity={:.2f}, "
                 "total_lp={:.4f}, {:.2f}ms",
                 result.perplexity, result.total_logprob, ms);
    for (int i = 0; i < result.n_logprobs; ++i) {
        logger->info("  logprob[{}]={:.4f}", i, result.logprobs[i]);
    }
 
    return result;
}
 
float InferenceBackend::compute_perplexity(
    const int32_t* tokens,
    int n_tokens)
{
    return evaluate_logprobs(tokens, n_tokens).perplexity;
}
 
// ── Hook helpers (v1.9.1) ──────────────────────────────────
 
bool InferenceBackend::fire_model_load_hook(const ModelConfig& config) {
    if (hooks_.fire_pre == nullptr) {
        return false;
    }
    std::string json = "{\"model_path\":\""
        + config.path.string() + "\"}";
    char* mod = nullptr;
    int rc = hooks_.fire_pre(hooks_.registry,
        ENTROPIC_HOOK_ON_MODEL_LOAD, json.c_str(), &mod);
    free(mod);
    if (rc != 0) {
        logger->info("[VRAM] ON_MODEL_LOAD hook cancelled");
    }
    return rc != 0;
}
 
// ── Queries ────────────────────────────────────────────────
 
int InferenceBackend::count_tokens(const std::string& text) const {
    if (is_loaded()) {
        return do_count_tokens(text);
    }
    return static_cast<int>(text.size()) / 4;
}
 
// ── Capability queries (v1.9.13) ───────────────────────────
 
bool InferenceBackend::supports(BackendCapability cap) const {
    return do_supports(cap);
}
 
std::vector<BackendCapability> InferenceBackend::capabilities() const {
    std::vector<BackendCapability> result;
    int count = static_cast<int>(BackendCapability::_COUNT);
    for (int i = 0; i < count; ++i) {
        auto cap = static_cast<BackendCapability>(i);
        if (supports(cap)) {
            result.push_back(cap);
        }
    }
    return result;
}
 
BackendInfo InferenceBackend::info() const {
    return do_info();
}
 
// ── Model state management (v1.9.13) ──────────────────────
 
bool InferenceBackend::save_state(
    int seq_id, std::vector<uint8_t>& buffer) const
{
    if (!is_active()) {
        logger->warn("save_state: not ACTIVE ({})", state_name(state()));
        return false;
    }
    auto start = entropic::log::now();
    bool ok = do_save_state(seq_id, buffer);
    if (ok) {
        logger->info("save_state: seq={} {}B {:.2f}ms",
                     seq_id, buffer.size(), entropic::log::elapsed_ms(start, entropic::log::now()));
    }
    return ok;
}
 
bool InferenceBackend::restore_state(
    int seq_id, const std::vector<uint8_t>& buffer)
{
    if (!is_active()) {
        logger->warn("restore_state: not ACTIVE ({})",
                     state_name(state()));
        return false;
    }
    auto start = entropic::log::now();
    bool ok = do_restore_state(seq_id, buffer);
    if (ok) {
        logger->info("restore_state: seq={} {}B {:.2f}ms",
                     seq_id, buffer.size(), entropic::log::elapsed_ms(start, entropic::log::now()));
    }
    return ok;
}
 
bool InferenceBackend::clear_state(int seq_id) {
    if (state() == ModelState::COLD) {
        logger->warn("clear_state: model is COLD");
        return false;
    }
    bool ok = do_clear_state(seq_id);
    if (ok) {
        logger->info("clear_state: seq={}", seq_id);
    }
    return ok;
}
 
// ── Multi-sequence generation (v1.9.13) ────────────────────
 
GenerationResult InferenceBackend::generate_seq(
    int seq_id,
    const std::vector<Message>& messages,
    const GenerationParams& params)
{
    if (!is_active()) {
        GenerationResult err;
        err.error_code = ENTROPIC_ERROR_INVALID_STATE;
        err.error_message = "generate_seq() requires ACTIVE state";
        err.finish_reason = "error";
        logger->error("{}", err.error_message);
        return err;
    }
 
    auto start = entropic::log::now();
    auto result = do_generate_seq(seq_id, messages, params);
    result.generation_time_ms = entropic::log::elapsed_ms(start, entropic::log::now());
    result.seq_id = seq_id;
    return result;
}
 
GenerationResult InferenceBackend::generate_streaming_seq(
    int seq_id,
    const std::vector<Message>& messages,
    const GenerationParams& params,
    std::function<void(std::string_view token)> on_token,
    std::atomic<bool>& cancel)
{
    if (!is_active()) {
        GenerationResult err;
        err.error_code = ENTROPIC_ERROR_INVALID_STATE;
        err.error_message =
            "generate_streaming_seq() requires ACTIVE state";
        err.finish_reason = "error";
        logger->error("{}", err.error_message);
        return err;
    }
 
    auto start = entropic::log::now();
    auto result = do_generate_streaming_seq(
        seq_id, messages, params, on_token, cancel);
    result.generation_time_ms = entropic::log::elapsed_ms(start, entropic::log::now());
    result.seq_id = seq_id;
    return result;
}
 
// ── Default virtual implementations (v1.9.13) ─────────────
 
bool InferenceBackend::do_supports(BackendCapability /*cap*/) const {
    return false;
}
 
BackendInfo InferenceBackend::do_info() const {
    BackendInfo bi;
    bi.name = do_backend_name();
    return bi;
}
 
bool InferenceBackend::do_save_state(
    int /*seq_id*/, std::vector<uint8_t>& /*buffer*/) const
{
    return false;
}
 
bool InferenceBackend::do_restore_state(
    int /*seq_id*/, const std::vector<uint8_t>& /*buffer*/)
{
    return false;
}
 
bool InferenceBackend::do_clear_state(int /*seq_id*/) {
    return false;
}
 
GenerationResult InferenceBackend::do_generate_seq(
    int /*seq_id*/,
    const std::vector<Message>& messages,
    const GenerationParams& params)
{
    return do_generate(messages, params);
}
 
GenerationResult InferenceBackend::do_generate_streaming_seq(
    int /*seq_id*/,
    const std::vector<Message>& messages,
    const GenerationParams& params,
    std::function<void(std::string_view token)> on_token,
    std::atomic<bool>& cancel)
{
    return do_generate_streaming(messages, params, on_token, cancel);
}
 
} // namespace entropic