orchestrator.cpp

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#include <entropic/inference/orchestrator.h>
#include <entropic/inference/speculative_compat.h>
#include <entropic/interfaces/i_inference_backend.h>
#include <entropic/types/logging.h>
 
#include "llama_cpp_backend.h"
#include "empty_content_diagnosis.h"
#include "device_memory.h"
#include "partial_offload.h"   // gh#148 refusal predicates
#include "vram_footprint.h"
#include "gguf_metadata.h"   // v2.13.1: auto reads the model's real shape
#include "auto_placement.h"   // v2.13.1: auto solves the footprint estimator
#include "response_parse.h"
#include "grammar_source.h"    // gh#154: provenance for the result record
#include "mtp_envelope.h"
#include <entropic/core/stream_think_filter.h>
#include "adapters/adapter_registry.h"
#include <entropic/inference/adapters/adapter_base.h>  // gh#88 recovery
 
#include <llama.h>
#include <nlohmann/json.hpp>
 
#include <cstdlib>
#include <filesystem>
 
namespace entropic {
 
namespace {
auto logger = entropic::log::get("inference.orchestrator");
using entropic::log::now;
using entropic::log::elapsed_ms;
 
std::string extract_latest_user_message(const std::vector<Message>& messages) {
    for (auto it = messages.rbegin(); it != messages.rend(); ++it) {
        if (it->role == "user") {
            return it->content;
        }
    }
    return "";
}
 
} // anonymous namespace
 
// ── Initialization ─────────────────────────────────────────
 
bool ModelOrchestrator::create_tier_backends(const ParsedConfig& config) {
    for (const auto& [name, tier_config] : config.models.tiers) {
        std::string path_key = tier_config.path.string();
        if (!std::filesystem::exists(tier_config.path)) {
            logger->error("Model file not found for tier '{}': {}",
                          name, path_key);
            logger->error("Place a GGUF file at the path above, or set "
                          "ENTROPIC_MODEL_DIR to a directory containing "
                          "it. Run `entropic download --list` to see "
                          "bundled model keys, then "
                          "`entropic download <key>` to fetch one.");
            return false;
        }
        if (model_pool_.find(path_key) == model_pool_.end()) {
            model_pool_[path_key] = std::make_shared<LlamaCppBackend>();
        }
        tiers_[name] = model_pool_[path_key];
        adapters_[name] = create_adapter(
            tier_config.adapter, name, "" /* prompt resolved later */);
    }
    // Router backend instantiation moved to SecondaryModelLoader
    // (gh#27, v2.1.11). The loader allocates the role slot lazily on
    // first ensure_loaded() call from activate_router().
    logger->info("Created {} unique backend(s) for {} tier(s)",
                 model_pool_.size(), tiers_.size());
    return true;
}
 
void ModelOrchestrator::build_routing_tables(const ParsedConfig& config) {
    for (const auto& [digit, tier_name] : config.routing.tier_map) {
        tier_map_[digit] = tier_name;
    }
    for (const auto& [src, targets] : config.routing.handoff_rules) {
        handoff_rules_[src] = std::unordered_set<std::string>(
            targets.begin(), targets.end());
    }
}
 
bool ModelOrchestrator::activate_default_tier(const ParsedConfig& config) {
    // gh#157: `models.defer_load` leaves the default tier COLD until the
    // first caller needs it. Two idle hosts previously held two copies of
    // the same 4.8 GB GGUF in VRAM without ever being sent a run.
    const bool skip = config.models.defer_load
        || tiers_.find(default_tier_) == tiers_.end();
    if (skip) {
        if (config.models.defer_load) {
            logger->info("[residency] models.defer_load=true — default tier "
                         "'{}' loads on first use", default_tier_);
        }
        return true;
    }
    if (get_model(default_tier_) == nullptr) {
        logger->error("Failed to activate default tier: {}", default_tier_);
        return false;
    }
    logger->info("Activated default tier: {}", default_tier_);
    return true;
}
 
void ModelOrchestrator::activate_router(const ParsedConfig& config) {
    if (!config.models.router) { return; }
    // Lifecycle now lives on SecondaryModelLoader (gh#27, v2.1.11).
    // Diagnostic-level logging is emitted by the loader itself.
    secondary_loader_.ensure_loaded("router", *config.models.router);
}
 
void ModelOrchestrator::activate_draft(const ParsedConfig& config) {
    const auto& spec = config.inference.speculative;
    if (!spec.enabled || spec.draft.path.empty()) { return; }
    // gh#106 (v2.9.0): under MTP the target owns the head (lazily, via
    // generate_mtp → setup_mtp_draft). Loading draft.path as a standalone
    // secondary backend here would double-load the head GGUF and never use
    // it — skip the gh#36 separate-draft activation entirely.
    if (spec.mtp) {
        logger->info("Speculative MTP: head '{}' is target-owned; skipping "
                     "separate draft activation", spec.draft.path.string());
        return;
    }
    // Full ModelConfig comes from the YAML's
    // `inference.speculative.draft:` block — every llama.cpp knob is
    // consumer-tunable. Defaults come from
    // `make_default_draft_model_config()` (gpu_layers=0,
    // flash_attn=false, context_length=8192, n_threads=4).
    secondary_loader_.ensure_loaded("draft", spec.draft);
}
 
void ModelOrchestrator::ensure_secondary_roles() {
    activate_router(config_);
    activate_draft(config_);
}
 
void ModelOrchestrator::route_ggml_logs(const ParsedConfig& config) {
    if (!config.ggml_logging) { return; }
    std::string path;
    if (!config.llama_log_path.empty()) {
        path = config.llama_log_path.string();
    } else if (!config.log_dir.empty()) {
        path = (config.log_dir / "llama_ggml.log").string();
    }
    if (!path.empty()) {
        entropic_inference_log_to_file(path.c_str());
        logger->info("ggml logging: {}", path);
    }
}
 
bool ModelOrchestrator::initialize(const ParsedConfig& config) {
    config_ = config;
    default_tier_ = config.models.default_tier;
    // gh#198: a deferred handle leaves the device alone until it loads.
    if (budget_resolves_at_init(config)) {
        ensure_vram_budget();
    } else {
        vram_budget_deferred_ = true;
        logger->info("[residency] defer_load: VRAM budget will be measured "
                     "at first admission, not now");
    }
 
    route_ggml_logs(config);  // before any model loading
 
    logger->info("Initializing model orchestrator");
 
    if (!create_tier_backends(config)) { return false; }
    build_routing_tables(config);
    // gh#157 (v2.13.0): a no-op when `models.defer_load` is set.
    if (!activate_default_tier(config)) { return false; }
    // gh#198: under defer_load the router and a separate draft were still
    // loaded here — whole models on an idle handle. The router now loads on
    // the first classification (classify_task) and the draft with the first
    // tier activation (get_model -> ensure_secondary_roles).
    if (!config.models.defer_load) {
        activate_router(config);
        activate_draft(config);  // Speculative draft slot (v2.1.11)
    }
 
    // LoRA adapter preload moved to first activation of the owning model
    // (gh#157): preloading here required the base model to already be
    // loaded, so it silently warned-and-skipped for every tier whose GGUF
    // was not the default one's — and `ensure_adapter_for_tier` then failed
    // with "not found or COLD" on the swap that needed it.
    load_bundled_grammars();     // Bundled grammars (v1.9.3)
    return true;
}
 
void ModelOrchestrator::shutdown() {
    logger->info("Shutting down model orchestrator");
 
    for (auto& [path, backend] : model_pool_) {
        if (backend->is_loaded()) {
            backend->unload();
        }
    }
 
    secondary_loader_.shutdown();
}
 
ModelOrchestrator::~ModelOrchestrator() {
    // Order matters (gh#58 close-out, v2.3.0):
    //   1. Backends first → frees llama_contexts.
    //   2. LoRA adapter handles after → safe because the contexts
    //      that may have held HOT adapter references are gone.
    shutdown();
    lora_manager_.unload_all();
}
 
bool ModelOrchestrator::resolve_mtp_effective(const std::string& tier_name) const {
    auto it = config_.models.tiers.find(tier_name);
    if (it != config_.models.tiers.end() && it->second.speculative_mtp) {
        return *it->second.speculative_mtp;
    }
    return config_.inference.speculative.mtp;
}
 
GenerationResult ModelOrchestrator::run_generate_dispatch(
    InferenceBackend* model,
    const std::vector<Message>& messages,
    const GenerationParams& params,
    const std::string& tier_name) {
    GenerationResult result;
    bool kernel_ran = config_.inference.speculative.enabled
        && try_speculative_route(model, messages, params, tier_name, result);
    if (!kernel_ran) {
        result = model->generate(messages, params);
    }
    return result;
}
 
bool ModelOrchestrator::try_mtp_route(
    InferenceBackend* model,
    const std::vector<Message>& messages,
    const GenerationParams& params,
    std::function<void(std::string_view)> on_token,
    std::atomic<bool>& cancel,
    GenerationResult& result)
{
    auto* llama_target = dynamic_cast<LlamaCppBackend*>(model);
    if (llama_target == nullptr) {
        // Fail loud — no silent plain-decode fallback (gh#108).
        result = GenerationResult{};
        result.error_code = ENTROPIC_ERROR_NOT_SUPPORTED;
        result.error_message = "speculative.mtp enabled but the target backend "
                               "is not llama.cpp; disable speculative.mtp";
        result.finish_reason = "error";
        logger->error("{}", result.error_message);
    } else {
        result = llama_target->generate_mtp(
            messages, params, on_token, cancel,
            config_.inference.speculative.draft.path.string(),
            config_.inference.speculative.n_draft);
    }
    return true;  // MTP owns the outcome — never fall back to plain decode
}
 
static bool mtp_head_guard_fires(LlamaCppBackend* draft,
                                 GenerationResult& result) {
    auto* dm = draft->llama_model_ptr();
    if (dm == nullptr) { return false; }
    int n = llama_model_n_layer(dm);
    if (!looks_like_mtp_head(n)) { return false; }
    result.error_code = ENTROPIC_ERROR_SPECULATIVE_INCOMPATIBLE_CONFIG;
    result.error_message = mtp_head_classical_path_error(n);
    result.finish_reason = "error";
    return true;
}
 
bool ModelOrchestrator::try_speculative_route_streaming(
    InferenceBackend* model,
    const std::vector<Message>& messages,
    const GenerationParams& params,
    const std::string& tier_name,
    std::function<void(std::string_view)> on_token,
    std::atomic<bool>& cancel,
    GenerationResult& result)
{
    // gh#106 (v2.9.0): MTP routes BEFORE the gh#36 compat/pair path — the
    // target owns the head (no separate draft backend), and MTP tolerates
    // shared-KV gemma4 archs the gh#36 compat gate rejects.
    if (resolve_mtp_effective(tier_name)) {  // gh#108 v2.10.0: grammar no longer blocks MTP
        return try_mtp_route(model, messages, params, on_token, cancel,
                             result);
    }
    auto compat = check_speculative_compat();
    bool kernel_ran = false;
    if (!compat.compatible) {
        logger->info("Speculative requested but pair incompatible "
                     "({}); using plain decode", compat.diagnostic);
    } else {
        auto* llama_target = dynamic_cast<LlamaCppBackend*>(model);
        auto* draft_be = secondary_loader_.get("draft");
        auto* llama_draft = dynamic_cast<LlamaCppBackend*>(draft_be);
        if (llama_target == nullptr || llama_draft == nullptr) {
            logger->info("Speculative compat passed but target/draft "
                         "is not llama.cpp; using plain decode");
        } else {
            // gh#107 (v2.10.0): guard against MTP head GGUFs on classical path.
            if (mtp_head_guard_fires(llama_draft, result)) {
                logger->error("{}", result.error_message);
                return true;
            }
            auto spec = llama_target->generate_speculative_with_draft(
                messages, params, on_token, cancel, *llama_draft,
                config_.inference.speculative.n_draft,
                config_.inference.speculative.draft.path.string());
            if (spec.error_code == ENTROPIC_ERROR_NOT_SUPPORTED) {
                logger->info("Speculative kernel returned NOT_SUPPORTED "
                             "({}); falling back", spec.error_message);
            } else {
                result = std::move(spec);
                kernel_ran = true;
            }
        }
    }
    return kernel_ran;
}
 
bool ModelOrchestrator::try_speculative_route(
    InferenceBackend* model,
    const std::vector<Message>& messages,
    const GenerationParams& params,
    const std::string& tier_name,
    GenerationResult& result)
{
    std::atomic<bool> local_cancel{false};
    // gh#108: pass an EMPTY std::function (not a bound no-op lambda) so the MTP
    // path can distinguish non-streaming from streaming via the callback's
    // bound-ness. gh#36's emit guards `if (on_token)`, so empty is equivalent.
    return try_speculative_route_streaming(
        model, messages, params, tier_name,
        std::function<void(std::string_view)>{}, local_cancel, result);
}
 
// ── Generation ─────────────────────────────────────────────
 
static void stage_active_tools(InferenceBackend* model,
                               const GenerationParams& params,
                               bool require_tool_call) {
    if (auto* llama = dynamic_cast<LlamaCppBackend*>(model)) {
        llama->set_active_tools(params.tools);
        llama->set_require_tool_call(require_tool_call);  // gh#134 (v2.10.4)
    }
}
 
static void apply_adapter_parse(InferenceBackend* model,
                                ChatAdapter* adapter,
                                GenerationResult& result) {
    if (result.content.empty()) { return; }
    result.raw_content = result.content;
    // gh#108 (v2.10.3): one template-first/adapter-second rule, shared with
    // interface_factory. See response_parse.h for why content composes and
    // tool calls do not.
    auto parsed = parse_model_response(
        dynamic_cast<LlamaCppBackend*>(model), adapter, result.content);
    result.content = std::move(parsed.content);
    result.tool_calls = std::move(parsed.tool_calls);
}
 
 
static void warn_if_content_vanished(const GenerationResult& result) {
    const auto cause = diagnose_empty_content(
        result.content.empty(), !result.raw_content.empty(),
        result.finish_reason, !result.tool_calls.empty());
    if (cause == EmptyContentCause::not_empty) { return; }
    if (cause == EmptyContentCause::tool_call_only) {
        logger->info("Turn delivered {} tool call(s) and no prose. {}",
                     result.tool_calls.size(), explain_empty_content(cause));
        return;
    }
    logger->warn("Turn produced {} raw chars but delivered no content. {}",
                 result.raw_content.size(), explain_empty_content(cause));
}
 
static void warn_if_budget_starved_required_turn(
    const GenerationResult& result,
    const std::string& tier_name,
    const std::unordered_map<std::string, TierConfig>& tiers) {
    if (!result.tool_calls.empty()) { return; }
    auto it = tiers.find(tier_name);
    if (it == tiers.end()
        || !it->second.require_tool_call.value_or(false)) { return; }
    if (result.finish_reason != "length") { return; }
    logger->error(
        "Tier '{}' requires a tool call but hit its token budget before "
        "emitting one (finish_reason=length, 0 tool calls). The tool-call "
        "grammar permits unbounded text BEFORE the mandated call, so the call "
        "must fit inside max_tokens. Raise max_tokens, or disable "
        "enable_thinking on this tier — the thinking channel is what consumes "
        "the preamble. This is a budget misconfiguration, not a model failure.",
        tier_name);
}
 
static void warn_turn_diagnostics(
    const GenerationResult& result,
    const std::string& tier_name,
    const std::unordered_map<std::string, TierConfig>& tiers) {
    warn_if_content_vanished(result);
    warn_if_budget_starved_required_turn(result, tier_name, tiers);
}
 
GenerationParams ModelOrchestrator::resolve_and_stage(
    InferenceBackend* model,
    const GenerationParams& params,
    const std::string& tier_name) {
    GenerationParams resolved = params;
    apply_tier_sampler_defaults(resolved, tier_name);  // gh#82
    // gh#134 (v2.10.4): per-tier, never global — front-office tiers
    // legitimately answer in prose.
    bool require_tc = false;
    if (auto it = config_.models.tiers.find(tier_name);
        it != config_.models.tiers.end()) {
        require_tc = it->second.require_tool_call.value_or(false);
    }
    stage_active_tools(model, resolved, require_tc);   // gh#87 3b, gh#134
    return resolved;
}
 
static void log_orchestration(const GenerationResult& result,
                              const std::string& selected,
                              const std::string& adapter_name,
                              const GenerationParams& params,
                              double routing_ms, double swap_ms) {
    logger->info("Orchestration: tier={}, adapter={}, grammar={}",
                 selected, adapter_name,
                 params.grammar.empty() ? "unconstrained"
                                        : params.grammar_key);
    logger->info("Total: {:.0f}ms (route={:.0f}ms, swap={:.0f}ms, "
                 "gen={:.0f}ms)",
                 result.total_ms, routing_ms, swap_ms,
                 result.generation_time_ms);
}
 
GenerationResult ModelOrchestrator::generate(
    const std::vector<Message>& messages,
    const GenerationParams& params,
    const std::string& tier_name)
{
    // gh#158 (v2.13.0): one lock, outermost on the generation path.
    // See `generation_mutex_` in orchestrator.h for the full order.
    std::lock_guard<std::recursive_mutex> gen_lock(generation_mutex_);
    auto t_start = now();
 
    // Route if no explicit tier
    std::string selected = tier_name;
    double routing_ms = 0.0;
    if (selected.empty()) {
        auto t_route = now();
        selected = route(messages);
        routing_ms = elapsed_ms(t_route, now());
    }
 
    // gh#154: an unregistered tier grammar refuses the run — before the
    // model is resolved, so a doomed run costs no swap and decodes nothing.
    GenerationParams resolved_params = params;
    auto refusal = refuse_unresolved_tier_grammar(resolved_params, selected);
    if (refusal.has_value()) { return *refusal; }
 
    // Get model (may trigger swap)
    auto t_swap = now();
    InferenceBackend* model = get_model(selected);
    double swap_ms = elapsed_ms(t_swap, now());
 
    if (!model) { return build_no_model_error(selected); }
 
    resolved_params =
        resolve_and_stage(model, resolved_params, selected);  // gh#87 3b
 
    // Generate — speculative routing applies here too (v2.1.11, gh#36)
    GenerationResult result = run_generate_dispatch(
        model, messages, resolved_params, selected);
 
    finish_generation(result, model, resolved_params, selected,
                      routing_ms, swap_ms, t_start);
    return result;
}
 
GenerationResult ModelOrchestrator::generate(
    const std::vector<Message>& messages,
    const GenerationParams& params,
    std::atomic<bool>& cancel,
    const std::string& tier_name)
{
    // gh#158 (v2.13.0): one lock, outermost on the generation path.
    // See `generation_mutex_` in orchestrator.h for the full order.
    std::lock_guard<std::recursive_mutex> gen_lock(generation_mutex_);
    auto t_start = now();
 
    std::string selected = tier_name;
    double routing_ms = 0.0;
    if (selected.empty()) {
        auto t_route = now();
        selected = route(messages);
        routing_ms = elapsed_ms(t_route, now());
    }
 
    // gh#154: same gate as the non-cancellable overload — one rule, every
    // entry point, always before a model is resolved.
    GenerationParams resolved_params = params;
    auto refusal = refuse_unresolved_tier_grammar(resolved_params, selected);
    if (refusal.has_value()) { return *refusal; }
 
    auto t_swap = now();
    InferenceBackend* model = get_model(selected);
    double swap_ms = elapsed_ms(t_swap, now());
 
    if (!model) { return build_no_model_error(selected); }
 
    resolved_params =
        resolve_and_stage(model, resolved_params, selected);  // gh#87 3b
 
    GenerationResult result = model->generate(
        messages, resolved_params, cancel);
 
    finish_generation(result, model, resolved_params, selected,
                      routing_ms, swap_ms, t_start);
    return result;
}
 
namespace {
const std::string& batch_arm_tier(const std::vector<std::string>& tiers,
                                  std::size_t i,
                                  const std::string& lead) {
    return (i < tiers.size() && !tiers[i].empty()) ? tiers[i] : lead;
}
}  // namespace
 
std::vector<GenerationResult> ModelOrchestrator::generate_batch(
    const std::vector<std::vector<Message>>& messages_list,
    const std::vector<GenerationParams>& params_list,
    const std::vector<std::string>& tiers,
    std::atomic<bool>& cancel)
{
    // gh#158 (v2.13.0): one lock, outermost on the generation path.
    // See `generation_mutex_` in orchestrator.h for the full order.
    std::lock_guard<std::recursive_mutex> gen_lock(generation_mutex_);
    const std::size_t n = messages_list.size();
    const std::string lead =
        (tiers.empty() || tiers[0].empty()) ? "default" : tiers[0];
 
    // gh#154: every arm's grammar resolves before any model is touched. A
    // shared prefill cannot run half-constrained, so one unresolved arm
    // refuses the whole batch.
    std::vector<GenerationParams> resolved;
    auto refusal = refuse_unresolved_batch_grammars(
        params_list, tiers, lead, resolved);
    if (refusal.has_value()) {
        return std::vector<GenerationResult>(n, *refusal);
    }
 
    InferenceBackend* model = get_model(lead);
    if (model == nullptr) {
        return std::vector<GenerationResult>(n, build_no_model_error(lead));
    }
    stage_batch_arms(model, tiers, lead, resolved);
 
    auto results = model->generate_batch(messages_list, resolved, cancel);
    for (std::size_t i = 0; i < results.size(); ++i) {
        apply_adapter_parse(
            model, get_adapter(batch_arm_tier(tiers, i, lead)), results[i]);
        // gh#154: a batch request is a generation. Each arm carries its
        // OWN resolved params, so each gets its own provenance — a
        // per-batch record would hide a tier whose grammar missed.
        record_generation(results[i], resolved[i], model);
    }
    return results;
}
 
void ModelOrchestrator::stage_batch_arms(
    InferenceBackend* model,
    const std::vector<std::string>& tiers,
    const std::string& lead,
    std::vector<GenerationParams>& resolved)
{
    for (std::size_t i = 0; i < resolved.size(); ++i) {
        resolved[i] = resolve_and_stage(
            model, resolved[i], batch_arm_tier(tiers, i, lead));
    }
}
 
static void stream_token_trampoline(const char* data, std::size_t len,
                                    void* ud) {
    (*static_cast<std::function<void(std::string_view)>*>(ud))(
        std::string_view(data, len));
}
 
GenerationResult ModelOrchestrator::generate_streaming(
    const std::vector<Message>& messages,
    const GenerationParams& params,
    std::function<void(std::string_view)> on_token,
    std::atomic<bool>& cancel,
    const std::string& tier_name)
{
    // gh#158 (v2.13.0): one lock, outermost on the generation path.
    // See `generation_mutex_` in orchestrator.h for the full order.
    std::lock_guard<std::recursive_mutex> gen_lock(generation_mutex_);
    std::string selected = tier_name.empty() ? route(messages) : tier_name;
 
    // gh#154: refuse an unregistered tier grammar before the model is
    // resolved — no swap, no prefill, and not one token streamed.
    GenerationParams resolved_params = params;
    auto refusal = refuse_unresolved_tier_grammar(resolved_params, selected);
    if (refusal.has_value()) { return *refusal; }
 
    InferenceBackend* model = get_model(selected);
 
    if (!model) {
        GenerationResult err;
        err.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
        err.error_message = "No model for tier: " + selected;
        err.finish_reason = "error";
        return err;
    }
 
    resolved_params =
        resolve_and_stage(model, resolved_params, selected);  // gh#87 3b
 
    // gh#108 (v2.10.3): strip this family's reasoning blocks from the live
    // stream. v2.10.0 added the filter but left it on its hardcoded `<think>`
    // pair, so gemma4's `<|channel>` passed straight through. Markers now come
    // from the resolved adapter, the same source the buffered strip uses.
    //
    // This is not cosmetic: ResponseGenerator::generate_streaming builds
    // result.content from its OWN token accumulator and discards what
    // apply_adapter_parse produced, so on the agent-loop streaming path this
    // filter is the only thing standing between raw reasoning and the
    // conversation history.
    auto* stream_adapter = get_adapter(selected);
    const auto markers = (stream_adapter != nullptr)
        ? stream_adapter->thinking_markers() : ThinkMarkers{};
    StreamThinkFilter filter(stream_token_trampoline, &on_token,
                             markers.open, markers.close);
    auto filtered = [&filter](std::string_view sv) {
        filter.on_token(sv.data(), sv.size());
    };
 
    GenerationResult result;
    bool routed = config_.inference.speculative.enabled
        && try_speculative_route_streaming(
               model, messages, resolved_params, selected, filtered, cancel,
               result);
    if (!routed) {
        result = model->generate_streaming(
            messages, resolved_params, filtered, cancel);
    }
    filter.flush();
    apply_adapter_parse(model, get_adapter(selected), result);
    // gh#134 (v2.10.4): name a budget-starved mandatory-tool turn.
    warn_turn_diagnostics(result, selected,
                                         config_.models.tiers);
    // gh#154: the streaming path runs the speculative/MTP kernels, so
    // leaving it unrecorded would omit exactly the decodes whose
    // drafted/accepted counts the record exists to carry.
    record_generation(result, resolved_params, model);
    return result;
}
 
// ── Routing ────────────────────────────────────────────────
 
std::string ModelOrchestrator::route(const std::vector<Message>& messages) {
    if (!config_.routing.enabled
        || !config_.models.router.has_value()) {
        logger->info("Route: routing disabled, using default '{}'",
                     default_tier_);
        last_routing_result_ = {default_tier_, "", "", "none", 0.0};
        return default_tier_;
    }
 
    auto [tier, raw] = classify_task(messages);
    last_routing_result_ = {tier, loaded_main_tier_, raw, "none", 0.0};
 
    // Track history
    tier_history_.push_back(tier);
    if (tier_history_.size() > 5) {
        tier_history_.erase(tier_history_.begin());
    }
 
    logger->info("[ROUTER] {} | raw='{}'", tier, raw);
    return tier;
}
 
std::pair<std::string, std::string> ModelOrchestrator::classify_task(
    const std::vector<Message>& messages)
{
    // gh#158 (v2.13.0): the router decodes, so this is a generation entry
    // point and takes the same outermost lock. Reached from `route()` inside
    // an already-locked `generate*`, which is exactly why the lock is
    // recursive — see `generation_mutex_` in orchestrator.h.
    std::lock_guard<std::recursive_mutex> gen_lock(generation_mutex_);
    std::string user_msg = extract_latest_user_message(messages);
 
    GenerationParams router_params;
    router_params.max_tokens = 1;
    router_params.temperature = 0.0f;
 
    // gh#198: a defer_load handle did not load the router at init. No-op
    // when it is already loaded or not configured; ensure_loaded locks its
    // own slots_mutex_, and this runs under generation_mutex_.
    activate_router(config_);
    auto* router_backend = secondary_loader_.get("router");
    if (router_backend == nullptr) {
        logger->warn("classify_task: router not loaded; returning empty");
        return {"", ""};
    }
    // audit task #71: a non-fine-tuned router fed the bare "<msg> ->" just
    // CONTINUES the text and never emits a routing digit, so classify_task
    // silently always fell back to the default tier. When the deployment
    // configures routing.classification_prompt, prepend it so a general
    // instruct model is actually told the digit scheme. (The trailing " ->"
    // still constrains it to a single digit, per build_classification_prompt.)
    std::string router_prompt = user_msg + " ->";
    const auto& cprompt = config_.routing.classification_prompt;
    if (cprompt.has_value() && !cprompt->empty()) {
        router_prompt = *cprompt + "\n" + user_msg + " ->";
        // A general instruct model emits a leading space before the digit;
        // max_tokens=1 would cut it off. 4 captures "<space>1"; the digit scan
        // below takes the first tier_map char. Only widened on the prompt path
        // so unconfigured deployments keep the original 1-token behavior.
        router_params.max_tokens = 4;
        // v2.8.1 (review #3): classification_prompt was parsed-but-never-read
        // before the v2.8.0 fix. Log when the active (prompt) path is taken so
        // a deployment carrying a stale prompt sees the inert->active switch +
        // the widened token budget instead of a silent behavior change.
        logger->info("classify_task: using configured classification_prompt "
                     "(router instructed; max_tokens widened to 4)");
    }
    auto result = router_backend->complete(router_prompt, router_params);
    std::string raw = result.content;
 
    // Trim whitespace
    auto start = raw.find_first_not_of(" \t\n\r");
    if (start != std::string::npos) {
        raw = raw.substr(start);
    }
 
    // Find matching tier
    for (char c : raw) {
        std::string digit(1, c);
        auto it = tier_map_.find(digit);
        if (it != tier_map_.end()) {
            logger->info("Route: digit='{}' -> tier='{}'",
                         digit, it->second);
            return {it->second, digit};
        }
    }
 
    logger->warn("Route: no valid digit in '{}', defaulting to {}",
                 raw, default_tier_);
    return {default_tier_, ""};
}
 
// ── Model access ───────────────────────────────────────────
 
void ModelOrchestrator::record_activation_reuse(
    const std::string& tier_name) {
    auto now_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
        std::chrono::steady_clock::now() - start_time_).count();
    bool tier_changed = (loaded_main_tier_ != tier_name);
    tier_last_activation_ms_[tier_name] = now_ms;
    if (!tier_changed) { return; }
    auto tier_it = config_.models.tiers.find(tier_name);
    std::string path = tier_it != config_.models.tiers.end()
        ? tier_it->second.path.string() : "";
    size_t footprint = tier_footprint_bytes_.count(tier_name)
        ? tier_footprint_bytes_[tier_name]
        : estimate_footprint_bytes(tier_name);
    tier_footprint_bytes_[tier_name] = footprint;
    loaded_main_tier_ = tier_name;
    fire_residency_observer(ResidencyEvent::ActivationSwap,
                            tier_name, path, footprint);
}
 
// Defined lower down, beside estimate_footprint_bytes which shares it.
static FootprintInputs footprint_inputs_for(
    const TierConfig& tier_cfg, uint64_t weights_bytes, int vram_reserve_mb);
 
void ModelOrchestrator::log_fit_recommendation(
    const std::string& tier_name) const {
    auto tier_it = config_.models.tiers.find(tier_name);
    if (tier_it == config_.models.tiers.end()) { return; }
    const auto& tier_cfg = tier_it->second;
    std::error_code ec;
    auto weights = std::filesystem::file_size(tier_cfg.path, ec);
    if (ec) { return; }
    FootprintInputs in = footprint_inputs_for(
        tier_cfg, weights, config_.vram_reserve_mb);
    int fits = recommend_context_length(in, vram_budget_bytes_);
    if (fits > 0) {
        logger->error("[residency] tier '{}' fits at context_length={} "
                      "(requested {}) — lower it, or reduce gpu_layers to "
                      "keep the requested context",
                      tier_name, fits, tier_cfg.context_length);
    } else {
        logger->error("[residency] tier '{}' does not fit at ANY context "
                      "length: weights{} alone exceed the {} MiB budget. "
                      "Reduce gpu_layers, or use a smaller quantization.",
                      tier_name,
                      in.mmproj_bytes > 0 ? " + vision projector" : "",
                      vram_budget_bytes_ / (1024 * 1024));
    }
}
 
void ModelOrchestrator::resolve_auto_gpu_layers(const std::string& tier_name) {
    auto it = config_.models.tiers.find(tier_name);
    if (it == config_.models.tiers.end() || !it->second.gpu_layers_auto) {
        return;
    }
    std::error_code ec;
    auto file_bytes = std::filesystem::file_size(it->second.path, ec);
    if (ec || vram_budget_bytes_ == 0) {
        logger->warn("[residency] tier '{}': gpu_layers=auto cannot be "
                     "resolved ({}) — keeping gpu_layers={}", tier_name,
                     ec ? "model file unreadable" : "free VRAM unknown",
                     it->second.gpu_layers);
        return;
    }
 
    // v2.13.1: read the model's actual shape instead of assuming 30 layers
    // and `file_bytes / 30` per layer. A metadata-only open costs the header
    // and the tensor index — no tensor data — regardless of file size.
    const GgufShape shape = read_gguf_shape(it->second.path);
    FootprintInputs base = footprint_inputs_for(
        it->second, static_cast<uint64_t>(file_bytes),
        config_.vram_reserve_mb);
    base.draft_bytes = resident_draft_bytes();
 
    const AutoPlacement placement =
        derive_auto_placement(shape, base, vram_budget_bytes_,
                              it->second.n_ubatch);
    if (!placement.known) {
        fall_back_to_weights_estimate(tier_name, file_bytes, placement.reason);
        return;
    }
 
    log_auto_placement(tier_name, shape, placement, file_bytes);
    it->second.gpu_layers = placement.gpu_layers;
    it->second.cpu_moe_layers = placement.cpu_moe_layers;
    // v2.13.2: only ever LOWERED, and only when that is what bought full
    // residency. 0 means auto did not touch it, so an operator's own value
    // is preserved unless it was the thing standing between them and the
    // whole model on the card.
    if (placement.n_ubatch > 0) { it->second.n_ubatch = placement.n_ubatch; }
}
 
uint64_t ModelOrchestrator::resident_draft_bytes() const {
    if (!config_.inference.speculative.enabled) { return 0; }
    std::error_code ec;
    const auto bytes = std::filesystem::file_size(
        config_.inference.speculative.draft.path, ec);
    return ec ? 0ull : static_cast<uint64_t>(bytes);
}
 
void ModelOrchestrator::fall_back_to_weights_estimate(
    const std::string& tier_name, uint64_t file_bytes, const char* why) {
    auto it = config_.models.tiers.find(tier_name);
    if (it == config_.models.tiers.end()) { return; }
    const int layers = partial_gpu_layers_for(file_bytes, vram_budget_bytes_);
    logger->warn("[residency] tier '{}': gpu_layers=auto could not read the "
                 "model's shape ({}) — falling back to the weights-only "
                 "estimate, {} layers", tier_name, why, layers);
    it->second.gpu_layers = layers;
}
 
void ModelOrchestrator::log_auto_placement(
    const std::string& tier_name, const GgufShape& shape,
    const AutoPlacement& placement, uint64_t file_bytes) const {
    const std::string experts = placement.cpu_moe_layers > 0
        ? fmt::format(", {} layers' experts host-side",
                      placement.cpu_moe_layers)
        : std::string();
    // v2.13.2: an operator who reads "30 of 30 layers" and then measures a
    // prefill rate half what they expected is owed the reason in the same
    // line that made the trade.
    const std::string ubatch = placement.n_ubatch > 0
        ? fmt::format(", n_ubatch lowered to {}", placement.n_ubatch)
        : std::string();
    logger->info("[residency] tier '{}': gpu_layers=auto -> {} of {} layers{}{} "
                 "— {} ({} MiB estimated of {} MiB free; {} MiB model, "
                 "{} experts)",
                 tier_name,
                 placement.fully_resident ? shape.block_count
                                          : placement.gpu_layers,
                 shape.block_count, experts, ubatch,
                 placement.reason, placement.bytes / (1024 * 1024),
                 vram_budget_bytes_ / (1024 * 1024),
                 file_bytes / (1024 * 1024), shape.expert_count);
    // Stated separately and always, because it is the figure that decides
    // whether the placement survives a longer prompt or one more tool
    // schema — and auto accepts a placement that merely fits.
    logger->info("[residency] tier '{}': {} MiB of VRAM left free after this "
                 "placement{}", tier_name,
                 placement.margin_bytes / (1024 * 1024),
                 placement.margin_bytes < 256ull * 1024 * 1024
                     ? " — thin; growth in prompt or tool schemas may push"
                       " this tier off the card"
                     : "");
}
 
bool ModelOrchestrator::config_admits(const std::string& tier_name) {
    auto it = config_.models.tiers.find(tier_name);
    if (it == config_.models.tiers.end()) { return true; }
    const auto& cfg = it->second;
    std::error_code ec;
    auto file_bytes = std::filesystem::file_size(cfg.path, ec);
    if (ec) { return true; }
 
    const uint64_t memlock = host_memlock_limit_bytes();
    bool refused = true;
    // v2.13.1: an expert split leaves expert weights HOST-resident while
    // gpu_layers reports -1, so the pinned bytes are theirs, not the file's.
    const uint64_t host_experts = host_expert_bytes_for(cfg);
    if (mlock_refused(cfg.use_mlock, file_bytes, cfg.gpu_layers, memlock,
                      host_experts)) {
        const uint64_t pinned = host_experts > 0 ? host_experts : file_bytes;
        refuse_residency(tier_name, ENTROPIC_ERROR_MLOCK_LIMIT_EXCEEDED,
            "use_mlock would pin " + std::to_string(pinned)
            + " bytes against an RLIMIT_MEMLOCK of "
            + std::to_string(memlock)
            + " bytes. Pinned pages cannot be reclaimed under pressure, so "
              "a partially offloaded model of this size is killed rather "
              "than paged. Set use_mlock: false for this tier.");
    } else if (gpu_offload_refused(file_bytes, cfg.gpu_layers,
                                   vram_budget_bytes_)) {
        refuse_residency(tier_name, ENTROPIC_ERROR_TIER_MODEL_TOO_LARGE,
            "gpu_layers=" + std::to_string(cfg.gpu_layers) + " cannot fit: "
            + std::to_string(vram_budget_bytes_ / (1024 * 1024))
            + " MiB of free VRAM is not enough for the compute buffers "
              "alone, so no positive layer count can work. Free VRAM, or "
              "set gpu_layers: 0 to run on the CPU.");
    } else {
        refused = false;
    }
    return !refused;
}
 
void ModelOrchestrator::refuse_residency(const std::string& tier_name,
                                         entropic_error_t code,
                                         const std::string& why) {
    logger->error("[residency] tier '{}' refused: {}", tier_name, why);
    last_residency_error_ = code;
    last_residency_message_ = "Tier '" + tier_name + "': " + why;
}
 
uint64_t ModelOrchestrator::host_expert_bytes_for(const ModelConfig& cfg) {
    if (cfg.cpu_moe_layers <= 0) { return 0; }
    const GgufShape shape = read_gguf_shape(cfg.path);
    if (!shape.known) { return 0; }
    return shape.expert_bytes_per_block()
         * static_cast<uint64_t>(cfg.cpu_moe_layers);
}
 
bool ModelOrchestrator::residency_admits(const std::string& tier_name) {
    // gh#198: a deferred handle measures here, on its first admission.
    ensure_vram_budget();
    // gh#148: resolve `gpu_layers: auto` and refuse a config that provably
    // cannot work, BEFORE pricing the footprint — both change what is being
    // priced, and both are loud rather than silently corrected.
    resolve_auto_gpu_layers(tier_name);
    if (!config_admits(tier_name)) { return false; }
 
    size_t footprint = estimate_footprint_bytes(tier_name);
    if (footprint > 0) {
        tier_footprint_bytes_[tier_name] = footprint;
    }
    if (vram_budget_bytes_ > 0 && footprint > vram_budget_bytes_) {
        logger->error("[residency] tier '{}' footprint {} bytes "
                      "exceeds VRAM budget {} bytes — "
                      "TIER_MODEL_TOO_LARGE (gh#57)",
                      tier_name, footprint, vram_budget_bytes_);
        // gh#142: a bare refusal leaves the operator with nothing to act on.
        // Say what WOULD fit, so the answer is a setting they can apply rather
        // than a wall. 0 means even an empty context does not fit, in which
        // case the context length is not the lever and saying so is honest.
        log_fit_recommendation(tier_name);
        last_residency_error_ = ENTROPIC_ERROR_TIER_MODEL_TOO_LARGE;
        return false;
    }
    return true;
}
 
GenerationResult ModelOrchestrator::build_no_model_error(
    const std::string& tier_name) {
    GenerationResult err;
    err.finish_reason = "error";
    if (last_residency_error_ != ENTROPIC_OK) {
        err.error_code = last_residency_error_;
        // gh#148: a typed refusal carries the setting to change. Fall back
        // to the gh#57 budget wording when nothing was stashed.
        err.error_message = !last_residency_message_.empty()
            ? last_residency_message_
            : "Tier '" + tier_name + "' model exceeds the "
              "engine's VRAM budget (gh#57)";
        last_residency_error_ = ENTROPIC_OK;
        last_residency_message_.clear();
    } else {
        err.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
        err.error_message = "No model available for tier: " + tier_name;
    }
    return err;
}
 
InferenceBackend* ModelOrchestrator::activate_and_track(
    const std::string& tier_name,
    const std::shared_ptr<InferenceBackend>& backend) {
    auto tier_it = config_.models.tiers.find(tier_name);
    bool activated = tier_it != config_.models.tiers.end()
        && backend->load_and_activate(tier_it->second);
    if (!activated) {
        logger->error("Failed to activate tier: {}", tier_name);
        return nullptr;
    }
    loaded_main_tier_ = tier_name;
    last_routing_result_.swap_action = "loaded";
    auto now_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
        std::chrono::steady_clock::now() - start_time_).count();
    tier_last_activation_ms_[tier_name] = now_ms;
    size_t footprint = tier_footprint_bytes_.count(tier_name)
        ? tier_footprint_bytes_[tier_name] : 0;
    // gh#157: the model exists now, so its adapters can bind. This is the
    // only moment at which that is true for a non-default tier.
    preload_adapters_for_model(backend.get());
    fire_residency_observer(ResidencyEvent::Loaded,
                            tier_name, tier_it->second.path.string(),
                            footprint);
    return backend.get();
}
 
InferenceBackend* ModelOrchestrator::get_model(const std::string& tier_name) {
    std::lock_guard<std::mutex> lock(swap_mutex_);
 
    auto it = tiers_.find(tier_name);
    std::string effective_tier = tier_name;
    if (it == tiers_.end()) {
        it = tiers_.find(config_.routing.fallback_tier);
        if (it != tiers_.end()) {
            effective_tier = config_.routing.fallback_tier;
        }
    }
 
    InferenceBackend* result = nullptr;
    if (it != tiers_.end() && it->second->is_active()) {
        last_routing_result_.swap_action = "reused";
        record_activation_reuse(effective_tier);
        result = it->second.get();
    } else if (it != tiers_.end() && residency_admits(effective_tier)) {
        deactivate_current_if_needed(it->second.get());
        result = activate_and_track(effective_tier, it->second);
    }
 
    // Ensure correct LoRA adapter for this tier (v1.9.2)
    if (result) {
        ensure_tier_lora(tier_name, result);
        // gh#164: `entropic_release_model(NULL)` drops the router and draft
        // too, and neither reloads itself — routing would degrade to the
        // default tier silently and permanently. No-op when they are loaded.
        ensure_secondary_roles();
    }
 
    return result;
}
 
void ModelOrchestrator::ensure_tier_lora(const std::string& tier_name,
                                         InferenceBackend* result) {
    auto* llama_backend = dynamic_cast<LlamaCppBackend*>(result);
    llama_context* ctx = llama_backend
        ? llama_backend->llama_context_ptr() : nullptr;
    double adapter_ms = ensure_adapter_for_tier(tier_name, ctx);
    last_routing_result_.adapter_swap_ms = adapter_ms;
    last_routing_result_.adapter_name = lora_manager_.active_adapter();
}
 
void ModelOrchestrator::deactivate_current_if_needed(InferenceBackend* incoming) {
    auto it = loaded_main_tier_.empty()
        ? tiers_.end() : tiers_.find(loaded_main_tier_);
 
    bool should_swap = it != tiers_.end()
        && it->second.get() != incoming
        && it->second->is_loaded();
 
    if (!should_swap) {
        return;
    }
 
    // Cascade: unload adapters for this base model (v1.9.2)
    auto* llama_backend = dynamic_cast<LlamaCppBackend*>(it->second.get());
    if (llama_backend) {
        lora_manager_.unload_all_for_model(
            llama_backend->llama_model_ptr(),
            llama_backend->llama_context_ptr());
    }
 
    unload_or_warm_current(it->second.get());
}
 
void ModelOrchestrator::unload_or_warm_current(InferenceBackend* current) {
    auto cfg_it = config_.models.tiers.find(loaded_main_tier_);
    bool keep_warm = cfg_it != config_.models.tiers.end()
        && cfg_it->second.keep_warm;
 
    if (keep_warm) {
        logger->info("Deactivating {} (keep_warm=true)", loaded_main_tier_);
        current->deactivate();
        return;
    }
    logger->info("Unloading {} (keep_warm=false)", loaded_main_tier_);
    std::string path = cfg_it != config_.models.tiers.end()
        ? cfg_it->second.path.string() : "";
    size_t footprint = tier_footprint_bytes_.count(loaded_main_tier_)
        ? tier_footprint_bytes_[loaded_main_tier_] : 0;
    std::string evicted_tier = loaded_main_tier_;
    current->unload();
    fire_residency_observer(ResidencyEvent::Evicted,
                            evicted_tier, path, footprint);
}
 
// ── Queries ────────────────────────────────────────────────
 
RoutingResult ModelOrchestrator::last_routing_result() const {
    return last_routing_result_;
}
 
std::string ModelOrchestrator::last_used_tier() const {
    return loaded_main_tier_;
}
 
std::vector<std::string> ModelOrchestrator::loaded_models() const {
    std::vector<std::string> result;
    for (const auto& [name, backend] : tiers_) {
        if (backend->is_loaded()) {
            result.push_back(name);
        }
    }
    if (secondary_loader_.is_loaded("router")) {
        result.push_back("router");
    }
    return result;
}
 
std::vector<std::string> ModelOrchestrator::available_models() const {
    std::vector<std::string> result;
    for (const auto& [name, _] : tiers_) {
        result.push_back(name);
    }
    if (config_.models.router.has_value()) {
        result.push_back("router");
    }
    return result;
}
 
InferenceBackend* ModelOrchestrator::get_backend(
    const std::string& tier_name) const {
    auto it = tiers_.find(tier_name);
    if (it == tiers_.end()) { return nullptr; }
    return it->second.get();
}
 
InferenceBackend* ModelOrchestrator::ensure_model(
    const std::string& tier_name) {
    return get_model(tier_name);
}
 
entropic_error_t ModelOrchestrator::release_models(
    const std::string& tier_name) {
    std::lock_guard<std::mutex> lock(swap_mutex_);
 
    if (!tier_name.empty()) {
        auto it = tiers_.find(tier_name);
        if (it == tiers_.end()) {
            logger->error("[residency] release: unknown tier '{}'",
                          tier_name);
            return ENTROPIC_ERROR_MODEL_NOT_FOUND;
        }
        release_backend(it->second.get());
        return ENTROPIC_OK;
    }
 
    std::unordered_set<const InferenceBackend*> seen;
    for (const auto& [name, backend] : tiers_) {
        (void)name;
        if (backend && seen.insert(backend.get()).second) {
            release_backend(backend.get());
        }
    }
    // Secondary roles (router, speculative draft). They re-ensure on the
    // next cold activation — see `activate_and_track`.
    secondary_loader_.shutdown();
    return ENTROPIC_OK;
}
 
void ModelOrchestrator::release_backend(InferenceBackend* backend) {
    if (backend == nullptr || !backend->is_loaded()) { return; }
 
    auto* llama_backend = dynamic_cast<LlamaCppBackend*>(backend);
    if (llama_backend != nullptr) {
        // KEEPS the registrations, unlike the swap path's
        // `unload_all_for_model` — see AdapterManager (gh#164).
        lora_manager_.release_handles_for_model(
            llama_backend->llama_model_ptr(),
            llama_backend->llama_context_ptr());
    }
    backend->unload();
    announce_eviction(backend);
}
 
void ModelOrchestrator::announce_eviction(const InferenceBackend* backend) {
    for (const auto& [name, bound] : tiers_) {
        if (bound.get() != backend) { continue; }
        auto cfg_it = config_.models.tiers.find(name);
        std::string path = cfg_it != config_.models.tiers.end()
            ? cfg_it->second.path.string() : "";
        auto fp_it = tier_footprint_bytes_.find(name);
        size_t footprint = fp_it != tier_footprint_bytes_.end()
            ? fp_it->second : 0;
        // Leave no stale incumbent: the next activation reads this to
        // decide whether a swap-out is needed.
        if (loaded_main_tier_ == name) { loaded_main_tier_.clear(); }
        fire_residency_observer(ResidencyEvent::Evicted, name, path,
                                footprint);
    }
}
 
bool ModelOrchestrator::tier_declares_vision(
    const std::string& tier_name) const {
    auto it = config_.models.tiers.find(tier_name);
    if (it == config_.models.tiers.end()) { return false; }
    return it->second.has_capability("vision")
        || !it->second.mmproj_path.empty();
}
 
bool ModelOrchestrator::can_handoff(
    const std::string& from, const std::string& to) const
{
    auto it = handoff_rules_.find(from);
    if (it == handoff_rules_.end()) {
        return false;
    }
    return it->second.count(to) > 0;
}
 
ChatAdapter* ModelOrchestrator::get_adapter(const std::string& tier_name) const {
    auto it = adapters_.find(tier_name);
    if (it != adapters_.end()) {
        return it->second.get();
    }
    return nullptr;
}
 
// ── LoRA adapter management (v1.9.2) ──────────────────────
 
bool ModelOrchestrator::deactivate_if_active(llama_context* ctx) {
    if (lora_manager_.active_adapter().empty()) {
        return false;
    }
    lora_manager_.deactivate(ctx);
    return true;
}
 
double ModelOrchestrator::ensure_adapter_for_tier(
    const std::string& tier_name, llama_context* ctx)
{
    auto tier_it = config_.models.tiers.find(tier_name);
    if (tier_it == config_.models.tiers.end()) {
        return 0.0;
    }
 
    const auto& tier_cfg = tier_it->second;
    auto t_start = now();
    bool needs_kv_clear = false;
 
    if (!tier_cfg.adapter_path) {
        needs_kv_clear = deactivate_if_active(ctx);
    } else if (lora_manager_.active_adapter() != tier_name) {
        needs_kv_clear = lora_manager_.swap(tier_name, ctx);
        if (!needs_kv_clear) {
            logger->warn("Adapter swap to '{}' failed", tier_name);
        }
    }
 
    if (needs_kv_clear && ctx) {
        llama_memory_clear(llama_get_memory(ctx), true);
        logger->info("Adapter swap for tier '{}' in {:.1f}ms",
                    tier_name, elapsed_ms(t_start, now()));
    }
 
    return elapsed_ms(t_start, now());
}
 
void ModelOrchestrator::preload_adapters_for_model(
    InferenceBackend* backend) {
    auto* llama_backend = dynamic_cast<LlamaCppBackend*>(backend);
    if (!llama_backend || !llama_backend->llama_model_ptr()) { return; }
 
    int loaded = 0;
    for (const auto& [name, tier_cfg] : config_.models.tiers) {
        auto tier_it = tiers_.find(name);
        bool mine = tier_cfg.adapter_path.has_value()
            && tier_it != tiers_.end()
            && tier_it->second.get() == backend
            && lora_manager_.state(name) == AdapterState::COLD;
        if (mine && lora_manager_.load(name, *tier_cfg.adapter_path,
                                       llama_backend->llama_model_ptr(),
                                       tier_cfg.adapter_scale)) {
            ++loaded;
        }
    }
 
    if (loaded > 0) {
        logger->info("Preloaded {} LoRA adapter(s) to WARM", loaded);
    }
}
 
// ── Grammar registry (v1.9.3) ──────────────────────────────
 
void ModelOrchestrator::load_bundled_grammars() {
    std::filesystem::path grammar_dir;
    if (!config_.config_dir.empty()) {
        grammar_dir = config_.config_dir / "grammars";
    }
    if (grammar_dir.empty() || !std::filesystem::is_directory(grammar_dir)) {
        // Fallback set by facade via load_grammars_from() if config_dir
        // doesn't have a grammars subdir. Check if already loaded.
        logger->info("No bundled grammar directory found, skipping");
        return;
    }
 
    size_t count = grammar_registry_.load_bundled(grammar_dir);
    logger->info("Grammar registry: {} grammar(s) loaded from {}",
                 count, grammar_dir.string());
}
 
size_t ModelOrchestrator::load_grammars_from(
    const std::filesystem::path& grammar_dir) {
    if (!std::filesystem::is_directory(grammar_dir)) {
        return 0;
    }
    auto count = grammar_registry_.load_bundled(grammar_dir);
    logger->info("Grammar registry: {} grammar(s) loaded from {}",
                 count, grammar_dir.string());
    return count;
}
 
void ModelOrchestrator::clear_all_prompt_caches() {
    for (auto& [_, backend] : model_pool_) {
        if (backend) { backend->clear_prompt_cache(); }
    }
    secondary_loader_.clear_all_prompt_caches();
    logger->info("Prompt caches invalidated across all backends "
                 "(identity change)");
}
 
bool ModelOrchestrator::has_vision_capable_tier() const {
    for (const auto& [_, tier] : config_.models.tiers) {
        if (tier.has_capability("vision")) { return true; }
    }
    return false;
}
 
std::string ModelOrchestrator::select_vision_tier() const {
    for (const auto& [name, tier] : config_.models.tiers) {
        if (tier.has_capability("vision")) { return name; }
    }
    return "";
}
 
static llama_model* resolve_target_model(
    const std::shared_ptr<InferenceBackend>& tier_backend) {
    if (!tier_backend || !tier_backend->is_loaded()) {
        return nullptr;
    }
    auto* llama_be = dynamic_cast<LlamaCppBackend*>(tier_backend.get());
    return (llama_be == nullptr) ? nullptr : llama_be->llama_model_ptr();
}
 
std::string ModelOrchestrator::resolve_speculative_pair(
    llama_model*& target_out, llama_model*& draft_out) const {
    target_out = nullptr;
    draft_out = nullptr;
    std::string err;
 
    auto tier_it = tiers_.find(loaded_main_tier_);
    if (tier_it == tiers_.end()) {
        err = "no main tier loaded";
    } else {
        target_out = resolve_target_model(tier_it->second);
        if (target_out == nullptr) {
            err = "main tier backend is not a llama.cpp backend or "
                  "is not loaded";
        } else {
            auto* draft_backend = secondary_loader_.get("draft");
            if (draft_backend == nullptr || !draft_backend->is_loaded()) {
                err = "no draft model configured for speculative "
                      "decoding "
                      "(set inference.speculative.draft_model)";
            } else {
                auto* d = dynamic_cast<LlamaCppBackend*>(draft_backend);
                draft_out = (d == nullptr) ? nullptr : d->llama_model_ptr();
                if (draft_out == nullptr) {
                    err = "draft backend is not a llama.cpp backend";
                }
            }
        }
    }
    return err;
}
 
ModelOrchestrator::SpeculativeCompatInfo
ModelOrchestrator::check_speculative_compat() const {
    SpeculativeCompatInfo info;
    llama_model* target_model = nullptr;
    llama_model* draft_model = nullptr;
    info.diagnostic = resolve_speculative_pair(target_model, draft_model);
    if (info.diagnostic.empty()) {
        auto result = entropic::speculative::check_compat(
            target_model, draft_model);
        info.compatible = result.compatible;
        info.diagnostic = std::move(result.diagnostic);
    }
    return info;
}
 
void ModelOrchestrator::finish_generation(
    GenerationResult& result,
    InferenceBackend* model,
    const GenerationParams& resolved_params,
    const std::string& selected,
    double routing_ms,
    double swap_ms,
    std::chrono::steady_clock::time_point t_start)
{
    apply_adapter_parse(model, get_adapter(selected), result);
    // gh#134 (v2.10.4): name a budget-starved mandatory-tool turn.
    warn_turn_diagnostics(result, selected, config_.models.tiers);
 
    result.routing_ms = routing_ms;
    result.swap_ms = swap_ms;
    result.total_ms = elapsed_ms(t_start, now());
    // gh#154: one record per generation, at the tail every orchestrated
    // path already shares — so a new decode path cannot ship unrecorded.
    record_generation(result, resolved_params, model);
    log_orchestration(result, selected, last_routing_result_.adapter_name,
                      resolved_params, routing_ms, swap_ms);
}
 
void ModelOrchestrator::record_generation(
    GenerationResult& result,
    const GenerationParams& resolved_params,
    const InferenceBackend* model)
{
    const std::string tool_grammar =
        model != nullptr ? model->active_tool_grammar() : std::string{};
    result.grammar = describe_grammar(resolved_params, tool_grammar);
 
    std::lock_guard<std::mutex> lock(records_mutex_);
    if (generation_records_.size() >= kMaxGenerationRecords) {
        generation_records_.erase(generation_records_.begin());
    }
    generation_records_.push_back(make_generation_record(result));
}
 
std::vector<GenerationRecord> ModelOrchestrator::generation_records() const
{
    std::lock_guard<std::mutex> lock(records_mutex_);
    return generation_records_;
}
 
static std::string normalize_grammar_key(const std::string& grammar_value) {
    std::filesystem::path p(grammar_value);
    if (p.extension() == ".gbnf") {
        return p.stem().string();
    }
    return grammar_value;
}
 
void ModelOrchestrator::resolve_grammar_key(
    GenerationParams& params, const std::string& tier_name)
{
    if (!params.grammar.empty()) {
        return;
    }
 
    // Try explicit grammar_key
    std::string key = params.grammar_key;
 
    // Fall back to tier config grammar field (frontmatter)
    if (key.empty()) {
        auto it = config_.models.tiers.find(tier_name);
        if (it != config_.models.tiers.end() && it->second.grammar) {
            key = normalize_grammar_key(it->second.grammar->string());
            // gh#154: remember WHERE the key came from. All three
            // request-side sources arrive as params.grammar, so without
            // this the result cannot say whether the tier asked or the
            // caller did.
            params.grammar_from_tier = true;
        }
    }
 
    if (key.empty()) {
        return;
    }
 
    // Recorded BEFORE the lookup, so a key that resolves to nothing is
    // still named on the result. A missing grammar leaves the decode
    // unconstrained (REQ-INFER-007) and the absence of a log line was,
    // until gh#154, the only signal that it had happened.
    params.resolved_grammar_key = key;
 
    std::string content = grammar_registry_.get(key);
    if (content.empty()) {
        logger->warn("Grammar key '{}' not found in registry — this decode "
                     "is UNCONSTRAINED; read generations[].grammar.resolved "
                     "from entropic_metrics_json to detect it", key);
        return;
    }
 
    logger->info("Grammar resolved: key='{}', {} bytes",
                 key, content.size());
    params.grammar = std::move(content);
}
 
std::optional<GenerationResult>
ModelOrchestrator::refuse_unresolved_tier_grammar(
    GenerationParams& params, const std::string& tier_name)
{
    resolve_grammar_key(params, tier_name);          // v1.9.3
    if (!tier_grammar_unresolved(params)) {
        return std::nullopt;
    }
 
    GenerationResult err;
    err.finish_reason = "error";
    err.error_code = ENTROPIC_ERROR_GRAMMAR_NOT_FOUND;
    err.error_message =
        "Tier '" + tier_name + "' declares grammar '"
        + params.resolved_grammar_key + "' and no grammar is registered "
        "under that key. Register it with entropic_grammar_register_file() "
        "or entropic_grammar_register() before running this tier, or place "
        + params.resolved_grammar_key + ".gbnf in a grammar search path. "
        "Refused rather than decoded unconstrained (gh#154): an "
        "unconstrained decode is indistinguishable from a constrained one "
        "in the output.";
    logger->error("{}", err.error_message);
    return err;
}
 
std::optional<GenerationResult>
ModelOrchestrator::refuse_unresolved_batch_grammars(
    const std::vector<GenerationParams>& params_list,
    const std::vector<std::string>& tiers,
    const std::string& lead,
    std::vector<GenerationParams>& out)
{
    out.clear();
    out.reserve(params_list.size());
    for (std::size_t i = 0; i < params_list.size(); ++i) {
        GenerationParams resolved = params_list[i];
        auto refusal = refuse_unresolved_tier_grammar(
            resolved, batch_arm_tier(tiers, i, lead));
        if (refusal.has_value()) {
            return refusal;
        }
        out.push_back(std::move(resolved));
    }
    return std::nullopt;
}
 
namespace {
template <typename T>
inline void apply_if_default(T& field, const std::optional<T>& ov, T dflt) {
    if (ov.has_value() && field == dflt) { field = *ov; }
}
}  // namespace
 
void apply_tier_sampler_overrides(
    GenerationParams& params, const TierSamplerOverrides& ov)
{
    // GenerationParams struct defaults (see types/config.h).
    apply_if_default(params.temperature,       ov.temperature,       0.7f);
    apply_if_default(params.max_tokens,        ov.max_output_tokens, 4096);
    apply_if_default(params.top_p,             ov.top_p,             0.9f);
    apply_if_default(params.top_k,             ov.top_k,             40);
    apply_if_default(params.min_p,             ov.min_p,             0.0f);
    apply_if_default(params.presence_penalty,  ov.presence_penalty,  0.0f);
    apply_if_default(params.frequency_penalty, ov.frequency_penalty, 0.0f);
    apply_if_default(params.repeat_penalty,    ov.repeat_penalty,    1.1f);  // gh#86
    apply_if_default(params.enable_thinking,   ov.enable_thinking,   true);  // gh#86
    apply_if_default(params.tool_call_mode,    ov.tool_call_mode, std::string{});  // gh#103
}
 
void ModelOrchestrator::apply_tier_sampler_defaults(
    GenerationParams& params, const std::string& tier_name)
{
    auto it = config_.models.tiers.find(tier_name);
    if (it == config_.models.tiers.end()) { return; }
    const auto& tier = it->second;
    TierSamplerOverrides ov;
    ov.temperature       = tier.temperature;
    ov.max_output_tokens = tier.max_output_tokens;
    ov.top_p             = tier.top_p;              // gh#85
    ov.top_k             = tier.top_k;              // gh#85
    ov.min_p             = tier.min_p;              // gh#85
    ov.presence_penalty  = tier.presence_penalty;   // gh#85
    ov.frequency_penalty = tier.frequency_penalty;  // gh#85
    ov.repeat_penalty    = tier.repeat_penalty;     // gh#86
    ov.enable_thinking   = tier.enable_thinking;    // gh#86
    ov.tool_call_mode    = tier.tool_call_mode;     // gh#103
    float before_temp = params.temperature;
    int before_max = params.max_tokens;
    apply_tier_sampler_overrides(params, ov);
    if (params.temperature != before_temp) {
        logger->info("Tier '{}' temperature applied: {}",
                     tier_name, params.temperature);
    }
    if (params.max_tokens != before_max) {
        logger->info("Tier '{}' max_output_tokens applied: {}",
                     tier_name, params.max_tokens);
    }
}
 
// ── VRAM-aware tier residency (v2.2.4, gh#57) ──────────────
 
size_t ModelOrchestrator::resolve_vram_budget_bytes() {
    const char* env = std::getenv("ENTROPIC_VRAM_BUDGET_BYTES");
    if (env == nullptr) {
        // Not set at all: fall through to the device.
        return static_cast<size_t>(query_device_free_vram_bytes());
    }
    // SET, so it takes control — including when it is empty or unparseable,
    // which resolve to 0 and therefore DISABLE the gate. That is the escape
    // hatch for an operator who wants the pre-gh#142 behaviour back, and it is
    // what the v2.3.10 contract already specified; the device fallback must not
    // quietly override an explicit setting.
    size_t budget = 0;
    if (*env != '\0') {
        try {
            long long v = std::stoll(env);
            budget = (v < 0) ? 0 : static_cast<size_t>(v);
        } catch (...) {
            budget = 0;
        }
    }
    return budget;
}
 
void ModelOrchestrator::ensure_vram_budget() {
    if (vram_budget_resolved_) { return; }
    const bool from_env = std::getenv("ENTROPIC_VRAM_BUDGET_BYTES") != nullptr;
    vram_budget_bytes_ = resolve_vram_budget_bytes();
    vram_budget_resolved_ = true;
    // The pre-2.14.0 line labelled every budget "(ENTROPIC_VRAM_BUDGET_BYTES)",
    // including the far more common device-measured one.
    logger->info("[residency] VRAM budget: {} bytes ({})", vram_budget_bytes_,
                 from_env ? "ENTROPIC_VRAM_BUDGET_BYTES" : "device free VRAM");
}
 
bool ModelOrchestrator::budget_resolves_at_init(const ParsedConfig& config) {
    return !config.models.defer_load
        || std::getenv("ENTROPIC_VRAM_BUDGET_BYTES") != nullptr;
}
 
 
static FootprintInputs footprint_inputs_for(
    const TierConfig& tier_cfg, uint64_t weights_bytes, int vram_reserve_mb) {
    FootprintInputs in;
    in.weights_bytes = weights_bytes;
    in.gpu_layers = tier_cfg.gpu_layers;
    in.context_length = tier_cfg.context_length;
    in.cache_type_k = tier_cfg.cache_type_k;
    in.cache_type_v = tier_cfg.cache_type_v;
    in.vram_reserve_mb = vram_reserve_mb;
    // gh#144 (v2.12.0): context_length is per session, so the pool's KV term
    // multiplies. Omitting this under-counts by exactly N and can admit a
    // configuration that aborts the host — what this gate exists to prevent.
    in.max_sessions = tier_cfg.max_sessions;
    if (!tier_cfg.mmproj_path.empty()) {
        std::error_code proj_ec;
        auto proj = std::filesystem::file_size(tier_cfg.mmproj_path, proj_ec);
        if (!proj_ec) { in.mmproj_bytes = proj; }
    }
    // v2.13.1: the GGUF's real shape, and what an expert split moves off the
    // card. Without these the budget gate re-prices a placement `auto` just
    // derived WITHOUT knowing about the split: it sees gpu_layers == -1,
    // calls that full residency, charges the whole file, and refuses a
    // configuration auto had already measured as fitting. Two estimators
    // disagreeing about one tier is the defect this release exists to
    // remove, so both go through this builder and both see the same shape.
    const GgufShape shape = read_gguf_shape(tier_cfg.path);
    if (shape.known) {
        in.block_count = shape.block_count;
        in.block_bytes = shape.block_bytes;
        in.non_block_bytes = shape.non_block_bytes;
        if (tier_cfg.cpu_moe_layers > 0) {
            in.host_expert_bytes =
                shape.expert_bytes_per_block()
                * static_cast<uint64_t>(tier_cfg.cpu_moe_layers);
        }
    }
    return in;
}
 
size_t ModelOrchestrator::estimate_footprint_bytes(
    const std::string& tier_name) const {
    auto tier_it = config_.models.tiers.find(tier_name);
    if (tier_it == config_.models.tiers.end()) { return 0; }
    const auto& tier_cfg = tier_it->second;
    std::error_code ec;
    auto weights = std::filesystem::file_size(tier_cfg.path, ec);
    if (ec) { return 0; }
    FootprintInputs in = footprint_inputs_for(
        tier_cfg, weights, config_.vram_reserve_mb);
    // gh#142: an unpriceable placement returns 0 = "unknown", which leaves the
    // gate open. Guessing here would refuse working configurations — a 13 GB
    // model at gpu_layers=15 runs fine on an 11 GB card.
    return static_cast<size_t>(estimate_vram_footprint(in).bytes);
}
 
size_t ModelOrchestrator::tier_footprint_bytes(
    const std::string& tier_name) const {
    std::lock_guard<std::mutex> lock(swap_mutex_);
    auto it = tier_footprint_bytes_.find(tier_name);
    if (it != tier_footprint_bytes_.end()) { return it->second; }
    size_t v = estimate_footprint_bytes(tier_name);
    if (v > 0) {
        tier_footprint_bytes_[tier_name] = v;
    }
    return v;
}
 
void ModelOrchestrator::set_residency_observer(ResidencyObserverFn cb) {
    std::lock_guard<std::mutex> lock(swap_mutex_);
    residency_observer_ = std::move(cb);
}
 
void ModelOrchestrator::fire_residency_observer(
    ResidencyEvent event,
    const std::string& tier_name,
    const std::string& model_path,
    size_t footprint) {
    const char* event_name = "unknown";
    switch (event) {
    case ResidencyEvent::Loaded:         event_name = "loaded"; break;
    case ResidencyEvent::Evicted:        event_name = "evicted"; break;
    case ResidencyEvent::ActivationSwap: event_name = "activation_swap"; break;
    }
    logger->info("[residency] {} tier='{}' path='{}' footprint={} bytes",
                 event_name, tier_name, model_path, footprint);
    if (residency_observer_) {
        residency_observer_(event, tier_name, model_path, footprint);
    }
}
 
static nlohmann::json make_residency_entry(
    const std::string& name, const std::filesystem::path& path,
    int context_length, size_t footprint, int vram_reserve_mb,
    long long last_ms) {
    std::error_code ec;
    auto weights = std::filesystem::file_size(path, ec);
    size_t weights_b = ec ? 0u : static_cast<size_t>(weights);
    size_t kv = static_cast<size_t>(context_length) * 16ull * 1024ull;
    size_t headroom = static_cast<size_t>(vram_reserve_mb)
        * 1024ull * 1024ull;
    return {
        {"tier",               name},
        {"model_path",         path.string()},
        {"footprint_bytes",    footprint},
        {"weights_bytes",      weights_b},
        {"kv_cache_bytes",     kv},
        {"headroom_bytes",     headroom},
        {"last_activation_ms", last_ms}
    };
}
 
void ModelOrchestrator::forget_session_kv(const std::string& session_key) {
    // Outermost lock: this mutates the live llama_context's KV cells, which
    // is precisely what a concurrent decode is reading. See the declaration
    // for why api_mutex -> generation_mutex_ closes no cycle.
    std::lock_guard<std::recursive_mutex> gen_lock(generation_mutex_);
    std::unordered_set<const InferenceBackend*> seen;
    for (const auto& [name, backend] : tiers_) {
        (void)name;
        if (!backend || !backend->is_loaded()) { continue; }
        if (!seen.insert(backend.get()).second) { continue; }
        backend->forget_session_kv(session_key);
    }
}
 
std::string ModelOrchestrator::residency_snapshot_json() const {
    std::lock_guard<std::mutex> lock(swap_mutex_);
    nlohmann::json j;
    j["vram_total_bytes"]     = vram_budget_bytes_;
    j["vram_budget_bytes"]    = vram_budget_bytes_;
    size_t in_use = 0;
    nlohmann::json arr = nlohmann::json::array();
    for (const auto& [name, backend] : tiers_) {
        if (!backend || !backend->is_loaded()) { continue; }
        auto tier_it = config_.models.tiers.find(name);
        if (tier_it == config_.models.tiers.end()) { continue; }
        auto fp_it = tier_footprint_bytes_.find(name);
        size_t footprint = (fp_it != tier_footprint_bytes_.end())
            ? fp_it->second : estimate_footprint_bytes(name);
        in_use += footprint;
        auto la = tier_last_activation_ms_.find(name);
        long long last_ms = (la != tier_last_activation_ms_.end())
            ? la->second : 0;
        arr.push_back(make_residency_entry(
            name, tier_it->second.path, tier_it->second.context_length,
            footprint, config_.vram_reserve_mb, last_ms));
    }
    j["residency"] = std::move(arr);
    j["vram_headroom_bytes"] = vram_budget_bytes_ > in_use
        ? vram_budget_bytes_ - in_use
        : 0u;
    // gh#198: a status read must never be the thing that creates the CUDA
    // context, so an unresolved budget reports as deferred rather than probing.
    const bool deferred = vram_budget_deferred_ && !vram_budget_resolved_;
    j["backend"] = deferred ? "deferred"
                 : (vram_budget_bytes_ > 0 ? "configured" : "unknown");
    return j.dump();
}
 
} // namespace entropic