llama_cpp_backend.cpp

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#include "llama_cpp_backend.h"
#include <entropic/mcp/utf8_sanitize.h>
#include "grammar_source.h"
#include "llama_cpp_sampler.h"
#include "llama_cpp_tokenizer.h"
#include "warm_keep_util.h"  // gh#96: common_prefix_len / warm_keep_cut
#include "tool_call_markers.h"  // gh#103: family-aware tool-call close marker
#include "batch_util.h"  // gh#98: batch_shared_prefix_len / batch_is_viable
#include "mtp_envelope.h"  // gh#108: mtp_unsupported_reason (fail-loud envelope)
 
#include <entropic/inference/adapters/adapter_base.h>  // gh#90 coerce_string_typed_args
#include <entropic/types/logging.h>
 
#include <common.h>
#include <chat.h>
#include <sampling.h>
#include <speculative.h>
#include <mtmd.h>
#include <mtmd-helper.h>
 
#include <nlohmann/json.hpp>
 
#include <cmath>
#include <cstring>
#include <optional>
#include <stdexcept>
 
namespace entropic {
 
namespace {
 
auto logger = entropic::log::get("inference.llama_cpp");
 
bool ends_with(const std::string& text, const std::string& suffix) {
    return text.size() >= suffix.size()
        && text.compare(text.size() - suffix.size(), suffix.size(), suffix) == 0;
}
 
bool check_stop_sequences(
    const std::string& text,
    const std::vector<std::string>& stop_sequences)
{
    for (const auto& stop : stop_sequences) {
        if (!stop.empty() && ends_with(text, stop)) {
            return true;
        }
    }
    return false;
}
 
GenerationResult prefill_error() {
    GenerationResult r;
    r.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
    r.error_message = "Prefill decode failed";
    r.finish_reason = "error";
    return r;
}
 
void log_sampler_config(const GenerationParams& params) {
    logger->info("Sampler: temp={:.2f}, top_k={}, top_p={:.2f}, "
                 "repeat_penalty={:.2f}, thinking={}",
                 params.temperature, params.top_k, params.top_p,
                 params.repeat_penalty, params.enable_thinking);
}
 
void finalize_result(GenerationResult& result,
    std::chrono::steady_clock::time_point start_time)
{
    auto end = entropic::log::now();
    result.generation_time_ms = entropic::log::elapsed_ms(
        start_time, end);
    if (result.token_count > 0 && result.generation_time_ms > 0.0) {
        result.throughput_tok_s =
            static_cast<double>(result.token_count)
            / result.generation_time_ms * 1000.0;
    }
    logger->info("Generated: {} tokens, finish={}, {:.0f}ms, "
                 "{:.1f} tok/s",
                 result.token_count, result.finish_reason,
                 result.generation_time_ms, result.throughput_tok_s);
    logger->info("Content:\n{}", result.content);
}
 
void finalize_generation(GenerationResult& result,
    const std::string& generated, int n_generated,
    const GenerationParams& params,
    std::chrono::steady_clock::time_point t0)
{
    if (n_generated >= params.max_tokens
            && result.finish_reason.empty()) {
        result.finish_reason = "length";
    }
    // gh#136 (4th type_error.316 recurrence): sanitize at INGRESS, where model
    // bytes first become a std::string, rather than at each of the ~18 .dump()
    // sites downstream. gh#112/113 were closed as "permanent closure of the
    // 316 family", then gh#114, gh#118, gh#132 and gh#136 each patched one more
    // egress. The exits keep multiplying; the entries do not. Guarding here
    // makes every downstream dump safe by construction, and a NEW .dump()
    // anywhere cannot reintroduce the bug.
    //
    // Once at finalization, never per-token: a multi-byte codepoint can split
    // across token boundaries and per-token sanitize would corrupt valid
    // output (see the same reasoning at response_generator.cpp:360, which has
    // guarded the streaming accumulator this way since v2.1.1).
    //
    // Safe for raw_content consumers: raw_content is a COPY of content
    // (orchestrator.cpp:514), and sanitize only replaces bytes that are
    // ALREADY invalid UTF-8 — which cannot form part of any valid JSON token,
    // so the gh#88 envelope recovery and fenced-JSON fallbacks are unaffected.
    result.content = entropic::mcp::sanitize_utf8(generated);
    result.token_count = n_generated;
    finalize_result(result, t0);
}
 
GenerationResult sampler_init_error(
    std::chrono::steady_clock::time_point t0)
{
    GenerationResult r;
    r.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
    r.error_message = "Sampler factory not initialized";
    r.finish_reason = "error";
    finalize_result(r, t0);
    return r;
}
 
ggml_type parse_kv_cache_type(const std::string& s) {
    static const std::pair<const char*, ggml_type> kTable[] = {
        {"f16",  GGML_TYPE_F16},
        {"f32",  GGML_TYPE_F32},
        {"bf16", GGML_TYPE_BF16},
        {"q8_0", GGML_TYPE_Q8_0},
        {"q4_0", GGML_TYPE_Q4_0},
    };
    for (const auto& [name, type] : kTable) {
        if (s == name) { return type; }
    }
    logger->warn("Unknown cache_type '{}' — defaulting to f16", s);
    return GGML_TYPE_F16;
}
 
llama_split_mode parse_split_mode(const std::string& s) {
    if (s.empty()) { return LLAMA_SPLIT_MODE_LAYER; }
    static const std::pair<const char*, llama_split_mode> kTable[] = {
        {"none",  LLAMA_SPLIT_MODE_NONE},
        {"layer", LLAMA_SPLIT_MODE_LAYER},
        {"row",   LLAMA_SPLIT_MODE_ROW},
    };
    for (const auto& [name, mode] : kTable) {
        if (s == name) { return mode; }
    }
    logger->warn("Unknown split_mode '{}' — defaulting to layer", s);
    return LLAMA_SPLIT_MODE_LAYER;
}
 
llama_model_params build_load_mparams(const entropic::ModelConfig& cfg) {
    llama_model_params m = llama_model_default_params();
    m.n_gpu_layers = cfg.gpu_layers;
    m.use_mmap = true;
    m.use_mlock = cfg.use_mlock;
    m.split_mode = parse_split_mode(cfg.split_mode);
    // gh#23 MVP item 7 (v2.3.19): main_gpu. Effective when split_mode
    // is "none" (pin) or "row" (small-tensor placement). 0 keeps
    // pre-v2.3.19 load bit-for-bit.
    m.main_gpu = cfg.main_gpu;
    return m;
}
 
} // anonymous namespace
 
// ── Lifecycle ──────────────────────────────────────────────
 
bool LlamaCppBackend::do_load(const ModelConfig& config) {
    llama_model_params mparams = llama_model_default_params();
    mparams.n_gpu_layers = 0;
    mparams.use_mmap = true;
    mparams.use_mlock = config.use_mlock;
 
    model_ = llama_model_load_from_file(config.path.c_str(), mparams);
    if (!model_) {
        last_error_ = "llama_model_load_from_file failed: " + config.path.string();
        return false;
    }
 
    vocab_ = llama_model_get_vocab(model_);
    is_recurrent_ = llama_model_is_recurrent(model_);
    is_hybrid_ = llama_model_is_hybrid(model_);  // gh#97: attn + recurrent/SSM
    // v2.3.10: wire the Tokenizer seam now that vocab_ is valid.
    // Lifetime: tokenizer_ borrows vocab_; do_unload resets
    // tokenizer_ BEFORE freeing the model so the borrow never dangles.
    tokenizer_ = std::make_unique<LlamaCppTokenizer>(vocab_);
    logger->info("Model loaded (CPU): {} tokens in vocab, recurrent={}",
              llama_vocab_n_tokens(vocab_), is_recurrent_);
    return true;
}
 
namespace {
llama_context_params build_cparams(const entropic::ModelConfig& cfg) {
    llama_context_params c = llama_context_default_params();
    c.n_ctx = static_cast<uint32_t>(cfg.context_length);
    c.n_batch = static_cast<uint32_t>(cfg.n_batch);
    // gh#23 MVP item 5 (v2.3.17): n_ubatch. 0 keeps llama.cpp's default
    // (== n_batch in practice), preserving pre-v2.3.17 chunking.
    if (cfg.n_ubatch > 0) {
        c.n_ubatch = static_cast<uint32_t>(cfg.n_ubatch);
    }
    c.n_threads = cfg.n_threads > 0
        ? static_cast<uint32_t>(cfg.n_threads)
        : std::thread::hardware_concurrency();
    c.flash_attn_type = cfg.flash_attn
        ? LLAMA_FLASH_ATTN_TYPE_ENABLED
        : LLAMA_FLASH_ATTN_TYPE_DISABLED;
    c.type_k = parse_kv_cache_type(cfg.cache_type_k);
    c.type_v = parse_kv_cache_type(cfg.cache_type_v);
    // gh#23 MVP item 8 (v2.3.20): offload_kqv. true (default) matches
    // llama.cpp's default — bit-identical for callers not opting out.
    c.offload_kqv = cfg.offload_kqv;
    // gh#23 MVP items 9 + 10 (v2.3.21 + v2.3.22): RoPE frequency
    // overrides. Both 0.0 = use model's trained value — bit-identical.
    c.rope_freq_base = cfg.rope_freq_base;
    c.rope_freq_scale = cfg.rope_freq_scale;
    // gh#23 MVP item 11 (v2.3.23): n_parallel maps to cparams.n_seq_max.
    // 1 (default) matches llama.cpp's default — bit-identical.
    c.n_seq_max = static_cast<uint32_t>(cfg.n_parallel);
    // gh#98 (v2.8.0): a unified KV buffer is REQUIRED for llama_memory_seq_cp
    // (the same-prefix batch fan-out) — seq_cp asserts on per-sequence buffers.
    // llama.cpp also recommends kv_unified exactly when sequences share a large
    // prefix (our case). Only enabled when batching is configured (n_parallel>1)
    // so single-sequence handles keep llama.cpp's default.
    c.kv_unified = (cfg.n_parallel > 1);
    // gh#108 (v2.9.2): llama_context_default_params() returns swa_full=true (a
    // full-context SWA cache), but the CLI default is false. For Gemma-4 (mostly
    // sliding-window: window=512, 5:1 SWA:global) the un-windowed cache wastes
    // ~5 GB at 128k. Set false — the memory-efficient windowed mode. Validated
    // against warm-keep / prompt-cache reuse over a >window prefix (the SWA layers
    // keep only the last `window` tokens, so KV reuse must not assume full-context
    // SWA residency — covered by the long-context warm-keep model test).
    c.swa_full = false;
    return c;
}
} // anonymous namespace
 
bool LlamaCppBackend::do_activate() {
    if (!load_gpu_model()) { return false; }
    if (!create_inference_context()) { return false; }
    // v2.3.10: wire the Sampler seam once ctx_ / vocab_ are live.
    // Lifetime: factory borrows ctx_ + vocab_; do_deactivate /
    // do_unload reset sampler_factory_ BEFORE freeing those handles
    // so the borrow never dangles.
    sampler_factory_ = std::make_unique<LlamaCppSamplerFactory>(
        ctx_, vocab_);
    init_mmproj_if_configured();
    return true;
}
 
bool LlamaCppBackend::load_gpu_model() {
    llama_model_params mparams = build_load_mparams(config());
 
    if (!config().tensor_split.empty()) {
        // TODO: parse tensor_split string into float array for multi-GPU
        logger->warn("tensor_split not yet implemented, ignoring");
    }
 
    // tokenizer_ borrows the old vocab_; reset it before the free so the
    // borrow never dangles. Then free the WARM model and null the
    // handles so a failed reload below leaves the backend in a clean,
    // recoverable state rather than a dangling one.
    tokenizer_.reset();
    if (model_ != nullptr) {
        llama_model_free(model_);
        model_ = nullptr;
        vocab_ = nullptr;
    }
 
    model_ = llama_model_load_from_file(config().path.c_str(), mparams);
    if (model_ == nullptr) {
        // llama.cpp returns null with no error string — the actual
        // reason (OOM, CUDA init failure, GGUF parse error, etc.) only
        // surfaces in ggml's log stream. Point the operator at it so
        // multi-handle GPU failures (gh#58 v2.2.7 follow-up) are
        // diagnosable without source-diving llama.cpp.
        last_error_ = "Failed to reload model with GPU layers "
                      "(path=" + config().path.string()
                    + ", gpu_layers=" + std::to_string(config().gpu_layers)
                    + ") — check llama_ggml.log in the engine's log_dir "
                      "for the underlying llama.cpp/CUDA error";
        return false;
    }
 
    vocab_ = llama_model_get_vocab(model_);
    tokenizer_ = std::make_unique<LlamaCppTokenizer>(vocab_);
    return true;
}
 
bool LlamaCppBackend::create_inference_context() {
    llama_context_params cparams = build_cparams(config());
 
    ctx_ = llama_init_from_model(model_, cparams);
    if (!ctx_) {
        last_error_ = "llama_init_from_model failed";
        return false;
    }
 
    logger->info("Context created: n_ctx={}, n_batch={}, "
                 "flash_attn={}, type_k={}, type_v={}",
                 config().context_length, config().n_batch,
                 config().flash_attn,
                 config().cache_type_k, config().cache_type_v);
 
    // Initialize prompt cache if not already created
    if (!prompt_cache_) {
        prompt_cache_ = std::make_unique<PromptCache>(
            prompt_cache_config_.max_bytes);
        logger->info("Prompt cache initialized: max_bytes={}",
                     prompt_cache_config_.max_bytes);
    }
    return true;
}
 
void LlamaCppBackend::init_mmproj_if_configured() {
    if (config().mmproj_path.empty()) {
        has_vision_ = false;
        return;
    }
    auto ctx_params = mtmd_context_params_default();
    ctx_params.use_gpu = (config().gpu_layers != 0);
    ctx_params.flash_attn_type = config().flash_attn
        ? LLAMA_FLASH_ATTN_TYPE_ENABLED
        : LLAMA_FLASH_ATTN_TYPE_DISABLED;
    ctx_params.print_timings = false;
    mtmd_ctx_ = mtmd_init_from_file(
        config().mmproj_path.c_str(), model_, ctx_params);
    if (mtmd_ctx_ == nullptr) {
        logger->error("mtmd_init_from_file failed for {} — "
                      "continuing in text-only mode",
                      config().mmproj_path.string());
        has_vision_ = false;
        return;
    }
    has_vision_ = mtmd_support_vision(mtmd_ctx_);
    logger->info("mmproj loaded from {} — vision={}",
                 config().mmproj_path.string(), has_vision_);
}
 
void LlamaCppBackend::teardown_mtp_draft() {
    if (mtp_draft_ctx_ != nullptr) {
        llama_free(mtp_draft_ctx_);
        mtp_draft_ctx_ = nullptr;
    }
    if (mtp_draft_model_ != nullptr) {
        llama_model_free(mtp_draft_model_);
        mtp_draft_model_ = nullptr;
    }
    mtp_head_path_.clear();
}
 
static int effective_n_draft(int n_max) {
    return (n_max > 0) ? n_max : 16;
}
 
bool LlamaCppBackend::setup_mtp_draft(const std::string& head_path, int n_max) {
    mtp_n_max_ = effective_n_draft(n_max);
    if (mtp_draft_ctx_ != nullptr && mtp_head_path_ == head_path) {
        return true;  // live head already bound to this ctx_
    }
    teardown_mtp_draft();
    return build_mtp_head(head_path);
}
 
bool LlamaCppBackend::build_mtp_head(const std::string& head_path) {
    if (ctx_ == nullptr) {
        last_error_ = "MTP setup requires an ACTIVE target context";
        return false;
    }
    if (head_path.empty()) {
        // gh#108: fail loud before llama_model_load_from_file("") — a bare
        // mtp=true with no draft.path is a config error, not a load to attempt.
        last_error_ = "MTP requires speculative.draft.path (the head GGUF); "
                      "none configured";
        return false;
    }
    llama_model_params mparams = llama_model_default_params();
    mparams.n_gpu_layers = config().gpu_layers;  // head is tiny — follow target
    mparams.use_mmap = true;
    mtp_draft_model_ = llama_model_load_from_file(head_path.c_str(), mparams);
    if (mtp_draft_model_ != nullptr) {
        llama_context_params cparams = build_cparams(config());
        cparams.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
        cparams.ctx_other = ctx_;   // share the target's KV memory
        cparams.n_rs_seq = 0;
        mtp_draft_ctx_ = llama_init_from_model(mtp_draft_model_, cparams);
    }
    bool ok = (mtp_draft_ctx_ != nullptr);
    if (ok) {
        mtp_head_path_ = head_path;
        logger->info("MTP head ready: {} (n_max={}, ctx_other=target, "
                     "shared-KV)", head_path, mtp_n_max_);
    } else {
        last_error_ = "MTP head setup failed: " + head_path;
        teardown_mtp_draft();
    }
    return ok;
}
 
void LlamaCppBackend::do_deactivate() {
    // gh#108 (v2.9.1): serialise vs an in-flight generate_mtp — it holds
    // mtp_mutex_ across its decode, so this blocks until that decode finishes
    // before freeing the MTP head + ctx_ (no deactivate-during-generate UAF).
    std::lock_guard<std::mutex> lk(mtp_mutex_);
    // gh#106 (v2.9.0): the MTP head borrows ctx_ via ctx_other — free it
    // FIRST so the borrow never dangles past the context.
    teardown_mtp_draft();
    // v2.3.10: sampler factory borrows ctx_ + vocab_. Release it
    // BEFORE freeing the context so the borrow never dangles.
    sampler_factory_.reset();
    // v2.1.8: mtmd holds a reference to the live llama_model — free
    // it before the GPU model is unloaded below.
    if (mtmd_ctx_ != nullptr) {
        mtmd_free(mtmd_ctx_);
        mtmd_ctx_ = nullptr;
        has_vision_ = false;
    }
    if (ctx_) {
        llama_free(ctx_);
        ctx_ = nullptr;
    }
    invalidate_resident_kv();  // gh#96: KV is gone with the context
 
    // Free the GPU model FIRST (releasing VRAM — the point of
    // deactivate), then reload CPU-only for the WARM state. tokenizer_
    // borrows the old vocab_, so reset it before the free.
    tokenizer_.reset();
    if (model_ != nullptr) {
        llama_model_free(model_);
        model_ = nullptr;
        vocab_ = nullptr;
    }
    reload_model_cpu_only();
}
 
void LlamaCppBackend::reload_model_cpu_only() {
    llama_model_params mparams = llama_model_default_params();
    mparams.n_gpu_layers = 0;
    mparams.use_mmap = true;
    mparams.use_mlock = config().use_mlock;
 
    model_ = llama_model_load_from_file(config().path.c_str(), mparams);
    if (model_ != nullptr) {
        vocab_ = llama_model_get_vocab(model_);
        tokenizer_ = std::make_unique<LlamaCppTokenizer>(vocab_);
    } else {
        // VRAM is released, but the warm-reload failed: leave the handle
        // null (state stays recoverable — the next activate reloads from
        // scratch). Error, not warn: a same-file CPU reload failing here
        // signals real trouble (disk/OOM).
        logger->error("Failed to reload CPU model during deactivate "
                      "(path={}); backend left unloaded until next activate",
                      config().path.string());
    }
}
 
LlamaCppBackend::~LlamaCppBackend() {
    do_unload();
}
 
void LlamaCppBackend::inject_tokenizer_for_test(
    std::unique_ptr<Tokenizer> tokenizer)
{
    tokenizer_ = std::move(tokenizer);
    state_.store(ModelState::WARM, std::memory_order_release);
}
 
void LlamaCppBackend::inject_sampler_factory_for_test(
    std::unique_ptr<SamplerFactory> factory)
{
    sampler_factory_ = std::move(factory);
}
 
void LlamaCppBackend::do_unload() {
    // gh#108 (v2.9.1): serialise vs in-flight generate_mtp (see do_deactivate).
    std::lock_guard<std::mutex> lk(mtp_mutex_);
 
    // gh#108 (v2.10.3): invalidate the sticky parse snapshot. parse_params_valid_
    // was set once by a tooled render (gh#105) and never cleared by anything —
    // not render_prompt (which clears only have_chat_params_), not unload. If a
    // backend instance were reused across a load of a different GGUF, the stale
    // PEG arena would survive AND parse_chat_format_ would still report the old
    // family, so common_chat_parse_reliable() would answer true and parse the
    // new model's output with the previous model's parser. Clearing here makes
    // that unreachable by construction rather than by luck of instance reuse.
    parse_params_valid_ = false;
    parse_chat_format_ = 0;
    parse_generation_prompt_.clear();
    parse_parser_.clear();
    // gh#134 (v2.10.4): same reasoning — a reused backend must not apply the
    // previous model's tool-call grammar to a new model's decode.
    tool_grammar_.clear();
    tool_grammar_lazy_ = false;
    if (prompt_cache_) {
        prompt_cache_->clear();
    }
    // gh#106 (v2.9.0): MTP head borrows ctx_ — free it before the context.
    teardown_mtp_draft();
    // v2.3.10: sampler factory borrows ctx_ + vocab_ — release it
    // BEFORE the context/model are freed below so the borrow never
    // points into freed memory. (do_deactivate normally releases
    // this earlier; this reset is the WARM→COLD safety net.)
    sampler_factory_.reset();
    // v2.3.10: tokenizer borrows vocab_ — release it BEFORE the model
    // is freed so the borrow never points into freed memory.
    tokenizer_.reset();
    // v2.1.8: mtmd must be freed before the underlying llama_model.
    if (mtmd_ctx_ != nullptr) {
        mtmd_free(mtmd_ctx_);
        mtmd_ctx_ = nullptr;
        has_vision_ = false;
    }
    if (ctx_) {
        llama_free(ctx_);
        ctx_ = nullptr;
    }
    invalidate_resident_kv();  // gh#96: KV is gone with the context
    if (model_) {
        llama_model_free(model_);
        model_ = nullptr;
    }
    vocab_ = nullptr;
}
 
// ── Tokenization ───────────────────────────────────────────
 
std::vector<llama_token> LlamaCppBackend::tokenize(
    const std::string& text, bool add_special) const
{
    // v2.3.10: route through the Tokenizer seam. tokenizer_ is set
    // in do_load (real impl) or via inject_tokenizer_for_test (mock).
    // Returns empty when no tokenizer is wired — matches the prior
    // failure-path return shape.
    if (!tokenizer_) { return {}; }
    auto ids = tokenizer_->tokenize(text, add_special);
    // llama_token is int32_t; vector conversion is a copy through
    // iterators since the value type matches.
    return {ids.begin(), ids.end()};
}
 
std::string LlamaCppBackend::detokenize(llama_token token) const {
    // v2.3.10: route through Tokenizer seam. The special=false /
    // gh#68 history + defensive rationale now lives in
    // LlamaCppTokenizer::detokenize. Returns empty when no
    // tokenizer is wired — matches prior failure-path return.
    if (!tokenizer_) { return {}; }
    return tokenizer_->detokenize(static_cast<int32_t>(token));
}
 
int LlamaCppBackend::do_count_tokens(const std::string& text) const {
    auto tokens = tokenize(text, false);
    return static_cast<int>(tokens.size());
}
 
std::vector<int32_t> LlamaCppBackend::tokenize_text(
    const std::string& text) const {
    auto tokens = tokenize(text, true);
    return {tokens.begin(), tokens.end()};
}
 
// ── Evaluation (v1.9.10) ──────────────────────────────────
 
LogprobResult LlamaCppBackend::do_evaluate_logprobs(
    const int32_t* tokens,
    int n_tokens)
{
    int n_vocab = llama_vocab_n_tokens(vocab_);
    LogprobResult result;
    result.tokens.assign(tokens, tokens + n_tokens);
    result.n_tokens = n_tokens;
    result.n_logprobs = n_tokens - 1;
    result.logprobs.reserve(result.n_logprobs);
 
    auto* mem = llama_get_memory(ctx_);
    llama_memory_clear(mem, true);
 
    for (int i = 0; i < n_tokens; i++) {
        llama_token tok = tokens[i];
        llama_batch batch = llama_batch_get_one(&tok, 1);
        int rc = llama_decode(ctx_, batch);
        if (rc != 0) {
            llama_memory_clear(mem, true);
            throw std::runtime_error("llama_decode failed at logprob pos");
        }
        if (i < n_tokens - 1) {
            const float* logits = llama_get_logits_ith(ctx_, -1);
            float lp = extract_token_logprob(
                logits, tokens[i + 1], n_vocab);
            result.logprobs.push_back(lp);
        }
    }
 
    float sum = 0.0f;
    for (float lp : result.logprobs) { sum += lp; }
    result.total_logprob = sum;
    result.perplexity = std::exp(
        -sum / static_cast<float>(result.n_logprobs));
 
    llama_memory_clear(mem, true);
    return result;
}
 
llama_seq_id LlamaCppBackend::allocate_temp_seq_id() {
    std::lock_guard<std::mutex> lock(seq_id_mutex_);
    if (!free_seq_ids_.empty()) {
        auto id = free_seq_ids_.back();
        free_seq_ids_.pop_back();
        return id;
    }
    return next_temp_seq_id_++;
}
 
void LlamaCppBackend::release_temp_seq_id(llama_seq_id seq_id) {
    std::lock_guard<std::mutex> lock(seq_id_mutex_);
    free_seq_ids_.push_back(seq_id);
}
 
float LlamaCppBackend::extract_token_logprob(
    const float* logits,
    int32_t next_token,
    int n_vocab)
{
    float max_logit = logits[0];
    for (int v = 1; v < n_vocab; v++) {
        if (logits[v] > max_logit) {
            max_logit = logits[v];
        }
    }
    float sum_exp = 0.0f;
    for (int v = 0; v < n_vocab; v++) {
        sum_exp += std::exp(logits[v] - max_logit);
    }
    float log_sum_exp = max_logit + std::log(sum_exp);
    return logits[next_token] - log_sum_exp;
}
 
// ── Chat template ──────────────────────────────────────────
 
static std::vector<llama_chat_message> to_llama_chat(
    const std::vector<Message>& messages) {
    std::vector<llama_chat_message> chat_msgs;
    chat_msgs.reserve(messages.size());
    for (const auto& msg : messages) {
        chat_msgs.push_back({msg.role.c_str(), msg.content.c_str()});
    }
    return chat_msgs;
}
 
static std::vector<common_chat_msg> to_common_chat(
    const std::vector<Message>& messages) {
    std::vector<common_chat_msg> out;
    out.reserve(messages.size());
    for (const auto& msg : messages) {
        common_chat_msg cm;
        cm.role = msg.role;
        cm.content = msg.content;
        out.push_back(std::move(cm));
    }
    return out;
}
 
static std::vector<common_chat_tool> mcp_tools_to_common_chat(
    const std::string& tools_json) {
    std::vector<common_chat_tool> out;
    if (tools_json.empty()) { return out; }
    auto arr = nlohmann::json::parse(tools_json, nullptr, false);
    if (!arr.is_array()) { return out; }
    for (const auto& t : arr) {
        common_chat_tool ct;
        ct.name = t.value("name", "");
        ct.description = t.value("description", "");
        if (t.contains("inputSchema")) {
            ct.parameters = t["inputSchema"].dump();
        }
        if (!ct.name.empty()) { out.push_back(std::move(ct)); }
    }
    return out;
}
 
static ToolCall to_entropic_tool_call(const common_chat_tool_call& cc) {
    ToolCall tc;
    tc.id = cc.id;
    tc.name = cc.name;
    tc.arguments_json = cc.arguments;
    auto j = nlohmann::json::parse(cc.arguments, nullptr, false);
    if (j.is_object()) {
        for (auto it = j.begin(); it != j.end(); ++it) {
            tc.arguments[it.key()] =
                it->is_string() ? it->get<std::string>() : it->dump();
        }
    }
    return tc;
}
 
static std::optional<common_chat_params> render_common_chat(
    llama_model* model,
    const std::vector<Message>& messages,
    const GenerationParams& params,
    const std::vector<common_chat_tool>& tools,
    bool require_tool_call) {
    if (model == nullptr) { return std::nullopt; }
    auto tmpls = common_chat_templates_init(model, "");
    std::optional<common_chat_params> out;
    if (tmpls) {
        common_chat_templates_inputs inputs;
        inputs.messages = to_common_chat(messages);
        inputs.add_generation_prompt = true;
        inputs.use_jinja = true;
        inputs.enable_thinking = params.enable_thinking;  // gh#86
        inputs.tools = tools;
        if (!tools.empty()) {
            // gh#134 (v2.10.4): was hardcoded AUTO, so a mandatory-tool tier
            // could always answer in prose. REQUIRED makes upstream build the
            // tool-call grammar eagerly (grammar_lazy=false), which the
            // sampler now actually applies — see to_common_sampling.
            inputs.tool_choice = require_tool_call
                ? COMMON_CHAT_TOOL_CHOICE_REQUIRED
                : COMMON_CHAT_TOOL_CHOICE_AUTO;
        }
        try {
            out = common_chat_templates_apply(tmpls.get(), inputs);
        } catch (const std::exception& e) {
            logger->warn("jinja chat template apply failed ({}); "
                         "falling back to low-level template", e.what());
        }
    }
    return out;
}
 
static std::string concat_messages_fallback(
    const std::vector<Message>& messages) {
    std::string fallback;
    for (const auto& msg : messages) {
        fallback += msg.role + ": " + msg.content + "\n";
    }
    return fallback;
}
 
std::string LlamaCppBackend::apply_chat_template(
    const std::vector<Message>& messages,
    const GenerationParams& params) const
{
    // No tools staged, so tool_choice is moot — pass false.
    auto rendered = render_common_chat(model_, messages, params, {}, false);
    return rendered ? rendered->prompt
                    : apply_chat_template_lowlevel(messages);
}
 
std::string LlamaCppBackend::render_prompt(
    const std::vector<Message>& messages,
    const GenerationParams& params)
{
    if (!active_tools_json_.empty()) {
        return render_with_tools(messages, params);
    }
    have_chat_params_ = false;
    return apply_chat_template(messages, params);
}
 
void LlamaCppBackend::set_active_tools(const std::string& tools_json) {
    active_tools_json_ = tools_json;
    logger->info("Active tools staged for common_chat render: {} bytes",
                 tools_json.size());
}
 
std::string LlamaCppBackend::render_with_tools(
    const std::vector<Message>& messages,
    const GenerationParams& params)
{
    have_chat_params_ = false;
    auto tools = mcp_tools_to_common_chat(active_tools_json_);
    auto rendered = render_common_chat(model_, messages, params, tools,
                                       require_tool_call_);
    std::string prompt;
    if (rendered) {
        last_chat_format_ = static_cast<int>(rendered->format);
        last_generation_prompt_ = rendered->generation_prompt;
        last_parser_ = rendered->parser;
        have_chat_params_ = true;
        // gh#105: snapshot this TOOLED render for the engine's later re-parse.
        // A toolless interleave (validator critique) clears have_chat_params_
        // but NOT this snapshot, so parse_response still decodes the main call.
        parse_chat_format_ = last_chat_format_;
        parse_generation_prompt_ = last_generation_prompt_;
        parse_parser_ = last_parser_;
        parse_params_valid_ = true;
        // gh#134 (v2.10.4): keep the tool-call grammar the render derived from
        // the staged schemas. Discarding it is why tools-staged tiers decode
        // unconstrained today.
        tool_grammar_ = rendered->grammar;
        tool_grammar_lazy_ = rendered->grammar_lazy;
        prompt = rendered->prompt;
        logger->info("render_with_tools: format={}, {} tool(s), captured "
                     "parser ({} bytes), grammar ({} bytes, lazy={})",
                     last_chat_format_, tools.size(), last_parser_.size(),
                     tool_grammar_.size(), tool_grammar_lazy_);
        // gh#138: the silent failure a consumer cannot see. A tier asking for
        // tool_choice=REQUIRED whose render yields no grammar is unconstrained
        // — the flag is set, tools are staged, and nothing enforces the call.
        // Loud, because the alternative is what gh#138 actually experienced:
        // prose-only turns with no diagnostic anywhere.
        if (require_tool_call_ && tool_grammar_.empty()) {
            logger->error(
                "require_tool_call is set for this tier but the chat template "
                "derived NO tool-call grammar from {} staged tool(s) "
                "(format={}). The turn will decode unconstrained and may end "
                "in prose with no tool call. This template may not support "
                "tool_choice=REQUIRED.",
                tools.size(), last_chat_format_);
        }
    } else {
        // gh#137: this fallback CANNOT honor enable_thinking — the low-level
        // llama_chat_apply_template API has no slot for it. Saying so is the
        // difference between a diagnosable config problem and a mystery: a
        // tier that set enable_thinking:false still reasons here, and on
        // gemma4 the whole generation can be one unterminated <|channel>
        // block, which the reasoning strip then correctly reduces to zero
        // characters. The consumer sees "0 chars, 0 tool calls" and no cause.
        if (!params.enable_thinking) {
            logger->warn(
                "Falling back to the low-level chat template, which cannot "
                "honor enable_thinking:false — this tier WILL still emit "
                "reasoning, and a generation that is entirely reasoning is "
                "stripped to empty content. The jinja render was unavailable "
                "for this turn (no tools staged, or the template could not be "
                "applied).");
        }
        prompt = apply_chat_template_lowlevel(messages);
    }
    return prompt;
}
 
bool LlamaCppBackend::common_chat_parse_reliable() const {
    // gh#105: read the sticky last-TOOLED snapshot, not the live capture — the
    // engine queries this AFTER a toolless validator render would have cleared
    // have_chat_params_, so the live flag is unreliable here.
    return parse_params_valid_
        && parse_chat_format_ == COMMON_CHAT_FORMAT_PEG_GEMMA4;
}
 
std::string LlamaCppBackend::tool_call_close_marker() const {
    // last_chat_format_ is stored as int (the captured common_chat_format).
    return have_chat_params_
        ? close_marker_for_format(
              static_cast<common_chat_format>(last_chat_format_))
        : "";
}
 
std::vector<std::string> LlamaCppBackend::effective_stop(
    const GenerationParams& params) const {
    GenerationParams p = params;
    const std::size_t before = p.stop.size();
    append_sequential_stop(p, tool_call_close_marker());
    if (p.stop.size() > before) {
        logger->info("Sequential tier: tool-call close marker injected "
                     "post-render (gh#105) — hard-stop at first tool call");
    }
    return p.stop;
}
 
void strip_thinking_channels(std::string& content, std::string* reasoning_out) {
    static const std::string kOpen = "<|channel>";
    static const std::string kClose = "<channel|>";
    bool stripped = false;
    bool truncated_unclosed = false;
    std::size_t pos;
    while ((pos = content.find(kOpen)) != std::string::npos) {
        stripped = true;
        std::size_t end = content.find(kClose, pos + kOpen.size());
        if (end == std::string::npos) { truncated_unclosed = true; }
        std::size_t span_end =
            (end == std::string::npos) ? content.size() : end + kClose.size();
        if (reasoning_out != nullptr) {
            std::size_t inner = pos + kOpen.size();
            std::size_t inner_end =
                (end == std::string::npos) ? content.size() : end;
            reasoning_out->append(content, inner, inner_end - inner);
        }
        content.erase(pos, span_end - pos);
    }
    if (stripped) {
        std::size_t nb = content.find_first_not_of(" \t\r\n");
        content.erase(0, nb == std::string::npos ? content.size() : nb);
    }
    if (truncated_unclosed && content.empty()) {
        logger->warn("strip_thinking_channels: a <|channel> reasoning block "
                     "was never closed, so the strip removed the whole "
                     "generation and content is empty. Not a parse error. The "
                     "orchestrator reports whether this was a token budget or "
                     "the model ending its own turn — only it holds "
                     "finish_reason. gh#137.");
    }
}
 
LlamaCppBackend::CommonChatResult LlamaCppBackend::parse_response(
    const std::string& raw) const
{
    CommonChatResult result;
    // gh#105: decode from the sticky last-TOOLED snapshot (parse_*), NOT the
    // live capture — a toolless validator render between the main generation
    // and this re-parse would have cleared the live params.
    if (!parse_params_valid_) {
        result.content = raw;
        return result;
    }
    common_chat_parser_params pp;
    pp.format = static_cast<common_chat_format>(parse_chat_format_);
    pp.generation_prompt = parse_generation_prompt_;
    pp.parser.load(parse_parser_);  // mandatory — see header
    try {
        auto msg = common_chat_parse(raw, /*is_partial=*/false, pp);
        result.content = msg.content;
        result.reasoning_content = msg.reasoning_content;
        // gh#106: Gemma 4 QAT reasoning channels common_chat doesn't parse.
        strip_thinking_channels(result.content, &result.reasoning_content);
        for (const auto& tc : msg.tool_calls) {
            result.tool_calls.push_back(to_entropic_tool_call(tc));
        }
        // gh#90: gemma <|"|> string-escape loses type through PEG_GEMMA4 —
        // restore string typing for params the staged schema declares string.
        coerce_string_typed_args(result.tool_calls, active_tools_json_);
    } catch (const std::exception& e) {
        logger->warn("common_chat_parse failed ({}); raw kept as content",
                     e.what());
        result.content = raw;
    }
    return result;
}
 
std::string LlamaCppBackend::apply_chat_template_lowlevel(
    const std::vector<Message>& messages) const
{
    auto chat_msgs = to_llama_chat(messages);
 
    int n = llama_chat_apply_template(
        nullptr, chat_msgs.data(), chat_msgs.size(),
        true, nullptr, 0);
    if (n < 0) {
        logger->error("llama_chat_apply_template failed (size query)");
        return concat_messages_fallback(messages);
    }
 
    std::vector<char> buf(static_cast<size_t>(n + 1));
    int written = llama_chat_apply_template(
        nullptr, chat_msgs.data(), chat_msgs.size(),
        true, buf.data(), static_cast<int32_t>(buf.size()));
    if (written < 0) {
        logger->error("llama_chat_apply_template failed (render)");
        return concat_messages_fallback(messages);
    }
 
    return std::string(buf.data(), static_cast<size_t>(written));
}
 
// ── Sampler ────────────────────────────────────────────────
 
std::unique_ptr<Sampler> LlamaCppBackend::create_sampler(
    const GenerationParams& params) const
{
    if (!sampler_factory_) { return nullptr; }
    return sampler_factory_->create(params);
}
 
// ── Decode loop ────────────────────────────────────────────
 
bool LlamaCppBackend::run_prefill(const std::vector<llama_token>& tokens) {
    llama_memory_clear(llama_get_memory(ctx_), true);
 
    const int n_batch = config().n_batch;
    const int n_tokens = static_cast<int>(tokens.size());
 
    for (int i = 0; i < n_tokens; i += n_batch) {
        int chunk = std::min(n_batch, n_tokens - i);
        std::vector<llama_token> slice(
            tokens.begin() + i, tokens.begin() + i + chunk);
        llama_batch batch = llama_batch_get_one(
            slice.data(), static_cast<int32_t>(chunk));
        if (llama_decode(ctx_, batch) != 0) {
            logger->error("Prefill decode failed at offset {}", i);
            return false;
        }
    }
    last_prefill_tokens_ += n_tokens;  // gh#96: count tokens decoded in prefill
    return true;
}
 
std::string LlamaCppBackend::step_token(
    Sampler& sampler,
    std::string& generated,
    std::function<void(std::string_view)>& on_token,
    const std::vector<std::string>& stop)
{
    llama_token new_token = sampler.sample();
 
    if (new_token == llama_vocab_eos(vocab_)
        || llama_vocab_is_eog(vocab_, new_token)) {
        return "eos";
    }
 
    std::string piece = detokenize(new_token);
    generated += piece;
    if (on_token) {
        on_token(std::string_view(piece));
    }
    if (check_stop_sequences(generated, stop)) {
        return "stop";
    }
 
    llama_token tok = new_token;
    llama_batch single = llama_batch_get_one(&tok, 1);
    return (llama_decode(ctx_, single) == 0) ? "continue" : "error";
}
 
GenerationResult LlamaCppBackend::decode_loop(
    const std::vector<llama_token>& tokens,
    const GenerationParams& params,
    std::function<void(std::string_view)> on_token,
    std::atomic<bool>* cancel)
{
    // v2.3.10: Sampler seam — factory installed in do_activate.
    auto sampler = create_sampler(params);
    if (!sampler) {
        GenerationResult result;
        result.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
        result.error_message = "Sampler factory not initialized";
        result.finish_reason = "error";
        return result;
    }
 
    if (!run_prefill(tokens)) {
        GenerationResult result;
        result.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
        result.error_message = "Prefill decode failed";
        result.finish_reason = "error";
        return result;
    }
 
    return generate_after_prefill(*sampler, params, std::move(on_token), cancel);
}
 
GenerationResult LlamaCppBackend::generate_after_prefill(
    Sampler& sampler,
    const GenerationParams& params,
    std::function<void(std::string_view)> on_token,
    std::atomic<bool>* cancel)
{
    GenerationResult result;
    std::string generated;
    int n_generated = 0;
    const auto stop = effective_stop(params);  // gh#105: per-call sequential marker
 
    while (n_generated < params.max_tokens) {
        bool cancelled = cancel && cancel->load(std::memory_order_acquire);
        if (cancelled) {
            result.finish_reason = "cancelled";
            result.error_code = ENTROPIC_ERROR_CANCELLED;
            break;
        }
 
        auto status = step_token(sampler, generated, on_token, stop);
        if (status == "continue") {
            ++n_generated;
        } else {
            result.finish_reason = (status == "error") ? "error" : "stop";
            if (status == "error") {
                result.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
            }
            break;
        }
    }
 
    if (n_generated >= params.max_tokens && result.finish_reason.empty()) {
        result.finish_reason = "length";
    }
 
    // gh#136 (4th type_error.316 recurrence): sanitize at INGRESS, where model
    // bytes first become a std::string, rather than at each of the ~18 .dump()
    // sites downstream. gh#112/113 were closed as "permanent closure of the
    // 316 family", then gh#114, gh#118, gh#132 and gh#136 each patched one more
    // egress. The exits keep multiplying; the entries do not. Guarding here
    // makes every downstream dump safe by construction, and a NEW .dump()
    // anywhere cannot reintroduce the bug.
    //
    // Once at finalization, never per-token: a multi-byte codepoint can split
    // across token boundaries and per-token sanitize would corrupt valid
    // output (see the same reasoning at response_generator.cpp:360, which has
    // guarded the streaming accumulator this way since v2.1.1).
    //
    // Safe for raw_content consumers: raw_content is a COPY of content
    // (orchestrator.cpp:514), and sanitize only replaces bytes that are
    // ALREADY invalid UTF-8 — which cannot form part of any valid JSON token,
    // so the gh#88 envelope recovery and fenced-JSON fallbacks are unaffected.
    result.content = entropic::mcp::sanitize_utf8(generated);
    result.token_count = n_generated;
    return result;
}
 
// ── gh#98: same-prefix multi-seq batched generation ────────
 
static GenerationResult batch_error_result(const std::string& msg) {
    GenerationResult e;
    e.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
    e.error_message = msg;
    e.finish_reason = "error";
    return e;
}
 
static void fill_batch_cell(llama_batch& b, int k, llama_token tok,
                            llama_pos pos, llama_seq_id seq, bool want_logits) {
    b.token[k] = tok;
    b.pos[k] = pos;
    b.n_seq_id[k] = 1;
    b.seq_id[k][0] = seq;
    b.logits[k] = want_logits ? 1 : 0;
}
 
bool LlamaCppBackend::prepare_batch_seqs(
    std::vector<BatchSeq>& seqs,
    const std::vector<GenerationParams>& params) {
    for (std::size_t i = 0; i < seqs.size(); ++i) {
        seqs[i].sampler = create_sampler(params[i]);
        auto* ls = dynamic_cast<LlamaCppSampler*>(seqs[i].sampler.get());
        if (ls == nullptr) { return false; }
        seqs[i].chain = ls->native_chain();
        seqs[i].seq_id = (i == 0) ? 0 : allocate_temp_seq_id();
        seqs[i].max_tokens = params[i].max_tokens;
    }
    return true;
}
 
bool LlamaCppBackend::prefill_shared_and_fanout(
    std::vector<BatchSeq>& seqs, const std::vector<llama_token>& seq0,
    std::size_t shared) {
    std::vector<llama_token> prefix(
        seq0.begin(), seq0.begin() + static_cast<long>(shared));
    if (!decode_tokens_from(prefix, 0)) { return false; }  // into seq 0
    auto* mem = llama_get_memory(ctx_);
    for (std::size_t i = 1; i < seqs.size(); ++i) {
        llama_memory_seq_cp(mem, 0, seqs[i].seq_id, 0,
                            static_cast<llama_pos>(shared));
    }
    for (auto& s : seqs) { s.pos = static_cast<int>(shared); }
    return true;
}
 
bool LlamaCppBackend::prefill_batch_suffixes(
    std::vector<BatchSeq>& seqs,
    const std::vector<std::vector<llama_token>>& toks,
    std::size_t shared) {
    int total = 0;
    // shared <= shortest-1 < every t.size() by batch_shared_prefix_len, but
    // guard the unsigned subtraction defensively (a bad `shared` would else
    // underflow to a huge alloc).
    for (const auto& t : toks) {
        total += static_cast<int>(t.size() - std::min(shared, t.size()));
    }
    llama_batch batch = llama_batch_init(total, 0,
                                         static_cast<int32_t>(seqs.size()));
    int k = 0;
    for (std::size_t i = 0; i < seqs.size(); ++i) {
        int len = static_cast<int>(toks[i].size());
        for (int p = static_cast<int>(shared); p < len; ++p) {
            fill_batch_cell(batch, k, toks[i][p], p, seqs[i].seq_id,
                            p == len - 1);
            if (p == len - 1) { seqs[i].logits_idx = k; }
            ++k;
        }
        seqs[i].pos = len;
    }
    batch.n_tokens = k;
    last_prefill_tokens_ += k;
    bool ok = (llama_decode(ctx_, batch) == 0);
    llama_batch_free(batch);
    return ok;
}
 
void LlamaCppBackend::sample_batch_active(std::vector<BatchSeq>& seqs) {
    for (auto& s : seqs) {
        if (!s.active) { continue; }
        // llama_sampler_sample() accepts the drawn token into the chain
        // internally (advancing grammar/penalties) — matching the single-seq
        // step_token path. A second accept would double-advance the grammar.
        llama_token tok = llama_sampler_sample(s.chain, ctx_, s.logits_idx);
        if (llama_vocab_is_eog(vocab_, tok)) {
            s.active = false;
            s.finish = "stop";
            continue;
        }
        s.out.push_back(tok);
        ++s.n_gen;
        if (s.n_gen >= s.max_tokens) { s.active = false; s.finish = "length"; }
    }
}
 
void LlamaCppBackend::run_batch_gen_loop(
    std::vector<BatchSeq>& seqs, int max_steps, std::atomic<bool>& cancel) {
    llama_batch batch = llama_batch_init(static_cast<int32_t>(seqs.size()), 0,
                                         static_cast<int32_t>(seqs.size()));
    for (int step = 0; step < max_steps; ++step) {
        if (cancel.load(std::memory_order_acquire)) { break; }
        sample_batch_active(seqs);
        int k = 0;
        for (auto& s : seqs) {
            if (!s.active) { continue; }
            fill_batch_cell(batch, k, s.out.back(), s.pos, s.seq_id, true);
            s.logits_idx = k;
            ++s.pos;
            ++k;
        }
        if (k == 0) { break; }
        batch.n_tokens = k;
        ++last_gen_decode_calls_;
        if (llama_decode(ctx_, batch) != 0) { break; }
    }
    llama_batch_free(batch);
}
 
std::vector<GenerationResult> LlamaCppBackend::build_batch_results(
    std::vector<BatchSeq>& seqs) {
    std::vector<GenerationResult> out;
    out.reserve(seqs.size());
    for (auto& s : seqs) {
        GenerationResult r;
        for (llama_token t : s.out) { r.content += detokenize(t); }
        r.token_count = s.n_gen;
        r.finish_reason = s.finish;
        out.push_back(std::move(r));
    }
    return out;
}
 
void LlamaCppBackend::release_temp_seqs(std::vector<BatchSeq>& seqs) {
    for (std::size_t i = 1; i < seqs.size(); ++i) {
        if (seqs[i].seq_id != 0) { release_temp_seq_id(seqs[i].seq_id); }
    }
}
 
std::vector<GenerationResult> LlamaCppBackend::run_batched_decode(
    const std::vector<std::vector<llama_token>>& toks,
    const std::vector<GenerationParams>& params,
    std::size_t shared,
    std::atomic<bool>& cancel)
{
    const std::size_t n = toks.size();
    std::vector<BatchSeq> seqs(n);
    if (!prepare_batch_seqs(seqs, params)) {
        release_temp_seqs(seqs);  // don't leak ids allocated before the failure
        return std::vector<GenerationResult>(
            n, batch_error_result("batch sampler init"));
    }
    int max_steps = 0;
    for (const auto& p : params) { max_steps = std::max(max_steps, p.max_tokens); }
 
    llama_memory_clear(llama_get_memory(ctx_), true);
    invalidate_resident_kv();
    last_prefill_tokens_ = 0;
    last_gen_decode_calls_ = 0;
 
    bool ok = prefill_shared_and_fanout(seqs, toks[0], shared)
           && prefill_batch_suffixes(seqs, toks, shared);
    if (ok) { run_batch_gen_loop(seqs, max_steps, cancel); }
 
    auto out = ok ? build_batch_results(seqs)
                  : std::vector<GenerationResult>(
                        n, batch_error_result("batch prefill"));
    release_temp_seqs(seqs);
    invalidate_resident_kv();
    logger->info("gh#98 batch: requests={} prefix.tokens_shared={} "
                 "prefix.tokens_saved={} total_prefill_tokens={} gen_decodes={}",
                 n, shared, shared * (n - 1), last_prefill_tokens_,
                 last_gen_decode_calls_);
    return out;
}
 
std::vector<GenerationResult> LlamaCppBackend::do_generate_batch(
    const std::vector<std::vector<Message>>& requests,
    const std::vector<GenerationParams>& params,
    std::atomic<bool>& cancel)
{
    const std::size_t n = requests.size();
    std::vector<std::vector<llama_token>> toks(n);
    for (std::size_t i = 0; i < n; ++i) {
        toks[i] = tokenize(render_prompt(requests[i], params[i]), true);
    }
    const std::size_t shared = batch_shared_prefix_len(toks);
    std::size_t total_suffix = 0;
    for (const auto& t : toks) { total_suffix += t.size() - shared; }
 
    const bool hybrid = is_hybrid_ || is_recurrent_;
    if (!batch_is_viable(n, config().n_parallel, shared, hybrid,
                         total_suffix, config().n_batch)) {
        return InferenceBackend::do_generate_batch(requests, params, cancel);
    }
    return run_batched_decode(toks, params, shared, cancel);
}
 
// ── Prompt cache helpers ───────────────────────────────────
 
std::string LlamaCppBackend::extract_system_prompt(
    const std::vector<Message>& messages)
{
    for (const auto& msg : messages) {
        if (msg.role == "system") {
            return msg.content;
        }
    }
    return "";
}
 
bool LlamaCppBackend::decode_tokens_from(
    const std::vector<llama_token>& tokens, int start_offset)
{
    int total = static_cast<int>(tokens.size());
    if (start_offset >= total) { return true; }
 
    int n_batch = llama_n_batch(ctx_);
    int n_remaining = total - start_offset;
    last_prefill_tokens_ += n_remaining;  // gh#96: count tokens decoded here
    for (int off = 0; off < n_remaining; off += n_batch) {
        int chunk = std::min(n_batch, n_remaining - off);
        llama_batch batch = llama_batch_get_one(
            const_cast<llama_token*>(tokens.data())
                + start_offset + off,
            chunk);
        if (llama_decode(ctx_, batch) != 0) {
            logger->error("Decode chunk failed (start={}, off={}, "
                          "chunk={})", start_offset, off, chunk);
            return false;
        }
    }
    return true;
}
 
bool LlamaCppBackend::restore_cached_prefix(
    const CacheEntry* cached,
    const std::vector<llama_token>& tokens)
{
    auto* mem = llama_get_memory(ctx_);
    llama_memory_clear(mem, true);
 
    size_t restored = llama_state_seq_set_data(
        ctx_, cached->data.data(), cached->data_size, 0);
    if (restored == 0) {
        logger->warn("KV state restore failed, falling back to full prefill");
        return false;
    }
 
    return decode_tokens_from(tokens, cached->token_count);
}
 
void LlamaCppBackend::save_prefix_to_cache(
    const CacheKey& key, int prefix_tokens)
{
    size_t state_size = llama_state_seq_get_size(ctx_, 0);
    if (state_size == 0) {
        return;
    }
 
    std::vector<uint8_t> buf(state_size);
    size_t written = llama_state_seq_get_data(
        ctx_, buf.data(), buf.size(), 0);
    if (written > 0) {
        buf.resize(written);
        prompt_cache_->store(key, std::move(buf), prefix_tokens);
    }
}
 
int LlamaCppBackend::compute_prefix_token_count(
    const std::vector<Message>& messages,
    const GenerationParams& params)
{
    std::vector<Message> sys_msgs;
    for (const auto& msg : messages) {
        if (msg.role == "system") {
            sys_msgs.push_back(msg);
        }
    }
    if (sys_msgs.empty()) {
        return 0;
    }
 
    std::string sys_prompt = apply_chat_template(sys_msgs, params);
    auto sys_tokens = tokenize(sys_prompt, true);
    return static_cast<int>(sys_tokens.size());
}
 
bool LlamaCppBackend::prefill_and_cache_prefix(
    const std::vector<llama_token>& tokens,
    int prefix_tokens,
    const CacheKey& key)
{
    int total = static_cast<int>(tokens.size());
    if (prefix_tokens <= 0 || prefix_tokens >= total) {
        return run_prefill(tokens);
    }
 
    // Pass 1: prefill only the prefix — `run_prefill` calls
    // llama_memory_clear, so seq 0 ends up holding exactly
    // prefix_tokens positions.
    std::vector<llama_token> prefix(
        tokens.begin(), tokens.begin() + prefix_tokens);
    if (!run_prefill(prefix)) {
        return false;
    }
 
    // Save now: state contains exactly the prefix.
    save_prefix_to_cache(key, prefix_tokens);
 
    // Pass 2: continue prefilling the remainder. No clear — decode
    // appends after the saved prefix positions.
    return decode_tokens_from(tokens, prefix_tokens);
}
 
bool LlamaCppBackend::run_prefill_cached(
    const std::vector<llama_token>& tokens,
    const std::string& system_prompt,
    const std::vector<Message>& messages,
    const GenerationParams& params)
{
    // gh#96 (v2.7.5): count tokens actually pushed through llama_decode during
    // prefill this turn. run_prefill / decode_tokens_from accumulate into
    // last_prefill_tokens_; a prompt-cache HIT restores the system prefix
    // without a decode, so this counts the re-decoded post-system remainder —
    // the per-turn waste that climbs today and should collapse to the appended
    // delta once warm-keep reuse lands. (llama_perf n_p_eval proved unreliable
    // across the state-restore boundary, so we count the decodes directly.)
    last_prefill_tokens_ = 0;
    last_input_tokens_ = static_cast<int>(tokens.size());  // gh#97
    auto t_pre = entropic::log::now();
    bool ok;
    if (is_hybrid_ || is_recurrent_) {
        // gh#97 (v2.7.6): hybrid/recurrent (SSM) memory rejects the partial
        // seq_rm warm-keep needs (state can't be partially erased at the tail),
        // and the prompt-cache restore lands non-contiguous cells — both desync
        // KV positions (pos_max inflates → eventual decode slot-failure with the
        // cache mostly empty). Plain full prefill (clear + contiguous decode) is
        // the only correct path for these archs at this llama.cpp pin. Mirrors
        // the speculative-decoding guard. Forfeits the gh#96 reuse for them.
        ok = run_prefill(tokens);
        invalidate_resident_kv();
    } else {
        // gh#96 warm-keep: reuse the resident KV prefix + decode only the delta;
        // fall back to a cold prefill (clear + system-prefix cache) when reuse
        // is off, the prefix diverged, or the KV was mutated out-of-band.
        ok = try_warm_reuse(tokens);
        if (!ok) {
            ok = prefill_dispatch(tokens, system_prompt, messages, params);
            if (ok) {
                resident_tokens_ = tokens;
            } else {
                invalidate_resident_kv();
            }
        }
    }
    last_prefill_ms_ = entropic::log::elapsed_ms(t_pre, entropic::log::now());
    logger->info("Prefill (gh#96): {} tokens / {:.1f} ms decoded this turn",
                 last_prefill_tokens_, last_prefill_ms_);
    return ok;
}
 
bool LlamaCppBackend::try_warm_reuse(const std::vector<llama_token>& tokens) {
    if (!prompt_cache_config_.warm_keep || ctx_ == nullptr) {
        return false;
    }
    auto* mem = llama_get_memory(ctx_);
    long pos_max = static_cast<long>(llama_memory_seq_pos_max(mem, 0));
    std::size_t cut = warm_keep_cut(resident_tokens_, tokens, pos_max);
    if (cut == 0) {
        return false;  // nothing reusable — cold prefill
    }
    // Drop the divergent tail (and any prior generated tokens past `cut`),
    // then decode only the appended delta. A single exit (returns <= 3 gate):
    // success records the new resident set; failure invalidates and reports it.
    llama_memory_seq_rm(mem, 0, static_cast<llama_pos>(cut), -1);
    bool ok = decode_tokens_from(tokens, static_cast<int>(cut));
    if (ok) {
        resident_tokens_ = tokens;
        if (prompt_cache_config_.log_hits) {
            logger->info("Warm-keep: reused {} resident tokens, decoded {} "
                         "delta (of {} total)", cut, tokens.size() - cut,
                         tokens.size());
        }
    } else {
        invalidate_resident_kv();
    }
    return ok;
}
 
void LlamaCppBackend::invalidate_resident_kv() {
    resident_tokens_.clear();
}
 
bool LlamaCppBackend::prefill_dispatch(
    const std::vector<llama_token>& tokens,
    const std::string& system_prompt,
    const std::vector<Message>& messages,
    const GenerationParams& params)
{
    bool cache_enabled = prompt_cache_
        && prompt_cache_config_.enabled
        && !system_prompt.empty();
 
    if (!cache_enabled) {
        return run_prefill(tokens);
    }
 
    CacheKey key = PromptCache::make_key(
        system_prompt, config().path.string());
    const CacheEntry* cached = prompt_cache_->lookup(key);
 
    if (cached != nullptr) {
        if (prompt_cache_config_.log_hits) {
            logger->info("Prompt cache HIT: {} bytes, {} prefix tokens",
                         cached->data_size, cached->token_count);
        }
        if (restore_cached_prefix(cached, tokens)) {
            return true;
        }
        logger->warn("Cache restore failed, falling back to full prefill");
    } else if (prompt_cache_config_.log_hits) {
        logger->info("Prompt cache MISS: processing full prompt");
    }
 
    int prefix_tokens = compute_prefix_token_count(messages, params);
    return prefill_and_cache_prefix(tokens, prefix_tokens, key);
}
 
// ── Multimodal generation (v1.9.11 Phases 5–7 + v2.1.8) ────
 
namespace {
 
bool any_image_in(const std::vector<Message>& messages) {
    for (const auto& m : messages) {
        if (has_images(m.content_parts)) { return true; }
    }
    return false;
}
 
std::vector<Message> strip_image_parts(
    const std::vector<Message>& messages) {
    std::vector<Message> out = messages;
    for (auto& m : out) {
        if (m.content_parts.empty()) { continue; }
        m.content = extract_text(m.content_parts);
        m.content_parts.clear();
    }
    return out;
}
 
std::vector<Message> substitute_image_markers(
    const std::vector<Message>& messages,
    ::mtmd_context* ctx,
    std::vector<::mtmd_bitmap*>& bitmaps_out) {
    std::vector<Message> out;
    out.reserve(messages.size());
    const std::string marker = mtmd_default_marker();
    for (const auto& m : messages) {
        Message copy;
        copy.role = m.role;
        if (m.content_parts.empty()) {
            copy.content = m.content;
            out.push_back(std::move(copy));
            continue;
        }
        std::string built;
        for (const auto& p : m.content_parts) {
            if (p.type != ContentPartType::IMAGE) {
                built += p.text;
                continue;
            }
            ::mtmd_bitmap* bm = nullptr;
            if (!p.image_path.empty()) {
                bm = mtmd_helper_bitmap_init_from_file(
                    ctx, p.image_path.c_str(), /*placeholder=*/false).bitmap;
            }
            if (bm == nullptr) { return {}; }
            bitmaps_out.push_back(bm);
            built += marker;
        }
        copy.content = std::move(built);
        out.push_back(std::move(copy));
    }
    return out;
}
 
} // anonymous namespace
 
entropic_error_t LlamaCppBackend::mtmd_prefill(
    const std::string& prompt,
    const std::vector<::mtmd_bitmap*>& bitmaps,
    std::string& err_msg)
{
    llama_memory_clear(llama_get_memory(ctx_), true);
    ::mtmd_input_text mt{prompt.c_str(), true, true};
    auto* chunks = mtmd_input_chunks_init();
    std::vector<const ::mtmd_bitmap*> bm_cptrs(
        bitmaps.begin(), bitmaps.end());
    int32_t tok_rc = mtmd_tokenize(
        mtmd_ctx_, chunks, &mt, bm_cptrs.data(), bm_cptrs.size());
    if (tok_rc != 0) {
        mtmd_input_chunks_free(chunks);
        err_msg = "mtmd_tokenize failed (rc="
            + std::to_string(tok_rc) + ")";
        return ENTROPIC_ERROR_GENERATE_FAILED;
    }
    llama_pos new_n_past = 0;
    int32_t eval_rc = mtmd_helper_eval_chunks(
        mtmd_ctx_, ctx_, chunks, 0, 0,
        static_cast<int32_t>(config().n_batch),
        true, &new_n_past);
    mtmd_input_chunks_free(chunks);
    if (eval_rc != 0) {
        err_msg = "mtmd_helper_eval_chunks failed (rc="
            + std::to_string(eval_rc) + ")";
        return ENTROPIC_ERROR_GENERATE_FAILED;
    }
    logger->info("Multimodal prefill complete: n_past={}", new_n_past);
    return ENTROPIC_OK;
}
 
GenerationResult LlamaCppBackend::run_sampling_loop(
    const GenerationParams& params,
    std::function<void(std::string_view token)> on_token,
    std::atomic<bool>* cancel,
    const std::chrono::steady_clock::time_point& t0)
{
    GenerationResult result;
    // v2.3.10: Sampler seam.
    auto sampler = create_sampler(params);
    if (!sampler) {
        result.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
        result.error_message = "Sampler factory not initialized";
        result.finish_reason = "error";
        finalize_result(result, t0);
        return result;
    }
    std::string generated;
    int n_generated = 0;
    const auto stop = effective_stop(params);  // gh#105: per-call sequential marker
    while (n_generated < params.max_tokens) {
        if (cancel != nullptr
                && cancel->load(std::memory_order_acquire)) {
            result.finish_reason = "cancelled";
            result.error_code = ENTROPIC_ERROR_CANCELLED;
            break;
        }
        auto status = step_token(
            *sampler, generated, on_token, stop);
        if (status == "continue") { ++n_generated; continue; }
        result.finish_reason = (status == "error") ? "error" : "stop";
        if (status == "error") {
            result.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
        }
        break;
    }
    finalize_generation(result, generated, n_generated, params, t0);
    return result;
}
 
GenerationResult LlamaCppBackend::generate_multimodal(
    const std::vector<Message>& messages,
    const GenerationParams& params,
    std::function<void(std::string_view token)> on_token,
    std::atomic<bool>* cancel)
{
    auto t0 = entropic::log::now();
    invalidate_resident_kv();  // gh#96: mtmd_prefill mutates seq 0 out-of-band
    std::vector<::mtmd_bitmap*> bitmaps;
    auto marked = substitute_image_markers(
        messages, mtmd_ctx_, bitmaps);
    if (marked.empty()) {
        for (auto* b : bitmaps) { mtmd_bitmap_free(b); }
        GenerationResult err;
        err.error_code = ENTROPIC_ERROR_IMAGE_LOAD_FAILED;
        err.error_message =
            "mtmd_helper_bitmap_init_from_file failed";
        return err;
    }
    auto prompt = render_prompt(marked, params);
    logger->info("Multimodal generate: {} images, prompt={} chars, max_tokens={}",
                 bitmaps.size(), prompt.size(), params.max_tokens);
    std::string prefill_err;
    auto rc = mtmd_prefill(prompt, bitmaps, prefill_err);
    for (auto* b : bitmaps) { mtmd_bitmap_free(b); }
    if (rc != ENTROPIC_OK) {
        GenerationResult err;
        err.error_code = rc;
        err.error_message = std::move(prefill_err);
        return err;
    }
    return run_sampling_loop(params, on_token, cancel, t0);
}
 
// ── Generation entry points ────────────────────────────────
 
GenerationResult LlamaCppBackend::do_generate(
    const std::vector<Message>& messages,
    const GenerationParams& params)
{
    if (!any_image_in(messages)) {
        return do_generate_text_only(messages, params);
    }
    if (has_vision_ && mtmd_ctx_ != nullptr) {
        return generate_multimodal(messages, params, nullptr, nullptr);
    }
    logger->warn("Image content present but model has no vision "
                 "capability — stripping image parts");
    return do_generate_text_only(strip_image_parts(messages), params);
}
 
GenerationResult LlamaCppBackend::do_generate_text_only(
    const std::vector<Message>& messages,
    const GenerationParams& params)
{
    auto t0 = entropic::log::now();
    std::string prompt = render_prompt(messages, params);
    auto tokens = tokenize(prompt, true);
    std::string sys = extract_system_prompt(messages);
 
    logger->info("Generate: {} input tokens, max_tokens={}",
              tokens.size(), params.max_tokens);
    log_sampler_config(params);
 
    // v2.3.10: Sampler seam.
    auto sampler = create_sampler(params);
    if (!sampler) { return sampler_init_error(t0); }
 
    if (!run_prefill_cached(tokens, sys, messages, params)) {
        return prefill_error();
    }
 
    GenerationResult result;
    std::string generated;
    int n_generated = 0;
    std::function<void(std::string_view)> no_cb = nullptr;
    const auto stop = effective_stop(params);  // gh#105: per-call sequential marker
 
    while (n_generated < params.max_tokens) {
        auto status = step_token(
            *sampler, generated, no_cb, stop);
        if (status == "continue") { ++n_generated; }
        else {
            result.finish_reason =
                (status == "error") ? "error" : "stop";
            if (status == "error") {
                result.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
            }
            break;
        }
    }
 
    finalize_generation(result, generated, n_generated, params, t0);
    return result;
}
 
GenerationResult LlamaCppBackend::do_generate(
    const std::vector<Message>& messages,
    const GenerationParams& params,
    std::atomic<bool>& cancel)
{
    if (!any_image_in(messages)) {
        return do_generate_text_only(messages, params, cancel);
    }
    if (has_vision_ && mtmd_ctx_ != nullptr) {
        return generate_multimodal(messages, params, nullptr, &cancel);
    }
    logger->warn("Image content present but model has no vision "
                 "capability — stripping image parts");
    return do_generate_text_only(strip_image_parts(messages), params, cancel);
}
 
GenerationResult LlamaCppBackend::do_generate_text_only(
    const std::vector<Message>& messages,
    const GenerationParams& params,
    std::atomic<bool>& cancel)
{
    auto t0 = entropic::log::now();
    std::string prompt = render_prompt(messages, params);
    auto tokens = tokenize(prompt, true);
    std::string sys = extract_system_prompt(messages);
 
    logger->info("Generate (cancellable): {} input tokens, max_tokens={}",
              tokens.size(), params.max_tokens);
    log_sampler_config(params);
 
    auto sampler = create_sampler(params);
    if (!sampler) { return sampler_init_error(t0); }
 
    if (!run_prefill_cached(tokens, sys, messages, params)) {
        return prefill_error();
    }
 
    GenerationResult result;
    std::string generated;
    int n_generated = 0;
    std::function<void(std::string_view)> no_cb = nullptr;
 
    const auto stop = effective_stop(params);  // gh#105: per-call sequential marker
    while (n_generated < params.max_tokens) {
        if (cancel.load(std::memory_order_acquire)) {
            result.finish_reason = "cancelled";
            result.error_code = ENTROPIC_ERROR_CANCELLED;
            break;
        }
        auto status = step_token(
            *sampler, generated, no_cb, stop);
        if (status == "continue") { ++n_generated; }
        else {
            result.finish_reason =
                (status == "error") ? "error" : "stop";
            if (status == "error") {
                result.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
            }
            break;
        }
    }
 
    finalize_generation(result, generated, n_generated, params, t0);
    return result;
}
 
GenerationResult LlamaCppBackend::do_generate_streaming(
    const std::vector<Message>& messages,
    const GenerationParams& params,
    std::function<void(std::string_view token)> on_token,
    std::atomic<bool>& cancel)
{
    if (!any_image_in(messages)) {
        return do_generate_streaming_text_only(
            messages, params, on_token, cancel);
    }
    if (has_vision_ && mtmd_ctx_ != nullptr) {
        return generate_multimodal(messages, params, on_token, &cancel);
    }
    logger->warn("Image content present but model has no vision "
                 "capability — stripping image parts");
    return do_generate_streaming_text_only(
        strip_image_parts(messages), params, on_token, cancel);
}
 
GenerationResult LlamaCppBackend::do_generate_streaming_text_only(
    const std::vector<Message>& messages,
    const GenerationParams& params,
    std::function<void(std::string_view token)> on_token,
    std::atomic<bool>& cancel)
{
    auto t0 = entropic::log::now();
    auto prompt = render_prompt(messages, params);
    auto tokens = tokenize(prompt, true);
    auto sys = extract_system_prompt(messages);
    logger->info("Stream: {} input tokens, max_tokens={}",
              tokens.size(), params.max_tokens);
    log_sampler_config(params);
 
    // v2.3.10: Sampler seam.
    auto sampler = create_sampler(params);
    if (!sampler) { return sampler_init_error(t0); }
    if (!run_prefill_cached(tokens, sys, messages, params)) {
        return prefill_error();
    }
    GenerationResult result;
    std::string generated;
    int n_generated = 0;
    const auto stop = effective_stop(params);  // gh#105: per-call sequential marker
    while (n_generated < params.max_tokens) {
        if (cancel.load(std::memory_order_acquire)) {
            result.finish_reason = "cancelled";
            result.error_code = ENTROPIC_ERROR_CANCELLED;
            break;
        }
        auto status = step_token(
            *sampler, generated, on_token, stop);
        if (status == "continue") { ++n_generated; }
        else {
            result.finish_reason =
                (status == "error") ? "error" : "stop";
            if (status == "error") {
                result.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
            }
            break;
        }
    }
    finalize_generation(result, generated, n_generated, params, t0);
    return result;
}
 
GenerationResult LlamaCppBackend::do_generate_speculative(
    const std::vector<Message>& /*messages*/,
    const GenerationParams& /*params*/,
    std::function<void(std::string_view)> /*on_token*/,
    std::atomic<bool>& /*cancel*/)
{
    GenerationResult result;
    result.error_code = ENTROPIC_ERROR_NOT_SUPPORTED;
    result.error_message =
        "LlamaCppBackend speculative requires an explicit draft "
        "backend handle — orchestrator dispatches via "
        "generate_speculative_with_draft";
    result.finish_reason = "error";
    return result;
}
 
namespace {
 
static void apply_grammar_source(
    common_params_sampling& cps,
    const GenerationParams& params,
    const std::string& tool_grammar,
    bool tool_grammar_lazy,
    const std::string& generation_prompt) {
    // gh#108 (v2.10.0): propagate GBNF grammar to the MTP sampler chain so
    // grammar-constrained tiers are correctly enforced under speculative.mtp.
    //
    // gh#134 (v2.10.4): a SECOND grammar source now reaches here — the
    // tool-call GBNF common_chat_templates_apply derives from the staged tool
    // schemas, which was previously discarded at the render. Precedence is
    // fail-loud rather than a silent pick: an explicit request grammar
    // alongside a tool-derived one is a config error, because the two
    // constrain the output to different languages and quietly honouring one
    // would make the other's absence undiagnosable.
    if (grammar_sources_collide(params.grammar, tool_grammar)) {
        logger->error(
            "Both a request grammar and a tool-call grammar are active. These "
            "constrain decoding to different languages and cannot compose; the "
            "request grammar is being used and the staged tools will NOT be "
            "structurally enforced. Drop one: clear params.grammar to let the "
            "tool schemas constrain, or unstage tools to use your grammar.");
    }
    const auto source = resolve_grammar_source(params.grammar, tool_grammar);
    if (source == GrammarSource::request) {
        cps.grammar = common_grammar(COMMON_GRAMMAR_TYPE_USER, params.grammar);
    } else if (source == GrammarSource::tool_call) {
        // TOOL_CALLS, not USER: common_grammar_needs_prefill() is true for this
        // type, so the sampler prefills generation_prompt into the grammar —
        // required because the model's output starts mid-template. The USER
        // type skips prefill and the grammar would reject from token one.
        cps.grammar = common_grammar(COMMON_GRAMMAR_TYPE_TOOL_CALLS,
                                     tool_grammar);
        cps.grammar_lazy = tool_grammar_lazy;
        // REQUIRED, not optional: common_sampler_init only prefills when
        // generation_prompt is non-empty (sampling.cpp:268, :285). Without it
        // the grammar sampler never accepts the tokens the chat template
        // already placed in the prompt, so it starts mid-rule and rejects the
        // very first sampled token — surfacing as GENERATE_FAILED with empty
        // content rather than as a grammar error. Measured exactly that on
        // gemma4 before this line existed.
        cps.generation_prompt = generation_prompt;
        // gh#138: the ONLY observable that the tool-call grammar is in force.
        // The orchestration line reports params.grammar, which is the REQUEST
        // grammar — it prints "unconstrained" even when this grammar is
        // active, so a consumer reading logs cannot tell the difference
        // between "tool grammar applied" and "nothing constrained the
        // decode". INFO, not DEBUG: this is the diagnostic that would have
        // answered gh#138 from the reporter's own logs.
        logger->info(
            "Tool-call grammar applied: {} bytes, lazy={}, prefill={} bytes",
            tool_grammar.size(), tool_grammar_lazy,
            generation_prompt.size());
    } else if (!tool_grammar.empty()) {
        // Tools were staged and a grammar derived, but the request grammar
        // won the precedence rule — say so rather than letting the tool
        // grammar vanish silently.
        logger->warn(
            "Tool-call grammar ({} bytes) NOT applied — an explicit request "
            "grammar takes precedence. Tool calls are not structurally "
            "enforced this turn.",
            tool_grammar.size());
    }
}
 
common_params_sampling to_common_sampling(
    const GenerationParams& params,
    const std::string& tool_grammar,
    bool tool_grammar_lazy,
    const std::string& generation_prompt) {
    common_params_sampling cps;
    cps.temp = params.temperature;
    apply_grammar_source(cps, params, tool_grammar, tool_grammar_lazy,
                         generation_prompt);
    cps.top_k = params.top_k;
    cps.top_p = params.top_p;
    cps.penalty_repeat = params.repeat_penalty;
    // gh#23 MVP items 2 + 3 (v2.3.14 + v2.3.15): wire presence +
    // frequency penalty into common-sampling. Counterparts of the
    // 3rd + 4th args to `llama_sampler_init_penalties` in the plain
    // decode path. Default 0.0f on both preserves bit-for-bit
    // speculative output.
    cps.penalty_freq    = params.frequency_penalty;
    cps.penalty_present = params.presence_penalty;
    // gh#23 MVP item 4 (v2.3.16): forward logit_bias to common-sampling.
    // Empty (default) leaves the speculative chain bit-for-bit
    // identical to pre-v2.3.16.
    for (auto& [tok, val] : params.logit_bias) {
        cps.logit_bias.push_back({tok, val});
    }
    if (params.seed >= 0) {
        cps.seed = static_cast<uint32_t>(params.seed);
    }
    cps.no_perf = true;
    // Mirror entropic's standard sampler chain ordering so the
    // speculative path produces output bit-identical to plain decode
    // (the v2.1.11 correctness contract). Entropic's `create_sampler`
    // builds: penalties → top_k → top_p → min_p → temperature → dist,
    // AND SKIPS the temperature sampler when temp == 0 (greedy mode).
    // common_sampler appends an extended-temperature sampler that
    // differs subtly from "no temp at all" — we omit it for temp=0
    // to match entropic exactly. min_p (v2.3.10, gh#23) appended only
    // when caller opted in (params.min_p > 0); 0.0 preserves the
    // pre-v2.3.10 chain shape bit-for-bit. Other extended filters
    // (top_n_sigma, dry, xtc, typical_p) remain stripped.
    cps.samplers = {COMMON_SAMPLER_TYPE_PENALTIES,
                    COMMON_SAMPLER_TYPE_TOP_K,
                    COMMON_SAMPLER_TYPE_TOP_P};
    if (params.min_p > 0.0f) {
        cps.samplers.push_back(COMMON_SAMPLER_TYPE_MIN_P);
    }
    if (params.temperature > 0.0f) {
        cps.samplers.push_back(COMMON_SAMPLER_TYPE_TEMPERATURE);
    }
    cps.min_p = params.min_p;
    cps.dry_multiplier = 0.0f;
    cps.top_n_sigma = -1.0f;
    return cps;
}
 
bool spec_prefill_minus_last(
    llama_context* ctx, const std::vector<llama_token>& tokens) {
    int total = static_cast<int>(tokens.size()) - 1;
    if (total <= 0) { return true; }
    int n_batch = llama_n_batch(ctx);
    for (int off = 0; off < total; off += n_batch) {
        int chunk = std::min(n_batch, total - off);
        llama_batch batch = llama_batch_get_one(
            const_cast<llama_token*>(tokens.data()) + off, chunk);
        if (llama_decode(ctx, batch) != 0) { return false; }
    }
    return true;
}
 
GenerationResult spec_error(entropic_error_t code, std::string msg) {
    logger->error("Speculative decode failed ({}): {}",
                  entropic_error_name(code), msg);
    GenerationResult r;
    r.error_code = code;
    r.error_message = std::move(msg);
    r.finish_reason = "error";
    return r;
}
 
} // anonymous namespace
 
struct SpeculativeRunState {
    common_speculative* spec = nullptr;
    common_sampler* smpl = nullptr;
    llama_context* ctx_tgt = nullptr;
    llama_context* ctx_dft = nullptr;
    llama_batch batch_tgt{};
    bool batch_initialized = false;
    llama_seq_id seq_id = 0;
    int n_past = 0;
    llama_token id_last = 0;
    std::vector<llama_token> prompt_tgt;
    std::vector<llama_token> draft;
    std::string generated;
    std::vector<std::string> stop;  
    int n_generated = 0;
    int n_drafted = 0;
    int n_accepted = 0;
    bool has_eos = false;
    std::string finish_reason;
    entropic_error_t error_code = ENTROPIC_OK;
    std::string error_message;
 
    // ── Checkpoint state (v2.1.11) ──────────────────────────
    // Activated when either context reports FULL-only seq_rm
    // (no partial removal). The kernel saves+restores draft/target
    // state across each speculative round so the underlying
    // memory module never sees an attempted partial removal.
    // Mirrors the use_ckpt_tgt / use_ckpt_dft flow in upstream's
    // speculative-simple example.
    bool use_ckpt_tgt = false;
    bool use_ckpt_dft = false;
    common_prompt_checkpoint ckpt;
};
 
static void spec_cleanup(SpeculativeRunState& state) {
    if (state.spec) { common_speculative_free(state.spec); }
    if (state.smpl) { common_sampler_free(state.smpl); }
    if (state.batch_initialized) {
        llama_batch_free(state.batch_tgt);
    }
}
 
static void spec_build_batch(SpeculativeRunState& state) {
    common_batch_clear(state.batch_tgt);
    common_batch_add(state.batch_tgt, state.id_last,
                     state.n_past, {state.seq_id}, true);
    int pos = state.n_past + 1;
    for (auto draft_token : state.draft) {
        common_batch_add(state.batch_tgt, draft_token, pos,
                         {state.seq_id}, true);
        ++pos;
    }
}
 
static bool spec_decode_both(SpeculativeRunState& state) {
    spec_build_batch(state);
    int rc_tgt = llama_decode(state.ctx_tgt, state.batch_tgt);
    if (rc_tgt != 0) {
        logger->error("Speculative target decode failed: rc={}, "
                      "n_past={}, draft_size={}",
                      rc_tgt, state.n_past, state.draft.size());
        state.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
        state.error_message = "target llama_decode failed";
        state.finish_reason = "error";
        return false;
    }
    int rc_dft = llama_decode(state.ctx_dft, state.batch_tgt);
    if (rc_dft != 0) {
        logger->error("Speculative draft decode failed: rc={}, "
                      "n_past={}, draft_size={}",
                      rc_dft, state.n_past, state.draft.size());
        state.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
        state.error_message = "draft llama_decode failed";
        state.finish_reason = "error";
        return false;
    }
    return true;
}
 
static int spec_run_draft(SpeculativeRunState& state) {
    auto& dp = common_speculative_get_draft_params(
        state.spec, state.seq_id);
    dp.drafting = true;
    dp.n_max = -1;
    dp.n_past = state.n_past;
    dp.id_last = state.id_last;
    dp.prompt = &state.prompt_tgt;
    dp.result = &state.draft;
    common_speculative_draft(state.spec);
    return static_cast<int>(state.draft.size());
}
 
static std::string spec_emit_token(
    SpeculativeRunState& state, llama_token id,
    const llama_vocab* vocab, int max_tokens,
    std::function<void(std::string_view)>& on_token,
    std::atomic<bool>& cancel)
{
    std::string signal;
    state.prompt_tgt.push_back(state.id_last);
    state.id_last = id;
    state.n_generated++;
    if (llama_vocab_is_eog(vocab, id)) {
        state.has_eos = true;
        state.finish_reason = "stop";
        signal = "eos";
    } else {
        const std::string piece =
            common_token_to_piece(state.ctx_tgt, id);
        state.generated += piece;
        if (on_token) { on_token(piece); }
        // gh#108: honor stop sequences (params.stop + gh#103 sequential-tool
        // close marker) so MTP stops where plain decode would, instead of
        // over-generating past the first tool call. state.stop is empty for the
        // gh#36 path, so this is a no-op there.
        if (check_stop_sequences(state.generated, state.stop)) {
            state.finish_reason = "stop";
            signal = "stop";
        } else if (cancel.load(std::memory_order_acquire)) {
            state.error_code = ENTROPIC_ERROR_CANCELLED;
            state.finish_reason = "cancelled";
            signal = "cancel";
        } else if (state.n_generated >= max_tokens) {
            state.finish_reason = "length";
            signal = "length";
        }
    }
    return signal;
}
 
static void spec_ckpt_save_dft(SpeculativeRunState& state) {
    state.ckpt.update_pos(
        static_cast<int64_t>(state.prompt_tgt.size()),
        llama_memory_seq_pos_min(
            llama_get_memory(state.ctx_tgt), state.seq_id),
        llama_memory_seq_pos_max(
            llama_get_memory(state.ctx_tgt), state.seq_id));
    if (state.use_ckpt_dft) {
        state.ckpt.update_dft(state.ctx_dft, state.seq_id,
            LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY
                | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
    }
}
 
static void spec_ckpt_save_tgt(SpeculativeRunState& state) {
    if (state.use_ckpt_tgt && !state.draft.empty()) {
        state.ckpt.update_tgt(state.ctx_tgt, state.seq_id,
            LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY
                | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
    }
}
 
static void spec_ckpt_restore_dft(SpeculativeRunState& state) {
    constexpr auto flags = LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY
                         | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE;
    if (state.use_ckpt_dft) {
        state.ckpt.load_dft(state.ctx_dft, state.seq_id, flags);
    }
    llama_memory_seq_rm(llama_get_memory(state.ctx_dft),
                        state.seq_id, state.ckpt.pos_max + 1, -1);
}
 
static void spec_rollback_partial(
    SpeculativeRunState& state, common_sampler* smpl_save,
    std::vector<llama_token>& ids) {
    constexpr auto flags = LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY
                         | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE;
    state.draft = std::move(ids);
    state.ckpt.load_tgt(state.ctx_tgt, state.seq_id, flags);
    llama_memory_seq_rm(llama_get_memory(state.ctx_tgt),
                        state.seq_id, state.ckpt.pos_max + 1, -1);
    state.ckpt.load_dft(state.ctx_dft, state.seq_id, flags);
    llama_memory_seq_rm(llama_get_memory(state.ctx_dft),
                        state.seq_id, state.ckpt.pos_max + 1, -1);
    state.prompt_tgt.resize(static_cast<size_t>(state.ckpt.n_tokens));
    state.n_past = static_cast<int>(state.prompt_tgt.size());
    // Sampler clone is non-null only when use_ckpt_tgt is set
    common_sampler_free(state.smpl);
    state.smpl = smpl_save;
}
 
static void spec_trim_rejected_drafts(SpeculativeRunState& state) {
    llama_memory_seq_rm(llama_get_memory(state.ctx_tgt),
                        state.seq_id, state.n_past, -1);
    llama_memory_seq_rm(llama_get_memory(state.ctx_dft),
                        state.seq_id, state.n_past, -1);
}
 
static bool spec_commit_accepted(
    SpeculativeRunState& state,
    const std::vector<llama_token>& ids,
    const llama_vocab* vocab, int max_tokens,
    std::function<void(std::string_view)>& on_token,
    std::atomic<bool>& cancel) {
    bool stop = false;
    for (auto id : ids) {
        auto signal = spec_emit_token(
            state, id, vocab, max_tokens, on_token, cancel);
        if (!signal.empty()) { stop = true; break; }
    }
    return stop;
}
 
static int spec_prepare_draft(SpeculativeRunState& state) {
    // Skip drafting if the previous round restored a partial accept
    // into state.draft (carry-over from rollback).
    if (!state.draft.empty()) {
        return static_cast<int>(state.draft.size());
    }
    spec_ckpt_save_dft(state);
    int drafted = spec_run_draft(state);
    spec_ckpt_save_tgt(state);
    spec_ckpt_restore_dft(state);
    return drafted;
}
 
static bool spec_accept_round(
    SpeculativeRunState& state,
    const llama_vocab* vocab,
    int max_tokens,
    std::function<void(std::string_view)>& on_token,
    std::atomic<bool>& cancel)
{
    int draft_size_before = spec_prepare_draft(state);
 
    if (!spec_decode_both(state)) { return false; }
 
    common_sampler* smpl_save = nullptr;
    if (state.use_ckpt_tgt) {
        smpl_save = common_sampler_clone(state.smpl);
    }
    auto ids = common_sampler_sample_and_accept_n(
        state.smpl, state.ctx_tgt, state.draft);
    int accepted = static_cast<int>(ids.size()) - 1;
    if (accepted < 0) { accepted = 0; }
 
    // Partial acceptance on a FULL-seq_rm context: rollback to
    // checkpoint, set draft = accepted, re-loop without emitting.
    if (state.use_ckpt_tgt
        && static_cast<int>(ids.size()) - 1
               < static_cast<int>(state.draft.size())) {
        spec_rollback_partial(state, smpl_save, ids);
        return true;
    }
    if (smpl_save) { common_sampler_free(smpl_save); }
 
    common_speculative_accept(state.spec, state.seq_id, accepted);
    state.n_drafted += draft_size_before;
    state.n_accepted += accepted;
    // n_past advances by ids.size() total: one slot for id_last
    // (the post-id_last position the next id will occupy), plus
    // `accepted` slots for the drafted tokens the sampler agreed
    // with. Matches speculative-simple's n_past++ in batch_add +
    // n_past += ids.size() - 1 sequence.
    state.n_past += static_cast<int>(ids.size());
 
    bool stop = spec_commit_accepted(
        state, ids, vocab, max_tokens, on_token, cancel);
    state.draft.clear();
    spec_trim_rejected_drafts(state);
    return !stop;
}
 
static std::string spec_check_preconditions(
    bool target_active, bool draft_active,
    llama_context* ctx_tgt, llama_context* ctx_dft) {
    // Defense-in-depth arch gate — orchestrator's
    // check_speculative_compat is the primary gate; a direct caller
    // into the kernel must also be refused on recurrent / hybrid
    // targets (Session 5 Gate A: hybrid SSM state diverges across
    // split-prefill boundaries; bit-identical unreachable at this pin).
    std::string err;
    const llama_model* model_tgt = llama_get_model(ctx_tgt);
    int cap_tgt = common_context_can_seq_rm(ctx_tgt);
    int cap_dft = common_context_can_seq_rm(ctx_dft);
    logger->info("Speculative seq_rm capability: target={}, draft={} "
                 "(0=NO, 1=PART, 2=FULL)", cap_tgt, cap_dft);
    if (!target_active || !draft_active) {
        err = "speculative requires ACTIVE target + draft";
    } else if (llama_model_is_recurrent(model_tgt)
               || llama_model_is_hybrid(model_tgt)) {
        err = "speculative refused: architecture (target is "
              "recurrent or hybrid; see proposal Implementation "
              "Log Gate A)";
    } else if (cap_tgt == COMMON_CONTEXT_SEQ_RM_TYPE_NO
               || cap_dft == COMMON_CONTEXT_SEQ_RM_TYPE_NO) {
        // NO is the only unsupported seq_rm case — the kernel has
        // both PART fast-path and FULL checkpoint paths.
        err = "speculative kernel requires at least FULL seq_rm "
              "(target/draft reported NO seq_rm at all)";
    }
    return err;
}
 
static std::string spec_init_sampler_and_decoder(
    SpeculativeRunState& state, llama_model* model_tgt,
    const GenerationParams& params, int n_draft_max,
    const std::string& draft_path,
    const std::string& tool_grammar,   // gh#134 (v2.10.4)
    bool tool_grammar_lazy,
    const std::string& generation_prompt) {
    auto common_sampling =
        to_common_sampling(params, tool_grammar, tool_grammar_lazy,
                           generation_prompt);
    state.smpl = common_sampler_init(model_tgt, common_sampling);
    if (!state.smpl) { return "common_sampler_init failed"; }
 
    common_params_speculative spec_params;
    spec_params.types = {COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE};
    spec_params.draft.n_max = (n_draft_max > 0) ? n_draft_max : 16;
    spec_params.draft.ctx_tgt = state.ctx_tgt;
    spec_params.draft.ctx_dft = state.ctx_dft;
    // Upstream gates DRAFT_SIMPLE on a non-empty draft path
    // (see common/speculative.cpp:875). Required even though we
    // provide already-loaded contexts.
    spec_params.draft.mparams.path = draft_path;
    state.spec = common_speculative_init(spec_params, 1);
    if (!state.spec) {
        common_sampler_free(state.smpl);
        state.smpl = nullptr;
        return "common_speculative_init failed";
    }
 
    common_speculative_begin(state.spec, state.seq_id, state.prompt_tgt);
    state.batch_tgt = llama_batch_init(llama_n_batch(state.ctx_tgt), 0, 1);
    state.batch_initialized = true;
    // Checkpoint flow lights up when either context can only do
    // FULL-sequence removal. Mirrors speculative-simple's
    // use_ckpt_{tgt,dft}.
    state.use_ckpt_tgt = common_context_can_seq_rm(state.ctx_tgt)
        == COMMON_CONTEXT_SEQ_RM_TYPE_FULL;
    state.use_ckpt_dft = common_context_can_seq_rm(state.ctx_dft)
        == COMMON_CONTEXT_SEQ_RM_TYPE_FULL;
    return "";
}
 
static std::string spec_init_run(
    SpeculativeRunState& state, llama_model* model_tgt,
    const std::vector<llama_token>& tokens,
    const GenerationParams& params, int n_draft_max,
    const std::string& draft_path,
    const std::string& tool_grammar, bool tool_grammar_lazy,
    const std::string& generation_prompt) {
    state.id_last = tokens.back();
    state.prompt_tgt.assign(tokens.begin(), tokens.end() - 1);
    state.n_past = static_cast<int>(tokens.size()) - 1;
 
    llama_memory_clear(llama_get_memory(state.ctx_tgt), true);
    llama_memory_clear(llama_get_memory(state.ctx_dft), true);
 
    if (!spec_prefill_minus_last(state.ctx_tgt, tokens)
        || !spec_prefill_minus_last(state.ctx_dft, tokens)) {
        return "speculative prefill failed";
    }
    return spec_init_sampler_and_decoder(
        state, model_tgt, params, n_draft_max, draft_path,
        tool_grammar, tool_grammar_lazy, generation_prompt);
}
 
static void spec_run_loop(
    SpeculativeRunState& state, const llama_vocab* vocab,
    int max_tokens,
    std::function<void(std::string_view)>& on_token,
    std::atomic<bool>& cancel) {
    while (state.n_generated < max_tokens) {
        if (cancel.load(std::memory_order_acquire)) {
            state.error_code = ENTROPIC_ERROR_CANCELLED;
            state.finish_reason = "cancelled";
            break;
        }
        if (!spec_accept_round(state, vocab, max_tokens,
                               on_token, cancel)) {
            break;
        }
    }
    if (state.finish_reason.empty()) {
        state.finish_reason = (state.n_generated >= max_tokens)
                                  ? "length" : "stop";
    }
}
 
static GenerationResult spec_finalize(
    SpeculativeRunState& state,
    std::chrono::steady_clock::time_point t0) {
    GenerationResult result;
    // gh#136 (4th type_error.316 recurrence): sanitize at INGRESS, where model
    // bytes first become a std::string, rather than at each of the ~18 .dump()
    // sites downstream. gh#112/113 were closed as "permanent closure of the
    // 316 family", then gh#114, gh#118, gh#132 and gh#136 each patched one more
    // egress. The exits keep multiplying; the entries do not. Guarding here
    // makes every downstream dump safe by construction, and a NEW .dump()
    // anywhere cannot reintroduce the bug.
    //
    // Once at finalization, never per-token: a multi-byte codepoint can split
    // across token boundaries and per-token sanitize would corrupt valid
    // output (see the same reasoning at response_generator.cpp:360, which has
    // guarded the streaming accumulator this way since v2.1.1).
    //
    // Safe for raw_content consumers: raw_content is a COPY of content
    // (orchestrator.cpp:514), and sanitize only replaces bytes that are
    // ALREADY invalid UTF-8 — which cannot form part of any valid JSON token,
    // so the gh#88 envelope recovery and fenced-JSON fallbacks are unaffected.
    result.content = entropic::mcp::sanitize_utf8(state.generated);
    result.token_count = state.n_generated;
    result.finish_reason = state.finish_reason;
    result.error_code = state.error_code;
    result.error_message = state.error_message;
    // gh#106: surface the draft/accept counts so callers (and the MTP
    // engagement test) can verify the kernel actually ran + accepted.
    result.n_drafted = state.n_drafted;
    result.n_accepted = state.n_accepted;
    result.generation_time_ms =
        entropic::log::elapsed_ms(t0, entropic::log::now());
    // gh#108: the speculative path previously left throughput_tok_s=0.0 — the
    // one metric the feature exists for. Compute it like finalize_result.
    if (result.token_count > 0 && result.generation_time_ms > 0.0) {
        result.throughput_tok_s =
            static_cast<double>(result.token_count)
            / result.generation_time_ms * 1000.0;
    }
    if (state.n_drafted > 0) {
        const float accept_rate =
            static_cast<float>(state.n_accepted)
                / static_cast<float>(state.n_drafted);
        logger->info("Speculative: generated={}, drafted={}, "
                     "accepted={}, accept_rate={:.3f}",
                     state.n_generated, state.n_drafted,
                     state.n_accepted, accept_rate);
    }
    spec_cleanup(state);
    return result;
}
 
static GenerationResult spec_run_from_tokens(
    llama_context* ctx_tgt, llama_context* ctx_dft, llama_model* model_tgt,
    const std::vector<llama_token>& tokens, const GenerationParams& params,
    std::function<void(std::string_view)>& on_token,
    std::atomic<bool>& cancel, int n_draft_max,
    const std::string& draft_path,
    std::chrono::steady_clock::time_point t0,
    const std::string& tool_grammar, bool tool_grammar_lazy,
    const std::string& generation_prompt) {
    SpeculativeRunState state;
    state.ctx_tgt = ctx_tgt;
    state.ctx_dft = ctx_dft;
    auto init_err = spec_init_run(state, model_tgt, tokens, params,
                                  n_draft_max, draft_path,
                                  tool_grammar, tool_grammar_lazy,
                                  generation_prompt);
    if (!init_err.empty()) {
        spec_cleanup(state);
        return spec_error(ENTROPIC_ERROR_GENERATE_FAILED,
                          std::move(init_err));
    }
    spec_run_loop(state, llama_model_get_vocab(model_tgt),
                  params.max_tokens, on_token, cancel);
    return spec_finalize(state, t0);
}
 
GenerationResult LlamaCppBackend::generate_speculative_with_draft(
    const std::vector<Message>& messages,
    const GenerationParams& params,
    std::function<void(std::string_view)> on_token,
    std::atomic<bool>& cancel,
    LlamaCppBackend& draft,
    int n_draft_max,
    const std::string& draft_path)
{
    auto t0 = entropic::log::now();
    invalidate_resident_kv();  // gh#96: speculative path manages seq 0 itself
    auto pre_err = spec_check_preconditions(
        is_active(), draft.is_active(), ctx_, draft.ctx_);
    GenerationResult result;
    if (!pre_err.empty()) {
        entropic_error_t code =
            (pre_err.find("requires ACTIVE") != std::string::npos)
                ? ENTROPIC_ERROR_INVALID_STATE
                : ENTROPIC_ERROR_NOT_SUPPORTED;
        result = spec_error(code, std::move(pre_err));
    } else {
        auto prompt = render_prompt(messages, params);
        auto tokens = tokenize(prompt, true);
        if (tokens.size() < 2) {
            result = spec_error(ENTROPIC_ERROR_GENERATE_FAILED,
                "speculative prompt must have at least 2 tokens");
        } else {
            logger->info("Speculative: {} input tokens, max_tokens={}, "
                         "n_draft_max={}",
                         tokens.size(), params.max_tokens, n_draft_max);
            result = spec_run_from_tokens(
                ctx_, draft.ctx_, model_, tokens, params, on_token,
                cancel, n_draft_max, draft_path, t0,
                tool_grammar_, tool_grammar_lazy_, parse_generation_prompt_);
        }
    }
    return result;
}
 
// ── gh#106 (v2.9.0): target-owned MTP speculative kernel ───────────
//
// Distinct from the gh#36 separate-draft kernel above. The MTP head
// (ctx_dft) shares the target's KV via ctx_other, so the CALLER only
// ever decodes ctx_tgt; the impl owns every ctx_dft decode. The loop is
// draft → decode(ctx_tgt) → process → sample_and_accept_n → accept,
// mirroring extern/llama.cpp/tools/server/server-context.cpp. Reuses the
// gh#36 file-local helpers (spec_build_batch / spec_emit_token /
// spec_commit_accepted / spec_trim_rejected_drafts / spec_finalize /
// spec_cleanup / spec_error / to_common_sampling) — only the decode step
// and the prefill differ. NO checkpoint dance: shared-KV gemma4 targets
// are PART-seq_rm, so the FULL-only rollback path never applies.
 
namespace {
 
int mtp_run_draft(SpeculativeRunState& state, int n_max) {
    auto& dp = common_speculative_get_draft_params(state.spec, state.seq_id);
    dp.drafting = true;
    dp.n_max = n_max;
    dp.n_past = state.n_past;
    dp.id_last = state.id_last;
    dp.prompt = &state.prompt_tgt;
    dp.result = &state.draft;
    common_speculative_draft(state.spec);
    return static_cast<int>(state.draft.size());
}
 
bool mtp_decode_and_process(SpeculativeRunState& state) {
    spec_build_batch(state);  // [id_last@n_past, draft@n_past+1 ...]
    if (llama_decode(state.ctx_tgt, state.batch_tgt) != 0) {
        state.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
        state.error_message = "MTP target decode failed";
        state.finish_reason = "error";
        return false;
    }
    if (!common_speculative_process(state.spec, state.batch_tgt)) {
        state.error_code = ENTROPIC_ERROR_GENERATE_FAILED;
        state.error_message = "common_speculative_process failed";
        state.finish_reason = "error";
        return false;
    }
    return true;
}
 
bool mtp_accept_round(
    SpeculativeRunState& state, int n_max, const llama_vocab* vocab,
    int max_tokens, std::function<void(std::string_view)>& on_token,
    std::atomic<bool>& cancel) {
    int drafted = mtp_run_draft(state, n_max);
    if (!mtp_decode_and_process(state)) { return false; }
    auto ids = common_sampler_sample_and_accept_n(
        state.smpl, state.ctx_tgt, state.draft);
    int accepted = static_cast<int>(ids.size()) - 1;
    if (accepted < 0) { accepted = 0; }
    // gh#108: only accept into the spec when this round actually drafted.
    // common_speculative_accept asserts impl_last[seq] (speculative.cpp:1650),
    // which is set ONLY for a non-empty draft (1604/1614) — a zero-draft round
    // would abort. The round still progresses by one token (the bonus in ids),
    // and process() already updated pending_h, so skipping accept is equivalent.
    if (drafted > 0) {
        common_speculative_accept(state.spec, state.seq_id, accepted);
    }
    state.n_drafted += drafted;
    state.n_accepted += accepted;
    // Same layout as gh#36: id_last fills one slot at n_past, the
    // `accepted` drafts fill the next slots — n_past advances by ids.size().
    state.n_past += static_cast<int>(ids.size());
    bool stop = spec_commit_accepted(
        state, ids, vocab, max_tokens, on_token, cancel);
    state.draft.clear();
    spec_trim_rejected_drafts(state);
    return !stop;
}
 
bool mtp_process_chunk(SpeculativeRunState& state, int off, int chunk) {
    common_batch_clear(state.batch_tgt);
    for (int j = 0; j < chunk; ++j) {
        common_batch_add(state.batch_tgt, state.prompt_tgt[off + j],
                         off + j, {state.seq_id}, false);
    }
    if (llama_decode(state.ctx_tgt, state.batch_tgt) != 0) { return false; }
    return common_speculative_process(state.spec, state.batch_tgt);
}
 
bool mtp_prefill_and_seed(SpeculativeRunState& state) {
    int total = static_cast<int>(state.prompt_tgt.size());
    if (total == 0) { return true; }  // 1-token prompt: round 1 drafts cold
    int n_batch = llama_n_batch(state.ctx_tgt);
    for (int off = 0; off < total; off += n_batch) {
        int chunk = std::min(n_batch, total - off);
        if (!mtp_process_chunk(state, off, chunk)) { return false; }
    }
    return true;
}
 
std::string mtp_init_decoder(
    SpeculativeRunState& state, llama_model* model_tgt,
    const GenerationParams& params, int n_max,
    const std::string& tool_grammar,   // gh#134 (v2.10.4)
    bool tool_grammar_lazy,
    const std::string& generation_prompt) {
    auto common_sampling =
        to_common_sampling(params, tool_grammar, tool_grammar_lazy,
                           generation_prompt);
    state.smpl = common_sampler_init(model_tgt, common_sampling);
    if (!state.smpl) { return "common_sampler_init failed"; }
    common_params_speculative sp;
    sp.types = {COMMON_SPECULATIVE_TYPE_DRAFT_MTP};
    sp.draft.n_max = n_max;
    sp.draft.ctx_tgt = state.ctx_tgt;
    sp.draft.ctx_dft = state.ctx_dft;
    state.spec = common_speculative_init(sp, 1);
    if (!state.spec) {
        common_sampler_free(state.smpl);
        state.smpl = nullptr;
        return "common_speculative_init (MTP) failed";
    }
    state.batch_tgt = llama_batch_init(llama_n_batch(state.ctx_tgt), 0, 1);
    state.batch_initialized = true;
    return "";
}
 
std::string mtp_init_run(
    SpeculativeRunState& state, llama_model* model_tgt,
    const std::vector<llama_token>& tokens,
    const GenerationParams& params, int n_max,
    const std::string& tool_grammar, bool tool_grammar_lazy,
    const std::string& generation_prompt) {
    state.id_last = tokens.back();
    state.prompt_tgt.assign(tokens.begin(), tokens.end() - 1);
    state.n_past = static_cast<int>(tokens.size()) - 1;
    llama_memory_clear(llama_get_memory(state.ctx_tgt), true);
    auto err = mtp_init_decoder(state, model_tgt, params, n_max,
                                tool_grammar, tool_grammar_lazy,
                                generation_prompt);
    if (!err.empty()) { return err; }
    if (!mtp_prefill_and_seed(state)) { return "MTP prefill/process failed"; }
    common_speculative_begin(state.spec, state.seq_id, state.prompt_tgt);
    return "";
}
 
void mtp_run_loop(
    SpeculativeRunState& state, int n_max, const llama_vocab* vocab,
    int max_tokens, std::function<void(std::string_view)>& on_token,
    std::atomic<bool>& cancel) {
    while (state.n_generated < max_tokens) {
        if (cancel.load(std::memory_order_acquire)) {
            state.error_code = ENTROPIC_ERROR_CANCELLED;
            state.finish_reason = "cancelled";
            break;
        }
        if (!mtp_accept_round(state, n_max, vocab, max_tokens,
                              on_token, cancel)) {
            break;
        }
    }
    if (state.finish_reason.empty()) {
        state.finish_reason = (state.n_generated >= max_tokens)
                                  ? "length" : "stop";
    }
}
 
GenerationResult mtp_run_from_tokens(
    llama_context* ctx_tgt, llama_context* ctx_dft, llama_model* model_tgt,
    const std::vector<llama_token>& tokens, const GenerationParams& params,
    std::function<void(std::string_view)>& on_token,
    std::atomic<bool>& cancel, int n_max,
    const std::vector<std::string>& stop,
    std::chrono::steady_clock::time_point t0,
    const std::string& tool_grammar, bool tool_grammar_lazy,
    const std::string& generation_prompt) {
    SpeculativeRunState state;
    state.ctx_tgt = ctx_tgt;
    state.ctx_dft = ctx_dft;
    state.stop = stop;  // gh#108: MTP honors stop sequences (effective_stop)
    auto init_err = mtp_init_run(state, model_tgt, tokens, params, n_max,
                                 tool_grammar, tool_grammar_lazy,
                                 generation_prompt);
    if (!init_err.empty()) {
        spec_cleanup(state);
        return spec_error(ENTROPIC_ERROR_GENERATE_FAILED,
                          std::move(init_err));
    }
    mtp_run_loop(state, n_max, llama_model_get_vocab(model_tgt),
                 params.max_tokens, on_token, cancel);
    return spec_finalize(state, t0);
}
 
}  // anonymous namespace
 
GenerationResult LlamaCppBackend::mtp_guard(
    const GenerationParams& params,
    const std::function<void(std::string_view)>& on_token,
    const std::string& head_path, int n_max) {
    GenerationResult r;  // ENTROPIC_OK by default → proceed
    std::string reason = mtp_unsupported_reason(
        params.temperature, !params.grammar.empty(),
        static_cast<bool>(on_token));
    if (!is_active()) {
        r = spec_error(ENTROPIC_ERROR_INVALID_STATE,
                       "MTP requires an ACTIVE target");
    } else if (!reason.empty()) {
        r = spec_error(ENTROPIC_ERROR_SPECULATIVE_INCOMPATIBLE_CONFIG, reason);
    } else if (1 + effective_n_draft(n_max) > llama_n_batch(ctx_)) {
        // Bound-check BEFORE setup_mtp_draft allocates. Ordering this the
        // other way made the guard's ANSWER depend on process history: an
        // out-of-envelope window (e.g. n_draft=100000) is still handed to
        // setup_mtp_draft, whose allocation SUCCEEDS in a fresh process —
        // so the guard fell through and returned the correct
        // SPECULATIVE_INCOMPATIBLE_CONFIG — but FAILS after a prior MTP
        // session, returning LOAD_FAILED for the same config. Same input,
        // two different typed errors, decided by allocator state. Measured
        // as `5 == 54` when a second MTP case ran in one process.
        //
        // Must use the same normalisation setup_mtp_draft applies, or the
        // check disagrees with the value actually used.
        r = spec_error(ENTROPIC_ERROR_SPECULATIVE_INCOMPATIBLE_CONFIG,
            "speculative.n_draft+1 ("
            + std::to_string(1 + effective_n_draft(n_max))
            + ") exceeds n_batch (" + std::to_string(llama_n_batch(ctx_))
            + "); reduce n_draft or raise n_batch");
    } else if (!setup_mtp_draft(head_path, n_max)) {
        r = spec_error(ENTROPIC_ERROR_LOAD_FAILED, last_error_);
    }
    return r;
}
 
GenerationResult LlamaCppBackend::generate_mtp(
    const std::vector<Message>& messages,
    const GenerationParams& params,
    std::function<void(std::string_view)> on_token,
    std::atomic<bool>& cancel,
    const std::string& head_path,
    int n_max)
{
    auto t0 = entropic::log::now();
    std::lock_guard<std::mutex> lk(mtp_mutex_);  // serialise vs teardown
    GenerationResult result = mtp_guard(params, on_token, head_path, n_max);
    if (result.error_code != ENTROPIC_OK) {
        return result;
    }
    invalidate_resident_kv();  // MTP kernel owns seq 0 itself
    auto tokens = tokenize(render_prompt(messages, params), true);
    if (tokens.size() < 2) {
        return spec_error(ENTROPIC_ERROR_GENERATE_FAILED,
            "MTP prompt must have at least 2 tokens");
    }
    logger->info("MTP: {} input tokens, max_tokens={}, n_max={}",
                 tokens.size(), params.max_tokens, mtp_n_max_);
    return mtp_run_from_tokens(ctx_, mtp_draft_ctx_, model_, tokens, params,
                               on_token, cancel, mtp_n_max_,
                               effective_stop(params), t0,  // gh#108: honor stops
                               tool_grammar_, tool_grammar_lazy_,   // gh#134
                               parse_generation_prompt_);
}
 
GenerationResult LlamaCppBackend::do_complete(
    const std::string& prompt,
    const GenerationParams& params)
{
    auto t0 = entropic::log::now();
    invalidate_resident_kv();  // gh#96: decode_loop/run_prefill mutate seq 0
    auto tokens = tokenize(prompt, false);
 
    logger->info("Complete: {} input tokens, max_tokens={}",
              tokens.size(), params.max_tokens);
    log_sampler_config(params);
    auto result = decode_loop(tokens, params, nullptr, nullptr);
    finalize_result(result, t0);
    return result;
}
 
// ── Architecture detection (v1.9.13) ───────────────────────
 
bool LlamaCppBackend::is_recurrent() const {
    return is_recurrent_;
}
 
// ── Capability overrides (v1.9.13) ─────────────────────────
 
bool LlamaCppBackend::do_supports(BackendCapability cap) const {
    int idx = static_cast<int>(cap);
    int count = static_cast<int>(BackendCapability::_COUNT);
    if (idx < 0 || idx >= count) {
        return false;
    }
 
    // Static capabilities: true = always supported. Length must equal
    // BackendCapability::_COUNT — trailing entries get appended as new
    // capabilities are introduced (gh#53 added AUDIO at index 12).
    static constexpr bool always[] = {
        false, false, true, true, true, true,
        false, true,  true, false, false, true,
        false,  // AUDIO — dynamic only (mtmd_support_audio)
    };
 
    // Dynamic capabilities override the static table
    bool result = always[idx];
    if (!result) {
        result = (cap == BackendCapability::KV_CACHE && !is_recurrent())
              || (cap == BackendCapability::HIDDEN_STATE && is_recurrent())
              || (cap == BackendCapability::VISION
                  && !config().mmproj_path.empty())
              || (cap == BackendCapability::AUDIO
                  && mtmd_ctx_ != nullptr
                  && mtmd_support_audio(mtmd_ctx_))
              || (cap == BackendCapability::SPECULATIVE_DECODING
                  && !is_recurrent());
    }
    return result;
}
 
std::string LlamaCppBackend::do_backend_name() const {
    return "llama.cpp";
}
 
BackendInfo LlamaCppBackend::do_info() const {
    BackendInfo bi;
    bi.name = "llama.cpp";
#if defined(ENTROPIC_BACKEND_CUDA)
    bi.compute_device = "cuda";
#elif defined(ENTROPIC_BACKEND_VULKAN)
    bi.compute_device = "vulkan";
#else
    bi.compute_device = "cpu";
#endif
    bi.model_format = "gguf";
 
    if (state() != ModelState::COLD && model_ != nullptr) {
        bi.architecture = is_recurrent() ? "recurrent" : "transformer";
        bi.max_context_length =
            is_recurrent() ? -1 : config().context_length;
        bi.parameter_count = llama_model_n_params(model_);
        bi.vram_bytes = 0;
        bi.ram_bytes = llama_model_size(model_);
 
        char desc[256] = {};
        llama_model_desc(model_, desc, sizeof(desc));
        bi.quantization = desc;
    }
    return bi;
}
 
bool LlamaCppBackend::do_clear_state(int seq_id) {
    if (ctx_ == nullptr) {
        return false;
    }
    auto mem = llama_get_memory(ctx_);
    if (seq_id < 0) {
        llama_memory_clear(mem, true);
    } else {
        llama_memory_seq_rm(mem, seq_id, -1, -1);
    }
    return true;
}
 
bool LlamaCppBackend::do_save_state(
    int seq_id, std::vector<uint8_t>& buffer) const {
    if (ctx_ == nullptr) { return false; }
    size_t sz = llama_state_seq_get_size(
        ctx_, static_cast<llama_seq_id>(seq_id));
    if (sz == 0) { return false; }
    buffer.resize(sz);
    size_t written = llama_state_seq_get_data(
        ctx_, buffer.data(), sz,
        static_cast<llama_seq_id>(seq_id));
    return written == sz;
}
 
bool LlamaCppBackend::do_restore_state(
    int seq_id, const std::vector<uint8_t>& buffer) {
    if (ctx_ == nullptr || buffer.empty()) { return false; }
    size_t result = llama_state_seq_set_data(
        ctx_, buffer.data(), buffer.size(),
        static_cast<llama_seq_id>(seq_id));
    return result > 0;
}
 
} // namespace entropic