llama_cpp_backend.h

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#pragma once
 
#include <entropic/inference/backend.h>
 
#include "prompt_cache.h"
 
#include <llama.h>
 
#include <atomic>
#include <chrono>
#include <cstdint>
#include <functional>
#include <memory>
#include <mutex>
#include <string>
#include <vector>
 
// Forward-declare libmtmd's opaque types at file scope so they
// resolve to ::mtmd_context / ::mtmd_bitmap (not entropic::mtmd_*)
// when referenced inside the class body. Full types live in
// extern/llama.cpp/tools/mtmd/mtmd.h and are only included from
// the implementation file. (v2.1.8, gh#37/v1.9.11 Phase 5)
extern "C" {
struct mtmd_context;
struct mtmd_bitmap;
}
 
namespace entropic {
 
class LlamaCppBackend : public InferenceBackend {
public:
    ~LlamaCppBackend() override;
 
    void set_prompt_cache_config(const PromptCacheConfig& config) {
        prompt_cache_config_ = config;
    }
 
    void clear_prompt_cache() override {
        if (prompt_cache_) { prompt_cache_->clear(); }
    }
 
    std::vector<int32_t> tokenize_text(
        const std::string& text) const override;
 
    /* ── llama.cpp handle accessors (v1.9.2) ────────────── */
 
    llama_model* llama_model_ptr() { return model_; }
 
    llama_context* llama_context_ptr() { return ctx_; }
 
protected:
    /* ── Lifecycle overrides ─────────────────────────────── */
 
    bool do_load(const ModelConfig& config) override;
    bool do_activate() override;
    void do_deactivate() override;
    void do_unload() override;
 
    /* ── Generation overrides ────────────────────────────── */
 
    GenerationResult do_generate(
        const std::vector<Message>& messages,
        const GenerationParams& params) override;
 
    GenerationResult do_generate_streaming(
        const std::vector<Message>& messages,
        const GenerationParams& params,
        std::function<void(std::string_view token)> on_token,
        std::atomic<bool>& cancel) override;
 
    GenerationResult do_generate_speculative(
        const std::vector<Message>& messages,
        const GenerationParams& params,
        std::function<void(std::string_view token)> on_token,
        std::atomic<bool>& cancel) override;
 
public:
    GenerationResult generate_speculative_with_draft(
        const std::vector<Message>& messages,
        const GenerationParams& params,
        std::function<void(std::string_view token)> on_token,
        std::atomic<bool>& cancel,
        LlamaCppBackend& draft,
        int n_draft_max,
        const std::string& draft_path);
 
protected:
    GenerationResult do_complete(
        const std::string& prompt,
        const GenerationParams& params) override;
 
    int do_count_tokens(const std::string& text) const override;
 
    /* ── Evaluation override (v1.9.10) ──────────────────── */
 
    LogprobResult do_evaluate_logprobs(
        const int32_t* tokens,
        int n_tokens) override;
 
    /* ── Capability overrides (v1.9.13) ──────────────────── */
 
    bool do_supports(BackendCapability cap) const override;
    std::string do_backend_name() const override;
    BackendInfo do_info() const override;
    bool do_clear_state(int seq_id) override;
 
    /* ── llama.cpp handles ───────────────────────────────── */
 
    llama_model* model_ = nullptr;             
    llama_context* ctx_ = nullptr;             
    const llama_vocab* vocab_ = nullptr;       
 
    /* ── Prompt cache ───────────────────────────────────── */
 
    PromptCacheConfig prompt_cache_config_;      
    std::unique_ptr<PromptCache> prompt_cache_;  
 
    /* ── Internal helpers ────────────────────────────────── */
 
    std::vector<llama_token> tokenize(
        const std::string& text, bool add_special) const;
 
    std::string detokenize(llama_token token) const;
 
    std::string apply_chat_template(
        const std::vector<Message>& messages,
        const GenerationParams& params) const;
 
    GenerationResult decode_loop(
        const std::vector<llama_token>& tokens,
        const GenerationParams& params,
        std::function<void(std::string_view)> on_token,
        std::atomic<bool>* cancel);
 
    bool run_prefill(const std::vector<llama_token>& tokens);
 
    std::string step_token(
        llama_sampler* sampler,
        std::string& generated,
        std::function<void(std::string_view)>& on_token,
        const std::vector<std::string>& stop);
 
    llama_sampler* create_sampler(const GenerationParams& params) const;
 
    static std::string extract_system_prompt(
        const std::vector<Message>& messages);
 
    bool run_prefill_cached(
        const std::vector<llama_token>& tokens,
        const std::string& system_prompt,
        const std::vector<Message>& messages,
        const GenerationParams& params);
 
    bool decode_tokens_from(
        const std::vector<llama_token>& tokens, int start_offset);
 
    bool restore_cached_prefix(
        const CacheEntry* cached,
        const std::vector<llama_token>& tokens);
 
    bool prefill_and_cache_prefix(
        const std::vector<llama_token>& tokens,
        int prefix_tokens,
        const CacheKey& key);
 
    void save_prefix_to_cache(const CacheKey& key, int prefix_tokens);
 
    int compute_prefix_token_count(
        const std::vector<Message>& messages,
        const GenerationParams& params);
 
    /* ── Evaluation helpers (v1.9.10) ───────────────────── */
 
    llama_seq_id allocate_temp_seq_id();
 
    void release_temp_seq_id(llama_seq_id seq_id);
 
    static float extract_token_logprob(
        const float* logits,
        int32_t next_token,
        int n_vocab);
 
    std::mutex seq_id_mutex_;                 
    std::vector<llama_seq_id> free_seq_ids_;  
 
    /* ── Architecture detection (v1.9.13) ──────────────── */
 
    bool is_recurrent_ = false;
 
    bool is_recurrent() const;
 
    /* ── Vision / multimodal (v1.9.11 Phases 5–7 + v2.1.8) ── */
 
    ::mtmd_context* mtmd_ctx_ = nullptr;
 
    bool has_vision_ = false;
 
    GenerationResult generate_multimodal(
        const std::vector<Message>& messages,
        const GenerationParams& params,
        std::function<void(std::string_view token)> on_token,
        std::atomic<bool>* cancel);
 
    void init_mmproj_if_configured();
 
    bool load_gpu_model();
 
    bool create_inference_context();
 
    entropic_error_t mtmd_prefill(
        const std::string& prompt,
        const std::vector<::mtmd_bitmap*>& bitmaps,
        std::string& err_msg);
 
    GenerationResult run_sampling_loop(
        const GenerationParams& params,
        std::function<void(std::string_view token)> on_token,
        std::atomic<bool>* cancel,
        const std::chrono::steady_clock::time_point& t0);
 
    GenerationResult do_generate_text_only(
        const std::vector<Message>& messages,
        const GenerationParams& params);
 
    GenerationResult do_generate_streaming_text_only(
        const std::vector<Message>& messages,
        const GenerationParams& params,
        std::function<void(std::string_view token)> on_token,
        std::atomic<bool>& cancel);
};
 
} // namespace entropic