backend.h

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#pragma once
 
#include <entropic/types/backend_capability.h>
#include <entropic/types/config.h>
#include <entropic/types/generation_result.h>
#include <entropic/types/logprob_result.h>
#include <entropic/types/message.h>
#include <entropic/interfaces/i_hook_handler.h>
 
#include <atomic>
#include <cstdint>
#include <functional>
#include <mutex>
#include <string>
#include <string_view>
#include <vector>
 
namespace entropic {
 
class InferenceBackend {
public:
    virtual ~InferenceBackend() = default;
 
    /* ── Lifecycle (template methods — own the state machine) ── */
 
    bool load(const ModelConfig& config);
 
    bool activate();
 
    void deactivate();
 
    void unload();
 
    bool load_and_activate(const ModelConfig& config);
 
    /* ── Generation (require ACTIVE state) ───────────────── */
 
    GenerationResult generate(
        const std::vector<Message>& messages,
        const GenerationParams& params);
 
    GenerationResult generate(
        const std::vector<Message>& messages,
        const GenerationParams& params,
        std::atomic<bool>& cancel);
 
    GenerationResult generate_streaming(
        const std::vector<Message>& messages,
        const GenerationParams& params,
        std::function<void(std::string_view token)> on_token,
        std::atomic<bool>& cancel);
 
    GenerationResult generate_speculative(
        const std::vector<Message>& messages,
        const GenerationParams& params,
        std::function<void(std::string_view token)> on_token,
        std::atomic<bool>& cancel);
 
    GenerationResult complete(
        const std::string& prompt,
        const GenerationParams& params);
 
    /* ── Evaluation (require ACTIVE state) ─────────────── */
 
    LogprobResult evaluate_logprobs(
        const int32_t* tokens,
        int n_tokens);
 
    float compute_perplexity(
        const int32_t* tokens,
        int n_tokens);
 
    /* ── Queries (lock-free) ─────────────────────────────── */
 
    ModelState state() const { return state_.load(std::memory_order_acquire); }
 
    bool is_active() const { return state() == ModelState::ACTIVE; }
 
    bool is_loaded() const { return state() != ModelState::COLD; }
 
    int count_tokens(const std::string& text) const;
 
    virtual std::vector<int32_t> tokenize_text(
        const std::string& text) const { return {}; }
 
    int context_length() const { return config_.context_length; }
 
    virtual void clear_prompt_cache() {}
 
    virtual std::string tool_call_close_marker() const { return ""; }
 
    const ModelConfig& config() const { return config_; }
 
    /* ── Capability queries (v1.9.13) ────────────────────── */
 
    bool supports(BackendCapability cap) const;
 
    std::vector<BackendCapability> capabilities() const;
 
    /* ── Backend metadata (v1.9.13) ──────────────────────── */
 
    BackendInfo info() const;
 
    virtual std::string active_tool_grammar() const { return {}; }
 
    /* ── Model state management (v1.9.13) ────────────────── */
 
    bool save_state(int seq_id, std::vector<uint8_t>& buffer) const;
 
    bool restore_state(int seq_id, const std::vector<uint8_t>& buffer);
 
    bool clear_state(int seq_id = -1);
 
    /* ── Multi-sequence generation (v1.9.13) ─────────────── */
 
    GenerationResult generate_seq(
        int seq_id,
        const std::vector<Message>& messages,
        const GenerationParams& params);
 
    GenerationResult generate_streaming_seq(
        int seq_id,
        const std::vector<Message>& messages,
        const GenerationParams& params,
        std::function<void(std::string_view token)> on_token,
        std::atomic<bool>& cancel);
 
    /* ── Same-prefix batch generation (gh#98, v2.8.0) ────── */
 
    std::vector<GenerationResult> generate_batch(
        const std::vector<std::vector<Message>>& requests,
        const std::vector<GenerationParams>& params,
        std::atomic<bool>& cancel);
 
protected:
    /* ── Subclass overrides (20%) ────────────────────────── */
 
    virtual bool do_load(const ModelConfig& config) = 0;
 
    virtual bool do_activate() = 0;
 
    virtual bool do_load_active(const ModelConfig& config) {
        return do_load(config) && do_activate();
    }
 
public:
    virtual void forget_session_kv(const std::string& session_key) {
        (void)session_key;
    }
 
protected:
 
    virtual void do_deactivate() = 0;
 
    virtual void do_unload() = 0;
 
    virtual GenerationResult do_generate(
        const std::vector<Message>& messages,
        const GenerationParams& params) = 0;
 
    virtual GenerationResult do_generate(
        const std::vector<Message>& messages,
        const GenerationParams& params,
        std::atomic<bool>& cancel) {
        (void)cancel;
        return do_generate(messages, params);
    }
 
    virtual std::vector<GenerationResult> do_generate_batch(
        const std::vector<std::vector<Message>>& requests,
        const std::vector<GenerationParams>& params,
        std::atomic<bool>& cancel) {
        std::vector<GenerationResult> out;
        out.reserve(requests.size());
        for (std::size_t i = 0; i < requests.size(); ++i) {
            out.push_back(do_generate(requests[i], params[i], cancel));
        }
        return out;
    }
 
    virtual GenerationResult do_generate_streaming(
        const std::vector<Message>& messages,
        const GenerationParams& params,
        std::function<void(std::string_view token)> on_token,
        std::atomic<bool>& cancel) = 0;
 
    virtual GenerationResult do_generate_speculative(
        const std::vector<Message>& messages,
        const GenerationParams& params,
        std::function<void(std::string_view token)> on_token,
        std::atomic<bool>& cancel);
 
    virtual GenerationResult do_complete(
        const std::string& prompt,
        const GenerationParams& params) = 0;
 
    virtual int do_count_tokens(const std::string& text) const = 0;
 
    virtual LogprobResult do_evaluate_logprobs(
        const int32_t* tokens,
        int n_tokens) = 0;
 
    /* ── New overridable methods (v1.9.13) ───────────────── */
 
    virtual bool do_supports(BackendCapability cap) const;
 
    virtual std::string do_backend_name() const = 0;
 
    virtual BackendInfo do_info() const;
 
    virtual bool do_save_state(int seq_id,
                               std::vector<uint8_t>& buffer) const;
 
    virtual bool do_restore_state(int seq_id,
                                  const std::vector<uint8_t>& buffer);
 
    virtual bool do_clear_state(int seq_id);
 
    virtual GenerationResult do_generate_seq(
        int seq_id,
        const std::vector<Message>& messages,
        const GenerationParams& params);
 
    virtual GenerationResult do_generate_streaming_seq(
        int seq_id,
        const std::vector<Message>& messages,
        const GenerationParams& params,
        std::function<void(std::string_view token)> on_token,
        std::atomic<bool>& cancel);
 
    std::string last_error_;  
 
    bool fire_model_load_hook(const ModelConfig& config);  
 
    bool promote_to_active();
 
    void set_hooks(const HookInterface& hooks) { hooks_ = hooks; }
 
protected:
    std::atomic<ModelState> state_{ModelState::COLD};
 
private:
    ModelConfig config_;
    std::mutex transition_mutex_;  
    std::mutex eval_mutex_;        
    HookInterface hooks_;          
};
 
} // namespace entropic