orchestrator.h

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#pragma once
 
#include <entropic/inference/backend.h>
#include <entropic/inference/adapter_manager.h>
#include <entropic/inference/grammar_registry.h>
#include <entropic/inference/profile_registry.h>
#include <entropic/inference/secondary_model_loader.h>
#include <entropic/inference/throughput_tracker.h>
#include <entropic/inference/adapters/adapter_base.h>
#include <entropic/types/config.h>
#include <entropic/types/error.h>
 
#include <chrono>
#include <functional>
#include <memory>
#include <mutex>
#include <optional>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
 
struct llama_context;  // Forward declaration for adapter management
struct llama_model;    // Forward declaration for speculative compat (v2.1.11)
 
namespace entropic {
 
// v2.13.1: `gpu_layers: auto` types. Declared, not included — they live in
// src/inference/ and the public header must not pull a private one in.
struct GgufShape;
struct AutoPlacement;
 
struct RoutingResult {
    std::string tier_name;                
    std::string previous_tier;            
    std::string model_raw;                
    std::string swap_action = "none";     
    double routing_ms = 0.0;              
    std::string adapter_name;             
    double adapter_swap_ms = 0.0;         
};
 
struct TierSamplerOverrides {
    std::optional<float> temperature;        
    std::optional<int>   max_output_tokens;  
    std::optional<float> top_p;              
    std::optional<int>   top_k;              
    std::optional<float> min_p;              
    std::optional<float> presence_penalty;   
    std::optional<float> frequency_penalty;  
    std::optional<float> repeat_penalty;     
    std::optional<bool>  enable_thinking;    
    std::optional<std::string> tool_call_mode;  
};
 
ENTROPIC_EXPORT void apply_tier_sampler_overrides(
    GenerationParams& params,
    const TierSamplerOverrides& ov);
 
class ModelOrchestrator {
public:
    bool initialize(const ParsedConfig& config);
 
    void shutdown();
 
    ~ModelOrchestrator();
 
    /* ── Generation ──────────────────────────────────────── */
 
    GenerationResult generate(
        const std::vector<Message>& messages,
        const GenerationParams& params,
        const std::string& tier_name = "");
 
    GenerationResult generate(
        const std::vector<Message>& messages,
        const GenerationParams& params,
        std::atomic<bool>& cancel,
        const std::string& tier_name = "");
 
    std::vector<GenerationResult> generate_batch(
        const std::vector<std::vector<Message>>& messages_list,
        const std::vector<GenerationParams>& params_list,
        const std::vector<std::string>& tiers,
        std::atomic<bool>& cancel);
 
    GenerationResult generate_streaming(
        const std::vector<Message>& messages,
        const GenerationParams& params,
        std::function<void(std::string_view)> on_token,
        std::atomic<bool>& cancel,
        const std::string& tier_name = "");
 
    /* ── Routing ─────────────────────────────────────────── */
 
    std::string route(const std::vector<Message>& messages);
 
    /* ── Queries ─────────────────────────────────────────── */
 
    RoutingResult last_routing_result() const;
 
    std::string last_used_tier() const;
 
    std::vector<std::string> loaded_models() const;
 
    std::vector<std::string> available_models() const;
 
    bool can_handoff(const std::string& from, const std::string& to) const;
 
    ChatAdapter* get_adapter(const std::string& tier_name) const;
 
    InferenceBackend* get_backend(const std::string& tier_name) const;
 
    InferenceBackend* ensure_model(const std::string& tier_name);
 
    entropic_error_t release_models(const std::string& tier_name);
 
    bool tier_declares_vision(const std::string& tier_name) const;
 
    AdapterManager& adapter_manager() { return lora_manager_; }
 
    GrammarRegistry& grammar_registry() { return grammar_registry_; }
 
    std::vector<GenerationRecord> generation_records() const;
 
    static constexpr size_t kMaxGenerationRecords = 64;
 
    ProfileRegistry& profile_registry() { return profile_registry_; }
 
    ThroughputTracker& throughput_tracker() { return throughput_tracker_; }
 
    size_t load_grammars_from(const std::filesystem::path& grammar_dir);
 
    void clear_all_prompt_caches();
 
    bool has_vision_capable_tier() const;
 
    struct SpeculativeCompatInfo {
        bool compatible = false;     
        std::string diagnostic;      
    };
 
    SpeculativeCompatInfo check_speculative_compat() const;
 
    void set_speculative_enabled(bool enabled) {
        config_.inference.speculative.enabled = enabled;
    }
 
    /* ── VRAM-aware tier residency (v2.2.4, gh#57) ────────── */
 
    enum class ResidencyEvent : int {
        Loaded = 0,
        Evicted = 1,
        ActivationSwap = 2,
    };
 
    using ResidencyObserverFn = std::function<void(
        ResidencyEvent event,
        const std::string& tier_name,
        const std::string& model_path,
        size_t footprint)>;
 
    void set_residency_observer(ResidencyObserverFn cb);
 
    std::string residency_snapshot_json() const;
 
    void forget_session_kv(const std::string& session_key);
 
    size_t vram_budget_bytes() const { return vram_budget_bytes_; }
 
    size_t tier_footprint_bytes(const std::string& tier_name) const;
 
    entropic_error_t last_residency_error() const { return last_residency_error_; }
 
    void clear_last_residency_error() { last_residency_error_ = ENTROPIC_OK; }
 
    std::string select_vision_tier() const;
 
    void apply_tier_sampler_defaults_for_test(GenerationParams& params,
                                              const std::string& tier_name) {
        apply_tier_sampler_defaults(params, tier_name);
    }
 
private:
    void record_generation(GenerationResult& result,
                           const GenerationParams& resolved_params,
                           const InferenceBackend* model);
 
    void finish_generation(GenerationResult& result,
                           InferenceBackend* model,
                           const GenerationParams& resolved_params,
                           const std::string& selected,
                           double routing_ms,
                           double swap_ms,
                           std::chrono::steady_clock::time_point t_start);
 
    /* ── gh#154: bounded per-generation metric ring ──────── */
    mutable std::mutex records_mutex_;
    std::vector<GenerationRecord> generation_records_;
 
    /* ── Model pool (one backend per unique path) ────────── */
    std::unordered_map<std::string, std::shared_ptr<InferenceBackend>> model_pool_;
 
    /* ── Tier → backend mapping (many-to-one) ────────────── */
    std::unordered_map<std::string, std::shared_ptr<InferenceBackend>> tiers_;
 
    /* ── Per-tier adapters (one-to-one, identity-specific) ── */
    std::unordered_map<std::string, std::unique_ptr<ChatAdapter>> adapters_;
 
    /* ── Secondary models (router, draft, future thinking) ── */
    SecondaryModelLoader secondary_loader_;
 
    /* ── Routing state ───────────────────────────────────── */
    std::unordered_map<std::string, std::string> tier_map_;  
    std::unordered_map<std::string, std::unordered_set<std::string>> handoff_rules_;
    std::string default_tier_;
    std::string loaded_main_tier_;
    RoutingResult last_routing_result_;
    std::vector<std::string> tier_history_;  
 
    mutable std::mutex swap_mutex_;  
 
    mutable std::recursive_mutex generation_mutex_;
 
    ParsedConfig config_;
 
    /* ── Residency tracking (v2.2.4, gh#57) ──────────────── */
 
    mutable std::unordered_map<std::string, size_t> tier_footprint_bytes_;
 
    std::unordered_map<std::string, long long> tier_last_activation_ms_;
 
    std::chrono::steady_clock::time_point start_time_{std::chrono::steady_clock::now()};
 
    size_t vram_budget_bytes_{0};
 
    bool vram_budget_resolved_{false};
 
    bool vram_budget_deferred_{false};
 
    ResidencyObserverFn residency_observer_;
 
    entropic_error_t last_residency_error_{ENTROPIC_OK};
 
    std::string last_residency_message_;
 
    size_t estimate_footprint_bytes(const std::string& tier_name) const;
 
    void log_fit_recommendation(const std::string& tier_name) const;
 
    static size_t resolve_vram_budget_bytes();
 
    void ensure_vram_budget();
 
    static bool budget_resolves_at_init(const ParsedConfig& config);
 
    static void route_ggml_logs(const ParsedConfig& config);
 
    void fire_residency_observer(
        ResidencyEvent event,
        const std::string& tier_name,
        const std::string& model_path,
        size_t footprint);
 
    /* ── LoRA adapter management (v1.9.2) ────────────────── */
    AdapterManager lora_manager_;  
 
    /* ── Grammar registry (v1.9.3) ────────────────────────── */
    GrammarRegistry grammar_registry_;  
 
    /* ── Profile registry (v2.0.0) ───────────────────────── */
    ProfileRegistry profile_registry_;  
 
    /* ── Throughput tracker (v2.0.0) ─────────────────────── */
    ThroughputTracker throughput_tracker_;  
 
    /* ── Internal ────────────────────────────────────────── */
 
    InferenceBackend* get_model(const std::string& tier_name);
 
    void record_activation_reuse(const std::string& tier_name);
 
    bool residency_admits(const std::string& tier_name);
 
    void resolve_auto_gpu_layers(const std::string& tier_name);
 
    uint64_t resident_draft_bytes() const;
 
    void fall_back_to_weights_estimate(const std::string& tier_name,
                                       uint64_t file_bytes, const char* why);
 
    static uint64_t host_expert_bytes_for(const ModelConfig& cfg);
 
    void log_auto_placement(const std::string& tier_name,
                            const GgufShape& shape,
                            const AutoPlacement& placement,
                            uint64_t file_bytes) const;
 
    bool config_admits(const std::string& tier_name);
 
    void refuse_residency(const std::string& tier_name,
                          entropic_error_t code,
                          const std::string& why);
 
    InferenceBackend* activate_and_track(
        const std::string& tier_name,
        const std::shared_ptr<InferenceBackend>& backend);
 
    GenerationResult build_no_model_error(const std::string& tier_name);
 
    void deactivate_current_if_needed(InferenceBackend* incoming);
 
    void ensure_tier_lora(const std::string& tier_name,
                          InferenceBackend* result);
 
    void unload_or_warm_current(InferenceBackend* current);
 
    std::pair<std::string, std::string> classify_task(
        const std::vector<Message>& messages);
 
    bool deactivate_if_active(llama_context* ctx);
 
    double ensure_adapter_for_tier(
        const std::string& tier_name, llama_context* ctx);
 
    void release_backend(InferenceBackend* backend);
 
    void announce_eviction(const InferenceBackend* backend);
 
    void ensure_secondary_roles();
 
    void preload_adapters_for_model(InferenceBackend* backend);
 
    bool create_tier_backends(const ParsedConfig& config);
 
    void build_routing_tables(const ParsedConfig& config);
 
    bool activate_default_tier(const ParsedConfig& config);
 
    void activate_router(const ParsedConfig& config);
 
    void activate_draft(const ParsedConfig& config);
 
    void load_bundled_grammars();
 
    void resolve_grammar_key(GenerationParams& params,
                             const std::string& tier_name);
 
    std::optional<GenerationResult> refuse_unresolved_tier_grammar(
        GenerationParams& params, const std::string& tier_name);
 
    std::optional<GenerationResult> refuse_unresolved_batch_grammars(
        const std::vector<GenerationParams>& params_list,
        const std::vector<std::string>& tiers,
        const std::string& lead,
        std::vector<GenerationParams>& out);
 
    void stage_batch_arms(InferenceBackend* model,
                          const std::vector<std::string>& tiers,
                          const std::string& lead,
                          std::vector<GenerationParams>& resolved);
 
    void apply_tier_sampler_defaults(GenerationParams& params,
                                     const std::string& tier_name);
 
    GenerationParams resolve_and_stage(InferenceBackend* model,
                                       const GenerationParams& params,
                                       const std::string& tier_name);
 
    std::string resolve_speculative_pair(
        llama_model*& target_out, llama_model*& draft_out) const;
 
    bool try_speculative_route(
        InferenceBackend* model,
        const std::vector<Message>& messages,
        const GenerationParams& params,
        const std::string& tier_name,
        GenerationResult& result);
 
    GenerationResult run_generate_dispatch(
        InferenceBackend* model,
        const std::vector<Message>& messages,
        const GenerationParams& params,
        const std::string& tier_name);
 
    bool try_speculative_route_streaming(
        InferenceBackend* model,
        const std::vector<Message>& messages,
        const GenerationParams& params,
        const std::string& tier_name,
        std::function<void(std::string_view)> on_token,
        std::atomic<bool>& cancel,
        GenerationResult& result);
 
    bool try_mtp_route(
        InferenceBackend* model,
        const std::vector<Message>& messages,
        const GenerationParams& params,
        std::function<void(std::string_view)> on_token,
        std::atomic<bool>& cancel,
        GenerationResult& result);
 
    bool resolve_mtp_effective(const std::string& tier_name) const;
};
 
} // namespace entropic