interface_factory.cpp

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#include <entropic/inference/interface_factory.h>
#include <entropic/inference/orchestrator.h>
#include <entropic/types/config.h>
#include <entropic/types/message.h>
#include <entropic/inference/adapters/adapter_base.h>  // gh#88 recovery
 
#include "llama_cpp_backend.h"  // gh#87 3b: common_chat parse routing
#include "tool_call_serialize.h"  // gh#93: shared (typed) tool-call serialization
 
#include <nlohmann/json.hpp>
 
#include <atomic>
#include <chrono>
#include <cstdlib>
#include <cstring>
#include <string>
#include <thread>
#include <vector>
 
namespace entropic {
 
// ── Context struct for callbacks ───────────────────────────
 
struct InterfaceContext {
    ModelOrchestrator* orchestrator; 
    std::string default_tier;        
};
 
// ── JSON helpers ───────────────────────────────────────────
 
static std::vector<Message> parse_msgs(const char* json_str) {
    std::vector<Message> msgs;
    if (!json_str) { return msgs; }
    auto arr = nlohmann::json::parse(json_str, nullptr, false);
    if (!arr.is_array()) { return msgs; }
    for (const auto& obj : arr) {
        Message m;
        m.role = obj.value("role", "");
        m.content = obj.value("content", "");
        msgs.push_back(std::move(m));
    }
    return msgs;
}
 
template <typename T>
static void assign_if_present(const nlohmann::json& j,
                              const char* key, T& dst) {
    if (j.contains(key)) { dst = j[key].get<T>(); }
}
 
static void parse_logit_bias_into(
    const nlohmann::json& j,
    std::unordered_map<int32_t, float>& dst)
{
    if (!j.contains("logit_bias") || !j["logit_bias"].is_object()) {
        return;
    }
    for (auto it = j["logit_bias"].begin(); it != j["logit_bias"].end(); ++it) {
        try {
            dst[std::stoi(it.key())] = it.value().get<float>();
        } catch (const std::exception&) {
            // skip un-parseable keys
        }
    }
}
 
static GenerationParams parse_params(const char* json_str) {
    GenerationParams p;
    if (!json_str) { return p; }
    auto j = nlohmann::json::parse(json_str, nullptr, false);
    if (!j.is_object()) { return p; }
    assign_if_present(j, "max_tokens",       p.max_tokens);
    assign_if_present(j, "temperature",      p.temperature);
    assign_if_present(j, "grammar_key",      p.grammar_key);
    assign_if_present(j, "enable_thinking",  p.enable_thinking);
    assign_if_present(j, "top_p",            p.top_p);
    assign_if_present(j, "top_k",            p.top_k);
    assign_if_present(j, "min_p",            p.min_p);
    assign_if_present(j, "presence_penalty", p.presence_penalty);
    assign_if_present(j, "frequency_penalty",p.frequency_penalty);
    assign_if_present(j, "repeat_penalty",   p.repeat_penalty);
    assign_if_present(j, "seed",             p.seed);
    assign_if_present(j, "tools",            p.tools);   // gh#87 3b
    parse_logit_bias_into(j, p.logit_bias);
    return p;
}
 
static std::string extract_tier(const char* json_str,
                                const std::string& default_tier) {
    if (!json_str) { return default_tier; }
    auto j = nlohmann::json::parse(json_str, nullptr, false);
    if (j.is_object() && j.contains("tier")) {
        return j["tier"].get<std::string>();
    }
    return default_tier;
}
 
static char* dup(const std::string& s) {
    return strdup(s.c_str());
}
 
// ── C-callable wrappers ────────────────────────────────────
 
static int iface_generate(const char* msgs_json,
                          const char* params_json,
                          char** result_json,
                          void* user_data) {
    auto* ctx = static_cast<InterfaceContext*>(user_data);
    auto messages = parse_msgs(msgs_json);
    auto params = parse_params(params_json);
    auto tier = extract_tier(params_json, ctx->default_tier);
    auto result = ctx->orchestrator->generate(
        messages, params, tier);
    auto& out = result.raw_content.empty()
        ? result.content : result.raw_content;
    *result_json = dup(out);
    return 0;
}
 
static int iface_generate_stream(
    const char* msgs_json, const char* params_json,
    void (*on_token)(const char*, size_t, void*),
    void* token_ud, int* cancel, void* user_data) {
    auto* ctx = static_cast<InterfaceContext*>(user_data);
    auto messages = parse_msgs(msgs_json);
    auto params = parse_params(params_json);
    std::atomic<bool> cancel_flag(false);
    auto cb = [on_token, token_ud, cancel, &cancel_flag]
              (std::string_view tok) {
        on_token(tok.data(), tok.size(), token_ud);
        if (cancel != nullptr && *cancel != 0) {
            cancel_flag.store(true, std::memory_order_release);
        }
    };
    auto tier = extract_tier(params_json, ctx->default_tier);
    ctx->orchestrator->generate_streaming(
        messages, params, cb, cancel_flag, tier);
    return 0;
}
 
static int iface_generate_with_cancel(
    const char* msgs_json, const char* params_json,
    char** result_json, int* cancel, void* user_data) {
    auto* ctx = static_cast<InterfaceContext*>(user_data);
    auto messages = parse_msgs(msgs_json);
    auto params = parse_params(params_json);
    auto tier = extract_tier(params_json, ctx->default_tier);
 
    std::atomic<bool> cancel_flag(false);
    std::atomic<bool> done(false);
    std::thread poller;
    if (cancel != nullptr) {
        poller = std::thread([cancel, &cancel_flag, &done]() {
            while (!done.load(std::memory_order_acquire)) {
                if (*cancel != 0) {
                    cancel_flag.store(true, std::memory_order_release);
                    return;
                }
                std::this_thread::sleep_for(
                    std::chrono::milliseconds(10));
            }
        });
    }
 
    auto result = ctx->orchestrator->generate(
        messages, params, cancel_flag, tier);
 
    done.store(true, std::memory_order_release);
    if (poller.joinable()) { poller.join(); }
 
    auto& out = result.raw_content.empty()
        ? result.content : result.raw_content;
    *result_json = dup(out);
    return 0;
}
 
static int iface_route(const char* msgs_json,
                       char** result_json, void* user_data) {
    auto* ctx = static_cast<InterfaceContext*>(user_data);
    auto messages = parse_msgs(msgs_json);
    auto tier = ctx->orchestrator->route(messages);
    *result_json = dup(tier);
    return 0;
}
 
static int iface_complete(const char* prompt,
                          const char* params_json,
                          char** result_json, void* user_data) {
    auto* ctx = static_cast<InterfaceContext*>(user_data);
    auto tier = extract_tier(params_json, ctx->default_tier);
    Message msg;
    msg.role = "user";
    msg.content = prompt;
    GenerationParams params{};
    params.max_tokens = 1;
    auto result = ctx->orchestrator->generate(
        {msg}, params, tier);
    *result_json = dup(result.content);
    return 0;
}
 
static int iface_parse_tool_calls(const char* raw,
                                  char** cleaned,
                                  char** tool_calls_json,
                                  void* user_data) {
    auto* ctx = static_cast<InterfaceContext*>(user_data);
    std::string raw_str = raw ? raw : "";
 
    auto tier = ctx->orchestrator->last_used_tier();
    if (tier.empty()) { tier = ctx->default_tier; }
    auto* llama = dynamic_cast<LlamaCppBackend*>(
        ctx->orchestrator->get_backend(tier));
 
    if (llama != nullptr && llama->common_chat_parse_reliable()) {
        auto parsed = llama->parse_response(raw_str);
        apply_action_envelope_recovery(parsed.tool_calls, raw_str);  // gh#88
        *cleaned = dup(parsed.content);
        *tool_calls_json = dup(serialize_tool_calls(parsed.tool_calls));
        return 0;
    }
 
    // gh#89: fall back on the SAME routed tier as the reliable branch above
    // (last_used_tier), not default_tier — a routed non-default autoparser tier
    // was parsing with the wrong tier's adapter.
    auto* adapter = ctx->orchestrator->get_adapter(tier);
    if (adapter == nullptr) {
        *cleaned = dup(raw_str);
        *tool_calls_json = dup("[]");
        return 0;
    }
    auto parsed = adapter->parse_tool_calls(raw_str);
    *cleaned = dup(parsed.cleaned_content);
    *tool_calls_json = dup(serialize_tool_calls(parsed.tool_calls));
    return 0;
}
 
static int iface_is_complete(const char* /*content*/,
                             const char* tool_calls_json,
                             void* /*user_data*/) {
    if (!tool_calls_json) { return 1; }
    auto tc = nlohmann::json::parse(tool_calls_json, nullptr, false);
    return (tc.is_array() && !tc.empty()) ? 0 : 1;
}
 
// ── Factory ────────────────────────────────────────────────
 
InferenceInterface build_orchestrator_interface(
    ModelOrchestrator* orchestrator,
    const std::string& default_tier,
    InterfaceContext** out_context) {
    auto* ctx = new InterfaceContext{orchestrator, default_tier};
    if (out_context) { *out_context = ctx; }
 
    InferenceInterface iface;
    iface.generate = iface_generate;
    iface.generate_cancellable = iface_generate_with_cancel;  // gh#81, v2.4.2
    iface.generate_stream = iface_generate_stream;
    iface.route = iface_route;
    iface.complete = iface_complete;
    iface.parse_tool_calls = iface_parse_tool_calls;
    iface.is_response_complete = iface_is_complete;
    iface.free_fn = free;
    iface.backend_data = ctx;
    iface.orchestrator_data = ctx;
    iface.adapter_data = ctx;
    return iface;
}
 
void destroy_orchestrator_interface(InterfaceContext* context) {
    delete context;
}
 
} // namespace entropic