interface_factory.cpp

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#include <entropic/inference/interface_factory.h>
#include <entropic/inference/orchestrator.h>
#include <entropic/types/config.h>
#include <entropic/types/message.h>
#include <entropic/inference/adapters/adapter_base.h>  // gh#88 recovery
 
#include "llama_cpp_backend.h"
#include "response_parse.h"  // gh#87 3b: common_chat parse routing
#include "tool_call_serialize.h"  // gh#93: shared (typed) tool-call serialization
 
#include <nlohmann/json.hpp>
 
#include <atomic>
#include <chrono>
#include <cstdlib>
#include <cstring>
#include <string>
#include <thread>
#include <vector>
 
namespace entropic {
 
// ── Context struct for callbacks ───────────────────────────
 
struct InterfaceContext {
    ModelOrchestrator* orchestrator; 
    std::string default_tier;        
};
 
// ── JSON helpers ───────────────────────────────────────────
 
static std::vector<Message> parse_msgs(const char* json_str) {
    std::vector<Message> msgs;
    if (!json_str) { return msgs; }
    auto arr = nlohmann::json::parse(json_str, nullptr, false);
    if (!arr.is_array()) { return msgs; }
    for (const auto& obj : arr) {
        Message m;
        m.role = obj.value("role", "");
        m.content = obj.value("content", "");
        msgs.push_back(std::move(m));
    }
    return msgs;
}
 
template <typename T>
static void assign_if_present(const nlohmann::json& j,
                              const char* key, T& dst) {
    if (j.contains(key)) { dst = j[key].get<T>(); }
}
 
static void parse_logit_bias_into(
    const nlohmann::json& j,
    std::unordered_map<int32_t, float>& dst)
{
    if (!j.contains("logit_bias") || !j["logit_bias"].is_object()) {
        return;
    }
    for (auto it = j["logit_bias"].begin(); it != j["logit_bias"].end(); ++it) {
        try {
            dst[std::stoi(it.key())] = it.value().get<float>();
        } catch (const std::exception&) {
            // skip un-parseable keys
        }
    }
}
 
static GenerationParams parse_params(const char* json_str) {
    GenerationParams p;
    if (!json_str) { return p; }
    auto j = nlohmann::json::parse(json_str, nullptr, false);
    if (!j.is_object()) { return p; }
    assign_if_present(j, "max_tokens",       p.max_tokens);
    assign_if_present(j, "temperature",      p.temperature);
    assign_if_present(j, "grammar_key",      p.grammar_key);
    // gh#144 (v2.12.0): caller-scoped session; absent means the default.
    assign_if_present(j, "session",          p.session_key);
    assign_if_present(j, "enable_thinking",  p.enable_thinking);
    assign_if_present(j, "top_p",            p.top_p);
    assign_if_present(j, "top_k",            p.top_k);
    assign_if_present(j, "min_p",            p.min_p);
    assign_if_present(j, "presence_penalty", p.presence_penalty);
    assign_if_present(j, "frequency_penalty",p.frequency_penalty);
    assign_if_present(j, "repeat_penalty",   p.repeat_penalty);
    assign_if_present(j, "seed",             p.seed);
    assign_if_present(j, "tools",            p.tools);   // gh#87 3b
    parse_logit_bias_into(j, p.logit_bias);
    return p;
}
 
static std::string extract_tier(const char* json_str,
                                const std::string& default_tier) {
    if (!json_str) { return default_tier; }
    auto j = nlohmann::json::parse(json_str, nullptr, false);
    if (j.is_object() && j.contains("tier")) {
        return j["tier"].get<std::string>();
    }
    return default_tier;
}
 
static char* dup(const std::string& s) {
    return strdup(s.c_str());
}
 
// ── C-callable wrappers ────────────────────────────────────
 
static int iface_generate(const char* msgs_json,
                          const char* params_json,
                          char** result_json,
                          void* user_data) {
    auto* ctx = static_cast<InterfaceContext*>(user_data);
    auto messages = parse_msgs(msgs_json);
    auto params = parse_params(params_json);
    auto tier = extract_tier(params_json, ctx->default_tier);
    auto result = ctx->orchestrator->generate(
        messages, params, tier);
    auto& out = result.raw_content.empty()
        ? result.content : result.raw_content;
    *result_json = dup(out);
    return 0;
}
 
static int iface_generate_stream(
    const char* msgs_json, const char* params_json,
    void (*on_token)(const char*, size_t, void*),
    void* token_ud, int* cancel, void* user_data) {
    auto* ctx = static_cast<InterfaceContext*>(user_data);
    auto messages = parse_msgs(msgs_json);
    auto params = parse_params(params_json);
    std::atomic<bool> cancel_flag(false);
    auto cb = [on_token, token_ud, cancel, &cancel_flag]
              (std::string_view tok) {
        on_token(tok.data(), tok.size(), token_ud);
        if (cancel != nullptr && *cancel != 0) {
            cancel_flag.store(true, std::memory_order_release);
        }
    };
    auto tier = extract_tier(params_json, ctx->default_tier);
    ctx->orchestrator->generate_streaming(
        messages, params, cb, cancel_flag, tier);
    return 0;
}
 
static int iface_generate_with_cancel(
    const char* msgs_json, const char* params_json,
    char** result_json, int* cancel, void* user_data) {
    auto* ctx = static_cast<InterfaceContext*>(user_data);
    auto messages = parse_msgs(msgs_json);
    auto params = parse_params(params_json);
    auto tier = extract_tier(params_json, ctx->default_tier);
 
    std::atomic<bool> cancel_flag(false);
    std::atomic<bool> done(false);
    std::thread poller;
    if (cancel != nullptr) {
        poller = std::thread([cancel, &cancel_flag, &done]() {
            while (!done.load(std::memory_order_acquire)) {
                if (*cancel != 0) {
                    cancel_flag.store(true, std::memory_order_release);
                    return;
                }
                std::this_thread::sleep_for(
                    std::chrono::milliseconds(10));
            }
        });
    }
 
    auto result = ctx->orchestrator->generate(
        messages, params, cancel_flag, tier);
 
    done.store(true, std::memory_order_release);
    if (poller.joinable()) { poller.join(); }
 
    auto& out = result.raw_content.empty()
        ? result.content : result.raw_content;
    *result_json = dup(out);
    return 0;
}
 
static int iface_route(const char* msgs_json,
                       char** result_json, void* user_data) {
    auto* ctx = static_cast<InterfaceContext*>(user_data);
    auto messages = parse_msgs(msgs_json);
    auto tier = ctx->orchestrator->route(messages);
    *result_json = dup(tier);
    return 0;
}
 
static int iface_complete(const char* prompt,
                          const char* params_json,
                          char** result_json, void* user_data) {
    auto* ctx = static_cast<InterfaceContext*>(user_data);
    auto tier = extract_tier(params_json, ctx->default_tier);
    Message msg;
    msg.role = "user";
    msg.content = prompt;
    GenerationParams params{};
    params.max_tokens = 1;
    auto result = ctx->orchestrator->generate(
        {msg}, params, tier);
    *result_json = dup(result.content);
    return 0;
}
 
static std::vector<ToolCall> tc_from_json_obj(const nlohmann::json& j) {
    // j.value() returns default when key absent or wrong type — fewer branches.
    std::string name = j.value("name", std::string{});
    if (name.empty()) { name = j.value("tool", std::string{}); }
 
    auto it = j.find("arguments");
    if (it == j.end() || !it->is_object()) { it = j.find("parameters"); }
    const nlohmann::json* args =
        (it != j.end() && it->is_object()) ? &(*it) : nullptr;
 
    std::vector<ToolCall> result;
    if (!name.empty() && args != nullptr) {
        ToolCall tc;
        tc.name = std::move(name);
        for (const auto& [k, v] : args->items()) {
            tc.arguments[k] = v.is_string() ? v.get<std::string>() : v.dump();
        }
        result.push_back(std::move(tc));
    }
    return result;
}
 
static std::vector<ToolCall> try_fenced_json_call(const std::string& raw) {
    const std::string kOpen = "```json";
    const std::string kClose = "```";
    auto ob = raw.find(kOpen);
    auto bs = (ob != std::string::npos) ? (ob + kOpen.size()) : std::string::npos;
    auto cb = (bs != std::string::npos) ? raw.find(kClose, bs) : std::string::npos;
    bool single = (ob != std::string::npos) && (cb != std::string::npos)
        && (raw.find(kOpen, ob + 1) == std::string::npos
            || raw.find(kOpen, ob + 1) > cb);
    if (!single) { return {}; }
    auto j = nlohmann::json::parse(raw.substr(bs, cb - bs), nullptr, false);
    if (!j.is_object()) { return {}; }
    return tc_from_json_obj(j);
}
 
static std::string find_unique_args_match(
    const nlohmann::json& obj,
    const nlohmann::json& tools) {
    std::string matched;
    for (const auto& t : tools) {
        auto name = t.value("name", std::string{});
        if (name.empty()) { continue; }
        auto req = t.value("inputSchema", nlohmann::json{})
                     .value("required", nlohmann::json::array());
        bool ok = true;
        for (const auto& r : req) {
            if (!r.is_string()
                || !obj.contains(r.get<std::string>())) {
                ok = false; break;
            }
        }
        if (!ok) { continue; }
        if (!matched.empty()) { return {}; }
        matched = name;
    }
    return matched;
}
 
static std::optional<nlohmann::json> extract_args_only_fence(
    const std::string& raw) {
    const std::string kOpen = "```json";
    const std::string kClose = "```";
    auto ob = raw.find(kOpen);
    auto bs = ob != std::string::npos ? ob + kOpen.size() : std::string::npos;
    auto cb = bs != std::string::npos ? raw.find(kClose, bs) : std::string::npos;
    bool single = ob != std::string::npos && cb != std::string::npos
        && (raw.find(kOpen, ob + 1) == std::string::npos
            || raw.find(kOpen, ob + 1) > cb);
    if (!single) { return std::nullopt; }
    auto j = nlohmann::json::parse(raw.substr(bs, cb - bs), nullptr, false);
    if (!j.is_object() || j.contains("name") || j.contains("tool")) {
        return std::nullopt;
    }
    return j;
}
 
static std::vector<ToolCall> try_fenced_args_object_call(
    const std::string& raw,
    const std::string& tools_json) {
    auto j_opt = extract_args_only_fence(raw);
    if (!j_opt) { return {}; }
    auto tools = nlohmann::json::parse(tools_json, nullptr, false);
    auto name = tools.is_array()
        ? find_unique_args_match(*j_opt, tools) : std::string{};
    if (name.empty()) { return {}; }
    ToolCall tc;
    tc.name = std::move(name);
    for (const auto& [k, v] : j_opt->items()) {
        tc.arguments[k] = v.is_string() ? v.get<std::string>() : v.dump();
    }
    return {std::move(tc)};
}
 
static void apply_fenced_fallbacks(const std::string& raw,
                                   LlamaCppBackend* llama,
                                   std::vector<ToolCall>& calls) {
    if (!calls.empty()) { return; }
    auto fb = try_fenced_json_call(raw);
    if (fb.empty() && llama != nullptr) {
        fb = try_fenced_args_object_call(raw, llama->active_tools_json());
    }
    if (!fb.empty()) { calls = std::move(fb); }
}
 
static void parse_via_adapter(ModelOrchestrator* orch,
                              const std::string& raw,
                              const std::string& tier,
                              std::string& out_cleaned,
                              std::vector<ToolCall>& out_calls) {
    auto* adapter = orch->get_adapter(tier);
    if (adapter == nullptr) { out_cleaned = raw; return; }
    auto p = adapter->parse_tool_calls(raw);
    out_cleaned = std::move(p.cleaned_content);
    out_calls = std::move(p.tool_calls);
}
 
static int iface_parse_tool_calls(const char* raw,
                                  char** cleaned,
                                  char** tool_calls_json,
                                  void* user_data) {
    auto* ctx = static_cast<InterfaceContext*>(user_data);
    std::string raw_str = raw ? raw : "";
    auto tier = ctx->orchestrator->last_used_tier();
    if (tier.empty()) { tier = ctx->default_tier; }
    auto* llama = dynamic_cast<LlamaCppBackend*>(
        ctx->orchestrator->get_backend(tier));
 
    // gh#108 (v2.10.3): same rule as the orchestrator's buffered parse —
    // this used to be a byte-for-byte duplicate of that branch, which is how
    // a fix could land on one path and miss the other.
    auto parsed = parse_model_response(
        llama, ctx->orchestrator->get_adapter(tier), raw_str);
    std::string cleaned_str = std::move(parsed.content);
    std::vector<ToolCall> calls = std::move(parsed.tool_calls);
 
    apply_fenced_fallbacks(raw_str, llama, calls);  // gh#122, gh#127
 
    *cleaned = dup(cleaned_str);
    *tool_calls_json = dup(serialize_tool_calls(calls));
    return 0;
}
 
static int iface_is_complete(const char* /*content*/,
                             const char* tool_calls_json,
                             void* /*user_data*/) {
    if (!tool_calls_json) { return 1; }
    auto tc = nlohmann::json::parse(tool_calls_json, nullptr, false);
    return (tc.is_array() && !tc.empty()) ? 0 : 1;
}
 
// ── Factory ────────────────────────────────────────────────
 
InferenceInterface build_orchestrator_interface(
    ModelOrchestrator* orchestrator,
    const std::string& default_tier,
    InterfaceContext** out_context) {
    auto* ctx = new InterfaceContext{orchestrator, default_tier};
    if (out_context) { *out_context = ctx; }
 
    InferenceInterface iface;
    iface.generate = iface_generate;
    iface.generate_cancellable = iface_generate_with_cancel;  // gh#81, v2.4.2
    iface.generate_stream = iface_generate_stream;
    iface.route = iface_route;
    iface.complete = iface_complete;
    iface.parse_tool_calls = iface_parse_tool_calls;
    iface.is_response_complete = iface_is_complete;
    iface.free_fn = free;
    iface.backend_data = ctx;
    iface.orchestrator_data = ctx;
    iface.adapter_data = ctx;
    return iface;
}
 
void destroy_orchestrator_interface(InterfaceContext* context) {
    delete context;
}
 
} // namespace entropic