inference_c_api.cpp

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#include <entropic/interfaces/i_inference_backend.h>
#include <entropic/entropic_export.h>
#include <entropic/types/logging.h>
#include <entropic/types/messages_json.h>
 
#include "llama_cpp_backend.h"
 
#include <nlohmann/json.hpp>
 
#include <atomic>
#include <cerrno>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <mutex>
#include <optional>
#include <string>
 
namespace {
 
auto logger = entropic::log::get("inference.c_api");
 
entropic::InferenceBackend* to_backend(entropic_inference_backend_t h) {
    return reinterpret_cast<entropic::InferenceBackend*>(h);
}
 
char* alloc_string(const std::string& s) {
    char* buf = static_cast<char*>(std::malloc(s.size() + 1));
    if (buf) {
        std::memcpy(buf, s.c_str(), s.size() + 1);
    }
    return buf;
}
 
template <typename T>
static void set_if(const nlohmann::json& j, const char* key, T& out) {
    if (j.contains(key)) { out = j[key].get<T>(); }
}
 
entropic::ModelConfig parse_config_json(const char* json_str) {
    entropic::ModelConfig config;
    auto j = nlohmann::json::parse(json_str);
 
    set_if(j, "path", config.path);
    set_if(j, "adapter", config.adapter);
    set_if(j, "context_length", config.context_length);
    set_if(j, "gpu_layers", config.gpu_layers);
    set_if(j, "keep_warm", config.keep_warm);
    set_if(j, "use_mlock", config.use_mlock);
    set_if(j, "n_batch", config.n_batch);
    set_if(j, "n_threads", config.n_threads);
    set_if(j, "flash_attn", config.flash_attn);
 
    return config;
}
 
entropic::GenerationParams parse_params_json(const char* json_str) {
    entropic::GenerationParams params;
    auto j = nlohmann::json::parse(json_str);
 
    if (j.contains("temperature"))    params.temperature = j["temperature"].get<float>();
    if (j.contains("top_p"))          params.top_p = j["top_p"].get<float>();
    if (j.contains("top_k"))          params.top_k = j["top_k"].get<int>();
    if (j.contains("repeat_penalty")) params.repeat_penalty = j["repeat_penalty"].get<float>();
    if (j.contains("max_tokens"))     params.max_tokens = j["max_tokens"].get<int>();
    if (j.contains("grammar"))        params.grammar = j["grammar"].get<std::string>();
 
    return params;
}
 
std::string serialize_result_json(const entropic::GenerationResult& result) {
    nlohmann::json j;
    j["content"] = result.content;
    j["finish_reason"] = result.finish_reason;
    j["token_count"] = result.token_count;
    j["generation_time_ms"] = result.generation_time_ms;
    j["error_code"] = static_cast<int>(result.error_code);
    j["error_message"] = result.error_message;
    return j.dump();
}
 
/* parse_content_part + parse_messages_json moved to the shared
 * utility include/entropic/types/messages_json.h (v2.1.8, gh#37) so
 * the facade's entropic_run_messages can reuse them. Calls below
 * dispatch to entropic::parse_messages_json directly. */
 
} // anonymous namespace
 
// ── C API Implementation ───────────────────────────────────
 
extern "C" {
 
ENTROPIC_EXPORT entropic_error_t entropic_inference_load(
    entropic_inference_backend_t backend,
    const char* config_json)
{
    logger->info("C API: inference_load");
    try {
        auto config = parse_config_json(config_json);
        auto rc = to_backend(backend)->load(config)
            ? ENTROPIC_OK : ENTROPIC_ERROR_LOAD_FAILED;
        logger->info("C API: inference_load -> {}",
                     static_cast<int>(rc));
        return rc;
    } catch (const std::exception& e) {
        logger->error("inference_load exception: {}", e.what());
        return ENTROPIC_ERROR_LOAD_FAILED;
    }
}
 
ENTROPIC_EXPORT entropic_error_t entropic_inference_activate(
    entropic_inference_backend_t backend)
{
    try {
        return to_backend(backend)->activate() ? ENTROPIC_OK : ENTROPIC_ERROR_LOAD_FAILED;
    } catch (const std::exception& e) {
        logger->error("inference_activate exception: {}", e.what());
        return ENTROPIC_ERROR_LOAD_FAILED;
    }
}
 
ENTROPIC_EXPORT entropic_error_t entropic_inference_deactivate(
    entropic_inference_backend_t backend)
{
    try {
        to_backend(backend)->deactivate();
        return ENTROPIC_OK;
    } catch (const std::exception& e) {
        logger->error("inference_deactivate exception: {}", e.what());
        return ENTROPIC_ERROR_INTERNAL;
    }
}
 
ENTROPIC_EXPORT entropic_error_t entropic_inference_unload(
    entropic_inference_backend_t backend)
{
    try {
        to_backend(backend)->unload();
        return ENTROPIC_OK;
    } catch (const std::exception& e) {
        logger->error("inference_unload exception: {}", e.what());
        return ENTROPIC_ERROR_INTERNAL;
    }
}
 
ENTROPIC_EXPORT int entropic_inference_state(
    entropic_inference_backend_t backend)
{
    return static_cast<int>(to_backend(backend)->state());
}
 
ENTROPIC_EXPORT entropic_error_t entropic_inference_generate(
    entropic_inference_backend_t backend,
    const char* messages_json,
    const char* params_json,
    char** result_json)
{
    logger->info("C API: inference_generate");
    try {
        auto msgs = entropic::parse_messages_json(messages_json);
        auto params = parse_params_json(params_json);
        auto result = to_backend(backend)->generate(msgs, params);
        *result_json = alloc_string(serialize_result_json(result));
        logger->info("C API: inference_generate -> {}",
                     result.ok() ? "ok" : "error");
        return result.ok() ? ENTROPIC_OK : result.error_code;
    } catch (const std::exception& e) {
        logger->error("inference_generate exception: {}", e.what());
        return ENTROPIC_ERROR_GENERATE_FAILED;
    }
}
 
ENTROPIC_EXPORT entropic_error_t entropic_inference_generate_streaming(
    entropic_inference_backend_t backend,
    const char* messages_json,
    const char* params_json,
    void (*on_token)(const char* token, size_t len, void* user_data),
    void* user_data,
    int* cancel_flag)
{
    try {
        auto msgs = entropic::parse_messages_json(messages_json);
        auto params = parse_params_json(params_json);
        std::atomic<bool> cancel{false};
 
        auto callback = [on_token, user_data, cancel_flag, &cancel]
            (std::string_view token) {
                on_token(token.data(), token.size(), user_data);
                if (cancel_flag && *cancel_flag) {
                    cancel.store(true, std::memory_order_release);
                }
            };
 
        auto result = to_backend(backend)->generate_streaming(
            msgs, params, callback, cancel);
        return result.ok() ? ENTROPIC_OK : result.error_code;
    } catch (const std::exception& e) {
        logger->error("inference_generate_streaming exception: {}", e.what());
        return ENTROPIC_ERROR_GENERATE_FAILED;
    }
}
 
ENTROPIC_EXPORT entropic_error_t entropic_inference_complete(
    entropic_inference_backend_t backend,
    const char* prompt,
    const char* params_json,
    char** result_json)
{
    try {
        auto params = parse_params_json(params_json);
        auto result = to_backend(backend)->complete(prompt, params);
        *result_json = alloc_string(serialize_result_json(result));
        return result.ok() ? ENTROPIC_OK : result.error_code;
    } catch (const std::exception& e) {
        logger->error("inference_complete exception: {}", e.what());
        return ENTROPIC_ERROR_GENERATE_FAILED;
    }
}
 
ENTROPIC_EXPORT int entropic_inference_count_tokens(
    entropic_inference_backend_t backend,
    const char* text,
    size_t text_len)
{
    try {
        return to_backend(backend)->count_tokens(std::string(text, text_len));
    } catch (...) {
        return static_cast<int>(text_len) / 4;
    }
}
 
ENTROPIC_EXPORT void entropic_inference_destroy(
    entropic_inference_backend_t backend)
{
    delete to_backend(backend);
}
 
ENTROPIC_EXPORT void entropic_inference_free(void* ptr) {
    std::free(ptr);
}
 
ENTROPIC_EXPORT entropic_inference_backend_t entropic_create_inference_backend() {
    return reinterpret_cast<entropic_inference_backend_t>(
        new entropic::LlamaCppBackend());
}
 
ENTROPIC_EXPORT int entropic_plugin_api_version() {
    return 1;
}
 
// ── Log redirect (v2.0.1) ──────────────────────────────────
 
static FILE* s_ggml_log_fp = nullptr;
// llama.cpp's llama_log_set is a single-slot process global. Track
// the active path so a second handle in the same process gets a
// predictable answer: same path → no-op (don't truncate the first
// handle's live log), conflicting path → reject with a warning
// (rather than clobber).
static std::mutex s_ggml_log_mu;
static std::optional<std::string> s_ggml_log_path;
 
static void ggml_log_to_file(enum ggml_log_level /*level*/,
                             const char* text, void* /*ud*/) {
    if (s_ggml_log_fp && text) {
        fputs(text, s_ggml_log_fp);
        fflush(s_ggml_log_fp);
    }
}
 
static void ggml_log_noop(enum ggml_log_level /*level*/,
                          const char* /*text*/, void* /*ud*/) {
}
 
static void ggml_log_silence_locked() {
    if (s_ggml_log_fp) {
        fclose(s_ggml_log_fp);
        s_ggml_log_fp = nullptr;
    }
    s_ggml_log_path.reset();
    llama_log_set(ggml_log_noop, nullptr);
}
 
static std::string canonicalize_or_passthrough(const char* path) {
    std::error_code ec;
    auto canonical = std::filesystem::weakly_canonical(path, ec).string();
    return ec ? std::string(path) : canonical;
}
 
void entropic_inference_log_to_file(const char* path) {
    std::lock_guard lk(s_ggml_log_mu);
 
    if (!path || path[0] == '\0') {
        ggml_log_silence_locked();
        return;
    }
    auto canonical = canonicalize_or_passthrough(path);
 
    // llama_log_set has one process-global slot; first-call wins so a
    // second handle's redirect cannot clobber the first.
    if (s_ggml_log_path && *s_ggml_log_path != canonical) {
        logger->warn(
            "ggml log redirect already wired to {}; ignoring request for {}",
            *s_ggml_log_path, canonical);
        return;
    }
 
    FILE* fp = fopen(path, "w");
    if (!fp) {
        logger->warn("ggml log fopen failed for {}: {}",
                     path, std::strerror(errno));
        return;
    }
    if (s_ggml_log_fp) { fclose(s_ggml_log_fp); }
    s_ggml_log_fp = fp;
    s_ggml_log_path = canonical;
    llama_log_set(ggml_log_to_file, nullptr);
}
 
void entropic_inference_log_silence(void) {
    std::lock_guard lk(s_ggml_log_mu);
    ggml_log_silence_locked();
}
 
} // extern "C"