secondary_model_loader.h

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#pragma once
 
#include <entropic/inference/backend.h>
#include <entropic/types/config.h>
 
#include <memory>
#include <mutex>
#include <string>
#include <unordered_map>
#include <vector>
 
namespace entropic {
 
class SecondaryModelLoader {
public:
    bool ensure_loaded(const std::string& role, const ModelConfig& config);
 
    InferenceBackend* get(const std::string& role) const;
 
    std::shared_ptr<InferenceBackend> get_shared(
        const std::string& role) const;
 
    bool release_role(const std::string& role);
 
    bool is_loaded(const std::string& role) const;
 
    std::vector<std::string> loaded_roles() const;
 
    void clear_all_prompt_caches();
 
    void shutdown();
 
private:
    mutable std::mutex slots_mutex_;
 
    std::unordered_map<std::string, std::shared_ptr<InferenceBackend>>
        slots_;
 
    std::unordered_map<std::string, std::string> slot_paths_;
};
 
} // namespace entropic