expert_offload_buft.h

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#pragma once
 
#include "expert_offload.h"
 
#include <llama.h>
 
#include <cstddef>
#include <string>
#include <vector>
 
namespace entropic {
 
class ExpertOffloadOverrides {
public:
    ExpertOffloadOverrides(int cpu_moe_layers,
                           ggml_backend_buffer_type_t cpu_buft)
        : patterns_(expert_offload_patterns(cpu_moe_layers)) {
        if (patterns_.empty()) { return; }
        // +1 for the terminator. Reserved up front so no push_back can
        // reallocate mid-fill; `patterns_` is never modified after the
        // member initializer, so every c_str() below stays valid.
        overrides_.reserve(patterns_.size() + 1);
        for (const std::string& pattern : patterns_) {
            overrides_.push_back({pattern.c_str(), cpu_buft});
        }
        overrides_.push_back({nullptr, nullptr});
    }
 
    ExpertOffloadOverrides(const ExpertOffloadOverrides&) = delete;
    ExpertOffloadOverrides& operator=(const ExpertOffloadOverrides&) = delete;
    ExpertOffloadOverrides(ExpertOffloadOverrides&&) = delete;
    ExpertOffloadOverrides& operator=(ExpertOffloadOverrides&&) = delete;
    ~ExpertOffloadOverrides() = default;
 
    const llama_model_tensor_buft_override* data() const {
        return overrides_.empty() ? nullptr : overrides_.data();
    }
 
    const std::vector<std::string>& patterns() const { return patterns_; }
 
    std::size_t size() const { return patterns_.size(); }
 
private:
    std::vector<std::string> patterns_;                      
    std::vector<llama_model_tensor_buft_override> overrides_; 
};
 
}  // namespace entropic