expert_offload.h

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#pragma once
 
#include <entropic/types/config.h>
 
#include <string>
#include <vector>
 
namespace entropic {
 
constexpr const char* kExpertTensorRegex =
    "\\.ffn_(up|down|gate|gate_up)_(ch|)exps";
 
inline std::string expert_block_pattern(int layer) {
    return "blk\\." + std::to_string(layer) + kExpertTensorRegex;
}
 
inline std::vector<std::string> expert_offload_patterns(int cpu_moe_layers) {
    std::vector<std::string> patterns;
    if (cpu_moe_layers <= 0) { return patterns; }
    patterns.reserve(static_cast<std::size_t>(cpu_moe_layers));
    for (int i = 0; i < cpu_moe_layers; ++i) {
        patterns.push_back(expert_block_pattern(i));
    }
    return patterns;
}
 
inline std::string expert_offload_conflict_reason(const ModelConfig& cfg) {
    const int n = cfg.cpu_moe_layers;
    std::string reason;
    if (n < 0) {
        reason = "cpu_moe_layers must be >= 0 (0 = off), got "
               + std::to_string(n)
               + ". There is no 'all layers' sentinel — name the layer "
                 "count you want host-side.";
    } else if (n > 0 && cfg.gpu_layers == 0) {
        reason = "cpu_moe_layers=" + std::to_string(n)
               + " with gpu_layers: 0 — the whole model is already on the "
                 "host, so there are no expert tensors on the card to move "
                 "off it. Raise gpu_layers, or drop cpu_moe_layers.";
    } else if (n > 0 && cfg.gpu_layers_auto) {
        // v2.13.1: auto now derives the expert split ITSELF, from the GGUF's
        // real expert-tensor bytes, and prefers moving experts host-side
        // over dropping a layer. So this is no longer "auto cannot model
        // this" — it is "you have asked for both, and only one can win".
        reason = "cpu_moe_layers=" + std::to_string(n)
               + " with gpu_layers: auto — auto derives the expert split "
                 "itself from the model's shape, so setting both asks two "
                 "things to decide one placement. Drop cpu_moe_layers and "
                 "let auto choose, or set gpu_layers to an explicit count "
                 "and keep your own split.";
    }
    return reason;
}
 
inline std::string expert_offload_load_refusal(int cpu_moe_layers,
                                               int n_layer, int n_expert) {
    std::string reason;
    if (cpu_moe_layers <= 0) { return reason; }
    if (n_expert <= 0) {
        reason = "cpu_moe_layers=" + std::to_string(cpu_moe_layers)
               + " was requested, but this model declares no experts "
                 "(<arch>.expert_count is absent or 0). Expert-tensor "
                 "offload has nothing to place; use gpu_layers to move "
                 "whole layers on a dense model.";
    } else if (cpu_moe_layers > n_layer) {
        reason = "cpu_moe_layers=" + std::to_string(cpu_moe_layers)
               + " exceeds the model's " + std::to_string(n_layer)
               + " layers. Lower it to at most " + std::to_string(n_layer)
               + "; it is not clamped, because a count that cannot be "
                 "honoured is a configuration error, not a preference.";
    }
    return reason;
}
 
}  // namespace entropic