warm_keep_util.h

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#ifndef ENTROPIC_INFERENCE_WARM_KEEP_UTIL_H
#define ENTROPIC_INFERENCE_WARM_KEEP_UTIL_H
 
#include <algorithm>
#include <cstddef>
#include <vector>
 
namespace entropic {
 
template <typename Tok>
inline std::size_t common_prefix_len(const std::vector<Tok>& a,
                                     const std::vector<Tok>& b) {
    std::size_t n = std::min(a.size(), b.size());
    std::size_t i = 0;
    while (i < n && a[i] == b[i]) {
        ++i;
    }
    return i;
}
 
template <typename Tok>
inline std::size_t warm_keep_cut(const std::vector<Tok>& resident,
                                 const std::vector<Tok>& incoming,
                                 long kv_pos_max) {
    if (resident.empty() || incoming.size() < 2 || kv_pos_max < 0) {
        return 0;
    }
    std::size_t common = common_prefix_len(resident, incoming);
    if (common == 0) {
        return 0;
    }
    if (static_cast<std::size_t>(kv_pos_max) + 1 < common) {
        return 0;  // KV does not actually hold the matched prefix
    }
    if (common >= incoming.size()) {
        common = incoming.size() - 1;  // always re-decode the last token
    }
    return common;
}
 
}  // namespace entropic
 
#endif  // ENTROPIC_INFERENCE_WARM_KEEP_UTIL_H