prompt_cache.h

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#pragma once
 
#include <cstddef>
#include <cstdint>
#include <list>
#include <mutex>
#include <string>
#include <string_view>
#include <unordered_map>
#include <vector>
 
namespace entropic {
 
struct CacheKey {
    uint64_t hash;  
 
    bool operator==(const CacheKey& other) const { return hash == other.hash; }
};
 
struct CacheKeyHash {
    size_t operator()(const CacheKey& key) const { return key.hash; }
};
 
struct CacheEntry {
    std::vector<uint8_t> data;   
    int token_count;             
    size_t data_size;            
};
 
struct CacheStats {
    uint64_t hits = 0;           
    uint64_t misses = 0;         
    uint64_t evictions = 0;      
    uint64_t stores = 0;         
    size_t peak_bytes = 0;       
};
 
class PromptCache {
public:
    explicit PromptCache(size_t max_bytes);
 
    bool store(const CacheKey& key,
               std::vector<uint8_t>&& data,
               int token_count);
 
    const CacheEntry* lookup(const CacheKey& key);
 
    void clear();
 
    size_t bytes_used() const;
 
    size_t entry_count() const;
 
    CacheStats stats() const;
 
    static CacheKey make_key(std::string_view prompt_text,
                             std::string_view model_path);
 
private:
    void evict_until(size_t needed_bytes);
 
    size_t max_bytes_;     
    size_t bytes_used_;    
    CacheStats stats_;     
 
    std::unordered_map<CacheKey, CacheEntry, CacheKeyHash> entries_;
 
    std::list<CacheKey> lru_;
 
    std::unordered_map<CacheKey, std::list<CacheKey>::iterator,
                       CacheKeyHash> lru_map_;
 
    mutable std::mutex mutex_;  
};
 
} // namespace entropic