throughput_tracker.h

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#pragma once
 
#include <atomic>
#include <cstdint>
#include <mutex>
 
namespace entropic {
 
class ThroughputTracker {
public:
    void record(int tokens_generated, int64_t elapsed_ms);
 
    double tok_per_sec() const;
 
    int64_t predict_ms(int token_count) const;
 
    int recommend_tokens(int64_t time_budget_ms,
                         float headroom = 0.9f,
                         int floor = 64) const;
 
    int sample_count() const;
 
    void reset();
 
private:
    static constexpr double kAlpha = 0.3;
 
    static constexpr int kMinTokens = 4;
 
    std::atomic<double> ewma_tok_s_{0.0};  
    std::atomic<int> samples_{0};          
    std::mutex mutex_;                     
};
 
} // namespace entropic