tokenizer.h

Go to the documentation of this file.

// SPDX-License-Identifier: Apache-2.0
#pragma once
 
#include <cstdint>
#include <string>
#include <vector>
 
namespace entropic {
 
class Tokenizer {
public:
    virtual ~Tokenizer() = default;
 
    virtual std::vector<int32_t> tokenize(
        const std::string& text, bool add_special) const = 0;
 
    virtual std::string detokenize(int32_t token) const = 0;
 
    virtual int count_tokens(const std::string& text) const {
        return static_cast<int>(tokenize(text, false).size());
    }
};
 
} // namespace entropic