entropic::Tokenizer Member List

This is the complete list of members for entropic::Tokenizer, including all inherited members.

count_tokens(const std::string &text) const	entropic::Tokenizer	inlinevirtual
detokenize(int32_t token) const =0	entropic::Tokenizer	pure virtual
tokenize(const std::string &text, bool add_special) const =0	entropic::Tokenizer	pure virtual
~Tokenizer()=default (defined in entropic::Tokenizer)	entropic::Tokenizer	virtual