tokenizers

Nov 9, 2024

Changes in this version

+ const TokBeginningOfSentence

+ const TokClassification

+ const TokEndOfSentence

+ const TokMask

+ const TokPad

+ const TokSpecialTokensCount

+ const TokUnknown

+ func GetConfig(repo *hub.Repo) (*api.Config, error)

+ func RegisterTokenizerClass(name string, constructor TokenizerConstructor)

+ type Config = api.Config

+ type SpecialToken = api.Tokenizer

+ type Tokenizer = api.Tokenizer

+ func New(repo *hub.Repo) (Tokenizer, error)

+ type TokenizerConstructor func(config *api.Config, repo *hub.Repo) (api.Tokenizer, error)