tokenizers

Mar 10, 2026

Changes in this version

+ var ErrTokenizerClosed = errors.New("tokenizer is nil or closed")

type Tokenizer

+ func (t *Tokenizer) DecodeErr(tokenIDs []uint32, skipSpecialTokens bool) (string, error)

+ func (t *Tokenizer) EncodeErr(str string, addSpecialTokens bool) ([]uint32, []string, error)

+ func (t *Tokenizer) EncodeWithOptionsErr(str string, addSpecialTokens bool, opts ...EncodeOption) (Encoding, error)

Jul 15, 2025

Changes in this version

type Tokenizer

+ func FromTiktoken(modelPath, configPath, pattern string) (*Tokenizer, error)

Nov 7, 2024

Changes in this version

+ const WANT_VERSION

type Tokenizer

+ func FromPretrained(modelID string, opts ...TokenizerConfigOption) (*Tokenizer, error)

+ type TokenizerConfigOption func(cfg *tokenizerConfig)

+ func WithAuthToken(token string) TokenizerConfigOption

+ func WithCacheDir(path string) TokenizerConfigOption

Aug 9, 2024

Changes in this version

type EncodeOption

+ func WithReturnOffsets() EncodeOption

type Encoding

+ Offsets []Offset

+ type Offset [2]uint

Apr 10, 2024

Changes in this version

+ type TokenizerOption func(to *tokenizerOpts)

+ func WithEncodeSpecialTokens() TokenizerOption

Jan 7, 2024

Changes in this version

+ type EncodeOption func(eo *EncodeOptions)

+ func WithReturnAllAttributes() EncodeOption

+ func WithReturnAttentionMask() EncodeOption

+ func WithReturnSpecialTokensMask() EncodeOption

+ func WithReturnTokens() EncodeOption

+ func WithReturnTypeIDs() EncodeOption

+ type EncodeOptions struct

+ AddSpecialTokens C.bool

+ ReturnAttentionMask C.bool

+ ReturnSpecialTokensMask C.bool

+ ReturnTokens C.bool

+ ReturnTypeIDs C.bool

+ type Encoding struct

+ AttentionMask []uint32

+ IDs []uint32

+ SpecialTokensMask []uint32

+ Tokens []string

+ TypeIDs []uint32

type Tokenizer

+ func (t *Tokenizer) EncodeWithOptions(str string, addSpecialTokens bool, opts ...EncodeOption) Encoding

May 4, 2023

Changes in this version

type Tokenizer

+ func FromBytesWithTruncation(data []byte, maxLen uint32, dir TruncationDirection) (*Tokenizer, error)

+ type TruncationDirection int

+ const TruncationDirectionLeft

+ const TruncationDirectionRight

Apr 3, 2023

Changes in this version

type Tokenizer

+ func FromBytes(data []byte) (*Tokenizer, error)

Apr 1, 2023

Changes in this version

+ type Tokenizer struct

+ func FromFile(path string) (*Tokenizer, error)

+ func (t *Tokenizer) Close() error

+ func (t *Tokenizer) Decode(tokenIDs []uint32, skipSpecialTokens bool) string

+ func (t *Tokenizer) Encode(str string, addSpecialTokens bool) []uint32

+ func (t *Tokenizer) VocabSize() uint32