tokenize

Sep 6, 2026

Changes in this version

type IDTokens

+ StopwordAfter []uint32

+ UnknownAfter []uint32

type Offset

+ func KnownTokenOffsetsAppend(input []byte, unknownAfter []uint32, stopwordAfter []uint32, offsets []Offset) []Offset

+ func TokenOffsetsAppend(input []byte, offsets []Offset) []Offset

type Vocabulary

+ func NewVocabularyFromWordsWithStopwords(words []string, stopwords []uint32) (*Vocabulary, error)

+ func NewVocabularyWithStopwords(texts [][]byte, stopwords []string) (*Vocabulary, error)

+ func (v *Vocabulary) TokenizeIDsAppend(input []byte, ids []ID, unknownAfter []uint32, stopwordAfter []uint32, ...) IDTokens

Aug 14, 2026

Changes in this version

+ type IDTokens struct

+ End int

+ IDs []ID

+ Start int

type Offset

+ func TokenOffsets(input []byte, tokenCount int) []Offset

type Vocabulary

+ func (v *Vocabulary) TokenizeIDs(input []byte) IDTokens

Jul 30, 2026

Changes in this version

+ type ID uint32

+ const Unknown

+ type Offset struct

+ End int

+ Start int

+ type Tokens struct

+ IDs []ID

+ Offsets []Offset

+ type Vocabulary struct

+ func NewVocabulary(texts [][]byte) (*Vocabulary, error)

+ func NewVocabularyFromWords(words []string) (*Vocabulary, error)

+ func (v *Vocabulary) Len() int

+ func (v *Vocabulary) Lookup(word string) (ID, bool)

+ func (v *Vocabulary) Tokenize(input []byte) Tokens

+ func (v *Vocabulary) Word(id ID) string

+ func (v *Vocabulary) Words() []string

+ type Word struct

+ End int

+ Start int

+ Text string

+ func Words(input []byte) []Word