tokenizers

package

v0.2.5 Latest Latest Go to latest Published: Sep 25, 2025 License: MIT Imports: 3 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/amikos-tech/chroma-go

Links

Open Source Insights

Documentation ¶

Index ¶

func LoadLibrary(path string) error
type EncodeOption
type EncodeResult
type Encoding
type Offset
type Tokenizer
type TokenizerOption
- func WithEncodeSpecialTokens() TokenizerOption
type TruncationDirection

Constants ¶

This section is empty.

Variables ¶

This section is empty.

Functions ¶

func LoadLibrary ¶

func LoadLibrary(path string) error

LoadLibrary loads the Tokenizer shared library at the specified path. This must be called first before using any other functions.

Types ¶

type EncodeOption ¶

type EncodeOption func(eo *encodeOpts)

func WithReturnAllAttributes ¶

func WithReturnAllAttributes() EncodeOption

func WithReturnAttentionMask ¶

func WithReturnAttentionMask() EncodeOption

func WithReturnOffsets ¶

func WithReturnOffsets() EncodeOption

func WithReturnSpecialTokensMask ¶

func WithReturnSpecialTokensMask() EncodeOption

func WithReturnTokens ¶

func WithReturnTokens() EncodeOption

func WithReturnTypeIDs ¶

func WithReturnTypeIDs() EncodeOption

type EncodeResult ¶

type EncodeResult struct {
	// contains filtered or unexported fields
}

EncodeResult represents the C struct returned by the encode function

type Encoding ¶

type Encoding struct {
	IDs               []uint32
	TypeIDs           []uint32
	SpecialTokensMask []uint32
	AttentionMask     []uint32
	Tokens            []string
	Offsets           []Offset
}

type Offset ¶

type Offset [2]uint

type Tokenizer ¶

type Tokenizer struct {
	// contains filtered or unexported fields
}

func FromBytes ¶

func FromBytes(data []byte, opts ...TokenizerOption) (*Tokenizer, error)

func FromBytesWithTruncation ¶

func FromBytesWithTruncation(data []byte, maxLen uint32, dir TruncationDirection) (*Tokenizer, error)

func FromFile ¶

func FromFile(path string) (*Tokenizer, error)

func (*Tokenizer) Close ¶

func (t *Tokenizer) Close() error

func (*Tokenizer) Decode ¶

func (t *Tokenizer) Decode(tokenIDs []uint32, skipSpecialTokens bool) (string, error)

func (*Tokenizer) Encode ¶

func (t *Tokenizer) Encode(str string, addSpecialTokens bool) ([]uint32, []string, error)

func (*Tokenizer) EncodeWithOptions ¶

func (t *Tokenizer) EncodeWithOptions(str string, addSpecialTokens bool, opts ...EncodeOption) (Encoding, error)

func (*Tokenizer) VocabSize ¶

func (t *Tokenizer) VocabSize() (uint32, error)

type TokenizerOption ¶

type TokenizerOption func(to *tokenizerOpts)

func WithEncodeSpecialTokens ¶

func WithEncodeSpecialTokens() TokenizerOption

type TruncationDirection ¶

type TruncationDirection int

const (
	TruncationDirectionLeft TruncationDirection = iota
	TruncationDirectionRight
)

Source Files ¶

View all Source files

tokenizer.go

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL