Documentation
¶
Index ¶
- func LoadLibrary(path string) error
- type EncodeOption
- type Encoding
- type Offset
- type Tokenizer
- func (t *Tokenizer) Close() error
- func (t *Tokenizer) Decode(tokenIDs []uint32, skipSpecialTokens bool) (string, error)
- func (t *Tokenizer) Encode(str string, addSpecialTokens bool) ([]uint32, []string, error)
- func (t *Tokenizer) EncodeWithOptions(str string, addSpecialTokens bool, opts ...EncodeOption) (Encoding, error)
- func (t *Tokenizer) VocabSize() (uint32, error)
- type TokenizerOption
- type TruncationDirection
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func LoadLibrary ¶
LoadLibrary is a no-op for backward compatibility pure-tokenizers handles library loading automatically
Types ¶
type EncodeOption ¶
type EncodeOption func(eo *encodeOpts)
func WithReturnAllAttributes ¶
func WithReturnAllAttributes() EncodeOption
func WithReturnAttentionMask ¶
func WithReturnAttentionMask() EncodeOption
func WithReturnOffsets ¶
func WithReturnOffsets() EncodeOption
func WithReturnSpecialTokensMask ¶
func WithReturnSpecialTokensMask() EncodeOption
func WithReturnTokens ¶
func WithReturnTokens() EncodeOption
func WithReturnTypeIDs ¶
func WithReturnTypeIDs() EncodeOption
type Encoding ¶
type Encoding struct {
IDs []uint32
TypeIDs []uint32
SpecialTokensMask []uint32
AttentionMask []uint32
Tokens []string
Offsets []Offset
}
Encoding represents the result of tokenizing text
type Tokenizer ¶
type Tokenizer struct {
// contains filtered or unexported fields
}
Tokenizer wraps pure-tokenizers with backward-compatible API
func FromBytes ¶
func FromBytes(data []byte, opts ...TokenizerOption) (*Tokenizer, error)
FromBytes creates a tokenizer from byte configuration
func FromBytesWithTruncation ¶
func FromBytesWithTruncation(data []byte, maxLen uint32, dir TruncationDirection) (*Tokenizer, error)
FromBytesWithTruncation creates a tokenizer with truncation settings
func (*Tokenizer) EncodeWithOptions ¶
func (t *Tokenizer) EncodeWithOptions(str string, addSpecialTokens bool, opts ...EncodeOption) (Encoding, error)
EncodeWithOptions tokenizes text with full control over encoding options
type TokenizerOption ¶
type TokenizerOption func(to *tokenizerOpts)
func WithEncodeSpecialTokens ¶
func WithEncodeSpecialTokens() TokenizerOption
type TruncationDirection ¶
type TruncationDirection int
const ( TruncationDirectionLeft TruncationDirection = iota TruncationDirectionRight )
Source Files
Click to show internal directories.
Click to hide internal directories.