Versions in this module Expand all Collapse all v1 v1.0.0 Sep 20, 2024 Changes in this version + type BpeDecoder struct + func DefaultBpeDecoder() *BpeDecoder + func NewBpeDecoder(suffix string) *BpeDecoder + func (bd *BpeDecoder) DecodeChain(tokens []string) []string + type ByteFallback struct + func NewByteFallback() *ByteFallback + func (d *ByteFallback) DecodeChain(tokens []string) []string + type CTC struct + Cleanup bool + PadToken string + WordDelimiterToken string + func DefaultCTC() *CTC + func NewCTC(padToken string, wordDelimiterToken string, cleanup bool) *CTC + func (d *CTC) DecodeChain(tokens []string) []string + type DecoderBase struct + func (d *DecoderBase) Decode(tokens []string) string + func (d *DecoderBase) DecodeChain(tokens []string) []string + type Fuse struct + func NewFuse() *Fuse + func (f *Fuse) DecodeChain(tokens []string) []string + type Sequence struct + func NewSequence(decoders []tokenizer.Decoder) *Sequence + func (d *Sequence) DecodeChain(tokens []string) []string + type Strip struct + Content string + Start int + Stop int + func NewStrip(content string, start, stop int) *Strip + func (d *Strip) DecodeChain(tokens []string) []string + type WordPieceDecoder struct + func DefaultWordpieceDecoder() *WordPieceDecoder + func NewWordPieceDecoder(prefix string, cleanup bool) *WordPieceDecoder + func (wd *WordPieceDecoder) Cleanup(tok string) string + func (wd *WordPieceDecoder) DecodeChain(tokens []string) []string
Go to main page