Versions in this module Expand all Collapse all v1 v1.0.0 Sep 20, 2024 Changes in this version + type BertProcessing struct + func NewBertProcessing(sep, cls PostToken) (retVal *BertProcessing) + func (bp *BertProcessing) AddedTokens(isPair bool) (retVal int) + func (bp *BertProcessing) Process(encoding, pairEncoding *tokenizer.Encoding, addSpecialTokens bool) (retVal *tokenizer.Encoding) + type ByteLevelProcessing struct + func NewByteLevelProcessing(pretok *pretokenizer.ByteLevel) (retVal *ByteLevelProcessing) + func (blp *ByteLevelProcessing) AddedTokens(isPair bool) (retVal int) + func (blp *ByteLevelProcessing) Process(encoding, pairEncoding *tokenizer.Encoding, addSpecialTokens bool) (retVal *tokenizer.Encoding) + type Piece interface + WithTypeId func(typeId int) + func NewPiece(s string) (Piece, error) + type PostToken struct + Id int + Value string + type RobertaProcessing struct + func DefaultRobertaProcessing() *RobertaProcessing + func NewRobertaProcessing(sep, cls PostToken, trimOffsets bool, addPrefixSpace bool) *RobertaProcessing + func (rp *RobertaProcessing) AddPrefixSpace(addPrefixSpace bool) + func (rp *RobertaProcessing) AddedTokens(isPair bool) int + func (rp *RobertaProcessing) Process(encoding, pairEncoding *tokenizer.Encoding, addSpecialTokens bool) *tokenizer.Encoding + func (rp *RobertaProcessing) TrimOffsets(trimOffsets bool) + type Sequence struct + func NewSequence(processors []tokenizer.PostProcessor) *Sequence + func (seq *Sequence) AddedTokens(isPair bool) (retVal int) + func (seq *Sequence) Process(encoding, pairEncoding *tokenizer.Encoding, addSpecialTokens bool) (retVal *tokenizer.Encoding) + type SequenceEnum int + const A + const B + type SequencePiece struct + Id SequenceEnum + TypeId int + func NewSequencePiece(id string, typeId int) *SequencePiece + func (p *SequencePiece) WithTypeId(v int) + type SpecialToken struct + Id string + Ids []int + Tokens []string + func NewSpecialToken(id string, ids []int, tokens []string) *SpecialToken + func NewSpecialTokenFrom(s string, id int) *SpecialToken + type SpecialTokenPiece struct + Id string + TypeId int + func NewSpecialTokenPiece(id string, typeId int) *SpecialTokenPiece + func (p *SpecialTokenPiece) WithTypeId(v int) + type Template []Piece + func NewTemplate(v interface{}) (Template, error) + func NewTemplateFromMulti(parts []string) (Template, error) + func NewTemplateFromOne(s string) (Template, error) + type TemplateProcessing struct + AddedPair int + AddedSingle int + Pair Template + Single Template + SpecialTokens *Tokens + func DefaultTemplateProcessing() *TemplateProcessing + func NewTemplateProcessing(single, pair Template, specialTokens *Tokens) *TemplateProcessing + func NewTemplateProcessingFrom(t *TemplateProcessingDeserializer) *TemplateProcessing + func (tp *TemplateProcessing) AddedTokens(isPair bool) int + func (tp *TemplateProcessing) ApplyTemplate(template []Piece, encodings []tokenizer.Encoding, addSpecialTokens bool) []tokenizer.Encoding + func (tp *TemplateProcessing) Builder() *TemplateProcessingBuilder + func (tp *TemplateProcessing) Process(encoding, pairEncoding *tokenizer.Encoding, addSpecialTokens bool) *tokenizer.Encoding + type TemplateProcessingBuilder struct + func (b *TemplateProcessingBuilder) DefaultAdded(isSingle bool) int + func (b *TemplateProcessingBuilder) NewPair(v interface{}) + func (b *TemplateProcessingBuilder) NewSingle(v interface{}) + func (b *TemplateProcessingBuilder) NewSpecialTokens(tokens []tokenizer.Token) + func (b *TemplateProcessingBuilder) Validate() error + func (tp *TemplateProcessingBuilder) Build() *TemplateProcessing + type TemplateProcessingDeserializer struct + Pair Template + Single Template + SpecialTokens *Tokens + type Tokens struct + TokenMap map[string]SpecialToken + func DefaultTokens() *Tokens + func NewTokens(toks []tokenizer.Token) *Tokens + func NewTokensFrom(toks []SpecialToken) *Tokens + func NewTokensFromMap(m map[string]SpecialToken) *Tokens + func (t *Tokens) GetItemByKey(id string) (SpecialToken, bool) + func (t *Tokens) GetItemByOrder(index int) (SpecialToken, bool)
Go to main page