processor

package

v0.1.7 Latest Latest Go to latest Published: Oct 16, 2020 License: Apache-2.0 Imports: 2 Imported by: 20

Details

This section is empty.

This section is empty.

This section is empty.

type BertProcessing struct {
	// contains filtered or unexported fields
}

func NewBertProcessing(sep, cls PostToken) (retVal *BertProcessing)

func (bp *BertProcessing) AddedTokens(isPair bool) (retVal int)

func (bp *BertProcessing) Process(encoding, pairEncoding *tokenizer.Encoding, addSpecialTokens bool) (retVal *tokenizer.Encoding)

type ByteLevelProcessing struct {
	// contains filtered or unexported fields
}

func NewByteLevelProcessing(pretok *pretokenizer.ByteLevel) (retVal *ByteLevelProcessing)

func (blp *ByteLevelProcessing) AddedTokens(isPair bool) (retVal int)

func (blp *ByteLevelProcessing) Process(encoding, pairEncoding *tokenizer.Encoding, addSpecialTokens bool) (retVal *tokenizer.Encoding)

type PostToken struct {
	Value string
	Id    int
}

type RobertaProcessing struct {
	// contains filtered or unexported fields
}

RobertaProcessing is a post post processor for Roberta model

func DefaultRobertaProcessing() *RobertaProcessing

DefaultRobertaProcessing creates a RobertaProcessing with default values

func NewRobertaProcessing(sep, cls PostToken) *RobertaProcessing

func (rp *RobertaProcessing) AddPrefixSpace(addPrefixSpace bool)

AddPrefixSpace set whether the processor will add a prefix space

func (rp *RobertaProcessing) AddedTokens(isPair bool) int

func (rp *RobertaProcessing) Process(encoding, pairEncoding *tokenizer.Encoding, addSpecialTokens bool) *tokenizer.Encoding

func (rp *RobertaProcessing) TrimOffsets(trimOffsets bool)

TrimOffsets set whether the processor will trim offsets