Documentation
¶
Index ¶
Constants ¶
View Source
const ( PadToken = "[PAD]" OMaskToken = "[O-MASK]" ClsToken = "[CLS]" SepToken = "[SEP]" )
View Source
const (
DefaultMaxOptions = 10
)
DefaultMaxOptions is the default value for some template attributes.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type BasePrompt ¶
func (BasePrompt) DoTruncate ¶
func (p BasePrompt) DoTruncate() bool
func (BasePrompt) Positions ¶
func (p BasePrompt) Positions() int
func (BasePrompt) TokenTypes ¶
func (p BasePrompt) TokenTypes() int
type ChoicesPrompt ¶
type ChoicesPrompt struct {
BasePrompt
AddOMask bool
AddPrompt bool
Length int
}
func (ChoicesPrompt) BuildText ¶
func (p ChoicesPrompt) BuildText(e Example) string
type ErnieTokenizer ¶
type ErnieTokenizer struct {
*aietokenizer.Tokenizer
}
func NewErnieTokenizer ¶
func NewErnieTokenizer(vocabFile string, lowerCase bool) (*ErnieTokenizer, error)
func (*ErnieTokenizer) AddSpecialTokenIDs ¶
func (t *ErnieTokenizer) AddSpecialTokenIDs(tokenIDsA, tokenIDsB []int, clsTokenID, sepTokenID int) []int
func (*ErnieTokenizer) ClsTokenID ¶
func (t *ErnieTokenizer) ClsTokenID() int
func (*ErnieTokenizer) OMaskTokenID ¶
func (t *ErnieTokenizer) OMaskTokenID() int
func (*ErnieTokenizer) PadTokenID ¶
func (t *ErnieTokenizer) PadTokenID() int
func (*ErnieTokenizer) SepTokenID ¶
func (t *ErnieTokenizer) SepTokenID() int
type Input ¶
func BuildInputsWithPrompt ¶
type Prediction ¶
func (Prediction) Best ¶
func (p Prediction) Best() (label string, score float32)
type PromptTokenizer ¶
type PromptTokenizer struct {
// contains filtered or unexported fields
}
func NewPromptTokenizer ¶
func NewPromptTokenizer(vocabFile string, doLowerCase bool, maxSeqLength int) (*PromptTokenizer, error)
func (*PromptTokenizer) Encode ¶
func (t *PromptTokenizer) Encode(inputs []Input) Encoding
type SepPrompt ¶
type SepPrompt struct {
BasePrompt
Sep string
}
type TextPrompt ¶
type TextPrompt struct {
BasePrompt
Text string
}
func (TextPrompt) BuildText ¶
func (p TextPrompt) BuildText(e Example) string
Click to show internal directories.
Click to hide internal directories.