model

type Base struct {
	// contains filtered or unexported fields
}

type BytePairEncoding struct {
	// contains filtered or unexported fields
}

type Model interface {
	Forward(ml.Context, Options) (ml.Tensor, error)

	Backend() ml.Backend
	Config() config
}

type Options struct {
	Inputs    []int32
	Positions []int32
	Sequences []int
	Outputs   []int32

	Images []image.Image
}

type Special int32

type Tag struct {
	Name      string
	Alternate []string
}

type TextProcessor interface {
	Encode(string) ([]int32, error)
	Decode([]int32) (string, error)
	Is(int32, Special) bool
}

type Vocabulary struct {
	Values []string
	Types  []uint32
	Scores []uint32
	Merges []string

	BOS, EOS int32
	// contains filtered or unexported fields
}