tokenizer

package

v0.0.0-...-7d7b408 Latest Latest Go to latest Published: Dec 25, 2024 License: MIT Imports: 1 Imported by: 0

Details

This section is empty.

This section is empty.

This section is empty.

type Key int

type Stream struct {
	// contains filtered or unexported fields
}

func (s *Stream) Scan() bool

func (s *Stream) Token() Key

type Tokenizer struct {
	// contains filtered or unexported fields
}

func New() *Tokenizer

func (t *Tokenizer) DefineTokens(key Key, tokens ...[]byte)

func (t *Tokenizer) DefineTokensString(key Key, tokens ...string)

func (t *Tokenizer) Parse(data []byte) *Stream

func (t *Tokenizer) WithOverlap(overlap bool) *Tokenizer