tokenizer

package

v0.61.0-test Latest Latest Go to latest Published: Sep 3, 2025 License: Apache-2.0 Imports: 5 Imported by: 0

Details

This section is empty.

This section is empty.

This section is empty.

type ExistsTokenizer struct{}

func NewExistsTokenizer() *ExistsTokenizer

func (t *ExistsTokenizer) Tokenize(tokens []frac.MetaToken, _, _ []byte, _ int) []frac.MetaToken

type KeywordTokenizer struct {
	// contains filtered or unexported fields
}

func NewKeywordTokenizer(maxTokenSize int, caseSensitive, partialIndexing bool) *KeywordTokenizer

func (t *KeywordTokenizer) Tokenize(tokens []frac.MetaToken, name, value []byte, maxTokenSize int) []frac.MetaToken

type PathTokenizer struct {
	// contains filtered or unexported fields
}

func NewPathTokenizer(
	maxTokenSize int,
	caseSensitive bool,
	partialIndexing bool,
) *PathTokenizer

func (t *PathTokenizer) Tokenize(tokens []frac.MetaToken, name, value []byte, maxTokenSize int) []frac.MetaToken

type TextTokenizer struct {
	// contains filtered or unexported fields
}

func NewTextTokenizer(maxTokenSize int, caseSensitive, partialIndexing bool, maxFieldValueLength int) *TextTokenizer

func (t *TextTokenizer) Tokenize(tokens []frac.MetaToken, name, value []byte, maxFieldValueLength int) []frac.MetaToken

type Tokenizer interface {
	Tokenize(tokens []frac.MetaToken, key, value []byte, maxLength int) []frac.MetaToken
}