tokenizer

package

v0.64.1 Latest Latest Go to latest Published: Dec 25, 2025 License: Apache-2.0 Imports: 6 Imported by: 0

Details

This section is empty.

This section is empty.

This section is empty.

type ExistsTokenizer struct{}

func NewExistsTokenizer() *ExistsTokenizer

func (t *ExistsTokenizer) Tokenize(tokens []MetaToken, _, _ []byte, _ int) []MetaToken

type KeywordTokenizer struct {
	// contains filtered or unexported fields
}

func NewKeywordTokenizer(maxTokenSize int, caseSensitive, partialIndexing bool) *KeywordTokenizer

func (t *KeywordTokenizer) Tokenize(tokens []MetaToken, name, value []byte, maxTokenSize int) []MetaToken

type MetaToken struct {
	Key   []byte
	Value []byte
}

func (m *MetaToken) MarshalBinaryTo(b []byte) []byte

func (m MetaToken) String() string

String used in tests for human-readable output.

func (m *MetaToken) UnmarshalBinary(b []byte) ([]byte, error)

type PathTokenizer struct {
	// contains filtered or unexported fields
}

func NewPathTokenizer(
	maxTokenSize int,
	caseSensitive bool,
	partialIndexing bool,
) *PathTokenizer

func (t *PathTokenizer) Tokenize(tokens []MetaToken, name, value []byte, maxTokenSize int) []MetaToken

type TextTokenizer struct {
	// contains filtered or unexported fields
}

func NewTextTokenizer(maxTokenSize int, caseSensitive, partialIndexing bool, maxFieldValueLength int) *TextTokenizer

func (t *TextTokenizer) Tokenize(tokens []MetaToken, name, value []byte, maxFieldValueLength int) []MetaToken

type Tokenizer interface {
	Tokenize(tokens []MetaToken, key, value []byte, maxLength int) []MetaToken
}