analysis

type Analyzer struct {
	CharFilters  []CharFilter
	Tokenizer    Tokenizer
	TokenFilters []TokenFilter
}

type ByteArrayConverter interface {
	Convert([]byte) (interface{}, error)
}

type CharFilter interface {
	Filter([]byte) []byte
}

type DateTimeParser interface {
	ParseDateTime(string) (time.Time, error)
}

type Token struct {
	// Start specifies the byte offset of the beginning of the term in the
	// field.
	Start int `json:"start"`

	// End specifies the byte offset of the end of the term in the field.
	End  int    `json:"end"`
	Term []byte `json:"term"`

	// Position specifies the 1-based index of the token in the sequence of
	// occurrences of its term in the field.
	Position int       `json:"position"`
	Type     TokenType `json:"type"`
	KeyWord  bool      `json:"keyword"`
}

type TokenFilter interface {
	Filter(TokenStream) TokenStream
}

type TokenFreq struct {
	Term      []byte
	Locations []*TokenLocation
	// contains filtered or unexported fields
}

type TokenFrequencies map[string]*TokenFreq

type TokenLocation struct {
	Field          string
	ArrayPositions []uint64
	Start          int
	End            int
	Position       int
}

type TokenMap map[string]bool

type TokenStream []*Token

type TokenType int

type Tokenizer interface {
	Tokenize([]byte) TokenStream
}