token

type Feature interface {
	Analyse(token *Token)
	Value(obj interface{})
	String() string
}

type Properties struct {
	// HasStartParens token starts with '('.
	HasStartParens bool

	// HasEndParens token end with '('.
	HasEndParens bool

	// HasStartSqParens token starts with '['.
	HasStartSqParens bool

	// HasEndSqParens token end with ']'.
	HasEndSqParens bool

	// HasEndDot token ends with '.'
	HasEndDot bool

	// HasEndComma token ends with ','
	HasEndComma bool

	// HasDigits token includes at least one '0-9'.
	HasDigits bool

	// HasLetters token includes at least one character for which
	// unicode.IsLetter(ch) is true.
	HasLetters bool

	// HasDash token includes '-'
	HasDash bool

	// HasSpecialChars internal part of a token includes non-letters, non-digits.
	HasSpecialChars bool

	// IsNumber internal part of a token has only numbers.
	IsNumber bool

	// IsWord internal part of a token includes only letters.
	IsWord bool
}

type Token struct {
	// Line line number in the text
	Line int

	// Raw is a verbatim presentation of a token as it appears in a text.
	Raw []rune

	// Start is the index of the first rune of a token. The first rune
	// does not have to be alpha-numeric.
	Start int

	// End is the index of the last rune of a token. The last rune does not
	// have to be alpha-numeric.
	End int

	// Cleaned is a presentation of a token after normalization.
	Cleaned string

	// Properties is a fixed set of general properties that we determine during
	// the text traversal.
	Properties

	// Features is the map of features as values with their string
	// representations as keys.
	Features map[string]Feature
	// contains filtered or unexported fields
}

type TokenJSON struct {
	Line    int    `json:"lineNumber"`
	Raw     string `json:"raw"`
	Cleaned string `json:"cleaned"`
	Start   int    `json:"start"`
	End     int    `json:"end"`
}