token

package

v1.2.39 Latest Latest Go to latest Published: Oct 25, 2022 License: MIT Imports: 5 Imported by: 0

Details

This section is empty.

View Source

var (
	Literals = map[string]struct{}{"<": {}, ">": {}}
)

This section is empty.

type Mode uint

A mode value is a set of flags (or 0). They control scanner behavior.

const (
	ModeCaseSensitive Mode = 1 << iota
	ModeRegexpPerl
	ModeRegexpPosix
)

type Token struct {
	Typ   Type
	Value string
}

func ComsumeRunesAny(inputs []rune, current int, runeType Type, expectRunes ...rune) (token Token, next int)

func ComsumeStringsAny(inputs []rune, current int, runeType Type, mode Mode, expectStrs ...string) (token Token, next int)

func ConsumeIdentifier(inputs []rune, current int, runeType Type) (token Token, next int)

func Tokenizer(inputs []rune, f func(inputs []rune, current int) (token Token, next int), strict bool) []Token

type <value>, type <value>

type TokenizerFunc interface {
	IsLiteral() bool
	IsOperator() bool
	IsKeyword() bool
}

type Type int

Token is the set of lexical tokens of the Go programming language.

const (
	// Special tokens
	TypeILLEGAL Type = iota
	TypeEOF
	TypeCOMMENT
	TypeIgnored

	TypeLiteral
	TypeOperator
	TypeKeyword
)