scanner

const (
	// TokenKindUnassigned is used to identify a TokenKind variable
	// which hasn't been assigned to yet. For this purpose, it must
	// be the zero-value of its type.
	TokenKindUnassigned TokenKind = iota

	// TokenKindWhitespace identifies a token which only contains
	// runes found in the runesWhitespace constant.
	TokenKindWhitespace

	// TokenKindNewline identifies a token which only contains
	// runes found in the runesWhitespace or runesNewline
	// constants.
	TokenKindNewline

	// TokenKindEscape identifies a 2-rune token which always
	// starts with the Escape rune.
	TokenKindEscape

	// TokenKindOther identifies all other possible tokens which are
	// not identified by the given TokenKind constants. This kind of
	// token is used to represent directory names, value types, and
	// data elements (numbers, strings, UUIDs, etc...).
	TokenKindOther

	// TokenKindEnd is returned from Scanner.Scan when the wrapped
	// io.Reader has been read to completion.
	TokenKindEnd

	// TokenKindKeyValSep identifies a token equal to KeyValSep.
	TokenKindKeyValSep

	// TokenKindDirSep identifies a token equal to DirSep.
	TokenKindDirSep

	// TokenKindTupStart identifies a token equal to TupStart.
	TokenKindTupStart

	// TokenKindTupEnd identifies a token equal to TupEnd.
	TokenKindTupEnd

	// TokenKindTupSep identifies a token equal to TupSep.
	TokenKindTupSep

	// TokenKindVarStart identifies a token equal to VarStart.
	TokenKindVarStart

	// TokenKindVarEnd identifies a token equal to VarEnd.
	TokenKindVarEnd

	// TokenKindVarSep identifies a token equal to VarSep.
	TokenKindVarSep

	// TokenKindStrMark identifies a token equal to StrMark.
	TokenKindStrMark

	// TokenKindStampStart identifies a token equal to StampStart.
	TokenKindStampStart

	// TokenKindStampSep identifies a token equal to StampSep.
	TokenKindStampSep

	// TokenKindReserved identifies a single-rune token which
	// isn't currently used by the language but reserved for
	// later use.
	TokenKindReserved
)