token

type Kind ¶

type Kind string

const (
	EOF           Kind = "eof"
	Identifier    Kind = "identifier"
	Number        Kind = "number"
	String        Kind = "string"
	NativeLiteral Kind = "native_literal"
	Comment       Kind = "comment"
	Newline       Kind = "newline"
	Operator      Kind = "operator"
	Punct         Kind = "punctuation"
)

type Position ¶

type Position struct {
	Offset int
	Line   int
	Column int
}

Position is a one-based source location with a zero-based byte offset.

func (p Position) String() string

type Span ¶

type Span struct {
	Start Position
	End   Position
}

Span is a half-open range in the original source.

type Token ¶

type Token struct {
	Kind   Kind
	Lexeme string
	Span   Span
}

Token retains its exact spelling so formatters and source-to-source backends never need to reconstruct comments or literals.