lexer

package

v0.1.1 Latest Latest Go to latest Published: Jul 8, 2026 License: AGPL-3.0 Imports: 8 Imported by: 0

Details

This section is empty.

View Source

var InvalidUTF8 = errors.New("Invalid UTF-8 char")

func IsFirstIdentRune(char rune) bool

func IsIdentRune(char rune) bool

type Lexer struct {
	// contains filtered or unexported fields
}

func New(source []byte) *Lexer

func NewFromReader(r io.Reader) (*Lexer, error)

func NewFromString(source string) *Lexer

func (l *Lexer) NextToken() (Token, error)

func (l *Lexer) Tokens() ([]Token, error)

type Token struct {
	Type     TokenType
	Lexeme   string
	Position diag.Position
}

func (t Token) String() string

type TokenType int

const (
	T_IDENT TokenType = iota
	T_STRING
	T_LPAREN
	T_RPAREN
	T_EOL
	T_EOF
)

func (t TokenType) String() string