lexer

package

v1.1.10 Latest Latest Go to latest Published: Dec 26, 2025 License: MIT Imports: 4 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/ArubikU/polyloft

Links

Documentation ¶

Index ¶

func FormatToken(item Item) string
func TokenName(tok Token) string
type Item
type Lexer
- func (l *Lexer) Scan(src []byte) []Item
type Token

Constants ¶

This section is empty.

Variables ¶

This section is empty.

Functions ¶

func FormatToken ¶

func FormatToken(item Item) string

FormatToken returns a formatted string for a token including its literal value if available.

func TokenName ¶

func TokenName(tok Token) string

TokenName returns a human-readable name for a token type.

Types ¶

type Item ¶

type Item struct {
	Tok   Token
	Lit   string
	Start ast.Position
	End   ast.Position
}

Item represents a scanned token with its literal text and position.

type Lexer ¶

type Lexer struct{}

Lexer implements a minimal state machine to scan tokens.

func (*Lexer) Scan ¶

func (l *Lexer) Scan(src []byte) []Item

type Token ¶

type Token int

Token identifies the kind of lexical item.

const (
	// Special tokens
	ILLEGAL Token = iota
	EOF

	// Literals/identifiers
	IDENT  // foo, Bar, _baz
	NUMBER // 10, 3.14, -5, 2.5e10
	INT    // 10, 42, -5
	HEX    // 0x1A, 0xFF
	BYTES  // 0b1010, 0b1101
	FLOAT  // 10.0f, 3.14f
	STRING
	INTERPOLATED_STRING // "text #{expr} more text"

	// Keywords
	KW_VAR
	KW_LET
	KW_CONST
	KW_FINAL
	KW_PUBLIC
	KW_PRIVATE
	KW_PROTECTED
	KW_STATIC
	KW_DEF
	KW_INTERFACE
	KW_CLASS
	KW_IMPORT
	KW_IMPLEMENTS
	KW_ABSTRACT
	KW_SEALED
	KW_RETURN
	KW_TRUE
	KW_FALSE
	KW_NIL
	KW_THREAD
	KW_SPAWN
	KW_JOIN
	KW_IF
	KW_ELIF
	KW_ELSE
	KW_FOR
	KW_IN
	KW_BREAK
	KW_CONTINUE
	KW_LOOP
	KW_END
	KW_DO
	KW_INSTANCEOF
	KW_THIS
	KW_SUPER
	KW_ENUM
	KW_RECORD
	KW_TRY
	KW_CATCH
	KW_FINALLY
	KW_THROW
	KW_DEFER
	KW_CHANNEL
	KW_SELECT
	KW_SWITCH
	KW_CASE
	KW_DEFAULT
	KW_CLOSED
	KW_WHERE
	KW_EXTENDS
	KW_OUT

	// Operators and delimiters
	ASSIGN       // =
	PLUS         // +
	MINUS        // -
	STAR         // *
	SLASH        // /
	PERCENT      // %
	PLUS_ASSIGN  // +=
	MINUS_ASSIGN // -=
	STAR_ASSIGN  // *=
	SLASH_ASSIGN // /=
	EQ           // ==
	NEQ          // !=
	LT           // <
	LTE          // <=
	GT           // >
	GTE          // >=
	AND          // &&
	OR           // ||
	NOT          // !
	ARROW        // =>
	RARROW       // ->
	COLONASSIGN  // :=

	COMMA    // ,
	COLON    // :
	SEMI     // ;
	QUESTION // ? (for ternary operator)

	LPAREN   // (
	RPAREN   // )
	LBRACE   // {
	RBRACE   // }
	LBRACK   // [
	RBRACK   // ]
	DOT      // .
	ELLIPSIS // ... (for variadic parameters)
	AT       // @ (for annotations)
	PIPE     // | (for union types)
)

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL