tokenizer

package

v0.0.0-...-6e34adf Latest Latest Go to latest Published: Jul 18, 2024 License: MIT Imports: 6 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/TheBinaryGuy/gox

Links

Open Source Insights

Documentation ¶

Index ¶

func PrintTokens(tokens []Token)
type LexingError
- func (e LexingError) String() string
type Token
- func Tokenize(fileContents []byte) ([]Token, error)
- func (t Token) String() string
type TokenType

Constants ¶

This section is empty.

Variables ¶

This section is empty.

Functions ¶

func PrintTokens ¶

func PrintTokens(tokens []Token)

Types ¶

type LexingError ¶

type LexingError struct {
	// contains filtered or unexported fields
}

func (LexingError) String ¶

func (e LexingError) String() string

type Token ¶

type Token struct {
	// contains filtered or unexported fields
}

func Tokenize ¶

func Tokenize(fileContents []byte) ([]Token, error)

func (Token) String ¶

func (t Token) String() string

type TokenType ¶

type TokenType int

const (
	EOF TokenType = iota
	LEFT_PAREN
	RIGHT_PAREN
	LEFT_BRACE
	RIGHT_BRACE
	COMMA
	DOT
	MINUS
	PLUS
	SEMICOLON
	STAR
	EQUAL
	EQUAL_EQUAL
	BANG
	BANG_EQUAL
	LESS
	LESS_EQUAL
	GREATER
	GREATER_EQUAL
	SLASH
	STRING
	NUMBER
	IDENTIFIER

	AND
	CLASS
	ELSE
	FALSE
	FOR
	FUN
	IF
	NIL
	OR
	PRINT
	RETURN
	SUPER
	THIS
	TRUE
	VAR
	WHILE
)

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL