lexer

package
v0.0.0-...-ae0b575 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 25, 2026 License: MIT Imports: 3 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Lexer

type Lexer struct {
	// contains filtered or unexported fields
}

func New

func New(input string) *Lexer

func NewWithDialect

func NewWithDialect(input string, d dialect.Dialect) *Lexer

func (*Lexer) NextToken

func (l *Lexer) NextToken() Token

type Token

type Token struct {
	Type     TokenType
	Literal  string
	Position int
	Line     int
	Column   int
}

func TokenizeSQL

func TokenizeSQL(input string) []Token

TokenizeSQL tokenizes a complete SQL string and returns all tokens Optimized version with pre-allocated slice

func TokenizeWithBuffer

func TokenizeWithBuffer(input string, buffer []Token) []Token

TokenizeWithBuffer reuses a provided buffer for better memory efficiency

func (Token) String

func (t Token) String() string

type TokenType

type TokenType int
const (
	ILLEGAL TokenType = iota
	EOF

	// Identifiers and literals
	IDENT  // table_name, column_name
	STRING // 'hello'
	NUMBER // 123, 123.45

	// SQL Keywords
	SELECT
	FROM
	WHERE
	JOIN
	INNER
	LEFT
	RIGHT
	FULL
	ON
	GROUP
	BY
	ORDER
	HAVING
	AS
	AND
	OR
	NOT
	IN
	EXISTS
	DISTINCT
	TOP
	LIMIT
	OFFSET
	UNION
	ALL
	INSERT
	INTO   // INSERT INTO
	VALUES // INSERT VALUES
	UPDATE
	SET // UPDATE SET
	DELETE
	CREATE
	DROP
	ALTER
	TABLE
	VIEW         // CREATE VIEW
	MATERIALIZED // MATERIALIZED VIEW (PostgreSQL)
	WITH         // CTE or WITH CHECK OPTION
	RECURSIVE    // Recursive CTE
	CHECK        // WITH CHECK OPTION
	OPTION       // WITH CHECK OPTION
	OVER         // Window functions
	PARTITION    // Window functions
	ROWS         // Window frame
	RANGE        // Window frame
	UNBOUNDED    // Window frame
	PRECEDING    // Window frame
	FOLLOWING    // Window frame
	CURRENT      // Window frame
	ROW          // Window frame
	INTERSECT    // Set operations
	EXCEPT       // Set operations
	CASE         // CASE expression
	WHEN         // CASE expression
	THEN         // CASE expression
	ELSE         // CASE expression
	END          // CASE/CTE end

	// DDL Keywords
	PRIMARY        // PRIMARY KEY
	FOREIGN        // FOREIGN KEY
	KEY            // KEY
	CONSTRAINT     // CONSTRAINT
	UNIQUE         // UNIQUE
	INDEX          // INDEX
	AUTO_INCREMENT // AUTO_INCREMENT (MySQL)
	AUTOINCREMENT  // AUTOINCREMENT (SQLite)
	IDENTITY       // IDENTITY (SQL Server)
	DEFAULT        // DEFAULT value
	REFERENCES     // FOREIGN KEY REFERENCES
	ADD            // ALTER TABLE ADD
	MODIFY         // ALTER TABLE MODIFY
	CHANGE         // ALTER TABLE CHANGE (MySQL)
	COLUMN         // COLUMN
	IF             // IF EXISTS/IF NOT EXISTS
	DATABASE       // DATABASE
	SCHEMA         // SCHEMA

	// Transaction Keywords
	BEGIN       // BEGIN TRANSACTION
	START       // START TRANSACTION
	COMMIT      // COMMIT
	ROLLBACK    // ROLLBACK
	SAVEPOINT   // SAVEPOINT
	RELEASE     // RELEASE SAVEPOINT
	WORK        // WORK (optional in COMMIT/ROLLBACK)
	TRANSACTION // TRANSACTION

	// Execution Plan Keywords
	EXPLAIN  // EXPLAIN
	ANALYZE  // ANALYZE (EXPLAIN ANALYZE)
	FORMAT   // FORMAT (EXPLAIN FORMAT=JSON)
	QUERY    // QUERY (EXPLAIN QUERY PLAN - SQLite)
	PLAN     // PLAN (EXPLAIN QUERY PLAN - SQLite)
	EXTENDED // EXTENDED (MySQL)

	// Stored Procedures and Functions Keywords
	PROCEDURE     // CREATE PROCEDURE
	FUNCTION      // CREATE FUNCTION
	RETURNS       // RETURNS (function return type)
	RETURN        // RETURN statement
	DECLARE       // DECLARE variables/cursors
	CURSOR        // CURSOR
	OPEN          // OPEN cursor
	FETCH         // FETCH cursor
	CLOSE         // CLOSE cursor
	INOUT         // INOUT parameter mode
	OUT           // OUT parameter mode
	LANGUAGE      // LANGUAGE (PostgreSQL)
	PLPGSQL       // PL/pgSQL (PostgreSQL)
	SQL           // SQL language
	REPLACE       // OR REPLACE
	SECURITY      // SECURITY DEFINER/INVOKER
	DEFINER       // DEFINER
	INVOKER       // INVOKER
	DETERMINISTIC // DETERMINISTIC function
	MODIFIES      // MODIFIES SQL DATA
	READS         // READS SQL DATA
	CONTAINS      // CONTAINS SQL
	NO            // NO SQL
	LOOP          // LOOP
	WHILE         // WHILE loop
	FOR           // FOR loop
	REVERSE       // REVERSE (FOR REVERSE)
	EXIT          // EXIT loop
	CONTINUE      // CONTINUE loop
	ITERATE       // ITERATE (MySQL synonym for CONTINUE)
	LABEL         // Loop label
	ELSEIF        // ELSEIF (MySQL)
	ELSIF         // ELSIF (PostgreSQL, Oracle)
	VARIADIC      // VARIADIC parameters (PostgreSQL)
	DO            // DO (WHILE...DO)
	REPEAT        // REPEAT...UNTIL (MySQL)
	UNTIL         // UNTIL (REPEAT...UNTIL)
	ENDIF         // END IF (some dialects)
	ENDWHILE      // END WHILE (some dialects)
	ENDFOR        // END FOR (some dialects)
	ENDLOOP       // END LOOP (some dialects)

	// Exception Handling Keywords
	TRY          // TRY (SQL Server)
	CATCH        // CATCH (SQL Server)
	EXCEPTION    // EXCEPTION (PostgreSQL, Oracle)
	HANDLER      // HANDLER (MySQL)
	SQLEXCEPTION // SQLEXCEPTION (MySQL)
	SQLWARNING   // SQLWARNING (MySQL)
	FOUND        // NOT FOUND (MySQL)
	THROW        // THROW (SQL Server)
	RAISE        // RAISE (PostgreSQL, Oracle)
	SIGNAL       // SIGNAL (MySQL)
	SQLSTATE     // SQLSTATE (error codes)
	MESSAGE_TEXT // MESSAGE_TEXT (MySQL SIGNAL)
	OTHERS       // OTHERS (PostgreSQL catch-all)

	// Trigger Keywords
	TRIGGER // CREATE TRIGGER
	BEFORE  // BEFORE INSERT/UPDATE/DELETE
	AFTER   // AFTER INSERT/UPDATE/DELETE
	INSTEAD // INSTEAD OF (SQL Server, Oracle)
	OF      // OF (INSTEAD OF)
	EACH    // FOR EACH ROW/STATEMENT
	NEW     // NEW (trigger references)
	OLD     // OLD (trigger references)

	// MERGE Statement Keywords
	MERGE   // MERGE INTO
	USING   // USING source
	MATCHED // WHEN MATCHED

	// Advanced Cursor Keywords
	NEXT       // FETCH NEXT
	PRIOR      // FETCH PRIOR
	FIRST      // FETCH FIRST
	LAST       // FETCH LAST
	ABSOLUTE   // FETCH ABSOLUTE n
	RELATIVE   // FETCH RELATIVE n
	DEALLOCATE // DEALLOCATE cursor
	ALLOCATE   // ALLOCATE cursor (SQL Server)
	SCROLL     // SCROLL cursor (can move forward/backward)

	// Operators
	ASSIGN  // =
	EQ      // ==
	NOT_EQ  // !=
	LT      // <
	GT      // >
	LTE     // <=
	GTE     // >=
	LIKE    // LIKE
	BETWEEN // BETWEEN
	IS      // IS
	NULL    // NULL

	// Delimiters
	COMMA     // ,
	SEMICOLON // ;
	LPAREN    // (
	RPAREN    // )
	DOT       // .
	ASTERISK  // *
	PLUS      // +
	MINUS     // -
	SLASH     // /
	PERCENT   // %
)

func LookupIdent

func LookupIdent(ident string) TokenType

LookupIdent checks if an identifier is a keyword

func (TokenType) String

func (tt TokenType) String() string

TokenTypeString returns the string representation of a token type

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL