Documentation
¶
Overview ¶
Package models provides core data structures for SQL tokenization and parsing, including tokens, spans, locations, and error types.
Index ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Keyword ¶
type Keyword struct {
Word string // The actual keyword text
Reserved bool // Whether this is a reserved keyword
}
Keyword represents a lexical keyword with its properties
type Location ¶
Location represents a position in the source code using 1-based indexing. Both Line and Column are 1-based to match SQL standards.
type Token ¶
type Token struct {
Type TokenType
Value string
Word *Word // For TokenTypeWord
Long bool // For TokenTypeNumber to indicate if it's a long number
Quote rune // For quoted strings and identifiers
}
Token represents a SQL token with its value and metadata
type TokenType ¶
type TokenType int
TokenType represents the type of a SQL token
const ( // Special tokens TokenTypeEOF TokenType = 0 TokenTypeUnknown TokenType = 1 // Basic token types (10-29) TokenTypeWord TokenType = 10 TokenTypeNumber TokenType = 11 TokenTypeChar TokenType = 12 TokenTypeWhitespace TokenType = 13 TokenTypeIdentifier TokenType = 14 TokenTypePlaceholder TokenType = 15 // String literals (30-49) TokenTypeString TokenType = 30 // Generic string type TokenTypeSingleQuotedString TokenType = 31 TokenTypeDoubleQuotedString TokenType = 32 TokenTypeTripleSingleQuotedString TokenType = 33 TokenTypeTripleDoubleQuotedString TokenType = 34 TokenTypeDollarQuotedString TokenType = 35 TokenTypeByteStringLiteral TokenType = 36 TokenTypeNationalStringLiteral TokenType = 37 TokenTypeEscapedStringLiteral TokenType = 38 TokenTypeUnicodeStringLiteral TokenType = 39 TokenTypeHexStringLiteral TokenType = 40 // Operators and punctuation (50-99) TokenTypeOperator TokenType = 50 // Generic operator TokenTypeComma TokenType = 51 TokenTypeEq TokenType = 52 TokenTypeDoubleEq TokenType = 53 TokenTypeNeq TokenType = 54 TokenTypeLt TokenType = 55 TokenTypeGt TokenType = 56 TokenTypeLtEq TokenType = 57 TokenTypeGtEq TokenType = 58 TokenTypeSpaceship TokenType = 59 TokenTypePlus TokenType = 60 TokenTypeMinus TokenType = 61 TokenTypeMul TokenType = 62 TokenTypeDiv TokenType = 63 TokenTypeDuckIntDiv TokenType = 64 TokenTypeMod TokenType = 65 TokenTypeStringConcat TokenType = 66 TokenTypeLParen TokenType = 67 TokenTypeLeftParen TokenType = 67 // Alias for compatibility TokenTypeRParen TokenType = 68 TokenTypeRightParen TokenType = 68 // Alias for compatibility TokenTypePeriod TokenType = 69 TokenTypeDot TokenType = 69 // Alias for compatibility TokenTypeColon TokenType = 70 TokenTypeDoubleColon TokenType = 71 TokenTypeAssignment TokenType = 72 TokenTypeSemicolon TokenType = 73 TokenTypeBackslash TokenType = 74 TokenTypeLBracket TokenType = 75 TokenTypeRBracket TokenType = 76 TokenTypeAmpersand TokenType = 77 TokenTypePipe TokenType = 78 TokenTypeCaret TokenType = 79 TokenTypeLBrace TokenType = 80 TokenTypeRBrace TokenType = 81 TokenTypeRArrow TokenType = 82 TokenTypeSharp TokenType = 83 TokenTypeTilde TokenType = 84 TokenTypeExclamationMark TokenType = 85 TokenTypeAtSign TokenType = 86 TokenTypeQuestion TokenType = 87 // Compound operators (100-149) TokenTypeTildeAsterisk TokenType = 100 TokenTypeExclamationMarkTilde TokenType = 101 TokenTypeExclamationMarkTildeAsterisk TokenType = 102 TokenTypeDoubleTilde TokenType = 103 TokenTypeDoubleTildeAsterisk TokenType = 104 TokenTypeExclamationMarkDoubleTilde TokenType = 105 TokenTypeExclamationMarkDoubleTildeAsterisk TokenType = 106 TokenTypeShiftLeft TokenType = 107 TokenTypeShiftRight TokenType = 108 TokenTypeOverlap TokenType = 109 TokenTypeDoubleExclamationMark TokenType = 110 TokenTypeCaretAt TokenType = 111 TokenTypePGSquareRoot TokenType = 112 TokenTypePGCubeRoot TokenType = 113 TokenTypeArrow TokenType = 114 TokenTypeLongArrow TokenType = 115 TokenTypeHashArrow TokenType = 116 TokenTypeHashLongArrow TokenType = 117 TokenTypeAtArrow TokenType = 118 TokenTypeArrowAt TokenType = 119 TokenTypeHashMinus TokenType = 120 TokenTypeAtQuestion TokenType = 121 TokenTypeAtAt TokenType = 122 TokenTypeQuestionAnd TokenType = 123 TokenTypeQuestionPipe TokenType = 124 TokenTypeCustomBinaryOperator TokenType = 125 // SQL Keywords (200-399) TokenTypeKeyword TokenType = 200 // Generic keyword TokenTypeSelect TokenType = 201 TokenTypeFrom TokenType = 202 TokenTypeWhere TokenType = 203 TokenTypeJoin TokenType = 204 TokenTypeInner TokenType = 205 TokenTypeLeft TokenType = 206 TokenTypeRight TokenType = 207 TokenTypeOuter TokenType = 208 TokenTypeOn TokenType = 209 TokenTypeAs TokenType = 210 TokenTypeAnd TokenType = 211 TokenTypeOr TokenType = 212 TokenTypeNot TokenType = 213 TokenTypeIn TokenType = 214 TokenTypeLike TokenType = 215 TokenTypeBetween TokenType = 216 TokenTypeIs TokenType = 217 TokenTypeNull TokenType = 218 TokenTypeTrue TokenType = 219 TokenTypeFalse TokenType = 220 TokenTypeCase TokenType = 221 TokenTypeWhen TokenType = 222 TokenTypeThen TokenType = 223 TokenTypeElse TokenType = 224 TokenTypeEnd TokenType = 225 TokenTypeGroup TokenType = 226 TokenTypeBy TokenType = 227 TokenTypeHaving TokenType = 228 TokenTypeOrder TokenType = 229 TokenTypeAsc TokenType = 230 TokenTypeDesc TokenType = 231 TokenTypeLimit TokenType = 232 TokenTypeOffset TokenType = 233 // Aggregate functions (250-269) TokenTypeCount TokenType = 250 TokenTypeSum TokenType = 251 TokenTypeAvg TokenType = 252 TokenTypeMin TokenType = 253 TokenTypeMax TokenType = 254 // Compound keywords (270-299) TokenTypeGroupBy TokenType = 270 TokenTypeOrderBy TokenType = 271 TokenTypeLeftJoin TokenType = 272 TokenTypeRightJoin TokenType = 273 TokenTypeInnerJoin TokenType = 274 TokenTypeOuterJoin TokenType = 275 )
Token type constants with explicit values to avoid collisions
type TokenWithSpan ¶
TokenWithSpan represents a token with its location in the source code
func NewEOFToken ¶
func NewEOFToken(pos Location) TokenWithSpan
NewEOFToken creates a new EOF token with span
func NewTokenWithSpan ¶
func NewTokenWithSpan(tokenType TokenType, value string, start, end Location) TokenWithSpan
NewTokenWithSpan creates a new TokenWithSpan with the given type, value, and location
func TokenAtLocation ¶
func TokenAtLocation(token Token, start, end Location) TokenWithSpan
TokenAtLocation creates a new TokenWithSpan from a Token and location
func WrapToken ¶
func WrapToken(token Token) TokenWithSpan
WrapToken wraps a token with an empty location
type TokenizerError ¶
TokenizerError represents an error during tokenization
func (TokenizerError) Error ¶
func (e TokenizerError) Error() string
type Whitespace ¶
type Whitespace struct {
Type WhitespaceType
Content string // For comments
Prefix string // For single line comments
}
Whitespace represents different types of whitespace tokens
type WhitespaceType ¶
type WhitespaceType int
WhitespaceType represents the type of whitespace
const ( WhitespaceTypeSpace WhitespaceType = iota WhitespaceTypeNewline WhitespaceTypeTab WhitespaceTypeSingleLineComment WhitespaceTypeMultiLineComment )
Source Files
Click to show internal directories.
Click to hide internal directories.