tokenizer

Nov 15, 2025

Changes in this version

+ const MaxInputSize

+ const MaxTokens

type Tokenizer

+ func (t *Tokenizer) TokenizeContext(ctx context.Context, input []byte) ([]models.TokenWithSpan, error)

Aug 22, 2025

Changes in this version

+ func PutTokenizer(t *Tokenizer)

+ type BufferPool struct

+ func NewBufferPool() *BufferPool

+ func (p *BufferPool) Get() []byte

+ func (p *BufferPool) Grow(buf []byte, n int) []byte

+ func (p *BufferPool) Put(buf []byte)

+ type DebugLogger interface

+ Debug func(format string, args ...interface{})

+ type Error struct

+ Location models.Location

+ Message string

+ func ErrorInvalidIdentifier(value string, location models.Location) *Error

+ func ErrorInvalidNumber(value string, location models.Location) *Error

+ func ErrorInvalidOperator(value string, location models.Location) *Error

+ func ErrorUnexpectedChar(ch byte, location models.Location) *Error

+ func ErrorUnterminatedString(location models.Location) *Error

+ func NewError(message string, location models.Location) *Error

+ func (e *Error) Error() string

+ type Position struct

+ Column int

+ Index int

+ LastNL int

+ Line int

+ func NewPosition(line, index int) Position

+ func (p *Position) AdvanceN(n int, lineStarts []int)

+ func (p *Position) AdvanceRune(r rune, size int)

+ func (p Position) Clone() Position

+ func (p Position) Location(t *Tokenizer) models.Location

+ type StringLiteralReader struct

+ func NewStringLiteralReader(input []byte, pos *Position, quote rune) *StringLiteralReader

+ func (r *StringLiteralReader) ReadStringLiteral() (models.Token, error)

+ type Tokenizer struct

+ func GetTokenizer() *Tokenizer

+ func New() (*Tokenizer, error)

+ func NewWithKeywords(kw *keywords.Keywords) (*Tokenizer, error)

+ func (t *Tokenizer) Reset()

+ func (t *Tokenizer) SetDebugLogger(logger DebugLogger)

+ func (t *Tokenizer) Tokenize(input []byte) ([]models.TokenWithSpan, error)

+ type TokenizerError struct

+ Location models.Location

+ Message string

+ func (e TokenizerError) Error() string