tokenizer

package module

Go to main page

Versions in this module

v1

v1.4.10

Jan 22, 2025

v1.4.9

Jan 15, 2025

v1.4.8

Jan 14, 2025

v1.4.7

Dec 4, 2024

v1.4.6

Nov 6, 2024

v1.4.5

Apr 5, 2024

v1.4.4

Apr 5, 2024

v1.4.0

Oct 23, 2023

Changes in this version

+ var DefaultSpecialString = []string

+ var DefaultWhiteSpaces = []byte

+ var Numbers = []rune

+ var Underscore = []rune

type StringSettings

+ func (q *StringSettings) AddSpecialStrings(special []string) *StringSettings

type Token

+ func (t *Token) ValueFloat64() float64

+ func (t *Token) ValueInt64() int64

type Tokenizer

+ func (t *Tokenizer) AllowKeywordSymbols(majorSymbols []rune, minorSymbols []rune) *Tokenizer

+ func (t *Tokenizer) AllowNumberUnderscore() *Tokenizer

v1.3.0

Sep 15, 2022

Changes in this version

type Stream

+ func (s *Stream) IsAnyNextSequence(keys ...[]TokenKey) bool

+ func (s *Stream) IsNextSequence(keys ...TokenKey) bool

v1.2.2

May 30, 2022

v1.2.1

Nov 16, 2021

Changes in this version

+ const BackSlash

+ const DefaultChunkSize

+ var DefaultStringEscapes = map[byte]byte

+ type QuoteInjectSettings struct

+ EndKey TokenKey

+ StartKey TokenKey

+ type Stream struct

+ func NewInfStream(p *parsing) *Stream

+ func NewStream(p *parsing) *Stream

+ func (s *Stream) Close()

+ func (s *Stream) CurrentToken() *Token

+ func (s *Stream) GetParsedLength() int

+ func (s *Stream) GetSnippet(before, after int) []Token

+ func (s *Stream) GetSnippetAsString(before, after, maxStringLength int) string

+ func (s *Stream) GoNext() *Stream

+ func (s *Stream) GoNextIfNextIs(key TokenKey, otherKeys ...TokenKey) bool

+ func (s *Stream) GoPrev() *Stream

+ func (s *Stream) GoTo(n int) *Stream

+ func (s *Stream) HeadToken() *Token

+ func (s *Stream) IsValid() bool

+ func (s *Stream) NextToken() *Token

+ func (s *Stream) PrevToken() *Token

+ func (s *Stream) SetHistorySize(size int) *Stream

+ func (s *Stream) String() string

+ type StringSettings struct

+ EndToken []byte

+ EscapeSymbol byte

+ Injects []QuoteInjectSettings

+ Key TokenKey

+ SpecSymbols map[byte]byte

+ StartToken []byte

+ func (q *StringSettings) AddInjection(startTokenKey, endTokenKey TokenKey) *StringSettings

+ func (q *StringSettings) SetEscapeSymbol(symbol byte) *StringSettings

+ func (q *StringSettings) SetSpecialSymbols(special map[byte]byte) *StringSettings

+ type Token struct

+ func (t *Token) ID() int

+ func (t *Token) Indent() []byte

+ func (t *Token) Is(key TokenKey, keys ...TokenKey) bool

+ func (t *Token) IsValid() bool

+ func (t *Token) Key() TokenKey

+ func (t *Token) Line() int

+ func (t *Token) Offset() int

+ func (t *Token) StringKey() TokenKey

+ func (t *Token) StringSettings() *StringSettings

+ func (t *Token) Value() []byte

+ func (t *Token) ValueFloat() float64

+ func (t *Token) ValueString() string

+ func (t *Token) ValueUnescaped() []byte

+ func (t *Token) ValueUnescapedString() string

+ func (t Token) IsFloat() bool

+ func (t Token) IsInteger() bool

+ func (t Token) IsKeyword() bool

+ func (t Token) IsNumber() bool

+ func (t Token) IsString() bool

+ func (t Token) String() string

+ func (t Token) ValueInt() int64

+ type TokenKey int

+ const TokenFloat

+ const TokenInteger

+ const TokenKeyword

+ const TokenString

+ const TokenStringFragment

+ const TokenUndef

+ const TokenUnknown

+ type Tokenizer struct

+ func New() *Tokenizer

+ func (t *Tokenizer) AllowKeywordUnderscore() *Tokenizer

+ func (t *Tokenizer) AllowNumbersInKeyword() *Tokenizer

+ func (t *Tokenizer) DefineStringToken(key TokenKey, startToken, endToken string) *StringSettings

+ func (t *Tokenizer) DefineTokens(key TokenKey, tokens []string) *Tokenizer

+ func (t *Tokenizer) ParseBytes(str []byte) *Stream

+ func (t *Tokenizer) ParseStream(r io.Reader, bufferSize uint) *Stream

+ func (t *Tokenizer) ParseString(str string) *Stream

+ func (t *Tokenizer) SetWhiteSpaces(ws []byte) *Tokenizer

+ func (t *Tokenizer) StopOnUndefinedToken() *Tokenizer

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL