tokenizer

package

Go to main page

Versions in this module

v0

v0.5.1

Oct 23, 2023

v0.5.0

Oct 23, 2023

v0.4.2

Oct 15, 2023

v0.4.1

Oct 11, 2023

v0.4.0

Oct 11, 2023

Changes in this version

type TokenizerStateMachine

+ func NewTokenizerStateMachineFromGoData(obj interface{}) (*TokenizerStateMachine, error)

v0.3.2

Oct 6, 2023

Changes in this version

+ var ErrorIncorrectToken = errors.New("")

+ func NewErrorIncorrectToken(mode StateMode, token token.TokenType) error

+ type PrimitiveStateProcessor interface

+ type PrimitiveValueTokenizer interface

+ type Tokenizer interface

+ GetMode func() StateMode

+ ProcessData func(provider constructor.TokenProvider) error

type TokenizerStateMachine

+ func NewTokenizerStateMachineFromIOReader(reader io.Reader) *TokenizerStateMachine

+ func (i *TokenizerStateMachine) GetASTBuilder() constructor.ASTManager

+ func (i *TokenizerStateMachine) RecordStateValue(valueType StateMode, nodeValue interface{}) error

v0.3.1

Oct 3, 2023

v0.3.0

Oct 2, 2023

v0.2.0

Sep 23, 2023

Changes in this version

+ var ErrorExtendedVariableFormatIncorrect = errors.New("variable should be of ${variableName} format")

+ var ErrorIncorrectCharacter = errors.New("incorrect character")

+ var ErrorIncorrectValueForState = errors.New("extracted value not match state")

+ var ErrorIncorrectValueTypeForConstructAST = errors.New("incorrect value type for construct ast")

+ var ErrorInternalASTProcotolChanged = errors.New(...)

+ var ErrorTokenRouteNotConfigure = errors.New("token route not configure")

+ var ErrorUnexpectedEOF = errors.New("unexpected EOF")

+ func IsSymbolToken(t TokenType) bool

+ func ShouldUnreadByte(t TokenType) bool

+ type ArrayState struct

+ func (i *ArrayState) GetMode() StateMode

+ func (i *ArrayState) ProcessData(dataSource *bufio.Reader) error

+ type BooleanState struct

+ func (i *BooleanState) GetMode() StateMode

+ func (i *BooleanState) ProcessData(dataSource *bufio.Reader) error

+ type InitState struct

+ func (i *InitState) GetMode() StateMode

+ func (i *InitState) ProcessData(dataSource *bufio.Reader) error

+ type JzonePrimitiveStateProcessor interface

+ type JzonePrimitiveTokenizer interface

+ type JzoneTokenizer interface

+ GetMode func() StateMode

+ ProcessData func(dataSource *bufio.Reader) error

+ type JzoneTokenizerStateMachine interface

+ GetASTConstructor func() *ast.ASTConstructor

+ GetCurrentMode func() StateMode

+ ProcessData func(dataSource io.Reader) error

+ RecordSyntaxSymbol func(b byte) error

+ RecordSyntaxValue func(t StateMode, value interface{}) error

+ Reset func()

+ SwitchStateByToken func(tokenType TokenType) error

+ SwitchToLatestState func() error

+ func NewTokenizerStateMachine() JzoneTokenizerStateMachine

+ type NullState struct

+ func (i *NullState) GetMode() StateMode

+ func (i *NullState) ProcessData(dataSource *bufio.Reader) error

+ type NumberState struct

+ func (i *NumberState) GetMode() StateMode

+ func (i *NumberState) ProcessData(dataSource *bufio.Reader) error

+ type ObjectState struct

+ func (i *ObjectState) GetMode() StateMode

+ func (i *ObjectState) ProcessData(dataSource *bufio.Reader) error

+ type PrimitiveValueTokenStateBase struct

+ func NewPrimitiveValueTokenStateBase(sm JzoneTokenizerStateMachine) PrimitiveValueTokenStateBase

+ type StateMode uint

+ const ARRAY_MODE

+ const BOOLEAN_MODE

+ const INIT_MODE

+ const NULL_MODE

+ const NUMBER_MODE

+ const OBJECT_MODE

+ const STRING_MODE

+ const STRING_VARIABLE_MODE

+ const VARIABLE_MODE

+ type StringState struct

+ func (i *StringState) GetMode() StateMode

+ func (i *StringState) ProcessData(dataSource *bufio.Reader) error

+ type TokenPreProcessor interface

+ PreprocessToken func(dataSource *bufio.Reader) (TokenType, error)

+ type TokenReader struct

+ func NewTokenReader(sm JzoneTokenizerStateMachine) TokenReader

+ func (i *TokenReader) PreprocessToken(dataSource *bufio.Reader) (TokenType, error)

+ type TokenType uint

+ const TOKEN_BOOLEAN

+ const TOKEN_COLON

+ const TOKEN_COMMA

+ const TOKEN_DOUBLE_QUOTATION

+ const TOKEN_DROP

+ const TOKEN_DUMMY

+ const TOKEN_LEFT_BRACE

+ const TOKEN_LEFT_BRACKET

+ const TOKEN_NULL

+ const TOKEN_NUMBER

+ const TOKEN_NUMBER_DECIMAL

+ const TOKEN_RIGHT_BRACE

+ const TOKEN_RIGHT_BRACKET

+ const TOKEN_SPACE

+ const TOKEN_STRING

+ const TOKEN_STRING_WITH_VARIABLE

+ const TOKEN_VARIABLE

+ func GetTokenTypeByStartCharacter(b byte) TokenType

+ type TokenizerStateMachine struct

+ func (i *TokenizerStateMachine) GetAST() ast.JsonNode

+ func (i *TokenizerStateMachine) GetASTConstructor() *ast.ASTConstructor

+ func (i *TokenizerStateMachine) GetCurrentMode() StateMode

+ func (i *TokenizerStateMachine) ProcessData(dataSource io.Reader) error

+ func (i *TokenizerStateMachine) RecordSyntaxSymbol(b byte) error

+ func (i *TokenizerStateMachine) RecordSyntaxValue(valueType StateMode, nodeValue interface{}) error

+ func (i *TokenizerStateMachine) Reset()

+ func (i *TokenizerStateMachine) SwitchStateByToken(tokenType TokenType) error

+ func (i *TokenizerStateMachine) SwitchToLatestState() error

+ type VariableState struct

+ func (i *VariableState) GetMode() StateMode

+ func (i *VariableState) ProcessData(dataSource *bufio.Reader) error

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL