tokenizer

package
v0.5.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 23, 2023 License: MIT Imports: 9 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var (
	ErrorIncorrectValueTypeForConstructAST = errors.New("incorrect value type for construct ast")
	ErrorInternalASTProcotolChanged        = errors.New("detect unexpect ast stack change, not kv, array, object, NodeType at top of stack")
	ErrorUnexpectedEOF                     = errors.New("unexpected EOF")
	ErrorTokenRouteNotConfigure            = errors.New("token route not configure")
	ErrorExtendedVariableFormatIncorrect   = errors.New("variable should be of ${variableName} format")
)

Functions

func NewErrorIncorrectToken added in v0.3.2

func NewErrorIncorrectToken(mode StateMode, token token.TokenType) error

Types

type ArrayState

type ArrayState struct {
	TokenReader
}

func (*ArrayState) GetMode

func (i *ArrayState) GetMode() StateMode

func (*ArrayState) ProcessData

func (i *ArrayState) ProcessData(provider astbuilder.TokenProvider) error

type BooleanState

type BooleanState struct {
	PrimitiveValueTokenStateBase
}

func (*BooleanState) GetMode

func (i *BooleanState) GetMode() StateMode

func (*BooleanState) ProcessData

func (i *BooleanState) ProcessData(provider astbuilder.TokenProvider) error

type InitState

type InitState struct {
	TokenReader
}

func (*InitState) GetMode

func (i *InitState) GetMode() StateMode

func (*InitState) ProcessData

func (i *InitState) ProcessData(provider astbuilder.TokenProvider) error

type NullState

type NullState struct {
	PrimitiveValueTokenStateBase
}

func (*NullState) GetMode

func (i *NullState) GetMode() StateMode

func (*NullState) ProcessData

func (i *NullState) ProcessData(provider astbuilder.TokenProvider) error

type NumberState

type NumberState struct {
	PrimitiveValueTokenStateBase
}

func (*NumberState) GetMode

func (i *NumberState) GetMode() StateMode

func (*NumberState) ProcessData

func (i *NumberState) ProcessData(provider astbuilder.TokenProvider) error

type ObjectState

type ObjectState struct {
	TokenReader
}

func (*ObjectState) GetMode

func (i *ObjectState) GetMode() StateMode

func (*ObjectState) ProcessData

func (i *ObjectState) ProcessData(provider astbuilder.TokenProvider) error

type PrimitiveStateProcessor added in v0.3.2

type PrimitiveStateProcessor interface {
	// contains filtered or unexported methods
}

for primitive value token, they hold primitive value (not array/object) store their value to somewhere based on extracted value, and the switch of state is based on AST current state, therefore no need for parameterized state change

type PrimitiveValueTokenStateBase

type PrimitiveValueTokenStateBase struct {
	// contains filtered or unexported fields
}

for the base, only provides what `PrimitiveStateProcessor` needs

type PrimitiveValueTokenizer added in v0.3.2

type PrimitiveValueTokenizer interface {
	Tokenizer
	PrimitiveStateProcessor
}

type StateMode

type StateMode uint
const (
	INIT_MODE StateMode = iota
	OBJECT_MODE
	ARRAY_MODE
	STRING_MODE
	NUMBER_MODE
	BOOLEAN_MODE
	NULL_MODE
	VARIABLE_MODE
	STRING_VARIABLE_MODE
)

type StringState

type StringState struct {
	PrimitiveValueTokenStateBase
}

func (*StringState) GetMode

func (i *StringState) GetMode() StateMode

func (*StringState) ProcessData

func (i *StringState) ProcessData(provider astbuilder.TokenProvider) error

type TokenReader

type TokenReader struct {
	// contains filtered or unexported fields
}

func NewTokenReader

func NewTokenReader(stateMachine *TokenizerStateMachine) TokenReader

type Tokenizer added in v0.3.2

type Tokenizer interface {
	ProcessData(provider astbuilder.TokenProvider) error
	GetMode() StateMode
}

common tokenizer implementation

type TokenizerStateMachine

type TokenizerStateMachine struct {
	// contains filtered or unexported fields
}

func NewTokenizerStateMachineFromGoData added in v0.4.0

func NewTokenizerStateMachineFromGoData(obj interface{}, options []astbuilder.TokenProviderOptions) (*TokenizerStateMachine, error)

func NewTokenizerStateMachineFromIOReader added in v0.3.2

func NewTokenizerStateMachineFromIOReader(reader io.Reader) *TokenizerStateMachine

func (*TokenizerStateMachine) GetAST

func (i *TokenizerStateMachine) GetAST() ast.JsonNode

func (*TokenizerStateMachine) GetASTBuilder added in v0.3.2

func (*TokenizerStateMachine) GetCurrentMode

func (i *TokenizerStateMachine) GetCurrentMode() StateMode

func (*TokenizerStateMachine) ProcessData

func (i *TokenizerStateMachine) ProcessData() error

func (*TokenizerStateMachine) RecordStateValue added in v0.3.2

func (i *TokenizerStateMachine) RecordStateValue(valueType StateMode, nodeValue interface{}) error

func (*TokenizerStateMachine) SwitchStateByToken

func (i *TokenizerStateMachine) SwitchStateByToken(tokenType token.TokenType) error

func (*TokenizerStateMachine) SwitchToLatestState

func (i *TokenizerStateMachine) SwitchToLatestState() error

use AST to switch the state of the machine when primitive values end of their processing

type VariableState

type VariableState struct {
	PrimitiveValueTokenStateBase
}

func (*VariableState) GetMode

func (i *VariableState) GetMode() StateMode

func (*VariableState) ProcessData

func (i *VariableState) ProcessData(provider astbuilder.TokenProvider) error

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL