parser

package
v0.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 15, 2025 License: MIT Imports: 18 Imported by: 5

Documentation

Index

Constants

View Source 
const MainName = "(main)"

Variables

View Source 
var DefaultMixedDelimiter = MixedDelimiter{
	Start: []rune("{%"),
	End:   []rune("%}"),
}

Functions

func RemoveSpaces 

func RemoveSpaces(t Token) bool

func StripCR 

func StripCR(b []byte, comment bool) []byte

StripCR removes carriage return characters.

func TrimSpace 

func TrimSpace(left, right bool, s string) string

TrimSpace returns a slice of the string s, with all leading and trailing white space removed, as defined by Unicode.

func Unquote 

func Unquote(s string) (string, error)

Unquote interprets s as a single-quoted, double-quoted, or backquoted Go string literal, returning the string value that s quotes. (If s is single-quoted, it would be a Go character literal; Unquote returns the corresponding one-character string.)

Types

type BlockEnd 

type BlockEnd struct {
	Token token.Token
	Next  bool
}

type BlockWrap 

type BlockWrap struct {
	Start token.Token
	Ends  []BlockEnd
}

type Error 

type Error struct {
	Pos source.FilePos
	Msg string
}

Error represents a parser error.

func (*Error) Error 

func (e *Error) Error() string

func (*Error) Format 

func (e *Error) Format(f fmt.State, verb rune)

type ErrorList 

type ErrorList []*Error

ErrorList is a collection of parser errors.

func (*ErrorList) Add 

func (p *ErrorList) Add(pos source.FilePos, msg string)

Add adds a new parser error to the collection.

func (ErrorList) Err 

func (p ErrorList) Err() error

Err returns an error.

func (ErrorList) Error 

func (p ErrorList) Error() string

func (ErrorList) Format 

func (p ErrorList) Format(f fmt.State, verb rune)

func (ErrorList) Len 

func (p ErrorList) Len() int

Len returns the number of elements in the collection.

func (ErrorList) Less 

func (p ErrorList) Less(i, j int) bool

func (ErrorList) Sort 

func (p ErrorList) Sort()

Sort sorts the collection.

func (ErrorList) Swap 

func (p ErrorList) Swap(i, j int)

type File 

type File struct {
	InputFile *source.File
	Stmts     node.Stmts
	Comments  []*ast.CommentGroup
}

File represents a file unit.

func Parse 

func Parse(input, fileName string, opts *ParserOptions, scannerOpts *ScannerOptions) (*File, error)

func ParseFile 

func ParseFile(pth string, opts *ParserOptions, scannerOpts *ScannerOptions) (file *File, err error)

func (*File) End 

func (n *File) End() source.Pos

End returns the position of first character immediately after the node.

func (*File) Pos 

func (n *File) Pos() source.Pos

Pos returns the position of first character belonging to the node.

func (*File) String 

func (n *File) String() string

func (*File) WriteCode 

func (n *File) WriteCode(ctx *node.CodeWriteContext)

type Handlers 

type Handlers struct {
	source.NextHandlers
	ScanHandler   func(ch rune) (t Token, insertSemi, ok bool)
	TokenHandlers TokenHandlers
}

func (*Handlers) CallTokenHandlers 

func (s *Handlers) CallTokenHandlers(t *Token)

func (*Handlers) TokenHandler 

func (s *Handlers) TokenHandler(f func(t *Token))

type MixedDelimiter 

type MixedDelimiter = source.StartEndDelimiter

type Mode 

type Mode int

Mode value is a set of flags for parser. 

const (
	// ParseComments parses comments and add them to AST
	ParseComments Mode = 1 << iota
	ParseMixed
	ParseConfigDisabled
	ParseMixedExprAsValue
	ParseFloatAsDecimal
	ParseCharAsString
)

func (*Mode) Clear 

func (b *Mode) Clear(flag Mode) *Mode

func (Mode) Has 

func (b Mode) Has(flag Mode) bool

func (*Mode) Set 

func (b *Mode) Set(flag Mode) *Mode

func (*Mode) Toggle 

func (b *Mode) Toggle(flag Mode) *Mode

type ParseListHandler 

type ParseListHandler func(start token.Token, ends ...BlockWrap) (list []node.Stmt, end *BlockEnd)

type Parser 

type Parser struct {
	File      *source.File
	Errors    ErrorList
	Scanner   ScannerInterface
	Token     Token
	PrevToken Token
	ExprLevel int // < 0: in control clause, >= 0: in expression

	Trace bool

	TraceOut io.Writer

	ParseStmtHandler func() node.Stmt
	InCode           bool
	BlockStart       token.Token
	BlockEnd         token.Token
	ScanFunc         func() Token
	// contains filtered or unexported fields
}

Parser parses the Tengo source files. It's based on ToInterface's parser implementation.

func NewParser 

func NewParser(file *source.File, trace io.Writer) *Parser

NewParser creates a Parser.

func NewParserWithOptions 

func NewParserWithOptions(
	file *source.File,
	opts *ParserOptions,
	scannerOptions *ScannerOptions,
) *Parser

NewParserWithOptions creates a Parser with parser mode flags.

func NewParserWithScanner 

func NewParserWithScanner(
	scanner ScannerInterface,
	opts *ParserOptions,
) *Parser

NewParserWithScanner creates a Parser with parser mode flags.

func NewSingleParser 

func NewSingleParser(input, fileName string, opts *ParserOptions, scannerOpts *ScannerOptions) *Parser

func (*Parser) AtComma 

func (p *Parser) AtComma(context string, follow token.Token) bool

func (*Parser) CallArgsOf 

func (p *Parser) CallArgsOf(lparen, rparen source.Pos, exprs ...node.Expr) (params *node.CallArgs)

func (*Parser) DefaultParseStmt 

func (p *Parser) DefaultParseStmt() (stmt node.Stmt)

func (*Parser) Error 

func (p *Parser) Error(pos source.Pos, msg string)

func (*Parser) ErrorExpected 

func (p *Parser) ErrorExpected(pos source.Pos, msg string)

func (*Parser) ErrorExpectedExpr 

func (p *Parser) ErrorExpectedExpr(expected, got node.Expr)

func (*Parser) Expect 

func (p *Parser) Expect(token token.Token) source.Pos

func (*Parser) ExpectLit 

func (p *Parser) ExpectLit(token token.Token) ast.Literal

func (*Parser) ExpectLits 

func (p *Parser) ExpectLits(toks ...token.Token) ast.Literal

func (*Parser) ExpectSemi 

func (p *Parser) ExpectSemi()

func (*Parser) ExpectToken 

func (p *Parser) ExpectToken(token token.Token) (tok Token)

func (*Parser) IsToken 

func (p *Parser) IsToken(toks ...token.Token) bool

func (*Parser) MakeExpr 

func (p *Parser) MakeExpr(s node.Stmt, want string) node.Expr

func (*Parser) Next 

func (p *Parser) Next() *Parser

func (*Parser) ParseAllParemExpr 

func (p *Parser) ParseAllParemExpr(lparenToken, rparenToken token.Token) node.Expr

func (*Parser) ParseArrayLitOrKeyValue 

func (p *Parser) ParseArrayLitOrKeyValue() node.Expr

func (*Parser) ParseBinaryExpr 

func (p *Parser) ParseBinaryExpr(prec1 int) node.Expr

func (*Parser) ParseBlockStmt 

func (p *Parser) ParseBlockStmt(ends ...token.Token) *node.BlockStmt

func (*Parser) ParseBody 

func (p *Parser) ParseBody() (b *node.BlockStmt, lambdaPos source.Pos, closure node.Expr)

func (*Parser) ParseBranchStmt 

func (p *Parser) ParseBranchStmt(tok token.Token) node.Stmt

func (*Parser) ParseCall 

func (p *Parser) ParseCall(x node.Expr) *node.CallExpr

func (*Parser) ParseCallArgs 

func (p *Parser) ParseCallArgs(start, end token.Token) *node.CallArgs

func (*Parser) ParseCatchStmt 

func (p *Parser) ParseCatchStmt() *node.CatchStmt

func (*Parser) ParseCharLit 

func (p *Parser) ParseCharLit() node.Expr

func (*Parser) ParseClosureExpr 

func (p *Parser) ParseClosureExpr(lambdaToken token.Token, paren *node.MultiParenExpr) *node.ClosureExpr

func (*Parser) ParseCondExpr 

func (p *Parser) ParseCondExpr(cond node.Expr) node.Expr

func (*Parser) ParseConfigStmt 

func (p *Parser) ParseConfigStmt() (c *node.ConfigStmt)

func (*Parser) ParseDecl 

func (p *Parser) ParseDecl() node.Decl

func (*Parser) ParseDictElementLit 

func (p *Parser) ParseDictElementLit() *node.DictElementLit

func (*Parser) ParseDictElementLitFunc 

func (p *Parser) ParseDictElementLitFunc() *node.DictElementFuncExpr

func (*Parser) ParseDictLit 

func (p *Parser) ParseDictLit() *node.DictExpr

func (*Parser) ParseElse 

func (p *Parser) ParseElse(ifs bool) node.Stmt

func (*Parser) ParseEmbedExpr 

func (p *Parser) ParseEmbedExpr() node.Expr

func (*Parser) ParseExpr 

func (p *Parser) ParseExpr() node.Expr

func (*Parser) ParseExprList 

func (p *Parser) ParseExprList() (list []node.Expr)

func (*Parser) ParseFile 

func (p *Parser) ParseFile() (file *File, err error)

ParseFile parses the source and returns an AST file unit.

func (*Parser) ParseFileH 

func (p *Parser) ParseFileH(listHandler ParseListHandler) (file *File, err error)

ParseFileH parses the source and returns an AST file unit.

func (*Parser) ParseFinallyStmt 

func (p *Parser) ParseFinallyStmt() *node.FinallyStmt

func (*Parser) ParseForStmt 

func (p *Parser) ParseForStmt() node.Stmt

func (*Parser) ParseFuncExpr 

func (p *Parser) ParseFuncExpr() node.Expr

func (*Parser) ParseFuncType 

func (p *Parser) ParseFuncType() *node.FuncType

func (*Parser) ParseGenDecl 

func (p *Parser) ParseGenDecl(
	keyword token.Token,
	fn func(token.Token, bool, []node.Spec, int) node.Spec,
) *node.GenDecl

func (*Parser) ParseIdent 

func (p *Parser) ParseIdent() *node.IdentExpr

func (*Parser) ParseIfHeader 

func (p *Parser) ParseIfHeader() (init node.Stmt, cond node.Expr)

func (*Parser) ParseIfStmt 

func (p *Parser) ParseIfStmt() node.Stmt

func (*Parser) ParseImportExpr 

func (p *Parser) ParseImportExpr() node.Expr

func (*Parser) ParseIndexOrSlice 

func (p *Parser) ParseIndexOrSlice(x node.Expr) node.Expr

func (*Parser) ParseKeyValueArrayLit 

func (p *Parser) ParseKeyValueArrayLit(lbrace source.Pos) *node.KeyValueArrayLit

func (*Parser) ParseKeyValueArrayLitAt 

func (p *Parser) ParseKeyValueArrayLitAt(lbrace source.Pos, rbraceToken token.Token) *node.KeyValueArrayLit

func (*Parser) ParseKeyValueLit 

func (p *Parser) ParseKeyValueLit() *node.KeyValueLit

func (*Parser) ParseKeyValuePairLit 

func (p *Parser) ParseKeyValuePairLit(endToken token.Token) *node.KeyValuePairLit

func (*Parser) ParseMixedTextStmt 

func (p *Parser) ParseMixedTextStmt() (t *node.MixedTextStmt)

func (*Parser) ParseMixedValue 

func (p *Parser) ParseMixedValue() (ett *node.MixedValueStmt)

func (*Parser) ParseNullishSelector 

func (p *Parser) ParseNullishSelector(x node.Expr) (sel node.Expr)

func (*Parser) ParseOperand 

func (p *Parser) ParseOperand() node.Expr

func (*Parser) ParseParamDecl 

func (p *Parser) ParseParamDecl() (d *node.GenDecl)

func (*Parser) ParseParamSpec 

func (p *Parser) ParseParamSpec(keyword token.Token, multi bool, prev []node.Spec, i int) (spec node.Spec)

func (*Parser) ParseParemExpr 

func (p *Parser) ParseParemExpr(lparenToken, rparenToken token.Token) node.ToMultiParenConverter

func (*Parser) ParsePipe 

func (p *Parser) ParsePipe(x node.Expr) node.Expr

func (*Parser) ParsePrimaryExpr 

func (p *Parser) ParsePrimaryExpr() node.Expr

func (*Parser) ParsePrimitiveOperand 

func (p *Parser) ParsePrimitiveOperand() node.Expr

func (*Parser) ParseRawHeredocLit 

func (p *Parser) ParseRawHeredocLit() (t *node.RawHeredocLit)

func (*Parser) ParseRawStringLit 

func (p *Parser) ParseRawStringLit() (t *node.RawStringLit)

func (*Parser) ParseReturn 

func (p *Parser) ParseReturn() (ret node.Return)

func (*Parser) ParseReturnExpr 

func (p *Parser) ParseReturnExpr() node.Expr

func (*Parser) ParseReturnStmt 

func (p *Parser) ParseReturnStmt() node.Stmt

func (*Parser) ParseScopedBlockStmt 

func (p *Parser) ParseScopedBlockStmt() *node.BlockStmt

func (*Parser) ParseSelector 

func (p *Parser) ParseSelector(x node.Expr) (sel node.Expr)

func (*Parser) ParseSelectorNode 

func (p *Parser) ParseSelectorNode(x node.Expr) (expr, sel node.Expr)

func (*Parser) ParseSimpleStmt 

func (p *Parser) ParseSimpleStmt(forIn bool) node.Stmt

func (*Parser) ParseStmt 

func (p *Parser) ParseStmt() (stmt node.Stmt)

func (*Parser) ParseStmtList 

func (p *Parser) ParseStmtList(end ...token.Token) (list []node.Stmt)

func (*Parser) ParseStringLit 

func (p *Parser) ParseStringLit() *node.StringLit

func (*Parser) ParseThrowExpr 

func (p *Parser) ParseThrowExpr() *node.ThrowExpr

func (*Parser) ParseThrowStmt 

func (p *Parser) ParseThrowStmt() node.Stmt

func (*Parser) ParseTryStmt 

func (p *Parser) ParseTryStmt() node.Stmt

func (*Parser) ParseType 

func (p *Parser) ParseType() (idents []*node.IdentExpr)

func (*Parser) ParseTypedIdent 

func (p *Parser) ParseTypedIdent() *node.TypedIdentExpr

func (*Parser) ParseUnaryExpr 

func (p *Parser) ParseUnaryExpr() node.Expr

func (*Parser) ParseValueSpec 

func (p *Parser) ParseValueSpec(keyword token.Token, multi bool, _ []node.Spec, i int) node.Spec

func (*Parser) PrintTrace 

func (p *Parser) PrintTrace(a ...any)

func (*Parser) SkipSpace 

func (p *Parser) SkipSpace()

func (*Parser) WrapPostScan 

func (p *Parser) WrapPostScan(f func(in func(p *Parser)) func(p *Parser))

type ParserOptions 

type ParserOptions struct {
	Trace io.Writer
	Mode  Mode
}

type ScanMode 

type ScanMode uint8

ScanMode represents a scanner mode. 

const (
	ScanComments ScanMode = 1 << iota
	DontInsertSemis
	ScanMixed
	ScanConfigDisabled
	ScanMixedExprAsValue
	ScanFloatAsDecimal
	ScanCharAsString
)

List of scanner modes.

func (*ScanMode) Clear 

func (b *ScanMode) Clear(flag ScanMode) *ScanMode

func (ScanMode) Has 

func (b ScanMode) Has(flag ScanMode) bool

func (*ScanMode) Set 

func (b *ScanMode) Set(flag ScanMode) *ScanMode

func (*ScanMode) Toggle 

func (b *ScanMode) Toggle(flag ScanMode) *ScanMode

type Scanner 

type Scanner struct {
	Handlers
	source.Reader
	MixedDelimiter MixedDelimiter // the mixed delimiters

	InsertSemi bool // insert a semicolon before next newline

	InCode             bool
	ToText             bool
	BraceCount         int
	BreacksCount       int
	ParenCount         int
	TokenPool          TokenPool
	SkipWhitespaceFunc func(s *Scanner)
	HandleMixed        func(textStart *int, rt func() *Token)
	EOF                *Token
	// contains filtered or unexported fields
}

Scanner reads the Gad source text. It's based on ToInterface's scanner implementation.

func NewScanner 

func NewScanner(
	file *source.File,
	opts *ScannerOptions,
) *Scanner

NewScanner creates a Scanner.

func (*Scanner) AddNextToken 

func (s *Scanner) AddNextToken(n ...Token) (r *Token)

func (*Scanner) AddNextTokenPtr 

func (s *Scanner) AddNextTokenPtr(n ...*Token) (r *Token)

func (*Scanner) Clone 

func (s *Scanner) Clone() (c *Scanner)

func (*Scanner) GetMixedDelimiter 

func (s *Scanner) GetMixedDelimiter() *MixedDelimiter

func (*Scanner) List 

func (s *Scanner) List() (ret []Token)

func (*Scanner) MixedCodeEnds 

func (s *Scanner) MixedCodeEnds(offsetInc int) bool

func (*Scanner) Mode 

func (s *Scanner) Mode() ScanMode

func (*Scanner) ModeP 

func (s *Scanner) ModeP() *ScanMode

func (Scanner) PeekScan 

func (s Scanner) PeekScan() (t Token)

func (*Scanner) Scan 

func (s *Scanner) Scan() (t Token)

func (*Scanner) ScanCodeBlock 

func (s *Scanner) ScanCodeBlock(leftText *Token) (code Token)

func (*Scanner) ScanComment 

func (s *Scanner) ScanComment() string

func (*Scanner) ScanIdentifier 

func (s *Scanner) ScanIdentifier() string

func (*Scanner) ScanNow 

func (s *Scanner) ScanNow() (t Token)

ScanNow returns a token, token literal and its position.

func (*Scanner) ScanNumber 

func (s *Scanner) ScanNumber(seenDecimalPoint bool) (tok token.Token, lit string)

func (*Scanner) SetMode 

func (s *Scanner) SetMode(m ScanMode)

func (*Scanner) Switch2 

func (s *Scanner) Switch2(tok0, tok1 token.Token) token.Token

func (*Scanner) Switch3 

func (s *Scanner) Switch3(
	tok0, tok1 token.Token,
	ch2 rune,
	tok2 token.Token,
) token.Token

func (*Scanner) Switch4 

func (s *Scanner) Switch4(
	tok0, tok1 token.Token,
	ch2 rune,
	tok2, tok3 token.Token,
) token.Token

type ScannerInterface 

type ScannerInterface interface {
	Scan() (t Token)
	Mode() ScanMode
	SetMode(m ScanMode)
	SourceFile() *source.File
	Source() []byte
	ErrorHandler(h ...source.ScannerErrorHandler)
	GetMixedDelimiter() *MixedDelimiter
}

type ScannerOptions 

type ScannerOptions struct {
	Mode           ScanMode
	MixedDelimiter MixedDelimiter
}

type TextFlag 

type TextFlag uint8

TextFlag represents a text flag. 

const (
	TrimLeft TextFlag = 1 << iota
	TrimRight
)

List of scanner modes.

func (*TextFlag) Clear 

func (b *TextFlag) Clear(flag TextFlag) *TextFlag

func (TextFlag) Has 

func (b TextFlag) Has(flag TextFlag) bool

func (*TextFlag) Set 

func (b *TextFlag) Set(flag TextFlag) *TextFlag

func (TextFlag) String 

func (b TextFlag) String() (s string)

func (*TextFlag) Toggle 

func (b *TextFlag) Toggle(flag TextFlag) *TextFlag

type Token 

type Token struct {
	Pos        source.Pos
	Token      token.Token
	Literal    string
	InsertSemi bool

	Prev []Token
	utils.Data
	// contains filtered or unexported fields
}

func (*Token) LiteralRemoveLinePrefix 

func (t *Token) LiteralRemoveLinePrefix(prefix string)

func (Token) String 

func (t Token) String() string

type TokenHandler 

type TokenHandler func(t *Token)

type TokenHandlers 

type TokenHandlers []TokenHandler

func (*TokenHandlers) Remove 

func (th *TokenHandlers) Remove(h TokenHandler)

type TokenPool 

type TokenPool []*Token

func (*TokenPool) Add 

func (p *TokenPool) Add(t ...*Token)

func (TokenPool) Empty 

func (p TokenPool) Empty() bool

func (TokenPool) Last 

func (p TokenPool) Last() (t *Token)

func (*TokenPool) Semi 

func (p *TokenPool) Semi()

func (*TokenPool) Shift 

func (p *TokenPool) Shift() (t *Token)

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.