token

package
v0.0.9 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 1, 2025 License: Apache-2.0 Imports: 12 Imported by: 0

Documentation

Overview

Package token provides tokenization support for Tony and related formats.

Tokenize is a function for tokenizing bytes.

Balance provides tree structure discovery based on indentation and normalizes the token sequence so that it is context free.

Index

Constants

View Source
const (
	MLitChomp = '-'
	MLitKeep  = '+'
)
View Source
const (
	TIndent = iota
	TInteger
	TFloat
	TColon
	TArrayElt
	TDocSep
	TComment
	TNull
	TTrue
	TFalse
	TTag
	TString
	TMString
	TLiteral
	TMLit
	TMergeKey
	TLCurl
	TRCurl
	TLSquare
	TRSquare
	TComma
)

Variables

View Source
var (
	ErrBadUTF8           = errors.New("bad utf8")
	ErrUnterminated      = errors.New("unterminated")
	ErrNumberLeadingZero = errors.New("leading zero")
	ErrNoIndent          = errors.New("indentation needed")
	ErrDocBalance        = errors.New("imbalanced document")
	ErrLiteral           = errors.New("bad literal")
	ErrBadEscape         = errors.New("bad escape")
	ErrBadUnicode        = errors.New("bad unicode")
	ErrUnicodeControl    = errors.New("unicode control")
	ErrMalformedMLit     = errors.New("malformed multiline literal")
	ErrColonSpace        = errors.New("colon should be followed by space")
	ErrEmptyDoc          = errors.New("empty document")
	ErrMultilineString   = errors.New("multiline string")
	ErrYAMLDoubleQuote   = errors.New("yaml double quote")
	ErrMLitPlacement     = errors.New("bad placement of |")
	ErrYAMLPlain         = errors.New("yaml plain string")
	ErrUnsupported       = errors.New("unsupported")
	ErrNumber            = errors.New("number")
)

Functions

func ExpectedErr

func ExpectedErr(what string, p *Pos) error

func LeadingZeroErr

func LeadingZeroErr(pos *Pos) error

func NeedsQuote

func NeedsQuote(v string) bool

func PrintTokens

func PrintTokens(toks []Token, msg string)

func Quote

func Quote(v string, autoSingle bool) string

func QuotedToString

func QuotedToString(d []byte) string

func UnexpectedErr

func UnexpectedErr(what string, p *Pos) error

func Unquote

func Unquote(v string) (string, error)

Types

type ErrImbalancedStructure

type ErrImbalancedStructure struct {
	Open, Close *Token
}

func (*ErrImbalancedStructure) Error

func (i *ErrImbalancedStructure) Error() string

func (*ErrImbalancedStructure) Unwrap

func (i *ErrImbalancedStructure) Unwrap() error

type Pos

type Pos struct {
	I int
	D *PosDoc
}

func (*Pos) Col

func (p *Pos) Col() int

func (*Pos) Line

func (p *Pos) Line() int

func (*Pos) LineCol

func (p *Pos) LineCol() (int, int)

func (Pos) String

func (p Pos) String() string

type PosDoc

type PosDoc struct {
	// contains filtered or unexported fields
}

func (*PosDoc) LineCol

func (p *PosDoc) LineCol(off int) (int, int)

func (*PosDoc) Pos

func (d *PosDoc) Pos(i int) *Pos

type Token

type Token struct {
	Type  TokenType
	Pos   *Pos
	Bytes []byte
}

func Balance

func Balance(toks []Token, f format.Format) ([]Token, error)

func Tokenize

func Tokenize(dst []Token, src []byte, opts ...TokenOpt) ([]Token, error)

func YAMLQuotedString

func YAMLQuotedString(d []byte, pos *Pos) (*Token, int, error)

func (*Token) Info

func (t *Token) Info() string

func (*Token) String

func (t *Token) String() string

type TokenOpt

type TokenOpt func(*tokenOpts)

func TokenJSON

func TokenJSON() TokenOpt

func TokenTony

func TokenTony() TokenOpt

func TokenYAML

func TokenYAML() TokenOpt

type TokenType

type TokenType int

func (TokenType) String

func (t TokenType) String() string

type TokenizeErr

type TokenizeErr struct {
	Err error
	Pos Pos
}

func NewTokenizeErr

func NewTokenizeErr(e error, p *Pos) *TokenizeErr

func (*TokenizeErr) Error

func (e *TokenizeErr) Error() string

func (*TokenizeErr) Unwrap

func (t *TokenizeErr) Unwrap() error

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL