Documentation
¶
Overview ¶
Package token provides tokenization support for Tony and related formats.
Tokenize is a function for tokenizing bytes.
Balance provides tree structure discovery based on indentation and normalizes the token sequence so that it is context free.
Index ¶
- Constants
- Variables
- func ExpectedErr(what string, p *Pos) error
- func LeadingZeroErr(pos *Pos) error
- func NeedsQuote(v string) bool
- func PrintTokens(toks []Token, msg string)
- func Quote(v string, autoSingle bool) string
- func QuotedToString(d []byte) string
- func UnexpectedErr(what string, p *Pos) error
- func Unquote(v string) (string, error)
- type ErrImbalancedStructure
- type Pos
- type PosDoc
- type Token
- type TokenOpt
- type TokenType
- type TokenizeErr
Constants ¶
View Source
const ( MLitChomp = '-' MLitKeep = '+' )
View Source
const ( TIndent = iota TInteger TFloat TColon TArrayElt TDocSep TComment TNull TTrue TFalse TTag TString TMString TLiteral TMLit TMergeKey TLCurl TRCurl TLSquare TRSquare TComma )
Variables ¶
View Source
var ( ErrBadUTF8 = errors.New("bad utf8") ErrUnterminated = errors.New("unterminated") ErrNumberLeadingZero = errors.New("leading zero") ErrNoIndent = errors.New("indentation needed") ErrDocBalance = errors.New("imbalanced document") ErrLiteral = errors.New("bad literal") ErrBadEscape = errors.New("bad escape") ErrBadUnicode = errors.New("bad unicode") ErrUnicodeControl = errors.New("unicode control") ErrMalformedMLit = errors.New("malformed multiline literal") ErrColonSpace = errors.New("colon should be followed by space") ErrEmptyDoc = errors.New("empty document") ErrMultilineString = errors.New("multiline string") ErrYAMLDoubleQuote = errors.New("yaml double quote") ErrMLitPlacement = errors.New("bad placement of |") ErrYAMLPlain = errors.New("yaml plain string") ErrUnsupported = errors.New("unsupported") ErrNumber = errors.New("number") )
Functions ¶
func ExpectedErr ¶
func LeadingZeroErr ¶
func NeedsQuote ¶
func PrintTokens ¶
func QuotedToString ¶
func UnexpectedErr ¶
Types ¶
type ErrImbalancedStructure ¶
type ErrImbalancedStructure struct {
Open, Close *Token
}
func (*ErrImbalancedStructure) Error ¶
func (i *ErrImbalancedStructure) Error() string
func (*ErrImbalancedStructure) Unwrap ¶
func (i *ErrImbalancedStructure) Unwrap() error
type TokenizeErr ¶
func NewTokenizeErr ¶
func NewTokenizeErr(e error, p *Pos) *TokenizeErr
func (*TokenizeErr) Error ¶
func (e *TokenizeErr) Error() string
func (*TokenizeErr) Unwrap ¶
func (t *TokenizeErr) Unwrap() error
Click to show internal directories.
Click to hide internal directories.