Documentation
¶
Overview ¶
Package token defines the lexical tokens of Kukel (syntax.ebnf, section 4).
Index ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Kind ¶
type Kind int
Kind is the kind of a token.
const ( ILLEGAL Kind = iota EOF COMMENT // only from a lexer with KeepComments (kukel fmt) // Literals and names. IDENT // main INT // 42, 0xFF, 0b1010 FLOAT // 1.5, .5, 2e10 CHAR // 'a' STRING // "abc" // Operators and punctuation. PLUS // + MINUS // - STAR // * SLASH // / PERCENT // % AMP // & PIPE // | CARET // ^ TILDE // ~ BANG // ! LT // < GT // > ASSIGN // = QUESTION // ? COLON // : COMMA // , SEMICOLON // ; LPAREN // ( RPAREN // ) LBRACK // [ RBRACK // ] LBRACE // { RBRACE // } LE // <= GE // >= EQ // == NE // != SHL // << SHR // >> ANDAND // && OROR // || INC // ++ DEC // -- ARROW // -> SCOPE // :: DOTDOT // .. ADD_ASSIGN // += SUB_ASSIGN // -= MUL_ASSIGN // *= DIV_ASSIGN // /= MOD_ASSIGN // %= AND_ASSIGN // &= OR_ASSIGN // |= XOR_ASSIGN // ^= SHL_ASSIGN // <<= SHR_ASSIGN // >>= BREAK CASE CATCH CLASS CONST CONTINUE DEFAULT DO ELSE FALSE FINALLY FN FOR GLOBAL IF IN LOCAL NEW NIL RETURN STATIC SWITCH THIS THROW TRUE TRY USE WHILE WITH YIELD )
type Pos ¶
type Pos struct {
File string
Line int
Col int
Offset int // byte offset in the (NUL-filtered) source
}
Pos is a source position. Line and Col are 1-based; Col counts bytes.
type Token ¶
type Token struct {
Kind Kind
Pos Pos
Text string // the source text of the token
// Values of literals:
// INT: Int holds the 64-bit pattern. IntMin is set for the decimal
// literal 9223372036854775808 (2^63), which is valid only
// directly after a unary minus (the parser checks that).
// CHAR: Int holds the character code.
// FLOAT: Float.
// STRING: Str holds the decoded bytes.
Int uint64
IntMin bool
Float float64
Str string
}
Token is one lexical token.
Source Files
Click to show internal directories.
Click to hide internal directories.