tokenizer

package
v0.18.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 15, 2023 License: MIT Imports: 0 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func Stringify added in v0.18.1

func Stringify(tokens []*Token) string

Types

type Token

type Token struct {
	Type  TokenType
	Value string
}

func NewToken

func NewToken(tp, text string) *Token

func Tokenize added in v0.13.1

func Tokenize(text string) []*Token

func (*Token) String added in v0.18.1

func (t *Token) String() string

type TokenType

type TokenType = string
const (
	Underline          TokenType = "_"
	Asterisk           TokenType = "*"
	Hash               TokenType = "#"
	Backtick           TokenType = "`"
	LeftSquareBracket  TokenType = "["
	RightSquareBracket TokenType = "]"
	LeftParenthesis    TokenType = "("
	RightParenthesis   TokenType = ")"
	ExclamationMark    TokenType = "!"
	Tilde              TokenType = "~"
	Dash               TokenType = "-"
	GreaterThan        TokenType = ">"
	Newline            TokenType = "\n"
	Space              TokenType = " "
)
const (
	Text TokenType = ""
)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL