tokenizer

package

v2.0.2 Latest Latest Go to latest Published: Aug 21, 2025 License: MIT Imports: 3 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

git.sr.ht/~ser/godjot

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
Variables
func Assertf(condition bool, format string, args ...any)
func DefaultType[T ~int]() T
type AttributeEntry
type Attributes
- func NewAttributes(entries ...AttributeEntry) Attributes
- func (a *Attributes) Append(key, value string)
- func (a *Attributes) Entries() []AttributeEntry
- func (a *Attributes) Get(key string) string
- func (a *Attributes) GoMap() map[string]string
- func (a *Attributes) MergeWith(other Attributes)
- func (a *Attributes) Set(key, value string)
- func (a *Attributes) Size() int
- func (a *Attributes) TryGet(key string) (string, bool)
type ByteMask
- func NewByteMask(set []byte) ByteMask
- func Union(masks ...ByteMask) ByteMask
- func (m ByteMask) And(other ByteMask) ByteMask
- func (m *ByteMask) Has(b byte) bool
- func (m ByteMask) Negate() ByteMask
- func (m ByteMask) Or(other ByteMask) ByteMask
type LineTokenizer
- func (tokenizer *LineTokenizer) Scan() (start, end int, eof bool)
type Range
type Ranges
- func (rs *Ranges) Push(r Range)
type ReaderState
type TextReader
- func (r TextReader) ByteRepeat(s ReaderState, b byte, minCount int) (ReaderState, bool)
- func (r TextReader) EmptyOrWhiteSpace(s ReaderState) (ReaderState, bool)
- func (r TextReader) HasByte(s ReaderState, b byte) bool
- func (r TextReader) HasMask(s ReaderState, mask ByteMask) bool
- func (r TextReader) HasToken(s ReaderState, token string) bool
- func (r TextReader) HasToken1(s ReaderState, token [1]byte) bool
- func (r TextReader) HasToken2(s ReaderState, token [2]byte) bool
- func (r TextReader) HasToken3(s ReaderState, token [3]byte) bool
- func (r TextReader) IsEmpty(s ReaderState) bool
- func (r TextReader) IsEmptyOrWhiteSpace(s ReaderState) bool
- func (r TextReader) Mask(s ReaderState, mask ByteMask) (ReaderState, bool)
- func (r TextReader) MaskRepeat(s ReaderState, mask ByteMask, minCount int) (ReaderState, bool)
- func (r TextReader) Peek(s ReaderState) (byte, bool)
- func (r TextReader) Select(start, end ReaderState) string
- func (r TextReader) Token(s ReaderState, token string) (ReaderState, bool)
- func (r TextReader) Token1(s ReaderState, token [1]byte) (ReaderState, bool)
- func (r TextReader) Token2(s ReaderState, token [2]byte) (ReaderState, bool)
- func (r TextReader) Token3(s ReaderState, token [3]byte) (ReaderState, bool)
type Token
- func (t Token[T]) Bytes(document []byte) []byte
- func (t Token[T]) IsDefault() bool
- func (t Token[T]) Length() int
- func (t Token[T]) PrefixLength(document []byte, b byte) int
type TokenList
- func (l *TokenList[T]) FillUntil(position int, tokenType T)
- func (l *TokenList[T]) FirstOrDefault() *Token[T]
- func (l TokenList[T]) GoString() string
- func (l *TokenList[T]) LastOrDefault() *Token[T]
- func (l *TokenList[T]) Push(token Token[T])
type TokenStack
- func NewTokenStack[T ~int]() TokenStack[T]
- func (s *TokenStack[T]) CloseLevelAt(token Token[T])
- func (s *TokenStack[T]) Empty() bool
- func (s *TokenStack[T]) LastLevel() *TokenList[T]
- func (s *TokenStack[T]) OpenLevelAt(token Token[T])
- func (s *TokenStack[T]) PopCommit()
- func (s *TokenStack[T]) PopForget()
- func (s *TokenStack[T]) PopForgetUntil(tokenType T) bool

Constants ¶

View Source

const Open = 1

Variables ¶

View Source

var (
	SpaceByteMask        = NewByteMask([]byte(" \t"))
	SpaceNewLineByteMask = NewByteMask([]byte(" \t\r\n"))
)

Functions ¶

func Assertf ¶

func Assertf(condition bool, format string, args ...any)

func DefaultType ¶

func DefaultType[T ~int]() T

Types ¶

type AttributeEntry ¶

type AttributeEntry struct{ Key, Value string }

type Attributes ¶

type Attributes struct {
	Keys []string
	Map  map[string]string
}

Attributes - is a map with support for deterministic order of enumeration

func NewAttributes ¶

func NewAttributes(entries ...AttributeEntry) Attributes

func (*Attributes) Append ¶

func (a *Attributes) Append(key, value string)

func (*Attributes) Entries ¶

func (a *Attributes) Entries() []AttributeEntry

func (*Attributes) Get ¶

func (a *Attributes) Get(key string) string

func (*Attributes) GoMap ¶

func (a *Attributes) GoMap() map[string]string

func (*Attributes) MergeWith ¶

func (a *Attributes) MergeWith(other Attributes)

func (*Attributes) Set ¶

func (a *Attributes) Set(key, value string)

func (*Attributes) Size ¶

func (a *Attributes) Size() int

func (*Attributes) TryGet ¶

func (a *Attributes) TryGet(key string) (string, bool)

type ByteMask ¶

type ByteMask [4]uint64

func NewByteMask ¶

func NewByteMask(set []byte) ByteMask

func Union ¶

func Union(masks ...ByteMask) ByteMask

func (ByteMask) And ¶

func (m ByteMask) And(other ByteMask) ByteMask

func (*ByteMask) Has ¶

func (m *ByteMask) Has(b byte) bool

func (ByteMask) Negate ¶

func (m ByteMask) Negate() ByteMask

func (ByteMask) Or ¶

func (m ByteMask) Or(other ByteMask) ByteMask

type LineTokenizer ¶

type LineTokenizer struct {
	Document []byte
	// contains filtered or unexported fields
}

func (*LineTokenizer) Scan ¶

func (tokenizer *LineTokenizer) Scan() (start, end int, eof bool)

type Range ¶

type Range struct{ Start, End int }

type Ranges ¶

type Ranges []Range

func (*Ranges) Push ¶

func (rs *Ranges) Push(r Range)

type ReaderState ¶

type ReaderState = int

type TextReader ¶

type TextReader []byte

func (TextReader) ByteRepeat ¶

func (r TextReader) ByteRepeat(s ReaderState, b byte, minCount int) (ReaderState, bool)

func (TextReader) EmptyOrWhiteSpace ¶

func (r TextReader) EmptyOrWhiteSpace(s ReaderState) (ReaderState, bool)

func (TextReader) HasByte ¶

func (r TextReader) HasByte(s ReaderState, b byte) bool

func (TextReader) HasMask ¶

func (r TextReader) HasMask(s ReaderState, mask ByteMask) bool

func (TextReader) HasToken ¶

func (r TextReader) HasToken(s ReaderState, token string) bool

func (TextReader) HasToken1 ¶

func (r TextReader) HasToken1(s ReaderState, token [1]byte) bool

func (TextReader) HasToken2 ¶

func (r TextReader) HasToken2(s ReaderState, token [2]byte) bool

func (TextReader) HasToken3 ¶

func (r TextReader) HasToken3(s ReaderState, token [3]byte) bool

func (TextReader) IsEmpty ¶

func (r TextReader) IsEmpty(s ReaderState) bool

func (TextReader) IsEmptyOrWhiteSpace ¶

func (r TextReader) IsEmptyOrWhiteSpace(s ReaderState) bool

func (TextReader) Mask ¶

func (r TextReader) Mask(s ReaderState, mask ByteMask) (ReaderState, bool)

func (TextReader) MaskRepeat ¶

func (r TextReader) MaskRepeat(s ReaderState, mask ByteMask, minCount int) (ReaderState, bool)

func (TextReader) Peek ¶

func (r TextReader) Peek(s ReaderState) (byte, bool)

func (TextReader) Select ¶

func (r TextReader) Select(start, end ReaderState) string

func (TextReader) Token ¶

func (r TextReader) Token(s ReaderState, token string) (ReaderState, bool)

func (TextReader) Token1 ¶

func (r TextReader) Token1(s ReaderState, token [1]byte) (ReaderState, bool)

func (TextReader) Token2 ¶

func (r TextReader) Token2(s ReaderState, token [2]byte) (ReaderState, bool)

func (TextReader) Token3 ¶

func (r TextReader) Token3(s ReaderState, token [3]byte) (ReaderState, bool)

type Token ¶

type Token[T comparable] struct {
	Type       T
	JumpToPair int
	Start, End int
	Attributes Attributes
}

func (Token[T]) Bytes ¶

func (t Token[T]) Bytes(document []byte) []byte

func (Token[T]) IsDefault ¶

func (t Token[T]) IsDefault() bool

func (Token[T]) Length ¶

func (t Token[T]) Length() int

func (Token[T]) PrefixLength ¶

func (t Token[T]) PrefixLength(document []byte, b byte) int

type TokenList ¶

type TokenList[T ~int] []Token[T]

func (*TokenList[T]) FillUntil ¶

func (l *TokenList[T]) FillUntil(position int, tokenType T)

func (*TokenList[T]) FirstOrDefault ¶

func (l *TokenList[T]) FirstOrDefault() *Token[T]

func (TokenList[T]) GoString ¶

func (l TokenList[T]) GoString() string

func (*TokenList[T]) LastOrDefault ¶

func (l *TokenList[T]) LastOrDefault() *Token[T]

func (*TokenList[T]) Push ¶

func (l *TokenList[T]) Push(token Token[T])

type TokenStack ¶

type TokenStack[T ~int] struct {
	Levels     []TokenList[T]
	TypeLevels map[T][]int
}

func NewTokenStack ¶

func NewTokenStack[T ~int]() TokenStack[T]

func (*TokenStack[T]) CloseLevelAt ¶

func (s *TokenStack[T]) CloseLevelAt(token Token[T])

func (*TokenStack[T]) Empty ¶

func (s *TokenStack[T]) Empty() bool

func (*TokenStack[T]) LastLevel ¶

func (s *TokenStack[T]) LastLevel() *TokenList[T]

func (*TokenStack[T]) OpenLevelAt ¶

func (s *TokenStack[T]) OpenLevelAt(token Token[T])

func (*TokenStack[T]) PopCommit ¶

func (s *TokenStack[T]) PopCommit()

func (*TokenStack[T]) PopForget ¶

func (s *TokenStack[T]) PopForget()

func (*TokenStack[T]) PopForgetUntil ¶

func (s *TokenStack[T]) PopForgetUntil(tokenType T) bool

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL