tokenizer

type SimpleTokenizer struct {
	Done bool
	Err  error
	// contains filtered or unexported fields
}

type Token struct {
	TokenBytes [1 + MAX_TOKEN_SIZE]byte
	TokenPos   int32
	BytePos    int32
}