token

package
v0.68.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 6, 2026 License: Apache-2.0 Imports: 11 Imported by: 0

Documentation

Index

Constants

View Source
const (
	TableEntrySize = unsafe.Sizeof(TableEntry{}) + unsafe.Sizeof(&TableEntry{})
	FieldDataSize  = unsafe.Sizeof(FieldData{}) + unsafe.Sizeof(&FieldData{})
)
View Source
const CacheKeyTable = 1

Variables

This section is empty.

Functions

This section is empty.

Types

type Block

type Block struct {
	Payload []byte
	Offsets []uint32
}

func (*Block) GetToken

func (b *Block) GetToken(index int) []byte

func (*Block) Len

func (b *Block) Len() int

func (Block) Pack

func (b Block) Pack(dst []byte) []byte

func (*Block) Size

func (b *Block) Size() int

func (*Block) Unpack

func (b *Block) Unpack(data []byte) error

type BlockLoader

type BlockLoader struct {
	// contains filtered or unexported fields
}

BlockLoader is responsible for Reading from disk, unpacking and caching tokens blocks. NOT THREAD SAFE. Do not use concurrently. Use your own BlockLoader instance for each search query

func NewBlockLoader

func NewBlockLoader(fracName string, reader *storage.IndexReader, c *cache.Cache[*Block]) *BlockLoader

func (*BlockLoader) Load

func (l *BlockLoader) Load(index uint32) *Block

type FieldData

type FieldData struct {
	MinVal  string
	Entries []*TableEntry // expect that TableEntry are necessarily ordered by StartTID here
}

type FieldTable

type FieldTable struct {
	Field   string
	Entries []*TableEntry // expect that TableEntry are necessarily ordered by StartTID here
}

type Provider

type Provider struct {
	// contains filtered or unexported fields
}

func NewProvider

func NewProvider(loader *BlockLoader, entries []*TableEntry) *Provider

func (*Provider) FirstTID

func (tp *Provider) FirstTID() uint32

func (*Provider) GetToken

func (tp *Provider) GetToken(tid uint32) []byte

func (*Provider) LastTID

func (tp *Provider) LastTID() uint32

func (*Provider) Ordered

func (tp *Provider) Ordered() bool

type Table

type Table map[string]*FieldData

func TableFromBlocks

func TableFromBlocks(blocks []TableBlock) Table

func (Table) GetEntryByTID

func (t Table) GetEntryByTID(tid uint32) *TableEntry

func (Table) SelectEntries

func (t Table) SelectEntries(field, hint string) []*TableEntry

SelectEntries returns monotonic and continuous sequence of token table entries

func (Table) Size

func (t Table) Size() int

Size calculates a very approximate amount of memory occupied

type TableBlock

type TableBlock struct {
	FieldsTables []FieldTable
}

TableBlock represents how token.Table is stored on disk

func (TableBlock) Pack

func (b TableBlock) Pack(buf []byte) []byte

func (*TableBlock) Unpack

func (b *TableBlock) Unpack(data []byte)

type TableEntry

type TableEntry struct {
	StartIndex uint32 // offset from the beginning of the block to the first token pointed to by the TableEntry
	StartTID   uint32 // first TID of TableEntry
	BlockIndex uint32 // sequence number of the physical block of tokens in the file
	ValCount   uint32

	MinVal string // only saved for the first entry in block
	MaxVal string
}

TableEntry is part of token.Table and points to a fragment of token.Block

func (*TableEntry) GetIndexInTokensBlock

func (t *TableEntry) GetIndexInTokensBlock(tid uint32) int

type TableLoader

type TableLoader struct {
	// contains filtered or unexported fields
}

func NewTableLoader

func NewTableLoader(fracName string, reader *storage.IndexReader, c *cache.Cache[Table]) *TableLoader

func (*TableLoader) Load

func (l *TableLoader) Load() Table

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL