textindex

package
v1.5.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 9, 2026 License: Apache-2.0 Imports: 26 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	ArrayContainerMaxSize = 4096
	BitSetContainerSize   = (1 << 16) / 64
)
View Source 
const (
	BitsetContainerType uint8 = 1
	ArrayContainerType  uint8 = 2
	RunContainerType    uint8 = 3
)

Variables

This section is empty.

Functions

func AddPostingToMem 

func AddPostingToMem(memElement *InvertMemElement, key []byte, rowId uint32) bool

func DecodeOffs 

func DecodeOffs(offs []uint32, i int) (start, end uint32)

func FreeFullTextIndexBuilder 

func FreeFullTextIndexBuilder(builder *FullTextIndexBuilder)

func GetPartHeaderSize 

func GetPartHeaderSize() int

func PutBlockData 

func PutBlockData(bd *BlockData)

func PutBlockReader 

func PutBlockReader(bh *BlockHeader)

func PutInvertMemElement 

func PutInvertMemElement(ele *InvertMemElement)

func PutPartHeader 

func PutPartHeader(ph *PartHeader)

func RetrievePostingList 

func RetrievePostingList(memElement *InvertMemElement, data *BlockData, bh *BlockHeader) bool

res[0]: first token start-offset res[1]: first token end-offset res[2]: last token start-offset res[3]: last token end-offset res[4]: keys size res[5]: keys total size = keys size + keys offs size res[6]: data size res[7]: data total size = data size + data offs size res[8]: items count

func TravelDirection 

func TravelDirection(a, b RunPair) int

direct = 1 : step b direct = -1 : step a direct = 0 : step a and b

func UnarshalUint32Slice 

func UnarshalUint32Slice(src []byte) []uint32

Types

type ArrayContainer 

type ArrayContainer struct {
	ContainerHead
	Rows []uint16
}

func NewArrayContainer 

func NewArrayContainer(head *ContainerHead) *ArrayContainer

func NewArrayContainerSize 

func NewArrayContainerSize(head *ContainerHead, size int) *ArrayContainer

func (*ArrayContainer) And 

func (c *ArrayContainer) And(a Container) Container

func (*ArrayContainer) GetCounts added in v1.5.0 

func (c *ArrayContainer) GetCounts(rowIds []uint32, counts []int64)

func (*ArrayContainer) Key 

func (c *ArrayContainer) Key() uint16

func (*ArrayContainer) Serialize 

func (c *ArrayContainer) Serialize() []uint32

func (*ArrayContainer) Unmarshal 

func (c *ArrayContainer) Unmarshal(src []byte) ([]byte, error)

type BitsetContainer 

type BitsetContainer struct {
	ContainerHead
	Bitset []uint64 // len is 1024
}

func NewBitsetContainer 

func NewBitsetContainer(head *ContainerHead) *BitsetContainer

func (*BitsetContainer) And 

func (c *BitsetContainer) And(a Container) Container

func (*BitsetContainer) GetCounts added in v1.5.0 

func (c *BitsetContainer) GetCounts(rowIds []uint32, counts []int64)

func (*BitsetContainer) Key 

func (c *BitsetContainer) Key() uint16

func (*BitsetContainer) Serialize 

func (c *BitsetContainer) Serialize() []uint32

func (*BitsetContainer) SetBitsetRange 

func (c *BitsetContainer) SetBitsetRange(start int, end int)

func (*BitsetContainer) Unmarshal 

func (c *BitsetContainer) Unmarshal(src []byte) ([]byte, error)

type BlockData 

type BlockData struct {
	Keys       []byte
	PackedKeys []byte
	Data       []byte
	PackedData []byte
}

func GetBlockData 

func GetBlockData() *BlockData

func NewBlockData 

func NewBlockData() *BlockData

func (*BlockData) GrowPackedBuf 

func (d *BlockData) GrowPackedBuf(keyDstLen, dataDstLen int)

func (*BlockData) Reset 

func (data *BlockData) Reset()

type BlockDataReader 

type BlockDataReader struct {
	// contains filtered or unexported fields
}

func NewBlockDataReader 

func NewBlockDataReader(path, file string, obsOpts *obs.ObsOptions) *BlockDataReader

func (*BlockDataReader) Close 

func (r *BlockDataReader) Close() error

func (*BlockDataReader) GetBlockReadData 

func (r *BlockDataReader) GetBlockReadData(bh *BlockHeader) (*BlockReadData, error)

func (*BlockDataReader) Open 

func (r *BlockDataReader) Open() error

type BlockHeadReader 

type BlockHeadReader struct {
	// contains filtered or unexported fields
}

read the BlockHeaders

func NewBlockReader 

func NewBlockReader(path, file string, obsOpts *obs.ObsOptions) *BlockHeadReader

func (*BlockHeadReader) ReadBlockHeaders 

func (r *BlockHeadReader) ReadBlockHeaders(partHeader *PartHeader) ([]BlockHeader, error)

type BlockHeader 

type BlockHeader struct {
	FirstItem   []byte
	LastItem    []byte
	MarshalType uint8
	ItemsCount  uint32
	// keys info
	KeysOffset     uint64
	KeysUnpackSize uint32 // Uncompressed length(keys+keysOffs)
	KeysPackSize   uint32 // Compressed length(keys+keysOffs)
	KeysSize       uint32 // Uncompressed length(only keys)
	// posting list info
	PostOffset     uint64
	PostUnpackSize uint32 // Uncompressed length(posts+postsOffs)
	PostPackSize   uint32 // Compressed length(posts+postsOffs)
	PostSize       uint32 // Uncompressed length(only posts)
}

func GetBlockReader 

func GetBlockReader() *BlockHeader

func NewBlockHeader 

func NewBlockHeader() *BlockHeader

func (*BlockHeader) Marshal 

func (bh *BlockHeader) Marshal(dst []byte) []byte

func (*BlockHeader) Reset 

func (bh *BlockHeader) Reset()

func (*BlockHeader) Unmarshal 

func (bh *BlockHeader) Unmarshal(src []byte) ([]byte, error)

type BlockReadData 

type BlockReadData struct {
	// contains filtered or unexported fields
}

func NewBlockReadData 

func NewBlockReadData() *BlockReadData

func (*BlockReadData) DecompressKeyData 

func (bd *BlockReadData) DecompressKeyData() error

func (*BlockReadData) DecompressPostData 

func (bd *BlockReadData) DecompressPostData() error

func (*BlockReadData) Init 

func (bd *BlockReadData) Init(bh *BlockHeader)

func (*BlockReadData) Query 

func (bd *BlockReadData) Query(queryStr string) ([]Container, error)

func (*BlockReadData) ResizePackData 

func (bd *BlockReadData) ResizePackData(size int)

func (*BlockReadData) ResizeUnackPostData 

func (bd *BlockReadData) ResizeUnackPostData(size int)

func (*BlockReadData) ResizeUnpackKeyData 

func (bd *BlockReadData) ResizeUnpackKeyData(size int)

type Container 

type Container interface {
	Key() uint16
	And(a Container) Container
	Unmarshal(src []byte) ([]byte, error)
	Serialize() []uint32
	GetCounts(rowIds []uint32, counts []int64)
}

func IntersectContainers 

func IntersectContainers(cntrs0, cntrs1 []Container) []Container

func UnmarshalContainer 

func UnmarshalContainer(src []byte) ([]Container, error)

type ContainerHead 

type ContainerHead struct {
	ContainerType uint8
	ContainerKey  uint16
	Cardinality   uint32
}

func (*ContainerHead) Unmarshal 

func (h *ContainerHead) Unmarshal(src []byte) ([]byte, error)

type FieldWriter added in v1.5.0 

type FieldWriter struct {
	// contains filtered or unexported fields
}

type FullTextIndexBuilder 

type FullTextIndexBuilder struct {
	// contains filtered or unexported fields
}

func NewFullTextIndexBuilder 

func NewFullTextIndexBuilder(splitChars string, hasChin bool) *FullTextIndexBuilder

func (*FullTextIndexBuilder) AddDocument 

func (b *FullTextIndexBuilder) AddDocument(val []byte, offset []uint32, startRow int, endRow int) (*InvertMemElement, error)

type GroupContainers 

type GroupContainers []Container

func (GroupContainers) TransToCounts added in v1.5.0 

func (cntrs GroupContainers) TransToCounts(header *PartHeader) []int64

func (GroupContainers) TransToPostingContainer 

func (cntrs GroupContainers) TransToPostingContainer(header *PartHeader) *PostingContainer

type InvertMemElement 

type InvertMemElement struct {
	// contains filtered or unexported fields
}

func GetMemElement 

func GetMemElement(groupSize uint32) *InvertMemElement

type PartContainer 

type PartContainer struct {
	// contains filtered or unexported fields
}

func NewPartContainers 

func NewPartContainers() *PartContainer

func (*PartContainer) FilterByQuery 

func (p *PartContainer) FilterByQuery(queryStr string) []BlockHeader

type PartHeader 

type PartHeader struct {
	FirstItemLen      uint8
	LastItemLen       uint8
	FirstItem         []byte // cap is 8
	LastItem          []byte // cap is 8
	Flag              uint16 // Flag[0:1] -> msharsalType, Flag[2:15] -> reserved
	BlockHeaderCnt    uint32 // BlockHeadersCount
	BlockHeaderOffset uint64
	BlockHeaderSize   uint32 // BlockHeaders + SegmentRanges
	SegmentRangeCnt   uint32
	SegmentRange      []uint32 // cap is segmentCntInPart, record the starting RowID of each segment
}

Metandex is stored together with BlockHeaders for attach scenes. For detach scenes, it is stored as a separate file

func GetPartHeader 

func GetPartHeader() *PartHeader

func GetPartHeaders 

func GetPartHeaders(path, file, field string, obsOpts *obs.ObsOptions) ([]*PartHeader, error)

read the part headers

func NewPartHeader 

func NewPartHeader() *PartHeader

func (*PartHeader) Contain 

func (ph *PartHeader) Contain(queryStr string) bool

func (*PartHeader) Marshal 

func (ph *PartHeader) Marshal(dst []byte) []byte

func (*PartHeader) Reset 

func (ph *PartHeader) Reset()

func (*PartHeader) Size 

func (ph *PartHeader) Size() int

func (*PartHeader) Unmarshal 

func (ph *PartHeader) Unmarshal(src []byte) ([]byte, error)

func (*PartHeader) UpdateFirstItem 

func (ph *PartHeader) UpdateFirstItem(firstItem []byte)

func (*PartHeader) UpdateLastItem 

func (ph *PartHeader) UpdateLastItem(lastItem []byte)

func (*PartHeader) UpdateSegmentRange 

func (ph *PartHeader) UpdateSegmentRange(rowsPerSegment []int, segId int)

type PostingContainer 

type PostingContainer struct {
	SegCntrs []SegRowContainer // idx: segmentId-startSegmentIdOfPart  -> SegRowContainer
}

func NewPostingContainer 

func NewPostingContainer(size int) *PostingContainer

type RowRange 

type RowRange struct {
	// contains filtered or unexported fields
}

type RunContainer 

type RunContainer struct {
	ContainerHead
	Nruns uint16
	Runs  []RunPair
}

func NewRunContainer 

func NewRunContainer(head *ContainerHead) *RunContainer

func NewRunContainerSize 

func NewRunContainerSize(head *ContainerHead, size int) *RunContainer

func (*RunContainer) And 

func (c *RunContainer) And(a Container) Container

func (*RunContainer) GetCounts added in v1.5.0 

func (c *RunContainer) GetCounts(rowIds []uint32, counts []int64)

func (*RunContainer) Key 

func (c *RunContainer) Key() uint16

func (*RunContainer) Serialize 

func (c *RunContainer) Serialize() []uint32

func (*RunContainer) Unmarshal 

func (c *RunContainer) Unmarshal(src []byte) ([]byte, error)

type RunPair 

type RunPair struct {
	Value uint16
	Len   uint16
}

func IntersectRunPair 

func IntersectRunPair(a, b RunPair) (RunPair, int, bool)

func (*RunPair) Unmarshal 

func (p *RunPair) Unmarshal(src []byte) ([]byte, error)

type SegRowContainer 

type SegRowContainer struct {
	RowIds []uint32
}

type TextIndexFilterReader 

type TextIndexFilterReader struct {
	// contains filtered or unexported fields
}

func NewTextIndexFilterReader 

func NewTextIndexFilterReader(path, file, field string, obsOpts *obs.ObsOptions, opt *influxql.IndexOption) (*TextIndexFilterReader, error)

func (*TextIndexFilterReader) Close 

func (r *TextIndexFilterReader) Close() error

func (*TextIndexFilterReader) FilterByPostinglist 

func (r *TextIndexFilterReader) FilterByPostinglist(partContainer *PartContainer, queryStr string) ([]Container, error)

func (*TextIndexFilterReader) GetPartContainer 

func (r *TextIndexFilterReader) GetPartContainer(partId int) (*PartContainer, error)

func (*TextIndexFilterReader) GetRowCount added in v1.5.0 

func (r *TextIndexFilterReader) GetRowCount(segId uint32, queryStr string) (int64, error)

func (*TextIndexFilterReader) IsExist 

func (r *TextIndexFilterReader) IsExist(segId uint32, queryStr string, elem *rpn.SKRPNElement) (bool, error)

func (*TextIndexFilterReader) UnMatchphaseHandle added in v1.5.0 

func (r *TextIndexFilterReader) UnMatchphaseHandle(header *PartHeader, counts []int64, segOff int) bool

type TextIndexFilterReaders 

type TextIndexFilterReaders struct {
	// contains filtered or unexported fields
}

func NewTextIndexFilterReaders 

func NewTextIndexFilterReaders(path, file string, schemas record.Schemas, ir *influxql.IndexRelation) *TextIndexFilterReaders

func (*TextIndexFilterReaders) Close 

func (r *TextIndexFilterReaders) Close() error

func (*TextIndexFilterReaders) GetRowCount added in v1.5.0 

func (r *TextIndexFilterReaders) GetRowCount(blockId int64, elem *rpn.SKRPNElement) (int64, error)

func (*TextIndexFilterReaders) IsExist 

func (r *TextIndexFilterReaders) IsExist(blockId int64, elem *rpn.SKRPNElement) (bool, error)

func (*TextIndexFilterReaders) StartSpan 

func (r *TextIndexFilterReaders) StartSpan(span *tracing.Span)

type TextIndexReader 

type TextIndexReader struct {
	// contains filtered or unexported fields
}

func NewTextIndexReader 

func NewTextIndexReader(rpnExpr *rpn.RPNExpr, schema record.Schemas, option hybridqp.Options, isCache bool) (*TextIndexReader, error)

func (*TextIndexReader) Close 

func (r *TextIndexReader) Close() error

Close is used to close the SKFileReader

func (*TextIndexReader) GetFragmentRowCount added in v1.5.0 

func (r *TextIndexReader) GetFragmentRowCount(fragId uint32) (int64, error)

func (*TextIndexReader) MayBeInFragment 

func (r *TextIndexReader) MayBeInFragment(fragId uint32) (bool, error)

MayBeInFragment determines whether a fragment in a file meets the query condition.

func (*TextIndexReader) ReInit 

func (r *TextIndexReader) ReInit(file interface{}) error

ReInit is used to that a SKFileReader is reused among multiple files.

func (*TextIndexReader) StartSpan 

func (r *TextIndexReader) StartSpan(span *tracing.Span)

type TextIndexReaderCreator 

type TextIndexReaderCreator struct {
}

func (*TextIndexReaderCreator) CreateSKFileReader 

func (index *TextIndexReaderCreator) CreateSKFileReader(rpnExpr *rpn.RPNExpr, schema record.Schemas, option hybridqp.Options, isCache bool) (sparseindex.SKFileReader, error)

type TextIndexWriter 

type TextIndexWriter struct {
	// contains filtered or unexported fields
}

func NewTextIndexWriter 

func NewTextIndexWriter(dir, msName, filePath, lockPath string, tokens string) *TextIndexWriter

func (*TextIndexWriter) Close 

func (w *TextIndexWriter) Close() error

func (*TextIndexWriter) CloseIndexWriters 

func (w *TextIndexWriter) CloseIndexWriters(dataWriter, headWriter, partWriter *colstore.IndexWriter)

func (*TextIndexWriter) CreateAttachIndex 

func (w *TextIndexWriter) CreateAttachIndex(writeRec *record.Record, schemaIdx, rowsPerSegment []int) error

func (*TextIndexWriter) CreateDetachIndex 

func (w *TextIndexWriter) CreateDetachIndex(writeRec *record.Record, schemaIdx, rowsPerSegment []int, dataBuf [][]byte) ([][]byte, []string)

func (*TextIndexWriter) Files added in v1.5.0 

func (w *TextIndexWriter) Files() []string

func (*TextIndexWriter) Flush added in v1.5.0 

func (w *TextIndexWriter) Flush() error

func (*TextIndexWriter) GetIndexFileWriters 

func (w *TextIndexWriter) GetIndexFileWriters(field string) (*colstore.IndexWriter, *colstore.IndexWriter, *colstore.IndexWriter, error)

func (*TextIndexWriter) GetTextIndexFilePath 

func (w *TextIndexWriter) GetTextIndexFilePath(field string, fileType int) string

func (*TextIndexWriter) Open 

func (w *TextIndexWriter) Open()

type TsspFile 

type TsspFile interface {
	Name() string
	Path() string
}

type WritePartInfo 

type WritePartInfo struct {
	// contains filtered or unexported fields
}

func NewWritePartInfo 

func NewWritePartInfo(size int) *WritePartInfo

func (*WritePartInfo) GetRowRanges 

func (wp *WritePartInfo) GetRowRanges() RowRange

func (*WritePartInfo) SetMemElements 

func (wp *WritePartInfo) SetMemElements(memElements *InvertMemElement, seq int)

func (*WritePartInfo) SetResError 

func (wp *WritePartInfo) SetResError(err error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.