readutil

package
v0.0.0-debug-20260702 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 2, 2026 License: Apache-2.0 Imports: 36 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	RangeLeftOpen = iota + math.MaxInt16
	RangeRightOpen
	RangeBothOpen
	PrefixRangeLeftOpen
	PrefixRangeRightOpen
	PrefixRangeBothOpen
)
View Source 
const (
	BatchPrefetchSize = 1000
)

Variables

View Source 
var ErrNoMore = moerr.NewInternalErrorNoCtx("no more")

Functions

func BuildEmptyRelData 

func BuildEmptyRelData() engine.RelData

func CompileFilterExpr 

func CompileFilterExpr(
	expr *plan.Expr,
	tableDef *plan.TableDef,
	fs fileservice.FileService,
) (
	fastFilterOp FastFilterOp,
	loadOp LoadOp,
	objectFilterOp ObjectFilterOp,
	blockFilterOp BlockFilterOp,
	seekOp SeekFirstBlockOp,
	canCompile bool,
	highSelectivityHint bool,
)

func CompileFilterExprs 

func CompileFilterExprs(
	exprs []*plan.Expr,
	tableDef *plan.TableDef,
	fs fileservice.FileService,
) (
	fastFilterOp FastFilterOp,
	loadOp LoadOp,
	objectFilterOp ObjectFilterOp,
	blockFilterOp BlockFilterOp,
	seekOp SeekFirstBlockOp,
	canCompile bool,
	highSelectivityHint bool,
)

func ConstructBlockPKFilter 

func ConstructBlockPKFilter(
	isFakePK bool,
	basePKFilter BasePKFilter,
	bf engine.MembershipFilter,
) (f objectio.BlockReadFilter, err error)

func ConstructInExpr 

func ConstructInExpr(
	ctx context.Context,
	colName string,
	colVec *vector.Vector,
) *plan.Expr

func EncodePrimaryKey 

func EncodePrimaryKey(v any, packer *types.Packer) []byte

func EncodePrimaryKeyVector 

func EncodePrimaryKeyVector(vec *vector.Vector, packer *types.Packer) (ret [][]byte)

func FastApplyDeletesByRowIds 

func FastApplyDeletesByRowIds(
	checkBid *objectio.Blockid,
	leftRows *[]int64,
	deletesMask *objectio.Bitmap,
	deletedRowIds []objectio.Rowid,
	isDeletedRowIdsSorted bool,
)

FastApplyDeletesByRowIds apply deleted RowIds on the leftRows or deletedMask if leftRows is nil. Note that this function is stable, means the relative order between elements in the leftRows would not change after applies. Example: [3,2,5,6,1,4,1] ==> after remove the even numbers ==> [3,5,1,1]

func FastApplyDeletesByRowOffsets 

func FastApplyDeletesByRowOffsets(
	leftRows *[]int64,
	deletedMask *objectio.Bitmap,
	offsets []int64,
)

FastApplyDeletesByRowOffsets apply deleted RowIds on the leftRows or deletedMask if leftRows is nil. Note that this function is stable, means the relative order between elements in the leftRows would not change after applies. Example: [3,2,5,6,1,4,1] ==> after remove the even numbers ==> [3,5,1,1]

func FilterObjects 

func FilterObjects(
	ctx context.Context,
	rangesParam engine.RangesParam,
	fastFilterOp FastFilterOp,
	loadOp LoadOp,
	objectFilterOp ObjectFilterOp,
	blockFilterOp BlockFilterOp,
	seekOp SeekFirstBlockOp,
	nextObjectFn func() (objectio.ObjectStats, error),
	latestObjects []objectio.ObjectStats,
	extraObjects []objectio.ObjectStats,
	outBlocks *objectio.BlockInfoSlice,
	highSelectivityHint bool,
	metaPrefetcher func(context.Context) bool,
	fs fileservice.FileService,
) (
	totalBlocks int,
	loadHit int,
	objectFilterTotal int,
	objectFilterHit int,
	blockFilterTotal int,
	blockFilterHit int,
	fastFilterTotal int,
	fastFilterHit int,
	err error,
)

func FilterTxnObjects 

func FilterTxnObjects(
	ctx context.Context,
	rangesParam engine.RangesParam,
	snapshotTS timestamp.Timestamp,
	fastFilterOp FastFilterOp,
	loadOp LoadOp,
	objectFilterOp ObjectFilterOp,
	blockFilterOp BlockFilterOp,
	seekOp SeekFirstBlockOp,
	objectList objectio.ObjectList,
	extraCommittedObjects []objectio.ObjectStats,
	uncommittedObjects []objectio.ObjectStats,
	outBlocks *objectio.BlockInfoSlice,
	metaPrefetcher func(context.Context) bool,
	fs fileservice.FileService,
	highSelectivityHint bool,
) (err error)

func ForeachObjectsExecute 

func ForeachObjectsExecute(
	onObject func(*objectio.ObjectStats) error,
	nextObjectFn func() (objectio.ObjectStats, error),
	latestObjects []objectio.ObjectStats,
	extraObjects []objectio.ObjectStats,
) (err error)

func GetThresholdForReader 

func GetThresholdForReader(readerNum int) uint64

func MakeColExprForTest 

func MakeColExprForTest(idx int32, typ types.T, colName ...string) *plan.Expr

for test

func MakeFunctionExprForTest 

func MakeFunctionExprForTest(name string, args []*plan.Expr) *plan.Expr

func MakeInExprForTest 

func MakeInExprForTest[T any](
	arg0 *plan.Expr, vals []T, oid types.T, mp *mpool.MPool,
) *plan.Expr

func NewCNTombstoneBatch 

func NewCNTombstoneBatch(
	pkType *types.Type,
	hidden objectio.HiddenColumnSelection,
) *batch.Batch

func NewColumnExpr 

func NewColumnExpr(pos int, typ plan.Type, name string) *plan.Expr

func NewEmptyTombstoneData 

func NewEmptyTombstoneData() *tombstoneData

func NewMergeReader 

func NewMergeReader(readers []engine.Reader) *mergeReader

func NewReader 

func NewReader(
	ctx context.Context,
	mp *mpool.MPool,
	packerPool *fileservice.Pool[*types.Packer],
	fs fileservice.FileService,
	tableDef *plan.TableDef,
	ts timestamp.Timestamp,
	expr *plan.Expr,

	source engine.DataSource,
	threshHold uint64,
	filterHint engine.FilterHint,
) (*reader, error)

-----------------------------------------------------------------

func NewSimpleReader 

func NewSimpleReader(
	ctx context.Context,
	ds engine.DataSource,
	fs fileservice.FileService,
	ts timestamp.Timestamp,
	opts ...ReaderOption,
) *reader

func RemoveIf 

func RemoveIf[T any](data []T, pred func(t T) bool) []T

RemoveIf removes the elements that pred is true.

func RowIdsToOffset 

func RowIdsToOffset(
	rowIds []types.Rowid,
	skipMask objectio.Bitmap,
) []int64

func SimpleMultiObjectsReader 

func SimpleMultiObjectsReader(
	ctx context.Context,
	fs fileservice.FileService,
	objs []objectio.ObjectStats,
	ts timestamp.Timestamp,
	opts ...ReaderOption,
) engine.Reader

TODO: optimize me to add object list relation data

func SimpleObjectReader 

func SimpleObjectReader(
	ctx context.Context,
	fs fileservice.FileService,
	obj *objectio.ObjectStats,
	ts timestamp.Timestamp,
	opts ...ReaderOption,
) engine.Reader

func SimpleReaderWithDataSource 

func SimpleReaderWithDataSource(
	ctx context.Context,
	fs fileservice.FileService,
	source engine.DataSource,
	ts timestamp.Timestamp,
	opts ...ReaderOption,
) engine.Reader

func SimpleTombstoneObjectReader 

func SimpleTombstoneObjectReader(
	ctx context.Context,
	fs fileservice.FileService,
	obj *objectio.ObjectStats,
	ts timestamp.Timestamp,
	opts ...ReaderOption,
) engine.Reader

func StreamBatchProcess 

func StreamBatchProcess(
	ctx context.Context,

	sourcer func(context.Context, *batch.Batch, *mpool.MPool) (bool, error),

	processor func(context.Context, *batch.Batch, *mpool.MPool) error,

	sinker func(context.Context, *batch.Batch) error,
	buffer containers.IBatchBuffer,
	mp *mpool.MPool,
) error

func TryFastFilterBlocks 

func TryFastFilterBlocks(
	ctx context.Context,
	snapshotTS timestamp.Timestamp,
	tableDef *plan.TableDef,
	rangesParam engine.RangesParam,
	objectList objectio.ObjectList,
	extraCommittedObjects []objectio.ObjectStats,
	uncommittedObjects []objectio.ObjectStats,
	outBlocks *objectio.BlockInfoSlice,
	metaPrefetcher func(context.Context) bool,
	fs fileservice.FileService,
) (ok bool, err error)

func UnmarshalRelationData 

func UnmarshalRelationData(data []byte) (engine.RelData, error)

func UnmarshalTombstoneData 

func UnmarshalTombstoneData(data []byte) (engine.Tombstoner, error)

func WithPartitionState 

func WithPartitionState(part any) func(*BlockListRelData)

Types

type BasePKFilter 

type BasePKFilter struct {
	Valid bool
	Op    int
	LB    []byte
	UB    []byte
	Vec   *vector.Vector
	Oid   types.T
	// Disjuncts holds OR-ed atomic filters; when non-empty, Op/LB/UB/Vec are ignored.
	Disjuncts []BasePKFilter
}

func ConstructBasePKFilter 

func ConstructBasePKFilter(
	expr *plan.Expr,
	tblDef *plan.TableDef,
	mp *mpool.MPool,
) (filter BasePKFilter, err error)

func (*BasePKFilter) String 

func (b *BasePKFilter) String() string

type BlockFilterOp 

type BlockFilterOp func(int, objectio.BlockObject, objectio.BloomFilter) (bool, bool, error)

type BlockListRelData 

type BlockListRelData struct {
	// contains filtered or unexported fields
}

func NewBlockListRelationData 

func NewBlockListRelationData(
	emptyCnt int,
	opts ...func(op *BlockListRelData),
) *BlockListRelData

emptyCnt is the number of empty blocks preserved

func NewBlockListRelationDataOfObject 

func NewBlockListRelationDataOfObject(
	obj *objectio.ObjectStats, withInMemory bool,
) *BlockListRelData

func (*BlockListRelData) AppendBlockInfo 

func (relData *BlockListRelData) AppendBlockInfo(blk *objectio.BlockInfo)

func (*BlockListRelData) AppendBlockInfoSlice 

func (relData *BlockListRelData) AppendBlockInfoSlice(slice objectio.BlockInfoSlice)

func (*BlockListRelData) AppendShardID 

func (relData *BlockListRelData) AppendShardID(id uint64)

func (*BlockListRelData) AttachTombstones 

func (relData *BlockListRelData) AttachTombstones(tombstones engine.Tombstoner) error

func (*BlockListRelData) BuildEmptyRelData 

func (relData *BlockListRelData) BuildEmptyRelData(i int) engine.RelData

func (*BlockListRelData) DataCnt 

func (relData *BlockListRelData) DataCnt() int

func (*BlockListRelData) DataSlice 

func (relData *BlockListRelData) DataSlice(i, j int) engine.RelData

func (*BlockListRelData) GetBlockInfo 

func (relData *BlockListRelData) GetBlockInfo(i int) objectio.BlockInfo

func (*BlockListRelData) GetBlockInfoSlice 

func (relData *BlockListRelData) GetBlockInfoSlice() objectio.BlockInfoSlice

func (*BlockListRelData) GetPState 

func (relData *BlockListRelData) GetPState() any

func (*BlockListRelData) GetShardID 

func (relData *BlockListRelData) GetShardID(i int) uint64

func (*BlockListRelData) GetShardIDList 

func (relData *BlockListRelData) GetShardIDList() []uint64

func (*BlockListRelData) GetTombstones 

func (relData *BlockListRelData) GetTombstones() engine.Tombstoner

func (*BlockListRelData) GetType 

func (relData *BlockListRelData) GetType() engine.RelDataType

func (*BlockListRelData) MarshalBinary 

func (relData *BlockListRelData) MarshalBinary() ([]byte, error)

func (*BlockListRelData) MarshalBinaryWithBuffer 

func (relData *BlockListRelData) MarshalBinaryWithBuffer(w *bytes.Buffer) (err error)

func (*BlockListRelData) SetBlockInfo 

func (relData *BlockListRelData) SetBlockInfo(i int, blk *objectio.BlockInfo)

func (*BlockListRelData) SetBlockList 

func (relData *BlockListRelData) SetBlockList(slice objectio.BlockInfoSlice)

func (*BlockListRelData) SetShardID 

func (relData *BlockListRelData) SetShardID(i int, id uint64)

func (*BlockListRelData) Split 

func (relData *BlockListRelData) Split(i int) []engine.RelData

func (*BlockListRelData) String 

func (relData *BlockListRelData) String() string

func (*BlockListRelData) UnmarshalBinary 

func (relData *BlockListRelData) UnmarshalBinary(data []byte) (err error)

type EmptyReader 

type EmptyReader struct {
}

func (*EmptyReader) Close 

func (r *EmptyReader) Close() error

func (*EmptyReader) GetOrderBy 

func (r *EmptyReader) GetOrderBy() []*plan.OrderBySpec

func (*EmptyReader) Read 

func (r *EmptyReader) Read(
	_ context.Context,
	_ []string,
	_ *plan.Expr,
	_ *mpool.MPool,
	_ *batch.Batch,
) (bool, error)

func (*EmptyReader) SetFilterZM 

func (r *EmptyReader) SetFilterZM(objectio.ZoneMap)

func (*EmptyReader) SetIndexParam 

func (r *EmptyReader) SetIndexParam(param *plan.IndexReaderParam)

func (*EmptyReader) SetOrderBy 

func (r *EmptyReader) SetOrderBy([]*plan.OrderBySpec)

type EmptyRelationData 

type EmptyRelationData struct {
	// contains filtered or unexported fields
}

func (*EmptyRelationData) AppendBlockInfo 

func (rd *EmptyRelationData) AppendBlockInfo(blk *objectio.BlockInfo)

func (*EmptyRelationData) AppendBlockInfoSlice 

func (rd *EmptyRelationData) AppendBlockInfoSlice(objectio.BlockInfoSlice)

func (*EmptyRelationData) AppendDataBlk 

func (rd *EmptyRelationData) AppendDataBlk(blk any)

func (*EmptyRelationData) AppendShardID 

func (rd *EmptyRelationData) AppendShardID(id uint64)

func (*EmptyRelationData) AttachTombstones 

func (rd *EmptyRelationData) AttachTombstones(tombstones engine.Tombstoner) error

func (*EmptyRelationData) BuildEmptyRelData 

func (rd *EmptyRelationData) BuildEmptyRelData(i int) engine.RelData

func (*EmptyRelationData) DataCnt 

func (rd *EmptyRelationData) DataCnt() int

func (*EmptyRelationData) DataSlice 

func (rd *EmptyRelationData) DataSlice(begin, end int) engine.RelData

func (*EmptyRelationData) ForeachDataBlk 

func (rd *EmptyRelationData) ForeachDataBlk(begin, end int, f func(blk any) error) error

func (*EmptyRelationData) GetBlockInfo 

func (rd *EmptyRelationData) GetBlockInfo(i int) objectio.BlockInfo

func (*EmptyRelationData) GetBlockInfoSlice 

func (rd *EmptyRelationData) GetBlockInfoSlice() objectio.BlockInfoSlice

func (*EmptyRelationData) GetDataBlk 

func (rd *EmptyRelationData) GetDataBlk(i int) any

func (*EmptyRelationData) GetShardID 

func (rd *EmptyRelationData) GetShardID(i int) uint64

func (*EmptyRelationData) GetShardIDList 

func (rd *EmptyRelationData) GetShardIDList() []uint64

func (*EmptyRelationData) GetTombstones 

func (rd *EmptyRelationData) GetTombstones() engine.Tombstoner

func (*EmptyRelationData) GetType 

func (rd *EmptyRelationData) GetType() engine.RelDataType

func (*EmptyRelationData) MarshalBinary 

func (rd *EmptyRelationData) MarshalBinary() ([]byte, error)

func (*EmptyRelationData) MarshalBinaryWithBuffer 

func (rd *EmptyRelationData) MarshalBinaryWithBuffer(w *bytes.Buffer) (err error)

func (*EmptyRelationData) SetBlockInfo 

func (rd *EmptyRelationData) SetBlockInfo(i int, blk *objectio.BlockInfo)

func (*EmptyRelationData) SetDataBlk 

func (rd *EmptyRelationData) SetDataBlk(i int, blk any)

func (*EmptyRelationData) SetShardID 

func (rd *EmptyRelationData) SetShardID(i int, id uint64)

func (*EmptyRelationData) Split 

func (rd *EmptyRelationData) Split(_ int) []engine.RelData

func (*EmptyRelationData) String 

func (rd *EmptyRelationData) String() string

func (*EmptyRelationData) UnmarshalBinary 

func (rd *EmptyRelationData) UnmarshalBinary(data []byte) (err error)

type FastFilterOp 

type FastFilterOp func(*objectio.ObjectStats) (bool, error)

type LoadOp 

type LoadOp = func(
	context.Context, *objectio.ObjectStats, objectio.ObjectMeta, objectio.BloomFilter,
) (objectio.ObjectMeta, objectio.BloomFilter, error)

type LoadOpFactory 

type LoadOpFactory func(fileservice.FileService) LoadOp

type MemPKFilter 

type MemPKFilter struct {
	TS types.TS

	FilterHint engine.FilterHint
	HasBF      bool
	BFSeqNum   int16
	// contains filtered or unexported fields
}

func NewMemPKFilter 

func NewMemPKFilter(
	tableDef *plan.TableDef,
	ts timestamp.Timestamp,
	packerPool *fileservice.Pool[*types.Packer],
	basePKFilter BasePKFilter,
	filterHint engine.FilterHint,
) (filter MemPKFilter, err error)

func (*MemPKFilter) Exact 

func (f *MemPKFilter) Exact() (bool, bool)

func (*MemPKFilter) FilterVector 

func (f *MemPKFilter) FilterVector(
	vec *vector.Vector,
	packer *types.Packer,
	skipMask *objectio.Bitmap,
)

func (*MemPKFilter) InKind 

func (f *MemPKFilter) InKind() (int, bool)

func (*MemPKFilter) Keys 

func (f *MemPKFilter) Keys() [][]byte

func (*MemPKFilter) Must 

func (f *MemPKFilter) Must() bool

func (*MemPKFilter) Op 

func (f *MemPKFilter) Op() int

func (*MemPKFilter) RecordExactHit 

func (f *MemPKFilter) RecordExactHit() bool

func (*MemPKFilter) SetFullData 

func (f *MemPKFilter) SetFullData(op int, isVec bool, val ...[]byte)

func (*MemPKFilter) SetNull 

func (f *MemPKFilter) SetNull()

func (*MemPKFilter) String 

func (f *MemPKFilter) String() string

func (*MemPKFilter) Valid 

func (f *MemPKFilter) Valid() bool

type ObjListRelData 

type ObjListRelData struct {
	PState         any
	NeedFirstEmpty bool

	TotalBlocks uint32
	Objlist     []objectio.ObjectStats
	Rsp         *engine.RangesShuffleParam
	// contains filtered or unexported fields
}

func (*ObjListRelData) AppendBlockInfo 

func (or *ObjListRelData) AppendBlockInfo(blk *objectio.BlockInfo)

func (*ObjListRelData) AppendBlockInfoSlice 

func (or *ObjListRelData) AppendBlockInfoSlice(slice objectio.BlockInfoSlice)

func (*ObjListRelData) AppendObj 

func (or *ObjListRelData) AppendObj(obj *objectio.ObjectStats)

func (*ObjListRelData) AppendShardID 

func (or *ObjListRelData) AppendShardID(id uint64)

func (*ObjListRelData) AttachTombstones 

func (or *ObjListRelData) AttachTombstones(tombstones engine.Tombstoner) error

func (*ObjListRelData) BuildEmptyRelData 

func (or *ObjListRelData) BuildEmptyRelData(i int) engine.RelData

func (*ObjListRelData) DataCnt 

func (or *ObjListRelData) DataCnt() int

func (*ObjListRelData) DataSlice 

func (or *ObjListRelData) DataSlice(i, j int) engine.RelData

func (*ObjListRelData) GetBlockInfo 

func (or *ObjListRelData) GetBlockInfo(i int) objectio.BlockInfo

func (*ObjListRelData) GetBlockInfoSlice 

func (or *ObjListRelData) GetBlockInfoSlice() objectio.BlockInfoSlice

func (*ObjListRelData) GetShardID 

func (or *ObjListRelData) GetShardID(i int) uint64

func (*ObjListRelData) GetShardIDList 

func (or *ObjListRelData) GetShardIDList() []uint64

func (*ObjListRelData) GetTombstones 

func (or *ObjListRelData) GetTombstones() engine.Tombstoner

func (*ObjListRelData) GetType 

func (or *ObjListRelData) GetType() engine.RelDataType

func (*ObjListRelData) MarshalBinary 

func (or *ObjListRelData) MarshalBinary() ([]byte, error)

func (*ObjListRelData) SetBlockInfo 

func (or *ObjListRelData) SetBlockInfo(i int, blk *objectio.BlockInfo)

func (*ObjListRelData) SetBlockList 

func (or *ObjListRelData) SetBlockList(slice objectio.BlockInfoSlice)

func (*ObjListRelData) SetShardID 

func (or *ObjListRelData) SetShardID(i int, id uint64)

func (*ObjListRelData) Split 

func (or *ObjListRelData) Split(cpunum int) []engine.RelData

func (*ObjListRelData) String 

func (or *ObjListRelData) String() string

func (*ObjListRelData) UnmarshalBinary 

func (or *ObjListRelData) UnmarshalBinary(buf []byte) error

type ObjectFilterOp 

type ObjectFilterOp func(objectio.ObjectMeta, objectio.BloomFilter) (bool, error)

type ReaderOption 

type ReaderOption func(*reader)

func WithBlockFilter 

func WithBlockFilter(
	filter objectio.BlockReadFilter, seqnum uint16, typ types.Type,
) ReaderOption

func WithColumns 

func WithColumns(
	seqnums []uint16, colTypes []types.Type,
) ReaderOption

type RemoteDataSource 

type RemoteDataSource struct {
	// contains filtered or unexported fields
}

func NewRemoteDataSource 

func NewRemoteDataSource(
	ctx context.Context,
	fs fileservice.FileService,
	snapshotTS timestamp.Timestamp,
	relData engine.RelData,
) (source *RemoteDataSource)

func (*RemoteDataSource) ApplyTombstones 

func (rs *RemoteDataSource) ApplyTombstones(
	ctx context.Context,
	bid *objectio.Blockid,
	rowsOffset []int64,
	applyPolicy engine.TombstoneApplyPolicy,
) (left []int64, err error)

func (*RemoteDataSource) Close 

func (rs *RemoteDataSource) Close()

func (*RemoteDataSource) GetOrderBy 

func (rs *RemoteDataSource) GetOrderBy() []*plan.OrderBySpec

func (*RemoteDataSource) GetTombstones 

func (rs *RemoteDataSource) GetTombstones(
	ctx context.Context, bid *objectio.Blockid,
) (mask objectio.Bitmap, err error)

func (*RemoteDataSource) Next 

func (rs *RemoteDataSource) Next(
	_ context.Context,
	_ []string,
	_ []types.Type,
	seqNums []uint16,
	_ int32,
	_ any,
	_ *mpool.MPool,
	_ *batch.Batch,
) (*objectio.BlockInfo, engine.DataState, error)

func (*RemoteDataSource) SetFilterZM 

func (rs *RemoteDataSource) SetFilterZM(_ objectio.ZoneMap)

func (*RemoteDataSource) SetOrderBy 

func (rs *RemoteDataSource) SetOrderBy(_ []*plan.OrderBySpec)

func (*RemoteDataSource) String 

func (rs *RemoteDataSource) String() string

type SeekFirstBlockOp 

type SeekFirstBlockOp func(objectio.ObjectDataMeta) int

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.