containers

package
v0.0.0-debug-20260702 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 2, 2026 License: Apache-2.0 Imports: 20 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func DedupSortedBatches 

func DedupSortedBatches(
	uniqueIdx int,
	batches []*batch.Batch,
) error

func ForeachVector added in v0.8.0 

func ForeachVector(vec Vector, op any, sel *nulls.Bitmap) (err error)

func ForeachVectorWindow added in v0.8.0 

func ForeachVectorWindow(
	vec Vector,
	start, length int,
	op1 any,
	op2 ItOp,
	sel *nulls.Bitmap,
) (err error)

func ForeachWindowBytes added in v0.8.0 

func ForeachWindowBytes(
	vec *vector.Vector,
	start, length int,
	op ItOpT[[]byte],
	sels *nulls.Bitmap,
) (err error)

func ForeachWindowFixed added in v0.8.0 

func ForeachWindowFixed[T any](
	vec *vector.Vector,
	start, length int,
	reverse bool,
	op ItOpT[T],
	opAny ItOp,
	sels *nulls.Bitmap,
) (err error)

func ForeachWindowVarlen added in v0.8.0 

func ForeachWindowVarlen(
	vec *vector.Vector,
	start, length int,
	reverse bool,
	op ItOpT[[]byte],
	opAny ItOp,
	sels *nulls.Bitmap,
) (err error)

func GenericUpdateBytes added in v0.6.0 

func GenericUpdateBytes(
	vec *vector.Vector, row uint32, v any, isNull bool, mp *mpool.MPool,
)

func GenericUpdateFixedValue added in v0.6.0 

func GenericUpdateFixedValue[T types.FixedSizeT](
	vec *vector.Vector, row uint32, v any, isNull bool, _ *mpool.MPool,
)

func GetDefaultVectorPoolALLocator added in v1.0.0 

func GetDefaultVectorPoolALLocator() *mpool.MPool

func MakeForeachVectorOp added in v0.8.0 

func MakeForeachVectorOp(t types.T, overloads map[types.T]any, args ...any) any

func NewConstBytes added in v1.0.0 

func NewConstBytes(typ types.Type, val []byte, length int, opts ...Options) *vectorWrapper

func NewConstFixed added in v1.0.0 

func NewConstFixed[T any](typ types.Type, val T, length int, opts ...Options) *vectorWrapper

func NewConstNullVector added in v1.0.0 

func NewConstNullVector(
	typ types.Type,
	length int,
	mp *mpool.MPool,
) *vectorWrapper

func NewVector 

func NewVector(typ types.Type, opts ...Options) *vectorWrapper

func SplitBatch added in v0.6.0 

func SplitBatch(bat *batch.Batch, cnt int) []*batch.Batch

func ToCNBatch added in v0.8.0 

func ToCNBatch(tnBat *Batch) *batch.Batch

func UpdateValue added in v0.6.0 

func UpdateValue(col *vector.Vector, row uint32, val any, isNull bool, mp *mpool.MPool)

func VectorsCopyToBatch 

func VectorsCopyToBatch(
	vecs Vectors,
	outputBat *batch.Batch,
	mp *mpool.MPool,
) (err error)

Types

type Batch 

type Batch struct {
	Attrs   []string
	Vecs    []Vector
	Deletes *nulls.Bitmap
	Nameidx map[string]int
	Pool    *VectorPool
	// DataRelease is called to free external data buffers (e.g. fileservice IOVectors)
	// when the batch was loaded with zero-copy (needCopy=false).
	DataRelease func()
}

func BuildBatch 

func BuildBatch(attrs []string, colTypes []types.Type, opts Options) *Batch

func BuildBatchWithPool added in v0.8.0 

func BuildBatchWithPool(
	attrs []string, colTypes []types.Type, capacity int, pool *VectorPool,
) *Batch

func MockBatch 

func MockBatch(vecTypes []types.Type, rows int, uniqueIdx int, provider *MockDataProvider) (bat *Batch)

func MockBatchWithAttrs 

func MockBatchWithAttrs(vecTypes []types.Type, attrs []string, rows int, uniqueIdx int, provider *MockDataProvider) (bat *Batch)

func MockBatchWithAttrsAndOffset 

func MockBatchWithAttrsAndOffset(vecTypes []types.Type, attrs []string, rows int, offset int) (bat *Batch)

func MockNullableBatch 

func MockNullableBatch(vecTypes []types.Type, rows int, uniqueIdx int, provider *MockDataProvider) (bat *Batch)

func NewBatch 

func NewBatch() *Batch

func NewBatchWithCapacity added in v0.8.0 

func NewBatchWithCapacity(cap int) *Batch

func NewEmptyBatch 

func NewEmptyBatch() *Batch

func NewNonNullBatchWithSharedMemory added in v0.7.0 

func NewNonNullBatchWithSharedMemory(b *batch.Batch, mp *mpool.MPool) *Batch

func ToTNBatch added in v1.0.0 

func ToTNBatch(cnBat *batch.Batch, mp *mpool.MPool) *Batch

func (*Batch) AddVector 

func (bat *Batch) AddVector(attr string, vec Vector)

func (*Batch) Allocated 

func (bat *Batch) Allocated() int

func (*Batch) Append added in v0.7.0 

func (bat *Batch) Append(src *Batch) (err error)

func (*Batch) AppendPlaceholder added in v0.8.0 

func (bat *Batch) AppendPlaceholder()

AddPlaceholder is used to consctruct batch sent to CN. The vectors in the batch are sorted by seqnum, if the seqnum was dropped, a zero value will be fill as placeholder. This is space-time tradeoff.

func (*Batch) ApproxSize added in v1.0.0 

func (bat *Batch) ApproxSize() int

func (*Batch) CleanOnlyData 

func (bat *Batch) CleanOnlyData()

func (*Batch) CloneWindow 

func (bat *Batch) CloneWindow(offset, length int, allocator ...*mpool.MPool) (cloned *Batch)

func (*Batch) CloneWindowWithPool added in v0.8.0 

func (bat *Batch) CloneWindowWithPool(offset, length int, pool *VectorPool) (cloned *Batch)

func (*Batch) Close 

func (bat *Batch) Close()

func (*Batch) Compact 

func (bat *Batch) Compact()

func (*Batch) Delete 

func (bat *Batch) Delete(i int)

func (*Batch) DeleteCnt 

func (bat *Batch) DeleteCnt() int

func (*Batch) Equals 

func (bat *Batch) Equals(o *Batch) bool

func (*Batch) Extend added in v0.6.0 

func (bat *Batch) Extend(src *Batch)

extend vector with same name, consume src batch

func (*Batch) GetVectorByName 

func (bat *Batch) GetVectorByName(name string) Vector

func (*Batch) HasDelete 

func (bat *Batch) HasDelete() bool

func (*Batch) IsDeleted 

func (bat *Batch) IsDeleted(i int) bool

func (*Batch) Length 

func (bat *Batch) Length() int

func (*Batch) PPString added in v0.6.0 

func (bat *Batch) PPString(num int) string

func (*Batch) RangeDelete 

func (bat *Batch) RangeDelete(start, end int)

func (*Batch) ReadFrom 

func (bat *Batch) ReadFrom(r io.Reader) (n int64, err error)

func (*Batch) ReadFromV1 added in v0.8.0 

func (bat *Batch) ReadFromV1(r io.Reader) (n int64, err error)

in version1, batch.Deletes is roaring.Bitmap

func (*Batch) ReadFromV2 

func (bat *Batch) ReadFromV2(r io.Reader) (n int64, err error)

ReadFromV2 in version2, vector.nulls.bitmap is v1

func (*Batch) Reset added in v1.0.0 

func (bat *Batch) Reset()

func (*Batch) Split 

func (bat *Batch) Split(cnt int) []*Batch

func (*Batch) String 

func (bat *Batch) String() string

func (*Batch) Window 

func (bat *Batch) Window(offset, length int) *Batch

func (*Batch) WindowDeletes added in v0.8.0 

func (bat *Batch) WindowDeletes(offset, length int, deep bool) *nulls.Bitmap

func (*Batch) WriteTo 

func (bat *Batch) WriteTo(w io.Writer) (n int64, err error)

func (*Batch) WriteToV2 

func (bat *Batch) WriteToV2(w io.Writer) (n int64, err error)

WriteToV2 in version 2, vector.Nulls.Bitmap is v1

type BatchFreeList 

type BatchFreeList struct {
	// contains filtered or unexported fields
}

BatchFreeList is a simple LIFO batch pool that implements IBatchBuffer. Unlike OneSchemaBatchBuffer, it never evicts batches based on size, ensuring pre-allocated vector capacity is always preserved for reuse. It is released when the owning sinker calls Close.

func NewBatchFreeList 

func NewBatchFreeList(
	attrs []string,
	typs []types.Type,
	offHeap bool,
) *BatchFreeList

func (*BatchFreeList) Close 

func (fl *BatchFreeList) Close(mp *mpool.MPool)

func (*BatchFreeList) Fetch 

func (fl *BatchFreeList) Fetch() *batch.Batch

func (*BatchFreeList) FetchWithSchema 

func (fl *BatchFreeList) FetchWithSchema(attrs []string, typs []types.Type) *batch.Batch

func (*BatchFreeList) Len 

func (fl *BatchFreeList) Len() int

func (*BatchFreeList) Putback 

func (fl *BatchFreeList) Putback(bat *batch.Batch, _ *mpool.MPool)

func (*BatchFreeList) Usage 

func (fl *BatchFreeList) Usage() (int, int, int, int)

type BatchSplitter added in v1.0.0 

type BatchSplitter struct {
	// contains filtered or unexported fields
}

BatchSplitter is used to split a batch into several batches with the same size.

func NewBatchSplitter added in v1.0.0 

func NewBatchSplitter(bat *Batch, sliceSize int) *BatchSplitter

func (*BatchSplitter) Next added in v1.0.0 

func (bs *BatchSplitter) Next() (*Batch, error)

type BatchWithVersion added in v0.8.0 

type BatchWithVersion struct {
	*Batch
	Seqnums    []uint16
	NextSeqnum uint16
	Version    uint32
}

func (*BatchWithVersion) Len added in v0.8.0 

func (b *BatchWithVersion) Len() int

func (*BatchWithVersion) Less added in v0.8.0 

func (b *BatchWithVersion) Less(i, j int) bool

Sort by seqnum

func (*BatchWithVersion) Swap added in v0.8.0 

func (b *BatchWithVersion) Swap(i, j int)

type GeneralBatchBuffer 

type GeneralBatchBuffer struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewGeneralBatchBuffer 

func NewGeneralBatchBuffer(
	sizeCap int,
	offHeap bool,
	maxOneFixedSize int,
	maxOneVarlen int,
) *GeneralBatchBuffer

func (*GeneralBatchBuffer) Close 

func (bb *GeneralBatchBuffer) Close(mp *mpool.MPool)

func (*GeneralBatchBuffer) Fetch 

func (bb *GeneralBatchBuffer) Fetch() *batch.Batch

func (*GeneralBatchBuffer) FetchWithSchema 

func (bb *GeneralBatchBuffer) FetchWithSchema(attrs []string, types []types.Type) *batch.Batch

func (*GeneralBatchBuffer) Len 

func (bb *GeneralBatchBuffer) Len() int

func (*GeneralBatchBuffer) Putback 

func (bb *GeneralBatchBuffer) Putback(bat *batch.Batch, mp *mpool.MPool)

func (*GeneralBatchBuffer) Usage 

func (bb *GeneralBatchBuffer) Usage() (int, int, int, int)

type IBatchBuffer 

type IBatchBuffer interface {
	FetchWithSchema([]string, []types.Type) *batch.Batch
	Fetch() *batch.Batch
	Putback(*batch.Batch, *mpool.MPool)
	Close(*mpool.MPool)
	Len() int
	Usage() (int, int, int, int)
}

type ItOp 

type ItOp = func(v any, isNull bool, row int) error

type ItOpT added in v0.8.0 

type ItOpT[T any] func(v T, isNull bool, row int) error

type MockDataProvider 

type MockDataProvider struct {
	// contains filtered or unexported fields
}

func NewMockDataProvider 

func NewMockDataProvider() *MockDataProvider

func (*MockDataProvider) AddColumnProvider 

func (p *MockDataProvider) AddColumnProvider(colIdx int, provider Vector)

func (*MockDataProvider) GetColumnProvider 

func (p *MockDataProvider) GetColumnProvider(colIdx int) Vector

func (*MockDataProvider) Reset 

func (p *MockDataProvider) Reset()

type OneSchemaBatchBuffer 

type OneSchemaBatchBuffer struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewOneSchemaBatchBuffer 

func NewOneSchemaBatchBuffer(
	sizeCap int,
	attrs []string,
	typs []types.Type,
	offHeap bool,
) *OneSchemaBatchBuffer

func (*OneSchemaBatchBuffer) Close 

func (bb *OneSchemaBatchBuffer) Close(mp *mpool.MPool)

func (*OneSchemaBatchBuffer) Fetch 

func (bb *OneSchemaBatchBuffer) Fetch() *batch.Batch

func (*OneSchemaBatchBuffer) FetchWithSchema 

func (bb *OneSchemaBatchBuffer) FetchWithSchema(attrs []string, types []types.Type) *batch.Batch

func (*OneSchemaBatchBuffer) Len 

func (bb *OneSchemaBatchBuffer) Len() int

func (*OneSchemaBatchBuffer) Putback 

func (bb *OneSchemaBatchBuffer) Putback(bat *batch.Batch, mp *mpool.MPool)

func (*OneSchemaBatchBuffer) Usage 

func (bb *OneSchemaBatchBuffer) Usage() (int, int, int, int)

type Options 

type Options struct {
	Capacity  int
	Allocator *mpool.MPool
}

type Vector 

type Vector interface {
	GetType() *types.Type

	IsConst() bool
	IsConstNull() bool

	// Deep copy ops
	Get(i int) any
	Append(v any, isNull bool)
	CloneWindow(offset, length int, allocator ...*mpool.MPool) Vector
	CloneWindowWithPool(offset, length int, pool *VectorPool) Vector
	PreExtend(length int) error

	WriteTo(w io.Writer) (int64, error)
	WriteToV1(w io.Writer) (int64, error)
	ReadFrom(r io.Reader) (int64, error)
	ReadFromV1(r io.Reader) (int64, error)

	// Shallow Ops
	ShallowGet(i int) any
	Window(offset, length int) Vector

	// Deepcopy if const
	TryConvertConst() Vector

	GetDownstreamVector() *cnVector.Vector

	Update(i int, v any, isNull bool)
	Compact(*roaring.Bitmap)
	CompactByBitmap(*nulls.Bitmap)

	Extend(o Vector)
	ExtendWithOffset(src Vector, srcOff, srcLen int)
	ExtendVec(o *cnVector.Vector) error

	Foreach(op ItOp, sels *nulls.Bitmap) error
	ForeachWindow(offset, length int, op ItOp, sels *nulls.Bitmap) error

	Length() int
	ApproxSize() int
	Allocated() int
	GetAllocator() *mpool.MPool

	IsNull(i int) bool
	HasNull() bool
	NullMask() *nulls.Nulls
	// NullCount will consider ConstNull and Const vector
	NullCount() int

	Close()

	// Test functions
	Equals(o Vector) bool
	String() string
	PPString(num int) string
	AppendMany(vs []any, isNulls []bool)
	Delete(i int)
	// contains filtered or unexported methods
}
var EMPTY_VECTOR Vector

func CloneVector added in v1.2.0 

func CloneVector(src *vector.Vector, mp *mpool.MPool, vp *VectorPool) (Vector, error)

func MakeVector 

func MakeVector(typ types.Type, mp *mpool.MPool) (vec Vector)

func MockVector 

func MockVector(t types.Type, rows int, unique bool, provider Vector) (vec Vector)

func MockVector2 

func MockVector2(typ types.Type, rows int, offset int) Vector

func ToTNVector added in v1.0.0 

func ToTNVector(v *vector.Vector, mp *mpool.MPool) Vector

type VectorPool added in v0.8.0 

type VectorPool struct {
	// contains filtered or unexported fields
}

func NewVectorPool added in v0.8.0 

func NewVectorPool(name string, cnt int, opts ...VectorPoolOption) *VectorPool

func (*VectorPool) Allocated added in v0.8.0 

func (p *VectorPool) Allocated() int

func (*VectorPool) Destory added in v1.0.0 

func (p *VectorPool) Destory()

Only for test It is not safe to call Destory

func (*VectorPool) FixedSizeAllocated added in v0.8.0 

func (p *VectorPool) FixedSizeAllocated() int

func (*VectorPool) FixedSizeUsed added in v0.8.0 

func (p *VectorPool) FixedSizeUsed(isUnsafe bool) (int, int)

func (*VectorPool) GetMPool added in v1.2.0 

func (p *VectorPool) GetMPool() *mpool.MPool

func (*VectorPool) GetVector added in v0.8.0 

func (p *VectorPool) GetVector(t *types.Type) *vectorWrapper

func (*VectorPool) String added in v0.8.0 

func (p *VectorPool) String() string

func (*VectorPool) Used added in v0.8.0 

func (p *VectorPool) Used(isUnsafe bool) (int, int)

func (*VectorPool) VarlenUsed added in v0.8.0 

func (p *VectorPool) VarlenUsed(isUnsafe bool) (int, int)

func (*VectorPool) VarlenaSizeAllocated added in v0.8.0 

func (p *VectorPool) VarlenaSizeAllocated() int

type VectorPoolOption added in v0.8.0 

type VectorPoolOption func(*VectorPool)

func WithAllocationLimit added in v0.8.0 

func WithAllocationLimit(maxv int) VectorPoolOption

func WithFixedSizeRatio added in v0.8.0 

func WithFixedSizeRatio(ratio float64) VectorPoolOption

func WithMPool added in v0.8.0 

func WithMPool(mp *mpool.MPool) VectorPoolOption

type Vectors 

type Vectors []vector.Vector

func NewVectors 

func NewVectors(n int) Vectors

func (Vectors) Allocated 

func (vs Vectors) Allocated() int

func (Vectors) Free 

func (vs Vectors) Free(mp *mpool.MPool)

func (Vectors) Rows 

func (vs Vectors) Rows() int

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.