Documentation
¶
Index ¶
- Constants
- Variables
- func AddVectorsToIndex(ctx context.Context, vectors []VectorRecord, vectorIndex VectorIndex) error
- func CalculateOptimalSegments(dims int) int
- func IsDynamic(indexType IndexType) bool
- func MultiVectorsEqual(vecA, vecB [][]float32) bool
- func VectorsEqual(vecA, vecB []float32) bool
- type BoltStore
- type BucketView
- type FS
- type File
- type GetViewThunk
- type GroupedPagedArray
- type IndexStats
- type IndexType
- type Iterator
- type MonotonicCounter
- type MultiVectorForID
- type MultipleVectorForID
- type PagedArray
- type PqMaxPool
- type QueryVectorDistancer
- type SearchByDistParams
- func (params *SearchByDistParams) Iterate()
- func (params *SearchByDistParams) MaxLimitReached() bool
- func (params *SearchByDistParams) MaximumSearchLimit() int64
- func (params *SearchByDistParams) OffsetCapacity(ids []uint64) int
- func (params *SearchByDistParams) TotalLimit() int
- func (params *SearchByDistParams) TotalLimitCapacity(ids []uint64) int
- type Sequence
- type SequenceStore
- type ShardedLocks
- func (sl *ShardedLocks) Hash(id uint64) uint64
- func (sl *ShardedLocks) Lock(id uint64)
- func (sl *ShardedLocks) LockAll()
- func (sl *ShardedLocks) Locked(id uint64, callback func())
- func (sl *ShardedLocks) LockedAll(callback func())
- func (sl *ShardedLocks) Unlock(id uint64)
- func (sl *ShardedLocks) UnlockAll()
- type ShardedRWLocks
- func (sl *ShardedRWLocks) Hash(id uint64) uint64
- func (sl *ShardedRWLocks) Lock(id uint64)
- func (sl *ShardedRWLocks) LockAll()
- func (sl *ShardedRWLocks) Locked(id uint64, callback func())
- func (sl *ShardedRWLocks) LockedAll(callback func())
- func (sl *ShardedRWLocks) RLock(id uint64)
- func (sl *ShardedRWLocks) RLockAll()
- func (sl *ShardedRWLocks) RLocked(id uint64, callback func())
- func (sl *ShardedRWLocks) RLockedAll(callback func())
- func (sl *ShardedRWLocks) RUnlock(id uint64)
- func (sl *ShardedRWLocks) RUnlockAll()
- func (sl *ShardedRWLocks) TryLock(id uint64) bool
- func (sl *ShardedRWLocks) Unlock(id uint64)
- func (sl *ShardedRWLocks) UnlockAll()
- type SharedGauge
- type SparseFisherYatesIterator
- type TargetTempVectorForID
- type TargetTempVectorForIDWithView
- type TargetVectorForID
- type TempVectorForID
- type TempVectorForIDWithView
- type TempVectorUint64Pool
- type TempVectorsPool
- type TestFS
- func (fs *TestFS) Create(name string) (File, error)
- func (fs *TestFS) Open(name string) (File, error)
- func (fs *TestFS) OpenFile(name string, flag int, perm os.FileMode) (File, error)
- func (fs *TestFS) ReadDir(name string) ([]os.DirEntry, error)
- func (fs *TestFS) Remove(name string) error
- func (fs *TestFS) RemoveAll(path string) error
- func (fs *TestFS) Rename(oldpath, newpath string) error
- func (fs *TestFS) Stat(name string) (os.FileInfo, error)
- func (fs *TestFS) Truncate(name string, size int64) error
- type TestFile
- type UnboundedChannel
- type Vector
- type VectorForID
- type VectorIndex
- type VectorIndexMulti
- type VectorRecord
- type VectorSlice
- type VectorUint64Slice
Constants ¶
const ( IndexTypeHNSW = "hnsw" IndexTypeFlat = "flat" IndexTypeNoop = "noop" IndexTypeDynamic = "dynamic" IndexTypeHFresh = "hfresh" )
const ( // DefaultSearchByDistInitialLimit : // the initial limit of 100 here is an // arbitrary decision, and can be tuned // as needed DefaultSearchByDistInitialLimit = 100 // DefaultSearchByDistLimitMultiplier : // the decision to increase the limit in // multiples of 10 here is an arbitrary // decision, and can be tuned as needed DefaultSearchByDistLimitMultiplier = 10 )
const ( DefaultShardedLocksCount = 512 DefaultPageSize = 1 )
Variables ¶
var ErrWrongDimensions = errors.New("vector dimensions do not match the index dimensions")
Functions ¶
func AddVectorsToIndex ¶ added in v1.29.0
func AddVectorsToIndex(ctx context.Context, vectors []VectorRecord, vectorIndex VectorIndex) error
func CalculateOptimalSegments ¶ added in v1.25.23
func MultiVectorsEqual ¶ added in v1.29.0
func VectorsEqual ¶ added in v1.24.0
Types ¶
type BoltStore ¶ added in v1.35.0
type BoltStore struct {
// contains filtered or unexported fields
}
BoltStore is a SequenceStore implementation that uses BoltDB as the backend.
type BucketView ¶ added in v1.34.11
type BucketView interface {
ReleaseView()
}
BucketView represents a consistent view of an LSM bucket that can be used for multiple reads without acquiring locks for each read. The caller must call ReleaseView() when done to avoid blocking compactions.
type FS ¶ added in v1.34.0
type FS interface {
Open(name string) (File, error)
OpenFile(name string, flag int, perm os.FileMode) (File, error)
Create(name string) (File, error)
MkdirAll(path string, perm os.FileMode) error
ReadDir(name string) ([]os.DirEntry, error)
Stat(name string) (os.FileInfo, error)
Remove(name string) error
RemoveAll(path string) error
Rename(oldpath, newpath string) error
Truncate(name string, size int64) error
}
type GetViewThunk ¶ added in v1.34.11
type GetViewThunk func() BucketView
GetViewThunk returns a consistent view of the underlying bucket.
type GroupedPagedArray ¶ added in v1.36.0
type GroupedPagedArray[T any] struct { // contains filtered or unexported fields }
GroupedPagedArray is a two-level paged array optimized for concurrent access
Level 1: a slice of atomic pointers to groups (allocated upfront, small). Level 2: each group holds 128 atomic pointers to pages (allocated on demand). Pages: fixed-size slices of T (allocated on demand).
Read path is lock-free (two atomic loads). Write path uses a mutex only for allocating new groups/pages via double-checked locking.
Caller is responsible for ensuring that reads and writes to elements within returned pages are performed safely using atomic operations or mutexes.
func NewGroupedPagedArray ¶ added in v1.36.0
func NewGroupedPagedArray[T any](maxPages, pageSize uint64) *GroupedPagedArray[T]
NewGroupedPagedArray creates a new GroupedPagedArray.
maxPages is the maximum number of pages that can be addressed. pageSize is the number of elements per page (rounded up to a power of 2, minimum 64).
Total addressable elements = maxPages × pageSize.
The upfront cost is len(groups) × 8 bytes, where len(groups) = ceil(maxPages / 128). For example, 16K max pages costs only 1KB upfront.
func (*GroupedPagedArray[T]) EnsurePageFor ¶ added in v1.36.0
func (p *GroupedPagedArray[T]) EnsurePageFor(id uint64) ([]T, int)
EnsurePageFor ensures that a page exists for the given ID, allocating the group and/or page if necessary. Returns the page and the index within the page.
Panics if the ID exceeds the maximum addressable capacity.
func (*GroupedPagedArray[T]) GetPageFor ¶ added in v1.36.0
func (p *GroupedPagedArray[T]) GetPageFor(id uint64) ([]T, int)
GetPageFor returns the page containing the given ID and the index within that page. Returns (nil, -1) if the page has not been allocated.
Lock-free: performs two atomic loads.
type IndexStats ¶ added in v1.24.23
type IndexStats interface {
IndexType() IndexType
}
type MonotonicCounter ¶ added in v1.34.0
type MonotonicCounter struct {
// contains filtered or unexported fields
}
MonotonicCounter is a thread-safe counter that increments monotonically. It is a simple wrapper to avoid accidental resets, decrements, overflow, or other operations that could break the monotonicity. The zero value is a valid counter that starts at 0.
func NewMonotonicCounter ¶ added in v1.34.0
func NewMonotonicCounter(start uint64) *MonotonicCounter
NewMonotonicCounter creates a new MonotonicCounter with the given starting value. Next call to Next() will return the starting value + 1. For a counter that starts at 0, you can use the zero value directly.
func (*MonotonicCounter) Next ¶ added in v1.34.0
func (c *MonotonicCounter) Next() uint64
Next returns the next value of the counter. If the counter has hit its maximum value, it panics.
func (*MonotonicCounter) NextN ¶ added in v1.34.0
func (c *MonotonicCounter) NextN(n uint64) (start uint64, end uint64)
NextN returns the next n values as a range: [start, end]. If n is 0, it returns (0, 0). Panics if the range would overflow.
func (*MonotonicCounter) TryNext ¶ added in v1.34.0
func (c *MonotonicCounter) TryNext() (uint64, bool)
TryNext returns (next, ok). When ok == false the counter has hit its maximum and next == zero(T).
func (*MonotonicCounter) TryNextN ¶ added in v1.34.0
func (c *MonotonicCounter) TryNextN(n uint64) (start, end uint64, ok bool)
TryNextN returns the next n values as a range [start, end]. If n is 0, it returns (0, 0, true). If the counter has hit its maximum value or the range would overflow, it returns (0, 0, false).
type MultiVectorForID ¶
type MultipleVectorForID ¶ added in v1.29.0
type PagedArray ¶ added in v1.34.0
type PagedArray[T any] struct { // contains filtered or unexported fields }
PagedArray is an array that stores elements in pages of a fixed size. It is optimized for concurrent access patterns where multiple goroutines may read and write to different pages simultaneously. The API is thread-safe and returns direct references to the pages, allowing for efficient concurrent access. Caller is responsible for ensuring that reads and writes to the returned pages are performed safely, using either atomic operations or mutexes.
func NewPagedArray ¶ added in v1.34.0
func NewPagedArray[T any](pages, pageSize uint64) *PagedArray[T]
NewPagedArray creates a new PagedArray with the given page size. It will round up to the next power of 2 and enforce a minimum size of 64.
func (*PagedArray[T]) EnsurePageFor ¶ added in v1.34.0
func (p *PagedArray[T]) EnsurePageFor(id uint64) ([]T, int)
EnsurePageFor makes sure that a page exists for the given ID. If the page already exists, it does nothing. It returns the page and its index within the page for the given ID.
func (*PagedArray[T]) GetPageFor ¶ added in v1.34.0
func (p *PagedArray[T]) GetPageFor(id uint64) ([]T, int)
GetPageFor takes an ID and returns the associated page and its index. If the page does not exist, it returns nil. It doesn't return a copy of the page, so modifications to the returned slice will affect the original data. Caller is responsible for ensuring that reads and writes to the returned page are performed safely, using either atomic operations or mutexes.
type PqMaxPool ¶
type PqMaxPool struct {
// contains filtered or unexported fields
}
func NewPqMaxPool ¶
type QueryVectorDistancer ¶ added in v1.26.0
func (*QueryVectorDistancer) Close ¶ added in v1.26.0
func (q *QueryVectorDistancer) Close()
func (*QueryVectorDistancer) DistanceToNode ¶ added in v1.26.0
func (q *QueryVectorDistancer) DistanceToNode(nodeID uint64) (float32, error)
type SearchByDistParams ¶
type SearchByDistParams struct {
// contains filtered or unexported fields
}
func NewSearchByDistParams ¶
func NewSearchByDistParams( offset int, limit int, totalLimit int, maximumSearchLimit int64, ) *SearchByDistParams
func (*SearchByDistParams) Iterate ¶
func (params *SearchByDistParams) Iterate()
func (*SearchByDistParams) MaxLimitReached ¶
func (params *SearchByDistParams) MaxLimitReached() bool
func (*SearchByDistParams) MaximumSearchLimit ¶
func (params *SearchByDistParams) MaximumSearchLimit() int64
func (*SearchByDistParams) OffsetCapacity ¶
func (params *SearchByDistParams) OffsetCapacity(ids []uint64) int
func (*SearchByDistParams) TotalLimit ¶
func (params *SearchByDistParams) TotalLimit() int
func (*SearchByDistParams) TotalLimitCapacity ¶
func (params *SearchByDistParams) TotalLimitCapacity(ids []uint64) int
type Sequence ¶ added in v1.35.0
type Sequence struct {
// contains filtered or unexported fields
}
Sequence represents a monotonic uint64 generator that can be used to generate unique incrementing ids for stored postings and other entities. It is inspired by Postgres Sequences, although much simpler and it's designed to ensure the following properties: 1. Monotonicity: Each generated id is guaranteed to be greater than the previously generated id. 2. Persistence: The state of the generator is persisted to disk. 3. Low disk I/O: The generator is designed to minimize the number of disk writes by allocating ranges of ids in memory. This allows it to serve most requests from memory without touching the disk. 4. Concurrency: The generator is safe for concurrent access. However because of its design, it does not guarantee gapless ids. A Sequence first allocates a range of ids and reserves them in the persistent storage. It then serves those ids from memory without touching the disk. If the sequence is closed gracefully, if persists the last used id, however in case of a crash, the sequence will start from the upper bound of the reserved range potentially leaving gaps. The range is configurable and can be tuned based on the expected throughput requirements.
func NewSequence ¶ added in v1.35.0
func NewSequence(store SequenceStore, rangeSize uint64) (*Sequence, error)
NewSequence loads the upper bound from the store and returns a ready to use Sequence.
func (*Sequence) Flush ¶ added in v1.35.0
Flush persists the last used id to the store, to be used as the next starting point when the sequence is re-opened. This must be called to gracefully close the sequence and avoid gaps after a restart. Callers need to ensure no concurrent calls to Next() are happening while Flush() is in progress.
type SequenceStore ¶ added in v1.35.0
SequenceStore defines the interface for persisting the state of a Sequence. Implementations don't need to be thread-safe.
type ShardedLocks ¶
type ShardedLocks struct {
PageSize uint64
// contains filtered or unexported fields
}
func NewDefaultShardedLocks ¶
func NewDefaultShardedLocks() *ShardedLocks
func NewShardedLocks ¶
func NewShardedLocks(count uint64) *ShardedLocks
func NewShardedLocksWithPageSize ¶ added in v1.28.0
func NewShardedLocksWithPageSize(pageSize uint64) *ShardedLocks
func (*ShardedLocks) Hash ¶ added in v1.34.0
func (sl *ShardedLocks) Hash(id uint64) uint64
func (*ShardedLocks) Lock ¶
func (sl *ShardedLocks) Lock(id uint64)
func (*ShardedLocks) LockAll ¶
func (sl *ShardedLocks) LockAll()
func (*ShardedLocks) Locked ¶
func (sl *ShardedLocks) Locked(id uint64, callback func())
func (*ShardedLocks) LockedAll ¶
func (sl *ShardedLocks) LockedAll(callback func())
func (*ShardedLocks) Unlock ¶
func (sl *ShardedLocks) Unlock(id uint64)
func (*ShardedLocks) UnlockAll ¶
func (sl *ShardedLocks) UnlockAll()
type ShardedRWLocks ¶ added in v1.24.0
type ShardedRWLocks struct {
PageSize uint64
// contains filtered or unexported fields
}
func NewDefaultShardedRWLocks ¶ added in v1.24.0
func NewDefaultShardedRWLocks() *ShardedRWLocks
func NewShardedRWLocks ¶ added in v1.24.0
func NewShardedRWLocks(count uint64) *ShardedRWLocks
func NewShardedRWLocksWith ¶ added in v1.34.0
func NewShardedRWLocksWith(pages, pageSize uint64) *ShardedRWLocks
func NewShardedRWLocksWithPageSize ¶ added in v1.28.0
func NewShardedRWLocksWithPageSize(pageSize uint64) *ShardedRWLocks
func (*ShardedRWLocks) Hash ¶ added in v1.34.0
func (sl *ShardedRWLocks) Hash(id uint64) uint64
func (*ShardedRWLocks) Lock ¶ added in v1.24.0
func (sl *ShardedRWLocks) Lock(id uint64)
func (*ShardedRWLocks) LockAll ¶ added in v1.24.0
func (sl *ShardedRWLocks) LockAll()
func (*ShardedRWLocks) Locked ¶ added in v1.24.0
func (sl *ShardedRWLocks) Locked(id uint64, callback func())
func (*ShardedRWLocks) LockedAll ¶ added in v1.24.0
func (sl *ShardedRWLocks) LockedAll(callback func())
func (*ShardedRWLocks) RLock ¶ added in v1.24.0
func (sl *ShardedRWLocks) RLock(id uint64)
func (*ShardedRWLocks) RLockAll ¶ added in v1.24.0
func (sl *ShardedRWLocks) RLockAll()
func (*ShardedRWLocks) RLocked ¶ added in v1.24.0
func (sl *ShardedRWLocks) RLocked(id uint64, callback func())
func (*ShardedRWLocks) RLockedAll ¶ added in v1.24.0
func (sl *ShardedRWLocks) RLockedAll(callback func())
func (*ShardedRWLocks) RUnlock ¶ added in v1.24.0
func (sl *ShardedRWLocks) RUnlock(id uint64)
func (*ShardedRWLocks) RUnlockAll ¶ added in v1.24.0
func (sl *ShardedRWLocks) RUnlockAll()
func (*ShardedRWLocks) TryLock ¶ added in v1.34.0
func (sl *ShardedRWLocks) TryLock(id uint64) bool
func (*ShardedRWLocks) Unlock ¶ added in v1.24.0
func (sl *ShardedRWLocks) Unlock(id uint64)
func (*ShardedRWLocks) UnlockAll ¶ added in v1.24.0
func (sl *ShardedRWLocks) UnlockAll()
type SharedGauge ¶ added in v1.24.22
type SharedGauge struct {
// contains filtered or unexported fields
}
SharedGauge is a thread-safe gauge that can be shared between multiple goroutines. It is used to track the number of running tasks, and allows to wait until all tasks are done.
func NewSharedGauge ¶ added in v1.24.22
func NewSharedGauge() *SharedGauge
func (*SharedGauge) Count ¶ added in v1.24.22
func (sc *SharedGauge) Count() int64
func (*SharedGauge) Decr ¶ added in v1.24.22
func (sc *SharedGauge) Decr() int64
Decr decrements the gauge and returns the new count.
func (*SharedGauge) Incr ¶ added in v1.24.22
func (sc *SharedGauge) Incr() int64
Incr increments the gauge and returns the new count.
type SparseFisherYatesIterator ¶ added in v1.27.0
type SparseFisherYatesIterator struct {
// contains filtered or unexported fields
}
SparseFisherYatesIterator implements the Iterator interface using the Sparse Fisher-Yates algorithm.
func NewSparseFisherYatesIterator ¶ added in v1.27.0
func NewSparseFisherYatesIterator(size int) *SparseFisherYatesIterator
NewSparseFisherYatesIterator creates a new SparseFisherYatesIterator with the given size.
func (*SparseFisherYatesIterator) IsDone ¶ added in v1.27.0
func (s *SparseFisherYatesIterator) IsDone() bool
IsDone checks if all elements have been sampled.
func (*SparseFisherYatesIterator) Next ¶ added in v1.27.0
func (s *SparseFisherYatesIterator) Next() *int
Next returns the next sampled index using the Sparse Fisher-Yates algorithm.
type TargetTempVectorForID ¶ added in v1.24.11
type TargetTempVectorForID[T []float32 | float32] struct { TargetVector string TempVectorForIDThunk func(ctx context.Context, id uint64, container *VectorSlice, targetVector string) ([]T, error) }
func (TargetTempVectorForID[T]) TempVectorForID ¶ added in v1.24.11
func (t TargetTempVectorForID[T]) TempVectorForID(ctx context.Context, id uint64, container *VectorSlice) ([]T, error)
type TargetTempVectorForIDWithView ¶ added in v1.34.11
type TargetTempVectorForIDWithView[T []float32 | float32] struct { TargetVector string TempVectorForIDWithViewThunk func(ctx context.Context, id uint64, container *VectorSlice, targetVector string, view BucketView) ([]T, error) }
TargetTempVectorForIDWithView wraps a view-aware vector thunk with a target vector name.
func (TargetTempVectorForIDWithView[T]) TempVectorForIDWithView ¶ added in v1.34.11
func (t TargetTempVectorForIDWithView[T]) TempVectorForIDWithView(ctx context.Context, id uint64, container *VectorSlice, view BucketView) ([]T, error)
TempVectorForIDWithView returns the view-aware vector lookup function.
type TargetVectorForID ¶ added in v1.24.11
type TargetVectorForID[T []float32 | float32 | byte | uint64] struct { TargetVector string VectorForIDThunk func(ctx context.Context, id uint64, targetVector string) ([]T, error) }
func (TargetVectorForID[T]) VectorForID ¶ added in v1.24.11
func (t TargetVectorForID[T]) VectorForID(ctx context.Context, id uint64) ([]T, error)
type TempVectorForID ¶
type TempVectorForIDWithView ¶ added in v1.34.11
type TempVectorForIDWithView[T []float32 | float32] func(ctx context.Context, id uint64, container *VectorSlice, view BucketView) ([]T, error)
TempVectorForIDWithView is like TempVectorForID but uses an existing bucket view.
type TempVectorUint64Pool ¶ added in v1.27.0
type TempVectorUint64Pool struct {
// contains filtered or unexported fields
}
func NewTempUint64VectorsPool ¶ added in v1.27.0
func NewTempUint64VectorsPool() *TempVectorUint64Pool
func (*TempVectorUint64Pool) Get ¶ added in v1.27.0
func (pool *TempVectorUint64Pool) Get(capacity int) *VectorUint64Slice
func (*TempVectorUint64Pool) Put ¶ added in v1.27.0
func (pool *TempVectorUint64Pool) Put(container *VectorUint64Slice)
type TempVectorsPool ¶
type TempVectorsPool struct {
// contains filtered or unexported fields
}
func NewTempVectorsPool ¶
func NewTempVectorsPool() *TempVectorsPool
func (*TempVectorsPool) Get ¶
func (pool *TempVectorsPool) Get(capacity int) *VectorSlice
func (*TempVectorsPool) Put ¶
func (pool *TempVectorsPool) Put(container *VectorSlice)
type TestFS ¶ added in v1.34.0
type TestFS struct {
FS
OnOpenFile func(f File) File
OnOpen func(f File) File
OnCreate func(f File) File
OnRename func(oldpath, newpath string) error
OnRemove func(name string) error
}
type TestFile ¶ added in v1.34.0
type UnboundedChannel ¶ added in v1.34.0
type UnboundedChannel[T any] struct { // contains filtered or unexported fields }
func MakeUnboundedChannel ¶ added in v1.34.0
func MakeUnboundedChannel[T any]() *UnboundedChannel[T]
MakeUnboundedChannel creates a channel with unbounded capacity. Any push is guaranteed to succeed without blocking, regardless of the number of pending reads on `out`. The implementation uses a background goroutine that continuously drains an internal queue to the output channel. If the channel is empty, the goroutine sleeps until a new item is added. This channel doesn't implement backpressure and should be used with caution to avoid excessive memory usage.
func (*UnboundedChannel[T]) Close ¶ added in v1.34.0
func (u *UnboundedChannel[T]) Close(ctx context.Context)
Close the channel and initiates the shutdown of the background goroutine. The provided context can be used to cancel the draining of remaining items to the output channel.
func (*UnboundedChannel[T]) Len ¶ added in v1.34.0
func (u *UnboundedChannel[T]) Len() int
Len returns the number of items currently buffered in the channel.
func (*UnboundedChannel[T]) Out ¶ added in v1.34.0
func (u *UnboundedChannel[T]) Out() <-chan T
Out returns a read-only channel from which values can be received.
func (*UnboundedChannel[T]) Push ¶ added in v1.34.0
func (u *UnboundedChannel[T]) Push(v T) bool
Push adds a value to the channel. This operation does not wait for a corresponding read and buffers the value internally until it can be sent. If the channel has been closed, Push returns false.
type Vector ¶ added in v1.29.0
func (*Vector[T]) Validate ¶ added in v1.29.0
func (v *Vector[T]) Validate(vectorIndex VectorIndex) error
type VectorForID ¶
type VectorIndex ¶ added in v1.29.0
type VectorIndexMulti ¶ added in v1.33.0
type VectorRecord ¶ added in v1.28.0
type VectorRecord interface {
Len() int
Validate(vectorIndex VectorIndex) error
}
type VectorSlice ¶
type VectorUint64Slice ¶ added in v1.27.0
type VectorUint64Slice struct {
Slice []uint64
}