flat

package
v1.35.21 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 21, 2026 License: BSD-3-Clause Imports: 30 Imported by: 0

Documentation

Index

Constants

View Source 
const (

	// RQ data serialization version
	RQDataVersion = 1

	// Compression type strings for persistence
	CompressionTypeRQ1 = "rq1"
	CompressionTypeRQ8 = "rq8"
)

Variables

This section is empty.

Functions

func New 

func New(cfg Config, uc flatent.UserConfig, store *lsmkv.Store) (*flat, error)

func ValidateUserConfigUpdate 

func ValidateUserConfigUpdate(initial, updated schemaConfig.VectorIndexConfig) error

Types

type BinaryQuantizerWrapper added in v1.34.0 

type BinaryQuantizerWrapper struct {
	compressionhelpers.BinaryQuantizer
}

BinaryQuantizerWrapper wraps BinaryQuantizer to implement Quantizer interface

func (*BinaryQuantizerWrapper) DistanceBetweenByteVectors added in v1.34.0 

func (b *BinaryQuantizerWrapper) DistanceBetweenByteVectors(x, y []byte) (float32, error)

func (*BinaryQuantizerWrapper) DistanceBetweenUint64Vectors added in v1.34.0 

func (b *BinaryQuantizerWrapper) DistanceBetweenUint64Vectors(x, y []uint64) (float32, error)

func (*BinaryQuantizerWrapper) EncodeBytes added in v1.34.0 

func (b *BinaryQuantizerWrapper) EncodeBytes(vector []float32) []byte

func (*BinaryQuantizerWrapper) EncodeUint64 added in v1.34.0 

func (b *BinaryQuantizerWrapper) EncodeUint64(vector []float32) []uint64

func (*BinaryQuantizerWrapper) FromCompressedBytesToBytes added in v1.34.0 

func (b *BinaryQuantizerWrapper) FromCompressedBytesToBytes(compressed []byte, buffer *[]byte) []byte

func (*BinaryQuantizerWrapper) FromCompressedBytesToUint64 added in v1.34.0 

func (b *BinaryQuantizerWrapper) FromCompressedBytesToUint64(compressed []byte, buffer *[]uint64) []uint64

func (*BinaryQuantizerWrapper) Type added in v1.34.0 

func (b *BinaryQuantizerWrapper) Type() QuantizerType

type BinaryRotationalQuantizerWrapper added in v1.34.0 

type BinaryRotationalQuantizerWrapper struct {
	*compressionhelpers.BinaryRotationalQuantizer
}

BinaryRotationalQuantizerWrapper wraps BinaryRotationalQuantizer to implement Quantizer interface

func (*BinaryRotationalQuantizerWrapper) DistanceBetweenByteVectors added in v1.34.0 

func (r *BinaryRotationalQuantizerWrapper) DistanceBetweenByteVectors(x, y []byte) (float32, error)

func (*BinaryRotationalQuantizerWrapper) DistanceBetweenUint64Vectors added in v1.34.0 

func (r *BinaryRotationalQuantizerWrapper) DistanceBetweenUint64Vectors(x, y []uint64) (float32, error)

func (*BinaryRotationalQuantizerWrapper) EncodeBytes added in v1.34.0 

func (r *BinaryRotationalQuantizerWrapper) EncodeBytes(vector []float32) []byte

func (*BinaryRotationalQuantizerWrapper) EncodeUint64 added in v1.34.0 

func (r *BinaryRotationalQuantizerWrapper) EncodeUint64(vector []float32) []uint64

func (*BinaryRotationalQuantizerWrapper) FromCompressedBytesToBytes added in v1.34.0 

func (r *BinaryRotationalQuantizerWrapper) FromCompressedBytesToBytes(compressed []byte, buffer *[]byte) []byte

func (*BinaryRotationalQuantizerWrapper) FromCompressedBytesToUint64 added in v1.34.0 

func (r *BinaryRotationalQuantizerWrapper) FromCompressedBytesToUint64(compressed []byte, buffer *[]uint64) []uint64

func (*BinaryRotationalQuantizerWrapper) Type added in v1.34.0 

func (r *BinaryRotationalQuantizerWrapper) Type() QuantizerType

type Cache added in v1.34.0 

type Cache struct {
	// contains filtered or unexported fields
}

Cache represents a cache that can work with different data types

func NewCache added in v1.34.0 

func NewCache(getUint64Vector func(ctx context.Context, id uint64) ([]uint64, error), getByteVector func(ctx context.Context, id uint64) ([]byte, error), maxObjects int, logger logrus.FieldLogger, allocChecker memwatch.AllocChecker, quantizerType QuantizerType) *Cache

NewCache creates a new cache

func (*Cache) Delete added in v1.34.0 

func (c *Cache) Delete(ctx context.Context, ids ...uint64)

Delete deletes vectors from the cache

func (*Cache) GetAllBytesInCurrentLock added in v1.34.0 

func (c *Cache) GetAllBytesInCurrentLock(ctx context.Context, start uint64, out [][]byte, errs []error) ([][]byte, []error, uint64, uint64)

GetAllBytesInCurrentLock gets all byte vectors in the current lock

func (*Cache) GetAllUint64InCurrentLock added in v1.34.0 

func (c *Cache) GetAllUint64InCurrentLock(ctx context.Context, start uint64, out [][]uint64, errs []error) ([][]uint64, []error, uint64, uint64)

GetAllUint64InCurrentLock gets all uint64 vectors in the current lock

func (*Cache) GetBytes added in v1.34.0 

func (c *Cache) GetBytes(ctx context.Context, id uint64) ([]byte, error)

GetBytes gets a byte vector from the cache

func (*Cache) GetUint64 added in v1.34.0 

func (c *Cache) GetUint64(ctx context.Context, id uint64) ([]uint64, error)

GetUint64 gets a uint64 vector from the cache

func (*Cache) Grow added in v1.34.0 

func (c *Cache) Grow(id uint64)

Grow grows the cache to accommodate the given ID

func (*Cache) IterateBytesWithAllowlist added in v1.34.0 

func (c *Cache) IterateBytesWithAllowlist(allow helpers.AllowList, action func(uint64, []byte) error) error

func (*Cache) IterateUint64WithAllowlist added in v1.34.0 

func (c *Cache) IterateUint64WithAllowlist(allow helpers.AllowList, action func(uint64, []uint64) error) error

func (*Cache) Len added in v1.34.0 

func (c *Cache) Len() int32

Len returns the current length of the cache

func (*Cache) LockAll added in v1.34.0 

func (c *Cache) LockAll()

LockAll locks all cache operations

func (*Cache) PageSize added in v1.34.0 

func (c *Cache) PageSize() uint64

PageSize returns the page size of the cache

func (*Cache) PreloadBytes added in v1.34.0 

func (c *Cache) PreloadBytes(id uint64, vector []byte)

PreloadBytes preloads a byte vector into the cache

func (*Cache) PreloadNoLockBytes added in v1.34.0 

func (c *Cache) PreloadNoLockBytes(id uint64, vector []byte)

PreloadNoLockBytes preloads a byte vector without locking

func (*Cache) PreloadNoLockUint64 added in v1.34.0 

func (c *Cache) PreloadNoLockUint64(id uint64, vector []uint64)

PreloadNoLockUint64 preloads a uint64 vector without locking

func (*Cache) PreloadUint64 added in v1.34.0 

func (c *Cache) PreloadUint64(id uint64, vector []uint64)

PreloadUint64 preloads a uint64 vector into the cache

func (*Cache) SetSizeAndGrowNoLockBytes added in v1.34.0 

func (c *Cache) SetSizeAndGrowNoLockBytes(maxID uint64)

SetSizeAndGrowNoLockBytes sets the byte cache size and grows without locking

func (*Cache) SetSizeAndGrowNoLockUint64 added in v1.34.0 

func (c *Cache) SetSizeAndGrowNoLockUint64(maxID uint64)

SetSizeAndGrowNoLockUint64 sets the uint64 cache size and grows without locking

func (*Cache) UnlockAll added in v1.34.0 

func (c *Cache) UnlockAll()

UnlockAll unlocks all cache operations

type CompressionType added in v1.34.0 

type CompressionType string

CompressionType represents the type of compression used 

const (
	CompressionNone CompressionType = "none"
	CompressionBQ   CompressionType = "bq"
	CompressionRQ1  CompressionType = "rq-1"
	CompressionRQ8  CompressionType = "rq-8"
)

Only supported quantizers are included

func (CompressionType) IsQuantized added in v1.34.0 

func (ct CompressionType) IsQuantized() bool

IsQuantized returns true if this compression type uses quantization

func (CompressionType) String added in v1.34.0 

func (ct CompressionType) String() string

String returns the string representation of the compression type

type Config 

type Config struct {
	ID                string
	RootPath          string
	TargetVector      string
	Logger            logrus.FieldLogger
	DistanceProvider  distancer.Provider
	AllocChecker      memwatch.AllocChecker
	MakeBucketOptions lsmkv.MakeBucketOptions
}

func (Config) Validate 

func (c Config) Validate() error

type Quantizer added in v1.34.0 

type Quantizer interface {
	EncodeUint64(vector []float32) []uint64
	EncodeBytes(vector []float32) []byte

	DistanceBetweenUint64Vectors(x, y []uint64) (float32, error)
	DistanceBetweenByteVectors(x, y []byte) (float32, error)

	PersistCompression(logger compressionhelpers.CommitLogger)
	Stats() compressionhelpers.CompressionStats
	Type() QuantizerType

	FromCompressedBytesToUint64(compressed []byte, buffer *[]uint64) []uint64
	FromCompressedBytesToBytes(compressed []byte, buffer *[]byte) []byte
}

Quantizer represents a quantizer that can work with different data types

type QuantizerBuilder added in v1.34.0 

type QuantizerBuilder struct {
	// contains filtered or unexported fields
}

QuantizerBuilder creates quantizers based on configuration

func NewQuantizerBuilder added in v1.34.0 

func NewQuantizerBuilder(distancerProvider distancer.Provider) *QuantizerBuilder

NewQuantizerBuilder creates a new quantizer builder

func (*QuantizerBuilder) CreateQuantizer added in v1.34.0 

func (qb *QuantizerBuilder) CreateQuantizer(compression CompressionType, dimensions int32) Quantizer

CreateQuantizer creates a quantizer based on the compression type and dimensions

type QuantizerType added in v1.34.0 

type QuantizerType int

QuantizerType represents the type of quantizer 

const (
	Uint64Quantizer QuantizerType = iota
	ByteQuantizer
)

type RQ1Data added in v1.34.0 

type RQ1Data struct {
	InputDim  uint32                      `msgpack:"input_dim"`
	OutputDim uint32                      `msgpack:"output_dim"`
	Rounds    uint32                      `msgpack:"rounds"`
	Swaps     [][]compressionhelpers.Swap `msgpack:"swaps"`
	Signs     [][]float32                 `msgpack:"signs"`
	Rounding  []float32                   `msgpack:"rounding"`
}

RQ1Data represents RQ1 (Binary Rotational Quantization) data

type RQ8Data added in v1.34.0 

type RQ8Data struct {
	InputDim  uint32                      `msgpack:"input_dim"`
	Bits      uint32                      `msgpack:"bits"`
	OutputDim uint32                      `msgpack:"output_dim"`
	Rounds    uint32                      `msgpack:"rounds"`
	Swaps     [][]compressionhelpers.Swap `msgpack:"swaps"`
	Signs     [][]float32                 `msgpack:"signs"`
}

RQ8Data represents RQ8 (8-bit Rotational Quantization) data

type RQDataContainer added in v1.34.0 

type RQDataContainer struct {
	Version         uint32      `msgpack:"version"`          // Serialization format version
	CompressionType string      `msgpack:"compression_type"` // Compression type for validation
	Data            interface{} `msgpack:"data"`             // The actual RQ data
}

RQDataContainer wraps RQ data with metadata for safe persistence

type RotationalQuantizerWrapper added in v1.34.0 

type RotationalQuantizerWrapper struct {
	*compressionhelpers.RotationalQuantizer
}

RotationalQuantizerWrapper wraps RotationalQuantizer to implement Quantizer interface

func (*RotationalQuantizerWrapper) DistanceBetweenByteVectors added in v1.34.0 

func (b *RotationalQuantizerWrapper) DistanceBetweenByteVectors(x, y []byte) (float32, error)

func (*RotationalQuantizerWrapper) DistanceBetweenUint64Vectors added in v1.34.0 

func (b *RotationalQuantizerWrapper) DistanceBetweenUint64Vectors(x, y []uint64) (float32, error)

func (*RotationalQuantizerWrapper) EncodeBytes added in v1.34.0 

func (b *RotationalQuantizerWrapper) EncodeBytes(vector []float32) []byte

func (*RotationalQuantizerWrapper) EncodeUint64 added in v1.34.0 

func (b *RotationalQuantizerWrapper) EncodeUint64(vector []float32) []uint64

func (*RotationalQuantizerWrapper) FromCompressedBytesToBytes added in v1.34.0 

func (b *RotationalQuantizerWrapper) FromCompressedBytesToBytes(compressed []byte, buffer *[]byte) []byte

func (*RotationalQuantizerWrapper) FromCompressedBytesToUint64 added in v1.34.0 

func (b *RotationalQuantizerWrapper) FromCompressedBytesToUint64(compressed []byte, buffer *[]uint64) []uint64

func (*RotationalQuantizerWrapper) Type added in v1.34.0 

func (b *RotationalQuantizerWrapper) Type() QuantizerType

type SliceStruct 

type SliceStruct[T any] struct {
	// contains filtered or unexported fields
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.