hnsw

package
v0.0.0-debug-20260702 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 2, 2026 License: Apache-2.0 Imports: 29 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func QuantizationToUsearch 

func QuantizationToUsearch(typid int32) (usearch.Quantization, error)

Types

type AddItem 

type AddItem[T types.RealNumbers] struct {
	// contains filtered or unexported fields
}

type HnswBuild 

type HnswBuild[T types.RealNumbers] struct {
	// contains filtered or unexported fields
}

func NewHnswBuild 

func NewHnswBuild[T types.RealNumbers](sqlproc *sqlexec.SqlProcess, uid string, nworker int32,
	cfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig) (info *HnswBuild[T], err error)

create HsnwBuild struct

func (*HnswBuild[T]) Add 

func (h *HnswBuild[T]) Add(key int64, vec []T) error

func (*HnswBuild[T]) CloseAndWait 

func (h *HnswBuild[T]) CloseAndWait() error

CloseAndWait closes the work queue, waits for all workers to drain it, and returns the first worker error (nil on success). It is idempotent; later calls return the same recorded error.

func (*HnswBuild[T]) Destroy 

func (h *HnswBuild[T]) Destroy() error

destroy

func (*HnswBuild[T]) GetIndexes 

func (h *HnswBuild[T]) GetIndexes() []*HnswModel[T]

func (*HnswBuild[T]) ToInsertSql 

func (h *HnswBuild[T]) ToInsertSql(ts int64) ([]string, error)

generate SQL to update the secondary index tables 1. sync the metadata table 2. sync the index file to index table

func (*HnswBuild[T]) WorkerErr 

func (h *HnswBuild[T]) WorkerErr() error

WorkerErr returns the recorded worker error (nil if none). Safe to call any time.

type HnswModel 

type HnswModel[T types.RealNumbers] struct {
	Id       string
	Index    *usearch.Index
	Path     string
	FileSize int64

	// info required for build
	MaxCapacity uint
	NThread     uint

	// from metadata.  info required for search
	Timestamp int64
	Checksum  string

	// for cdc update
	Dirty atomic.Bool
	View  bool
	Len   atomic.Int64
	// contains filtered or unexported fields
}

HnswModel struct - This structure shares with Search, Build and Sync

func LoadMetadata 

func LoadMetadata[T types.RealNumbers](sqlproc *sqlexec.SqlProcess, dbname string, metatbl string) ([]*HnswModel[T], error)

load metadata from database

func NewHnswModelForBuild 

func NewHnswModelForBuild[T types.RealNumbers](id string, cfg vectorindex.IndexConfig, nthread int, max_capacity uint) (*HnswModel[T], error)

New HnswModel struct

func (*HnswModel[T]) Add 

func (idx *HnswModel[T]) Add(key int64, vec []T) error

add vector to the index

func (*HnswModel[T]) AddWithoutIncr 

func (idx *HnswModel[T]) AddWithoutIncr(key int64, vec []T) error

add vector without increment the counter. concurrency add will increment the counter before Add

func (*HnswModel[T]) Contains 

func (idx *HnswModel[T]) Contains(key int64) (found bool, err error)

contains key

func (*HnswModel[T]) Destroy 

func (idx *HnswModel[T]) Destroy() error

Destroy the struct

func (*HnswModel[T]) Empty 

func (idx *HnswModel[T]) Empty() (bool, error)

is the index empty

func (*HnswModel[T]) Full 

func (idx *HnswModel[T]) Full() (bool, error)

check the index is full, i.e. 10K vectors

func (*HnswModel[T]) LoadIndex 

func (idx *HnswModel[T]) LoadIndex(
	sqlproc *sqlexec.SqlProcess,
	idxcfg vectorindex.IndexConfig,
	tblcfg vectorindex.IndexTableConfig,
	nthread int64,
	view bool) (err error)

load index from database TODO: loading file is tricky. 1. we need to know the size of the file. 2. Write Zero to file to have a pre-allocated size 3. SELECT chunk_id, data from index_table WHERE index_id = id. Result will be out of order 4. according to the chunk_id, seek to the offset and write the chunk 5. check the checksum to verify the correctness of the file

func (*HnswModel[T]) LoadIndexFromBuffer 

func (idx *HnswModel[T]) LoadIndexFromBuffer(
	sqlproc *sqlexec.SqlProcess,
	idxcfg vectorindex.IndexConfig,
	tblcfg vectorindex.IndexTableConfig,
	nthread int64,
	view bool) (err error)

func (*HnswModel[T]) Remove 

func (idx *HnswModel[T]) Remove(key int64) error

remove key

func (*HnswModel[T]) SaveToFile 

func (idx *HnswModel[T]) SaveToFile() error

Save the index to file

func (*HnswModel[T]) Search 

func (idx *HnswModel[T]) Search(query []T, limit uint) (keys []usearch.Key, distances []float32, err error)

Call usearch.Search

func (*HnswModel[T]) ToDeleteSql 

func (idx *HnswModel[T]) ToDeleteSql(cfg vectorindex.IndexTableConfig) ([]string, error)

func (*HnswModel[T]) ToSql 

func (idx *HnswModel[T]) ToSql(cfg vectorindex.IndexTableConfig) ([]string, error)

Generate the SQL to update the secondary index tables. 1. store the index file into the index table

func (*HnswModel[T]) Unload 

func (idx *HnswModel[T]) Unload() error

unload

type HnswSearch 

type HnswSearch[T types.RealNumbers] struct {
	Idxcfg        vectorindex.IndexConfig
	Tblcfg        vectorindex.IndexTableConfig
	Indexes       []*HnswModel[T]
	Concurrency   atomic.Int64
	Mutex         sync.Mutex
	Cond          *sync.Cond
	ThreadsSearch int64
}

This is the HNSW search implementation that implement VectorIndexSearchIf interface

func NewHnswSearch 

func NewHnswSearch[T types.RealNumbers](idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig) *HnswSearch[T]

func (*HnswSearch[T]) Contains 

func (s *HnswSearch[T]) Contains(key int64) (bool, error)

func (*HnswSearch[T]) Destroy 

func (s *HnswSearch[T]) Destroy()

Destroy HnswSearch (implement VectorIndexSearch.Destroy)

func (*HnswSearch[T]) Load 

func (s *HnswSearch[T]) Load(sqlproc *sqlexec.SqlProcess) error

load index from database (implement VectorIndexSearch.LoadFromDatabase)

func (*HnswSearch[T]) LoadIndex 

func (s *HnswSearch[T]) LoadIndex(sqlproc *sqlexec.SqlProcess, indexes []*HnswModel[T]) ([]*HnswModel[T], error)

load index from database

func (*HnswSearch[T]) Search 

func (s *HnswSearch[T]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt vectorindex.RuntimeConfig) (keys any, distances []float64, err error)

Search the hnsw index (implement VectorIndexSearch.Search)

func (*HnswSearch[T]) SearchFloat32 

func (s *HnswSearch[T]) SearchFloat32(proc *sqlexec.SqlProcess, query any, rt vectorindex.RuntimeConfig, outKeys []int64, outDists []float32) error

check config and update some parameters such as ef_search

func (*HnswSearch[T]) UpdateConfig 

func (s *HnswSearch[T]) UpdateConfig(newalgo cache.VectorIndexSearchIf) error

type HnswSync 

type HnswSync[T types.RealNumbers] struct {
	// contains filtered or unexported fields
}

func NewHnswSync 

func NewHnswSync[T types.RealNumbers](sqlproc *sqlexec.SqlProcess,
	db string,
	tbl string,
	idxname string,
	idxdefs []*plan.IndexDef,
	vectype int32,
	dimension int32) (*HnswSync[T], error)

func (*HnswSync[T]) Destroy 

func (s *HnswSync[T]) Destroy()

func (*HnswSync[T]) DownloadAll 

func (s *HnswSync[T]) DownloadAll(sqlproc *sqlexec.SqlProcess) (err error)

func (*HnswSync[T]) RunOnce 

func (s *HnswSync[T]) RunOnce(sqlproc *sqlexec.SqlProcess, cdc *vectorindex.VectorIndexCdc[T]) (err error)

func (*HnswSync[T]) Save 

func (s *HnswSync[T]) Save(sqlproc *sqlexec.SqlProcess) error

func (*HnswSync[T]) ToSql 

func (s *HnswSync[T]) ToSql(ts int64) ([]string, error)

generate SQL to update the secondary index tables 1. sync the metadata table 2. sync the index file to index table

func (*HnswSync[T]) Update 

func (s *HnswSync[T]) Update(sqlproc *sqlexec.SqlProcess, cdc *vectorindex.VectorIndexCdc[T]) error

Source Files

View all Source files

Directories

Path Synopsis
Package plugin is the HNSW vector index integration.
 Package plugin is the HNSW vector index integration.
compile
Package compile implements the HNSW plugin's compile-layer (DDL) hooks.
 Package compile implements the HNSW plugin's compile-layer (DDL) hooks.
idxcron
Package idxcron is HNSW's idxcron hook implementation.
 Package idxcron is HNSW's idxcron hook implementation.
iscp
Package iscp provides HNSW's ISCP hook layer: writer construction and consumer-loop dispatch.
 Package iscp provides HNSW's ISCP hook layer: writer construction and consumer-loop dispatch.
plan
Package plan implements the HNSW plugin's plan-layer hooks.
 Package plan implements the HNSW plugin's plan-layer hooks.
runtime
Package runtime holds the HNSW algorithm's catalog-side metadata.
 Package runtime holds the HNSW algorithm's catalog-side metadata.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.