indexing

type DocHash struct {
	// ID of the document.
	ID string
	// ContentHash is the hash of the document content and metadata.
	ContentHash string
}

type Document struct {
	// ID is an optional stable identifier for the document.
	ID string
	// PageContent is the text of the document.
	PageContent string
	// Metadata holds arbitrary metadata used in the content hash.
	Metadata map[string]any
}

type InMemoryRecordManager struct {
	// contains filtered or unexported fields
}

type InMemoryVectorStore struct {
	// contains filtered or unexported fields
}

type IndexResult struct {
	// Added documents newly indexed.
	Added []Document
	// Updated documents whose content changed.
	Updated []Document
	// Skipped documents already up to date.
	Skipped []Document
	// Deleted documents removed from the store.
	Deleted []string
}

type Indexer struct {
	// RecordManager tracks indexed documents.
	RecordManager RecordManager
	// Store is the destination vector store.
	Store VectorStore
	// Namespace separates different indexes.
	Namespace string
	// CleanupMode controls handling of documents no longer present. "incremental"
	// deletes store entries whose ids are no longer in the source.
	CleanupMode string
	// WithID assigns ids to documents. If nil, the content hash is used as id.
	WithID func(doc Document) string
}

type RecordManager interface {
	// Update records the given document hashes as indexed.
	Update(ctx context.Context, namespace string, docs []DocHash) error
	// Exists reports whether a document id with the given content hash is
	// already indexed.
	Exists(ctx context.Context, namespace string, id string) (bool, error)
	// GetHash returns the stored content hash for an id, or "" if absent.
	GetHash(ctx context.Context, namespace string, id string) (string, error)
	// Delete removes document ids from the index.
	Delete(ctx context.Context, namespace string, ids []string) error
	// Keys returns all indexed ids in the namespace.
	Keys(ctx context.Context, namespace string) ([]string, error)
}

type VectorStore interface {
	// AddDocuments stores documents and returns their ids.
	AddDocuments(ctx context.Context, docs []Document) ([]string, error)
	// Delete removes documents by id.
	Delete(ctx context.Context, ids []string) error
}