worker

type Options struct {
	// Pool is released before every source call, so one connection
	// suffices. Required.
	Pool   *pgxpool.Pool
	Schema string
	Tenant string

	// SupportedLanguages are the document languages backfill enumerates. Required.
	SupportedLanguages []string
	// Sources build the documents, each of its kinds; a dirty row of a kind
	// no source documents deletes its document. Required.
	Sources []Source

	// Batch sizing (defaults are conservative).
	DirtyBatchSize   int
	BackfillPageSize int
	// Upper bound on how much cursor backfill work to do per SyncOnce.
	BackfillMaxPages int
}

type Source interface {
	// DocumentKinds are the content kinds it documents.
	DocumentKinds() []string
	// ListDocuments enumerates documented references of one kind and
	// language for backfill: one bounded page from an opaque cursor, done
	// when exhausted.
	ListDocuments(ctx context.Context, kind, language, cursor string, limit int) (refs []contentref.ContentRef, nextCursor string, done bool, err error)
	// BuildDocuments returns the current document of every reference that
	// has one in language; a missing one deletes its document. The worker
	// records invalid documents individually; an error is only for a failed
	// or incomplete read, never a partial result.
	BuildDocuments(ctx context.Context, kind, language string, refs []contentref.ContentRef) ([]search.KeywordDocument, error)
}