eval

type Config struct {
	LLM      provider.LLM
	Embedder provider.Embedder
	Store    store.Store
	Judge    provider.LLM // semantic match oracle; defaults to LLM
	K        int          // search top-k; defaults to 3
	Versions []string     // prompt versions; defaults to all registered
}

type Fixture struct {
	Name string `json:"name"`
	// Messages is the conversation fed to Add.
	Messages []types.Message `json:"messages"`
	// ExpectedFacts are the durable facts a good extractor should produce.
	ExpectedFacts []string `json:"expected_facts"`
	// Queries probe search recall (optional).
	Queries []Query `json:"queries"`
	// RequiredTokens are substrings that must survive verbatim into some
	// extracted fact — the deterministic specificity check (optional).
	RequiredTokens []string `json:"required_tokens,omitempty"`
	// SkipDedup opts a fixture out of the dedup-correctness check (e.g. the
	// nothing-to-extract case, where there is nothing to re-dedup).
	SkipDedup bool `json:"skip_dedup,omitempty"`
}

type Judge struct {
	LLM provider.LLM
}

type Query struct {
	Q            string   `json:"q"`
	ShouldRecall []string `json:"should_recall"`
}

type Result struct {
	Name      string
	Extracted []string

	Recall    float64
	Precision float64

	HasSpecificity bool
	Specificity    float64

	HasSearch    bool
	SearchRecall float64

	HasDedup      bool
	DedupNewFacts int // 0 is ideal: re-ingesting added nothing new
}

type VersionReport struct {
	Version string
	Results []Result
}