eval

type AgentTarget struct {
	// Count is what a correct measurement of this defect returns.
	Count int64 `yaml:"count"`
	// Equivalent are other true measurements of the same defect.
	//
	// Two orphaned rows sharing one bad code are two rows and one value, and a
	// model that counts distinct offenders rather than affected rows has found
	// the same defect. Where a target admits more than one true figure the
	// manifest has to say so: the alternative is a correct model scoring zero,
	// which would show up as the model's failure rather than the fixture's.
	//
	// This is a concession and it should stay a small one. A target whose
	// figure genuinely depends on how the question is asked is a target that
	// cannot be credited reliably, and the fixture is usually the thing to fix.
	Equivalent []int64 `yaml:"equivalent"`
	// Query is a SELECT returning Count, so the manifest's own claim can be
	// re-run rather than believed.
	//
	// It is not shown to the model and it is not how a finding is credited —
	// the model has to write its own query and the engine has to run that one.
	// It is here because a target with a wrong count is a target nothing can
	// ever match, and the eval would report a model missing it forever without
	// anything saying why. Evidence that re-runs is the rule everywhere else in
	// Veritix, and a manifest making claims about a dataset is not the place to
	// make it an exception.
	Query string `yaml:"query"`
}

type ChecksScore struct {
	// Found are the ids of defects their nominated rule caught.
	Found []string
	// Missed are the defects a rule was supposed to catch and did not.
	Missed []Defect
	// FalsePositives are clean locations where a check fired anyway.
	FalsePositives []Clean
	// Uncovered are the defects no check proposes, listed rather than
	// counted: they are not failures of the deterministic pass, they are the
	// reason the agentic tier exists.
	Uncovered []Defect
	// Converted are agent targets an accepted rule now measures, so the
	// deterministic pass finds them and no model is needed.
	//
	// They are kept apart from Found, which credits a built-in check named in
	// the manifest, and apart from the model's recall, which stays
	// agent-origin only. Folding either way would lose the fact worth
	// reporting: a target here was the agent's to find and is not any more,
	// because somebody accepted a rule. That is the whole return on paying a
	// model to audit data — once per class of defect rather than once per
	// audit — and it is invisible in every other number on the scorecard.
	Converted []string
}

type Claim struct {
	Rule  string
	Where string
	Count int64
	// Covers names the deterministic rule already responsible for this
	// location, when one is. A model spending its budget restating what the
	// checks found is not wrong, but it is not doing the job the agentic tier
	// was added for, and averaging that into "unclassified" would hide it.
	Covers string
	// Known is a manifest noise entry's reason, when the claim is one somebody
	// has already adjudicated. It explains the claim; it does not grade it.
	Known string
}

type Clean struct {
	// Rule is the check that must not fire here.
	Rule string `yaml:"rule"`
	// Where is the location, in the same form as Defect.Where.
	Where string `yaml:"where"`
	// Why explains why this location is clean.
	Why string `yaml:"why"`
}

type Context struct {
	// ID names the document, and is what a target's NeedsContext refers to.
	ID string `yaml:"id"`
	// File is its path relative to the manifest's own directory.
	File string `yaml:"file"`
	// Why says what this document carries and which targets need it.
	Why string `yaml:"why"`
}

type Defect struct {
	// ID names the defect. It appears in the scorecard and is stable.
	ID string `yaml:"id"`
	// Where is the location, as "<display>" or "<display>.<column>".
	Where string `yaml:"where"`
	// Why explains the defect to a person reading a failure.
	Why string `yaml:"why"`
	// CaughtBy is the deterministic rule that must find it, or "none" when no
	// check proposes it.
	CaughtBy string `yaml:"caught_by"`
	// Agent is set when a model is expected to find this one.
	Agent *AgentTarget `yaml:"agent"`
	// NeedsContext names the context documents without which this defect is
	// invisible. Empty means the export alone is enough to find it.
	//
	// It is on the defect rather than on the target because it is a claim
	// about the dataset, not about the model: these rows are indistinguishable
	// from correct ones until something outside the export says what the
	// column is for. Splitting the scorecard on it is what turns "context
	// helped" from an impression into a number, and — because a fixture also
	// carries targets that need nothing — lets the same run show whether
	// filling the transcript with documents cost the model the ones it could
	// already find.
	NeedsContext []string `yaml:"needs_context"`
}

type Manifest struct {
	// Version is the document format version. Only 1 exists.
	Version int `yaml:"version"`
	// Dataset names the fixture, for the scorecard's header.
	Dataset string `yaml:"dataset"`
	// Description says what the dataset is for.
	Description string `yaml:"description"`
	// Defects are the problems planted in it.
	Defects []Defect `yaml:"defects"`
	// Clean are places a check must stay quiet. A check that fires on
	// everything is useless, and only this half of the manifest catches one.
	Clean []Clean `yaml:"clean"`
	// Noise are true observations that are not defects, so the scorecard can
	// say so instead of asking a reader to adjudicate the same claim again
	// every run.
	Noise []Noise `yaml:"noise"`
	// Context are the customer's own documents, for a fixture whose defects
	// are not all in the data.
	Context []Context `yaml:"context"`

	// Dir is the directory the manifest was loaded from, so a context
	// document's relative path can be resolved. It is not part of the
	// document.
	Dir string `yaml:"-"`
}

type Noise struct {
	// Where is the location, in the same form as Defect.Where.
	Where string `yaml:"where"`
	// Count is the figure the engine returns for it.
	Count int64 `yaml:"count"`
	// Why explains why this is not a defect.
	Why string `yaml:"why"`
}

type Options struct {
	// Paths are the files and directories to audit as one dataset.
	Paths []string
	// Manifest is the ground truth to score against.
	Manifest *Manifest
	// Engine configures the DuckDB instance and its limits.
	Engine config.Engine
	// Profile controls the depth of profiling.
	Profile profile.Options
	// Rules are expectations to apply before scoring — in practice, rules
	// accepted from an earlier run's proposals.
	//
	// This is what makes the rule-proposal loop measurable rather than merely
	// plausible. A model finds a defect on one run in three; a rule accepted
	// from that run finds it on every run, with no model. Scoring an audit
	// with those rules loaded is the only way to see the conversion, and the
	// scorecard reports it separately from what the model found so that an
	// accepted rule can never be mistaken for the model earning its keep.
	Rules *rules.File
	// Agent is the model under evaluation. Nil scores the deterministic
	// auditor alone, which is a useful thing to measure and the only thing CI
	// can measure without a model.
	Agent *agent.Options
	// Context names the MCP servers the agent may read the customer's own
	// documents from. It is what the aided half of a scorecard measures: the
	// same fixture, the same model, scored with the documents reachable and
	// without them.
	Context config.Context
	// Runs is how many times to audit the dataset. One is enough for the
	// deterministic pass and is not enough for a model.
	Runs int
}

type RunScore struct {
	// Detected and Missed are agent-target defect ids.
	Detected []string
	Missed   []string
	// Unclassified are agent findings that matched no target.
	Unclassified []Claim
	// Checks is how the deterministic pass did on the same run.
	Checks ChecksScore
	// Trace is what the agent run cost, when one ran.
	Trace *agent.Trace
	// Err is set when the run itself failed.
	Err string
}

type Score struct {
	Dataset  string
	Provider string
	Model    string
	Runs     []RunScore
	Targets  []TargetScore
	// Checks is the deterministic result, taken from the first run that
	// completed. It is reported once because it is supposed to be the same
	// every time.
	Checks ChecksScore
	// ChecksUnstable is set when it was not. That is a defect in Veritix
	// rather than a score for the model, and it invalidates the run it appears
	// in, so it is surfaced rather than averaged away.
	ChecksUnstable bool
}

type TargetScore struct {
	Defect Defect
	// Hits is how many runs found it, out of Runs.
	Hits int
	Runs int
}