eval

type Agreement struct {
	Compared int `json:"compared"`
	Same     int `json:"same"`
	// Unanswered counts pairs one side or the other did not judge.
	Unanswered int `json:"unanswered"`
	// Confusion[a][b] is how often the first set said a and the second said b.
	Confusion map[string]map[string]int `json:"confusion"`

	// SameEffect counts pairs where the two verdicts do the same thing to the graph,
	// whether or not they used the same word. See verifier.Relation.EffectOf: five
	// relations collapse to three effects, because "supports", "refines" and
	// "unrelated" are all read by nothing.
	//
	// This is the number to judge a judge by, and it is not a softer version of
	// Rate. Measured on claude-haiku-4-5 over 37 pairs judged twice: 84% raw, 97%
	// by effect. Five of the six disagreements were supports/refines/unrelated
	// shuffles that change nothing downstream; one was contradicts-vs-supports,
	// which changes a confidence score and queues a research lead. Acting on 84%
	// would mean rewriting a prompt to chase six problems when there was one.
	SameEffect int `json:"same_effect"`
}

type CitationProblem struct {
	Verdict Verdict `json:"verdict"`
	Source  string  `json:"source"`
	Claim   string  `json:"claim"`
	Detail  string  `json:"detail"`
}

type CitationReport struct {
	Verified    int `json:"verified"`
	Mismatch    int `json:"mismatch"`
	OffsetDrift int `json:"offset_drift"`
	Unreachable int `json:"unreachable"`
	Skipped     int `json:"skipped"`

	Problems []CitationProblem `json:"problems,omitempty"`
}

type Corpus struct {
	Name      string     `json:"name"`
	Questions []Question `json:"questions"`
}

type CorpusReport struct {
	Corpus  string           `json:"corpus"`
	Results []QuestionResult `json:"results"`

	// Aggregate holds the mean of every metric measured on at least one question.
	Aggregate []Metric `json:"aggregate"`
}

type Delta struct {
	Name     string  `json:"name"`
	Baseline float64 `json:"baseline"`
	Current  float64 `json:"current"`
	Change   float64 `json:"change"`
	Unit     string  `json:"unit,omitempty"`
}

type DumpOptions struct {
	// Kinds restricts the dump to pairs the model gave these verdicts. Empty means every
	// pair that carries an edge.
	Kinds []string
	// All includes pairs with no edge — the ones the model called unrelated.
	//
	// Off by default because they dominate: a 25-claim session produces a few dozen
	// edges and several hundred candidate pairs, and a labelling task nobody finishes
	// measures nothing. Turn it on to measure RECALL — whether real disagreements were
	// missed — once precision is known.
	All bool
	// MaxCandidatesPerClaim mirrors the Verifier's retrieval cap, so the candidate space
	// is regenerated exactly as the run saw it.
	MaxCandidatesPerClaim int
}

type JudgeOptions struct {
	Model     string
	BatchSize int
	// Kinds restricts which stored verdicts get re-judged. Empty means all of them.
	Kinds []string
	// Labels carries labels across from an earlier set, matched by pair key.
	//
	// The reason the whole harness is worth having: label once, and every future model
	// or prompt is scored against the same judgements rather than needing the work done
	// again.
	Labels map[string]string
}

type LabelledPair struct {
	// Pair is the canonical key, stable across dumps of the same session.
	Pair string `json:"pair"`

	A string `json:"a"`
	B string `json:"b"`

	SourceA string `json:"source_a,omitempty"`
	SourceB string `json:"source_b,omitempty"`

	// Model is what the adjudicator said. "unrelated" means no edge was stored, which
	// covers both "judged unrelated" and "never reached" — Judged distinguishes them.
	Model string `json:"model"`
	// Baseline is what the STORED graph said, when this set came from re-judging with a
	// different model. Kept so a comparison shows both verdicts on one line.
	Baseline string `json:"baseline,omitempty"`
	// Why is the model's own rationale, which is often the clearest evidence that a
	// verdict is wrong.
	Why string `json:"why,omitempty"`

	// Judged is false when one of the claims was never verified, so the pair was never
	// put to the model at all. Scoring skips those: they say nothing about the judge.
	Judged bool `json:"judged"`

	// Label is the truth, filled in by hand. Empty means unlabelled.
	Label string `json:"label"`
	// Note is for the labeller.
	Note string `json:"note,omitempty"`
}

type Metric struct {
	Name   string `json:"name"`
	Status Status `json:"status"`

	// Value and Unit are set when Status is Measured.
	Value float64 `json:"value,omitempty"`
	Unit  string  `json:"unit,omitempty"`

	// Detail is the arithmetic behind the number, so a reader can check it.
	Detail string `json:"detail,omitempty"`
	// Reason explains a Blocked status: what is missing and which milestone
	// brings it.
	Reason string `json:"reason,omitempty"`

	// Regression marks a metric that should fail a build. Only ever set on
	// metrics that are objectively wrong, never on quality judgements.
	Regression bool `json:"regression,omitempty"`
}

type Options struct {
	// Citations, when set, re-reads every cited source and checks the quote is
	// actually in it (§14.3 citation accuracy).
	//
	// Off by default because it costs a fetch per source. Under cassette
	// replay it is free and deterministic, which is what §14.1 was built for.
	Citations SourceReader
}

type PairScore struct {
	Labelled   int `json:"labelled"`
	Unlabelled int `json:"unlabelled"`
	// Skipped is pairs that were never put to the model.
	Skipped int `json:"skipped"`
	Correct int `json:"correct"`

	// Confusion[modelVerdict][trueLabel] is how often the model said one thing and the
	// truth was another.
	Confusion map[string]map[string]int `json:"confusion"`

	// Precision and Recall per relation. Precision is what matters for a judge that
	// over-reports: of the pairs it called X, how many were X.
	Precision map[string]float64 `json:"precision"`
	Recall    map[string]float64 `json:"recall"`

	// CorrectEffect counts verdicts that do the right thing to the graph even when
	// they used the wrong word, on the same reasoning as Agreement.SameEffect.
	//
	// Measured on claude-haiku-4-5 over 37 labelled pairs: 76% raw, 97% by effect.
	// Eight of its nine errors were calling an unrelated pair "supports" or
	// "refines", which builds an edge nothing reads. The ninth was a false
	// contradiction, which costs a confidence penalty and a research lead.
	CorrectEffect int `json:"correct_effect"`
}

type PairSet struct {
	Session string         `json:"session"`
	Model   string         `json:"adjudicator,omitempty"`
	Pairs   []LabelledPair `json:"pairs"`
}

type PipelineReader struct {
	Fetch   fetch.Fetcher
	Extract extract.Extractor
}

type Question struct {
	// ID is stable across runs so a baseline can be compared question by question.
	ID       string `json:"id"`
	Question string `json:"question"`

	// Tags group results in the aggregate. §14.2's categories — settled, disputed,
	// stale, plausible-but-wrong — are the intended vocabulary, but nothing here
	// enforces a list: a corpus that cannot describe its own questions is less useful
	// than one with an unexpected tag in it.
	Tags []string `json:"tags,omitempty"`

	// Notes is for whoever maintains the corpus. Never sent to a model.
	Notes string `json:"notes,omitempty"`
}

type QuestionResult struct {
	ID       string   `json:"id"`
	Question string   `json:"question"`
	Tags     []string `json:"tags,omitempty"`

	SessionID string `json:"session_id,omitempty"`
	// Err is set when the run itself failed. Distinct from a run that produced a poor
	// answer: one is a broken pipeline, the other is a research outcome, and folding
	// them together is how a corpus reports 100% success on a build that cannot run.
	Err string `json:"error,omitempty"`

	// Status and Claims are how a corpus tells "ran and found nothing" from "ran".
	//
	// Neither is visible in the scorecard: its hard regressions are budget overshoot,
	// ledger drift and stranded holds, and a session that produced zero claims has none
	// of them. The first replay through this runner reported a green tick for a run
	// whose planner call had failed outright — the exact regression the runner exists to
	// catch, passing its own gate.
	Status string `json:"status,omitempty"`
	Claims int    `json:"claims"`

	Card Scorecard `json:"scorecard"`
}

type Scorecard struct {
	SessionID string   `json:"session_id"`
	Metrics   []Metric `json:"metrics"`

	// Citations is the per-claim detail behind the citation accuracy metric,
	// present only when it was computed.
	Citations *CitationReport `json:"citations,omitempty"`
}

type SourceReader interface {
	Text(ctx context.Context, rawURL string) (string, error)
}

type Status string

type StoredReader struct {
	Store     store.Store
	SessionID string
	// Fallback reads a source the store does not have. Nil means such a source is
	// simply unreadable, which is the honest answer when there is no network path
	// configured.
	Fallback SourceReader
	// contains filtered or unexported fields
}

type Verdict string