bench

type Answer struct {
	QuestionID string    `json:"question_id"`
	Condition  Condition `json:"condition"`
	Text       string    `json:"text"`
	Usage      llm.Usage `json:"usage"`
	LatencyMS  int64     `json:"latency_ms"`
	Refused    bool      `json:"refused,omitempty"`
	Error      string    `json:"error,omitempty"`

	// Accuracy and FactsFound are filled by the grader.
	Accuracy    float64  `json:"accuracy"`
	FactsFound  []string `json:"facts_found,omitempty"`
	FactsMissed []string `json:"facts_missed,omitempty"`
	GraderNote  string   `json:"grader_note,omitempty"`

	// Graded records that a grade was actually reached.
	//
	// An answer that was produced but never graded -- the grader rate limited,
	// out of quota, or returning unparseable JSON -- has an Accuracy of zero
	// that means "not measured", not "wrong". The distinction is the same one
	// Error draws for the answering call, and it was missing here: a run that
	// answered every question and graded none reported an accuracy of 0.000
	// for both conditions, which reads as total failure of the extraction when
	// nothing had been measured at all.
	Graded bool `json:"graded"`
}

type Band string

type CallFailure struct {
	Reason string `json:"reason"`
	Count  int    `json:"count"`
}

type Condition string

type ConditionMetrics struct {
	Questions int `json:"questions"`
	// Answered is how many questions actually produced an answer. It differs
	// from Questions whenever a call failed, and the accuracy below is the mean
	// over these rather than over all of them.
	Answered int `json:"answered"`
	// Scored is how many of those answers were graded, and Ungraded how many
	// were not. Accuracy is the mean over Scored: an answer the grader never
	// reached is excluded rather than counted as wrong.
	Scored         int              `json:"scored"`
	Ungraded       int              `json:"ungraded"`
	MeanAccuracy   float64          `json:"mean_accuracy"`
	MeanInputToks  float64          `json:"mean_input_tokens"`
	TotalInputToks int64            `json:"total_input_tokens"`
	MeanLatencyMS  float64          `json:"mean_latency_ms"`
	Refusals       int              `json:"refusals"`
	Errors         int              `json:"errors"`
	ByBand         map[Band]float64 `json:"accuracy_by_band"`
}

type CoverageResult struct {
	Coverage float64     `json:"coverage"`
	Total    int         `json:"total"`
	Found    int         `json:"found"`
	Facts    []FactCheck `json:"facts"`
}

type FactCheck struct {
	QuestionID string `json:"question_id"`
	Fact       string `json:"fact"`
	Present    bool   `json:"present"`
	// Missing lists the fact's distinctive words that appear nowhere in the
	// artifact. It is what turns "coverage 0.81" into something actionable:
	// usually either a real gap in the extraction or a fact whose wording was
	// never on the page to begin with.
	Missing []string `json:"missing,omitempty"`
}

type Input struct {
	Set *Set
	// Artifact is the distilled graph.
	Artifact *graph.Graph
	// RawHTML is what an unaided agent would have had to read.
	RawHTML string
}

type Options struct {
	Model       string
	GraderModel string
	APIKey      string
	// Budget caps total tokens across the whole run.
	Budget int64
	// Concurrency bounds simultaneous API calls.
	Concurrency int
	// BaseURL points the run at an OpenAI-compatible provider. Empty means
	// Anthropic, or whatever LLM_BASE_URL names.
	BaseURL string
	// RawContextTokens caps how much of the raw page the control is given.
	//
	// Without a cap the control simply errors on any large page, so the
	// benchmark could measure everything except the case the tool exists for:
	// pear.no serves around four hundred and seventy thousand tokens and no
	// model accepts it. Truncating models what an unaided agent actually gets
	// -- the top of the document and no more -- and the report says how much
	// was left behind, so nobody mistakes the handicap for a result.
	RawContextTokens int
	// GraderRepeats re-grades a sample of answers to measure how much the
	// grader agrees with itself. Zero disables it.
	GraderRepeats int
	Logf          func(format string, args ...any)
}

type Question struct {
	ID   string `yaml:"id" json:"id"`
	Band Band   `yaml:"band" json:"band"`
	Ask  string `yaml:"ask" json:"ask"`
	// Expect is the ground-truth answer, written by hand against the real page.
	Expect string `yaml:"expect" json:"expect"`
	// Facts are the atomic claims a correct answer must contain. They are what
	// coverage is measured against, and they are why coverage means something
	// here and not in the self-audit.
	Facts []string `yaml:"facts" json:"facts"`
}

type Report struct {
	Target      string    `json:"target"`
	GeneratedAt time.Time `json:"generated_at"`
	Model       string    `json:"model"`
	GraderModel string    `json:"grader_model"`
	ContentHash string    `json:"content_hash"`
	Tier        string    `json:"tier"`

	// RawTokens is the size of the page an unaided agent would have been
	// handed, and RawTokensSent is how much of it actually fitted.
	//
	// These differ on exactly the pages sieve exists for. A context ceiling is
	// not a limitation of this benchmark, it is the condition the control is
	// under in real use -- an agent given a four-hundred-thousand-token page
	// does not read four hundred thousand tokens of it either. Recording both
	// is what keeps that honest rather than hidden.
	RawTokens     int  `json:"raw_tokens"`
	RawTokensSent int  `json:"raw_tokens_sent"`
	RawTruncated  bool `json:"raw_truncated"`

	// RawVisibleChars is how much readable text the served page carried, with
	// markup and scripts removed. It decides which criteria can be judged at
	// all: a page that serves nothing can neither verify the artifact nor be
	// reduced in size.
	RawVisibleChars int `json:"raw_visible_chars"`

	Answers []Answer `json:"answers"`

	// CallFailures groups the distinct reasons calls failed, commonest first.
	//
	// A failure count on its own is a dead end. The reason is always in the
	// report's per-answer records, but a reader watching the terminal sees only
	// a number, and the difference between a wrong model name, an expired key
	// and a rate limit is the difference between a five-second fix and an
	// afternoon. The provider already says which it is; this is only a matter
	// of not throwing it away.
	CallFailures []CallFailure `json:"call_failures,omitempty"`

	Metrics struct {
		Raw      ConditionMetrics `json:"raw"`
		Artifact ConditionMetrics `json:"artifact"`
	} `json:"metrics"`

	// Coverage is measured against ground-truth facts, not against what the
	// capture happened to observe.
	Coverage float64 `json:"coverage"`
	// Fidelity is the share of sampled artifact statements verifiable in the
	// source. It gates the release: a distiller that invents content is worse
	// than no distiller.
	Fidelity float64 `json:"fidelity"`
	// FidelityMeasured distinguishes a fidelity of zero from a check that could
	// not run. They are opposite conclusions -- one says the artifact invented
	// everything, the other that nothing could be verified -- and collapsing
	// them into a single number condemned sieve for extracting content which
	// was correctly absent from the served HTML.
	FidelityMeasured bool     `json:"fidelity_measured"`
	FidelityNotes    []string `json:"fidelity_notes,omitempty"`

	// GraderAgreement is how often the grader gave the same verdict when asked
	// twice about the same answer. It is the error bar on every accuracy figure
	// above: a grader that disagrees with itself a fifth of the time cannot
	// support a claim of a five-point difference, and without measuring it
	// there is no way to know which claims are safe.
	GraderAgreement float64 `json:"grader_agreement,omitempty"`
	GraderRegraded  int     `json:"grader_regraded,omitempty"`

	// Stability is reported only when a stability run was requested.
	Stability *Stability `json:"stability,omitempty"`

	// Verdict states plainly whether the run met the success criteria.
	Verdict Verdict `json:"verdict"`
}

type Runner struct {
	// contains filtered or unexported fields
}

type Set struct {
	URL       string     `yaml:"url" json:"url"`
	Name      string     `yaml:"name" json:"name"`
	Questions []Question `yaml:"questions" json:"questions"`
}

type Stability struct {
	// TierStable reports whether both runs chose the same escalation tier.
	// It is separate from content stability on purpose: a tool that wavers
	// between tiers and one that extracts inconsistently are different bugs.
	TierStable bool   `json:"tier_stable"`
	TierA      string `json:"tier_a"`
	TierB      string `json:"tier_b"`

	// HashStable reports whether the semantic content hash matched.
	HashStable bool `json:"hash_stable"`
	// BlockAgreement is the share of blocks present in both runs with identical
	// text. A page that is genuinely personalised will score low here, and
	// correctly so.
	BlockAgreement float64  `json:"block_agreement"`
	BlocksA        int      `json:"blocks_a"`
	BlocksB        int      `json:"blocks_b"`
	OnlyInA        []string `json:"only_in_a,omitempty"`
	OnlyInB        []string `json:"only_in_b,omitempty"`
}

type Verdict struct {
	TokenReduction float64 `json:"token_reduction"`
	// AccuracyGain is the mean difference over questions answered under both
	// conditions, not the difference of the two means. See judge.
	AccuracyGain float64 `json:"accuracy_gain"`
	// ComparedQuestions is how many questions that pairing covered.
	ComparedQuestions int  `json:"compared_questions"`
	MetTokenTarget    bool `json:"met_token_target"`
	MetAccuracyTarget bool `json:"met_accuracy_target"`
	MetCoverageTarget bool `json:"met_coverage_target"`
	MetFidelityTarget bool `json:"met_fidelity_target"`
	// TokenReductionApplies is false on a page that served no readable text,
	// where there was nothing to reduce. MetTokenTarget is then true by
	// vacancy rather than by achievement, and this is how to tell them apart.
	TokenReductionApplies bool `json:"token_reduction_applies"`

	Passed  bool   `json:"passed"`
	Summary string `json:"summary"`
	// Notes record criteria that did not apply, so a pass is never mistaken
	// for having cleared a bar that was never raised.
	Notes []string `json:"notes,omitempty"`
}