eval

type Case struct {
	Name string
	Task string // the goal handed to the agent.
	// TaskSteer, when non-empty, is appended verbatim to Task by RunTrial as a
	// final paragraph. It is an eval-arm knob rather than part of a case's task.
	TaskSteer string
	Fixture   Fixture      // the starting state, materialized fresh per trial.
	Oracle    Oracle       // grades the finished run against ground truth (not self-report).
	Protocol  string       // "tools" (default) or "text".
	Config    agent.Config // knob template (MaxIterations, MaxTokens, RunTimeout, Verify*, …).

	// Sandbox, when non-nil, builds the per-trial sandbox over the freshly
	// materialized fixture dir — the seam for cases whose execution environment is
	// not the host (SWE-bench instances exec inside their official per-instance
	// Docker image, rooted at the extracted /testbed). The trial owns the returned
	// sandbox's lifetime (Close). nil => the default host-local sandbox.
	Sandbox func(ctx context.Context, dir string) (sandbox.Sandbox, error)

	// Setup, when non-nil, runs after the fixture is materialized and the
	// per-trial sandbox exists, but before the agent/runner starts. It can do
	// case-specific in-sandbox measurement and return a per-trial LadderVerify
	// command. The returned command overrides LadderVerify for this trial,
	// including the empty string (used to refuse ladder runs whose signal could
	// not be validated).
	Setup func(ctx context.Context, root string, sb sandbox.Sandbox) (SetupResult, error)

	// Tools, when non-nil, builds the agent's toolset from the per-trial sandbox.
	// It is the seam for replaying a case under the PRODUCTION toolset of the binary
	// it mirrors: commit-msg and issue-bot run ReadOnlyTools (no write/edit), not the
	// full DefaultTools the loop falls back to — so without this a "review" case
	// would be handed mutation tools it never has in production, and could behave
	// (and be graded) unfaithfully. nil => the loop's DefaultTools.
	Tools func(sb sandbox.Sandbox, runTimeout time.Duration) map[string]agent.Tool

	// VCSWorkspace is true when the fixture materializes a real git work tree
	// suitable for VCS-backed runners such as the escalation ladder. Pure
	// in-memory fixtures (MapFixture) must leave this false so the ladder arm
	// can reject them cleanly rather than half-working.
	VCSWorkspace bool

	// LadderVerify is the in-run command the ladder uses as the red/green signal
	// for each rung attempt. When empty, the case verifies only via the held-out
	// oracle and is NOT ladder-compatible — validateLadderCases rejects it. The
	// case constructor sets this when it can provide a meaningful per-trial
	// verify command (e.g. a suite that the agent can pass by completing the
	// task). It is independent of the CLI -verify-cmd flag, which is an optional
	// self-check for the single-model path and is not a substitute for a
	// case-declared ladder signal.
	LadderVerify string
}

type Cell struct {
	Case   string
	Model  string
	Trials []Trial

	// PricingCoverage_ and UnpricedModels_ are additive report fields
	// populated by WriteFiles before JSON serialisation so downstream
	// consumers of report.json can see per-cell pricing honesty without
	// recomputing from trials. The methods PricingCoverage() and
	// UnpricedModels() compute the same values on the fly for callers that
	// don't go through WriteFiles (tests, markdown).
	PricingCoverage_ string   `json:"pricing_coverage,omitempty"`
	UnpricedModels_  []string `json:"unpriced_models,omitempty"`
}

type CommandOracle struct {
	Label   string        // short name for the report Detail (e.g. "suite").
	Cmd     string        // the success command; exit 0 == pass.
	Timeout time.Duration // 0 = defaultOracleTimeout.
}

type Fixture interface {
	Materialize(dir string) error
	Describe() string
}

type GitFixture struct {
	RepoRoot string // path to the live git repo to archive FROM (the source of truth).
	Ref      string // commit-ish to materialize (e.g. "ba3ba65^" for "the parent of the fix").
}

type Grade struct {
	Pass   bool
	Detail string

	// NoAttempt marks a run that left NOTHING gradable — e.g. an unchanged
	// working tree (SWE-bench's empty model patch). Distinct from a wrong
	// attempt: best-of-N selection (SelectBest) demotes these below every trial
	// that did work, because an `answered` over a no-op is a confident no-op.
	NoAttempt bool
}

type GradeInput struct {
	Root    string
	Sandbox sandbox.Sandbox
	Result  *agent.RunResult

	// SetupCreated records untracked paths that Setup created, keyed by path.
	// The entry is the complete post-Setup git tree identity; an oracle may
	// exclude a path only while its current entry remains exactly this value.
	SetupCreated map[string]TreeEntry
}

type HeldOut struct {
	Label   string        // names this oracle in Detail (e.g. "held-out").
	Drop    Fixture       // the held-out files, written into Root after the run.
	Verify  string        // the command run over the augmented fixture; exit 0 == pass.
	Timeout time.Duration // 0 = defaultOracleTimeout.
}

type Manifest struct {
	SchemaVersion  string   `json:"schema_version"` // bump when Trial/Cell/Manifest shape changes, so a diff tool can refuse to compare incompatible reports instead of misreading absent-vs-zero fields.
	GeneratedAt    string   `json:"generated_at"`
	GitSHA         string   `json:"git_sha"`
	GoVersion      string   `json:"go_version"`
	TrialsPer      int      `json:"trials_per_cell"`
	Models         []string `json:"models"`
	Cases          []string `json:"cases"`
	TaskSteerArmed bool     `json:"task_steer_armed,omitempty"`
}

type MapFixture map[string]string

type Model struct {
	Label       string
	Provider    llm.Provider
	Run         TrialRunFunc
	RequiresVCS bool
}

type Oracle interface {
	Grade(ctx context.Context, in GradeInput) Grade
}

type Price struct {
	InPerM        float64 // USD per 1M prompt tokens.
	OutPerM       float64 // USD per 1M completion tokens.
	CacheReadPerM float64 // USD per 1M cache-read prompt tokens.
}

type Report struct {
	Manifest Manifest `json:"manifest"`
	Cells    []Cell   `json:"cells"`
}

type ReviewFateStats struct {
	Runs     int // runs whose report named this reviewer.
	Rounds   int // total review rounds spent.
	Repaired int // blocked, fed back, cleared on a later round.
	Refuted  int // repro PASSED — the claim was execution-refuted.
	Expired  int // still blocking when rounds/run ran out.
	Notes    int // never blocked (severity note / under the confidence gate).
	Dropped  int // failed deterministic validation (quote/fence).
}

type SetupResult struct {
	// LadderVerify, when non-nil, overrides Case.LadderVerify for this trial.
	// A pointer distinguishes "no override" from "measured empty; refuse".
	LadderVerify *string
}

type TreeEntry struct {
	Mode   string
	Type   string
	Object string
}

type Trial struct {
	Case      string
	Model     string
	RunID     string        // the agent run's stable ID (RunResult.ID) — the P1 spine, so a Trial can be correlated with its persisted transcript.
	Index     int           // 1-based trial number within the cell.
	Outcome   agent.Outcome // the agent's self-reported terminal state.
	Answer    string
	Iters     int
	Usage     llm.Usage
	Pass      bool   // the ORACLE's verdict — the actual grade.
	Detail    string // the oracle's evidence.
	NoAttempt bool   // the oracle found nothing gradable (Grade.NoAttempt) — demoted by best-of selection.
	Err       string // infra failure (provider/transport), distinct from a non-pass.

	Cost   float64 // USD cost of this trial's Usage at the pinned Pricing (meaningless unless Priced).
	Priced bool    // whether the model had a Pricing entry — false => render "—", not $0.

	// Plan is the opening plan-stage report (nil when Config.Planner was unset).
	// Persisted per trial so planner activity and cost are recorded facts.
	Plan       *agent.PlanReport `json:"plan,omitempty"`
	PlanCost   float64           // USD of Plan.Usage at the PLANNER model's pinned Pricing.
	PlanPriced bool              // false when the planner model has no Pricing entry (or Plan is nil).

	// Review is the closing review-gate report (nil when Config.Reviewer was
	// unset). Persisted per trial so review activity is a recorded fact, not
	// something to infer from outcome flips and billing deltas (the R4b gap).
	Review       *agent.ReviewReport `json:"review,omitempty"`
	ReviewCost   float64             // USD of Review.Usage at the REVIEWER model's pinned Pricing.
	ReviewPriced bool                // false when the reviewer model has no Pricing entry (or Review is nil).

	// LatencyMs is the agent's wall-clock for this trial (summed model + tool time
	// across steps). Derived from Steps so it survives in report.json after the full
	// trace is dropped — lets the report separate slow runs from token-heavy ones.
	LatencyMs int64

	// Protocol is the loop actually used: "tools", "text", or "refused" (infra
	// failure when a tools case met a no-tools model).
	Protocol string `json:"protocol"`

	// Ladder holds per-trial escalation-ladder metadata when the trial was
	// executed by a ladder arm. nil for normal single-model trials — the
	// omitempty tag keeps existing report JSON stable.
	Ladder *TrialLadder `json:"ladder,omitempty"`

	// Steps is the full think->act->observe trace, kept for per-trial archival but
	// EXCLUDED from report.json (json:"-") so the aggregate report stays compact —
	// WriteFiles writes each trace to its own file under traces/ instead.
	Steps []agent.Step `json:"-"`
}

type TrialLadder struct {
	Attempts   int     // total attempts across all rungs.
	WinnerRung int     // 1-based rung index of the winning attempt; 0 when nothing won.
	Escalated  bool    // true when any attempt went past rung 1 (the NORTH-STAR guard).
	Cost       float64 // USD cost of all ladder attempts summed; valid only when Priced is true.
	Priced     bool    // whether at least one attempt had a cost known to the price table.

	// CostByModel is the per-model dollar attribution: solver cost under its rung
	// model id, reviewer cost under the reviewer model id, planner under the
	// planner model id. The sum across entries equals Cost (modulo pricing
	// gaps). nil for non-ladder trials (omitempty keeps JSON compact).
	CostByModel map[string]float64 `json:"cost_by_model,omitempty"`
}

type TrialRunFunc func(context.Context, agent.Config, string) (*agent.RunResult, *TrialLadder, error)