benchmark

type AgentFactory func(runDir string, spec ModelSpec) (*agent.Agent, error)

type DelegationMetrics struct {
	// TaskCount is the number of delegated tasks the run recorded.
	TaskCount int `json:"task_count"`
	// Verified / Escalated / Failed are the per-outcome task counts.
	Verified  int `json:"verified"`
	Escalated int `json:"escalated"`
	Failed    int `json:"failed"`
	// WorkerTokens is the tokens the worker model completed (verified
	// without escalation), and TotalTokens the tokens across all delegated
	// tasks.
	WorkerTokens int `json:"worker_tokens"`
	TotalTokens  int `json:"total_tokens"`
	// EscalationRate is Escalated/TaskCount (0 when there is no delegated
	// work): the share of ALL delegated tasks the cheap model could not
	// finish and the coder model had to take over. The denominator is every
	// delegated task (not only worker-role ones), so the rate reads as the
	// run's overall escalation share. This is the headline number for
	// whether a cheaper model "holds up".
	EscalationRate float64 `json:"escalation_rate"`
	// WorkerTokenShare is WorkerTokens/TotalTokens (0 when no delegated
	// work): the share of delegated work the worker model handled.
	WorkerTokenShare float64 `json:"worker_token_share"`
}

type DelegationReport struct {
	// RunMode is the mode these runs used (RunModeCoder when unset).
	RunMode string `json:"run_mode"`
	// Runs is the number of runs in this mode.
	Runs int `json:"runs"`
	// Passed is how many of those runs passed.
	Passed int `json:"passed"`
	// PassRate is Passed/Runs (0 when there are no runs).
	PassRate float64 `json:"pass_rate"`
	// DelegatedTasks is the pooled delegated-task count.
	DelegatedTasks int `json:"delegated_tasks"`
	// Escalated is the pooled escalated-task count, and EscalationRate the
	// pooled share (0 when no delegated tasks).
	Escalated      int     `json:"escalated"`
	EscalationRate float64 `json:"escalation_rate"`
	// Tokens and Cost are the pooled totals across the mode's runs.
	Tokens int     `json:"tokens"`
	Cost   float64 `json:"cost"`
}

type KeepRuns string

type Meta struct {
	RunDate string
	Version string
}

type ModelSpec struct {
	// Model is the model id to request. Empty → the run's configuration
	// default for the provider.
	Model string
	// Provider is the provider name to run against (e.g. "anthropic").
	// Empty → the run's configuration default provider.
	Provider string
	// RunMode selects how the run delegates work: RunModeCoder (or empty —
	// the default), RunModeDelegated, or RunModeEconomy. See delegation.go.
	RunMode string
	// WorkerModel / WorkerProvider point the worker role at a cheaper model
	// for RunModeDelegated / RunModeEconomy. An empty WorkerModel leaves the
	// worker role unset, in which case delegation is a no-op and the run
	// behaves like RunModeCoder. An empty WorkerProvider uses the run's own
	// provider (the run's active provider, pinned by configureRun).
	WorkerModel    string
	WorkerProvider string
}

type Report struct {
	Meta Meta `json:"meta"`
	// Models is the suite's model list (SuiteModels order) — recorded
	// as passed in, never re-derived from the runs.
	Models []ModelSpec `json:"models"`
	// Tasks are the (task, model, provider) pair reports in first-seen
	// order (the suite's model × task order when the runs come from
	// RunSuite).
	Tasks []TaskReport `json:"tasks"`
	// Starters pool the (task × run) matrix per (starter, model,
	// provider) in first-seen order — the "pass rate per starter
	// per model".
	Starters []StarterReport `json:"starters"`
	// FailureCategories counts the failed runs per failure mode (see
	// failureCategories); nil — omitted from the JSON — when nothing
	// failed.
	FailureCategories map[string]int `json:"failure_categories,omitempty"`
	// Delegation compares the suite's runs by delegation mode (coder only,
	// worker delegation, economy): task success, escalation rate, tokens
	// and cost. Empty when no run recorded a mode.
	Delegation []DelegationReport `json:"delegation,omitempty"`
}

type Run struct {
	// TaskID is the benchmark task this run executed (Task.ID).
	TaskID string
	// Starter is the task's starter id (Task.Starter).
	Starter string
	// Model is the requested model (ModelSpec.Model, "" when the run's
	// configuration default applies).
	Model string
	// Provider is the requested provider (ModelSpec.Provider, "" when
	// the run's configuration default applies).
	Provider string
	// RunNumber is the 1-based run number within this task+model.
	RunNumber int
	// Passed is the run's acceptance outcome, derived only from Result.
	Passed bool
	// Result is the turn's verification result, stored by the
	// turn-end hook, or nil when the hook never ran (verification
	// disabled, the turn changed no code, a setup error before the turn,
	// or a verify runner setup error).
	Result *verify.Result
	// NotVerifiedReason states WHY a failed run has no verification
	// result (set only when Result is nil): "no code changes this turn"
	// (the hook's change gate stayed closed), "verification setup error"
	// (the hook entered but its runner failed to set up), "verification
	// did not run this turn" (the hook still did not run), or "verify
	// timed out" (the run's turn exceeded Runner.Timeout). Empty when
	// Result is non-nil, and also when verification was disabled — which
	// a benchmark run forces on, so an empty reason on a nil result
	// means there is nothing honest to record beyond the category.
	NotVerifiedReason string
	// Err is a run-level failure (fresh-copy setup, plan write, agent
	// build, or the agent turn itself). Non-nil means the run did not
	// complete cleanly; such a run is never Passed.
	Err error
	// StartedAt is when the run started (fresh-copy creation).
	StartedAt time.Time
	// FinishedAt is when the run finished (the copy is removed).
	FinishedAt time.Time
	// Turns is the number of top-level agent turns the runner issued for
	// this run (1 for the current single-request runner; the field grows
	// naturally when multi-request tasks land).
	Turns int
	// RepairRounds is the number of turn-end verification repair rounds
	// the hook ran within the turn (0 when verification never ran).
	RepairRounds int
	// RepairAttempts is the per-check repair attempts the hook consumed
	// against the stopping rule, keyed like the hook's per-check counters
	// (empty when verification never ran).
	RepairAttempts map[string]int
	// RepairLimit is the stopping-rule limit that was in effect for the
	// turn (0 when verification never ran).
	RepairLimit int
	// Tokens is the run's total token usage: the agent's conversation
	// total — the run's agent is fresh, so the conversation total is the
	// run's usage (existing cost tracking).
	Tokens int
	// Cost is the run's total cost (the agent's conversation total cost).
	Cost float64
	// LangChecks is how many final messages the language guard
	// judged for the run's model during the run (a delta of the
	// process-wide per-model metric).
	LangChecks int64
	// LangMismatches is how many of those judged messages were a
	// reliable language mismatch (a delta).
	LangMismatches int64
	// RoleUsage is the run's per-role token/cost breakdown: the run's
	// agent's per-role usage, captured from its metrics.
	// Empty when the run errored before the agent's turn recorded usage.
	RoleUsage []agent.RoleUsage
	// RunMode is the delegation mode the run used (RunModeCoder when unset).
	RunMode string
	// Delegation is the run's delegation outcome (task/verified/escalated/
	// failed counts and the worker share). Zero for a coder-only run.
	Delegation DelegationMetrics
}

type Runner struct {
	// ConfigManager owns the per-run configuration. Before each agent
	// build the runner writes the run's requirements into it with
	// UpdateConfigNoSave — verification forced on (the benchmark's
	// pass/fail source must run; the manager's other verification
	// settings are preserved) and the spec's model/provider where set —
	// and a benchmark run never persists a run's model choice into the
	// user's configuration. The writes are narrow, never clearing: an
	// empty spec field leaves the manager's current value, and writes
	// persist across RunTask calls on the same runner — a caller that
	// wants isolated defaults per model supplies a fresh manager.
	// nil is allowed with a custom AgentFactory that manages its own
	// configuration.
	ConfigManager *configuration.Manager
	// AgentFactory builds the headless agent for one run (the default
	// factory, above, is used when nil).
	AgentFactory AgentFactory
	// ShapeCopy is an optional per-copy hook (tests / custom harnesses)
	// run after instantiation + plan write and before agent creation —
	// e.g. to override the copy's .sprout/starter.json commands. nil by
	// default: production runs use the starter's own manifest.
	ShapeCopy func(runDir string, task *Task) error
	// RunsPerTask is the runs-per-model count. 0 (or
	// negative) → defaultRunsPerTask (3).
	RunsPerTask int
	// WorkDir is the parent directory for the fresh copies. "" →
	// os.TempDir(). It is created (with parents) if missing.
	WorkDir string
	// Models is the suite's model list: the models a
	// suite run benchmarks against. Empty → the default list (the
	// provider catalog's recommended_model entries, models.go); set it
	// to override the default. SuiteModels is the single resolution
	// point.
	Models []ModelSpec
	// Timeout is the per-run wall-clock limit: a run whose turn exceeds
	// it is stopped through the agent's real interrupt (TriggerInterrupt,
	// the CLI's Ctrl+C mechanism) and recorded as failed, wrapping
	// ErrRunTimeout in Run.Err — never Passed. 0 (or negative) →
	// defaultTaskTimeout (10m). Each run carries its own timer.
	Timeout time.Duration
	// KeepRuns is the evidence policy: which runs leave their failed
	// working-copy diff, agent transcript and verification output under
	// EvidenceDir. "" resolves to KeepRunsFailed — the default, so a run
	// never silently discards the evidence a failure needs.
	KeepRuns KeepRuns
	// EvidenceDir is the root directory the per-run evidence lands under
	// (EvidenceDir/runs/<task>-<model>-<n>/). "" → no evidence is kept,
	// regardless of KeepRuns: a runner with no evidence root has nowhere
	// to write the evidence. Evidence writing is best-effort — a
	// failure to write never changes a run's verdict.
	EvidenceDir string
}

type StarterReport struct {
	Starter  string
	Model    string
	Provider string
	// Tasks is how many distinct tasks of this starter the report
	// covers.
	Tasks int
	// RunsTotal is the pooled task × run matrix size.
	RunsTotal int
	Passed    int
	// PassRate is the pooled Passed/RunsTotal (0 when RunsTotal is 0).
	PassRate float64
}

type Task struct {
	// ID is the task identifier: non-empty, a single safe path
	// segment. Convention: the file name without .json.
	ID string `json:"id"`
	// Request is the plain-language request the agent receives.
	Request string `json:"request"`
	// Starter is the starter ID the task runs against.
	Starter string `json:"starter"`
	// Plan is the frozen plan: the machine-readable
	// plancontract.Plan. It must validate (plancontract.Validate);
	// its acceptance criteria are the task's acceptance criteria.
	Plan plancontract.Plan `json:"plan"`
}

type TaskReport struct {
	TaskID string
	// Starter is the task's starter id.
	Starter  string
	Model    string
	Provider string
	// Runs are the pair's runs in run number order.
	Runs []Run
	// Passed is how many of the runs passed.
	Passed int
	// PassRate is Passed/len(Runs) (0 when there are no runs).
	PassRate float64
}

type ValidationError struct {
	Problems []string
}