eval

type Action struct {
	Calls []Call `json:"calls,omitempty"`
	// Malformed: the model tried to call a tool and produced something the
	// API rejected (Gemini MALFORMED_FUNCTION_CALL, Converse malformed_tool_use).
	// An invalid call, not a stop.
	Malformed bool `json:"malformed,omitempty"`
	// Truncated: the output token cap cut the step; it says nothing about the
	// model's decision, so the step is excluded.
	Truncated bool `json:"truncated,omitempty"`
}

type AgentArm struct {
	// ToolCallRate: share of steps that called at least one tool.
	ToolCallRate float64 `json:"toolCallRate"`
	// InvalidToolRate: steps calling a tool the request did not offer
	// (hallucinated tool names).
	InvalidToolRate float64 `json:"invalidToolRate"`
	// SchemaErrorRate: steps whose arguments miss a required field or are
	// not a JSON object.
	SchemaErrorRate float64 `json:"schemaErrorRate"`
	// InvalidCallRate: steps with either problem (what the exported
	// halo.eval.agent.invalid_call_rate samples per session).
	InvalidCallRate float64 `json:"invalidCallRate"`
	// PrematureStopRate: among referenced steps where production kept
	// working, the share where this arm ended its turn instead.
	PrematureStopRate float64 `json:"prematureStopRate"`
	// Fidelity: agreement (same tools) with the action production actually
	// took. The control's fidelity is the re-sampling noise floor.
	Fidelity Rate `json:"fidelity"`
}

type AgentResult struct {
	Steps    int `json:"steps"`    // pairs whose both actions parsed
	Sessions int `json:"sessions"` // distinct sessions among them
	// MultiTurn: steps past the first turn (turns: all).
	MultiTurn int `json:"multiTurn"`
	// Excluded, not steps: Auxiliary requests that offer no tools (compaction
	// summaries, next-speaker checks: both arms just answer), Truncated steps
	// (an arm hit its output cap), Retries (byte-identical resends of a step).
	Auxiliary int `json:"auxiliary"`
	Truncated int `json:"truncated"`
	Retries   int `json:"retries"`
	// ToolAgreement / ArgAgreement: candidate vs control replay, same prefix.
	ToolAgreement Rate     `json:"toolAgreement"`
	ArgAgreement  Rate     `json:"argAgreement"`
	Control       AgentArm `json:"control"`
	Candidate     AgentArm `json:"candidate"`
	// Referenced: steps whose production action was recovered.
	Referenced int `json:"referenced"`
	// FidelityDelta: candidate - control fidelity, paired by step,
	// session-clustered CI. Below zero beyond the noise floor = regression.
	FidelityDelta MetricDelta `json:"fidelityDelta"`
	// Diverged: share of sessions where the candidate's action differed from
	// the control replay at some step; FirstDivergence is the mean step at
	// which that first happened (among diverged sessions).
	Diverged        float64       `json:"diverged"`
	FirstDivergence float64       `json:"firstDivergence"`
	ByDepth         []DepthBucket `json:"byDepth"`
	// PerStep are the analysis samples the exporter may send, one per step.
	PerStep []StepSample `json:"-"`
}

type ArmStats struct {
	ErrorRate    float64 `json:"errorRate"`
	LatencyMS    float64 `json:"latencyMs"`
	OutputTokens float64 `json:"outputTokens"`
}

type Call struct {
	Name  string `json:"name"`
	Input string `json:"input,omitempty"`
}

type CaseSummary struct {
	ID           string     `json:"id"`
	Reasons      []string   `json:"reasons"`
	ExpectedFrom string     `json:"expectedFrom"`
	Expected     []string   `json:"expectedTools"`
	Provenance   Provenance `json:"provenance"`
}

type ClaudeDriver struct{}

type CodexDriver struct{}

type Command struct {
	Args  []string
	Stdin string
}

type Comparison struct {
	Variant string  `json:"variant"`
	Control string  `json:"control"`
	Tasks   int     `json:"tasks"`
	Delta   float64 `json:"delta"` // mean per-task pass rate: variant - control
	CILo    float64 `json:"ciLo"`
	CIHi    float64 `json:"ciHi"`
	Verdict string  `json:"verdict"`

	// Deltas are paired bootstrap CIs per metric over non-flaky tasks
	// (MetricPass, MetricCost, ...); Gate is the ship/hold/block verdict.
	Deltas map[string]MetricDelta `json:"deltas,omitempty"`
	Gate   *Gate                  `json:"gate,omitempty"`
}

type Criterion struct {
	Name        string  `yaml:"name" json:"name"`
	Description string  `yaml:"description" json:"description"`
	Weight      float64 `yaml:"weight,omitempty" json:"weight,omitempty"` // default 1
}

type DepthBucket struct {
	Steps         string  `json:"steps"` // e.g. "0", "1-4"
	N             int     `json:"n"`
	ToolAgreement float64 `json:"toolAgreement"`
}

type DockerRunner struct {
	Bin string // default "docker"
	// Image returns the image for a variant; default
	// ghcr.io/dshakes/eval-<harness>:<version>.
	Image func(Variant) string
	// PassEnv names host env vars forwarded into the agent step only.
	PassEnv []string
	// Network is passed as --network; default "none".
	Network string
	// Memory (default "4g"), CPUs (default "2") and User (default
	// "1000:1000", non-root) bound each container.
	Memory, CPUs, User string
	// SandboxProbes maps harness -> a command that must exit 0 inside a
	// container hardened like the trials before any trial of that harness
	// runs (once per runner and image). nil = DefaultSandboxProbes; an empty
	// map disables probing.
	SandboxProbes map[string][]string
	// contains filtered or unexported fields
}

type Driver interface {
	Name() string
	Command(t *Task, v Variant) Command
	// SettingsPath is where the rendered managed settings must appear inside
	// the environment for the CLI to pick them up.
	SettingsPath() string
	Parse(stdout []byte) (Usage, error)
}

type Env interface {
	Exec(ctx context.Context, c Command) (ExecResult, error)
	Close() error
}

type ExecResult struct {
	Stdout, Stderr []byte
	ExitCode       int
}

type FilePairs struct {
	Path string
	Keys [][]byte
	// AllowPlaintext reads plaintext rows even with Keys set (a store sealed
	// part way through its life). Off by default: with keys configured an
	// unsealed row is refused (shadow.ErrPlaintextRow).
	AllowPlaintext bool
}

type Gate struct {
	Verdict   string     `json:"verdict"` // ship | hold | block
	Reasons   []string   `json:"reasons"`
	Flaky     []TaskDiff `json:"flaky,omitempty"`
	Regressed []TaskDiff `json:"regressed,omitempty"`
	Tasks     []TaskDiff `json:"tasks,omitempty"`
}

type GeminiDriver struct{}

type Grade struct {
	Grader string   `json:"grader"` // type, plus path/rubric when useful
	Pass   bool     `json:"pass"`
	Score  *float64 `json:"score,omitempty"` // judge only
	Rubric string   `json:"rubric,omitempty"`
	Detail string   `json:"detail,omitempty"`
	// Error is a grader failure (e.g. judge schema violation): never a pass.
	Error string `json:"error,omitempty"`
}

type GraderSpec struct {
	Type string `yaml:"type" json:"type"`
	// command
	Run string `yaml:"run,omitempty" json:"run,omitempty"`
	// file: Path relative to the work dir; Exists defaults to true.
	Path        string   `yaml:"path,omitempty" json:"path,omitempty"`
	Exists      *bool    `yaml:"exists,omitempty" json:"exists,omitempty"`
	Contains    []string `yaml:"contains,omitempty" json:"contains,omitempty"`
	NotContains []string `yaml:"not_contains,omitempty" json:"notContains,omitempty"`
	Matches     string   `yaml:"matches,omitempty" json:"matches,omitempty"` // RE2
	// diff: globs (path.Match, slash-separated) over changed paths.
	MaxFilesChanged int      `yaml:"max_files_changed,omitempty" json:"maxFilesChanged,omitempty"`
	MustChange      []string `yaml:"must_change,omitempty" json:"mustChange,omitempty"`
	MustNotChange   []string `yaml:"must_not_change,omitempty" json:"mustNotChange,omitempty"`
	// judge: rubric file relative to the declaring task or suite dir (e.g.
	// ../rubrics/x.yaml), never outside the evals root.
	Rubric string `yaml:"rubric,omitempty" json:"rubric,omitempty"`
	// contains filtered or unexported fields
}

type HTTPLLM struct {
	BaseURL, Wire, APIKey string
	HTTP                  *http.Client // default: 120s timeout
}

type HarvestOptions struct {
	// Experiment limits the harvest to one experiment ("" = all).
	Experiment string
	// Select lists the reasons that make a step a case (default DefaultSelect).
	Select []string
	// Scores are judge scores per pair id (HistoryScores); needed by judge-loss.
	Scores map[string]PairScore
	// JudgeLoss: control minus candidate judge score above which a step is
	// selected (default 0.2).
	JudgeLoss float64
	// PerDepth caps the stratified sample per depth bucket (default 5).
	PerDepth int
	Seed     uint64
	// Redactor is required: nothing is harvested unredacted.
	Redactor *Redactor
	Now      func() time.Time
}

type HarvestResult struct {
	Steps int `json:"steps"` // parsed steps considered
	// Skipped: selected steps whose expected action was itself invalid
	// (a tool not offered, a missing argument), so not a usable test.
	Skipped  int            `json:"skipped"`
	ByReason map[string]int `json:"byReason"`
	Cases    []StepCase     `json:"-"`
}

type HistoryEntry struct {
	Time     time.Time      `json:"time"`
	Kind     string         `json:"kind"` // offline | online
	Suite    string         `json:"suite,omitempty"`
	Gate     *Gate          `json:"gate,omitempty"`
	Variants []VariantScore `json:"variants,omitempty"`
	Online   *OnlineResult  `json:"online,omitempty"`
}

type Job struct {
	Task    *Task
	Variant Variant
	// Settings is the host path of the rendered managed settings (may be empty)
	// and SettingsDest where it must appear in the environment.
	Settings     string
	SettingsDest string
}

type Judge struct {
	LLM   LLM
	Model string
	Cache *JudgeCache // may be nil
}

type JudgeCache struct {
	Dir string
	// contains filtered or unexported fields
}

type JudgeConfig struct {
	URL   string `yaml:"url" json:"url"`     // gateway base URL; /v1/messages or /v1/responses is appended
	Wire  string `yaml:"wire" json:"wire"`   // anthropic-messages | openai-responses
	Model string `yaml:"model" json:"model"` // exact, pinned model id (no "latest" aliases)
	// APIKeyEnv names the env var holding the gateway credential.
	APIKeyEnv string `yaml:"api_key_env,omitempty" json:"apiKeyEnv,omitempty"`
	// Cache is a directory (relative to the suite) for cached verdicts.
	Cache string `yaml:"cache,omitempty" json:"cache,omitempty"`
}

type JudgeInfo struct {
	Model   string   `json:"model"`
	Wire    string   `json:"wire,omitempty"`
	Rubrics []string `json:"rubrics"`
}

type LLM interface {
	Complete(ctx context.Context, req LLMRequest) (string, error)
}

type LLMRequest struct {
	Model, System, Prompt string
	MaxTokens             int
}

type LocalRunner struct{}

type Matrix struct {
	// Baseline is the control cell name; default the first cell.
	Baseline  string           `yaml:"baseline,omitempty" json:"baseline,omitempty"`
	Harnesses []MatrixHarness  `yaml:"harnesses" json:"harnesses"`
	Providers []MatrixProvider `yaml:"providers,omitempty" json:"providers,omitempty"`
}

type MatrixHarness struct {
	Harness  string   `yaml:"harness" json:"harness"`
	Version  string   `yaml:"version" json:"version"`
	Models   []string `yaml:"models" json:"models"`
	Settings string   `yaml:"settings,omitempty" json:"settings,omitempty"`
}

type MatrixProvider struct {
	Name    string            `yaml:"name" json:"name"`
	Aliases map[string]string `yaml:"aliases,omitempty" json:"aliases,omitempty"`
}

type MetricDelta struct {
	Mean     float64 `json:"mean"`
	CILo     float64 `json:"ciLo"`
	CIHi     float64 `json:"ciHi"`
	Baseline float64 `json:"baseline"`
}

type OTLPExporter struct {
	URL   string // collector base URL; /v1/metrics is appended
	Token string // bearer token (optional)
	HTTP  *http.Client
}

type OnlineOptions struct {
	Judge  *Judge
	Rubric *Rubric
	// Sample caps graded pairs per experiment (0 = all); the sample is a
	// seeded shuffle, so a rerun with the same seed grades the same pairs.
	Sample int
	Seed   uint64
	// Experiment limits grading to one experiment ("" = all).
	Experiment string
	// Skip holds pair ids already graded (from the history file). It limits
	// the judge only: the step-level reading needs every step of a session.
	Skip map[string]bool
	// AgentSkip holds step pair ids whose agent samples were already exported.
	AgentSkip map[string]bool
	// AgentQuiet: a step with no recovered production action is exported only
	// once it is this old (its next step may still arrive). Default
	// DefaultAgentQuiet.
	AgentQuiet time.Duration
	Now        func() time.Time
	// ControlNames maps experiment -> its control variant name (from policy).
	// halo-shadow does not label the control side, and analysis matches arms by
	// the experiment's variant names; default "control".
	ControlNames map[string]string
}

type OnlineResult struct {
	Experiment  string  `json:"experiment"`
	Rubric      string  `json:"rubric"`
	JudgeModel  string  `json:"judgeModel"`
	Pairs       int     `json:"pairs"`  // eligible pairs seen
	Graded      int     `json:"graded"` // pairs with both sides judged
	JudgeErrors int     `json:"judgeErrors"`
	Control     float64 `json:"controlScore"`
	Candidate   float64 `json:"candidateScore"`
	// Delta is candidate - control score, paired by pair, with a bootstrap CI.
	Delta   MetricDelta `json:"delta"`
	WinRate float64     `json:"winRate"` // share of pairs the candidate scored higher (ties count half)
	// ControlVariant/CandidateVariant name the arms (shadow target variants).
	ControlVariant   string   `json:"controlVariant"`
	CandidateVariant string   `json:"candidateVariant"`
	PairIDs          []string `json:"pairIds"`
	// Scores are the per-pair readings the exporter sends; history keeps them
	// (pair ids and numbers only) for harvest's judge-loss selection.
	Scores []PairScore `json:"scores,omitempty"`
	// Ops are the operational readings over every eligible pair, errored ones
	// included (they are the point): the refusal, latency and cost regressions
	// ADR-0004 says shadow catches beside quality.
	Ops OpsResult `json:"ops"`
	// Agent is the step-level reading (ADR-0014): next-action agreement,
	// fidelity to production's action, tool-call validity. Nil when no pair's
	// actions parsed.
	Agent *AgentResult `json:"agent,omitempty"`
	// AgentPairIDs are the steps whose agent samples this run exports (each
	// step exactly once across runs: the history records them).
	AgentPairIDs []string `json:"agentPairIds,omitempty"`
	// AgentSteps are those samples (not kept in history).
	AgentSteps []StepSample `json:"-"`
}

type OpsResult struct {
	Pairs     int      `json:"pairs"`
	Control   ArmStats `json:"control"`
	Candidate ArmStats `json:"candidate"`
}

type PairScore struct {
	ID        string  `json:"id"`
	Session   string  `json:"-"`
	Control   float64 `json:"control"`
	Candidate float64 `json:"candidate"`
}

type PairSource interface {
	Pairs(ctx context.Context) ([]shadow.Pair, error)
}

type Part struct {
	Title     string
	Body      string
	Untrusted bool
}

type Provenance struct {
	PairID         string    `json:"pairId"`
	Experiment     string    `json:"experiment"`
	Session        string    `json:"session"`
	Depth          int       `json:"depth"`
	HarvestedAt    time.Time `json:"harvestedAt"`
	RedactionRules string    `json:"redactionRules"`
}

type Rate struct {
	Mean float64 `json:"mean"`
	CILo float64 `json:"ciLo"`
	CIHi float64 `json:"ciHi"`
	N    int     `json:"n"`
}

type Redactor struct {
	// contains filtered or unexported fields
}

type Rubric struct {
	ID           string      `yaml:"id" json:"id"`
	Version      string      `yaml:"version" json:"version"`
	Instructions string      `yaml:"instructions" json:"instructions"`
	Criteria     []Criterion `yaml:"criteria" json:"criteria"`
	// PassThreshold is the weighted score (0..1) at or above which a judge
	// grader passes; default 0.7.
	PassThreshold float64 `yaml:"pass_threshold,omitempty" json:"passThreshold"`
	// contains filtered or unexported fields
}

type RunOptions struct {
	Parallel int
	// Judge grades tasks' judge graders; required if any task has one.
	Judge *Judge
}

type Runner interface {
	Start(ctx context.Context, j Job) (Env, error)
}

type Scorecard struct {
	Suite       string         `json:"suite"`
	Control     string         `json:"control"`
	Variants    []VariantScore `json:"variants"`
	Comparisons []Comparison   `json:"comparisons"`
	Trials      []Trial        `json:"trials"`

	Seed uint64 `json:"seed"`
	K    int    `json:"k,omitempty"`
	// Gate is the worst comparison gate; nil with a single variant.
	Gate       *Gate       `json:"gate,omitempty"`
	Thresholds *Thresholds `json:"thresholds,omitempty"`
	Judge      *JudgeInfo  `json:"judge,omitempty"`
}

type StepCase struct {
	ID       string          `json:"id"`
	Protocol string          `json:"protocol"`
	Request  json.RawMessage `json:"request"` // redacted
	// Tools are the offered tools: name -> required argument keys (null for
	// freeform and server tools).
	Tools        map[string][]string `json:"tools"`
	Expected     Action              `json:"expected"`
	ExpectedFrom string              `json:"expectedFrom"`
	// Alternatives are also accepted: the control replay, when it differed
	// from production's action but was valid (the incumbent's own choice).
	Alternatives []Action   `json:"alternatives,omitempty"`
	Match        string     `json:"match"`
	Reasons      []string   `json:"reasons"`
	Provenance   Provenance `json:"provenance"`
}

type StepModel struct {
	URL    string // base URL; https, or http to loopback only
	Model  string
	APIKey string
	HTTP   *http.Client // default: 300s timeout
}

type StepResult struct {
	ID        string `json:"id"`
	Session   string `json:"session"`
	Depth     int    `json:"depth"`
	Pass      bool   `json:"pass"`
	SameTools bool   `json:"sameTools"`
	SameCalls bool   `json:"sameCalls"`
	Valid     bool   `json:"valid"`
	// PrematureStop: the expected action kept working and the model stopped.
	PrematureStop bool     `json:"prematureStop"`
	Actual        Action   `json:"actual"`
	Error         string   `json:"error,omitempty"`
	JudgeScore    *float64 `json:"judgeScore,omitempty"`
	JudgeError    string   `json:"judgeError,omitempty"`
	// contains filtered or unexported fields
}

type StepRunOptions struct {
	Parallel int // concurrent model calls (default 4)
	Seed     uint64
	// Judge and Rubric, both or neither: a secondary rubric score per answered
	// case. It never changes Pass.
	Judge  *Judge
	Rubric *Rubric
}

type StepSample struct {
	PairID, Session string
	Time            time.Time
	// HasRef: production's action was recovered; the fidelity fields are set
	// only then.
	HasRef                      bool
	ControlFidelity, CandFid    float64
	ControlInvalid, CandInvalid float64 // 1 when the arm's call was invalid
}

type StepScorecard struct {
	Suite    string `json:"suite"`
	Model    string `json:"model"`
	Cases    int    `json:"cases"`
	Sessions int    `json:"sessions"`
	Errors   int    `json:"errors"`
	// Pass counts an errored case as failed; the other rates are over
	// answered cases (PrematureStop: those whose expected action continued).
	Pass          Rate         `json:"pass"`
	SameTools     Rate         `json:"sameTools"`
	SameCalls     Rate         `json:"sameCalls"`
	Valid         Rate         `json:"valid"`
	PrematureStop Rate         `json:"prematureStop"`
	Rubric        string       `json:"rubric,omitempty"`
	Judge         *Rate        `json:"judge,omitempty"`
	JudgeErrors   int          `json:"judgeErrors,omitempty"`
	Results       []StepResult `json:"results"`
}

type Suite struct {
	Name string `yaml:"name" json:"name"`
	// Tasks are task directory names under <suite dir>/../tasks.
	Tasks []string `yaml:"tasks" json:"tasks"`
	// Control names the baseline variant; defaults to the first.
	Control  string    `yaml:"control,omitempty" json:"control"`
	Repeats  int       `yaml:"repeats,omitempty" json:"repeats"`
	Variants []Variant `yaml:"variants" json:"variants"`
	// Seed is the base seed for per-trial seeds and bootstrap CIs. No pinned
	// CLI exposes a sampling seed, so it makes the run order and statistics
	// reproducible, not the agent itself.
	Seed uint64 `yaml:"seed,omitempty" json:"seed,omitempty"`
	// K is the k of pass@k and pass^k; default Repeats.
	K int `yaml:"k,omitempty" json:"k,omitempty"`
	// Gate holds the ship/hold/block thresholds.
	Gate Thresholds `yaml:"gate,omitempty" json:"gate"`
	// Judge configures the LLM-as-judge grader used by tasks' judge graders.
	Judge *JudgeConfig `yaml:"judge,omitempty" json:"judge,omitempty"`
	// Matrix, used by `halo eval run --matrix`, replaces Variants with the
	// harness x model x provider cross product.
	Matrix *Matrix `yaml:"matrix,omitempty" json:"matrix,omitempty"`
	// Graders apply to every task in addition to its own (e.g. a judge
	// rubric for the whole suite, so shared tasks stay judge-free).
	Graders []GraderSpec `yaml:"graders,omitempty" json:"graders,omitempty"`

	// Dir is the directory of the suite file; set by LoadSuite.
	Dir string `yaml:"-" json:"-"`
}

type Task struct {
	ID        string        `yaml:"id" json:"id"`
	Repo      string        `yaml:"repo" json:"repo"` // git URL, or directory relative to the task dir
	Setup     []string      `yaml:"setup,omitempty" json:"setup,omitempty"`
	Prompt    string        `yaml:"prompt" json:"prompt"`
	Check     string        `yaml:"check" json:"check"` // shell command, exit 0 = pass
	Timeout   time.Duration `yaml:"timeout,omitempty" json:"timeout,omitempty"`
	BudgetUSD float64       `yaml:"budget_usd,omitempty" json:"budgetUSD,omitempty"`
	MaxTurns  int           `yaml:"max_turns,omitempty" json:"maxTurns,omitempty"`
	Tags      []string      `yaml:"tags,omitempty" json:"tags,omitempty"`
	// Graders all have to pass too; check may be empty when graders are set.
	Graders []GraderSpec `yaml:"graders,omitempty" json:"graders,omitempty"`

	// Dir is the directory holding task.yaml; set by LoadTask.
	Dir string `yaml:"-" json:"-"`
}

type TaskDiff struct {
	Task      string  `json:"task"`
	Baseline  float64 `json:"baseline"`  // pass rate over trials
	Candidate float64 `json:"candidate"` // pass rate over trials
	Status    string  `json:"status"`    // regressed | improved | same | flaky
	Reason    string  `json:"reason,omitempty"`
}

type Thresholds struct {
	// MaxPassDrop is the non-inferiority margin on the paired per-task pass
	// rate: ship needs the CI lower bound >= -MaxPassDrop. Default 0.05.
	MaxPassDrop          *float64 `yaml:"max_pass_drop,omitempty" json:"maxPassDrop,omitempty"`
	MaxCostIncrease      *float64 `yaml:"max_cost_increase,omitempty" json:"maxCostIncrease,omitempty"`
	MaxTokensIncrease    *float64 `yaml:"max_tokens_increase,omitempty" json:"maxTokensIncrease,omitempty"`
	MaxWallIncrease      *float64 `yaml:"max_wall_increase,omitempty" json:"maxWallIncrease,omitempty"`
	MaxToolCallsIncrease *float64 `yaml:"max_tool_calls_increase,omitempty" json:"maxToolCallsIncrease,omitempty"`
	// TaskRegression is the per-task pass-rate drop that marks a task as
	// regressed (default 0.5); more than MaxRegressedTasks (default 0) blocks.
	TaskRegression    float64 `yaml:"task_regression,omitempty" json:"taskRegression"`
	MaxRegressedTasks int     `yaml:"max_regressed_tasks,omitempty" json:"maxRegressedTasks"`
	// MinTasks is the fewest non-flaky tasks a verdict other than hold needs
	// (default 1).
	MinTasks int `yaml:"min_tasks,omitempty" json:"minTasks"`
}

type TokenUsage struct {
	InputTokens  int `json:"input_tokens"`
	OutputTokens int `json:"output_tokens"`
}

type Trial struct {
	Task       string  `json:"task"`
	Variant    string  `json:"variant"`
	Repeat     int     `json:"repeat"`
	Pass       bool    `json:"pass"`
	CostUSD    float64 `json:"costUSD"`
	Turns      int     `json:"turns"`
	WallMs     int64   `json:"wallMs"`
	Tokens     int64   `json:"tokens"`
	ToolErrors int     `json:"toolErrors"`
	ToolCalls  int     `json:"toolCalls"`
	// Seed is derived from the suite seed, task and repeat (identical across
	// variants, so arms stay paired). Recorded; no pinned CLI accepts one.
	Seed uint64 `json:"seed,omitempty"`
	// Grades are the task's graders' results (absent for check-only tasks).
	Grades []Grade `json:"grades,omitempty"`
	// Unavailable is set when the cell could not be measured on this host
	// (ErrSandboxUnavailable): the trial never ran, is excluded from every
	// statistic, and makes the gate hold. Never a failure.
	Unavailable string `json:"unavailable,omitempty"`
	// GraderError is set when a grader could not produce a verdict (e.g. the
	// judge's reply failed schema validation). The trial does not pass.
	GraderError string `json:"graderError,omitempty"`
	// Error records setup/agent/parse problems. The trial is still scored by
	// the check command where possible; infra failures count as not passed.
	Error string `json:"error,omitempty"`
}

type Usage struct {
	CostUSD    float64
	Turns      int
	DurationMs int64
	Tokens     int64 // input + output (+ cache) tokens
	ToolErrors int
	ToolCalls  int
	Failed     bool   // harness reported the run as failed/errored
	Error      string // message when Failed
}

type Variant struct {
	Name    string `yaml:"name" json:"name"`
	Harness string `yaml:"harness" json:"harness"` // claude | codex | gemini
	Version string `yaml:"version" json:"version"`
	Model   string `yaml:"model" json:"model"` // gateway model alias
	// Provider labels the upstream the alias routes to (matrix runs); recorded only.
	Provider string `yaml:"provider,omitempty" json:"provider,omitempty"`
	// Settings is a path (relative to the suite file) to the rendered managed
	// settings for this variant, e.g. the output of `halo render`.
	Settings string `yaml:"settings,omitempty" json:"settings,omitempty"`
	// Dir is the suite dir Settings must live in; set by LoadSuite.
	Dir string `yaml:"-" json:"-"`
}

type VariantScore struct {
	Name        string  `json:"name"`
	Trials      int     `json:"trials"`
	Pass1       float64 `json:"pass1"`
	Pass1CILo   float64 `json:"pass1CILo"`
	Pass1CIHi   float64 `json:"pass1CIHi"`
	CostPerTask float64 `json:"costPerTaskUSD"` // mean cost per trial
	WallP50Ms   int64   `json:"wallP50Ms"`
	WallP95Ms   int64   `json:"wallP95Ms"`
	Tokens      int64   `json:"tokens"`
	ToolErrors  int     `json:"toolErrors"`
	Errors      int     `json:"errors"` // trials with a recorded Error

	Harness  string `json:"harness,omitempty"`
	Version  string `json:"version,omitempty"`
	Model    string `json:"model,omitempty"`
	Provider string `json:"provider,omitempty"`
	// PassAtK / PassHatK are the per-task unbiased pass@k (any of k trials
	// passes) and pass^k (all k pass), averaged over tasks.
	K           int      `json:"k,omitempty"`
	PassAtK     float64  `json:"passAtK"`
	PassHatK    float64  `json:"passHatK"`
	ToolCalls   int      `json:"toolCalls"`
	GradeErrors int      `json:"gradeErrors"`
	JudgeScore  *float64 `json:"judgeScore,omitempty"` // mean judge score over judged trials
	// Unavailable counts trials this host could not measure (Trial.Unavailable,
	// e.g. codex without Landlock). They are excluded from every statistic; a
	// variant with no measured trials shows UnavailableReason, not 0%.
	Unavailable       int    `json:"unavailable,omitempty"`
	UnavailableReason string `json:"unavailableReason,omitempty"`
}

type Verdict struct {
	Score     float64            `json:"score"` // weighted mean of Scores
	Scores    map[string]float64 `json:"scores"`
	Rationale string             `json:"rationale"`
	Cached    bool               `json:"cached,omitempty"`
}