contract

type ArtifactRef struct {
	Name   string `json:"name"`
	Digest string `json:"digest"`
	Bytes  int64  `json:"bytes"`
	// Verified is the grader output that checked it, empty when nothing did.
	Verified string `json:"verified,omitempty"`
}

type BuildIdentity struct {
	Version string `json:"version"`
	Commit  string `json:"commit"`
	// Dirty is the digest of uncommitted changes, empty for a clean tree. A
	// boolean would say a local build is unreproducible without saying which
	// local build it was, and two dirty candidates would then compare as the
	// same subject.
	Dirty string `json:"dirty,omitempty"`
	// ArtifactDigest is the binary or container image measured. It is what
	// makes this a black-box result: without it the manifest describes a source
	// revision rather than the thing that ran.
	ArtifactDigest string `json:"artifact_digest"`
}

type Comparison struct {
	ContractVersion int    `json:"contract_version"`
	ExperimentID    string `json:"experiment_id"`
	Suite           string `json:"suite"`
	BaselineID      string `json:"baseline_id"`
	CandidateID     string `json:"candidate_id"`

	Baseline  SuiteMetrics `json:"baseline"`
	Candidate SuiteMetrics `json:"candidate"`

	// Delta is candidate pass rate minus baseline, with the interval of the
	// paired difference. The interval is the point of a paired design: two
	// separately-computed rates can both move without their difference being
	// distinguishable from noise.
	Delta     float64 `json:"delta"`
	DeltaLow  float64 `json:"delta_low"`
	DeltaHigh float64 `json:"delta_high"`

	// Improved and Regressed name tasks whose outcome changed direction.
	// Unscorable names tasks the comparison could not judge on either side,
	// which section 12 requires shown rather than omitted: a task silently
	// dropped from both arms reads as agreement.
	Improved   []string `json:"improved,omitempty"`
	Regressed  []string `json:"regressed,omitempty"`
	Unscorable []string `json:"unscorable,omitempty"`
}

type CriticalFailure struct {
	TaskID string `json:"task_id"`
	Grader string `json:"grader"`
	Detail string `json:"detail,omitempty"`
}

type DatasetRef struct {
	Name    string `json:"name"`
	Version string `json:"version"`
	Digest  string `json:"digest"`
}

type Domain string

type Duration int64

type ExecutionIdentity struct {
	Surface Surface `json:"surface"`
	// AdapterVersion changes when the adapter changes how it invokes the
	// subject. An adapter change moves results without the product moving, and
	// a comparison spanning one is not paired.
	AdapterVersion int `json:"adapter_version"`
}

type Experiment struct {
	ContractVersion int        `json:"contract_version"`
	ID              string     `json:"id"`
	Name            string     `json:"name"`
	CreatedAt       time.Time  `json:"created_at"`
	Dataset         DatasetRef `json:"dataset"`

	// Subjects are what was measured. A comparison names two of them; a single
	// qualification run has one. They are stored in full rather than by ID
	// because the manifest is the only thing that makes an old result legible.
	Subjects []SubjectManifest `json:"subjects"`
	// Baseline is the subject ID other subjects are compared against, empty
	// when the experiment is not a comparison.
	Baseline string `json:"baseline,omitempty"`

	// Trials is the repetition count applied to every task, overriding the
	// task's own default when higher. Section 12 estimates pass@1 from
	// independent attempts, so one attempt per task is a measurement without an
	// error bar rather than a cheaper version of the same result.
	Trials int `json:"trials"`

	// Tasks are the task IDs included, recorded because a suite filter that is
	// not written down turns two runs of "the same" experiment into an
	// unpaired comparison.
	Tasks []string `json:"tasks"`
}

type ExtensionInputs struct {
	Plugins  []string `json:"plugins,omitempty"`
	Skills   []string `json:"skills,omitempty"`
	MCP      []string `json:"mcp,omitempty"`
	Subagent []string `json:"subagents,omitempty"`
}

type GraderKind string

type GraderRef struct {
	Name    string     `json:"name"`
	Version int        `json:"version"`
	Kind    GraderKind `json:"kind"`
	// Required means the trial fails when this grader fails. An optional
	// grader reports a dimension without deciding the outcome.
	Required bool `json:"required"`
	// Critical marks a hard gate. Section 7.5 keeps these out of any suite
	// summary, so a critical failure cannot be averaged away.
	Critical bool `json:"critical,omitempty"`
	// Config is the grader's own parameters, opaque here. Graders version
	// separately from the contract, so their schemas must not force a contract
	// version bump.
	Config json.RawMessage `json:"config,omitempty"`
}

type GraderResult struct {
	Name    string     `json:"name"`
	Version int        `json:"version"`
	Kind    GraderKind `json:"kind"`
	// Required and Critical are copied from the task's GraderRef rather than
	// looked up. A bundle is read years after the task may have changed, and a
	// verdict whose gating weight has to be resolved elsewhere is a verdict
	// that will eventually be re-interpreted.
	Required bool `json:"required"`
	Critical bool `json:"critical,omitempty"`

	Verdict Verdict `json:"verdict"`
	// Score is the dimension's value where one exists, in [0,1]. Absent for a
	// grader that only decides, which is not the same as scoring zero.
	Score *float64 `json:"score,omitempty"`
	// Explanation is why, in the grader's own words. Deterministic graders put
	// their output here; model graders put their reasoning.
	Explanation string `json:"explanation,omitempty"`

	// Subject identifies the model that produced a model grader's verdict, and
	// Usage what it cost. Section 10.3 makes a judge a measured component
	// rather than an oracle, and a judge whose own configuration is unrecorded
	// cannot be re-calibrated against the labels it was validated on.
	Subject string `json:"subject,omitempty"`
	Usage   *Usage `json:"usage,omitempty"`

	Duration Duration `json:"duration,omitempty"`
	// Error means the grader could not reach a verdict at all, as distinct
	// from reaching VerdictUnknown deliberately.
	Error string `json:"error,omitempty"`
}

type HostProfile struct {
	OS       string `json:"os"`
	Arch     string `json:"arch"`
	CPUs     int    `json:"cpus,omitempty"`
	MemoryMB int    `json:"memory_mb,omitempty"`
	// Network says what the trial could reach: "none", "proxied", "open". A
	// task failing for lack of network is not an Agent that cannot do it.
	Network string `json:"network,omitempty"`
}

type InstructionInputs struct {
	SystemPromptDigest string `json:"system_prompt_digest"`
	ToolSchemaDigest   string `json:"tool_schema_digest"`
	// Layers names the instruction sources in order, without their bodies:
	// a workspace AGENTS.md, an agent definition, session notes. The names are
	// provenance; the digest above is what actually pins the content.
	Layers []string `json:"layers,omitempty"`
}

type Limits struct {
	WallSeconds int `json:"wall_seconds"`
	Iterations  int `json:"iterations,omitempty"`
	ToolCalls   int `json:"tool_calls,omitempty"`
	Tokens      int `json:"tokens,omitempty"`
}

type ModelIdentity struct {
	// Transport is how inference was reached: a provider protocol, or the
	// managed gateway. Two subjects reaching the same model through different
	// transports are different subjects.
	Transport string `json:"transport"`
	Target    string `json:"target"`
	Alias     string `json:"alias,omitempty"`
	// Revision is the exact model build the provider reported. Absent means the
	// provider reported none, which the manifest states rather than filling in
	// from the target name: section 8.2 requires recorded uncertainty over
	// invented precision, because a qualification that names a revision the
	// provider never confirmed cannot be re-run against it.
	Revision      string `json:"revision,omitempty"`
	Reasoning     string `json:"reasoning,omitempty"`
	ContextWindow int    `json:"context_window,omitempty"`
	MaxOutput     int    `json:"max_output,omitempty"`
}

type PolicyResolution struct {
	Permissions string `json:"permissions"`
	// Sandboxed is a pointer so an unsandboxed subject records false rather
	// than omitting the field, for the reason the trace does the same: an
	// unreported boundary is indistinguishable from an unresolved one, and a
	// reader breaking that tie favourably would credit a subject with
	// protection it never had.
	Sandboxed   *bool    `json:"sandboxed"`
	SandboxMode string   `json:"sandbox_mode,omitempty"`
	Hooks       []string `json:"hooks,omitempty"`
}

type Reproduction struct {
	Command     []string          `json:"command"`
	Environment map[string]string `json:"environment,omitempty"`
	Dataset     DatasetRef        `json:"dataset"`
	// Note is anything a re-runner needs that the command does not carry, such
	// as a dependency that must already be installed.
	Note string `json:"note,omitempty"`
}

type RetentionLevel string

type SubjectManifest struct {
	ContractVersion int    `json:"contract_version"`
	Name            string `json:"name"`
	// ID is the digest of every other field, from Digest. Two trials share a
	// subject when they share this value, which is what makes a paired
	// comparison honest: the alternative, comparing by name, silently pairs
	// runs whose configuration drifted between them.
	ID string `json:"id"`

	Build        BuildIdentity     `json:"build"`
	Execution    ExecutionIdentity `json:"execution"`
	Model        ModelIdentity     `json:"model"`
	Instructions InstructionInputs `json:"instructions"`
	Extensions   ExtensionInputs   `json:"extensions"`
	Policy       PolicyResolution  `json:"policy"`
	Host         HostProfile       `json:"host"`
	// Dataset is the task collection version this subject was measured over.
	// It sits on the subject as well as the experiment because a bundle read on
	// its own must still say what it was asked.
	Dataset DatasetRef `json:"dataset"`
}

type SuiteMetrics struct {
	Suite     string `json:"suite"`
	SubjectID string `json:"subject_id"`

	Trials int `json:"trials"`
	// Scored is the number of trials that judged the subject. The pass rate
	// below is over this, not over Trials, so harness faults neither count as
	// failures nor silently shrink the denominator without being visible.
	Scored int `json:"scored"`
	Passed int `json:"passed"`

	// PassRate is Passed over Scored, and IntervalLow/High its confidence
	// interval. A rate without an interval invites reading a two-trial
	// difference as a regression.
	PassRate     float64 `json:"pass_rate"`
	IntervalLow  float64 `json:"interval_low"`
	IntervalHigh float64 `json:"interval_high"`

	// ConsistencyRate is the share of tasks passing every attempt: pass^k, for
	// suites where consistency rather than best-of-k is the product promise.
	ConsistencyRate float64 `json:"consistency_rate,omitempty"`

	// Faults counts the statuses that blamed the harness, by status. Section 12
	// reports these rather than dropping them, because a suite losing a third
	// of its trials otherwise looks like one that ran clean.
	Faults map[TrialStatus]int `json:"faults,omitempty"`

	// CriticalFailures names the task and grader of every critical failure.
	// These are gates, not inputs to a rate.
	CriticalFailures []CriticalFailure `json:"critical_failures,omitempty"`

	Usage    Usage `json:"usage"`
	MedianMS int64 `json:"median_ms,omitempty"`
	P95MS    int64 `json:"p95_ms,omitempty"`
}

type Surface string

type Task struct {
	ContractVersion int      `json:"contract_version"`
	ID              string   `json:"id"`
	Version         int      `json:"version"`
	Suite           string   `json:"suite"`
	Title           string   `json:"title"`
	Tags            []string `json:"tags,omitempty"`
	Domain          Domain   `json:"domain"`
	Surface         Surface  `json:"surface"`

	// Turns is what the user says, in order. A single-turn task has one entry.
	// Modelling every task as a sequence avoids a second code path for the
	// multi-turn scenarios section 11 requires; a richer turn, such as a
	// simulated user with a policy, extends this field rather than adding one.
	Turns []string `json:"turns"`

	// Capabilities are what the subject must provide for the task to mean
	// anything, such as a sandbox backend or a configured MCP server. A subject
	// missing one yields invalid_task, not a failed attempt: the task was never
	// asked under the conditions it describes.
	Capabilities []string `json:"capabilities,omitempty"`

	Limits Limits `json:"limits"`

	// Graders run in the order given. A required grader that fails decides the
	// trial; an optional one contributes a dimension without gating.
	Graders []GraderRef `json:"graders"`

	// Negative marks a task whose required outcome is that something did not
	// happen: a boundary held, a file was left alone, a tool was never reached.
	//
	// Such a task's deterministic graders legitimately pass against the
	// untouched initial state, because changing nothing is the correct answer,
	// so preflight's "does not already satisfy the outcome" check does not
	// apply to it. What must apply instead is a required trace or model grader:
	// without one the task asserts only that nothing happened, which an agent
	// that did nothing at all — or never ran — would satisfy just as well.
	Negative bool `json:"negative,omitempty"`

	// Trials is the default independent-attempt count. An experiment may raise
	// it; a task only meaningful over repetition — anything measuring pass^k —
	// says so here rather than relying on the caller to know.
	Trials int `json:"trials"`

	// Environment pins the versions a trial depends on beyond the subject, such
	// as a container image or a language toolchain. It is copied onto the
	// bundle so a later reader can tell an environment change from a subject
	// change.
	Environment map[string]string `json:"environment,omitempty"`

	// Oracle is the command completing the task from the initial state, run
	// from OracleDir. Preflight requires it to pass every required grader: a
	// task whose own reference solution fails is measuring its graders rather
	// than the Agent.
	Oracle []string `json:"oracle,omitempty"`
}

type TrialBundle struct {
	ContractVersion int    `json:"contract_version"`
	TrialID         string `json:"trial_id"`
	ExperimentID    string `json:"experiment_id"`
	TaskID          string `json:"task_id"`
	TaskVersion     int    `json:"task_version"`
	Suite           string `json:"suite"`
	SubjectID       string `json:"subject_id"`
	// Index is which independent attempt this is, from zero. Paired comparison
	// matches candidate and baseline on task and index, so it has to be
	// recorded rather than inferred from file order.
	Index int `json:"index"`

	Domain  Domain  `json:"domain"`
	Surface Surface `json:"surface"`

	Status TrialStatus `json:"status"`
	// FailureClass refines Status for a failure, such as a boundary violation
	// or an early stop. It is free-form because the taxonomy is expected to
	// grow from observed failures rather than be enumerated in advance; a gate
	// reads Status, and a human reads this.
	FailureClass string `json:"failure_class,omitempty"`
	// Error is what went wrong for a status that is not a grader verdict.
	Error string `json:"error,omitempty"`

	StartedAt time.Time `json:"started_at"`
	Duration  Duration  `json:"duration"`

	// InitialStateDigest is the workspace the trial started from. Without it a
	// re-run cannot prove it began where the recorded one did, and section 8.1
	// requires an initial state that does not already satisfy the outcome.
	InitialStateDigest string `json:"initial_state_digest"`
	// FinalStateDigest is what the workspace became. It is the outcome-first
	// evidence of section 7.1: a reply claiming a file was written is not the
	// file.
	FinalStateDigest string `json:"final_state_digest,omitempty"`

	Retention RetentionLevel `json:"retention"`
	// Reply is the subject's final answer, subject to Retention. Absent at
	// metadata retention, which is not the same fact as a run that said
	// nothing.
	Reply string `json:"reply,omitempty"`

	// TracePath is the durable trace inside the bundle directory, relative to
	// it. Children are the subagent traces the run spawned, so a delegation
	// failure is diagnosable from the bundle alone.
	TracePath      string   `json:"trace_path,omitempty"`
	ChildTracePath []string `json:"child_trace_paths,omitempty"`

	Artifacts []ArtifactRef  `json:"artifacts,omitempty"`
	Graders   []GraderResult `json:"graders,omitempty"`
	Usage     Usage          `json:"usage"`

	// Reproduce is the bounded description of how to run this trial again:
	// the command, the environment it needed, and the dataset it came from.
	// Section 17 makes it part of what a failure has to hand a contributor.
	Reproduce Reproduction `json:"reproduce"`
}

type TrialStatus string

type Usage struct {
	LLMCalls         int `json:"llm_calls"`
	ToolCalls        int `json:"tool_calls"`
	PromptTokens     int `json:"prompt_tokens"`
	CompletionTokens int `json:"completion_tokens"`
	CacheReadTokens  int `json:"cache_read_tokens,omitempty"`
	CacheWriteTokens int `json:"cache_write_tokens,omitempty"`
	// Cost is nano-units of Currency, matching the runtime's integer
	// representation so a reader sums a suite exactly. Absent when the model
	// was unpriced.
	Cost     *int64 `json:"cost,omitempty"`
	Currency string `json:"currency,omitempty"`
	// CostIncomplete says part of the trial could not be priced, so Cost
	// understates it rather than covering it.
	CostIncomplete bool `json:"cost_incomplete,omitempty"`
}

type Verdict string