bench

type Arm string

type ArmReport struct {
	Attempted        int
	Valid            int
	TaskDone         int
	InvariantPass    int
	ContextTokens    int64
	CostUSD          float64
	HookMatches      int
	HookInjections   int
	HookRepeated     int
	HookSuppressed   int
	HookContextBytes int
}

type BenchmarkReport struct {
	ResultSchemaVersion int
	ClaimSchemaVersion  int
	EvidenceFrom        string
	EvidenceTo          string
	Inputs              []ReportInput
	Cohorts             []CohortReport
	Assessments         []ClaimAssessment
}

type CheckResult struct {
	Command  string `json:"command"`
	Pass     bool   `json:"pass"`
	TimedOut bool   `json:"timed_out,omitempty"`
	Output   string `json:"output,omitempty"`
}

type Claim struct {
	ID            string `yaml:"id"`
	Claim         string `yaml:"claim"`
	PrimaryMetric string `yaml:"primary_metric"`
	Comparison    string `yaml:"comparison"`
	// TreatmentInstance and ControlInstance name the two variants a
	// difference_in_differences claim compares. Both must appear in
	// Instances; empty for hook-on_vs_hook-off claims.
	TreatmentInstance            string   `yaml:"treatment_instance,omitempty"`
	ControlInstance              string   `yaml:"control_instance,omitempty"`
	Direction                    string   `yaml:"direction"`
	RequiredModel                string   `yaml:"required_model"`
	RequiredEffort               string   `yaml:"required_effort"`
	RequireCleanSeamark          bool     `yaml:"require_clean_seamark"`
	MinimumEffect                float64  `yaml:"minimum_effect"`
	MinimumInstanceEffect        float64  `yaml:"minimum_instance_effect"`
	MaximumHarmfulInterference   float64  `yaml:"maximum_harmful_interference"`
	MinimumInstances             int      `yaml:"minimum_instances"`
	MinimumValidPairsPerInstance int      `yaml:"minimum_valid_pairs_per_instance"`
	Instances                    []string `yaml:"instances"`
}

type ClaimAssessment struct {
	ID                  string
	Definition          Claim
	Status              string
	Reason              string
	QualifyingInstances int
	MeanEffect          float64
	WorstInstanceEffect float64
	HarmfulInterference float64
	// AnchorNote carries the component effects behind a factorial claim. It is
	// report-only; the frozen verdict uses their difference.
	AnchorNote string
}

type ClaimRegistry struct {
	SchemaVersion int     `yaml:"schema_version"`
	Claims        []Claim `yaml:"claims"`
}

type CohortReport struct {
	Instance            string
	Fingerprint         string
	Task                string
	TaskSHA             string
	Pin                 string
	Fixture             string
	RequestedModel      string
	Model               string
	SeamarkVersion      string
	SeamarkSHA          string
	AgentVersion        string
	Effort              string
	HookDelivery        HookDeliveryMode
	HookExposure        HookExposureExpectation
	ComparisonFamily    string
	ProtocolFingerprint string
	RuntimeID           string
	MaxBudgetUSD        float64
	Rows                int
	ValidPairs          int
	FavorablePairs      int
	UnfavorablePairs    int
	TiedPairs           int
	HarmfulPairs        int
	HookOn              ArmReport
	HookOff             ArmReport
}

type Command struct {
	Name    string
	Args    []string
	Timeout time.Duration
}

type HookDeliveryMode = reviews.HookDeliveryMode

type HookExposureExpectation string

type Instance struct {
	ID          string
	Rule        string
	Task        string
	LessonYAML  string
	PlaceboYAML string

	Generate  func(string) error
	Judge     func(string) (Verdict, error)
	ApplyGold func(string) error
	// ApplyNaive installs a task-complete solution that deliberately omits the
	// owner invariant. Preflight uses it to prove that the two judges actually
	// discriminate the failure mode the experiment claims to measure.
	ApplyNaive func(string) error
	// JudgeVersion must change whenever verdict semantics change. It binds
	// persisted rows to the exact interpretation used by this instance.
	JudgeVersion string
	Checks       []Command
	// HookExposure controls validation of hooked arms. Empty means required,
	// preserving the original benchmark contract. Optional is reserved for a
	// scope-control whose valid outcome may be zero matching edits.
	HookExposure HookExposureExpectation
	// ComparisonFamily groups intentionally different instances that form one
	// factorial experiment. Empty means the instance is not cross-compared.
	ComparisonFamily string
	// ProtocolInstance names the canonical instance whose full fingerprint
	// defines the shared protocol for ComparisonFamily. Every family variant
	// points to the same canonical instance.
	ProtocolInstance string

	// ExploreFiles are repository-relative paths whose appearance in an
	// assistant message is useful diagnostic evidence. They do not affect the
	// verdict.
	ExploreFiles []string

	// Prepare materializes any external, pinned source required by Generate.
	// Synthetic instances leave it nil. Public-repository instances use an
	// explicit preparation step so paid runs and their agents never depend on
	// network access.
	Prepare func(context.Context) (string, error)
	// contains filtered or unexported fields
}

type ModelUsage struct {
	InputTokens         int64   `json:"input_tokens,omitempty"`
	CacheReadTokens     int64   `json:"cache_read_input_tokens,omitempty"`
	CacheCreationTokens int64   `json:"cache_creation_input_tokens,omitempty"`
	OutputTokens        int64   `json:"output_tokens,omitempty"`
	CostUSD             float64 `json:"cost_usd,omitempty"`
}

type ReportInput struct {
	Path   string
	SHA256 string
	Rows   int
}

type Row struct {
	SchemaVersion int    `json:"schema_version"`
	TS            string `json:"ts"`
	RunID         string `json:"run_id"`
	Instance      string `json:"instance"`
	TaskSHA       string `json:"task_sha256"`
	Pin           string `json:"pin"`
	Arm           Arm    `json:"arm"`
	Trial         int    `json:"trial"`
	TaskDone      bool   `json:"task_pass"`
	Avoided       bool   `json:"invariant_pass"`
	Notes         string `json:"notes,omitempty"`

	// Valid says the agent session and treatment were actually delivered.
	// PairValid additionally says every requested arm in this trial number was
	// valid. Only rows satisfying both enter effect tallies.
	Valid                 bool   `json:"valid"`
	PairValid             bool   `json:"pair_valid"`
	InvalidReason         string `json:"invalid_reason,omitempty"`
	InfrastructureFailure bool   `json:"infrastructure_failure,omitempty"`
	// Fixture is the generated repo's full HEAD commit. Generation
	// is deterministic, so this identifies the exact fixture content a
	// row was measured against — rows from different fixture versions
	// must never be pooled as one series.
	Fixture string `json:"fixture,omitempty"`
	// HookFirings is how many firing records the trial repo's own
	// audit log holds after the run (hook-on arm only). Zero means the
	// injection never reached the agent and the arms were effectively
	// identical — the row proves its treatment happened instead of
	// assuming it.
	HookFirings int `json:"hook_firings,omitempty"`
	// HookAuditRows records every audit row, including unrelated or malformed
	// firings. HookFirings counts only rows proving that the selected lesson
	// reached an in-region edit through the expected hook surface.
	HookAuditRows int `json:"hook_audit_rows,omitempty"`
	// Schema v6 delivery intensity. Matches counts matching edit-hook
	// invocations; each match either injected context or was fully suppressed.
	HookMatches      int `json:"hook_matches"`
	HookInjections   int `json:"hook_injections"`
	HookRepeated     int `json:"hook_repeated_injections"`
	HookSuppressed   int `json:"hook_suppressed"`
	HookContextBytes int `json:"hook_context_bytes"`
	// Transcript is where this trial's raw agent output was saved;
	// StderrLog and Patch keep the rest of the audit record, so a
	// verdict stays checkable after the trial dir is deleted.
	Transcript    string `json:"transcript,omitempty"`
	TranscriptSHA string `json:"transcript_sha256,omitempty"`
	StderrLog     string `json:"stderr,omitempty"`
	StderrSHA     string `json:"stderr_sha256,omitempty"`
	Patch         string `json:"patch,omitempty"`
	PatchSHA      string `json:"patch_sha256,omitempty"`
	// Checks are public repository-local validation commands. Hidden task and
	// invariant judges are represented by TaskDone and Avoided above.
	ChecksPass bool          `json:"checks_pass"`
	Checks     []CheckResult `json:"checks,omitempty"`
	// LessonFileRead: the agent named .seamark/lessons.yaml in its own
	// messages. In arms without a lesson file this must be false; in a
	// control arm it would mean contamination.
	LessonFileRead bool `json:"lesson_file_read,omitempty"`
	// Explored lists instance-selected files the agent named in its own
	// messages and tool calls, in first-mention order. It is diagnostic
	// evidence only and never affects a verdict.
	Explored            []string                `json:"explored,omitempty"`
	RequestedModel      string                  `json:"requested_model,omitempty"`
	Model               string                  `json:"model,omitempty"`
	ModelUsage          map[string]ModelUsage   `json:"model_usage,omitempty"`
	InputTokens         int64                   `json:"input_tokens,omitempty"`
	CacheReadTokens     int64                   `json:"cache_read_input_tokens,omitempty"`
	CacheCreationTokens int64                   `json:"cache_creation_input_tokens,omitempty"`
	ContextTokens       int64                   `json:"context_tokens,omitempty"`
	OutputTokens        int64                   `json:"output_tokens,omitempty"`
	Turns               int                     `json:"turns,omitempty"`
	PermissionDenials   int                     `json:"permission_denials,omitempty"`
	CostUSD             float64                 `json:"cost_usd,omitempty"`
	DurationMS          int64                   `json:"duration_ms,omitempty"`
	AgentExit           int                     `json:"agent_exit"`
	TimedOut            bool                    `json:"timed_out,omitempty"`
	AgentError          bool                    `json:"agent_error,omitempty"`
	InitSeen            bool                    `json:"init_seen,omitempty"`
	ResultSeen          bool                    `json:"result_seen,omitempty"`
	Tools               []string                `json:"tools,omitempty"`
	Plugins             []string                `json:"plugins,omitempty"`
	MCPServers          []string                `json:"mcp_servers,omitempty"`
	SeamarkVersion      string                  `json:"seamark_version,omitempty"`
	SeamarkSHA          string                  `json:"seamark_sha256,omitempty"`
	AgentVersion        string                  `json:"agent_version,omitempty"`
	Effort              string                  `json:"effort,omitempty"`
	HookDelivery        HookDeliveryMode        `json:"hook_delivery,omitempty"`
	HookExposure        HookExposureExpectation `json:"hook_exposure,omitempty"`
	MaxBudgetUSD        float64                 `json:"max_budget_usd,omitempty"`
	RuntimeID           string                  `json:"runtime_id,omitempty"`
	Fingerprint         string                  `json:"fingerprint,omitempty"`
	ProtocolFingerprint string                  `json:"protocol_fingerprint,omitempty"`
	ComparisonFamily    string                  `json:"comparison_family,omitempty"`
}

type RunConfig struct {
	Trials     int           // trials per arm
	Arms       []Arm         // arms to run; nil means both
	Instance   Instance      // zero value selects SchemaSyncInstance
	AgentArgv  []string      // agent command; the task prompt is appended as the last argument
	SeamarkBin string        // absolute path to the seamark binary (hook command + index)
	Timeout    time.Duration // per-trial agent timeout; 0 means 10 minutes
	Out        string        // results JSONL path, appended one row per trial
	WorkDir    string        // parent for trial dirs; "" means a fresh temp dir
	Keep       bool          // keep trial dirs after judging, for inspection
	// TranscriptDir saves each trial's raw agent stdout (the full
	// stream-json transcript when the agent emits one) for reading WHY
	// a verdict came out the way it did. Empty disables saving.
	TranscriptDir string
	// PrepareIndex runs `seamark index` in hook-on trials so the hook
	// has a store to read. Hermetic tests turn it off.
	PrepareIndex bool
	Version      string // seamark version stamped into rows
	SeamarkSHA   string // exact binary digest stamped into rows
	AgentVersion string // agent CLI version stamped into rows
	Model        string // exact requested primary model; empty for custom agents
	Effort       string // requested effort level
	MaxBudgetUSD float64
	RuntimeID    string // sandbox/toolchain identity
	Fingerprint  string // immutable instance + runtime configuration hash
	// ProtocolFingerprint omits the intentionally varied instance scope while
	// binding every shared part of a cross-instance experiment.
	ProtocolFingerprint string
	// RunID groups rows and makes transcript names unique across concurrent
	// invocations. Empty asks Run to generate a cryptographically random ID.
	RunID string
	// HookDelivery selects the edit-hook repeat policy. Empty means always.
	HookDelivery HookDeliveryMode

	// RequireStructuredResult and RequireCleanInit are true for the
	// default Claude adapter. Stub/custom adapters may leave them false.
	RequireStructuredResult bool
	RequireCleanInit        bool
	Log                     func(string, ...any) // progress lines; nil silences
}

type Summary struct {
	Rows          []Row
	ByArm         map[Arm]Tally
	RunID         string
	Instance      string
	Rule          string
	HookExposure  HookExposureExpectation
	StoppedReason string
}

type Tally struct {
	Attempted    int
	Ran          int
	Invalid      int
	Completed    int // trials where the task was done at all
	Avoided      int // completed trials where the owner invariant passed
	Firings      int // hook firing records across the arm's trials
	Matches      int
	Injections   int
	Repeated     int
	Suppressed   int
	ContextBytes int
	MeanInput    int64
}

type Verdict struct {
	// TaskDone means the agent completed the visible task.
	TaskDone bool
	// Avoided means a task-complete solution also preserved the owner invariant.
	Avoided bool
	// Notes concisely explains the deterministic judgment.
	Notes string
}