bench

type Arm string

type ArmReport struct {
	Attempted        int
	Valid            int
	TaskDone         int
	InvariantPass    int
	ContextTokens    int64
	CostUSD          float64
	HookMatches      int
	HookInjections   int
	HookRepeated     int
	HookSuppressed   int
	HookContextBytes int
}

type BenchmarkReport struct {
	ResultSchemaVersion int
	ClaimSchemaVersion  int
	EvidenceFrom        string
	EvidenceTo          string
	Inputs              []ReportInput
	Cohorts             []CohortReport
	Assessments         []ClaimAssessment
}

type CheckResult struct {
	Command  string `json:"command"`
	Pass     bool   `json:"pass"`
	TimedOut bool   `json:"timed_out,omitempty"`
	Output   string `json:"output,omitempty"`
}

type Claim struct {
	ID                           string   `yaml:"id"`
	Claim                        string   `yaml:"claim"`
	PrimaryMetric                string   `yaml:"primary_metric"`
	Comparison                   string   `yaml:"comparison"`
	Direction                    string   `yaml:"direction"`
	RequiredModel                string   `yaml:"required_model"`
	RequiredEffort               string   `yaml:"required_effort"`
	RequireCleanSeamark          bool     `yaml:"require_clean_seamark"`
	MinimumEffect                float64  `yaml:"minimum_effect"`
	MinimumInstanceEffect        float64  `yaml:"minimum_instance_effect"`
	MaximumHarmfulInterference   float64  `yaml:"maximum_harmful_interference"`
	MinimumInstances             int      `yaml:"minimum_instances"`
	MinimumValidPairsPerInstance int      `yaml:"minimum_valid_pairs_per_instance"`
	Instances                    []string `yaml:"instances"`
}

type ClaimAssessment struct {
	ID                  string
	Definition          Claim
	Status              string
	Reason              string
	QualifyingInstances int
	MeanEffect          float64
	WorstInstanceEffect float64
	HarmfulInterference float64
}

type ClaimRegistry struct {
	SchemaVersion int     `yaml:"schema_version"`
	Claims        []Claim `yaml:"claims"`
}

type CohortReport struct {
	Instance         string
	Fingerprint      string
	Task             string
	TaskSHA          string
	Pin              string
	Fixture          string
	RequestedModel   string
	Model            string
	SeamarkVersion   string
	SeamarkSHA       string
	AgentVersion     string
	Effort           string
	HookDelivery     HookDeliveryMode
	RuntimeID        string
	MaxBudgetUSD     float64
	Rows             int
	ValidPairs       int
	FavorablePairs   int
	UnfavorablePairs int
	TiedPairs        int
	HarmfulPairs     int
	HookOn           ArmReport
	HookOff          ArmReport
}

type Command struct {
	Name    string
	Args    []string
	Timeout time.Duration
}

type HookDeliveryMode = reviews.HookDeliveryMode

type Instance struct {
	ID          string
	Rule        string
	Task        string
	LessonYAML  string
	PlaceboYAML string

	Generate  func(string) error
	Judge     func(string) (Verdict, error)
	ApplyGold func(string) error
	// ApplyNaive installs a task-complete solution that deliberately omits the
	// owner invariant. Preflight uses it to prove that the two judges actually
	// discriminate the failure mode the experiment claims to measure.
	ApplyNaive func(string) error
	// JudgeVersion must change whenever verdict semantics change. It binds
	// persisted rows to the exact interpretation used by this instance.
	JudgeVersion string
	Checks       []Command

	// ExploreFiles are repository-relative paths whose appearance in an
	// assistant message is useful diagnostic evidence. They do not affect the
	// verdict.
	ExploreFiles []string
	// contains filtered or unexported fields
}

type ModelUsage struct {
	InputTokens         int64   `json:"input_tokens,omitempty"`
	CacheReadTokens     int64   `json:"cache_read_input_tokens,omitempty"`
	CacheCreationTokens int64   `json:"cache_creation_input_tokens,omitempty"`
	OutputTokens        int64   `json:"output_tokens,omitempty"`
	CostUSD             float64 `json:"cost_usd,omitempty"`
}

type ReportInput struct {
	Path   string
	SHA256 string
	Rows   int
}

type Row struct {
	SchemaVersion int    `json:"schema_version"`
	TS            string `json:"ts"`
	RunID         string `json:"run_id"`
	Instance      string `json:"instance"`
	TaskSHA       string `json:"task_sha256"`
	Pin           string `json:"pin"`
	Arm           Arm    `json:"arm"`
	Trial         int    `json:"trial"`
	TaskDone      bool   `json:"task_pass"`
	Avoided       bool   `json:"invariant_pass"`
	Notes         string `json:"notes,omitempty"`

	// Valid says the agent session and treatment were actually delivered.
	// PairValid additionally says every requested arm in this trial number was
	// valid. Only rows satisfying both enter effect tallies.
	Valid                 bool   `json:"valid"`
	PairValid             bool   `json:"pair_valid"`
	InvalidReason         string `json:"invalid_reason,omitempty"`
	InfrastructureFailure bool   `json:"infrastructure_failure,omitempty"`
	// Fixture is the generated repo's full HEAD commit. Generation
	// is deterministic, so this identifies the exact fixture content a
	// row was measured against — rows from different fixture versions
	// must never be pooled as one series.
	Fixture string `json:"fixture,omitempty"`
	// HookFirings is how many firing records the trial repo's own
	// audit log holds after the run (hook-on arm only). Zero means the
	// injection never reached the agent and the arms were effectively
	// identical — the row proves its treatment happened instead of
	// assuming it.
	HookFirings int `json:"hook_firings,omitempty"`
	// HookAuditRows records every audit row, including unrelated or malformed
	// firings. HookFirings counts only rows proving that the selected lesson
	// reached an in-region edit through the expected hook surface.
	HookAuditRows int `json:"hook_audit_rows,omitempty"`
	// Schema v6 delivery intensity. Matches counts matching edit-hook
	// invocations; each match either injected context or was fully suppressed.
	HookMatches      int `json:"hook_matches"`
	HookInjections   int `json:"hook_injections"`
	HookRepeated     int `json:"hook_repeated_injections"`
	HookSuppressed   int `json:"hook_suppressed"`
	HookContextBytes int `json:"hook_context_bytes"`
	// Transcript is where this trial's raw agent output was saved;
	// StderrLog and Patch keep the rest of the audit record, so a
	// verdict stays checkable after the trial dir is deleted.
	Transcript    string `json:"transcript,omitempty"`
	TranscriptSHA string `json:"transcript_sha256,omitempty"`
	StderrLog     string `json:"stderr,omitempty"`
	StderrSHA     string `json:"stderr_sha256,omitempty"`
	Patch         string `json:"patch,omitempty"`
	PatchSHA      string `json:"patch_sha256,omitempty"`
	// Checks are public repository-local validation commands. Hidden task and
	// invariant judges are represented by TaskDone and Avoided above.
	ChecksPass bool          `json:"checks_pass"`
	Checks     []CheckResult `json:"checks,omitempty"`
	// LessonFileRead: the agent named .seamark/lessons.yaml in its own
	// messages. In arms without a lesson file this must be false; in a
	// control arm it would mean contamination.
	LessonFileRead bool `json:"lesson_file_read,omitempty"`
	// Explored lists instance-selected files the agent named in its own
	// messages and tool calls, in first-mention order. It is diagnostic
	// evidence only and never affects a verdict.
	Explored            []string              `json:"explored,omitempty"`
	RequestedModel      string                `json:"requested_model,omitempty"`
	Model               string                `json:"model,omitempty"`
	ModelUsage          map[string]ModelUsage `json:"model_usage,omitempty"`
	InputTokens         int64                 `json:"input_tokens,omitempty"`
	CacheReadTokens     int64                 `json:"cache_read_input_tokens,omitempty"`
	CacheCreationTokens int64                 `json:"cache_creation_input_tokens,omitempty"`
	ContextTokens       int64                 `json:"context_tokens,omitempty"`
	OutputTokens        int64                 `json:"output_tokens,omitempty"`
	Turns               int                   `json:"turns,omitempty"`
	PermissionDenials   int                   `json:"permission_denials,omitempty"`
	CostUSD             float64               `json:"cost_usd,omitempty"`
	DurationMS          int64                 `json:"duration_ms,omitempty"`
	AgentExit           int                   `json:"agent_exit"`
	TimedOut            bool                  `json:"timed_out,omitempty"`
	AgentError          bool                  `json:"agent_error,omitempty"`
	InitSeen            bool                  `json:"init_seen,omitempty"`
	ResultSeen          bool                  `json:"result_seen,omitempty"`
	Tools               []string              `json:"tools,omitempty"`
	Plugins             []string              `json:"plugins,omitempty"`
	MCPServers          []string              `json:"mcp_servers,omitempty"`
	SeamarkVersion      string                `json:"seamark_version,omitempty"`
	SeamarkSHA          string                `json:"seamark_sha256,omitempty"`
	AgentVersion        string                `json:"agent_version,omitempty"`
	Effort              string                `json:"effort,omitempty"`
	HookDelivery        HookDeliveryMode      `json:"hook_delivery,omitempty"`
	MaxBudgetUSD        float64               `json:"max_budget_usd,omitempty"`
	RuntimeID           string                `json:"runtime_id,omitempty"`
	Fingerprint         string                `json:"fingerprint,omitempty"`
}

type RunConfig struct {
	Trials     int           // trials per arm
	Arms       []Arm         // arms to run; nil means both
	Instance   Instance      // zero value selects SchemaSyncInstance
	AgentArgv  []string      // agent command; the task prompt is appended as the last argument
	SeamarkBin string        // absolute path to the seamark binary (hook command + index)
	Timeout    time.Duration // per-trial agent timeout; 0 means 10 minutes
	Out        string        // results JSONL path, appended one row per trial
	WorkDir    string        // parent for trial dirs; "" means a fresh temp dir
	Keep       bool          // keep trial dirs after judging, for inspection
	// TranscriptDir saves each trial's raw agent stdout (the full
	// stream-json transcript when the agent emits one) for reading WHY
	// a verdict came out the way it did. Empty disables saving.
	TranscriptDir string
	// PrepareIndex runs `seamark index` in hook-on trials so the hook
	// has a store to read. Hermetic tests turn it off.
	PrepareIndex bool
	Version      string // seamark version stamped into rows
	SeamarkSHA   string // exact binary digest stamped into rows
	AgentVersion string // agent CLI version stamped into rows
	Model        string // exact requested primary model; empty for custom agents
	Effort       string // requested effort level
	MaxBudgetUSD float64
	RuntimeID    string // sandbox/toolchain identity
	Fingerprint  string // immutable instance + runtime configuration hash
	// RunID groups rows and makes transcript names unique across concurrent
	// invocations. Empty asks Run to generate a cryptographically random ID.
	RunID string
	// HookDelivery selects the edit-hook repeat policy. Empty means always.
	HookDelivery HookDeliveryMode

	// RequireStructuredResult and RequireCleanInit are true for the
	// default Claude adapter. Stub/custom adapters may leave them false.
	RequireStructuredResult bool
	RequireCleanInit        bool
	Log                     func(string, ...any) // progress lines; nil silences
}

type Summary struct {
	Rows          []Row
	ByArm         map[Arm]Tally
	RunID         string
	Instance      string
	Rule          string
	StoppedReason string
}

type Tally struct {
	Attempted    int
	Ran          int
	Invalid      int
	Completed    int // trials where the task was done at all
	Avoided      int // completed trials where the owner invariant passed
	Firings      int // hook firing records across the arm's trials
	Matches      int
	Injections   int
	Repeated     int
	Suppressed   int
	ContextBytes int
	MeanInput    int64
}

type Verdict struct {
	// TaskDone means the agent completed the visible task.
	TaskDone bool
	// Avoided means a task-complete solution also preserved the owner invariant.
	Avoided bool
	// Notes concisely explains the deterministic judgment.
	Notes string
}