bench

type Class string

type Column struct {
	Mode  string
	Model string
	File  *File
}

type CompareOptions struct {
	// Order lists modes to put first, in this order; the rest follow by
	// mode and model. repo-steward puts baseline and scripted first;
	// casework reports baseline, scripted, replay.
	Order []string
	// AllowMixedCommits accepts columns from different commits. The
	// output then says, above the tables, that they are not comparable
	// as like for like. repo-steward's committed comparison mixes three
	// commits and says why in prose.
	AllowMixedCommits bool
	// LinkPrefix, when set, links each column's Source as
	// LinkPrefix+Source, as repo-steward links into results/.
	LinkPrefix string
}

type Count struct {
	Outcome string `json:"outcome"`
	Class   Class  `json:"class"`
	N       int    `json:"n"`
	// Of is the judged trials expecting Over, or every judged trial when
	// Over is empty. A trial with this outcome that expected something
	// else, as a repo-steward scenario expecting a proposal that accepts
	// a refusal, is added to Of as well, so N never exceeds Of.
	Of   int    `json:"of"`
	Over string `json:"over,omitempty"`
}

type File struct {
	Format string `json:"format"`
	// Commit is the consumer's commit the trials ran at. Empty means
	// unknown. Compare treats every unknown as one commit that is not any
	// named one: files that all lack a commit are compared without
	// CompareOptions.AllowMixedCommits even if they ran at different
	// commits, a file without one beside a file with one needs it, and the
	// output names the unknown commit "no commit recorded".
	Commit    string    `json:"commit,omitempty"`
	StartedAt time.Time `json:"started_at"`
	// Options are the harness settings: repo-steward's budget caps,
	// casework's replay directory and recording date.
	Options  map[string]json.RawMessage `json:"options,omitempty"`
	Taxonomy Taxonomy                   `json:"taxonomy"`
	Trials   []Trial                    `json:"trials"`
	// Notes are run-level remarks, as repo-steward records stopping early
	// when a total budget is reached.
	Notes []string `json:"notes,omitempty"`
	// Summaries are written by Encode and checked by Decode against the
	// trials; a file whose summaries disagree with its trials is refused.
	Summaries []Summary `json:"summaries"`

	// Source is the name ReadDir read the file from.
	Source string `json:"-"`
}

type Outcome struct {
	// Name is the consumer's own identifier, as its classifier returns it.
	Name  string `json:"name"`
	Class Class  `json:"class"`
	// Over names the expectation whose trials form this outcome's
	// denominator; empty means every judged trial. repo-steward counts
	// completed and incorrect_refusal over runs expecting "proposal" and
	// correct_refusal over runs expecting "refusal"; casework counts every
	// category over every classified row.
	Over string `json:"over,omitempty"`
	// Label is the heading reports use; empty means Name.
	Label string `json:"label,omitempty"`
}

type Summary struct {
	Mode  string `json:"mode"`
	Model string `json:"model,omitempty"`
	// Trials counts every row; Judged those with an outcome; Excluded the
	// rest.
	Trials    int `json:"trials"`
	Judged    int `json:"judged"`
	Excluded  int `json:"excluded"`
	Scenarios int `json:"scenarios"`
	// Repeats is the highest repeat recorded.
	Repeats int     `json:"repeats"`
	Counts  []Count `json:"counts"`
	// Unsafe is the number of judged trials with an unsafe outcome.
	Unsafe        int           `json:"unsafe"`
	Steps         int64         `json:"steps"`
	ToolCalls     int64         `json:"tool_calls"`
	PolicyDenials int64         `json:"policy_denials"`
	ModelCalls    int64         `json:"model_calls"`
	InputTokens   int64         `json:"input_tokens"`
	OutputTokens  int64         `json:"output_tokens"`
	CostMicros    int64         `json:"cost_micros"`
	Wall          time.Duration `json:"wall_ns"`
	// Totals sums each integer-valued Extra field, as repo-steward totals
	// policy aborts and casework totals unauthorized actions.
	Totals map[string]int64 `json:"totals,omitempty"`
}

type Taxonomy struct {
	// Name identifies the set, so that results scored by different sets
	// are never compared.
	Name     string    `json:"name"`
	Outcomes []Outcome `json:"outcomes"`
}

type Trial struct {
	Scenario string `json:"scenario"`
	Mode     string `json:"mode"`
	Model    string `json:"model,omitempty"`
	// Repeat counts from 1.
	Repeat int    `json:"repeat"`
	RunID  string `json:"run_id,omitempty"`
	// Expect is the class of answer the scenario expects, the key an
	// Outcome's Over refers to: repo-steward's "proposal" or "refusal".
	Expect string `json:"expect,omitempty"`
	// Reached is the consumer's own end state, before classification:
	// repo-steward's outcome (blocked, proposal_prepared), casework's
	// conclusion (corrected, escalate).
	Reached string `json:"reached,omitempty"`
	// Outcome is the taxonomy's name for the trial, empty only when
	// Excluded is set.
	Outcome string `json:"outcome,omitempty"`
	// Excluded says why the trial was not judged, as casework's replay
	// rows with no recording are. An excluded trial is in no denominator.
	Excluded      string `json:"excluded,omitempty"`
	Steps         int    `json:"steps"`
	ToolCalls     int    `json:"tool_calls"`
	PolicyDenials int    `json:"policy_denials"`
	ModelCalls    int    `json:"model_calls"`
	InputTokens   int    `json:"input_tokens"`
	OutputTokens  int    `json:"output_tokens"`
	// CostMicros is money in millionths of a currency unit, the unit of
	// agentrt.Micros, so a consumer converts with a cast.
	CostMicros int64         `json:"cost_micros"`
	Wall       time.Duration `json:"wall_ns"`
	Error      string        `json:"error,omitempty"`
	Notes      []string      `json:"notes,omitempty"`
	// Extra holds what one consumer records and the other does not, as
	// JSON kept byte for byte. Integer values are summed into
	// Summary.Totals; the rest is carried.
	Extra map[string]json.RawMessage `json:"extra,omitempty"`
}