eval

type ClaudeDriver struct{}

type CodexDriver struct{}

type Command struct {
	Args  []string
	Stdin string
}

type Comparison struct {
	Variant string  `json:"variant"`
	Control string  `json:"control"`
	Tasks   int     `json:"tasks"`
	Delta   float64 `json:"delta"` // mean per-task pass rate: variant - control
	CILo    float64 `json:"ciLo"`
	CIHi    float64 `json:"ciHi"`
	Verdict string  `json:"verdict"`

	// Deltas are paired bootstrap CIs per metric over non-flaky tasks
	// (MetricPass, MetricCost, ...); Gate is the ship/hold/block verdict.
	Deltas map[string]MetricDelta `json:"deltas,omitempty"`
	Gate   *Gate                  `json:"gate,omitempty"`
}

type Criterion struct {
	Name        string  `yaml:"name" json:"name"`
	Description string  `yaml:"description" json:"description"`
	Weight      float64 `yaml:"weight,omitempty" json:"weight,omitempty"` // default 1
}

type DockerRunner struct {
	Bin string // default "docker"
	// Image returns the image for a variant; default
	// ghcr.io/dshakes/eval-<harness>:<version>.
	Image func(Variant) string
	// PassEnv names host env vars forwarded into the agent step only.
	PassEnv []string
	// Network is passed as --network; default "none".
	Network string
	// Memory (default "4g"), CPUs (default "2") and User (default
	// "1000:1000", non-root) bound each container.
	Memory, CPUs, User string
	// SandboxProbes maps harness -> a command that must exit 0 inside a
	// container hardened like the trials before any trial of that harness
	// runs (once per runner and image). nil = DefaultSandboxProbes; an empty
	// map disables probing.
	SandboxProbes map[string][]string
	// contains filtered or unexported fields
}

type Driver interface {
	Name() string
	Command(t *Task, v Variant) Command
	// SettingsPath is where the rendered managed settings must appear inside
	// the environment for the CLI to pick them up.
	SettingsPath() string
	Parse(stdout []byte) (Usage, error)
}

type Env interface {
	Exec(ctx context.Context, c Command) (ExecResult, error)
	Close() error
}

type ExecResult struct {
	Stdout, Stderr []byte
	ExitCode       int
}

type FilePairs struct {
	Path string
	Keys [][]byte
}

type Gate struct {
	Verdict   string     `json:"verdict"` // ship | hold | block
	Reasons   []string   `json:"reasons"`
	Flaky     []TaskDiff `json:"flaky,omitempty"`
	Regressed []TaskDiff `json:"regressed,omitempty"`
	Tasks     []TaskDiff `json:"tasks,omitempty"`
}

type GeminiDriver struct{}

type Grade struct {
	Grader string   `json:"grader"` // type, plus path/rubric when useful
	Pass   bool     `json:"pass"`
	Score  *float64 `json:"score,omitempty"` // judge only
	Rubric string   `json:"rubric,omitempty"`
	Detail string   `json:"detail,omitempty"`
	// Error is a grader failure (e.g. judge schema violation): never a pass.
	Error string `json:"error,omitempty"`
}

type GraderSpec struct {
	Type string `yaml:"type" json:"type"`
	// command
	Run string `yaml:"run,omitempty" json:"run,omitempty"`
	// file: Path relative to the work dir; Exists defaults to true.
	Path        string   `yaml:"path,omitempty" json:"path,omitempty"`
	Exists      *bool    `yaml:"exists,omitempty" json:"exists,omitempty"`
	Contains    []string `yaml:"contains,omitempty" json:"contains,omitempty"`
	NotContains []string `yaml:"not_contains,omitempty" json:"notContains,omitempty"`
	Matches     string   `yaml:"matches,omitempty" json:"matches,omitempty"` // RE2
	// diff: globs (path.Match, slash-separated) over changed paths.
	MaxFilesChanged int      `yaml:"max_files_changed,omitempty" json:"maxFilesChanged,omitempty"`
	MustChange      []string `yaml:"must_change,omitempty" json:"mustChange,omitempty"`
	MustNotChange   []string `yaml:"must_not_change,omitempty" json:"mustNotChange,omitempty"`
	// judge: rubric file relative to the declaring task or suite dir (e.g.
	// ../rubrics/x.yaml), never outside the evals root.
	Rubric string `yaml:"rubric,omitempty" json:"rubric,omitempty"`
	// contains filtered or unexported fields
}

type HTTPLLM struct {
	BaseURL, Wire, APIKey string
	HTTP                  *http.Client // default: 120s timeout
}

type HistoryEntry struct {
	Time     time.Time      `json:"time"`
	Kind     string         `json:"kind"` // offline | online
	Suite    string         `json:"suite,omitempty"`
	Gate     *Gate          `json:"gate,omitempty"`
	Variants []VariantScore `json:"variants,omitempty"`
	Online   *OnlineResult  `json:"online,omitempty"`
}

type Job struct {
	Task    *Task
	Variant Variant
	// Settings is the host path of the rendered managed settings (may be empty)
	// and SettingsDest where it must appear in the environment.
	Settings     string
	SettingsDest string
}

type Judge struct {
	LLM   LLM
	Model string
	Cache *JudgeCache // may be nil
}

type JudgeCache struct {
	Dir string
	// contains filtered or unexported fields
}

type JudgeConfig struct {
	URL   string `yaml:"url" json:"url"`     // gateway base URL; /v1/messages or /v1/responses is appended
	Wire  string `yaml:"wire" json:"wire"`   // anthropic-messages | openai-responses
	Model string `yaml:"model" json:"model"` // exact, pinned model id (no "latest" aliases)
	// APIKeyEnv names the env var holding the gateway credential.
	APIKeyEnv string `yaml:"api_key_env,omitempty" json:"apiKeyEnv,omitempty"`
	// Cache is a directory (relative to the suite) for cached verdicts.
	Cache string `yaml:"cache,omitempty" json:"cache,omitempty"`
}

type JudgeInfo struct {
	Model   string   `json:"model"`
	Wire    string   `json:"wire,omitempty"`
	Rubrics []string `json:"rubrics"`
}

type LLM interface {
	Complete(ctx context.Context, req LLMRequest) (string, error)
}

type LLMRequest struct {
	Model, System, Prompt string
	MaxTokens             int
}

type LocalRunner struct{}

type Matrix struct {
	// Baseline is the control cell name; default the first cell.
	Baseline  string           `yaml:"baseline,omitempty" json:"baseline,omitempty"`
	Harnesses []MatrixHarness  `yaml:"harnesses" json:"harnesses"`
	Providers []MatrixProvider `yaml:"providers,omitempty" json:"providers,omitempty"`
}

type MatrixHarness struct {
	Harness  string   `yaml:"harness" json:"harness"`
	Version  string   `yaml:"version" json:"version"`
	Models   []string `yaml:"models" json:"models"`
	Settings string   `yaml:"settings,omitempty" json:"settings,omitempty"`
}

type MatrixProvider struct {
	Name    string            `yaml:"name" json:"name"`
	Aliases map[string]string `yaml:"aliases,omitempty" json:"aliases,omitempty"`
}

type MetricDelta struct {
	Mean     float64 `json:"mean"`
	CILo     float64 `json:"ciLo"`
	CIHi     float64 `json:"ciHi"`
	Baseline float64 `json:"baseline"`
}

type OTLPExporter struct {
	URL   string // collector base URL; /v1/metrics is appended
	Token string // bearer token (optional)
	HTTP  *http.Client
}

type OnlineOptions struct {
	Judge  *Judge
	Rubric *Rubric
	// Sample caps graded pairs per experiment (0 = all); the sample is a
	// seeded shuffle, so a rerun with the same seed grades the same pairs.
	Sample int
	Seed   uint64
	// Experiment limits grading to one experiment ("" = all).
	Experiment string
	// Skip holds pair ids already graded (from the history file).
	Skip map[string]bool
	Now  func() time.Time
	// ControlNames maps experiment -> its control variant name (from policy).
	// halo-shadow does not label the control side, and analysis matches arms by
	// the experiment's variant names; default "control".
	ControlNames map[string]string
}

type OnlineResult struct {
	Experiment  string  `json:"experiment"`
	Rubric      string  `json:"rubric"`
	JudgeModel  string  `json:"judgeModel"`
	Pairs       int     `json:"pairs"`  // eligible pairs seen
	Graded      int     `json:"graded"` // pairs with both sides judged
	JudgeErrors int     `json:"judgeErrors"`
	Control     float64 `json:"controlScore"`
	Candidate   float64 `json:"candidateScore"`
	// Delta is candidate - control score, paired by pair, with a bootstrap CI.
	Delta   MetricDelta `json:"delta"`
	WinRate float64     `json:"winRate"` // share of pairs the candidate scored higher (ties count half)
	// ControlVariant/CandidateVariant name the arms (shadow target variants).
	ControlVariant   string   `json:"controlVariant"`
	CandidateVariant string   `json:"candidateVariant"`
	PairIDs          []string `json:"pairIds"`
	// Scores are the per-pair readings the exporter sends (not kept in history).
	Scores []PairScore `json:"-"`
}

type PairScore struct {
	ID                 string
	Control, Candidate float64
}

type PairSource interface {
	Pairs(ctx context.Context) ([]shadow.Pair, error)
}

type Part struct {
	Title     string
	Body      string
	Untrusted bool
}

type Rubric struct {
	ID           string      `yaml:"id" json:"id"`
	Version      string      `yaml:"version" json:"version"`
	Instructions string      `yaml:"instructions" json:"instructions"`
	Criteria     []Criterion `yaml:"criteria" json:"criteria"`
	// PassThreshold is the weighted score (0..1) at or above which a judge
	// grader passes; default 0.7.
	PassThreshold float64 `yaml:"pass_threshold,omitempty" json:"passThreshold"`
	// contains filtered or unexported fields
}

type RunOptions struct {
	Parallel int
	// Judge grades tasks' judge graders; required if any task has one.
	Judge *Judge
}

type Runner interface {
	Start(ctx context.Context, j Job) (Env, error)
}

type Scorecard struct {
	Suite       string         `json:"suite"`
	Control     string         `json:"control"`
	Variants    []VariantScore `json:"variants"`
	Comparisons []Comparison   `json:"comparisons"`
	Trials      []Trial        `json:"trials"`

	Seed uint64 `json:"seed"`
	K    int    `json:"k,omitempty"`
	// Gate is the worst comparison gate; nil with a single variant.
	Gate       *Gate       `json:"gate,omitempty"`
	Thresholds *Thresholds `json:"thresholds,omitempty"`
	Judge      *JudgeInfo  `json:"judge,omitempty"`
}

type Suite struct {
	Name string `yaml:"name" json:"name"`
	// Tasks are task directory names under <suite dir>/../tasks.
	Tasks []string `yaml:"tasks" json:"tasks"`
	// Control names the baseline variant; defaults to the first.
	Control  string    `yaml:"control,omitempty" json:"control"`
	Repeats  int       `yaml:"repeats,omitempty" json:"repeats"`
	Variants []Variant `yaml:"variants" json:"variants"`
	// Seed is the base seed for per-trial seeds and bootstrap CIs. No pinned
	// CLI exposes a sampling seed, so it makes the run order and statistics
	// reproducible, not the agent itself.
	Seed uint64 `yaml:"seed,omitempty" json:"seed,omitempty"`
	// K is the k of pass@k and pass^k; default Repeats.
	K int `yaml:"k,omitempty" json:"k,omitempty"`
	// Gate holds the ship/hold/block thresholds.
	Gate Thresholds `yaml:"gate,omitempty" json:"gate"`
	// Judge configures the LLM-as-judge grader used by tasks' judge graders.
	Judge *JudgeConfig `yaml:"judge,omitempty" json:"judge,omitempty"`
	// Matrix, used by `halo eval run --matrix`, replaces Variants with the
	// harness x model x provider cross product.
	Matrix *Matrix `yaml:"matrix,omitempty" json:"matrix,omitempty"`
	// Graders apply to every task in addition to its own (e.g. a judge
	// rubric for the whole suite, so shared tasks stay judge-free).
	Graders []GraderSpec `yaml:"graders,omitempty" json:"graders,omitempty"`

	// Dir is the directory of the suite file; set by LoadSuite.
	Dir string `yaml:"-" json:"-"`
}

type Task struct {
	ID        string        `yaml:"id" json:"id"`
	Repo      string        `yaml:"repo" json:"repo"` // git URL, or directory relative to the task dir
	Setup     []string      `yaml:"setup,omitempty" json:"setup,omitempty"`
	Prompt    string        `yaml:"prompt" json:"prompt"`
	Check     string        `yaml:"check" json:"check"` // shell command, exit 0 = pass
	Timeout   time.Duration `yaml:"timeout,omitempty" json:"timeout,omitempty"`
	BudgetUSD float64       `yaml:"budget_usd,omitempty" json:"budgetUSD,omitempty"`
	MaxTurns  int           `yaml:"max_turns,omitempty" json:"maxTurns,omitempty"`
	Tags      []string      `yaml:"tags,omitempty" json:"tags,omitempty"`
	// Graders all have to pass too; check may be empty when graders are set.
	Graders []GraderSpec `yaml:"graders,omitempty" json:"graders,omitempty"`

	// Dir is the directory holding task.yaml; set by LoadTask.
	Dir string `yaml:"-" json:"-"`
}

type TaskDiff struct {
	Task      string  `json:"task"`
	Baseline  float64 `json:"baseline"`  // pass rate over trials
	Candidate float64 `json:"candidate"` // pass rate over trials
	Status    string  `json:"status"`    // regressed | improved | same | flaky
	Reason    string  `json:"reason,omitempty"`
}

type Thresholds struct {
	// MaxPassDrop is the non-inferiority margin on the paired per-task pass
	// rate: ship needs the CI lower bound >= -MaxPassDrop. Default 0.05.
	MaxPassDrop          *float64 `yaml:"max_pass_drop,omitempty" json:"maxPassDrop,omitempty"`
	MaxCostIncrease      *float64 `yaml:"max_cost_increase,omitempty" json:"maxCostIncrease,omitempty"`
	MaxTokensIncrease    *float64 `yaml:"max_tokens_increase,omitempty" json:"maxTokensIncrease,omitempty"`
	MaxWallIncrease      *float64 `yaml:"max_wall_increase,omitempty" json:"maxWallIncrease,omitempty"`
	MaxToolCallsIncrease *float64 `yaml:"max_tool_calls_increase,omitempty" json:"maxToolCallsIncrease,omitempty"`
	// TaskRegression is the per-task pass-rate drop that marks a task as
	// regressed (default 0.5); more than MaxRegressedTasks (default 0) blocks.
	TaskRegression    float64 `yaml:"task_regression,omitempty" json:"taskRegression"`
	MaxRegressedTasks int     `yaml:"max_regressed_tasks,omitempty" json:"maxRegressedTasks"`
	// MinTasks is the fewest non-flaky tasks a verdict other than hold needs
	// (default 1).
	MinTasks int `yaml:"min_tasks,omitempty" json:"minTasks"`
}

type Trial struct {
	Task       string  `json:"task"`
	Variant    string  `json:"variant"`
	Repeat     int     `json:"repeat"`
	Pass       bool    `json:"pass"`
	CostUSD    float64 `json:"costUSD"`
	Turns      int     `json:"turns"`
	WallMs     int64   `json:"wallMs"`
	Tokens     int64   `json:"tokens"`
	ToolErrors int     `json:"toolErrors"`
	ToolCalls  int     `json:"toolCalls"`
	// Seed is derived from the suite seed, task and repeat (identical across
	// variants, so arms stay paired). Recorded; no pinned CLI accepts one.
	Seed uint64 `json:"seed,omitempty"`
	// Grades are the task's graders' results (absent for check-only tasks).
	Grades []Grade `json:"grades,omitempty"`
	// Unavailable is set when the cell could not be measured on this host
	// (ErrSandboxUnavailable): the trial never ran, is excluded from every
	// statistic, and makes the gate hold. Never a failure.
	Unavailable string `json:"unavailable,omitempty"`
	// GraderError is set when a grader could not produce a verdict (e.g. the
	// judge's reply failed schema validation). The trial does not pass.
	GraderError string `json:"graderError,omitempty"`
	// Error records setup/agent/parse problems. The trial is still scored by
	// the check command where possible; infra failures count as not passed.
	Error string `json:"error,omitempty"`
}

type Usage struct {
	CostUSD    float64
	Turns      int
	DurationMs int64
	Tokens     int64 // input + output (+ cache) tokens
	ToolErrors int
	ToolCalls  int
	Failed     bool   // harness reported the run as failed/errored
	Error      string // message when Failed
}

type Variant struct {
	Name    string `yaml:"name" json:"name"`
	Harness string `yaml:"harness" json:"harness"` // claude | codex | gemini
	Version string `yaml:"version" json:"version"`
	Model   string `yaml:"model" json:"model"` // gateway model alias
	// Provider labels the upstream the alias routes to (matrix runs); recorded only.
	Provider string `yaml:"provider,omitempty" json:"provider,omitempty"`
	// Settings is a path (relative to the suite file) to the rendered managed
	// settings for this variant, e.g. the output of `halo render`.
	Settings string `yaml:"settings,omitempty" json:"settings,omitempty"`
	// Dir is the suite dir Settings must live in; set by LoadSuite.
	Dir string `yaml:"-" json:"-"`
}

type VariantScore struct {
	Name        string  `json:"name"`
	Trials      int     `json:"trials"`
	Pass1       float64 `json:"pass1"`
	Pass1CILo   float64 `json:"pass1CILo"`
	Pass1CIHi   float64 `json:"pass1CIHi"`
	CostPerTask float64 `json:"costPerTaskUSD"` // mean cost per trial
	WallP50Ms   int64   `json:"wallP50Ms"`
	WallP95Ms   int64   `json:"wallP95Ms"`
	Tokens      int64   `json:"tokens"`
	ToolErrors  int     `json:"toolErrors"`
	Errors      int     `json:"errors"` // trials with a recorded Error

	Harness  string `json:"harness,omitempty"`
	Version  string `json:"version,omitempty"`
	Model    string `json:"model,omitempty"`
	Provider string `json:"provider,omitempty"`
	// PassAtK / PassHatK are the per-task unbiased pass@k (any of k trials
	// passes) and pass^k (all k pass), averaged over tasks.
	K           int      `json:"k,omitempty"`
	PassAtK     float64  `json:"passAtK"`
	PassHatK    float64  `json:"passHatK"`
	ToolCalls   int      `json:"toolCalls"`
	GradeErrors int      `json:"gradeErrors"`
	JudgeScore  *float64 `json:"judgeScore,omitempty"` // mean judge score over judged trials
	// Unavailable counts trials this host could not measure (Trial.Unavailable,
	// e.g. codex without Landlock). They are excluded from every statistic; a
	// variant with no measured trials shows UnavailableReason, not 0%.
	Unavailable       int    `json:"unavailable,omitempty"`
	UnavailableReason string `json:"unavailableReason,omitempty"`
}

type Verdict struct {
	Score     float64            `json:"score"` // weighted mean of Scores
	Scores    map[string]float64 `json:"scores"`
	Rationale string             `json:"rationale"`
	Cached    bool               `json:"cached,omitempty"`
}