evals

type ArmAvail struct {
	Name   string
	Skip   string // empty = available
	Binary string
}

type ArmSummary struct {
	Cases    int     `json:"cases"`
	PassAt1  float64 `json:"pass_at_1"`
	PassHatK float64 `json:"pass_hat_k"`
	CostUSD  float64 `json:"cost_usd"`
	Flips    string  `json:"flips,omitempty"`
}

type Case struct {
	ID           string            `yaml:"id"`
	Task         string            `yaml:"task"`
	Tier         string            `yaml:"tier"`
	Gating       *bool             `yaml:"gating"`
	Tags         []string          `yaml:"tags"`
	Intention    string            `yaml:"intention"`
	EditPath     string            `yaml:"edit_path"`
	Arms         []string          `yaml:"arms"`
	Trials       int               `yaml:"trials"`
	BudgetUSD    float64           `yaml:"budget_usd"`
	Timeout      string            `yaml:"timeout"`
	Setup        []any             `yaml:"setup"`
	Teardown     []any             `yaml:"teardown"`
	Fixtures     []string          `yaml:"fixtures"`
	Connections  []string          `yaml:"conns"`
	Env          map[string]string `yaml:"env"`
	Artifact     string            `yaml:"artifact"`
	Graders      CaseGraders       `yaml:"graders"`
	Mock         MockHints         `yaml:"mock"`
	SeedFiles    map[string]string `yaml:"seed_files"`
	ResetSchemas []string          `yaml:"reset_schemas"`

	Path    string `yaml:"-"` // case.yaml path
	CaseDir string `yaml:"-"` // folder that holds case.yaml
}

type CaseGraders struct {
	Required []GraderSpec `yaml:"required"`
	Optional []GraderSpec `yaml:"optional"`
	Judge    []string     `yaml:"judge"`
}

type CaseMeta struct {
	Gating bool
	Tier   string
}

type CaseMetrics struct {
	Case       string  `json:"case"`
	Arm        string  `json:"arm"`
	Trials     int     `json:"trials"`
	Passed     int     `json:"passed"`
	PassAtK    bool    `json:"pass_at_k"`
	PassHatK   bool    `json:"pass_hat_k"`
	MeanScore  float64 `json:"mean_score"`
	MeanCost   float64 `json:"mean_cost"`
	MeanDur    float64 `json:"mean_duration_s"`
	SkipReason string  `json:"skip_reason,omitempty"`
	Gating     bool    `json:"gating"`
	Tier       string  `json:"tier,omitempty"`
	Timeouts   int     `json:"timeouts,omitempty"`
}

type CompareResult struct {
	Path    string
	Pass    bool
	Message string
	Want    any
	Got     any
}

type FixtureDef struct {
	Connection string       `yaml:"connection"`
	Requires   []string     `yaml:"requires"`
	Provision  []GraderSpec `yaml:"provision"`
	ReadyCheck ReadyCheck   `yaml:"ready_check"`
}

type FixtureServer struct {
	URL string
	// contains filtered or unexported fields
}

type Flags struct {
	Arms          []string
	Tags          []string
	Tiers         []string
	Cases         []string
	Trials        int
	Baseline      string
	MaxSuiteUSD   float64
	Parallel      int
	ResetFixtures bool
	RetryFailed   bool
}

type Flip struct {
	Case  string  `json:"case"`
	Arm   string  `json:"arm"`
	From  string  `json:"from"` // pass | fail
	To    string  `json:"to"`   // pass | fail | removed
	Delta float64 `json:"score_delta"`
}

type GradeContext struct {
	WorkDir        string
	Artifact       string // relative to WorkDir
	CaseDir        string // case folder with case.yaml, expected/, mutants/
	Transcript     string
	SlingBin       string
	Env            []string
	ConnDown       map[string]bool // name → down
	SkipExecute    bool            // mock arm: do not run L5 execute graders
	FixtureBaseURL string
	FixtureToken   string
	APIPageCap     int
	APITimeout     time.Duration
}

type GraderResult struct {
	Name     string `json:"name"`
	Pass     bool   `json:"pass"`
	Skip     bool   `json:"skip,omitempty"`
	Optional bool   `json:"optional,omitempty"`
	Judge    bool   `json:"judge,omitempty"`
	Detail   string `json:"detail,omitempty"`
	Critique string `json:"critique,omitempty"`
}

type GraderSpec map[string]any

type MatchPolicy struct {
	MustMatch    []string `yaml:"must_match"`
	MustNotExist []string `yaml:"must_not_exist"`
}

type MockHints struct {
	Transcript       string `yaml:"transcript"`
	MutantTranscript string `yaml:"mutant_transcript"`
}

type Provisioner struct {
	Bin      string
	Env      []string
	Logf     func(string, ...any)
	Registry map[string]FixtureDef
	// contains filtered or unexported fields
}

type ReadyCheck struct {
	Connection string `yaml:"connection"`
	SQL        string `yaml:"sql"`
	Equals     any    `yaml:"equals"`
}

type ResultsWriter struct {
	// contains filtered or unexported fields
}

type Suite struct {
	Flags       Flags
	Bin         string
	Writer      *ResultsWriter
	Logf        func(string, ...any)
	Server      *FixtureServer
	Provisioner *Provisioner
	// contains filtered or unexported fields
}

type SuiteSummary struct {
	Run          string                `json:"run"`
	Arms         map[string]ArmSummary `json:"arms"`
	Cases        []CaseMetrics         `json:"cases"`
	Flips        []Flip                `json:"flips,omitempty"`
	Threshold    float64               `json:"threshold"`
	TierRates    map[string]float64    `json:"tier_rates,omitempty"`
	Verdict      string                `json:"verdict"`
	CostUSD      float64               `json:"cost_usd"`
	SkillsDelta  string                `json:"skills_delta,omitempty"`
	CostByFamily map[string]float64    `json:"cost_by_family,omitempty"`
	Judge        string                `json:"judge,omitempty"`
	Binary       string                `json:"binary,omitempty"`
	Timeouts     int                   `json:"timeouts,omitempty"`
	GatingN      int                   `json:"gating_n,omitempty"`
	GatingPass   int                   `json:"gating_pass,omitempty"`
	GatingRate   float64               `json:"gating_rate,omitempty"`
	SchemaReset  string                `json:"schema_reset,omitempty"`
}

type TrialResult struct {
	Run           string         `json:"run"`
	Case          string         `json:"case"`
	Arm           string         `json:"arm"`
	Trial         int            `json:"trial"`
	Invariant     string         `json:"invariant,omitempty"` // passable | sensitivity (mock)
	Pass          bool           `json:"pass"`
	Score         float64        `json:"score"`
	Graders       []GraderResult `json:"graders"`
	CostUSD       float64        `json:"cost_usd"`
	DurationS     float64        `json:"duration_s"`
	Turns         int            `json:"turns,omitempty"`
	PromptMD5     string         `json:"prompt_md5,omitempty"`
	SkillsVersion string         `json:"skills_version,omitempty"`
	Model         string         `json:"model,omitempty"`
	SkipReason    string         `json:"skip_reason,omitempty"`
	Error         string         `json:"error,omitempty"`
	InfraError    bool           `json:"infra_error,omitempty"`
	Retried       bool           `json:"retried,omitempty"`
	Timeout       bool           `json:"timeout,omitempty"`
	Binary        string         `json:"binary,omitempty"`
	Transcript    string         `json:"-"`
}