eval

type Call struct {
	Name   string
	Args   string
	Output string
	Err    string
}

type Campaign struct {
	N          int
	Aggregates []CaseAggregate
}

type Case struct {
	Name     string            `yaml:"name"`
	Prompt   string            `yaml:"prompt"` // the incident description (seeds the loop)
	Tools    map[string]string `yaml:"tools"`  // tool name -> recorded evidence the tool returns
	Expected Expected          `yaml:"expected"`
	// GroundTruth is optional live-scenario ground truth carried into replay. When
	// present it unlocks the richer scoring the model-comparison benchmark reports:
	// data-source coverage (expected_sources) and blind LLM-judge rubric grading
	// (root_cause / expected_action). Absent ⇒ keyword-only scoring, as before.
	GroundTruth *GroundTruth `yaml:"ground_truth,omitempty"`

	// Workload is the incident's affected workload (namespace + name). It seeds the
	// request and — when a catalog fixture is present — drives the recall structural
	// gate (resource agreement). Optional; zero for alerts without a workload.
	Workload *CaseWorkload `yaml:"workload,omitempty"`
	// CatalogDir, when set, points at a directory of knowledge-base markdown entries
	// RELATIVE to the case file. Its presence seeds an instant-recall catalog for this
	// case and wires Recall + the adversarial verify pass into the replay loop exactly
	// as production does — so the closed recall→verify loop is exercised mechanically in
	// the replay eval. Absent ⇒ the case replays with no recall, unchanged.
	CatalogDir string `yaml:"catalog_dir,omitempty"`
	// Recall optionally tunes the recall gates for this case (mirrors config
	// instant_recall). Absent (or a zero field) ⇒ the production default. Consulted only
	// when CatalogDir is set.
	Recall *CaseRecall `yaml:"recall,omitempty"`
	// ExpectRecall asserts the recall outcome mechanically and fails the case when unmet:
	//   short_circuit — recall fired and its answer was delivered (loop skipped)
	//   withdrawn     — recall fired but the verify pass rejected it and the loop fell
	//                   through to a full investigation
	//   fired         — recall fired (either short_circuit or withdrawn)
	//   rejected      — a recall gate rejected the hit: recall never fired
	// Empty ⇒ no recall assertion (existing cases are unaffected).
	ExpectRecall string `yaml:"expect_recall,omitempty"`
	// contains filtered or unexported fields
}

type CaseAggregate struct {
	Name        string
	Runs        int
	PassRate    float64  // fraction of repeats whose Result.Pass is true
	Reached     bool     // PassRate >= evalMinPassRate
	Flaky       bool     // PassRate in (1-evalMinPassRate, evalMinPassRate): runs disagree
	Confidence  float64  // median confidence over repeats
	Missing     []string // union of missing keywords/entities across repeats
	OverClaimed []string // union of over-claimed distractors across repeats
}

type CaseRecall struct {
	MinScore             float64 `yaml:"min_score"`
	MarginGap            float64 `yaml:"margin_gap"`
	SoloFloor            float64 `yaml:"solo_floor"`
	RequireWorkloadMatch bool    `yaml:"require_workload_match"`
	OutcomePrior         float64 `yaml:"outcome_prior"`
	OutcomeFloor         float64 `yaml:"outcome_floor"`
}

type CaseWorkload struct {
	Namespace string `yaml:"namespace"`
	Name      string `yaml:"name"`
}

type CompareCaseRow struct {
	Name     string  `json:"name"`
	Runs     int     `json:"runs"`
	PassRate float64 `json:"pass_rate"`
	Reached  bool    `json:"reached"`
	Flaky    bool    `json:"flaky"`
	Coverage float64 `json:"coverage"` // median coverage ratio over the case's runs
}

type CompareSpec struct {
	Judge  *JudgeSpec   `yaml:"judge,omitempty"`
	Models []ModelEntry `yaml:"models"`
}

type ComparedCase struct {
	Name string
	Runs []ComparedRun
}

type ComparedRun struct {
	Result   Result
	Coverage Coverage
	Verdict  Verdict
	Graded   bool
}

type ComparisonReport struct {
	At     string            `json:"at"`
	N      int               `json:"n"`     // runs per case
	Judge  string            `json:"judge"` // judge model disclosure, e.g. "anthropic/claude-…"
	Models []ModelComparison `json:"models"`
}

type ComparisonRunner struct {
	Model providers.ModelProvider // the entry under test (wrap with CountingModel for token totals)
	Judge Judge                   // fixed across entries; nil skips rubric grading
	Log   *slog.Logger
}

type CountingModel struct {
	Inner providers.ModelProvider
	// contains filtered or unexported fields
}

type Coverage struct {
	Touched     []string // mandatory source groups actually exercised
	Missing     []string // mandatory groups never touched
	Bonus       []string // optional groups touched
	CrossSignal bool     // >=2 distinct source groups exercised
	ToolErrors  []string // distinct tool names that returned an error
	Ratio       float64  // |touched| / |expected|  (1.0 when no expected sources)
}

type Dimension struct {
	Key string
	Max int
}

type Expected struct {
	MustContain       []string `yaml:"must_contain"`        // keywords that must appear in the findings (recall, over full findings text)
	MinConfidence     float64  `yaml:"min_confidence"`      // confidence floor (0 = no floor)
	RootCauseEntities []string `yaml:"root_cause_entities"` // entities that MUST be named as the cause (entity recall, over claim text)
	Distractors       []string `yaml:"distractors"`         // plausible-but-wrong entities that must NOT be blamed (over-claim/FP); only evaluated when root_cause_entities is non-empty
}

type GroundTruth struct {
	RootCause       string   `yaml:"root_cause"`
	ExpectedSources []string `yaml:"expected_sources"` // MANDATORY data-source groups -> coverage gate
	OptionalSources []string `yaml:"optional_sources"` // bonus if touched, never gates
	ExpectedAction  string   `yaml:"expected_action"`
	MustReachRoot   bool     `yaml:"must_reach_root"`
}

type Judge interface {
	Grade(ctx context.Context, scn Scenario, inv providers.Investigation, transcript ...string) (Verdict, error)
}

type JudgeSpec struct {
	Provider  string `yaml:"provider"`
	BaseURL   string `yaml:"base_url"`
	Model     string `yaml:"model"`
	APIKeyEnv string `yaml:"api_key_env"`
}

type LiveReport struct {
	At      string       `json:"at"`
	N       int          `json:"n"`   // runs per scenario (live mode); 0 in replay mode
	Ran     int          `json:"ran"` // scenarios actually investigated (not skipped)
	Passed  int          `json:"passed"`
	Skipped int          `json:"skipped"`
	Results []LiveResult `json:"results"`
}

type LiveResult struct {
	Scenario       string
	Skipped        bool
	SkipReason     string
	Runs           []RunOutcome
	CoverageRatio  float64        // median
	DimMedian      map[string]int // median per rubric dimension
	DimVariance    map[string]float64
	ConfidentWrong bool     // any run confident-wrong
	Flaky          bool     // root_cause scores vary too much across runs to trust
	ToolErrors     []string // union across runs
	Pass           bool
}

type LiveRunner struct {
	Model     providers.ModelProvider
	BaseTools []investigate.Tool
	Judge     Judge
	Steps     StepRunner
	Log       *slog.Logger
	N         int                    // runs per scenario (default 1 if 0)
	OnRecord  func(Scenario, []Call) // optional: persist the run's calls (replay corpus)
	Recall    *investigate.Recall    // optional; when set, runOnce takes the instant-recall short-circuit (production path). nil ⇒ no recall.
}

type ModelComparison struct {
	Name     string `json:"name"` // the entry's report label
	Provider string `json:"provider"`
	Model    string `json:"model"`
	Effort   string `json:"effort,omitempty"`

	// Aggregate scores.
	PassRate       float64            `json:"pass_rate"`               // fraction of cases that reached the k-of-n bar
	Reached        int                `json:"reached"`                 // cases whose pass-rate met the k-of-n bar
	Total          int                `json:"total"`                   // cases run
	RubricMedian   map[string]float64 `json:"rubric_median,omitempty"` // per-dimension median over graded runs; nil when ungraded
	GradedRuns     int                `json:"graded_runs"`             // runs the judge graded (0 ⇒ rubric omitted)
	CoverageMedian float64            `json:"coverage_median"`         // median coverage ratio over all runs
	ConfidentWrong int                `json:"confident_wrong"`         // graded runs flagged confident-and-wrong

	// Usage + cost.
	InputTokens  int      `json:"input_tokens"`
	OutputTokens int      `json:"output_tokens"`
	CostUSD      *float64 `json:"cost_usd,omitempty"` // present only when the entry supplied prices

	Cases []CompareCaseRow `json:"cases"`
}

type ModelEntry struct {
	Name      string `yaml:"name"`     // report label; must be unique
	Provider  string `yaml:"provider"` // "openai" (default) | "anthropic" | "gemini"
	BaseURL   string `yaml:"base_url"`
	Model     string `yaml:"model"`
	APIKeyEnv string `yaml:"api_key_env"` // env var holding the API key (empty = keyless)
	// Effort is the OpenAI-compatible reasoning_effort request field (e.g. "low",
	// "medium", "high"). Only valid for the openai provider; other providers reject it.
	Effort string `yaml:"effort,omitempty"`
	// Prices enables the estimated-cost column; omit it to omit the column.
	Prices *Prices `yaml:"prices,omitempty"`
}

type ModelJudge struct {
	Model providers.ModelProvider
}

type Prices struct {
	InputUSD  float64 `yaml:"input_usd" json:"input_usd"`
	OutputUSD float64 `yaml:"output_usd" json:"output_usd"`
}

type Recorder struct {
	// contains filtered or unexported fields
}

type Result struct {
	Name        string
	Pass        bool
	Confidence  float64
	Missing     []string // expected keywords/entities not found (or an error note); includes "over-claimed: <e>" markers
	OverClaimed []string // distractor entities the investigation wrongly blamed (over-claim/FP)

	// Recall telemetry (populated only for cases with a catalog fixture): whether
	// instant recall fired, and whether its answer short-circuited the loop. Surfaced
	// so recall behaviour is asserted mechanically, not inferred from the finding.
	RecallFired        bool
	RecallShortCircuit bool
}

type RunOutcome struct {
	Investigation providers.Investigation
	Coverage      Coverage
	Verdict       Verdict
}

type Runner struct {
	Model providers.ModelProvider
	Log   *slog.Logger
}

type Scenario struct {
	ID          string      `yaml:"id"`
	Category    string      `yaml:"category"` // what-changed | saturation | network | cloud | dependency | cert | dns | storage | instant-recall
	Description string      `yaml:"description"`
	Invasive    bool        `yaml:"invasive"` // true => has setup/teardown; false => natural failure
	Precheck    string      `yaml:"precheck"` // optional shell; non-zero exit => SKIP (natural scenarios)
	Setup       []string    `yaml:"setup"`    // shell steps (kubectl/flux) to induce the fault
	Trigger     Trigger     `yaml:"trigger"`
	GroundTruth GroundTruth `yaml:"ground_truth"`
	Teardown    []string    `yaml:"teardown"` // shell steps to revert; always run
}

type StepRunner interface {
	Run(ctx context.Context, step string) error
}

type Trigger struct {
	Mode      string `yaml:"mode"`      // "cli" (default) | "webhook"
	Symptom   string `yaml:"symptom"`   // free-text incident description
	Namespace string `yaml:"namespace"` // affected namespace (optional)
}

type Verdict struct {
	Scores         map[string]int `json:"scores"`
	ConfidentWrong bool           `json:"confident_wrong"`
	Rationale      string         `json:"rationale"`
}