eval

type Call struct {
	Name   string
	Args   string
	Output string
	Err    string
}

type Campaign struct {
	N          int
	Aggregates []CaseAggregate
}

type Case struct {
	Name     string            `yaml:"name"`
	Prompt   string            `yaml:"prompt"` // the incident description (seeds the loop)
	Tools    map[string]string `yaml:"tools"`  // tool name -> recorded evidence the tool returns
	Expected Expected          `yaml:"expected"`
}

type CaseAggregate struct {
	Name        string
	Runs        int
	PassRate    float64  // fraction of repeats whose Result.Pass is true
	Reached     bool     // PassRate >= evalMinPassRate
	Flaky       bool     // PassRate in (1-evalMinPassRate, evalMinPassRate): runs disagree
	Confidence  float64  // median confidence over repeats
	Missing     []string // union of missing keywords/entities across repeats
	OverClaimed []string // union of over-claimed distractors across repeats
}

type Coverage struct {
	Touched     []string // mandatory source groups actually exercised
	Missing     []string // mandatory groups never touched
	Bonus       []string // optional groups touched
	CrossSignal bool     // >=2 distinct source groups exercised
	ToolErrors  []string // distinct tool names that returned an error
	Ratio       float64  // |touched| / |expected|  (1.0 when no expected sources)
}

type Dimension struct {
	Key string
	Max int
}

type Expected struct {
	MustContain       []string `yaml:"must_contain"`        // keywords that must appear in the findings (recall, over full findings text)
	MinConfidence     float64  `yaml:"min_confidence"`      // confidence floor (0 = no floor)
	RootCauseEntities []string `yaml:"root_cause_entities"` // entities that MUST be named as the cause (entity recall, over claim text)
	Distractors       []string `yaml:"distractors"`         // plausible-but-wrong entities that must NOT be blamed (over-claim/FP); only evaluated when root_cause_entities is non-empty
}

type GroundTruth struct {
	RootCause       string   `yaml:"root_cause"`
	ExpectedSources []string `yaml:"expected_sources"` // MANDATORY data-source groups -> coverage gate
	OptionalSources []string `yaml:"optional_sources"` // bonus if touched, never gates
	ExpectedAction  string   `yaml:"expected_action"`
	MustReachRoot   bool     `yaml:"must_reach_root"`
}

type Judge interface {
	Grade(ctx context.Context, scn Scenario, inv providers.Investigation) (Verdict, error)
}

type LiveReport struct {
	At      string       `json:"at"`
	N       int          `json:"n"`   // runs per scenario (live mode); 0 in replay mode
	Ran     int          `json:"ran"` // scenarios actually investigated (not skipped)
	Passed  int          `json:"passed"`
	Skipped int          `json:"skipped"`
	Results []LiveResult `json:"results"`
}

type LiveResult struct {
	Scenario       string
	Skipped        bool
	SkipReason     string
	Runs           []RunOutcome
	CoverageRatio  float64        // median
	DimMedian      map[string]int // median per rubric dimension
	DimVariance    map[string]float64
	ConfidentWrong bool     // any run confident-wrong
	Flaky          bool     // root_cause scores vary too much across runs to trust
	ToolErrors     []string // union across runs
	Pass           bool
}

type LiveRunner struct {
	Model     providers.ModelProvider
	BaseTools []investigate.Tool
	Judge     Judge
	Steps     StepRunner
	Log       *slog.Logger
	N         int                    // runs per scenario (default 1 if 0)
	OnRecord  func(Scenario, []Call) // optional: persist the run's calls (replay corpus)
	Recall    *investigate.Recall    // optional; when set, runOnce takes the instant-recall short-circuit (production path). nil ⇒ no recall.
}

type ModelJudge struct {
	Model providers.ModelProvider
}

type Recorder struct {
	// contains filtered or unexported fields
}

type Report struct {
	Results []Result
}

type Result struct {
	Name        string
	Pass        bool
	Confidence  float64
	Missing     []string // expected keywords/entities not found (or an error note); includes "over-claimed: <e>" markers
	OverClaimed []string // distractor entities the investigation wrongly blamed (over-claim/FP)
}

type RunOutcome struct {
	Investigation providers.Investigation
	Coverage      Coverage
	Verdict       Verdict
}

type Runner struct {
	Model providers.ModelProvider
	Log   *slog.Logger
}

type Scenario struct {
	ID          string      `yaml:"id"`
	Category    string      `yaml:"category"` // what-changed | saturation | network | cloud | dependency | cert | dns | storage | instant-recall
	Description string      `yaml:"description"`
	Invasive    bool        `yaml:"invasive"` // true => has setup/teardown; false => natural failure
	Precheck    string      `yaml:"precheck"` // optional shell; non-zero exit => SKIP (natural scenarios)
	Setup       []string    `yaml:"setup"`    // shell steps (kubectl/flux) to induce the fault
	Trigger     Trigger     `yaml:"trigger"`
	GroundTruth GroundTruth `yaml:"ground_truth"`
	Teardown    []string    `yaml:"teardown"` // shell steps to revert; always run
}

type StepRunner interface {
	Run(ctx context.Context, step string) error
}

type Trigger struct {
	Mode      string `yaml:"mode"`      // "cli" (default) | "webhook"
	Symptom   string `yaml:"symptom"`   // free-text incident description
	Namespace string `yaml:"namespace"` // affected namespace (optional)
}

type Verdict struct {
	Scores         map[string]int `json:"scores"`
	ConfidentWrong bool           `json:"confident_wrong"`
	Rationale      string         `json:"rationale"`
}