eval

type CapabilityScorecard struct {
	// HardPassRate is Summary.Passed / Summary.Total (0–1).
	HardPassRate float64 `json:"hard_pass_rate"`
	// PassAtK mirrors report pass@k when set (0–1); appendix only.
	PassAtK *float64 `json:"pass_at_k,omitempty"`
	// PassHatK mirrors report pass^k when set (0–1); appendix only.
	PassHatK *float64 `json:"pass_hat_k,omitempty"`
	// Reliability is kept as appendix: 0.6*pass^k + 0.4*pass@k when both set; else HardPassRate.
	Reliability float64 `json:"reliability"`
	// PassAt1 is successful trials / (cases × k) (0–1).
	PassAt1 float64 `json:"pass_at_1"`
	// PassAt1CI is Wilson 95% CI for PassAt1 when trial counts are known.
	PassAt1CI *WilsonCI `json:"pass_at_1_ci,omitempty"`
	// L1Score is compliance rate (0–1).
	L1Score float64 `json:"l1_score"`
	// L2Score is (ToolCallAcc + RepairRate) / 2, or ToolCallAcc when no repair cases.
	L2Score float64 `json:"l2_score"`
	// L3Score is the equal-weight mean of available L3 dimensions.
	L3Score float64 `json:"l3_score"`
	// Total is 100 * (0.2*L1 + 0.5*L2 + 0.3*L3).
	Total float64 `json:"total"`
	// ToolCallAcc is tool hard-gate accuracy among eligible trials (0–1).
	ToolCallAcc *float64 `json:"tool_call_acc,omitempty"`
	// RepairRate is Pass@1 among tier=repair cases (0–1).
	RepairRate *float64 `json:"repair_rate,omitempty"`
	// RepairRateCI is Wilson 95% CI for RepairRate when set.
	RepairRateCI *WilsonCI `json:"repair_rate_ci,omitempty"`
	// LatencyScore is mean clamp(budget/actual) for trials with DurationMs > 0.
	LatencyScore *float64 `json:"latency_score,omitempty"`
	// TokenScore is mean clamp(budget/actual) for trials with TotalTokens > 0.
	TokenScore *float64 `json:"token_score,omitempty"`
	// NaturalRepairRate is appendix: pass rate among trials with ≥1 tool error.
	NaturalRepairRate *float64 `json:"natural_repair_rate,omitempty"`
	// ToolSelectionRate is appendix soft ExpectedTools order match rate.
	ToolSelectionRate *float64 `json:"tool_selection_rate,omitempty"`
	// CorrectnessAvg is mean judge correctness (1–5) when quality scoring is enabled.
	CorrectnessAvg *float64 `json:"correctness_avg,omitempty"`
	// FaithfulnessAvg is mean judge faithfulness (1–5).
	FaithfulnessAvg *float64 `json:"faithfulness_avg,omitempty"`
	// HelpfulnessAvg is mean judge helpfulness (1–5).
	HelpfulnessAvg *float64 `json:"helpfulness_avg,omitempty"`
	// SafetyAvg is mean judge safety (1–5).
	SafetyAvg *float64 `json:"safety_avg,omitempty"`
	// QualityAvg is mean of the four dimensions (1–5); appendix only (not in Total).
	QualityAvg *float64 `json:"quality_avg,omitempty"`
	// JudgeGoldAgreement mirrors report agreement when set (0–1).
	JudgeGoldAgreement *float64 `json:"judge_gold_agreement,omitempty"`
	// QualityEnabled is false for fake/scripted judges.
	QualityEnabled bool `json:"quality_enabled"`
	// TotalCI is set when suite-level repeats ≥ 2 were aggregated externally.
	TotalCI *MeanCI `json:"total_ci,omitempty"`
	// L1CI is suite-repeat CI when set.
	L1CI *MeanCI `json:"l1_ci,omitempty"`
	// L2CI is suite-repeat CI when set.
	L2CI *MeanCI `json:"l2_ci,omitempty"`
	// L3CI is suite-repeat CI when set.
	L3CI *MeanCI `json:"l3_ci,omitempty"`
	// Notes explains how to interpret the scorecard.
	Notes string `json:"notes,omitempty"`
}

type CaseResult struct {
	// Name identifies the case.
	Name string `json:"name"`
	// Difficulty is easy, medium, or hard when set.
	Difficulty string `json:"difficulty,omitempty"`
	// Tier is basic, combo, system, or repair when set.
	Tier string `json:"tier,omitempty"`
	// Passed is the hard gate result.
	Passed bool `json:"passed"`
	// Metrics holds detailed scores (last trial for multi-trial cases).
	Metrics Metrics `json:"metrics"`
	// TrialMetrics holds per-trial metrics for multi-trial cases.
	TrialMetrics []Metrics `json:"trial_metrics,omitempty"`
	// TranscriptSummary is a short human-readable trajectory excerpt (failed cases).
	TranscriptSummary string `json:"transcript_summary,omitempty"`
	// Error is a run-level error message when present.
	Error string `json:"error,omitempty"`
	// Judge holds model-grader scores when used.
	Judge *JudgeScores `json:"judge,omitempty"`
	// Gold holds human gold scores when present.
	Gold *GoldScores `json:"gold,omitempty"`
	// TrialPasses lists per-trial hard pass for multi-trial cases.
	TrialPasses []bool `json:"trial_passes,omitempty"`
}

type CompareDiff struct {
	BaselineSuite      string              `json:"baseline_suite"`
	CandidateSuite     string              `json:"candidate_suite"`
	Improved           []string            `json:"improved"`
	Regressed          []string            `json:"regressed"`
	UnchangedPass      []string            `json:"unchanged_pass"`
	UnchangedFail      []string            `json:"unchanged_fail"`
	OnlyBaseline       []string            `json:"only_baseline"`
	OnlyCandidate      []string            `json:"only_candidate"`
	BaselineScorecard  CapabilityScorecard `json:"baseline_scorecard"`
	CandidateScorecard CapabilityScorecard `json:"candidate_scorecard"`
	TotalDelta         float64             `json:"total_delta"`
	L1Delta            float64             `json:"l1_delta"`
	L2Delta            float64             `json:"l2_delta"`
	L3Delta            float64             `json:"l3_delta"`
	ReliabilityDelta   float64             `json:"reliability_delta"`
	QualityDelta       *float64            `json:"quality_delta,omitempty"`
}

type DockerSandbox struct {
	// contains filtered or unexported fields
}

type DockerSandboxConfig struct {
	Image       string
	Network     string
	Memory      string
	ServerURL   string
	AccessToken string
}

type EvalReport struct {
	// Suite is regression or capability.
	Suite string `json:"suite"`
	// GeneratedAt is RFC3339 timestamp.
	GeneratedAt string `json:"generated_at"`
	// Cases holds per-case results.
	Cases []CaseResult `json:"cases"`
	// Summary aggregates hard pass counts.
	Summary ReportSummary `json:"summary"`
	// PassAtK is set for live multi-trial runs (optional).
	PassAtK *float64 `json:"pass_at_k,omitempty"`
	// PassHatK is set for live multi-trial runs (optional).
	PassHatK *float64 `json:"pass_hat_k,omitempty"`
	// Trials is k when multi-trial aggregation was used.
	Trials int `json:"trials,omitempty"`
	// TotalDurationMs sums trial wall times for the suite.
	TotalDurationMs int64 `json:"total_duration_ms,omitempty"`
	// TotalTokens sums reported token usage across trials.
	TotalTokens int `json:"total_tokens,omitempty"`
	// JudgeGoldAgreement is the fraction of gold dimensions within tolerance.
	JudgeGoldAgreement *float64 `json:"judge_gold_agreement,omitempty"`
	// JudgeMode is "fake", "none", or "model:<name>" for interpreting quality scores.
	JudgeMode string `json:"judge_mode,omitempty"`
	// LatencyBudgetMs is the L3 latency budget used for this report when set.
	LatencyBudgetMs int64 `json:"latency_budget_ms,omitempty"`
	// TokenBudget is the L3 token budget used for this report when set.
	TokenBudget int `json:"token_budget,omitempty"`
	// Scorecard aggregates L1/L2/L3 Total for optimization A/B.
	Scorecard *CapabilityScorecard `json:"scorecard,omitempty"`
}

type Expectation struct {
	// RequiredTools lists tool names that must each appear at least once (hard coverage).
	RequiredTools []string
	// ForbiddenTools lists tool names that must not appear (hard).
	ForbiddenTools []string
	// ExpectedTools lists tool names that should appear in order among executed tools.
	// Soft by default; hard only when StrictToolOrder is true.
	ExpectedTools []string
	// StrictToolOrder makes ExpectedTools order a hard gate.
	StrictToolOrder bool
	// RequiredArgs maps tool name to required argument keys that must be non-empty.
	RequiredArgs map[string][]string
	// MaxSteps bounds assistant turns (0 disables). Hard-fails unless SoftMaxSteps.
	MaxSteps int
	// SoftMaxSteps records MaxSteps overflow without failing the hard gate (capability live).
	SoftMaxSteps bool
	// RequireCompletion requires a non-error final assistant text response.
	RequireCompletion bool
	// Outcome holds additional outcome assertions (files, final text).
	Outcome OutcomeAsserts
}

type FileAssert struct {
	// Path is relative to WorkspaceRoot.
	Path string
	// Contains requires the file to exist and include this substring.
	Contains string
	// Equals, when non-empty, requires exact file content match.
	Equals string
}

type GoldScores struct {
	Correctness  int    `json:"correctness"`
	Faithfulness int    `json:"faithfulness"`
	Helpfulness  int    `json:"helpfulness"`
	Safety       int    `json:"safety"`
	Rationale    string `json:"rationale,omitempty"`
}

type JudgeScores struct {
	Correctness  int    `json:"correctness"`
	Faithfulness int    `json:"faithfulness"`
	Helpfulness  int    `json:"helpfulness"`
	Safety       int    `json:"safety"`
	Reasoning    string `json:"reasoning,omitempty"`
	Unknown      bool   `json:"unknown,omitempty"`
}

type LiveOptions struct {
	// Trials is k (default 3).
	Trials int
	// ModelName is sent as llms.WithModel on each request.
	// Empty defaults to "eval" (safe for FakeModel); real providers need the configured name.
	ModelName string
	// JudgeModel scores open quality dimensions when non-nil.
	JudgeModel llms.Model
	// GoldByCase maps case name to gold scores for agreement.
	GoldByCase map[string]GoldScores
	// OnProgress reports case/trial progress when set.
	OnProgress ProgressFunc
	// JudgeMode labels quality scoring: "fake", "none", or "model:<name>".
	JudgeMode string
	// LatencyBudgetMs overrides default L3 latency budget when > 0.
	LatencyBudgetMs int64
	// TokenBudget overrides default L3 token budget when > 0.
	TokenBudget int
}

type LoadOptions struct {
	Sandbox Sandbox
}

type MeanCI struct {
	// Mean is the sample mean.
	Mean float64 `json:"mean"`
	// Low is mean − 1.96*SEM.
	Low float64 `json:"low"`
	// High is mean + 1.96*SEM.
	High float64 `json:"high"`
	// N is the number of suite repeats.
	N int `json:"n"`
}

type Metrics struct {
	// ToolSelectionCorrect is true when ExpectedTools appear in order (soft metric).
	ToolSelectionCorrect bool
	// ToolSelectionTracked is true when ExpectedTools was non-empty (soft metric denominator).
	ToolSelectionTracked bool
	// RequiredToolsCovered is true when every RequiredTools entry was called.
	RequiredToolsCovered bool
	// ForbiddenToolsClear is true when no ForbiddenTools entry was called.
	ForbiddenToolsClear bool
	// ArgsValid is true when required tool arguments were present and non-empty.
	ArgsValid bool
	// OutcomeOK is true when OutcomeAsserts passed (or none were set).
	OutcomeOK bool
	// TextOutcomeOK is true when final-text asserts passed (or none were set).
	TextOutcomeOK bool
	// FileOutcomeOK is true when file asserts passed (or none were set).
	FileOutcomeOK bool
	// StepCount is the number of assistant turns observed.
	StepCount int
	// Completed is true when the run finished with a final assistant message and no error.
	Completed bool
	// ToolsCalled lists tool names executed during the run.
	ToolsCalled []string
	// FinalText is the last non-empty assistant text content.
	FinalText string
	// AssistantText joins all non-empty assistant texts (used for outcome asserts).
	AssistantText string
	// StepsWithinLimit is true when MaxSteps is 0 or StepCount <= MaxSteps.
	StepsWithinLimit bool
	// DurationMs is wall time for the scenario run.
	DurationMs int64
	// TotalTokens sums assistant Usage.TotalTokens when reported.
	TotalTokens int
	// ToolErrorCount is the number of tool results with IsError.
	ToolErrorCount int
	// NaturalRepairEligible is true when ToolErrorCount >= 1.
	NaturalRepairEligible bool
	// NaturalRepairSuccess is true when eligible and Passed.
	NaturalRepairSuccess bool
	// ComplianceEligible is true when this trial counts toward L1.
	ComplianceEligible bool
	// CompliancePassed is the L1 gate result for this trial.
	CompliancePassed bool
	// ToolAccEligible is true when this trial counts toward ToolCallAcc.
	ToolAccEligible bool
	// ToolAccPassed is the ToolCallAcc hard-gate result for this trial.
	ToolAccPassed bool
	// Passed is the hard CI gate (required/forbidden/args/outcome/completion/max steps; order only if strict).
	Passed bool
}

type OutcomeAsserts struct {
	// FinalTextContains requires each substring in the last assistant text (case-sensitive).
	FinalTextContains []string
	// FinalTextContainsAny requires at least one substring (case-insensitive) when non-empty.
	FinalTextContainsAny []string
	// Files asserts workspace file outcomes (true outcome when tools wrote them).
	Files []FileAssert
}

type ProgressEvent struct {
	// Phase is "case_start", "trial", or "case_done".
	Phase string
	// CaseName is the scenario name.
	CaseName string
	// CaseIndex is 1-based index among scenarios.
	CaseIndex int
	// CaseTotal is the number of scenarios.
	CaseTotal int
	// Trial is 1-based trial index (live only; 0 when N/A).
	Trial int
	// Trials is k for live runs.
	Trials int
	// Passed is set for trial and case_done phases.
	Passed bool
	// Duration is wall time for the trial or whole case.
	Duration time.Duration
	// Detail is an optional failure reason.
	Detail string
}

type ProgressFunc func(ProgressEvent)

type ReportFile struct {
	// Path is the absolute or relative path to the JSON file.
	Path string
	// Suite is capability or regression (from filename).
	Suite string
	// Stamp is the UTC timestamp segment from the filename.
	Stamp string
	// Report is the loaded report with scorecard filled when missing.
	Report EvalReport
}

type ReportSummary struct {
	Total  int `json:"total"`
	Passed int `json:"passed"`
	Failed int `json:"failed"`
}

type RunResult struct {
	// Messages is the full transcript.
	Messages []msg.AgentMessage
	// Metrics are scored outcomes.
	Metrics Metrics
	// Err is the loop error when present.
	Err error
}

type RuntimeSandbox interface {
	Sandbox
	ExecutionEnv(root string) env.ExecutionEnv
}

type Sandbox interface {
	// Prepare creates the sandbox root for a case name under parent.
	Prepare(parent, caseName string) (string, error)
	// Reset clears the root and rewrites fixtures.
	Reset(root string, fixtures []WorkspaceFixture) error
	// Root returns the prepared root path (may be empty when unused).
	Root() string
}

type Scenario struct {
	// Name identifies the scenario in table tests.
	Name string
	// Suite is "regression" or "capability" (for reports).
	Suite string
	// Difficulty is easy, medium, or hard (capability ladder).
	Difficulty string
	// Tier is basic, combo, system, or repair.
	Tier string
	// MetricsTags lists optional metric tags (e.g. compliance).
	MetricsTags []string
	// Prompt is the user message.
	Prompt string
	// Scripts are FakeModel responses in order.
	Scripts []agentllm.ResponseScript
	// Tools are registered for the run.
	Tools []tool.Tool
	// WorkspaceRoot is the isolated workspace for file outcome checks and coding tools.
	WorkspaceRoot string
	// ExecEnv is the tool execution environment bound to WorkspaceRoot.
	ExecEnv env.ExecutionEnv
	// Sandbox tracks workspace lifecycle for per-trial resets.
	Sandbox Sandbox
	// Fixtures seed WorkspaceRoot before each trial (live multi-trial isolation).
	Fixtures []WorkspaceFixture
	// Expect defines scoring criteria.
	Expect Expectation
}

type ScorecardOptions struct {
	// LatencyBudgetMs is the per-trial latency budget (default DefaultLatencyBudgetMs).
	LatencyBudgetMs int64
	// TokenBudget is the per-trial token budget (default DefaultTokenBudget).
	TokenBudget int
}

type TaskDraft struct {
	Name              string `yaml:"name"`
	Suite             string `yaml:"suite"`
	Prompt            string `yaml:"prompt"`
	Notes             string `yaml:"notes"`
	TranscriptExcerpt string `yaml:"transcript_excerpt"`
	SourceError       string `yaml:"source_error,omitempty"`
	ExportedAt        string `yaml:"exported_at"`
}

type WilsonCI struct {
	// Low is the lower bound (0–1).
	Low float64 `json:"low"`
	// High is the upper bound (0–1).
	High float64 `json:"high"`
}

type WorkspaceFixture struct {
	// Path is relative to WorkspaceRoot.
	Path string
	// Content is the file body.
	Content string
}

type WorkspaceSandbox struct {
	// contains filtered or unexported fields
}