models

type ActionSequenceGraderParameters struct {
	MatchingMode    ActionSequenceMatchingMode `yaml:"matching_mode,omitempty" json:"matching_mode,omitempty"`
	ExpectedActions []string                   `yaml:"expected_actions,omitempty" json:"expected_actions,omitempty"`
}

type ActionSequenceMatchingMode string

type AdversarialConfig struct {
	// Packs is the list of built-in pack identifiers to run (e.g.
	// "prompt-injection", "scope-bypass"). Unknown identifiers are rejected
	// by `waza adversarial`; the current schema does not support
	// out-of-tree custom packs. Required; must contain at least one entry.
	Packs []string `yaml:"packs" json:"packs"`
	// OnUnsafeOutcome controls whether unsafe outcomes fail the run
	// ("fail", default) or only warn ("warn"). When empty, "fail" is used.
	OnUnsafeOutcome AdversarialOnUnsafeOutcome `yaml:"on_unsafe_outcome,omitempty" json:"on_unsafe_outcome,omitempty"`
}

type AdversarialOnUnsafeOutcome string

type BehaviorGraderParameters struct {
	MaxToolCalls   int      `yaml:"max_tool_calls,omitempty" json:"max_tool_calls,omitempty"`
	MaxTokens      int      `yaml:"max_tokens,omitempty" json:"max_tokens,omitempty"`
	RequiredTools  []string `yaml:"required_tools,omitempty" json:"required_tools,omitempty"`
	ForbiddenTools []string `yaml:"forbidden_tools,omitempty" json:"forbidden_tools,omitempty"`
	MaxDurationMS  int64    `yaml:"max_duration_ms,omitempty" json:"max_duration_ms,omitempty"`
}

type BehaviorRules struct {
	MaxToolInvocations int      `yaml:"max_tool_calls,omitempty" json:"max_tool_invocations,omitempty"`
	MaxRounds          int      `yaml:"max_iterations,omitempty" json:"max_rounds,omitempty"`
	MaxTokens          int      `yaml:"max_tokens,omitempty" json:"max_tokens,omitempty"`
	MaxResponseTimeMs  int64    `yaml:"max_response_time_ms,omitempty" json:"max_response_time_ms,omitempty"`
	MustUseTool        []string `yaml:"required_tools,omitempty" json:"must_use_tool,omitempty"`
	ForbidTool         []string `yaml:"forbidden_tools,omitempty" json:"forbid_tool,omitempty"`
}

type Checkpoint struct {
	// AfterTurn is the 1-based turn index at whose boundary the graders run.
	// Turn 1 is the initial prompt; turns 2..N are follow-ups or responder
	// replies. Must be >= 1.
	AfterTurn int `yaml:"after_turn" json:"after_turn"`
	// Graders is the slice of grader configurations to run at this turn
	// boundary. Required. Reuses the same ValidatorInline shape as task-level
	// `graders:` so any existing grader type is supported.
	Graders []ValidatorInline `yaml:"graders" json:"graders"`
	// OnFailure controls multi-turn behavior when any grader in this
	// checkpoint fails. "continue" (default) records the failure and keeps
	// going. "stop" short-circuits the multi-turn loop after the failure.
	OnFailure CheckpointOnFailure `yaml:"on_failure,omitempty" json:"on_failure,omitempty"`
}

type CheckpointOnFailure string

type CheckpointOutcome struct {
	// AfterTurn echoes Checkpoint.AfterTurn so consumers can sort/group.
	AfterTurn int `json:"after_turn"`
	// Status is StatusPassed when every grader in this checkpoint passed,
	// StatusFailed when at least one grader failed.
	Status Status `json:"status"`
	// Validations maps grader identifier to result, identical to
	// RunResult.Validations.
	Validations map[string]GraderResults `json:"validations"`
	// Stopped is true when this checkpoint had `on_failure: stop` and at
	// least one grader failed, terminating the multi-turn loop after this
	// turn.
	Stopped bool `json:"stopped,omitempty"`
}

type Config struct {
	TrialsPerTask int `yaml:"trials_per_task" json:"runs_per_test"`
	TimeoutSec    int `yaml:"timeout_seconds" json:"timeout_sec"`
	// FirstEventTimeoutSec bounds how long to wait for the first session event
	// before treating a run as a session-start hang (the engine launched but
	// never started the agent's first turn). It is much shorter than
	// TimeoutSec, which must cover the slowest legitimate full turn. 0 (the
	// default) disables the check. Per-task overrides live on TestCase.
	FirstEventTimeoutSec int            `yaml:"first_event_timeout_seconds,omitempty" json:"first_event_timeout_sec,omitempty"`
	Concurrent           bool           `yaml:"parallel" json:"concurrent"`
	Workers              int            `yaml:"workers,omitempty" json:"workers,omitempty"`
	StopOnError          bool           `yaml:"fail_fast,omitempty" json:"stop_on_error,omitempty"`
	EngineType           string         `yaml:"executor" json:"engine_type"`
	ModelID              string         `yaml:"model" json:"model_id"`
	SkillPaths           []string       `yaml:"skill_directories,omitempty" json:"skill_paths,omitempty"`
	InstructionFiles     []string       `yaml:"instruction_files,omitempty" json:"instruction_files,omitempty"`
	InjectSkillBody      *bool          `yaml:"inject_skill_body,omitempty" json:"inject_skill_body,omitempty"`
	DisabledSkills       []string       `yaml:"disabled_skills,omitempty" json:"disabled_skills,omitempty"`
	RequiredSkills       []string       `yaml:"required_skills,omitempty" json:"required_skills,omitempty"`
	ServerConfigs        map[string]any `yaml:"mcp_servers,omitempty" json:"server_configs,omitempty"`
	MaxAttempts          int            `yaml:"max_attempts,omitempty" json:"max_attempts,omitempty"`
	GroupBy              string         `yaml:"group_by,omitempty" json:"group_by,omitempty"`
	JudgeModel           string         `yaml:"judge_model,omitempty" json:"judge_model,omitempty"`
}

type DiffExpectedFileParameters struct {
	// Path is the workspace-relative path to the file being checked.
	Path string `yaml:"path" json:"path"`

	// Snapshot is the path (relative to context/fixtures dir) of the expected file content.
	// When set, the workspace file must match this snapshot exactly.
	Snapshot string `yaml:"snapshot,omitempty" json:"snapshot,omitempty"`

	// Contains lists line fragments that must appear in the workspace file.
	// Prefixed with "+" means the line must be present; "-" means it must be absent.
	Contains []string `yaml:"contains,omitempty" json:"contains,omitempty"`
}

type DiffGraderParameters struct {
	ExpectedFiles   []DiffExpectedFileParameters `yaml:"expected_files,omitempty" json:"expected_files,omitempty"`
	ContextDir      string                       `yaml:"context_dir,omitempty" json:"context_dir,omitempty"`
	UpdateSnapshots bool                         `yaml:"update_snapshots,omitempty" json:"update_snapshots,omitempty"`
}

type EvalSpec struct {
	SchemaVersion string `yaml:"schemaVersion,omitempty" json:"schemaVersion,omitempty"`
	SpecIdentity  `yaml:",inline"`
	SkillName     string             `yaml:"skill"`
	Version       string             `yaml:"version"`
	Config        Config             `yaml:"config"`
	Hooks         hooks.HooksConfig  `yaml:"hooks,omitempty"`
	MCPMocks      []MCPMockConfig    `yaml:"mcp_mocks,omitempty" json:"mcp_mocks,omitempty"`
	Adversarial   *AdversarialConfig `yaml:"adversarial,omitempty" json:"adversarial,omitempty"`
	Inputs        map[string]string  `yaml:"inputs,omitempty" json:"inputs,omitempty"`
	TasksFrom     string             `yaml:"tasks_from,omitempty" json:"tasks_from,omitempty"`
	Range         [2]int             `yaml:"range,omitempty" json:"range,omitempty"`
	Graders       []GraderConfig     `yaml:"graders"`
	Metrics       []MeasurementDef   `yaml:"metrics"`
	Tasks         []string           `yaml:"tasks"`
	Baseline      bool               `yaml:"baseline,omitempty" json:"baseline,omitempty"`
}

type EvaluationOutcome struct {
	SchemaVersion   string                   `json:"schemaVersion"`
	RunID           string                   `json:"eval_id"`
	SkillTested     string                   `json:"skill"`
	BenchName       string                   `json:"eval_name"`
	Timestamp       time.Time                `json:"timestamp"`
	Setup           OutcomeSetup             `json:"config"`
	Digest          OutcomeDigest            `json:"summary"`
	Measures        map[string]MeasureResult `json:"metrics"`
	TestOutcomes    []TestOutcome            `json:"tasks"`
	TriggerMetrics  *TriggerMetrics          `json:"trigger_metrics,omitempty"`
	TriggerResults  []TriggerResult          `json:"trigger_results,omitempty"`
	Metadata        map[string]any           `json:"metadata,omitempty"`
	IsBaseline      bool                     `json:"is_baseline,omitempty"`
	BaselineOutcome *EvaluationOutcome       `json:"baseline_outcome,omitempty"`
}

type FailureArtifacts struct {
	StdErr        string            `json:"stderr,omitempty"`
	StdOut        string            `json:"stdout,omitempty"`
	ExitCode      int               `json:"exit_code,omitempty"`
	FailedGraders []string          `json:"failed_graders,omitempty"`
	ErrorPatterns []string          `json:"error_patterns,omitempty"`
	TriageSummary string            `json:"triage_summary,omitempty"`
	CapturedAt    time.Time         `json:"captured_at"`
	Context       map[string]string `json:"context,omitempty"`
}

type FileContentPatternParameters struct {
	Path         string   `yaml:"path" json:"path"`
	MustMatch    []string `yaml:"must_match,omitempty" json:"must_match,omitempty"`
	MustNotMatch []string `yaml:"must_not_match,omitempty" json:"must_not_match,omitempty"`
}

type FileGraderParameters struct {
	MustExist       []string                       `yaml:"must_exist,omitempty" json:"must_exist,omitempty"`
	MustNotExist    []string                       `yaml:"must_not_exist,omitempty" json:"must_not_exist,omitempty"`
	ContentPatterns []FileContentPatternParameters `yaml:"content_patterns,omitempty" json:"content_patterns,omitempty"`
}

type GenericGraderParameters map[string]any

type GitResource struct {
	// Type selects the materialization strategy. Required. Only "worktree"
	// is currently supported.
	Type GitResourceType `yaml:"type" json:"type"`
	// Source is the local path to a git repository. Required for the
	// "worktree" strategy.
	Source string `yaml:"source" json:"source"`
	// Commit is an optional commit SHA, branch, or tag to check out.
	// Defaults to the source repo's HEAD when empty.
	Commit string `yaml:"commit,omitempty" json:"commit,omitempty"`
	// Dest is an optional subdirectory under the workspace to materialize
	// the repo into. When empty, the repo is materialized at the workspace
	// root. Must be a relative path that does not escape the workspace.
	Dest string `yaml:"dest,omitempty" json:"dest,omitempty"`
}

type GitResourceType string

type GradeOutcome struct {
	OverallScore   float64                 `json:"overall_score"`
	Passed         bool                    `json:"passed"`
	Tasks          map[string]GradeOutcome `json:"tasks,omitempty"`
	GraderAverages map[string]float64      `json:"grader_averages,omitempty"`
}

type GraderConfig struct {
	Kind       GraderKind       `yaml:"type" json:"kind"`
	Identifier string           `yaml:"name" json:"identifier"`
	ScriptPath string           `yaml:"script,omitempty" json:"script_path,omitempty"`
	Rubric     string           `yaml:"rubric,omitempty" json:"rubric,omitempty"`
	ModelID    string           `yaml:"model,omitempty" json:"model_id,omitempty"`
	Weight     float64          `yaml:"weight,omitempty" json:"weight,omitempty"`
	Parameters GraderParameters `yaml:"config,omitempty" json:"parameters,omitempty"`
}

type GraderKind string

type GraderParameters interface {
	// contains filtered or unexported methods
}

type GraderResults struct {
	Name       string         `json:"identifier"`
	Type       GraderKind     `json:"type"`
	Score      float64        `json:"score"`
	Weight     float64        `json:"weight"`
	Passed     bool           `json:"passed"`
	Feedback   string         `json:"feedback"`
	Details    map[string]any `json:"details,omitempty"`
	DurationMs int64          `json:"duration_ms"`
}

type GroupStats struct {
	Name     string  `json:"name"`
	Passed   int     `json:"passed"`
	Total    int     `json:"total"`
	AvgScore float64 `json:"avg_score"`
}

type InlineScriptGraderParameters struct {
	Assertions []string `yaml:"assertions,omitempty" json:"assertions,omitempty"`

	// Language indicates which language the Assertions are written for. Defaults to [LanguagePython]
	Language Language `yaml:"language,omitempty" json:"language,omitempty"`
}

type JSONSchemaGraderParameters struct {
	// Schema is an inline JSON schema object used for validation.
	Schema map[string]any `yaml:"schema,omitempty" json:"schema,omitempty"`

	// SchemaFile is a path to a JSON schema file. Used when Schema is not provided.
	SchemaFile string `yaml:"schema_file,omitempty" json:"schema_file,omitempty"`

	// ExtractJSON allows the grader to validate a single JSON document embedded
	// in prose or a Markdown JSON code block. The default requires pure JSON.
	ExtractJSON bool `yaml:"extract_json,omitempty" json:"extract_json,omitempty"`
}

type Language string

type MCPMockConfig struct {
	Name     string                 `yaml:"name" json:"name"`
	Fixtures string                 `yaml:"fixtures,omitempty" json:"fixtures,omitempty"`
	Tools    map[string]MCPMockTool `yaml:"tools,omitempty" json:"tools,omitempty"`
}

type MCPMockResponse struct {
	Match       map[string]any    `yaml:"match,omitempty" json:"match,omitempty"`
	MatchSchema map[string]any    `yaml:"match_schema,omitempty" json:"match_schema,omitempty"`
	MatchRegex  map[string]string `yaml:"match_regex,omitempty" json:"match_regex,omitempty"`
	Return      any               `yaml:"return,omitempty" json:"return,omitempty"`
	Error       string            `yaml:"error,omitempty" json:"error,omitempty"`
}

type MCPMockTool struct {
	Description string            `yaml:"description,omitempty" json:"description,omitempty"`
	InputSchema map[string]any    `yaml:"input_schema,omitempty" json:"input_schema,omitempty"`
	Responses   []MCPMockResponse `yaml:"responses,omitempty" json:"responses,omitempty"`
}

type MeasureResult struct {
	Identifier string         `json:"identifier"`
	Value      float64        `json:"value"`
	Threshold  float64        `json:"threshold"`
	Passed     bool           `json:"passed"`
	Weight     float64        `json:"weight"`
	Details    map[string]any `json:"details,omitempty"`
}

type MeasurementDef struct {
	Identifier string  `yaml:"name" json:"identifier"`
	Weight     float64 `yaml:"weight" json:"weight"`
	Threshold  float64 `yaml:"threshold" json:"threshold"`
	Enabled    bool    `yaml:"enabled,omitempty" json:"enabled,omitempty"`
	Desc       string  `yaml:"description,omitempty" json:"desc,omitempty"`
}

type ModelScore struct {
	ModelID        string             `json:"model_id"`
	HeuristicScore float64            `json:"heuristic_score"`
	Rank           int                `json:"rank"`
	Scores         map[string]float64 `json:"component_scores,omitempty"`
}

type ModelUsage struct {
	InputTokens      int     `json:"input_tokens"`
	OutputTokens     int     `json:"output_tokens"`
	CacheReadTokens  int     `json:"cache_read_tokens"`
	CacheWriteTokens int     `json:"cache_write_tokens"`
	RequestCount     float64 `json:"request_count"`
	RequestCost      float64 `json:"request_cost"`
}

type MultiSkillSummary struct {
	Timestamp time.Time      `json:"timestamp"`
	Skills    []SkillSummary `json:"skills"`
	Overall   OverallSummary `json:"overall"`
}

type OutcomeDigest struct {
	TotalTests     int          `json:"total_tests"`
	Succeeded      int          `json:"succeeded"`
	Failed         int          `json:"failed"`
	Errors         int          `json:"errors"`
	Skipped        int          `json:"skipped"`
	SuccessRate    float64      `json:"success_rate"`
	AggregateScore float64      `json:"aggregate_score"`
	WeightedScore  float64      `json:"weighted_score"`
	MinScore       float64      `json:"min_score"`
	MaxScore       float64      `json:"max_score"`
	StdDev         float64      `json:"std_dev"`
	DurationMs     int64        `json:"duration_ms"`
	Groups         []GroupStats `json:"groups,omitempty"`
	Usage          *UsageStats  `json:"usage,omitempty"`

	// Statistical summary populated when trials_per_task > 1
	Statistics *StatisticalSummary `json:"statistics,omitempty"`
}

type OutcomeSetup struct {
	RunsPerTest int    `json:"runs_per_test"`
	ModelID     string `json:"model_id"`
	EngineType  string `json:"engine_type"`
	TimeoutSec  int    `json:"timeout_sec"`
	JudgeModel  string `json:"judge_model,omitempty"`
}

type OutcomeSpec struct {
	Category  string `yaml:"type" json:"category"`
	Value     any    `yaml:"value,omitempty" json:"value,omitempty"`
	Predicate string `yaml:"condition,omitempty" json:"predicate,omitempty"`
}

type OverallSummary struct {
	TotalSkills       int     `json:"total_skills"`
	TotalModels       int     `json:"total_models"`
	AvgPassRate       float64 `json:"avg_pass_rate"`
	AvgAggregateScore float64 `json:"avg_aggregate_score"`
}

type PairwiseResult struct {
	Winner             string `json:"winner"`    // "baseline", "skill", or "tie"
	Magnitude          string `json:"magnitude"` // "much-better", "slightly-better", "equal", etc.
	Reasoning          string `json:"reasoning"`
	PositionConsistent bool   `json:"position_consistent"` // true if result held after position swap
}

type ProgramGraderParameters struct {
	Command string   `yaml:"command,omitempty" json:"command,omitempty"`
	Args    []string `yaml:"args,omitempty" json:"args,omitempty"`
	Timeout int      `yaml:"timeout,omitempty" json:"timeout,omitempty"`
}

type PromptGraderMode string

type PromptGraderParameters struct {
	Prompt          string           `yaml:"prompt,omitempty" json:"prompt,omitempty"`
	Model           string           `yaml:"model,omitempty" json:"model,omitempty"`
	ContinueSession bool             `yaml:"continue_session,omitempty" json:"continue_session,omitempty"`
	Mode            PromptGraderMode `yaml:"mode,omitempty" json:"mode,omitempty"`
	// Rubric is an optional reference to a reusable rubric. It can be either
	// the name of a built-in rubric ("groundedness") or a path to a local
	// rubric markdown file ("./rubrics/my-custom.md"). When set, the rubric's
	// body becomes the judge prompt unless an inline Prompt is also supplied.
	Rubric string `yaml:"rubric,omitempty" json:"rubric,omitempty"`
}

type Recommendation struct {
	RecommendedModel string                `json:"recommended_model"`
	HeuristicScore   float64               `json:"heuristic_score"`
	Reason           string                `json:"reason"`
	WinnerMarginPct  float64               `json:"winner_margin_pct"`
	Weights          RecommendationWeights `json:"weights"`
	ModelScores      []ModelScore          `json:"all_models"`
}

type RecommendationWeights struct {
	AggregateScore float64 `json:"aggregate_score"`
	PassRate       float64 `json:"pass_rate"`
	Consistency    float64 `json:"consistency"`
	Speed          float64 `json:"speed"`
}

type ResourceRef struct {
	Location string `yaml:"path,omitempty" json:"location,omitempty"`
	Body     string `yaml:"content,omitempty" json:"body,omitempty"`
}

type ResponderConfig struct {
	// Model is the model used for the responder LLM. Optional; when empty the
	// eval-level config.model is used.
	Model string `yaml:"model,omitempty" json:"model,omitempty"`
	// Instructions describe the target configuration the responder represents
	// and the rule for abstaining. Required.
	Instructions string `yaml:"instructions" json:"instructions"`
	// MaxFollowups caps how many times the responder may reply before the loop
	// stops. Required; must be >= 1.
	MaxFollowups int `yaml:"max_followups" json:"max_followups"`
}

type ResponderInfo struct {
	// FollowupsSent is the number of responder answers sent to the agent.
	FollowupsSent int `json:"followups_sent"`
	// Outcome is one of: stopped, abstained, cap_exhausted, error.
	Outcome string `json:"outcome"`
	// Reason holds the responder's reason when Outcome == "abstained" or an
	// error message when Outcome == "error".
	Reason string `json:"reason,omitempty"`
}

type RunResult struct {
	RunNumber int `json:"run_number"`
	Attempts  int `json:"attempts"`
	// Status contains the overall status of the run.
	// NOTE: if Status == [StatusError], then [ErrorMsg] will be set to the
	// message from the error.
	Status           Status                   `json:"status"`
	DurationMs       int64                    `json:"duration_ms"`
	Validations      map[string]GraderResults `json:"validations"`
	SessionDigest    SessionDigest            `json:"session_digest"`
	Transcript       []TranscriptEvent        `json:"transcript,omitempty"`
	FinalOutput      string                   `json:"final_output"`
	ErrorMsg         string                   `json:"error_msg,omitempty"`
	SkillInvocations []SkillInvocation        `json:"skill_invocations,omitempty"`
	Usage            *UsageStats              `json:"usage,omitempty"`
	WorkspaceDir     string                   `json:"workspace_dir,omitempty"`
	FailureArtifacts *FailureArtifacts        `json:"failure_artifacts,omitempty"`
	Responder        *ResponderInfo           `json:"responder,omitempty"`
	// Checkpoints captures per-turn checkpoint grader results, one entry per
	// configured TestCase.Checkpoint that actually ran (i.e., turn index was
	// reached). Empty / omitted when the task defines no checkpoints.
	Checkpoints []CheckpointOutcome `json:"checkpoints,omitempty"`

	// ToolEvents is a normalized, replay-friendly record of every tool call
	// made during the run. Added in schema version 1.1 as an additive field
	// (see issue #366). The legacy SessionDigest.ToolCalls is preserved for
	// backward compatibility; new consumers should prefer ToolEvents.
	ToolEvents []ToolEvent `json:"tool_events,omitempty"`

	// SnapshotPath is the on-disk path of the self-contained snapshot.json
	// artifact for this run, as returned by snapshot.Writer.Write. It is
	// rooted at the --snapshot directory the user passed to `waza run` (so
	// it may be absolute, or relative to the working directory at run time,
	// depending on what the user supplied). Consumers that need a
	// results.json-relative path should compute it themselves with
	// filepath.Rel against the results.json directory. Empty when
	// `waza run --snapshot` was not used. Added in schema version 1.2 as
	// an additive field (see issue #367).
	SnapshotPath string `json:"snapshot_path,omitempty"`
}

type SessionDigest struct {
	ToolCallCount int         `json:"tool_call_count"`
	ToolsUsed     []string    `json:"tools_used"`
	ToolCalls     []ToolCall  `json:"tool_calls,omitempty"`
	Errors        []string    `json:"errors"`
	Usage         *UsageStats `json:"usage,omitempty"`
	SessionID     string      `json:"session_id,omitempty"`
}

type SkillImpactMetric struct {
	PassRateWithSkills float64         `json:"pass_rate_with_skills"`
	PassRateBaseline   float64         `json:"pass_rate_baseline"`
	Delta              float64         `json:"delta"`
	PercentChange      float64         `json:"percent_change"`
	Pairwise           *PairwiseResult `json:"pairwise,omitempty"`
}

type SkillInvocation struct {
	Name string `json:"name"`
	Path string `json:"path,omitempty"`
}

type SkillInvocationGraderParameters struct {
	RequiredSkills  []string                    `yaml:"required_skills,omitempty" json:"required_skills,omitempty"`
	ForbiddenSkills []string                    `yaml:"forbidden_skills,omitempty" json:"forbidden_skills,omitempty"`
	Mode            SkillInvocationMatchingMode `yaml:"mode,omitempty" json:"mode,omitempty"`
	AllowExtra      *bool                       `yaml:"allow_extra,omitempty" json:"allow_extra,omitempty"`
}

type SkillInvocationMatchingMode string

type SkillSummary struct {
	SkillName      string   `json:"skill_name"`
	Models         []string `json:"models"`
	PassRate       float64  `json:"pass_rate"`
	AggregateScore float64  `json:"aggregate_score"`
	OutputFiles    []string `json:"output_files"`
}

type SpecIdentity struct {
	Name        string `yaml:"name" json:"name"`
	Description string `yaml:"description,omitempty" json:"description,omitempty"`
}

type StatisticalSummary struct {
	BootstrapCI    statistics.ConfidenceInterval `json:"bootstrap_ci"`
	IsSignificant  bool                          `json:"is_significant"`
	NormalizedGain *float64                      `json:"normalized_gain,omitempty"`
}

type Status string

type TaskExpectation struct {
	OutcomeSpecs    []OutcomeSpec  `yaml:"outcomes,omitempty" json:"outcome_specs,omitempty"`
	ToolPatterns    map[string]any `yaml:"tool_calls,omitempty" json:"tool_patterns,omitempty"`
	BehaviorRules   BehaviorRules  `yaml:"behavior,omitempty" json:"behavior_rules,omitempty"`
	MustInclude     []string       `yaml:"output_contains,omitempty" json:"must_include,omitempty"`
	MustExclude     []string       `yaml:"output_not_contains,omitempty" json:"must_exclude,omitempty"`
	MayInclude      []string       `yaml:"output_contains_any,omitempty" json:"may_include,omitempty"`
	ExpectedTrigger *bool          `yaml:"should_trigger,omitempty" json:"expected_trigger,omitempty"`
}

type TaskStimulus struct {
	Message     string            `yaml:"prompt" json:"message"`
	MessageFile string            `yaml:"prompt_file,omitempty" json:"message_file,omitempty"`
	Metadata    map[string]any    `yaml:"context,omitempty" json:"metadata,omitempty"`
	Resources   []ResourceRef     `yaml:"files,omitempty" json:"resources,omitempty"`
	Repos       []GitResource     `yaml:"repos,omitempty" json:"repos,omitempty"`
	WorkDir     string            `yaml:"workdir,omitempty" json:"workdir,omitempty"`
	Environment map[string]string `yaml:"environment,omitempty" json:"environment,omitempty"`
	FollowUps   []string          `yaml:"follow_up_prompts,omitempty" json:"follow_ups,omitempty"`
	Responder   *ResponderConfig  `yaml:"responder,omitempty" json:"responder,omitempty"`
}

type TaskTranscript struct {
	TaskID      string                   `json:"task_id"`
	TaskName    string                   `json:"task_name"`
	Status      Status                   `json:"status"`
	StartedAt   time.Time                `json:"started_at"`
	CompletedAt time.Time                `json:"completed_at"`
	DurationMs  int64                    `json:"duration_ms"`
	Prompt      string                   `json:"prompt"`
	FinalOutput string                   `json:"final_output"`
	Transcript  []TranscriptEvent        `json:"transcript"`
	Validations map[string]GraderResults `json:"validations,omitempty"`
	Session     SessionDigest            `json:"session"`
	ErrorMsg    string                   `json:"error_msg,omitempty"`
}

type TestCase struct {
	Active           *bool           `yaml:"enabled,omitempty" json:"active,omitempty"`
	ContextRoot      string          `yaml:"context_dir,omitempty" json:"context_root,omitempty"`
	DisplayName      string          `yaml:"name" json:"display_name"`
	Expectation      TaskExpectation `yaml:"expected,omitempty" json:"expectation,omitempty"`
	InstructionFiles []string        `yaml:"instruction_files,omitempty" json:"instruction_files,omitempty"`
	SkillPaths       []string        `yaml:"skill_directories,omitempty" json:"skill_paths,omitempty"`
	Stimulus         TaskStimulus    `yaml:"inputs" json:"stimulus"`
	Summary          string          `yaml:"description,omitempty" json:"summary,omitempty"`
	Tags             []string        `yaml:"tags,omitempty" json:"labels,omitempty"`
	TestID           string          `yaml:"id" json:"test_id"`
	// Golden marks a task as a "must always pass" regression gate. When set
	// to true, `waza gate` treats any non-pass status as a hard failure
	// (exit code 2) regardless of the aggregate pass-rate threshold.
	Golden     bool `yaml:"golden,omitempty" json:"golden,omitempty"`
	TimeoutSec *int `yaml:"timeout_seconds,omitempty" json:"timeout_sec,omitempty"`
	// FirstEventTimeoutSec overrides Config.FirstEventTimeoutSec for this task.
	// nil inherits the eval-level value; 0 disables the check for this task.
	FirstEventTimeoutSec *int              `yaml:"first_event_timeout_seconds,omitempty" json:"first_event_timeout_sec,omitempty"`
	Validators           []ValidatorInline `yaml:"graders,omitempty" json:"validators,omitempty"`
	// Checkpoints attach graders to intermediate turn boundaries in a
	// multi-turn run. Each checkpoint specifies `after_turn: N` (1-based,
	// where turn 1 is the initial prompt) and a list of graders that run
	// against the cumulative conversation state at the end of that turn.
	// Checkpoints are additive — task-level `graders:` still run against
	// the final state after all turns complete.
	Checkpoints []Checkpoint `yaml:"checkpoints,omitempty" json:"checkpoints,omitempty"`
}

type TestOutcome struct {
	TestID      string `json:"test_id"`
	DisplayName string `json:"display_name"`
	Group       string `json:"group,omitempty"`
	// Golden mirrors TestCase.Golden so `waza gate` can identify
	// must-pass tasks directly from results.json without re-reading YAML.
	Golden      bool               `json:"golden,omitempty"`
	Status      Status             `json:"status"`
	Runs        []RunResult        `json:"runs"`
	Stats       *TestStats         `json:"stats,omitempty"`
	SkillImpact *SkillImpactMetric `json:"skill_impact,omitempty"`
}

type TestStats struct {
	PassRate         float64 `json:"pass_rate"`
	FlakinessPercent float64 `json:"flakiness_percent"`
	PassedRuns       int     `json:"passed_runs"`
	FailedRuns       int     `json:"failed_runs"`
	ErrorRuns        int     `json:"error_runs"`
	TotalRuns        int     `json:"total_runs"`
	AvgScore         float64 `json:"avg_score"`
	AvgWeightedScore float64 `json:"avg_weighted_score"`
	MinScore         float64 `json:"min_score"`
	MaxScore         float64 `json:"max_score"`
	StdDevScore      float64 `json:"std_dev_score"`
	ScoreVariance    float64 `json:"score_variance"`
	CI95Lo           float64 `json:"ci95_lo"`
	CI95Hi           float64 `json:"ci95_hi"`
	Flaky            bool    `json:"flaky"`
	AvgDurationMs    int64   `json:"avg_duration_ms"`

	// Bootstrap confidence interval over weighted scores (populated when trials > 1)
	BootstrapCI   *statistics.ConfidenceInterval `json:"bootstrap_ci,omitempty"`
	IsSignificant *bool                          `json:"is_significant,omitempty"`
}

type TextGraderParameters struct {
	// Contains lists substrings that must appear in the output (case-insensitive).
	Contains []string `yaml:"contains,omitempty" json:"contains,omitempty"`

	// NotContains lists substrings that must NOT appear in the output (case-insensitive).
	NotContains []string `yaml:"not_contains,omitempty" json:"not_contains,omitempty"`

	// ContainsCS lists substrings that must appear in the output (case-sensitive).
	ContainsCS []string `yaml:"contains_cs,omitempty" json:"contains_cs,omitempty"`

	// NotContainsCS lists substrings that must NOT appear in the output (case-sensitive).
	NotContainsCS []string `yaml:"not_contains_cs,omitempty" json:"not_contains_cs,omitempty"`

	// RegexMatch lists regex patterns that must match somewhere in the output.
	RegexMatch []string `yaml:"regex_match,omitempty" json:"regex_match,omitempty"`

	// RegexNotMatch lists regex patterns that must NOT match anywhere in the output.
	RegexNotMatch []string `yaml:"regex_not_match,omitempty" json:"regex_not_match,omitempty"`
}

type ToolCall struct {
	// ID is the engine-assigned identifier for this call (e.g. the
	// Copilot SDK's ToolCallID). Used to correlate tool invocations in
	// observability backends.
	ID        string                               `json:"id,omitempty"`
	Name      string                               `json:"name"`
	Arguments ToolCallArgs                         `json:"arguments,omitempty"`
	Result    *copilot.ToolExecutionCompleteResult `json:"result,omitempty"`
	Success   bool                                 `json:"success"`
}

type ToolCallArgs struct {
	// these are filled out for file-based tools (view/edit)
	Path     string `json:"path"      mapstructure:"path"`
	FileText string `json:"file_text" mapstructure:"file_text"`

	// filled out for tools like bash or powershell
	Command     string `json:"command"     mapstructure:"command"`
	Description string `json:"description" mapstructure:"description"`

	// filled out for skill invocations
	Skill string `json:"skill" mapstructure:"skill"`

	// Extra captures any tool-specific argument keys that aren't part of
	// the fixed fields above (e.g. `query`/`limit` for search tools,
	// MCP-specific payloads). Populated automatically by mapstructure's
	// ",remain" support so argument matchers in tool_calls /
	// tool_constraint graders can see the full argument bag.
	//
	// Excluded from JSON marshaling: callers consuming arguments as a
	// generic map should use graders.normalizeToolCallArgs, which merges
	// Extra into the known fields. Persisting the raw bag in results.json
	// happens through RunResult.ToolEvents instead, which preserves the
	// engine's original argument value verbatim.
	Extra map[string]any `json:"-" mapstructure:",remain"`
}

type ToolCallsGraderParameters struct {
	RequiredTools  []string `yaml:"required_tools,omitempty" json:"required_tools,omitempty"`
	ForbiddenTools []string `yaml:"forbidden_tools,omitempty" json:"forbidden_tools,omitempty"`
	MinCalls       *int     `yaml:"min_calls,omitempty" json:"min_calls,omitempty"`
	MaxCalls       *int     `yaml:"max_calls,omitempty" json:"max_calls,omitempty"`

	// Expect is an ordered list of tool-call expectations. Each entry names
	// the expected tool and (optionally) structured matchers for argument
	// fields. An expectation passes when there is at least one actual tool
	// call whose name matches and whose arguments satisfy every matcher.
	//
	// Unlike RequiredTools (which only checks names), Expect lets a grader
	// assert input correctness — e.g. "list_dir was called with
	// path matching ^src/".
	Expect []ToolExpectation `yaml:"expect,omitempty" json:"expect,omitempty"`
}

type ToolConstraintGraderParameters struct {
	ExpectTools []ToolSpecParameters `yaml:"expect_tools,omitempty" json:"expect_tools,omitempty"`
	RejectTools []ToolSpecParameters `yaml:"reject_tools,omitempty" json:"reject_tools,omitempty"`
}

type ToolEvent struct {
	// Turn is the 1-based ordinal of the assistant turn that produced this
	// call. Multiple tool calls may share the same turn. 0 means unknown
	// (engines that do not surface turn boundaries).
	Turn int `json:"turn"`

	// Sequence is the 1-based ordinal of this call within the run, in the
	// order the engine emitted them. Stable across replays.
	Sequence int `json:"sequence"`

	// ToolCallID is the engine's unique identifier for this call. May be
	// empty for engines that do not assign IDs.
	ToolCallID string `json:"tool_call_id,omitempty"`

	// ToolName is the canonical name of the tool that was invoked.
	ToolName string `json:"tool_name"`

	// Args is the call's argument payload normalised to a JSON-compatible
	// value (object, array, scalar, or null). The exact shape depends on
	// the tool; argmatcher matchers in tool_calls / tool_constraint graders
	// run against this field.
	Args any `json:"args,omitempty"`

	// Result is the tool's response payload normalised to a JSON-compatible
	// value, or nil when the engine did not surface a result.
	Result any `json:"result,omitempty"`

	// Success reports whether the engine considered the call to have
	// completed without error.
	Success bool `json:"success"`

	// Error is the engine's failure message when Success is false. Empty
	// for successful calls.
	Error string `json:"error,omitempty"`

	// DurationMs is the wall-clock time between the tool-start and
	// tool-complete events when available. 0 when timing is unavailable.
	DurationMs int64 `json:"duration_ms,omitempty"`
}

type ToolExpectation struct {
	// Tool is the expected tool name (matched as a Go regexp against the
	// recorded tool name). A plain string is a substring match anchored by
	// the regexp engine, so use `^foo$` for exact match.
	Tool string `yaml:"tool" json:"tool"`

	// Args holds matcher specs for tool-call argument fields. All matchers
	// must pass for the expectation to be considered satisfied.
	Args map[string]argmatcher.Matcher `yaml:"args,omitempty" json:"args,omitempty"`
}

type ToolSpecParameters struct {
	Tool           string `yaml:"tool" json:"tool"`
	CommandPattern string `yaml:"command_pattern,omitempty" json:"command_pattern,omitempty"`
	SkillPattern   string `yaml:"skill_pattern,omitempty" json:"skill_pattern,omitempty"`
	PathPattern    string `yaml:"path_pattern,omitempty" json:"path_pattern,omitempty"`

	// Args constrains specific argument fields on the tool call using
	// structured matchers (equals / regex / contains / range / json_schema).
	// Keys are argument names as they appear on ToolCall.Arguments after JSON
	// normalisation; values are matcher specs. All matchers must pass for the
	// call to satisfy the spec.
	Args map[string]argmatcher.Matcher `yaml:"args,omitempty" json:"args,omitempty"`
}

type TranscriptEvent struct {
	copilot.SessionEvent `json:"-"`
}

type TriggerHeuristicGraderParameters struct {
	SkillPath string   `yaml:"skill_path" json:"skill_path"`
	Mode      string   `yaml:"mode" json:"mode"`
	Threshold *float64 `yaml:"threshold,omitempty" json:"threshold,omitempty"`
}

type TriggerMetrics struct {
	TP        int     `json:"true_positives"`
	FP        int     `json:"false_positives"`
	TN        int     `json:"true_negatives"`
	FN        int     `json:"false_negatives"`
	Errors    int     `json:"errors,omitempty"`
	Precision float64 `json:"precision"`
	Recall    float64 `json:"recall"`
	F1        float64 `json:"f1"`
	Accuracy  float64 `json:"accuracy"`
}

type TriggerResult struct {
	Prompt        string            `json:"prompt"`
	Confidence    string            `json:"confidence,omitempty"`
	ShouldTrigger bool              `json:"should_trigger"`
	DidTrigger    bool              `json:"did_trigger"`
	ErrorMsg      string            `json:"error_msg,omitempty"`
	FinalOutput   string            `json:"final_output,omitempty"`
	Transcript    []TranscriptEvent `json:"transcript,omitempty"`
	ToolCalls     []ToolCall        `json:"tool_calls,omitempty"`
	SessionID     string            `json:"session_id,omitempty"`
}

type UsageStats struct {
	Turns            int                   `json:"turns"`
	InputTokens      int                   `json:"input_tokens"`
	OutputTokens     int                   `json:"output_tokens"`
	CacheReadTokens  int                   `json:"cache_read_tokens"`
	CacheWriteTokens int                   `json:"cache_write_tokens"`
	PremiumRequests  float64               `json:"premium_requests"`
	Provider         string                `json:"provider,omitempty"`
	ProviderHost     string                `json:"provider_host,omitempty"`
	ModelMetrics     map[string]ModelUsage `json:"model_metrics,omitempty"`
}

type ValidatorInline struct {
	Identifier string           `yaml:"name" json:"identifier"`
	Kind       GraderKind       `yaml:"type,omitempty" json:"kind,omitempty"`
	Checks     []string         `yaml:"assertions,omitempty" json:"checks,omitempty"`
	Rubric     string           `yaml:"rubric,omitempty" json:"rubric,omitempty"`
	Weight     float64          `yaml:"weight,omitempty" json:"weight,omitempty"`
	Parameters GraderParameters `yaml:"config,omitempty" json:"parameters,omitempty"`
}