models

type AcquiredOfflineDocument struct {
	// contains filtered or unexported fields
}

type ActionSequenceGraderParameters struct {
	MatchingMode    ActionSequenceMatchingMode `yaml:"matching_mode,omitempty" json:"matching_mode,omitempty"`
	ExpectedActions []string                   `yaml:"expected_actions,omitempty" json:"expected_actions,omitempty"`
}

type ActionSequenceMatchingMode string

type AdversarialConfig struct {
	// Packs is the list of built-in pack identifiers to run (e.g.
	// "prompt-injection", "scope-bypass"). Unknown identifiers are rejected
	// by `waza adversarial`; the current schema does not support
	// out-of-tree custom packs. Required; must contain at least one entry.
	Packs []string `yaml:"packs" json:"packs"`
	// OnUnsafeOutcome controls whether unsafe outcomes fail the run
	// ("fail", default) or only warn ("warn"). When empty, "fail" is used.
	OnUnsafeOutcome AdversarialOnUnsafeOutcome `yaml:"on_unsafe_outcome,omitempty" json:"on_unsafe_outcome,omitempty"`
}

type AdversarialOnUnsafeOutcome string

type BehaviorGraderParameters struct {
	MaxToolCalls   int      `yaml:"max_tool_calls,omitempty" json:"max_tool_calls,omitempty"`
	MaxTokens      int      `yaml:"max_tokens,omitempty" json:"max_tokens,omitempty"`
	RequiredTools  []string `yaml:"required_tools,omitempty" json:"required_tools,omitempty"`
	ForbiddenTools []string `yaml:"forbidden_tools,omitempty" json:"forbidden_tools,omitempty"`
	MaxDurationMS  int64    `yaml:"max_duration_ms,omitempty" json:"max_duration_ms,omitempty"`
}

type BehaviorRules struct {
	MaxToolInvocations int      `yaml:"max_tool_calls,omitempty" json:"max_tool_invocations,omitempty"`
	MaxRounds          int      `yaml:"max_iterations,omitempty" json:"max_rounds,omitempty"`
	MaxTokens          int      `yaml:"max_tokens,omitempty" json:"max_tokens,omitempty"`
	MaxResponseTimeMs  int64    `yaml:"max_response_time_ms,omitempty" json:"max_response_time_ms,omitempty"`
	MustUseTool        []string `yaml:"required_tools,omitempty" json:"must_use_tool,omitempty"`
	ForbidTool         []string `yaml:"forbidden_tools,omitempty" json:"forbid_tool,omitempty"`
}

type CheckExplanation struct {
	Check       RequirementCheck    `json:"check"`
	Observation string              `json:"observation"`
	Category    string              `json:"category"`
	Message     string              `json:"message"`
	References  []EvidenceReference `json:"references,omitempty"`
}

type Checkpoint struct {
	// AfterTurn is the 1-based turn index at whose boundary the graders run.
	// Turn 1 is the initial prompt; turns 2..N are follow-ups or responder
	// replies. Must be >= 1.
	AfterTurn int `yaml:"after_turn" json:"after_turn"`
	// Graders is the slice of grader configurations to run at this turn
	// boundary. Required. Reuses the same ValidatorInline shape as task-level
	// `graders:` so any existing grader type is supported.
	Graders []ValidatorInline `yaml:"graders" json:"graders"`
	// OnFailure controls multi-turn behavior when any grader in this
	// checkpoint fails. "continue" (default) records the failure and keeps
	// going. "stop" short-circuits the multi-turn loop after the failure.
	OnFailure CheckpointOnFailure `yaml:"on_failure,omitempty" json:"on_failure,omitempty"`
}

type CheckpointOnFailure string

type CheckpointOutcome struct {
	// AfterTurn echoes Checkpoint.AfterTurn so consumers can sort/group.
	AfterTurn int `json:"after_turn"`
	// Status is StatusPassed when every grader in this checkpoint passed,
	// StatusFailed when at least one grader failed.
	Status Status `json:"status"`
	// Validations maps grader identifier to result, identical to
	// RunResult.Validations.
	Validations map[string]GraderResults `json:"validations"`
	// Stopped is true when this checkpoint had `on_failure: stop` and at
	// least one grader failed, terminating the multi-turn loop after this
	// turn.
	Stopped bool `json:"stopped,omitempty"`
}

type CommandInvocation struct {
	Command string   `json:"command"`
	Args    []string `json:"args,omitempty"`
	// A started attempt interrupted before completion has no observed exit code.
	ExitCode      *int `json:"exit_code,omitempty"`
	ResponseIndex int  `json:"response_index"`
}

type CommandMockConfig struct {
	Name        string                `yaml:"name" json:"name"`
	Responses   []CommandMockResponse `yaml:"responses" json:"responses"`
	ExpectCalls *int                  `yaml:"expect_calls,omitempty" json:"expect_calls,omitempty"`
}

type CommandMockResponse struct {
	Args        []string          `yaml:"args,omitempty" json:"args,omitempty"`
	ArgsRegex   []string          `yaml:"args_regex,omitempty" json:"args_regex,omitempty"`
	Environment map[string]string `yaml:"environment,omitempty" json:"environment,omitempty"`
	WorkDir     string            `yaml:"workdir,omitempty" json:"workdir,omitempty"`
	Stdout      any               `yaml:"stdout,omitempty" json:"stdout,omitempty"`
	Stderr      string            `yaml:"stderr,omitempty" json:"stderr,omitempty"`
	ExitCode    int               `yaml:"exit_code,omitempty" json:"exit_code,omitempty"`
	Fixture     string            `yaml:"fixture,omitempty" json:"fixture,omitempty"`
	Sequence    []CommandMockStep `yaml:"sequence,omitempty" json:"sequence,omitempty"`
}

type CommandMockStep struct {
	Stdout   any    `yaml:"stdout,omitempty" json:"stdout,omitempty"`
	Stderr   string `yaml:"stderr,omitempty" json:"stderr,omitempty"`
	ExitCode int    `yaml:"exit_code" json:"exit_code"`
	Fixture  string `yaml:"fixture,omitempty" json:"fixture,omitempty"`
	DelayMS  int64  `yaml:"delay_ms,omitempty" json:"delay_ms,omitempty"`
}

type Config struct {
	TrialsPerTask int `yaml:"trials_per_task" json:"runs_per_test"`
	TimeoutSec    int `yaml:"timeout_seconds" json:"timeout_sec"`
	// FirstEventTimeoutSec bounds how long to wait for the first session event
	// before treating a run as a session-start hang (the engine launched but
	// never started the agent's first turn). It is much shorter than
	// TimeoutSec, which must cover the slowest legitimate full turn. 0 (the
	// default) disables the check. Per-task overrides live on TestCase.
	FirstEventTimeoutSec int            `yaml:"first_event_timeout_seconds,omitempty" json:"first_event_timeout_sec,omitempty"`
	Concurrent           bool           `yaml:"parallel" json:"concurrent"`
	Workers              int            `yaml:"workers,omitempty" json:"workers,omitempty"`
	StopOnError          bool           `yaml:"fail_fast,omitempty" json:"stop_on_error,omitempty"`
	EngineType           string         `yaml:"executor" json:"engine_type"`
	ModelID              string         `yaml:"model" json:"model_id"`
	ReasoningEffort      string         `yaml:"reasoning_effort,omitempty" json:"reasoning_effort,omitempty"`
	SkillPaths           []string       `yaml:"skill_directories,omitempty" json:"skill_paths,omitempty"`
	InstructionFiles     []string       `yaml:"instruction_files,omitempty" json:"instruction_files,omitempty"`
	InjectSkillBody      *bool          `yaml:"inject_skill_body,omitempty" json:"inject_skill_body,omitempty"`
	TriggerSkillRouting  bool           `yaml:"trigger_skill_routing,omitempty" json:"trigger_skill_routing,omitempty"`
	DisabledSkills       []string       `yaml:"disabled_skills,omitempty" json:"disabled_skills,omitempty"`
	RequiredSkills       []string       `yaml:"required_skills,omitempty" json:"required_skills,omitempty"`
	ServerConfigs        map[string]any `yaml:"mcp_servers,omitempty" json:"server_configs,omitempty"`
	MaxAttempts          int            `yaml:"max_attempts,omitempty" json:"max_attempts,omitempty"`
	GroupBy              string         `yaml:"group_by,omitempty" json:"group_by,omitempty"`
	JudgeModel           string         `yaml:"judge_model,omitempty" json:"judge_model,omitempty"`
	JudgeReasoningEffort string         `yaml:"judge_reasoning_effort,omitempty" json:"judge_reasoning_effort,omitempty"`
}

type DependencyMode struct {
	Kind string `json:"kind"`
	Name string `json:"name"`
	Mode string `json:"mode"`
}

type DiffExpectedFileParameters struct {
	// Path is the workspace-relative path to the file being checked.
	Path string `yaml:"path" json:"path"`

	// Snapshot is the path (relative to context/fixtures dir) of the expected file content.
	// When set, the workspace file must match this snapshot exactly.
	Snapshot string `yaml:"snapshot,omitempty" json:"snapshot,omitempty"`

	// Contains lists line fragments that must appear in the workspace file.
	// Prefixed with "+" means the line must be present; "-" means it must be absent.
	Contains []string `yaml:"contains,omitempty" json:"contains,omitempty"`
}

type DiffGraderParameters struct {
	ExpectedFiles   []DiffExpectedFileParameters `yaml:"expected_files,omitempty" json:"expected_files,omitempty"`
	ContextDir      string                       `yaml:"context_dir,omitempty" json:"context_dir,omitempty"`
	UpdateSnapshots bool                         `yaml:"update_snapshots,omitempty" json:"update_snapshots,omitempty"`
}

type EvalSpec struct {
	SchemaVersion string `yaml:"schemaVersion,omitempty" json:"schemaVersion,omitempty"`
	SpecIdentity  `yaml:",inline"`
	Scenario      string              `yaml:"scenario,omitempty" json:"scenario,omitempty"`
	SkillName     string              `yaml:"skill"`
	Version       string              `yaml:"version"`
	Config        Config              `yaml:"config"`
	Hooks         hooks.HooksConfig   `yaml:"hooks,omitempty"`
	MCPMocks      []MCPMockConfig     `yaml:"mcp_mocks,omitempty" json:"mcp_mocks,omitempty"`
	CommandMocks  []CommandMockConfig `yaml:"command_mocks,omitempty" json:"command_mocks,omitempty"`
	Adversarial   *AdversarialConfig  `yaml:"adversarial,omitempty" json:"adversarial,omitempty"`
	Inputs        map[string]string   `yaml:"inputs,omitempty" json:"inputs,omitempty"`
	TasksFrom     string              `yaml:"tasks_from,omitempty" json:"tasks_from,omitempty"`
	Range         [2]int              `yaml:"range,omitempty" json:"range,omitempty"`
	Graders       []GraderConfig      `yaml:"graders"`
	Metrics       []MeasurementDef    `yaml:"metrics"`
	Tasks         []string            `yaml:"tasks"`
	Baseline      bool                `yaml:"baseline,omitempty" json:"baseline,omitempty"`
	// contains filtered or unexported fields
}

type EvaluationOutcome struct {
	SchemaVersion   string                   `json:"schemaVersion"`
	RunID           string                   `json:"eval_id"`
	SkillTested     string                   `json:"skill"`
	BenchName       string                   `json:"eval_name"`
	Timestamp       time.Time                `json:"timestamp"`
	Setup           OutcomeSetup             `json:"config"`
	Digest          OutcomeDigest            `json:"summary"`
	Measures        map[string]MeasureResult `json:"metrics"`
	TestOutcomes    []TestOutcome            `json:"tasks"`
	TriggerMetrics  *TriggerMetrics          `json:"trigger_metrics,omitempty"`
	TriggerResults  []TriggerResult          `json:"trigger_results,omitempty"`
	Metadata        map[string]any           `json:"metadata,omitempty"`
	IsBaseline      bool                     `json:"is_baseline,omitempty"`
	BaselineOutcome *EvaluationOutcome       `json:"baseline_outcome,omitempty"`
	// EvaluationUsage is present for current evaluations, including cache-only
	// runs. Its sessions, not cached task diagnostics, define billing scope.
	EvaluationUsage *EvaluationUsage `json:"evaluation_usage,omitempty"`
	RegradeUsage    *EvaluationUsage `json:"regrade_usage,omitempty"`
	// contains filtered or unexported fields
}

type EvaluationSessionUsage struct {
	SessionID       string      `json:"session_id,omitempty"`
	Usage           *UsageStats `json:"usage,omitempty"`
	InitialUsage    *UsageStats `json:"initial_usage,omitempty"`
	UnknownBaseline bool        `json:"unknown_baseline,omitempty"`
	Cumulative      bool        `json:"cumulative,omitempty"`
	UsageRevision   uint64      `json:"usage_revision,omitempty"`
}

type EvaluationUsage struct {
	Sessions []EvaluationSessionUsage `json:"sessions"`
}

type EvidenceArtifact struct {
	ID            string          `json:"id"`
	Kind          string          `json:"kind"`
	Availability  string          `json:"availability"`
	Completeness  string          `json:"completeness"`
	Document      string          `json:"document,omitempty"`
	Pointer       string          `json:"pointer,omitempty"`
	ContentDigest *EvidenceDigest `json:"content_digest,omitempty"`
	SourceDigest  *EvidenceDigest `json:"source_digest,omitempty"`
	Reason        string          `json:"reason,omitempty"`
	Redacted      bool            `json:"redacted,omitempty"`
}

type EvidenceDiagnostic struct {
	Category string `json:"category"`
	Code     string `json:"code"`
	Message  string `json:"message"`
}

type EvidenceDigest struct {
	SHA256   string `json:"sha256"`
	Encoding string `json:"encoding"`
}

type EvidenceManifest struct {
	Version              string               `json:"version"`
	SHA256               string               `json:"sha256"`
	SourceManifestSHA256 string               `json:"source_manifest_sha256,omitempty"`
	Origin               EvidenceOrigin       `json:"origin"`
	Runtime              EvidenceRuntime      `json:"runtime"`
	Redaction            EvidenceRedaction    `json:"redaction"`
	Artifacts            []EvidenceArtifact   `json:"artifacts"`
	Diagnostics          []EvidenceDiagnostic `json:"diagnostics,omitempty"`
	// contains filtered or unexported fields
}

type EvidenceOrigin struct {
	EvalID        string `json:"eval_id"`
	TaskID        string `json:"task_id"`
	RunNumber     int    `json:"run_number"`
	AttemptCount  int    `json:"attempt_count"`
	PriorAttempts string `json:"prior_attempts"`
}

type EvidenceRedaction struct {
	Policy       string   `json:"policy"`
	AppliedRules []string `json:"applied_rules,omitempty"`
	MatchCount   int      `json:"match_count"`
	Limitations  []string `json:"limitations"`
}

type EvidenceReference struct {
	Origin     EvidenceOrigin `json:"origin"`
	ArtifactID string         `json:"artifact_id"`
	Pointer    string         `json:"pointer,omitempty"`
}

type EvidenceRuntime struct {
	WazaVersion           string  `json:"waza_version"`
	RequestedEngine       string  `json:"requested_engine"`
	RequestedModel        string  `json:"requested_model"`
	ExecutionMode         string  `json:"execution_mode"`
	DependencyMode        string  `json:"dependency_mode"`
	RequestedPolicy       string  `json:"requested_policy"`
	VerifiedEnforcement   string  `json:"verified_enforcement"`
	GoVersion             string  `json:"go_version"`
	Platform              string  `json:"platform"`
	SDKVersion            *string `json:"sdk_version"`
	EffectiveModelVersion *string `json:"effective_model_version"`
	NoSkills              *bool   `json:"no_skills"`
	NativeSkillControl    string  `json:"native_skill_control"`
}

type FailureArtifacts struct {
	StdErr        string            `json:"stderr,omitempty"`
	StdOut        string            `json:"stdout,omitempty"`
	ExitCode      int               `json:"exit_code,omitempty"`
	FailedGraders []string          `json:"failed_graders,omitempty"`
	ErrorPatterns []string          `json:"error_patterns,omitempty"`
	TriageSummary string            `json:"triage_summary,omitempty"`
	CapturedAt    time.Time         `json:"captured_at"`
	Context       map[string]string `json:"context,omitempty"`
}

type FaultSourceFormat string

type FaultSourceObject map[string]json.RawMessage

type FileContentPatternParameters struct {
	Path         string   `yaml:"path" json:"path"`
	MustMatch    []string `yaml:"must_match,omitempty" json:"must_match,omitempty"`
	MustNotMatch []string `yaml:"must_not_match,omitempty" json:"must_not_match,omitempty"`
}

type FileGraderParameters struct {
	MustExist       []string                       `yaml:"must_exist,omitempty" json:"must_exist,omitempty"`
	MustNotExist    []string                       `yaml:"must_not_exist,omitempty" json:"must_not_exist,omitempty"`
	ContentPatterns []FileContentPatternParameters `yaml:"content_patterns,omitempty" json:"content_patterns,omitempty"`
}

type GenericGraderParameters map[string]any

type GitResource struct {
	// Type selects the materialization strategy. Required. Only "worktree"
	// is currently supported.
	Type GitResourceType `yaml:"type" json:"type"`
	// Source is the local path to a git repository. Required for the
	// "worktree" strategy.
	Source string `yaml:"source" json:"source"`
	// Commit is an optional commit SHA, branch, or tag to check out.
	// Defaults to the source repo's HEAD when empty.
	Commit string `yaml:"commit,omitempty" json:"commit,omitempty"`
	// Dest is an optional subdirectory under the workspace to materialize
	// the repo into. When empty, the repo is materialized at the workspace
	// root. Must be a relative path that does not escape the workspace.
	Dest string `yaml:"dest,omitempty" json:"dest,omitempty"`
}

type GitResourceType string

type GradeOutcome struct {
	OverallScore   float64                 `json:"overall_score"`
	Passed         bool                    `json:"passed"`
	Tasks          map[string]GradeOutcome `json:"tasks,omitempty"`
	GraderAverages map[string]float64      `json:"grader_averages,omitempty"`
	Status         Status                  `json:"status,omitempty"`
	Error          string                  `json:"error,omitempty"`
}

type GraderConfig struct {
	Ref        string           `yaml:"ref,omitempty" json:"ref,omitempty"`
	Kind       GraderKind       `yaml:"type" json:"kind"`
	Identifier string           `yaml:"name" json:"identifier"`
	ScriptPath string           `yaml:"script,omitempty" json:"script_path,omitempty"`
	Rubric     string           `yaml:"rubric,omitempty" json:"rubric,omitempty"`
	ModelID    string           `yaml:"model,omitempty" json:"model_id,omitempty"`
	Weight     float64          `yaml:"weight,omitempty" json:"weight,omitempty"`
	Parameters GraderParameters `yaml:"config,omitempty" json:"parameters,omitempty"`
}

type GraderKind string

type GraderParameters interface {
	// contains filtered or unexported methods
}

type GraderResults struct {
	Name       string         `json:"identifier"`
	Type       GraderKind     `json:"type"`
	Score      float64        `json:"score"`
	Weight     float64        `json:"weight"`
	Passed     bool           `json:"passed"`
	Feedback   string         `json:"feedback"`
	Details    map[string]any `json:"details,omitempty"`
	DurationMs int64          `json:"duration_ms"`
}

type GroupStats struct {
	Name     string  `json:"name"`
	Passed   int     `json:"passed"`
	Total    int     `json:"total"`
	AvgScore float64 `json:"avg_score"`
}

type InlineScriptGraderParameters struct {
	Assertions []string `yaml:"assertions,omitempty" json:"assertions,omitempty"`

	// Language indicates which language the Assertions are written for. Defaults to [LanguagePython]
	Language Language `yaml:"language,omitempty" json:"language,omitempty"`
}

type JSONSchemaGraderParameters struct {
	// Schema is an inline JSON schema object used for validation.
	Schema map[string]any `yaml:"schema,omitempty" json:"schema,omitempty"`

	// SchemaFile is a path to a JSON schema file. Used when Schema is not provided.
	SchemaFile string `yaml:"schema_file,omitempty" json:"schema_file,omitempty"`

	// ExtractJSON allows the grader to validate a single JSON document embedded
	// in prose or a Markdown JSON code block. The default requires pure JSON.
	ExtractJSON bool `yaml:"extract_json,omitempty" json:"extract_json,omitempty"`
}

type JudgeEvidence struct {
	Version       string                                       `json:"version"`
	SHA256        string                                       `json:"sha256"`
	Origin        EvidenceOrigin                               `json:"origin"`
	Boundary      string                                       `json:"boundary"`
	ExecutionMode string                                       `json:"execution_mode"`
	Task          JudgeEvidenceSection[string]                 `json:"task"`
	Output        JudgeEvidenceSection[string]                 `json:"output"`
	Messages      JudgeEvidenceSection[[]JudgeEvidenceMessage] `json:"messages"`
	ToolEvents    JudgeEvidenceSection[[]ToolEvent]            `json:"tool_events"`
	Files         JudgeEvidenceSection[[]JudgeEvidenceFile]    `json:"files"`
	Selection     PromptEvidenceContext                        `json:"selection"`
	Redaction     EvidenceRedaction                            `json:"redaction"`
	Limits        JudgeEvidenceLimits                          `json:"limits"`
	Diagnostics   []EvidenceDiagnostic                         `json:"diagnostics,omitempty"`
}

type JudgeEvidenceFile struct {
	Path    string `json:"path"`
	Content string `json:"content"`
	SHA256  string `json:"sha256"`
}

type JudgeEvidenceLimits struct {
	MaxFiles       int `json:"max_files"`
	MaxFileBytes   int `json:"max_file_bytes"`
	MaxBundleBytes int `json:"max_bundle_bytes"`
}

type JudgeEvidenceMessage struct {
	Role    string `json:"role"`
	Content string `json:"content"`
}

type JudgeEvidenceSection[T any] struct {
	Availability string `json:"availability"`
	Completeness string `json:"completeness"`
	Items        *T     `json:"items"`
	Reason       string `json:"reason,omitempty"`
}

type JudgeFileCapture struct {
	Files       []JudgeEvidenceFile
	Diagnostics []EvidenceDiagnostic
	Redaction   EvidenceRedaction
}

type Language string

type Lockfile struct {
	SchemaVersion int              `yaml:"schema_version"`
	Graders       []LockfileGrader `yaml:"graders"`
	// contains filtered or unexported fields
}

type LockfileGrader struct {
	Ref     string `yaml:"ref"`
	Commit  string `yaml:"commit"`
	Digest  string `yaml:"digest"`
	URL     string `yaml:"url"`
	Trusted bool   `yaml:"trusted,omitempty"`
}

type MCPMockConfig struct {
	Name     string                 `yaml:"name" json:"name"`
	Fixtures string                 `yaml:"fixtures,omitempty" json:"fixtures,omitempty"`
	Tools    map[string]MCPMockTool `yaml:"tools,omitempty" json:"tools,omitempty"`
}

type MCPMockResponse struct {
	Match       map[string]any    `yaml:"match,omitempty" json:"match,omitempty"`
	MatchSchema map[string]any    `yaml:"match_schema,omitempty" json:"match_schema,omitempty"`
	MatchRegex  map[string]string `yaml:"match_regex,omitempty" json:"match_regex,omitempty"`
	Return      any               `yaml:"return,omitempty" json:"return,omitempty"`
	Error       string            `yaml:"error,omitempty" json:"error,omitempty"`
	Sequence    []MCPMockStep     `yaml:"sequence,omitempty" json:"sequence,omitempty"`
}

type MCPMockStep struct {
	Return  any    `yaml:"return,omitempty" json:"return,omitempty"`
	Error   string `yaml:"error,omitempty" json:"error,omitempty"`
	DelayMS int64  `yaml:"delay_ms,omitempty" json:"delay_ms,omitempty"`
}

type MCPMockTool struct {
	Description string            `yaml:"description,omitempty" json:"description,omitempty"`
	InputSchema map[string]any    `yaml:"input_schema,omitempty" json:"input_schema,omitempty"`
	Responses   []MCPMockResponse `yaml:"responses,omitempty" json:"responses,omitempty"`
}

type MeasureResult struct {
	Identifier string         `json:"identifier"`
	Value      float64        `json:"value"`
	Threshold  float64        `json:"threshold"`
	Passed     bool           `json:"passed"`
	Weight     float64        `json:"weight"`
	Details    map[string]any `json:"details,omitempty"`
}

type MeasurementDef struct {
	Identifier string  `yaml:"name" json:"identifier"`
	Weight     float64 `yaml:"weight" json:"weight"`
	Threshold  float64 `yaml:"threshold" json:"threshold"`
	Enabled    bool    `yaml:"enabled,omitempty" json:"enabled,omitempty"`
	Desc       string  `yaml:"description,omitempty" json:"desc,omitempty"`
}

type MockResponseSource struct {
	Kind    string
	Command string
	Server  string
	Tool    string
	Index   int
	Data    []byte
	Finite  bool
}

type ModelScore struct {
	ModelID        string             `json:"model_id"`
	HeuristicScore float64            `json:"heuristic_score"`
	Rank           int                `json:"rank"`
	Scores         map[string]float64 `json:"component_scores,omitempty"`
}

type ModelUsage struct {
	InputTokens      int     `json:"input_tokens"`
	OutputTokens     int     `json:"output_tokens"`
	CacheReadTokens  int     `json:"cache_read_tokens"`
	CacheWriteTokens int     `json:"cache_write_tokens"`
	RequestCount     float64 `json:"request_count"`
	RequestCost      float64 `json:"request_cost"`
	// AICredits is the final AI-credit total the Copilot SDK attributed to this
	// model, or nil when the SDK did not report one.
	AICredits *float64 `json:"ai_credits,omitempty"`
}

type MultiSkillSummary struct {
	Timestamp time.Time      `json:"timestamp"`
	Skills    []SkillSummary `json:"skills"`
	Overall   OverallSummary `json:"overall"`
}

type OutcomeDigest struct {
	TotalTests     int          `json:"total_tests"`
	Succeeded      int          `json:"succeeded"`
	Failed         int          `json:"failed"`
	Errors         int          `json:"errors"`
	Skipped        int          `json:"skipped"`
	SuccessRate    float64      `json:"success_rate"`
	AggregateScore float64      `json:"aggregate_score"`
	WeightedScore  float64      `json:"weighted_score"`
	MinScore       float64      `json:"min_score"`
	MaxScore       float64      `json:"max_score"`
	StdDev         float64      `json:"std_dev"`
	DurationMs     int64        `json:"duration_ms"`
	Groups         []GroupStats `json:"groups,omitempty"`
	Usage          *UsageStats  `json:"usage,omitempty"`

	// Statistical summary populated when trials_per_task > 1
	Statistics *StatisticalSummary `json:"statistics,omitempty"`
	// contains filtered or unexported fields
}

type OutcomeSetup struct {
	RunsPerTest          int    `json:"runs_per_test"`
	ModelID              string `json:"model_id"`
	EngineType           string `json:"engine_type"`
	TimeoutSec           int    `json:"timeout_sec"`
	JudgeModel           string `json:"judge_model,omitempty"`
	ReasoningEffort      string `json:"reasoning_effort,omitempty"`
	JudgeReasoningEffort string `json:"judge_reasoning_effort,omitempty"`
}

type OutcomeSpec struct {
	Category  string `yaml:"type" json:"category"`
	Value     any    `yaml:"value,omitempty" json:"value,omitempty"`
	Predicate string `yaml:"condition,omitempty" json:"predicate,omitempty"`
}

type OverallSummary struct {
	TotalSkills       int     `json:"total_skills"`
	TotalModels       int     `json:"total_models"`
	AvgPassRate       float64 `json:"avg_pass_rate"`
	AvgAggregateScore float64 `json:"avg_aggregate_score"`
}

type PairwiseResult struct {
	Winner             string `json:"winner"`    // "baseline", "skill", or "tie"
	Magnitude          string `json:"magnitude"` // "much-better", "slightly-better", "equal", etc.
	Reasoning          string `json:"reasoning"`
	PositionConsistent bool   `json:"position_consistent"` // true if result held after position swap
}

type ProgramGraderParameters struct {
	Command string   `yaml:"command,omitempty" json:"command,omitempty"`
	Args    []string `yaml:"args,omitempty" json:"args,omitempty"`
	Timeout int      `yaml:"timeout,omitempty" json:"timeout,omitempty"`
}

type PromptEvidenceContext struct {
	Files         []string `yaml:"files,omitempty" json:"files,omitempty"`
	EvaluatorOnly []string `yaml:"evaluator_only,omitempty" json:"evaluator_only,omitempty"`
}

type PromptGraderMode string

type PromptGraderParameters struct {
	Prompt          string                 `yaml:"prompt,omitempty" json:"prompt,omitempty"`
	Model           string                 `yaml:"model,omitempty" json:"model,omitempty"`
	ReasoningEffort string                 `yaml:"reasoning_effort,omitempty" json:"reasoning_effort,omitempty"`
	ContinueSession bool                   `yaml:"continue_session,omitempty" json:"continue_session,omitempty"`
	Mode            PromptGraderMode       `yaml:"mode,omitempty" json:"mode,omitempty"`
	EvidenceContext *PromptEvidenceContext `yaml:"evidence_context,omitempty" json:"evidence_context,omitempty"`
	// Rubric is an optional reference to a reusable rubric. It can be either
	// the name of a built-in rubric ("groundedness") or a path to a local
	// rubric markdown file ("./rubrics/my-custom.md"). When set, the rubric's
	// body becomes the judge prompt unless an inline Prompt is also supplied.
	Rubric string `yaml:"rubric,omitempty" json:"rubric,omitempty"`
}

type PromptJudgeObservation struct {
	Grader             string                `json:"grader"`
	BundleSHA256       string                `json:"bundle_sha256,omitempty"`
	DeliverySHA256     string                `json:"delivery_sha256,omitempty"`
	Context            string                `json:"context"`
	State              string                `json:"state"`
	SubmissionObserved bool                  `json:"submission_observed"`
	Inspection         string                `json:"inspection"`
	EvidenceContext    PromptEvidenceContext `json:"evidence_context"`
	Purpose            string                `json:"purpose"`
	SessionID          string                `json:"session_id,omitempty"`
	Model              string                `json:"model,omitempty"`
	Usage              *UsageStats           `json:"usage,omitempty"`
	DurationMs         int64                 `json:"duration_ms"`
	Diagnostics        []EvidenceDiagnostic  `json:"diagnostics,omitempty"`
}

type Recommendation struct {
	RecommendedModel string                `json:"recommended_model"`
	HeuristicScore   float64               `json:"heuristic_score"`
	Reason           string                `json:"reason"`
	WinnerMarginPct  float64               `json:"winner_margin_pct"`
	Weights          RecommendationWeights `json:"weights"`
	ModelScores      []ModelScore          `json:"all_models"`
}

type RecommendationWeights struct {
	AggregateScore float64 `json:"aggregate_score"`
	PassRate       float64 `json:"pass_rate"`
	Consistency    float64 `json:"consistency"`
	Speed          float64 `json:"speed"`
}

type Requirement struct {
	ID          string             `yaml:"id" json:"id"`
	Category    string             `yaml:"category" json:"category"`
	Description string             `yaml:"description" json:"description"`
	Checks      []RequirementCheck `yaml:"checks,omitempty" json:"checks,omitempty"`
}

type RequirementCheck struct {
	Scope     string `yaml:"scope" json:"scope"`
	Grader    string `yaml:"grader" json:"grader"`
	AfterTurn int    `yaml:"after_turn,omitempty" json:"after_turn,omitempty"`
}

type RequirementExplanation struct {
	TaskID        string             `json:"task_id"`
	RequirementID string             `json:"requirement_id"`
	Checks        []CheckExplanation `json:"checks"`
}

type ResourceRef struct {
	Location string `yaml:"path,omitempty" json:"location,omitempty"`
	Body     string `yaml:"content,omitempty" json:"body,omitempty"`
}

type ResponderConfig struct {
	// Model is the model used for the responder LLM. Optional; when empty the
	// eval-level config.model is used.
	Model string `yaml:"model,omitempty" json:"model,omitempty"`
	// Instructions describe the target configuration the responder represents
	// and the rule for abstaining. Required.
	Instructions string `yaml:"instructions" json:"instructions"`
	// MaxFollowups caps how many times the responder may reply before the loop
	// stops. Required; must be >= 1.
	MaxFollowups int `yaml:"max_followups" json:"max_followups"`
}

type ResponderInfo struct {
	// FollowupsSent is the number of responder answers sent to the agent.
	FollowupsSent int `json:"followups_sent"`
	// Outcome is one of: stopped, abstained, cap_exhausted, error.
	Outcome string `json:"outcome"`
	// Reason holds the responder's reason when Outcome == "abstained" or an
	// error message when Outcome == "error".
	Reason string `json:"reason,omitempty"`
}

type RunResult struct {
	JudgeEvidence *JudgeEvidence           `json:"judge_evidence,omitempty"`
	PromptJudging []PromptJudgeObservation `json:"prompt_judging,omitempty"`
	RunNumber     int                      `json:"run_number"`
	Attempts      int                      `json:"attempts"`
	// Prompt is the resolved initial task prompt for this run. It may be
	// recorded before execution starts when request setup fails.
	Prompt string `json:"prompt,omitempty"`
	// Status contains the overall status of the run.
	// NOTE: if Status == [StatusError], then [ErrorMsg] will be set to the
	// message from the error.
	Status           Status                   `json:"status"`
	DurationMs       int64                    `json:"duration_ms"`
	Validations      map[string]GraderResults `json:"validations"`
	SessionDigest    SessionDigest            `json:"session_digest"`
	GraderSessions   []SessionDigest          `json:"grader_sessions,omitempty"`
	Transcript       []TranscriptEvent        `json:"transcript,omitempty"`
	FinalOutput      string                   `json:"final_output"`
	ErrorMsg         string                   `json:"error_msg,omitempty"`
	SkillInvocations []SkillInvocation        `json:"skill_invocations,omitempty"`
	Usage            *UsageStats              `json:"usage,omitempty"`
	WorkspaceDir     string                   `json:"workspace_dir,omitempty"`
	FailureArtifacts *FailureArtifacts        `json:"failure_artifacts,omitempty"`
	Responder        *ResponderInfo           `json:"responder,omitempty"`
	// Checkpoints captures per-turn checkpoint grader results, one entry per
	// configured TestCase.Checkpoint that actually ran (i.e., turn index was
	// reached). Empty / omitted when the task defines no checkpoints.
	Checkpoints []CheckpointOutcome `json:"checkpoints,omitempty"`

	// ToolEvents is a normalized, replay-friendly record of every tool call
	// made during the run. Added in schema version 1.1 as an additive field
	// (see issue #366). The legacy SessionDigest.ToolCalls is preserved for
	// backward compatibility; new consumers should prefer ToolEvents.
	ToolEvents []ToolEvent `json:"tool_events,omitempty"`

	// CommandInvocations records sanitized calls handled by declarative
	// command_mocks. Response output is intentionally excluded.
	CommandInvocations []CommandInvocation `json:"command_invocations,omitempty"`
	// DependencyModes records the configured mode for declared dependencies.
	// It describes configuration, not proof that a dependency was invoked.
	DependencyModes []DependencyMode `json:"dependency_modes,omitempty"`

	// SnapshotPath is the on-disk path of the self-contained snapshot.json
	// artifact for this run, as returned by snapshot.Writer.Write. It is
	// rooted at the --snapshot directory the user passed to `waza run` (so
	// it may be absolute, or relative to the working directory at run time,
	// depending on what the user supplied). Consumers that need a
	// results.json-relative path should compute it themselves with
	// filepath.Rel against the results.json directory. Empty when
	// `waza run --snapshot` was not used. Added in schema version 1.2 as
	// an additive field (see issue #367).
	SnapshotPath            string                   `json:"snapshot_path,omitempty"`
	Evidence                *EvidenceManifest        `json:"evidence,omitempty"`
	RequirementExplanations []RequirementExplanation `json:"requirement_explanations,omitempty"`
}

type SessionDigest struct {
	ToolCallCount int         `json:"tool_call_count"`
	ToolsUsed     []string    `json:"tools_used"`
	ToolCalls     []ToolCall  `json:"tool_calls,omitempty"`
	Errors        []string    `json:"errors"`
	Usage         *UsageStats `json:"usage,omitempty"`
	SessionID     string      `json:"session_id,omitempty"`

	// ToolPolicyMode is the effective .agent.md `tools:` runtime policy
	// applied to this session: "unrestricted", "deny_all", or "allow_list".
	// Empty when no .agent.md tool policy was resolved for the run.
	ToolPolicyMode string `json:"tool_policy_mode,omitempty"`
	// ToolPolicyDenials records every tool-execution attempt denied by an
	// active tool policy across all turns of this session (initial,
	// follow-up, and responder-driven alike).
	ToolPolicyDenials []ToolPolicyDenial `json:"tool_policy_denials,omitempty"`
}

type SkillImpactMetric struct {
	PassRateWithSkills float64         `json:"pass_rate_with_skills"`
	PassRateBaseline   float64         `json:"pass_rate_baseline"`
	Delta              float64         `json:"delta"`
	PercentChange      float64         `json:"percent_change"`
	Pairwise           *PairwiseResult `json:"pairwise,omitempty"`
}

type SkillInvocation struct {
	Name string `json:"name"`
	Path string `json:"path,omitempty"`
}

type SkillInvocationGraderParameters struct {
	RequiredSkills  []string                    `yaml:"required_skills,omitempty" json:"required_skills,omitempty"`
	ForbiddenSkills []string                    `yaml:"forbidden_skills,omitempty" json:"forbidden_skills,omitempty"`
	Mode            SkillInvocationMatchingMode `yaml:"mode,omitempty" json:"mode,omitempty"`
	AllowExtra      *bool                       `yaml:"allow_extra,omitempty" json:"allow_extra,omitempty"`
}

type SkillInvocationMatchingMode string

type SkillSummary struct {
	SkillName      string   `json:"skill_name"`
	Models         []string `json:"models"`
	PassRate       float64  `json:"pass_rate"`
	AggregateScore float64  `json:"aggregate_score"`
	OutputFiles    []string `json:"output_files"`
}

type SpecIdentity struct {
	Name        string `yaml:"name" json:"name"`
	Description string `yaml:"description,omitempty" json:"description,omitempty"`
}

type StatisticalSummary struct {
	BootstrapCI *statistics.ConfidenceInterval `json:"bootstrap_ci,omitempty"`
	// Wilson 95% interval on SuccessRate, counting each task as one sample.
	SuccessRateCI *statistics.ConfidenceInterval `json:"success_rate_ci,omitempty"`
	// PassHatK[k-1] is the τ-bench pass^k: the chance that k fresh trials of a
	// task all pass, averaged over tasks. k runs up to the fewest trials of any task.
	PassHatK       []float64 `json:"pass_hat_k,omitempty"`
	NormalizedGain *float64  `json:"normalized_gain,omitempty"`

	// Deprecated: no longer populated (see TestStats.IsSignificant). A pointer
	// so older artifacts round-trip a stored false.
	IsSignificant *bool `json:"is_significant,omitempty"`
}

type Status string

type TaskExpectation struct {
	OutcomeSpecs    []OutcomeSpec  `yaml:"outcomes,omitempty" json:"outcome_specs,omitempty"`
	ToolPatterns    map[string]any `yaml:"tool_calls,omitempty" json:"tool_patterns,omitempty"`
	BehaviorRules   BehaviorRules  `yaml:"behavior,omitempty" json:"behavior_rules,omitempty"`
	MustInclude     []string       `yaml:"output_contains,omitempty" json:"must_include,omitempty"`
	MustExclude     []string       `yaml:"output_not_contains,omitempty" json:"must_exclude,omitempty"`
	MayInclude      []string       `yaml:"output_contains_any,omitempty" json:"may_include,omitempty"`
	ExpectedTrigger *bool          `yaml:"should_trigger,omitempty" json:"expected_trigger,omitempty"`
}

type TaskStimulus struct {
	Message     string            `yaml:"prompt" json:"message"`
	MessageFile string            `yaml:"prompt_file,omitempty" json:"message_file,omitempty"`
	Metadata    map[string]any    `yaml:"context,omitempty" json:"metadata,omitempty"`
	Resources   []ResourceRef     `yaml:"files,omitempty" json:"resources,omitempty"`
	Repos       []GitResource     `yaml:"repos,omitempty" json:"repos,omitempty"`
	WorkDir     string            `yaml:"workdir,omitempty" json:"workdir,omitempty"`
	Environment map[string]string `yaml:"environment,omitempty" json:"environment,omitempty"`
	FollowUps   []string          `yaml:"follow_up_prompts,omitempty" json:"follow_ups,omitempty"`
	Responder   *ResponderConfig  `yaml:"responder,omitempty" json:"responder,omitempty"`
}

type TaskTranscript struct {
	TaskID      string                   `json:"task_id"`
	TaskName    string                   `json:"task_name"`
	Status      Status                   `json:"status"`
	StartedAt   time.Time                `json:"started_at"`
	CompletedAt time.Time                `json:"completed_at"`
	DurationMs  int64                    `json:"duration_ms"`
	Prompt      string                   `json:"prompt"`
	FinalOutput string                   `json:"final_output"`
	Transcript  []TranscriptEvent        `json:"transcript"`
	Validations map[string]GraderResults `json:"validations,omitempty"`
	Session     SessionDigest            `json:"session"`
	ErrorMsg    string                   `json:"error_msg,omitempty"`
}

type TestCase struct {
	Active           *bool           `yaml:"enabled,omitempty" json:"active,omitempty"`
	ContextRoot      string          `yaml:"context_dir,omitempty" json:"context_root,omitempty"`
	DisplayName      string          `yaml:"name" json:"display_name"`
	Expectation      TaskExpectation `yaml:"expected,omitempty" json:"expectation,omitempty"`
	InstructionFiles []string        `yaml:"instruction_files,omitempty" json:"instruction_files,omitempty"`
	SkillPaths       []string        `yaml:"skill_directories,omitempty" json:"skill_paths,omitempty"`
	Stimulus         TaskStimulus    `yaml:"inputs" json:"stimulus"`
	Summary          string          `yaml:"description,omitempty" json:"summary,omitempty"`
	Tags             []string        `yaml:"tags,omitempty" json:"labels,omitempty"`
	TestID           string          `yaml:"id" json:"test_id"`
	// Golden marks a task as a "must always pass" regression gate. When set
	// to true, `waza gate` treats any non-pass status as a hard failure
	// (exit code 2) regardless of the aggregate pass-rate threshold.
	Golden     bool `yaml:"golden,omitempty" json:"golden,omitempty"`
	TimeoutSec *int `yaml:"timeout_seconds,omitempty" json:"timeout_sec,omitempty"`
	// FirstEventTimeoutSec overrides Config.FirstEventTimeoutSec for this task.
	// nil inherits the eval-level value; 0 disables the check for this task.
	FirstEventTimeoutSec *int                 `yaml:"first_event_timeout_seconds,omitempty" json:"first_event_timeout_sec,omitempty"`
	CommandMocks         *[]CommandMockConfig `yaml:"command_mocks,omitempty" json:"command_mocks,omitempty"`
	Validators           []ValidatorInline    `yaml:"graders,omitempty" json:"validators,omitempty"`
	// Checkpoints attach graders to intermediate turn boundaries in a
	// multi-turn run. Each checkpoint specifies `after_turn: N` (1-based,
	// where turn 1 is the initial prompt) and a list of graders that run
	// against the cumulative conversation state at the end of that turn.
	// Checkpoints are additive — task-level `graders:` still run against
	// the final state after all turns complete.
	Checkpoints  []Checkpoint  `yaml:"checkpoints,omitempty" json:"checkpoints,omitempty"`
	Requirements []Requirement `yaml:"requirements,omitempty" json:"requirements,omitempty"`
	// contains filtered or unexported fields
}

type TestOutcome struct {
	Cached      bool   `json:"cached,omitempty"`
	TestID      string `json:"test_id"`
	DisplayName string `json:"display_name"`
	Group       string `json:"group,omitempty"`
	// Golden mirrors TestCase.Golden so `waza gate` can identify
	// must-pass tasks directly from results.json without re-reading YAML.
	Golden      bool               `json:"golden,omitempty"`
	Status      Status             `json:"status"`
	Runs        []RunResult        `json:"runs"`
	Stats       *TestStats         `json:"stats,omitempty"`
	SkillImpact *SkillImpactMetric `json:"skill_impact,omitempty"`
}

type TestStats struct {
	PassRate         float64 `json:"pass_rate"`
	FlakinessPercent float64 `json:"flakiness_percent"`
	PassedRuns       int     `json:"passed_runs"`
	FailedRuns       int     `json:"failed_runs"`
	ErrorRuns        int     `json:"error_runs"`
	TotalRuns        int     `json:"total_runs"`
	AvgScore         float64 `json:"avg_score"`
	AvgWeightedScore float64 `json:"avg_weighted_score"`
	MinScore         float64 `json:"min_score"`
	MaxScore         float64 `json:"max_score"`
	StdDevScore      float64 `json:"std_dev_score"`
	ScoreVariance    float64 `json:"score_variance"`
	CI95Lo           float64 `json:"ci95_lo"`
	CI95Hi           float64 `json:"ci95_hi"`
	Flaky            bool    `json:"flaky"`
	AvgDurationMs    int64   `json:"avg_duration_ms"`

	// Bootstrap confidence interval over weighted scores (populated when trials > 1)
	BootstrapCI *statistics.ConfidenceInterval `json:"bootstrap_ci,omitempty"`
	// Wilson 95% interval on PassRate over this task's trials (populated when trials > 1)
	PassRateCI *statistics.ConfidenceInterval `json:"pass_rate_ci,omitempty"`

	// Deprecated: no longer populated. It tested whether a CI on the task's own
	// 0–1 score excluded zero, which compares an absolute score with zero instead
	// of testing a difference, so it was not a significance test. Kept so older
	// results files load without unknown-field warnings.
	IsSignificant *bool `json:"is_significant,omitempty"`
	// contains filtered or unexported fields
}

type TextGraderParameters struct {
	// Contains lists substrings that must appear in the output (case-insensitive).
	Contains []string `yaml:"contains,omitempty" json:"contains,omitempty"`

	// NotContains lists substrings that must NOT appear in the output (case-insensitive).
	NotContains []string `yaml:"not_contains,omitempty" json:"not_contains,omitempty"`

	// ContainsCS lists substrings that must appear in the output (case-sensitive).
	ContainsCS []string `yaml:"contains_cs,omitempty" json:"contains_cs,omitempty"`

	// NotContainsCS lists substrings that must NOT appear in the output (case-sensitive).
	NotContainsCS []string `yaml:"not_contains_cs,omitempty" json:"not_contains_cs,omitempty"`

	// RegexMatch lists regex patterns that must match somewhere in the output.
	RegexMatch []string `yaml:"regex_match,omitempty" json:"regex_match,omitempty"`

	// RegexNotMatch lists regex patterns that must NOT match anywhere in the output.
	RegexNotMatch []string `yaml:"regex_not_match,omitempty" json:"regex_not_match,omitempty"`
}

type ToolCall struct {
	// ID is the engine-assigned identifier for this call (e.g. the
	// Copilot SDK's ToolCallID). Used to correlate tool invocations in
	// observability backends.
	ID        string                               `json:"id,omitempty"`
	Name      string                               `json:"name"`
	Arguments ToolCallArgs                         `json:"arguments,omitempty"`
	Result    *copilot.ToolExecutionCompleteResult `json:"result,omitempty"`
	Success   bool                                 `json:"success"`
}

type ToolCallArgs struct {
	// these are filled out for file-based tools (view/edit)
	Path     string `json:"path"      mapstructure:"path"`
	FileText string `json:"file_text" mapstructure:"file_text"`

	// filled out for tools like bash or powershell
	Command     string `json:"command"     mapstructure:"command"`
	Description string `json:"description" mapstructure:"description"`

	// filled out for skill invocations
	Skill string `json:"skill" mapstructure:"skill"`

	// Extra captures any tool-specific argument keys that aren't part of
	// the fixed fields above (e.g. `query`/`limit` for search tools,
	// MCP-specific payloads). Populated automatically by mapstructure's
	// ",remain" support so argument matchers in tool_calls /
	// tool_constraint graders can see the full argument bag.
	//
	// Excluded from JSON marshaling: callers consuming arguments as a
	// generic map should use graders.normalizeToolCallArgs, which merges
	// Extra into the known fields. Persisting the raw bag in results.json
	// happens through RunResult.ToolEvents instead, which preserves the
	// engine's original argument value verbatim.
	Extra map[string]any `json:"-" mapstructure:",remain"`
}

type ToolCallsGraderParameters struct {
	RequiredTools  []string `yaml:"required_tools,omitempty" json:"required_tools,omitempty"`
	ForbiddenTools []string `yaml:"forbidden_tools,omitempty" json:"forbidden_tools,omitempty"`
	MinCalls       *int     `yaml:"min_calls,omitempty" json:"min_calls,omitempty"`
	MaxCalls       *int     `yaml:"max_calls,omitempty" json:"max_calls,omitempty"`

	// Expect is an ordered list of tool-call expectations. Each entry names
	// the expected tool and (optionally) structured matchers for argument
	// fields. An expectation passes when there is at least one actual tool
	// call whose name matches and whose arguments satisfy every matcher.
	//
	// Unlike RequiredTools (which only checks names), Expect lets a grader
	// assert input correctness — e.g. "list_dir was called with
	// path matching ^src/".
	Expect []ToolExpectation `yaml:"expect,omitempty" json:"expect,omitempty"`
}

type ToolConstraintGraderParameters struct {
	ExpectTools []ToolSpecParameters `yaml:"expect_tools,omitempty" json:"expect_tools,omitempty"`
	RejectTools []ToolSpecParameters `yaml:"reject_tools,omitempty" json:"reject_tools,omitempty"`

	// AllowOnly declares a policy allow-list of tool specs. When non-nil,
	// every observed tool call in the session digest must match at least one
	// entry; any undeclared tool used by the agent counts as a policy
	// violation. Declared tools do NOT need to be exercised — this is an
	// upper bound, not a lower bound.
	//
	// AllowOnly is distinct from ExpectTools:
	//
	//   - ExpectTools requires every listed tool to have been used at least
	//     once (a lower bound). Extra tools not listed are ignored.
	//   - AllowOnly forbids any tool that is not listed (an upper bound).
	//     Missing listed tools are not a violation.
	//
	// Semantics of the pointer:
	//
	//   - nil: no allow-list check is applied.
	//   - non-nil empty slice (`&[]ToolSpecParameters{}`): deny-all — any
	//     observed tool call is a violation.
	//   - non-nil populated: only observed calls whose tool name matches an
	//     entry (exact / case-insensitive on the Tool field, with optional
	//     pattern qualifiers on other fields) are allowed.
	//
	// Tool-name matching is exact (case-insensitive) rather than regex, so
	// entries injected from an .agent.md `tools:` frontmatter list are not
	// silently reinterpreted as patterns.
	AllowOnly *[]ToolSpecParameters `yaml:"allow_only,omitempty" json:"allow_only,omitempty"`
}

type ToolEvent struct {
	// Turn is the 1-based ordinal of the assistant turn that produced this
	// call. Multiple tool calls may share the same turn. 0 means unknown
	// (engines that do not surface turn boundaries).
	Turn int `json:"turn"`

	// Sequence is the 1-based ordinal of this call within the run, in the
	// order the engine emitted them. Stable across replays.
	Sequence int `json:"sequence"`

	// ToolCallID is the engine's unique identifier for this call. May be
	// empty for engines that do not assign IDs.
	ToolCallID string `json:"tool_call_id,omitempty"`

	// ToolName is the canonical name of the tool that was invoked.
	ToolName string `json:"tool_name"`

	// Args is the call's argument payload normalised to a JSON-compatible
	// value (object, array, scalar, or null). The exact shape depends on
	// the tool; argmatcher matchers in tool_calls / tool_constraint graders
	// run against this field.
	Args any `json:"args,omitempty"`

	// Result is the tool's response payload normalised to a JSON-compatible
	// value, or nil when the engine did not surface a result.
	Result any `json:"result,omitempty"`

	// Success reports whether the engine considered the call to have
	// completed without error.
	Success bool `json:"success"`

	// Error is the engine's failure message when Success is false. Empty
	// for successful calls.
	Error string `json:"error,omitempty"`

	// DurationMs is the wall-clock time between the tool-start and
	// tool-complete events when available. 0 when timing is unavailable.
	DurationMs int64 `json:"duration_ms,omitempty"`
}

type ToolExpectation struct {
	// Tool is the expected tool name (matched as a Go regexp against the
	// recorded tool name). A plain string is a substring match anchored by
	// the regexp engine, so use `^foo$` for exact match.
	Tool string `yaml:"tool" json:"tool"`

	// Args holds matcher specs for tool-call argument fields. All matchers
	// must pass for the expectation to be considered satisfied.
	Args map[string]argmatcher.Matcher `yaml:"args,omitempty" json:"args,omitempty"`
}

type ToolPolicyDenial struct {
	Tool   string `json:"tool"`
	Kind   string `json:"kind"`
	Reason string `json:"reason"`
}

type ToolSpecParameters struct {
	Tool           string `yaml:"tool" json:"tool"`
	CommandPattern string `yaml:"command_pattern,omitempty" json:"command_pattern,omitempty"`
	SkillPattern   string `yaml:"skill_pattern,omitempty" json:"skill_pattern,omitempty"`
	PathPattern    string `yaml:"path_pattern,omitempty" json:"path_pattern,omitempty"`

	// Args constrains specific argument fields on the tool call using
	// structured matchers (equals / regex / contains / range / json_schema).
	// Keys are argument names as they appear on ToolCall.Arguments after JSON
	// normalisation; values are matcher specs. All matchers must pass for the
	// call to satisfy the spec.
	Args map[string]argmatcher.Matcher `yaml:"args,omitempty" json:"args,omitempty"`
}

type TranscriptEvent struct {
	copilot.SessionEvent `json:"-"`
}

type TriggerHeuristicGraderParameters struct {
	SkillPath string   `yaml:"skill_path" json:"skill_path"`
	Mode      string   `yaml:"mode" json:"mode"`
	Threshold *float64 `yaml:"threshold,omitempty" json:"threshold,omitempty"`
}

type TriggerMetrics struct {
	TP        int     `json:"true_positives"`
	FP        int     `json:"false_positives"`
	TN        int     `json:"true_negatives"`
	FN        int     `json:"false_negatives"`
	Errors    int     `json:"errors,omitempty"`
	Precision float64 `json:"precision"`
	Recall    float64 `json:"recall"`
	F1        float64 `json:"f1"`
	Accuracy  float64 `json:"accuracy"`
}

type TriggerResult struct {
	Prompt        string            `json:"prompt"`
	Confidence    string            `json:"confidence,omitempty"`
	ShouldTrigger bool              `json:"should_trigger"`
	DidTrigger    bool              `json:"did_trigger"`
	ErrorMsg      string            `json:"error_msg,omitempty"`
	FinalOutput   string            `json:"final_output,omitempty"`
	Transcript    []TranscriptEvent `json:"transcript,omitempty"`
	ToolCalls     []ToolCall        `json:"tool_calls,omitempty"`
	SessionID     string            `json:"session_id,omitempty"`
}

type UsageStats struct {
	Turns            int     `json:"turns"`
	InputTokens      int     `json:"input_tokens"`
	OutputTokens     int     `json:"output_tokens"`
	CacheReadTokens  int     `json:"cache_read_tokens"`
	CacheWriteTokens int     `json:"cache_write_tokens"`
	PremiumRequests  float64 `json:"premium_requests"`
	// AICredits is the final GitHub Copilot AI-credit total reported by the
	// Copilot SDK for the session (nano-AI units converted with
	// [AICreditsFromNanoAIU]). It is nil when the SDK or bundled runtime did not
	// report a final credit total — for example artifacts produced by older
	// runtimes, or custom providers — and consumers must render that as an
	// explicit unavailable state rather than a computed estimate.
	AICredits    *float64              `json:"ai_credits,omitempty"`
	Provider     string                `json:"provider,omitempty"`
	ProviderHost string                `json:"provider_host,omitempty"`
	ModelMetrics map[string]ModelUsage `json:"model_metrics,omitempty"`
}

type ValidatorInline struct {
	Identifier string           `yaml:"name" json:"identifier"`
	Kind       GraderKind       `yaml:"type,omitempty" json:"kind,omitempty"`
	Checks     []string         `yaml:"assertions,omitempty" json:"checks,omitempty"`
	Rubric     string           `yaml:"rubric,omitempty" json:"rubric,omitempty"`
	Weight     float64          `yaml:"weight,omitempty" json:"weight,omitempty"`
	Parameters GraderParameters `yaml:"config,omitempty" json:"parameters,omitempty"`
}