eval_harness

type AIAgent struct {
	// contains filtered or unexported fields
}

type AICheckCheckResult struct {
	Passed     bool `json:"passed"`
	ErrorCount int  `json:"error_count"`
}

type AICheckResult struct {
	File   string              `json:"file"`
	Check  AICheckCheckResult  `json:"check"`
	Verify AICheckVerifyResult `json:"verify"`
}

type AICheckVerifyResult struct {
	Available      bool `json:"available"`
	Verified       int  `json:"verified"`
	Counterexample int  `json:"counterexample"`
	Skipped        int  `json:"skipped"`
	Errors         int  `json:"errors"`
}

type AILANGRunner struct {
	// contains filtered or unexported fields
}

type AgentBenchmarkConfig struct {
	MaxConcurrent      int           // Max parallel Claude sessions
	RequestsPerSecond  int           // API rate limit
	TimeoutSeconds     int           // Timeout per benchmark
	WorkspaceDir       string        // Base workspace directory
	AllowedTools       []string      // Tools agent can use
	ClaudePath         string        // Path to claude CLI
	ClaudeModel        string        // Claude model to use (haiku, sonnet, opus, or full name)
	Verify             bool          // Enable contract verification (M-CONTRACT-EVAL)
	DevtoolsPrompt     string        // Devtools prompt content to append to system prompt (M-CONTRACT-EVAL)
	AgentPromptContent string        // Agent coding prompt content (replaces teaching prompt when UseAgentPrompt condition is active)
	Condition          EvalCondition // Experimental condition (overrides Verify/DevtoolsPrompt when set)
	MicroragMode       MicroragMode  // μRAG subprocess env mode (M-BRAIN-MICRORAG): on/off/auto
}

type AgentBenchmarkResult struct {
	BenchmarkID   string
	Executor      string // Executor used: "claude", "gemini", etc.
	Success       bool
	Iterations    int     // Number of agent turns
	Cost          float64 // Total cost in USD
	DurationMS    int     // Total time in milliseconds
	NumTurns      int     // Conversation turns
	ToolCallCount int     // Number of tool invocations (validates agentic behavior)
	Error         string  // Error message if failed
	SessionID     string  // Session ID from executor
	Result        string  // Final result text from agent

	// Token usage details
	Usage      TokenUsage            `json:"usage"`
	ModelUsage map[string]ModelStats `json:"modelUsage"`

	// Solution and session log for inspection
	SolutionCode  string `json:"solution_code,omitempty"`  // Generated solution code
	SessionLog    string `json:"session_log,omitempty"`    // Full Claude session log
	PromptVersion string `json:"prompt_version,omitempty"` // Version of teaching prompt used

	// Validation flags (match standard eval format for downstream compatibility)
	CompileOk bool   `json:"compile_ok"` // Did solution parse/compile?
	RuntimeOk bool   `json:"runtime_ok"` // Did solution run without error?
	StdoutOk  bool   `json:"stdout_ok"`  // Did output match expected?
	Stdout    string `json:"stdout,omitempty"`
	Stderr    string `json:"stderr,omitempty"`

	// Timing breakdown
	TTFTSeconds float64 `json:"ttft_seconds,omitempty"` // Time to first token in seconds

	// Cross-harness grouping
	ModelFamily string `json:"model_family,omitempty"` // Logical model family (e.g. "claude-sonnet-4-6"); empty = no grouping

	// Contract verification results (M-CONTRACT-EVAL)
	VerifyOk        bool   `json:"verify_ok"`             // All contracts verified
	VerifyVerified  int    `json:"verify_verified"`       // Count of verified functions
	VerifyCounterex int    `json:"verify_counterexample"` // Count of counterexamples
	VerifySkipped   int    `json:"verify_skipped"`        // Count of skipped functions
	VerifyErrors    int    `json:"verify_errors"`         // Count of Z3 errors
	VerifyJSON      string `json:"verify_json,omitempty"` // Full ai-check JSON output
}

type BenchmarkSpec struct {
	ID           string            `yaml:"id"`
	Description  string            `yaml:"description"`
	Languages    []string          `yaml:"languages"`
	Entrypoint   string            `yaml:"entrypoint"`
	Caps         []string          `yaml:"caps"`
	Prompt       string            `yaml:"prompt"`        // Inline prompt text (language-agnostic)
	PromptFiles  map[string]string `yaml:"prompt_files"`  // Language-specific prompt files: {ailang: "prompts/v0.3.0.md"}
	TaskPrompt   string            `yaml:"task_prompt"`   // Task-specific prompt appended after base prompt
	ContractSpec string            `yaml:"contract_spec"` // Optional: AILANG contract specification for Z3 verification
	Z3Hints      string            `yaml:"z3_hints"`      // Optional: Pre-computed Z3 counterexample descriptions for known traps
	ExpectedOut  string            `yaml:"expected_stdout"`
	Difficulty   string            `yaml:"difficulty"`
	ExpectedGain string            `yaml:"expected_gain"`
	Timeout      int               `yaml:"timeout"` // Agent timeout in seconds (default: 60)

	// Test infrastructure: stdin, CLI args, and input files
	Stdin      string            `yaml:"stdin,omitempty"`       // Stdin data to pipe to the program
	CliArgs    []string          `yaml:"cli_args,omitempty"`    // CLI arguments to pass after the script
	InputFiles map[string]string `yaml:"input_files,omitempty"` // Files to create in workspace: {filename: content}

	// Eval suite classification (M-EVAL-SUITE-PREP, v0.14.0)
	Tier string   `yaml:"tier,omitempty"` // One of: smoke|core|stretch|vision. Missing defaults to "core".
	Tags []string `yaml:"tags,omitempty"` // 1-3 tags from ValidTagTaxonomy. May be empty during migration.
}

type ClaudeHeadlessResult struct {
	Type              string                `json:"type"`
	Subtype           string                `json:"subtype"`
	IsError           bool                  `json:"is_error"`
	DurationMS        int                   `json:"duration_ms"`
	DurationAPIMS     int                   `json:"duration_api_ms"`
	NumTurns          int                   `json:"num_turns"`
	Result            string                `json:"result"`
	SessionID         string                `json:"session_id"`
	TotalCostUSD      float64               `json:"total_cost_usd"`
	Usage             TokenUsage            `json:"usage"`
	ModelUsage        map[string]ModelStats `json:"modelUsage"`
	PermissionDenials []interface{}         `json:"permission_denials"`
	UUID              string                `json:"uuid"`
	Transcript        string                `json:"-"` // Full conversation transcript (not in JSON, set by streaming)
}

type ErrCode string

type ErrUvMissing struct {
	// contains filtered or unexported fields
}

type EvalCondition struct {
	Name                string // "baseline", "contract", "z3_guided", "full", "tool_aware", "agent_prompt", or "" for legacy
	IncludeContract     bool   // Include contract_spec in prompt
	IncludeZ3Hints      bool   // Include z3_hints in prompt
	IncludeDevtools     bool   // Append devtools prompt to system prompt
	IncludeToolGuidance bool   // Include general contract-writing + ai-check guidance (no spec given)
	EnableVerify        bool   // Enable Z3 verification (standard mode repair + post-hoc check)
	UseAgentPrompt      bool   // Use compact agent coding prompt instead of full teaching prompt
}

type GenerateResult struct {
	Code         string
	InputTokens  int // Prompt tokens (input to LLM)
	OutputTokens int // Completion tokens (generated code)
	TotalTokens  int // Total tokens (for billing)
	Model        string
}

type GoRunner struct {
	// contains filtered or unexported fields
}

type JSRunner struct {
	// contains filtered or unexported fields
}

type LanguageRunner interface {
	Run(code string, timeout time.Duration) (*RunResult, error)
	Language() string
}

type LimitedWriter struct {
	// contains filtered or unexported fields
}

type MetricsLogger struct {
	// contains filtered or unexported fields
}

type MicroragMode string

type MockAIAgent struct {
	// contains filtered or unexported fields
}

type ModelConfig struct {
	APIName                  string  `yaml:"api_name"`
	Provider                 string  `yaml:"provider"`
	Description              string  `yaml:"description"`
	EnvVar                   string  `yaml:"env_var"`
	AgentCLI                 *string `yaml:"agent_cli"`          // CLI command for agent eval (e.g., "claude", "openai", "gemini"), nil if not supported
	AgentModelName           *string `yaml:"agent_model_name"`   // Model name to pass to agent CLI (e.g., "haiku", "sonnet")
	MaxOutputTokens          int     `yaml:"max_output_tokens"`  // Max output tokens (0 = handler default 4096)
	TTFTTimeoutSeconds       int     `yaml:"ttft_timeout"`       // Prefill budget in seconds (0 = executor default 30s)
	GenerationTimeoutSeconds int     `yaml:"generation_timeout"` // Per-token idle budget after first event (0 = executor default 3m)
	ModelFamily              string  `yaml:"model_family"`       // Logical model family for cross-harness grouping (e.g., "claude-sonnet-4-6"); empty = no grouping
	GCPProject               string  `yaml:"gcp_project"`        // Override GOOGLE_CLOUD_PROJECT for this model's evals (e.g. "ailang-dev")
	GCPLocation              string  `yaml:"gcp_location"`       // Override GOOGLE_CLOUD_LOCATION (e.g. "us-central1")
	Pricing                  Pricing `yaml:"pricing"`
	Notes                    string  `yaml:"notes"`
}

type ModelStats struct {
	InputTokens              int     `json:"inputTokens"`
	OutputTokens             int     `json:"outputTokens"`
	CacheReadInputTokens     int     `json:"cacheReadInputTokens"`
	CacheCreationInputTokens int     `json:"cacheCreationInputTokens"`
	WebSearchRequests        int     `json:"webSearchRequests"`
	CostUSD                  float64 `json:"costUSD"`
	ContextWindow            int     `json:"contextWindow"`
}

type ModelsConfig struct {
	Models           map[string]ModelConfig `yaml:"models"`
	Default          string                 `yaml:"default"`
	BenchmarkSuite   []string               `yaml:"benchmark_suite"`
	ExtendedSuite    []string               `yaml:"extended_suite"`
	DevModels        []string               `yaml:"dev_models"`
	AgentSuite       []string               `yaml:"agent_suite"`
	OllamaSuite      []string               `yaml:"ollama_suite"`
	HarnessSuite     []string               `yaml:"harness_suite"`
	LangHarnessSuite []string               `yaml:"lang_harness_suite"`
}

type MultiExecutorConfig struct {
	AgentBenchmarkConfig

	// ExecutorName specifies which executor to use (e.g., "claude", "gemini")
	// If empty, uses the model's agent_cli from models.yml
	ExecutorName string

	// ModelName is the model to use (e.g., "claude-sonnet-4-5", "gemini-3-flash")
	ModelName string

	// ConfigKey is the models.yml lookup key for per-model config (e.g., "opencode-gemma4-e4b").
	// When set, overrides ModelName for timeout/config lookups. Needed when ModelName is the
	// resolved API model name (e.g., "ollama/gemma4:e4b") rather than the models.yml key.
	ConfigKey string

	// ExtraHandler is an additional event handler composed with the debug handler.
	// Used for ObservatoryWriter to capture structured tool calls during streaming.
	// When nil, only the debug handler is used (no behavior change).
	ExtraHandler executor.EventHandler
}

type Pricing struct {
	InputPer1K  float64 `yaml:"input_per_1k"`
	OutputPer1K float64 `yaml:"output_per_1k"`
}

type PromptLoader struct {
	// contains filtered or unexported fields
}

type PromptRegistry struct {
	SchemaVersion string                   `json:"schema_version"`
	Versions      map[string]PromptVersion `json:"versions"`
	Active        string                   `json:"active"`
	Notes         []string                 `json:"notes"`
}

type PromptVersion struct {
	File        string   `json:"file"`
	Hash        string   `json:"hash"`
	Description string   `json:"description"`
	Created     string   `json:"created"`
	Tags        []string `json:"tags"`
	Notes       string   `json:"notes"`
}

type PythonRunner struct {
	// contains filtered or unexported fields
}

type RepairHint struct {
	Title string // Short description of the error
	Why   string // Explanation of why the error occurred
	How   string // Concrete steps to fix the error
}

type RepairLog struct {
	Wrapped       bool     // True if bare expression was wrapped in module scaffold
	AddedModule   bool     // True if module declaration was added
	AddedImports  []string // List of imports that were injected
	CallFixes     int      // Number of bare function calls that were fixed with parens
	AddedMainFunc bool     // True if main function was synthesized
}

type RepairRunner struct {
	// contains filtered or unexported fields
}

type RetryConfig struct {
	MaxRetries int
	BaseDelay  time.Duration
}

type RunMetrics struct {
	ID             string    `json:"id"`
	Lang           string    `json:"lang"`
	Model          string    `json:"model"`
	Executor       string    `json:"executor,omitempty"` // Executor used: "claude", "gemini", etc. (agent mode only)
	Seed           int64     `json:"seed"`
	InputTokens    int       `json:"input_tokens"`  // Prompt tokens (recorded but not primary metric)
	OutputTokens   int       `json:"output_tokens"` // Generated code tokens (PRIMARY METRIC)
	TotalTokens    int       `json:"total_tokens"`  // Total for billing
	CostUSD        float64   `json:"cost_usd"`
	CompileOk      bool      `json:"compile_ok"`
	RuntimeOk      bool      `json:"runtime_ok"`
	StdoutOk       bool      `json:"stdout_ok"`
	DurationMs     int64     `json:"duration_ms"`    // Total time (startup + compile + execution)
	CompileMs      int64     `json:"compile_ms"`     // Time spent in compilation (if separate)
	ExecuteMs      int64     `json:"execute_ms"`     // Time spent in execution (if measurable)
	ErrorCategory  string    `json:"error_category"` // compile_error | runtime_error | logic_error | none
	Stdout         string    `json:"stdout,omitempty"`
	Stderr         string    `json:"stderr,omitempty"`
	ExpectedStdout string    `json:"expected_stdout,omitempty"`
	Timestamp      time.Time `json:"timestamp"`
	Code           string    `json:"code,omitempty"` // Generated code (optional, for debugging)

	// Self-repair metrics (M-EVAL-LOOP)
	FirstAttemptOk  bool   `json:"first_attempt_ok"`            // Did first attempt succeed?
	RepairUsed      bool   `json:"repair_used"`                 // Did we attempt a repair?
	RepairOk        bool   `json:"repair_ok"`                   // Did repair succeed?
	ErrCode         string `json:"err_code,omitempty"`          // Error code from taxonomy (PAR_001, etc.)
	RepairTokensIn  int    `json:"repair_tokens_in,omitempty"`  // Input tokens for repair attempt
	RepairTokensOut int    `json:"repair_tokens_out,omitempty"` // Output tokens for repair attempt

	// Prompt versioning (M-EVAL-LOOP)
	PromptVersion string `json:"prompt_version,omitempty"` // Prompt version used (v0.3.0-hints, etc.)

	// Reproducibility (M-EVAL-LOOP)
	BinaryHash string   `json:"binary_hash,omitempty"` // SHA256 of ailang binary
	StdlibHash string   `json:"stdlib_hash,omitempty"` // SHA256 of stdlib
	Caps       []string `json:"caps,omitempty"`        // Capabilities granted

	// Contract verification results (M-CONTRACT-EVAL)
	VerifyOk        bool   `json:"verify_ok"`             // All contracts verified
	VerifyVerified  int    `json:"verify_verified"`       // Count of verified functions
	VerifyCounterex int    `json:"verify_counterexample"` // Count of counterexamples
	VerifySkipped   int    `json:"verify_skipped"`        // Count of skipped functions
	VerifyErrors    int    `json:"verify_errors"`         // Count of Z3 errors
	VerifyJSON      string `json:"verify_json,omitempty"` // Full ai-check JSON output

	// Agent mode KPIs (M-EVAL-AGENT)
	AgentTurns      int    `json:"agent_turns,omitempty"`      // Number of conversation turns (agent mode only)
	AgentTranscript string `json:"agent_transcript,omitempty"` // Full Claude conversation transcript (agent mode only)
	EvalMode        string `json:"eval_mode,omitempty"`        // Evaluation mode: "standard" or "agent"

	// Experimental condition (M-CONTRACT-EVAL conditions dimension)
	Condition string `json:"condition,omitempty"` // Experimental condition: "baseline", "contract", "z3_guided", "full"

	// μRAG state for this run (M-BRAIN-MICRORAG)
	// Values: "on" | "off" | "auto" | "disabled" | "" (legacy / not set).
	// Lets eval-report break results down with vs. without JIT knowledge injection.
	MicroragState string `json:"microrag_state,omitempty"`

	// Cross-harness grouping (M-EVAL-CROSS-HARNESS)
	// Populated from models.yml model_family field. Enables --group-by=model-family
	// in eval-matrix to compare same model across different harnesses (e.g. claude vs opencode).
	ModelFamily string `json:"model_family,omitempty"`
}

type RunResult struct {
	Stdout       string
	Stderr       string
	ExitCode     int
	Duration     time.Duration // Total time (startup + compile + execution)
	CompileTime  time.Duration // Time spent in compilation/type-checking (if separate)
	ExecuteTime  time.Duration // Time spent in actual code execution (if measurable)
	CompileOk    bool
	RuntimeOk    bool
	StdoutOk     bool
	TimedOut     bool
	CodeHash     string // SHA256 hash of executed code (for validation)
	WorkspaceDir string // Path to isolated workspace (for debugging)
}

type TelemetryReporter struct {
	// contains filtered or unexported fields
}

type TokenUsage struct {
	InputTokens              int `json:"input_tokens"`
	CacheCreationInputTokens int `json:"cache_creation_input_tokens"`
	CacheReadInputTokens     int `json:"cache_read_input_tokens"`
	OutputTokens             int `json:"output_tokens"`
}

type ValidationResult struct {
	CompileOk bool
	RuntimeOk bool
	StdoutOk  bool
	Stdout    string
	Stderr    string
}

type Watchdog struct {
	MaxAge      time.Duration // Kill processes older than this
	CheckPeriod time.Duration // How often to check
	Pattern     string        // Process pattern to match
	KilledCount int           // Number of orphans killed
	Enabled     bool          // Whether watchdog is active
}