benchmark

type AggregatedStats struct {
	BenchmarkID string  `json:"benchmark_id"`
	TotalRuns   int     `json:"total_runs"`
	Successes   int     `json:"successes"`
	Failures    int     `json:"failures"`
	SuccessRate float64 `json:"success_rate"`

	// LLM Calls
	LLMCallsMin    int     `json:"llm_calls_min"`
	LLMCallsMax    int     `json:"llm_calls_max"`
	LLMCallsMean   float64 `json:"llm_calls_mean"`
	LLMCallsMedian float64 `json:"llm_calls_median"`
	LLMCallsP5     float64 `json:"llm_calls_p5"`
	LLMCallsP95    float64 `json:"llm_calls_p95"`
	LLMCallsStdDev float64 `json:"llm_calls_stddev"`

	// Tokens (total: prompt + completion)
	TokensMin    int     `json:"tokens_min"`
	TokensMax    int     `json:"tokens_max"`
	TokensMean   float64 `json:"tokens_mean"`
	TokensMedian float64 `json:"tokens_median"`
	TokensP5     float64 `json:"tokens_p5"`
	TokensP95    float64 `json:"tokens_p95"`
	TokensStdDev float64 `json:"tokens_stddev"`

	// Generated Tokens (completion tokens only)
	GeneratedTokensMin    int     `json:"generated_tokens_min"`
	GeneratedTokensMax    int     `json:"generated_tokens_max"`
	GeneratedTokensMean   float64 `json:"generated_tokens_mean"`
	GeneratedTokensMedian float64 `json:"generated_tokens_median"`
	GeneratedTokensP5     float64 `json:"generated_tokens_p5"`
	GeneratedTokensP95    float64 `json:"generated_tokens_p95"`
	GeneratedTokensStdDev float64 `json:"generated_tokens_stddev"`

	// Context Used (max context window used)
	ContextUsedMin    int     `json:"context_used_min"`
	ContextUsedMax    int     `json:"context_used_max"`
	ContextUsedMean   float64 `json:"context_used_mean"`
	ContextUsedMedian float64 `json:"context_used_median"`
	ContextUsedP5     float64 `json:"context_used_p5"`
	ContextUsedP95    float64 `json:"context_used_p95"`

	// Processed Tokens (prompt - cached)
	ProcessedTokensMean float64 `json:"processed_tokens_mean"`

	// Speed metrics (tokens/sec, weighted averages across all runs)
	PromptSpeed     float64 `json:"prompt_speed"`     // processed tokens / prompt time
	GenerationSpeed float64 `json:"generation_speed"` // generated tokens / generation time

	// Cost
	CostMin    float64 `json:"cost_min"`
	CostMax    float64 `json:"cost_max"`
	CostMean   float64 `json:"cost_mean"`
	CostTotal  float64 `json:"cost_total"`
	CostStdDev float64 `json:"cost_stddev"`

	// Duration
	DurationMinMS    int64   `json:"duration_min_ms"`
	DurationMaxMS    int64   `json:"duration_max_ms"`
	DurationMeanMS   float64 `json:"duration_mean_ms"`
	DurationMedianMS float64 `json:"duration_median_ms"`
	DurationP5MS     float64 `json:"duration_p5_ms"`
	DurationP95MS    float64 `json:"duration_p95_ms"`
	DurationStdDevMS float64 `json:"duration_stddev_ms"`

	// Tool usage
	ToolCallCounts map[string]int `json:"tool_call_counts"`

	// Errors
	ErrorCounts map[string]int `json:"error_counts"`
}

type BenchmarkConfig struct {
	Enabled       bool     `yaml:"enabled"`
	OutputDir     string   `yaml:"output_dir"`
	RunsPerTask   int      `yaml:"runs_per_task"`
	TimeoutPerRun int      `yaml:"timeout_per_run"` // seconds
	ReportFormat  string   `yaml:"report_format"`   // "markdown" or "json"
	Categories    []string `yaml:"categories"`      // Which categories to run
	BenchmarkIDs  []string `yaml:"benchmark_ids"`   // Specific benchmarks to run
	WarmupTask    string   `yaml:"warmup_task"`     // Warmup task (default: "Say hello")
	NoResume      bool     `yaml:"no_resume"`       // Force fresh start
}

type BenchmarkDef struct {
	ID         string            `yaml:"id"`
	Name       string            `yaml:"name"`
	Category   string            `yaml:"category"`
	Goal       string            `yaml:"goal"`
	Setup      []SetupFile       `yaml:"setup"`
	Task       string            `yaml:"task"`
	Validation []ValidationCheck `yaml:"validation"`
	Tags       []string          `yaml:"tags"`
	Haystack   string            `yaml:"haystack,omitempty"` // For needle_retrieval: haystack ID to use
}

type BenchmarksFile struct {
	Haystacks  map[string]HaystackDef `yaml:"haystacks"`
	Benchmarks []BenchmarkDef         `yaml:"benchmarks"`
}

type CLIFlags struct {
	Enabled     bool
	Runs        int
	Category    string
	BenchmarkID string
	OutputFile  string
	NoResume    bool
	Suffix      string
}

type CSVWriter struct {
	// contains filtered or unexported fields
}

type ClassStats struct {
	Class                string
	TotalRuns            int
	Successes            int
	Failures             int
	SuccessRate          float64
	TotalDurationMS      int64
	TotalGeneratedTokens int64
}

type Environment struct {
	BaseDir      string // .kvit-coder-benchmark
	SetupDir     string // .kvit-coder-benchmark/setup
	WorkspaceDir string // .kvit-coder-benchmark/workspace
	ExpectedDir  string // .kvit-coder-benchmark/expected
	ResultsDir   string // .kvit-coder-benchmark/results
}

type ExecuteResult struct {
	RunResult    *RunResult
	FinalOutput  string
	ToolCalls    []ToolCallLog
	Messages     []llm.Message
	WorkspaceDir string
}

type Executor struct {
	// contains filtered or unexported fields
}

type FailureDetail struct {
	BenchmarkID  string   `json:"benchmark_id"`
	Run          int      `json:"run"`
	Errors       []string `json:"errors"`
	LastToolCall string   `json:"last_tool_call,omitempty"`
}

type HaystackBenchmarkDef struct {
	ID         string            `yaml:"id"`
	Name       string            `yaml:"name"`
	Haystack   string            `yaml:"haystack"`
	Task       string            `yaml:"task"`
	Validation []ValidationCheck `yaml:"validation"`
	Tags       []string          `yaml:"tags"`
}

type HaystackCLIFlags struct {
	Enabled     bool
	Runs        int
	BenchmarkID string
	OutputFile  string
	NoResume    bool
	Suffix      string
}

type HaystackDef struct {
	File        string `yaml:"file"`        // Path to haystack file (relative to benchmarks dir)
	Generate    string `yaml:"generate"`    // Optional script to generate the file
	Source      string `yaml:"source"`      // Optional download URL
	Description string `yaml:"description"` // Human-readable description
}

type HaystackFile struct {
	Haystacks  map[string]HaystackDef `yaml:"haystacks"`
	Benchmarks []HaystackBenchmarkDef `yaml:"benchmarks"`
}

type Progress struct {
	// contains filtered or unexported fields
}

type Report struct {
	Version    string            `json:"version"`
	Date       time.Time         `json:"date"`
	Config     string            `json:"config"` // Full config.yaml contents
	Summary    []AggregatedStats `json:"summary"`
	Failures   []FailureDetail   `json:"failures"`
	RawResults []RunResult       `json:"raw_results,omitempty"` // Optional detailed results
}

type ReportGenerator struct {
	// contains filtered or unexported fields
}

type ResumeInfo struct {
	CompletedRuns map[string]map[int]bool // map[benchmarkID]map[runNumber]completed
	FirstStarted  time.Time               // When benchmarking started
	LastCompleted time.Time               // Last completion time
}

type RunResult struct {
	BenchmarkID     string        `json:"benchmark_id" csv:"benchmark_id"`
	Run             int           `json:"run" csv:"run"`
	Success         bool          `json:"success" csv:"success"`
	LLMCalls        int           `json:"llm_calls" csv:"llm_calls"`
	Tokens          int           `json:"tokens" csv:"tokens"`
	PromptTokens    int           `json:"prompt_tokens" csv:"prompt_tokens"`
	GeneratedTokens int           `json:"generated_tokens" csv:"generated_tokens"`
	CachedTokens    int           `json:"cached_tokens" csv:"cached_tokens"`
	ContextUsed     int           `json:"context_used" csv:"context_used"`
	Cost            float64       `json:"cost" csv:"cost"`
	DurationMS      int64         `json:"duration_ms" csv:"duration_ms"`
	PromptMS        float64       `json:"prompt_ms" csv:"prompt_ms"`
	GenerationMS    float64       `json:"generation_ms" csv:"generation_ms"`
	ToolCalls       []ToolCallLog `json:"tool_calls" csv:"-"`
	Errors          []string      `json:"errors" csv:"-"`
	StartedAt       time.Time     `json:"started_at" csv:"started_at"`
	CompletedAt     time.Time     `json:"completed_at" csv:"completed_at"`

	// For CSV serialization
	ToolCallsJSON string `json:"-" csv:"tool_calls"`
	ErrorsJSON    string `json:"-" csv:"errors"`
}

type Runner struct {
	// contains filtered or unexported fields
}

type Sandbox struct {
	Tool       string   // "bwrap", "firejail", or "none"
	ExecPrefix []string // argv prepended before "sh -c <cmd>" ({workdir} substituted at run time)
}

type SetupFile struct {
	File    string `yaml:"file"`
	Content string `yaml:"content"`
	Binary  []byte `yaml:"binary,omitempty"` // For binary file tests
	Dir     bool   `yaml:"dir,omitempty"`    // Create directory instead of file
}

type TBCSVWriter struct {
	// contains filtered or unexported fields
}

type TBCheck struct {
	ID     string `json:"id"`
	Desc   string `json:"desc"`
	Detail string `json:"detail"`
	Passed bool   `json:"passed"`
}

type TBExecutor struct {
	// contains filtered or unexported fields
}

type TBManifestEntry struct {
	Slug      string `json:"slug"`
	Type      string `json:"type"`
	Brief     string `json:"brief"`
	Observed  bool   `json:"observed"`
	NumChecks int    `json:"num_checks"`
	HasSetup  bool   `json:"has_setup"`
}

type TBResults struct {
	RunIDs           []int64                        `json:"run_ids"`
	Models           []string                       `json:"models"`
	Trials           int                            `json:"trials"`
	NumGradedTasks   int                            `json:"num_graded_tasks"`
	NumObservedTasks int                            `json:"num_observed_tasks"`
	NumRuns          int                            `json:"num_runs"`
	EffortMode       string                         `json:"effort_mode"`
	Config           map[string]any                 `json:"config"`
	OverallGraded    map[string]tbRollup            `json:"overall_graded"`
	ByType           map[string]map[string]tbRollup `json:"by_type"`
	ByTask           map[string]tbByTaskEntry       `json:"by_task"`
	Observed         map[string]map[string]tbRollup `json:"observed"`
}

type TBRunResult struct {
	Slug     string
	Type     string
	Run      int
	Observed bool

	// Scoring (graded tasks only; observed tasks leave these zero).
	Score    float64 // 0..1 = passed/total
	Passed   int
	Total    int
	FullPass bool // Score == 1.0 — the binary "solve" analogue
	ImportOK bool

	// Metrics (reused from agent stats, mirroring RunResult).
	DurationMS      int64
	Tokens          int
	PromptTokens    int
	GeneratedTokens int
	CachedTokens    int
	ContextUsed     int
	Cost            float64
	LLMCalls        int

	// TimedOut records that the run was stopped by thinkbench.timeout_per_run
	// rather than finishing. Such a run is still graded on whatever it wrote,
	// so a nonzero Score here does not mean the agent completed the task.
	TimedOut bool

	Errors       []string
	FailedChecks []string // "check_id: detail" for each grader check that did not pass
	WorkspaceDir string   // persisted path for observed tasks / grading failures
}

type TBScorecard struct {
	Task     string    `json:"task"`
	ImportOK bool      `json:"import_ok"`
	Passed   int       `json:"passed"`
	Total    int       `json:"total"`
	Score    float64   `json:"score"`
	Checks   []TBCheck `json:"checks"`
}

type TBTask struct {
	Slug       string
	Type       TBType
	Brief      string // brief.txt contents (the only prompt the agent sees)
	Observed   bool   // ambiguous-spec → true (no grader, persist-only)
	NumChecks  int    // from manifest (0 for observed)
	HasSetup   bool   // has a setup/ starter tree
	SetupDir   string // abs path to suite/tasks/<slug>/setup (may not exist)
	GraderPath string // abs path to suite/tasks/<slug>/grade.py (held out; "" for observed)
}

type TBType string

type ThinkbenchCLIFlags struct {
	Enabled     bool
	Runs        int    // trials per task (already resolved: -n / config.trials / default)
	BenchmarkID string // comma-separated slugs
	Types       string // comma-separated type override
	SuiteDir    string // override suite directory
	OutputFile  string
	NoResume    bool // accepted for parity; thinkbench is fresh-by-default so this is a no-op
	Resume      bool // opt in to resuming the most recent interrupted run
	Suffix      string
}

type ToolCallLog struct {
	Tool string          `json:"tool"`
	Args json.RawMessage `json:"args,omitempty"`
}

type ValidationCheck struct {
	Type     string `yaml:"type"`     // "file_contains", "file_equals", "file_exists", "file_not_exists", "file_line_count", "tool_called", "tool_called_with", "output_contains", "output_not_contains", "multi_tool_calls", "run_command"
	Target   string `yaml:"target"`   // File path or "output"
	Expected string `yaml:"expected"` // Expected value, pattern, or tool name
	Args     string `yaml:"args"`     // Expected args pattern (for tool_called_with)
	Command  string `yaml:"command"`  // Command to run (for run_command)
	Count    int    `yaml:"count"`    // Expected count (for file_line_count, multi_tool_calls)
	Line     int    `yaml:"line"`     // Specific line number (for file_line_equals)
	Negate   bool   `yaml:"negate"`   // Check for absence instead of presence
}

type ValidationResult struct {
	Check   ValidationCheck
	Passed  bool
	Message string
}

type Validator struct {
	// contains filtered or unexported fields
}