eval

type BenchmarkSuite struct {
	Name    string
	Tasks   []BenchmarkTask
	Results []TaskResult
}

type BenchmarkTask struct {
	ID          string
	Description string
	SetupFn     func(workDir string) error
	ValidateFn  func(workDir string) (bool, string)
	Prompt      string
	TimeLimit   time.Duration
	Tags        []string
	MaxAttempts int
	Filters     []Filter
}

type Cache struct {
	Dir string
}

type CacheEntry struct {
	Model    string  `json:"model"`
	Prompt   string  `json:"prompt"`
	Response string  `json:"response"`
	Tokens   int     `json:"tokens"`
	CostUSD  float64 `json:"cost_usd"`
}

type CoverageAnalyzer struct {
	ProjectDir string
	// contains filtered or unexported fields
}

type CoverageReport struct {
	TotalLines         int
	CoveredLines       int
	Percentage         float64
	Files              []FileCoverage
	UncoveredFunctions []string
	Suggestions        []TestSuggestion
}

type FileCoverage struct {
	Path            string
	TotalLines      int
	CoveredLines    int
	Percentage      float64
	UncoveredRanges []LineRange
}

type Filter func(string) string

type GroupResult struct {
	Name     string  `json:"name"`
	Total    int     `json:"total"`
	Passed   int     `json:"passed"`
	PassRate float64 `json:"pass_rate"`
}

type LLMClient interface {
	Complete(ctx context.Context, model, prompt string) (response string, tokens int, cost float64, err error)
}

type LineRange struct {
	Start        int
	End          int
	FunctionName string
}

type ModelBenchmark struct {
	Name    string
	Models  []ModelConfig
	Tasks   []BenchmarkTask
	Runs    int
	Results map[string]*ModelResult
}

type ModelConfig struct {
	Name        string
	Provider    string
	Model       string
	Temperature float64
	MaxTokens   int
}

type ModelResult struct {
	Model       string
	PassRate    float64
	AvgTokens   int
	AvgCostUSD  float64
	AvgDuration time.Duration
	P50Duration time.Duration
	P95Duration time.Duration
	TaskResults []TaskResult
	Strengths   []string
	Weaknesses  []string
}

type PackageResult struct {
	Package  string
	Passed   int
	Failed   int
	Skipped  int
	Duration time.Duration
	Output   string
	Tests    []TestCaseResult
	Error    string
}

type ParallelRunner struct {
	MaxWorkers int
	Timeout    time.Duration
	Results    map[string]*PackageResult
	// contains filtered or unexported fields
}

type ResultHash struct {
	TasksHash  string `json:"tasks_hash"`
	PromptHash string `json:"prompt_hash"`
	GitCommit  string `json:"git_commit"`
	GoVersion  string `json:"go_version"`
	OS         string `json:"os"`
	Arch       string `json:"arch"`
}

type ResultStore struct {
	Dir string
}

type ResultSummary struct {
	TotalTasks    int     `json:"total_tasks"`
	Passed        int     `json:"passed"`
	Failed        int     `json:"failed"`
	PassRate      float64 `json:"pass_rate"`
	TotalDuration string  `json:"total_duration"`
	TotalTokens   int     `json:"total_tokens"`
	TotalCostUSD  float64 `json:"total_cost_usd"`
}

type Runner struct {
	Model       string
	Provider    string
	MaxAttempts int
	Timeout     time.Duration
	LLM         LLMClient
	Cache       *Cache
	NoCache     bool
	Filters     []Filter
}

type RunnerResult struct {
	Packages     []PackageResult
	TotalPassed  int
	TotalFailed  int
	TotalSkipped int
	Duration     time.Duration
	Parallel     int
}

type StoredResult struct {
	Version   string             `json:"version"`
	Timestamp time.Time          `json:"timestamp"`
	Suite     string             `json:"suite"`
	Model     string             `json:"model"`
	Provider  string             `json:"provider"`
	Hash      *ResultHash        `json:"hash,omitempty"`
	Summary   ResultSummary      `json:"summary"`
	Tasks     []StoredTaskResult `json:"tasks"`
}

type StoredTaskResult struct {
	TaskID   string   `json:"task_id"`
	Passed   bool     `json:"passed"`
	Duration string   `json:"duration"`
	Tokens   int      `json:"tokens"`
	CostUSD  float64  `json:"cost_usd"`
	Attempts int      `json:"attempts"`
	Error    string   `json:"error,omitempty"`
	Tags     []string `json:"tags,omitempty"`
}

type SuiteResult struct {
	Suite         string
	TotalTasks    int
	Passed        int
	Failed        int
	TotalDuration time.Duration
	TotalTokens   int
	TotalCostUSD  float64
	PassRate      float64
	Results       []TaskResult
}

type TaskGroup struct {
	Name  string
	Tags  []string // tasks matching any of these tags belong to this group
	Tasks []BenchmarkTask
}

type TaskResult struct {
	TaskID     string
	Passed     bool
	Duration   time.Duration
	TokensUsed int
	CostUSD    float64
	Attempts   int
	Error      string
}

type TestCaseResult struct {
	Name     string
	Passed   bool
	Duration time.Duration
	Output   string
}

type TestSuggestion struct {
	Function string
	File     string
	Priority string
	Reason   string
	Template string
}

type YAMLTask struct {
	Task        string            `yaml:"task"`
	Description string            `yaml:"description"`
	Language    string            `yaml:"language"`
	Tags        []string          `yaml:"tags"`
	Timeout     string            `yaml:"timeout"`
	MaxAttempts int               `yaml:"max_attempts"`
	Setup       string            `yaml:"setup"`
	Prompt      string            `yaml:"prompt"`
	Validate    []string          `yaml:"validate"`
	Files       map[string]string `yaml:"files"`
	Filters     []string          `yaml:"filters"`
}