bench

type ProviderResult struct {
	ProviderName string
	Model        string
	// Averaged across runs.
	AvgLatencyMS    float64
	MinLatencyMS    float64
	MaxLatencyMS    float64
	AvgInputTokens  float64
	AvgOutputTokens float64
	TotalCostUSD    float64
	// QualityScore is 1-10 as judged by the JudgeProvider. 0 means not rated.
	QualityScore float64
	// LastResponse is the last successful response (used for quality scoring).
	LastResponse string
	// Error is set when the provider failed on all runs.
	Error string
	// SuccessfulRuns counts how many runs succeeded.
	SuccessfulRuns int
}

type Report struct {
	Prompt    string
	Runs      int
	Timestamp time.Time
	Results   []*ProviderResult
}

type RunConfig struct {
	// Prompt is the text sent to all providers.
	Prompt string
	// Providers is the list of provider names to benchmark.
	Providers []string
	// Models maps provider name → model override (empty = use provider default).
	Models map[string]string
	// Runs is the number of times each provider is called (results are averaged).
	Runs int
	// JudgeProvider is the provider used to rate response quality (empty = skip).
	JudgeProvider string
	// Timeout per individual completion call.
	Timeout time.Duration
}