benchmark

package
v0.0.0-...-667cd2e Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 4, 2026 License: MIT Imports: 23 Imported by: 0

Documentation

Overview

Package benchmark provides benchmarking infrastructure for testing LLM tool usage.

Index

Constants

This section is empty.

Variables

View Source 
var CSVHeaders = []string{
	"benchmark_id",
	"run",
	"success",
	"llm_calls",
	"tokens",
	"prompt_tokens",
	"generated_tokens",
	"cached_tokens",
	"context_used",
	"cost",
	"duration_ms",
	"prompt_ms",
	"generation_ms",
	"tool_calls",
	"errors",
	"started_at",
	"completed_at",
}

CSVHeaders are the column headers for the benchmark CSV.

Functions

func FindBenchmarksFile 

func FindBenchmarksFile(workspaceRoot string) string

FindBenchmarksFile looks for benchmarks.yaml in standard locations.

func GetBenchmarksByCategory 

func GetBenchmarksByCategory(benchmarks []BenchmarkDef) map[string][]BenchmarkDef

GetBenchmarksByCategory groups benchmarks by category.

func GetCSVPath 

func GetCSVPath(outputPath string) string

GetCSVPath returns the CSV path for a given output path. If output is benchmark-results.md, returns benchmark-results.csv

func ListBenchmarks 

func ListBenchmarks(cfg *config.Config) error

ListBenchmarks prints available benchmarks.

func ListCategories 

func ListCategories(benchmarks []BenchmarkDef) []string

ListCategories returns unique categories from benchmarks.

func Run 

func Run(ctx context.Context, flags CLIFlags, runner *agent.Runner, cfg *config.Config, systemPrompt string, version string, originalWorkspaceRoot string) error

Run executes the benchmark CLI with the given configuration. originalWorkspaceRoot is the workspace before benchmark override (for finding benchmarks.yaml)

Types

type AggregatedStats 

type AggregatedStats struct {
	BenchmarkID string  `json:"benchmark_id"`
	TotalRuns   int     `json:"total_runs"`
	Successes   int     `json:"successes"`
	Failures    int     `json:"failures"`
	SuccessRate float64 `json:"success_rate"`

	// LLM Calls
	LLMCallsMin    int     `json:"llm_calls_min"`
	LLMCallsMax    int     `json:"llm_calls_max"`
	LLMCallsMean   float64 `json:"llm_calls_mean"`
	LLMCallsMedian float64 `json:"llm_calls_median"`
	LLMCallsP5     float64 `json:"llm_calls_p5"`
	LLMCallsP95    float64 `json:"llm_calls_p95"`
	LLMCallsStdDev float64 `json:"llm_calls_stddev"`

	// Tokens (total: prompt + completion)
	TokensMin    int     `json:"tokens_min"`
	TokensMax    int     `json:"tokens_max"`
	TokensMean   float64 `json:"tokens_mean"`
	TokensMedian float64 `json:"tokens_median"`
	TokensP5     float64 `json:"tokens_p5"`
	TokensP95    float64 `json:"tokens_p95"`
	TokensStdDev float64 `json:"tokens_stddev"`

	// Generated Tokens (completion tokens only)
	GeneratedTokensMin    int     `json:"generated_tokens_min"`
	GeneratedTokensMax    int     `json:"generated_tokens_max"`
	GeneratedTokensMean   float64 `json:"generated_tokens_mean"`
	GeneratedTokensMedian float64 `json:"generated_tokens_median"`
	GeneratedTokensP5     float64 `json:"generated_tokens_p5"`
	GeneratedTokensP95    float64 `json:"generated_tokens_p95"`
	GeneratedTokensStdDev float64 `json:"generated_tokens_stddev"`

	// Context Used (max context window used)
	ContextUsedMin    int     `json:"context_used_min"`
	ContextUsedMax    int     `json:"context_used_max"`
	ContextUsedMean   float64 `json:"context_used_mean"`
	ContextUsedMedian float64 `json:"context_used_median"`
	ContextUsedP5     float64 `json:"context_used_p5"`
	ContextUsedP95    float64 `json:"context_used_p95"`

	// Processed Tokens (prompt - cached)
	ProcessedTokensMean float64 `json:"processed_tokens_mean"`

	// Speed metrics (tokens/sec, weighted averages across all runs)
	PromptSpeed     float64 `json:"prompt_speed"`     // processed tokens / prompt time
	GenerationSpeed float64 `json:"generation_speed"` // generated tokens / generation time

	// Cost
	CostMin    float64 `json:"cost_min"`
	CostMax    float64 `json:"cost_max"`
	CostMean   float64 `json:"cost_mean"`
	CostTotal  float64 `json:"cost_total"`
	CostStdDev float64 `json:"cost_stddev"`

	// Duration
	DurationMinMS    int64   `json:"duration_min_ms"`
	DurationMaxMS    int64   `json:"duration_max_ms"`
	DurationMeanMS   float64 `json:"duration_mean_ms"`
	DurationMedianMS float64 `json:"duration_median_ms"`
	DurationP5MS     float64 `json:"duration_p5_ms"`
	DurationP95MS    float64 `json:"duration_p95_ms"`
	DurationStdDevMS float64 `json:"duration_stddev_ms"`

	// Tool usage
	ToolCallCounts map[string]int `json:"tool_call_counts"`

	// Errors
	ErrorCounts map[string]int `json:"error_counts"`
}

AggregatedStats holds aggregated statistics for a benchmark across all runs.

type BenchmarkConfig 

type BenchmarkConfig struct {
	Enabled       bool     `yaml:"enabled"`
	OutputDir     string   `yaml:"output_dir"`
	RunsPerTask   int      `yaml:"runs_per_task"`
	TimeoutPerRun int      `yaml:"timeout_per_run"` // seconds
	ReportFormat  string   `yaml:"report_format"`   // "markdown" or "json"
	Categories    []string `yaml:"categories"`      // Which categories to run
	BenchmarkIDs  []string `yaml:"benchmark_ids"`   // Specific benchmarks to run
	WarmupTask    string   `yaml:"warmup_task"`     // Warmup task (default: "Say hello")
	NoResume      bool     `yaml:"no_resume"`       // Force fresh start
}

BenchmarkConfig holds configuration for benchmark runs.

func DefaultBenchmarkConfig 

func DefaultBenchmarkConfig() *BenchmarkConfig

DefaultBenchmarkConfig returns default benchmark configuration.

type BenchmarkDef 

type BenchmarkDef struct {
	ID         string            `yaml:"id"`
	Name       string            `yaml:"name"`
	Category   string            `yaml:"category"`
	Goal       string            `yaml:"goal"`
	Setup      []SetupFile       `yaml:"setup"`
	Task       string            `yaml:"task"`
	Validation []ValidationCheck `yaml:"validation"`
	Tags       []string          `yaml:"tags"`
}

BenchmarkDef defines a single benchmark test case.

func FilterBenchmarks 

func FilterBenchmarks(benchmarks []BenchmarkDef, categories []string, ids []string) []BenchmarkDef

FilterBenchmarks filters benchmarks by category and/or IDs.

func LoadBenchmarks 

func LoadBenchmarks(path string) ([]BenchmarkDef, error)

LoadBenchmarks loads benchmark definitions from a YAML file.

type BenchmarksFile 

type BenchmarksFile struct {
	Benchmarks []BenchmarkDef `yaml:"benchmarks"`
}

BenchmarksFile represents the structure of benchmarks.yaml

type CLIFlags 

type CLIFlags struct {
	Enabled     bool
	Runs        int
	Category    string
	BenchmarkID string
	OutputFile  string
	NoResume    bool
	Suffix      string
}

CLIFlags holds the command-line flags for benchmarks.

type CSVWriter 

type CSVWriter struct {
	// contains filtered or unexported fields
}

CSVWriter handles writing benchmark results to CSV.

func NewCSVWriter 

func NewCSVWriter(path string, resume bool) (*CSVWriter, error)

NewCSVWriter creates a new CSV writer for benchmark results.

func (*CSVWriter) Close 

func (w *CSVWriter) Close() error

Close closes the CSV writer.

func (*CSVWriter) WriteResult 

func (w *CSVWriter) WriteResult(result *RunResult) error

WriteResult writes a single benchmark result to CSV.

type ClassStats 

type ClassStats struct {
	Class           string
	TotalRuns       int
	Successes       int
	Failures        int
	SuccessRate     float64
	TotalDurationMS int64
}

ClassStats holds aggregated statistics for a benchmark class (e.g., C, E, R, S, W)

type Environment 

type Environment struct {
	BaseDir      string // .kvit-coder-benchmark
	SetupDir     string // .kvit-coder-benchmark/setup
	WorkspaceDir string // .kvit-coder-benchmark/workspace
	ExpectedDir  string // .kvit-coder-benchmark/expected
	ResultsDir   string // .kvit-coder-benchmark/results
}

Environment manages the benchmark test environment.

func NewEnvironment 

func NewEnvironment(baseDir string) *Environment

NewEnvironment creates a new benchmark environment.

func (*Environment) CleanupAll 

func (e *Environment) CleanupAll() error

CleanupAll removes all benchmark workspaces.

func (*Environment) CleanupBenchmark 

func (e *Environment) CleanupBenchmark(benchmark BenchmarkDef, runID int) error

CleanupBenchmark cleans the workspace after a benchmark run.

func (*Environment) FileExistsInWorkspace 

func (e *Environment) FileExistsInWorkspace(benchmarkID string, runID int, relativePath string) bool

FileExistsInWorkspace checks if a file exists in the benchmark workspace.

func (*Environment) GetWorkspaceDir 

func (e *Environment) GetWorkspaceDir(benchmarkID string, runID int) string

GetWorkspaceDir returns the workspace directory for a specific run.

func (*Environment) Initialize 

func (e *Environment) Initialize() error

Initialize creates the benchmark directory structure.

func (*Environment) ListFilesInWorkspace 

func (e *Environment) ListFilesInWorkspace(benchmarkID string, runID int) ([]string, error)

ListFilesInWorkspace lists all files in the benchmark workspace.

func (*Environment) LoadExpected 

func (e *Environment) LoadExpected(benchmarkID string) ([]byte, error)

LoadExpected loads expected results for validation.

func (*Environment) ReadFileFromWorkspace 

func (e *Environment) ReadFileFromWorkspace(benchmarkID string, runID int, relativePath string) ([]byte, error)

ReadFileFromWorkspace reads a file from the benchmark workspace.

func (*Environment) SaveExpected 

func (e *Environment) SaveExpected(benchmarkID string, content []byte) error

SaveExpected saves expected results for validation.

func (*Environment) SetupBenchmark 

func (e *Environment) SetupBenchmark(benchmark BenchmarkDef, runID int) (string, error)

SetupBenchmark prepares the workspace for a specific benchmark run. All benchmarks share the same workspace directory, which is cleaned before each run.

type ExecuteResult 

type ExecuteResult struct {
	RunResult    *RunResult
	FinalOutput  string
	ToolCalls    []ToolCallLog
	Messages     []llm.Message
	WorkspaceDir string
}

ExecuteResult contains the result of executing a benchmark.

type Executor 

type Executor struct {
	// contains filtered or unexported fields
}

Executor handles running a single benchmark.

func NewExecutor 

func NewExecutor(runner *agent.Runner, cfg *config.Config, systemPrompt string, env *Environment, timeout time.Duration, stdoutWriter, stderrWriter io.Writer) *Executor

NewExecutor creates a new benchmark executor.

func (*Executor) Execute 

func (e *Executor) Execute(ctx context.Context, benchmark BenchmarkDef, runID int) (*ExecuteResult, error)

Execute runs a single benchmark and returns the result.

func (*Executor) IsExternalCommand 

func (e *Executor) IsExternalCommand() bool

IsExternalCommand returns true if the executor is configured to use an external command.

func (*Executor) Warmup 

func (e *Executor) Warmup(ctx context.Context, task string) error

Warmup runs a warmup task to prime the cache.

type FailureDetail 

type FailureDetail struct {
	BenchmarkID  string   `json:"benchmark_id"`
	Run          int      `json:"run"`
	Errors       []string `json:"errors"`
	LastToolCall string   `json:"last_tool_call,omitempty"`
}

FailureDetail captures information about a failed benchmark run.

type Progress 

type Progress struct {
	// contains filtered or unexported fields
}

Progress tracks and displays benchmark progress.

func NewProgress 

func NewProgress(writer io.Writer, totalBenchmarks, runsPerBenchmark int) *Progress

NewProgress creates a new progress tracker.

func (*Progress) CompleteRun 

func (p *Progress) CompleteRun(duration time.Duration, result *RunResult)

CompleteRun marks a run as complete and shows pass/fail status with statistics.

func (*Progress) Display 

func (p *Progress) Display()

Display shows the current progress.

func (*Progress) Finish 

func (p *Progress) Finish()

Finish displays the final progress state.

func (*Progress) PrintSummary 

func (p *Progress) PrintSummary(results []RunResult)

PrintSummary prints a summary after all benchmarks complete.

func (*Progress) SetResumePoint 

func (p *Progress) SetResumePoint(completedRuns int, originalStartTime time.Time)

SetResumePoint sets the starting point when resuming.

func (*Progress) StartRun 

func (p *Progress) StartRun(benchmarkID string, run int, prompt string)

StartRun marks the start of a new benchmark run.

type Report 

type Report struct {
	Version    string            `json:"version"`
	Date       time.Time         `json:"date"`
	Config     string            `json:"config"` // Full config.yaml contents
	Summary    []AggregatedStats `json:"summary"`
	Failures   []FailureDetail   `json:"failures"`
	RawResults []RunResult       `json:"raw_results,omitempty"` // Optional detailed results
}

Report holds the complete benchmark report.

type ReportGenerator 

type ReportGenerator struct {
	// contains filtered or unexported fields
}

ReportGenerator generates benchmark reports.

func NewReportGenerator 

func NewReportGenerator(results []RunResult, benchmarks []BenchmarkDef, version, configYAML string, numRuns int) *ReportGenerator

NewReportGenerator creates a new report generator.

func (*ReportGenerator) GenerateMarkdown 

func (g *ReportGenerator) GenerateMarkdown() string

GenerateMarkdown generates a markdown report.

func (*ReportGenerator) WriteMarkdown 

func (g *ReportGenerator) WriteMarkdown(path string) error

WriteMarkdown writes the markdown report to a file.

type ResumeInfo 

type ResumeInfo struct {
	CompletedRuns map[string]map[int]bool // map[benchmarkID]map[runNumber]completed
	FirstStarted  time.Time               // When benchmarking started
	LastCompleted time.Time               // Last completion time
}

ResumeInfo contains information about where to resume benchmark runs.

func GetResumeInfo 

func GetResumeInfo(results []RunResult) *ResumeInfo

GetResumeInfo analyzes existing results to determine resume point.

func (*ResumeInfo) CountCompleted 

func (r *ResumeInfo) CountCompleted() int

CountCompleted returns the total number of completed runs.

func (*ResumeInfo) IsCompleted 

func (r *ResumeInfo) IsCompleted(benchmarkID string, run int) bool

IsCompleted checks if a specific benchmark run is already completed.

type RunResult 

type RunResult struct {
	BenchmarkID     string        `json:"benchmark_id" csv:"benchmark_id"`
	Run             int           `json:"run" csv:"run"`
	Success         bool          `json:"success" csv:"success"`
	LLMCalls        int           `json:"llm_calls" csv:"llm_calls"`
	Tokens          int           `json:"tokens" csv:"tokens"`
	PromptTokens    int           `json:"prompt_tokens" csv:"prompt_tokens"`
	GeneratedTokens int           `json:"generated_tokens" csv:"generated_tokens"`
	CachedTokens    int           `json:"cached_tokens" csv:"cached_tokens"`
	ContextUsed     int           `json:"context_used" csv:"context_used"`
	Cost            float64       `json:"cost" csv:"cost"`
	DurationMS      int64         `json:"duration_ms" csv:"duration_ms"`
	PromptMS        float64       `json:"prompt_ms" csv:"prompt_ms"`
	GenerationMS    float64       `json:"generation_ms" csv:"generation_ms"`
	ToolCalls       []ToolCallLog `json:"tool_calls" csv:"-"`
	Errors          []string      `json:"errors" csv:"-"`
	StartedAt       time.Time     `json:"started_at" csv:"started_at"`
	CompletedAt     time.Time     `json:"completed_at" csv:"completed_at"`

	// For CSV serialization
	ToolCallsJSON string `json:"-" csv:"tool_calls"`
	ErrorsJSON    string `json:"-" csv:"errors"`
}

RunResult captures metrics from a single benchmark run.

func LoadResults 

func LoadResults(path string) ([]RunResult, error)

LoadResults loads all benchmark results from a CSV file.

type Runner 

type Runner struct {
	// contains filtered or unexported fields
}

Runner orchestrates the execution of all benchmarks.

func NewRunner 

func NewRunner(executor *Executor, env *Environment, config *BenchmarkConfig, benchmarks []BenchmarkDef, writer io.Writer, csvPath string) *Runner

NewRunner creates a new benchmark runner.

func (*Runner) RunAll 

func (r *Runner) RunAll(ctx context.Context) ([]RunResult, error)

RunAll executes all benchmarks according to configuration. Press Esc to skip current benchmark, Ctrl+C to exit.

func (*Runner) RunSingle 

func (r *Runner) RunSingle(ctx context.Context, benchmarkID string) (*ExecuteResult, error)

RunSingle executes a single benchmark for debugging.

type SetupFile 

type SetupFile struct {
	File    string `yaml:"file"`
	Content string `yaml:"content"`
	Binary  []byte `yaml:"binary,omitempty"` // For binary file tests
	Dir     bool   `yaml:"dir,omitempty"`    // Create directory instead of file
}

SetupFile defines a file to create for benchmark setup.

type ToolCallLog 

type ToolCallLog struct {
	Tool string          `json:"tool"`
	Args json.RawMessage `json:"args,omitempty"`
}

ToolCallLog captures a single tool call made during a benchmark run.

func ExtractToolCalls 

func ExtractToolCalls(messages []json.RawMessage) []ToolCallLog

ExtractToolCalls extracts tool call logs from agent messages.

type ValidationCheck 

type ValidationCheck struct {
	Type     string `yaml:"type"`     // "file_contains", "file_equals", "file_exists", "file_not_exists", "file_line_count", "tool_called", "tool_called_with", "output_contains", "output_not_contains", "multi_tool_calls", "run_command"
	Target   string `yaml:"target"`   // File path or "output"
	Expected string `yaml:"expected"` // Expected value, pattern, or tool name
	Args     string `yaml:"args"`     // Expected args pattern (for tool_called_with)
	Command  string `yaml:"command"`  // Command to run (for run_command)
	Count    int    `yaml:"count"`    // Expected count (for file_line_count, multi_tool_calls)
	Line     int    `yaml:"line"`     // Specific line number (for file_line_equals)
	Negate   bool   `yaml:"negate"`   // Check for absence instead of presence
}

ValidationCheck defines a validation condition for benchmark success.

type ValidationResult 

type ValidationResult struct {
	Check   ValidationCheck
	Passed  bool
	Message string
}

ValidationResult holds the result of a single validation check.

type Validator 

type Validator struct {
	// contains filtered or unexported fields
}

Validator validates benchmark results against expected conditions.

func NewValidator 

func NewValidator(workspaceDir, output string, toolCalls []ToolCallLog) *Validator

NewValidator creates a new validator for a benchmark run.

func (*Validator) Validate 

func (v *Validator) Validate(checks []ValidationCheck) (bool, []string)

Validate runs all validation checks and returns results.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.