benchmark

package
v0.10.257 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 15, 2026 License: MIT Imports: 14 Imported by: 0

Documentation

Overview

Package benchmark parses and validates code-review benchmark suites.

Index

Constants

This section is empty.

Variables

View Source 
var (
	// ErrInvalid means a benchmark suite is malformed or violates the schema.
	ErrInvalid = errors.New("benchmark: invalid")
)

Functions

func Normalize 

func Normalize(suite *SuiteFile)

Normalize trims user-facing scalar fields in a suite document.

func ResolveSuitePath added in v0.10.228 

func ResolveSuitePath(baseDir, path string) string

ResolveSuitePath resolves a slash-separated path relative to a benchmark suite directory.

func Select 

func Select(suite SuiteFile, candidateIDs, caseIDs []string) ([]Candidate, []Case, error)

Select returns suite-order candidates and cases after optional ID filtering.

func Validate 

func Validate(suite SuiteFile, cfg config.File) error

Validate checks suite schema and profile/case compatibility without assuming a specific benchmark command mode.

func ValidateForRun added in v0.3.66 

func ValidateForRun(suite SuiteFile, cfg config.File) error

ValidateForRun applies the stricter full-pipeline requirements used by the current benchmark validate/doctor/run commands.

func ValidateForSelection added in v0.3.68 

func ValidateForSelection(suite SuiteFile, cfg config.File) error

ValidateForSelection applies the stricter selector-only requirements used by benchmark select.

Types

type Anchor 

type Anchor struct {
	ID    string `yaml:"id" json:"id"`
	File  string `yaml:"file" json:"file"`
	Side  string `yaml:"side" json:"side"`
	Lines []int  `yaml:"lines" json:"lines"`
}

Anchor is optional non-scoring placement metadata for a case.

type Candidate 

type Candidate struct {
	ID             string          `yaml:"id" json:"id"`
	Profile        string          `yaml:"profile" json:"profile"`
	Stages         CandidateStages `yaml:"stages" json:"stages"`
	MaxAgents      int             `yaml:"max_agents,omitempty" json:"max_agents,omitempty"`
	MaxConcurrency int             `yaml:"max_concurrency,omitempty" json:"max_concurrency,omitempty"`
	// contains filtered or unexported fields
}

Candidate is one review configuration to try against each selected case.

func (*Candidate) UnmarshalYAML 

func (c *Candidate) UnmarshalYAML(value *yaml.Node) error

UnmarshalYAML tracks candidate field presence for validation.

type CandidateStages added in v0.3.66 

type CandidateStages struct {
	Selection SelectionStage `yaml:"selection" json:"selection"`
	Reviewers ReviewerStage  `yaml:"reviewers,omitempty" json:"reviewers,omitempty"`
	Synthesis SelectionStage `yaml:"synthesis,omitempty" json:"synthesis,omitempty"`
	// contains filtered or unexported fields
}

CandidateStages groups the per-phase runtime recipes for a benchmark candidate.

func (*CandidateStages) UnmarshalYAML added in v0.3.66 

func (s *CandidateStages) UnmarshalYAML(value *yaml.Node) error

UnmarshalYAML tracks per-stage field presence for validation.

type Case 

type Case struct {
	ID              string   `yaml:"id" json:"id"`
	PR              string   `yaml:"pr" json:"pr"`
	ReviewBaseSHA   string   `yaml:"review_base_sha,omitempty" json:"review_base_sha,omitempty"`
	ReviewHeadSHA   string   `yaml:"review_head_sha,omitempty" json:"review_head_sha,omitempty"`
	ExpectedBaseSHA string   `yaml:"expected_base_sha,omitempty" json:"expected_base_sha,omitempty"`
	ExpectedHeadSHA string   `yaml:"expected_head_sha,omitempty" json:"expected_head_sha,omitempty"`
	Anchors         []Anchor `yaml:"anchors,omitempty" json:"anchors,omitempty"`
	// contains filtered or unexported fields
}

Case is one pull request to review during a benchmark.

func (*Case) UnmarshalYAML 

func (c *Case) UnmarshalYAML(value *yaml.Node) error

UnmarshalYAML tracks optional SHA field presence so explicitly blank values can be rejected after normalization.

type CostMetrics added in v0.3.55 

type CostMetrics struct {
	Available  bool    `json:"available"`
	Input      float64 `json:"input"`
	Output     float64 `json:"output"`
	CacheRead  float64 `json:"cache_read"`
	CacheWrite float64 `json:"cache_write"`
	Total      float64 `json:"total"`
}

CostMetrics records provider-reported cost estimates.

type PhaseMetrics added in v0.3.55 

type PhaseMetrics struct {
	Name        string       `json:"name"`
	Role        string       `json:"role,omitempty"`
	LogPath     string       `json:"log_path"`
	Provider    string       `json:"provider,omitempty"`
	Model       string       `json:"model,omitempty"`
	StopReason  string       `json:"stop_reason,omitempty"`
	LLMCalls    int          `json:"llm_calls"`
	Turns       int          `json:"turns"`
	ToolCalls   int          `json:"tool_calls"`
	ToolResults int          `json:"tool_results"`
	Tokens      TokenMetrics `json:"tokens"`
	Cost        CostMetrics  `json:"cost"`
}

PhaseMetrics summarizes one agent log.

type ReviewerStage added in v0.3.66 

type ReviewerStage struct {
	Model     string   `yaml:"model,omitempty" json:"model,omitempty"`
	ModelTier string   `yaml:"model_tier,omitempty" json:"model_tier,omitempty"`
	Effort    string   `yaml:"effort,omitempty" json:"effort,omitempty"`
	AgentDirs []string `yaml:"agent_dirs,omitempty" json:"agent_dirs,omitempty"`
	// contains filtered or unexported fields
}

ReviewerStage configures the benchmark reviewer execution phase.

func (*ReviewerStage) UnmarshalYAML added in v0.3.66 

func (s *ReviewerStage) UnmarshalYAML(value *yaml.Node) error

UnmarshalYAML accepts the canonical agent_dirs field and the draft agents_dir alias inside reviewer stage recipes, but rejects documents that use both.

type RunMetrics added in v0.3.55 

type RunMetrics struct {
	LLMCalls    int            `json:"llm_calls"`
	Turns       int            `json:"turns"`
	ToolCalls   int            `json:"tool_calls"`
	ToolResults int            `json:"tool_results"`
	Tokens      TokenMetrics   `json:"tokens"`
	Cost        CostMetrics    `json:"cost"`
	Phases      []PhaseMetrics `json:"phases,omitempty"`
}

RunMetrics summarizes usage and tool activity for one review run.

func ExtractRunMetrics added in v0.3.55 

func ExtractRunMetrics(artifactPath string) (RunMetrics, error)

ExtractRunMetrics reads agent JSONL logs from a review artifact directory.

func (*RunMetrics) Add added in v0.10.228 

func (m *RunMetrics) Add(other RunMetrics)

Add accumulates aggregate usage and activity from other into m.

func (RunMetrics) HasCostUsage added in v0.3.55 

func (m RunMetrics) HasCostUsage() bool

HasCostUsage reports whether provider cost telemetry was captured.

func (RunMetrics) HasData added in v0.3.55 

func (m RunMetrics) HasData() bool

HasData reports whether metrics contain provider usage or activity.

func (RunMetrics) HasTokenUsage added in v0.3.55 

func (m RunMetrics) HasTokenUsage() bool

HasTokenUsage reports whether provider token telemetry was captured.

type SelectionStage added in v0.3.66 

type SelectionStage struct {
	Model  string `yaml:"model,omitempty" json:"model,omitempty"`
	Effort string `yaml:"effort,omitempty" json:"effort,omitempty"`
	Prompt string `yaml:"prompt,omitempty" json:"prompt,omitempty"`
	// contains filtered or unexported fields
}

SelectionStage configures the benchmark selection/orchestration phase.

func (*SelectionStage) UnmarshalYAML added in v0.3.66 

func (s *SelectionStage) UnmarshalYAML(value *yaml.Node) error

UnmarshalYAML tracks selection stage field presence for validation.

type Suite 

type Suite struct {
	ID      string `yaml:"id" json:"id"`
	Name    string `yaml:"name,omitempty" json:"name,omitempty"`
	Version int    `yaml:"version,omitempty" json:"version,omitempty"`
}

Suite identifies a collection of benchmark cases and candidates.

type SuiteFile 

type SuiteFile struct {
	Path       string      `yaml:"-" json:"-"`
	Suite      Suite       `yaml:"suite" json:"suite"`
	Candidates []Candidate `yaml:"candidates" json:"candidates"`
	Cases      []Case      `yaml:"cases" json:"cases"`
}

SuiteFile is one benchmark suite document.

func Load 

func Load(data []byte) (SuiteFile, error)

Load parses a benchmark suite document.

func LoadFile 

func LoadFile(path string) (SuiteFile, error)

LoadFile reads and parses a benchmark suite file.

type TokenMetrics added in v0.3.55 

type TokenMetrics struct {
	Available   bool  `json:"available"`
	Input       int64 `json:"input"`
	Output      int64 `json:"output"`
	CacheRead   int64 `json:"cache_read"`
	CacheWrite  int64 `json:"cache_write"`
	TotalTokens int64 `json:"total_tokens"`
}

TokenMetrics records provider token usage.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.