Help us shape the future of Go! Take the 2026 Go Developer Survey

benchmarks

package
v0.4.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 7, 2026 License: Apache-2.0 Imports: 23 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	// ErrBenchmarkSessionNotFound indicates that the requested benchmark session does not exist.
	ErrBenchmarkSessionNotFound = errors.New("benchmark session not found")
	// ErrBenchmarkRunNotFound indicates that the requested benchmark run does not exist.
	ErrBenchmarkRunNotFound = errors.New("benchmark run not found")
	// ErrBenchmarkComparisonNotFound indicates that no benchmark data matched the requested comparison scope.
	ErrBenchmarkComparisonNotFound = errors.New("benchmark comparison not found")
)

Benchmark read/query errors exposed to higher-level API and UI callers.

Functions

func FindRepoRoot 

func FindRepoRoot(start string) (string, error)

FindRepoRoot walks upward from start until it finds a repo root containing go.mod.

func ResolveDefaultOutputRoot 

func ResolveDefaultOutputRoot(start string) (string, error)

ResolveDefaultOutputRoot uses a local .centian benchmark workspace under the working directory instead of the repo test fixture tree.

func ValidateSuite 

func ValidateSuite(root string, suite *SuiteDefinition) error

ValidateSuite validates a suite and all referenced cases structurally.

Types

type AgentMetadata 

type AgentMetadata struct {
	Format               string                     `json:"format,omitempty"`
	LogPath              string                     `json:"logPath,omitempty"`
	SelectedModel        string                     `json:"selectedModel,omitempty"`
	SessionID            string                     `json:"sessionId,omitempty"`
	ThreadID             string                     `json:"threadId,omitempty"`
	NumTurns             *int                       `json:"numTurns,omitempty"`
	DurationMilliseconds *int64                     `json:"durationMilliseconds,omitempty"`
	TotalCostUSD         *float64                   `json:"totalCostUsd,omitempty"`
	Usage                AgentUsageMetadata         `json:"usage,omitempty"`
	ModelUsage           map[string]AgentModelUsage `json:"modelUsage,omitempty"`
}

AgentMetadata captures parsed usage/session data from one agent run log.

type AgentModelUsage 

type AgentModelUsage struct {
	InputTokens              *int64   `json:"inputTokens,omitempty"`
	OutputTokens             *int64   `json:"outputTokens,omitempty"`
	CacheReadInputTokens     *int64   `json:"cacheReadInputTokens,omitempty"`
	CacheCreationInputTokens *int64   `json:"cacheCreationInputTokens,omitempty"`
	CostUSD                  *float64 `json:"costUsd,omitempty"`
}

AgentModelUsage stores model-specific usage details when the agent exposes them.

type AgentScorecard 

type AgentScorecard struct {
	Agent                      string   `json:"agent"`
	Model                      string   `json:"model,omitempty"`
	Models                     []string `json:"models,omitempty"`
	RunCount                   int      `json:"runCount"`
	TotalTaskToolCalls         int      `json:"totalTaskToolCalls"`
	TotalDownstreamToolCalls   int      `json:"totalDownstreamToolCalls"`
	MedianTaskToolCalls        int      `json:"medianTaskToolCalls"`
	MedianDownstreamToolCalls  int      `json:"medianDownstreamToolCalls"`
	TotalCentianErrors         int      `json:"totalCentianErrors"`
	TotalDownstreamToolErrors  int      `json:"totalDownstreamToolErrors"`
	MedianCentianErrors        int      `json:"medianCentianErrors"`
	MedianDownstreamToolErrors int      `json:"medianDownstreamToolErrors"`
	MedianDurationMillis       int64    `json:"medianDurationMillis"`
	SuccessRate                float64  `json:"successRate"`
	FirstPassRate              float64  `json:"firstPassRate"`
}

AgentScorecard is the agent-level scorecard built from persisted benchmark runs.

type AgentUsageMetadata 

type AgentUsageMetadata struct {
	InputTokens              *int64 `json:"inputTokens,omitempty"`
	OutputTokens             *int64 `json:"outputTokens,omitempty"`
	CachedInputTokens        *int64 `json:"cachedInputTokens,omitempty"`
	CacheCreationInputTokens *int64 `json:"cacheCreationInputTokens,omitempty"`
	CacheReadInputTokens     *int64 `json:"cacheReadInputTokens,omitempty"`
}

AgentUsageMetadata stores normalized token metadata observed in agent logs.

type AggregateSummary 

type AggregateSummary struct {
	Key                             string  `json:"key"`
	SessionPath                     string  `json:"sessionPath,omitempty"`
	CaseID                          string  `json:"caseId,omitempty"`
	Agent                           string  `json:"agent,omitempty"`
	TemplateVariant                 string  `json:"templateVariant,omitempty"`
	RunCount                        int     `json:"runCount"`
	ScoredRunCount                  int     `json:"scoredRunCount"`
	SuccessRate                     float64 `json:"successRate"`
	FirstPassSuccessRate            float64 `json:"firstPassSuccessRate"`
	FinalVerificationPassRate       float64 `json:"finalVerificationPassRate"`
	InvariantViolationRate          float64 `json:"invariantViolationRate"`
	RestartFailTimeoutRate          float64 `json:"restartFailTimeoutRate"`
	MedianWallClockSeconds          float64 `json:"medianWallClockSeconds"`
	MedianTotalToolCalls            float64 `json:"medianTotalToolCalls"`
	MedianInputTokens               float64 `json:"medianInputTokens"`
	MedianOutputTokens              float64 `json:"medianOutputTokens"`
	MedianFailedTaskToolCalls       float64 `json:"medianFailedTaskToolCalls"`
	MedianFailedDownstreamToolCalls float64 `json:"medianFailedDownstreamToolCalls"`
	MedianEditedFilesCount          float64 `json:"medianEditedFilesCount"`
	TotalTaskToolCalls              int     `json:"totalTaskToolCalls"`
	TotalDownstreamToolCalls        int     `json:"totalDownstreamToolCalls"`
}

AggregateSummary stores aggregate comparison metrics for one grouping key.

type BenchmarkComparisonView 

type BenchmarkComparisonView struct {
	SuiteID      string                `json:"suiteId"`
	SuiteName    string                `json:"suiteName,omitempty"`
	TemplateID   string                `json:"templateId,omitempty"`
	TemplateName string                `json:"templateName,omitempty"`
	Filters      BenchmarkRunFilters   `json:"filters"`
	SessionCount int                   `json:"sessionCount"`
	RunCount     int                   `json:"runCount"`
	Sessions     []ComparisonSession   `json:"sessions"`
	Runs         []BenchmarkRunSummary `json:"runs"`
	Aggregates   ComparisonAggregates  `json:"aggregates"`
}

BenchmarkComparisonView is the read-side comparison result used by the API and UI.

type BenchmarkRunDetail 

type BenchmarkRunDetail struct {
	ScorecardID  string        `json:"scorecardId"`
	SessionID    string        `json:"sessionId"`
	SessionPath  string        `json:"sessionPath"`
	SuiteName    string        `json:"suiteName,omitempty"`
	TemplateName string        `json:"templateName,omitempty"`
	CaseName     string        `json:"caseName,omitempty"`
	Scored       bool          `json:"scored"`
	ScoreErrors  []string      `json:"scoreErrors,omitempty"`
	Scorecard    *RunScorecard `json:"scorecard,omitempty"`
}

BenchmarkRunDetail returns the full stored scorecard for one run.

type BenchmarkRunFilters 

type BenchmarkRunFilters struct {
	SuiteID         string `json:"suiteId,omitempty"`
	TemplateID      string `json:"templateId,omitempty"`
	SessionID       string `json:"sessionId,omitempty"`
	CaseID          string `json:"caseId,omitempty"`
	Agent           string `json:"agent,omitempty"`
	TemplateVariant string `json:"templateVariant,omitempty"`
}

BenchmarkRunFilters restrict benchmark read queries to one run subset.

type BenchmarkRunSummary 

type BenchmarkRunSummary struct {
	ScorecardID               string         `json:"scorecardId"`
	SessionID                 string         `json:"sessionId"`
	SessionPath               string         `json:"sessionPath"`
	SuiteID                   string         `json:"suiteId"`
	SuiteName                 string         `json:"suiteName,omitempty"`
	TemplateID                string         `json:"templateId"`
	TemplateName              string         `json:"templateName,omitempty"`
	CaseID                    string         `json:"caseId"`
	CaseName                  string         `json:"caseName,omitempty"`
	Agent                     string         `json:"agent"`
	SelectedModel             string         `json:"selectedModel,omitempty"`
	TemplateVariant           string         `json:"templateVariant"`
	Attempt                   int            `json:"attempt"`
	RawStatus                 string         `json:"rawStatus"`
	Scored                    bool           `json:"scored"`
	LinkedTaskRunIDs          []string       `json:"linkedTaskRunIds,omitempty"`
	CompletedSuccessfully     bool           `json:"completedSuccessfully"`
	FinalVerificationPassed   bool           `json:"finalVerificationPassed"`
	FirstPassSuccess          bool           `json:"firstPassSuccess"`
	InvariantViolation        bool           `json:"invariantViolation"`
	RestartOccurred           bool           `json:"restartOccurred"`
	FailOccurred              bool           `json:"failOccurred"`
	TimeoutOccurred           bool           `json:"timeoutOccurred"`
	WallClockSeconds          float64        `json:"wallClockSeconds"`
	TotalToolCalls            int            `json:"totalToolCalls"`
	TotalTaskToolCalls        int            `json:"totalTaskToolCalls"`
	TotalDownstreamToolCalls  int            `json:"totalDownstreamToolCalls"`
	InputTokens               *int64         `json:"inputTokens,omitempty"`
	OutputTokens              *int64         `json:"outputTokens,omitempty"`
	FailedTaskToolCalls       int            `json:"failedTaskToolCalls"`
	FailedDownstreamToolCalls int            `json:"failedDownstreamToolCalls"`
	EditedFilesCount          int            `json:"editedFilesCount"`
	AgentMetadata             *AgentMetadata `json:"agentMetadata,omitempty"`
	Warnings                  []string       `json:"warnings,omitempty"`
	Errors                    []string       `json:"errors,omitempty"`
}

BenchmarkRunSummary is the compact scorecard-backed row used in list views.

type BenchmarkSessionDetail 

type BenchmarkSessionDetail struct {
	SessionID          string                   `json:"sessionId"`
	SuiteID            string                   `json:"suiteId"`
	SuiteName          string                   `json:"suiteName,omitempty"`
	TemplateID         string                   `json:"templateId,omitempty"`
	TemplateName       string                   `json:"templateName,omitempty"`
	SessionPath        string                   `json:"sessionPath"`
	GeneratedAt        time.Time                `json:"generatedAt"`
	RunCount           int                      `json:"runCount"`
	ScoredRunCount     int                      `json:"scoredRunCount"`
	FailedToScoreCount int                      `json:"failedToScoreCount"`
	Agents             []string                 `json:"agents,omitempty"`
	CaseIDs            []string                 `json:"caseIds,omitempty"`
	CaseNames          []string                 `json:"caseNames,omitempty"`
	TemplateVariants   []string                 `json:"templateVariants,omitempty"`
	Aggregates         SessionSummaryAggregates `json:"aggregates"`
	Runs               []BenchmarkRunSummary    `json:"runs,omitempty"`
}

BenchmarkSessionDetail is the session-level API/UI read model.

type BenchmarkSuiteSummary 

type BenchmarkSuiteSummary struct {
	SuiteID           string    `json:"suiteId"`
	SuiteName         string    `json:"suiteName,omitempty"`
	TemplateID        string    `json:"templateId,omitempty"`
	TemplateName      string    `json:"templateName,omitempty"`
	LatestGeneratedAt time.Time `json:"latestGeneratedAt"`
	SessionCount      int       `json:"sessionCount"`
	RunCount          int       `json:"runCount"`
	Agents            []string  `json:"agents,omitempty"`
	CaseIDs           []string  `json:"caseIds,omitempty"`
	CaseNames         []string  `json:"caseNames,omitempty"`
	TemplateVariants  []string  `json:"templateVariants,omitempty"`
}

BenchmarkSuiteSummary is the suite-level API/UI read model.

type CaseConstraints 

type CaseConstraints struct {
	LockedPaths            []string `yaml:"lockedPaths"`
	AllowedAdditionalPaths []string `yaml:"allowedAdditionalPaths"`
}

CaseConstraints describes path-level restrictions or allowances for the case.

type CaseDefinition 

type CaseDefinition struct {
	Version      string            `yaml:"version"`
	Case         CaseMetadata      `yaml:"case"`
	PromptFile   string            `yaml:"promptFile"`
	Fixture      FixtureDefinition `yaml:"fixture"`
	Expectations CaseExpectations  `yaml:"expectations"`
	Constraints  CaseConstraints   `yaml:"constraints"`
}

CaseDefinition is the repo-tracked definition of one benchmark case.

func LoadCase 

func LoadCase(suiteRoot string, ref SuiteCaseRef) (*CaseDefinition, error)

LoadCase loads and validates one benchmark case referenced by the suite.

type CaseExpectations 

type CaseExpectations struct {
	SelectedCommand       string            `yaml:"selectedCommand"`
	RedSignal             SignalExpectation `yaml:"redSignal"`
	GreenConditionSummary string            `yaml:"greenConditionSummary"`
}

CaseExpectations defines the contract the benchmark runner later verifies.

type CaseMetadata 

type CaseMetadata struct {
	ID          string `yaml:"id"`
	Name        string `yaml:"name"`
	Description string `yaml:"description"`
}

CaseMetadata describes one benchmark case.

type ComparisonAggregates 

type ComparisonAggregates struct {
	BySession          []AggregateSummary `json:"bySession"`
	ByCase             []AggregateSummary `json:"byCase"`
	ByAgent            []AggregateSummary `json:"byAgent"`
	ByTemplateVariant  []AggregateSummary `json:"byTemplateVariant"`
	ByCaseAgentVariant []AggregateSummary `json:"byCaseAgentVariant"`
}

ComparisonAggregates contains grouped aggregates across scored sessions.

type ComparisonSession 

type ComparisonSession struct {
	SessionPath        string    `json:"sessionPath"`
	GeneratedAt        time.Time `json:"generatedAt"`
	RunCount           int       `json:"runCount"`
	ScoredRunCount     int       `json:"scoredRunCount"`
	FailedToScoreCount int       `json:"failedToScoreCount"`
}

ComparisonSession is the compact per-session row preserved in comparison output.

type FixtureDefinition 

type FixtureDefinition struct {
	SeedPath                 string `yaml:"seedPath"`
	ResetMode                string `yaml:"resetMode"`
	StartingRepoStateSummary string `yaml:"startingRepoStateSummary"`
}

FixtureDefinition describes the seeded starting state of a benchmark case.

type PromptDefinition 

type PromptDefinition = common.PromptDefinition

PromptDefinition is the user-style prompt for one benchmark case.

func LoadPrompt 

func LoadPrompt(caseRoot, promptFile string) (*PromptDefinition, error)

LoadPrompt loads and validates the prompt file for one benchmark case.

type QueryService 

type QueryService struct {
	// contains filtered or unexported fields
}

QueryService is the single live benchmark query/scoring service used by the API and UI.

func NewQueryService 

func NewQueryService(store benchmarkQueryStore) *QueryService

NewQueryService builds a live benchmark query service.

func (*QueryService) GetComparison 

func (s *QueryService) GetComparison(ctx context.Context, suiteID string, filters BenchmarkRunFilters) (*BenchmarkComparisonView, error)

GetComparison returns a live comparison view for one suite and filter set.

func (*QueryService) GetRun 

func (s *QueryService) GetRun(ctx context.Context, suiteID, scorecardID string) (*BenchmarkRunDetail, error)

GetRun returns one scored benchmark run detail by suite and run id.

func (*QueryService) GetSession 

func (s *QueryService) GetSession(ctx context.Context, suiteID, sessionID string) (*BenchmarkSessionDetail, error)

GetSession returns one scored benchmark session by suite and session id.

func (*QueryService) ListAgentScorecards 

func (s *QueryService) ListAgentScorecards(ctx context.Context) ([]AgentScorecard, error)

ListAgentScorecards returns agent-level scorecards for all persisted benchmark runs.

func (*QueryService) ListRuns 

func (s *QueryService) ListRuns(ctx context.Context, suiteID string, filters BenchmarkRunFilters) ([]BenchmarkRunSummary, error)

ListRuns returns scored benchmark runs for one suite, filtered by the provided dimensions.

func (*QueryService) ListSessions 

func (s *QueryService) ListSessions(ctx context.Context, suiteID string, filters BenchmarkRunFilters, includeRuns bool) ([]BenchmarkSessionDetail, error)

ListSessions returns suite sessions, optionally including their scored run summaries.

func (*QueryService) ListSuites 

func (s *QueryService) ListSuites(ctx context.Context, filters BenchmarkRunFilters) ([]BenchmarkSuiteSummary, error)

ListSuites returns benchmark suite summaries derived from persisted benchmark sessions and runs.

func (*QueryService) ListTemplateScorecards 

func (s *QueryService) ListTemplateScorecards(ctx context.Context) ([]TemplateScorecard, error)

ListTemplateScorecards returns template-level scorecards for all persisted benchmark runs.

type ReadService 

type ReadService struct {
	// contains filtered or unexported fields
}

ReadService exposes benchmark read models for the API/UI.

func NewReadService 

func NewReadService(store benchmarkQueryStore) *ReadService

NewReadService builds a read-only benchmark service on top of the shared live query layer.

func (*ReadService) GetComparison 

func (s *ReadService) GetComparison(ctx context.Context, suiteID string, filters BenchmarkRunFilters) (*BenchmarkComparisonView, error)

GetComparison derives a filtered comparison view from live benchmark run summaries.

func (*ReadService) GetRun 

func (s *ReadService) GetRun(ctx context.Context, suiteID, scorecardID string) (*BenchmarkRunDetail, error)

GetRun returns one live run detail.

func (*ReadService) GetSession 

func (s *ReadService) GetSession(ctx context.Context, suiteID, sessionID string) (*BenchmarkSessionDetail, error)

GetSession returns one session detail for a suite.

func (*ReadService) ListAgentScorecards 

func (s *ReadService) ListAgentScorecards(ctx context.Context) ([]AgentScorecard, error)

ListAgentScorecards returns agent scorecards over all persisted benchmark runs.

func (*ReadService) ListRuns 

func (s *ReadService) ListRuns(ctx context.Context, suiteID string, filters BenchmarkRunFilters) ([]BenchmarkRunSummary, error)

ListRuns returns live benchmark runs for one suite.

func (*ReadService) ListSessions 

func (s *ReadService) ListSessions(ctx context.Context, suiteID string, filters BenchmarkRunFilters) ([]BenchmarkSessionDetail, error)

ListSessions returns benchmark sessions for one suite.

func (*ReadService) ListSuites 

func (s *ReadService) ListSuites(ctx context.Context, filters BenchmarkRunFilters) ([]BenchmarkSuiteSummary, error)

ListSuites returns suite-level benchmark summaries.

func (*ReadService) ListTemplateScorecards 

func (s *ReadService) ListTemplateScorecards(ctx context.Context) ([]TemplateScorecard, error)

ListTemplateScorecards returns generic template scorecards over all persisted task runs.

type RunArtifactPaths 

type RunArtifactPaths struct {
	RunDir               string `json:"runDir"`
	ProjectDir           string `json:"projectDir"`
	LogsDir              string `json:"logsDir"`
	AgentDir             string `json:"agentDir"`
	ConfigPath           string `json:"configPath"`
	EventStoreMode       string `json:"eventStoreMode,omitempty"`
	EventStorePath       string `json:"eventStorePath"`
	RequestLogPath       string `json:"requestLogPath,omitempty"`
	SelectedTemplatePath string `json:"selectedTemplatePath,omitempty"`
}

RunArtifactPaths lists the raw artifacts captured for one run.

type RunManifest 

type RunManifest struct {
	SuiteID          string           `json:"suiteId"`
	SuiteName        string           `json:"suiteName,omitempty"`
	CaseID           string           `json:"caseId"`
	CaseName         string           `json:"caseName,omitempty"`
	TemplateID       string           `json:"templateId"`
	TemplateName     string           `json:"templateName,omitempty"`
	TemplateVariant  TemplateVariant  `json:"templateVariant"`
	AgentID          string           `json:"agentId"`
	Attempt          int              `json:"attempt"`
	SelectedModel    string           `json:"selectedModel,omitempty"`
	StartedAt        time.Time        `json:"startedAt"`
	EndedAt          time.Time        `json:"endedAt"`
	Status           string           `json:"status"`
	UIPublicURL      string           `json:"uiPublicUrl,omitempty"`
	CentianPID       int              `json:"centianPid,omitempty"`
	LinkedTaskRunIDs []string         `json:"linkedTaskRunIds,omitempty"`
	ArtifactPaths    RunArtifactPaths `json:"artifactPaths"`
	ErrorSummary     string           `json:"errorSummary,omitempty"`
}

RunManifest describes one concrete benchmark run.

type RunOptions 

type RunOptions struct {
	SuitePath         string
	CaseIDs           []string
	Executions        []agentrunner.AgentExecutionOptions
	Repeat            int
	TemplateVariants  []TemplateVariant
	OutputRoot        string
	Timeout           time.Duration
	CentianBinaryPath string
	CentianConfigPath string
	SessionLabel      string
	OnCentianReady    func(*RunManifest)
	AfterRun          func(*RunManifest) error
}

RunOptions configures one benchmark invocation.

type RunScorecard 

type RunScorecard struct {
	SuiteID          string              `json:"suiteId"`
	SuiteName        string              `json:"suiteName,omitempty"`
	CaseID           string              `json:"caseId"`
	CaseName         string              `json:"caseName,omitempty"`
	TemplateID       string              `json:"templateId"`
	TemplateName     string              `json:"templateName,omitempty"`
	TemplateVariant  string              `json:"templateVariant"`
	Agent            string              `json:"agent"`
	SelectedModel    string              `json:"selectedModel,omitempty"`
	Attempt          int                 `json:"attempt"`
	RunManifestPath  string              `json:"runManifestPath"`
	SessionPath      string              `json:"sessionPath"`
	EventStoreMode   string              `json:"eventStoreMode,omitempty"`
	EventStorePath   string              `json:"eventStorePath,omitempty"`
	RequestLogPath   string              `json:"requestLogPath,omitempty"`
	AgentStdoutPath  string              `json:"agentStdoutPath,omitempty"`
	AgentStderrPath  string              `json:"agentStderrPath,omitempty"`
	RawStatus        string              `json:"rawStatus"`
	LinkedTaskRunIDs []string            `json:"linkedTaskRunIds,omitempty"`
	Outcome          ScorecardOutcome    `json:"outcome"`
	Process          ScorecardProcess    `json:"process"`
	Efficiency       ScorecardEfficiency `json:"efficiency"`
	AgentMetadata    *AgentMetadata      `json:"agentMetadata,omitempty"`
	ScoreVersion     string              `json:"scoreVersion"`
	GeneratedAt      time.Time           `json:"generatedAt"`
	Warnings         []string            `json:"warnings,omitempty"`
	Errors           []string            `json:"errors,omitempty"`
}

RunScorecard stores derived metrics for one concrete benchmark run.

type RunSummaryRow 

type RunSummaryRow struct {
	SessionPath               string         `json:"sessionPath,omitempty"`
	CaseID                    string         `json:"caseId"`
	Agent                     string         `json:"agent"`
	SelectedModel             string         `json:"selectedModel,omitempty"`
	TemplateVariant           string         `json:"templateVariant"`
	Attempt                   int            `json:"attempt"`
	EventStoreMode            string         `json:"eventStoreMode,omitempty"`
	EventStorePath            string         `json:"eventStorePath,omitempty"`
	RawStatus                 string         `json:"rawStatus"`
	LinkedTaskRunIDs          []string       `json:"linkedTaskRunIds,omitempty"`
	Scored                    bool           `json:"scored"`
	CompletedSuccessfully     bool           `json:"completedSuccessfully"`
	FinalVerificationPassed   bool           `json:"finalVerificationPassed"`
	FirstPassSuccess          bool           `json:"firstPassSuccess"`
	InvariantViolation        bool           `json:"invariantViolation"`
	RestartOccurred           bool           `json:"restartOccurred"`
	FailOccurred              bool           `json:"failOccurred"`
	TimeoutOccurred           bool           `json:"timeoutOccurred"`
	WallClockSeconds          float64        `json:"wallClockSeconds"`
	TotalToolCalls            int            `json:"totalToolCalls"`
	TotalTaskToolCalls        int            `json:"totalTaskToolCalls"`
	TotalDownstreamToolCalls  int            `json:"totalDownstreamToolCalls"`
	InputTokens               *int64         `json:"inputTokens,omitempty"`
	OutputTokens              *int64         `json:"outputTokens,omitempty"`
	FailedTaskToolCalls       int            `json:"failedTaskToolCalls"`
	FailedDownstreamToolCalls int            `json:"failedDownstreamToolCalls"`
	EditedFilesCount          int            `json:"editedFilesCount"`
	AgentMetadata             *AgentMetadata `json:"agentMetadata,omitempty"`
	Warnings                  []string       `json:"warnings,omitempty"`
	Errors                    []string       `json:"errors,omitempty"`
}

RunSummaryRow is the compact per-run row used by the session summary.

type Runner 

type Runner struct {
	Now                  func() time.Time
	AllocatePort         func() (string, error)
	StartCentian         func(context.Context, StartCentianOptions) (*StartedCentian, error)
	LaunchAgent          func(context.Context, *agentrunner.RunOptions) (*agentrunner.RunResult, error)
	FetchTaskRuns        func(string) ([]persistence.TaskRunSummary, error)
	FetchTaskRunEvents   func(string, string) ([]persistence.TaskRunEvent, error)
	FindLatestRequestLog func(string) (string, error)
	PersistSession       func(context.Context, string, *persistence.BenchmarkSessionRecord) error
	PersistRun           func(context.Context, string, *persistence.BenchmarkRunRecord) error
	PersistRunScore      func(context.Context, string, *persistence.BenchmarkRunScoreRecord) error
}

Runner executes benchmark suites locally.

func NewRunner 

func NewRunner() *Runner

NewRunner returns a benchmark runner with the default local execution hooks.

func (*Runner) RunSuite 

func (r *Runner) RunSuite(ctx context.Context, opts *RunOptions) (*SessionManifest, error)

RunSuite executes the requested benchmark matrix and preserves raw artifacts for every run.

type ScorecardEfficiency 

type ScorecardEfficiency struct {
	WallClockSeconds     float64  `json:"wallClockSeconds"`
	TotalToolCalls       int      `json:"totalToolCalls"`
	InputTokens          *int64   `json:"inputTokens,omitempty"`
	OutputTokens         *int64   `json:"outputTokens,omitempty"`
	EditedFilesCount     int      `json:"editedFilesCount"`
	EditedFiles          []string `json:"editedFiles,omitempty"`
	ObservedCommandCalls int      `json:"observedCommandCalls"`
}

ScorecardEfficiency contains efficiency metrics for one run.

type ScorecardOutcome 

type ScorecardOutcome struct {
	CompletedSuccessfully   bool `json:"completedSuccessfully"`
	FinalVerificationPassed bool `json:"finalVerificationPassed"`
	FirstPassSuccess        bool `json:"firstPassSuccess"`
	RestartOccurred         bool `json:"restartOccurred"`
	FailOccurred            bool `json:"failOccurred"`
	TimeoutOccurred         bool `json:"timeoutOccurred"`
	InvariantViolation      bool `json:"invariantViolation"`
}

ScorecardOutcome contains outcome metrics for one run.

type ScorecardProcess 

type ScorecardProcess struct {
	FailedTaskToolCalls       int `json:"failedTaskToolCalls"`
	FailedDownstreamToolCalls int `json:"failedDownstreamToolCalls"`
	TotalTaskToolCalls        int `json:"totalTaskToolCalls"`
	TotalDownstreamToolCalls  int `json:"totalDownstreamToolCalls"`
	RestartCount              int `json:"restartCount"`
	FailCount                 int `json:"failCount"`
	TimeoutCount              int `json:"timeoutCount"`
}

ScorecardProcess contains process metrics for one run.

type SessionManifest 

type SessionManifest struct {
	SuiteID          string                    `json:"suiteId"`
	SuiteName        string                    `json:"suiteName,omitempty"`
	TemplateID       string                    `json:"templateId"`
	TemplateName     string                    `json:"templateName,omitempty"`
	SuitePath        string                    `json:"suitePath"`
	InvocationDir    string                    `json:"invocationDir"`
	OutputRoot       string                    `json:"outputRoot"`
	StartedAt        time.Time                 `json:"startedAt"`
	EndedAt          time.Time                 `json:"endedAt"`
	Status           string                    `json:"status"`
	Repeat           int                       `json:"repeat"`
	Agents           []string                  `json:"agents"`
	CaseIDs          []string                  `json:"caseIds"`
	TemplateVariants []TemplateVariant         `json:"templateVariants"`
	Runs             []SessionRunManifestEntry `json:"runs"`
}

SessionManifest describes one benchmark invocation and all concrete runs.

type SessionRunManifestEntry 

type SessionRunManifestEntry struct {
	CaseID          string `json:"caseId"`
	CaseName        string `json:"caseName,omitempty"`
	AgentID         string `json:"agentId"`
	TemplateVariant string `json:"templateVariant"`
	Attempt         int    `json:"attempt"`
	RelativeRunDir  string `json:"relativeRunDir"`
	Status          string `json:"status"`
	ErrorSummary    string `json:"errorSummary,omitempty"`
}

SessionRunManifestEntry summarizes one concrete run under a session.

type SessionSummary 

type SessionSummary struct {
	ScoreVersion       string                   `json:"scoreVersion"`
	SessionPath        string                   `json:"sessionPath"`
	SuiteID            string                   `json:"suiteId"`
	GeneratedAt        time.Time                `json:"generatedAt"`
	RunCount           int                      `json:"runCount"`
	ScoredRunCount     int                      `json:"scoredRunCount"`
	FailedToScoreCount int                      `json:"failedToScoreCount"`
	Runs               []RunSummaryRow          `json:"runs"`
	Aggregates         SessionSummaryAggregates `json:"aggregates"`
}

SessionSummary stores comparison-friendly derived metrics for one scored session.

type SessionSummaryAggregates 

type SessionSummaryAggregates struct {
	ByCase             []AggregateSummary `json:"byCase"`
	ByAgent            []AggregateSummary `json:"byAgent"`
	ByTemplateVariant  []AggregateSummary `json:"byTemplateVariant"`
	ByCaseAgentVariant []AggregateSummary `json:"byCaseAgentVariant"`
}

SessionSummaryAggregates contains grouped score summaries.

type SignalExpectation 

type SignalExpectation struct {
	Type  string `yaml:"type"`
	Value string `yaml:"value"`
}

SignalExpectation describes the expected red baseline signal.

type StartCentianOptions 

type StartCentianOptions struct {
	BinaryPath string
	ConfigPath string
	ProjectDir string
	LogsDir    string
	BaseURL    string
	MCPURL     string
}

StartCentianOptions configures one benchmark-local Centian child process.

type StartedCentian 

type StartedCentian struct {
	PID  int
	Stop func() error
}

StartedCentian describes a running child Centian process.

type SuiteCaseRef 

type SuiteCaseRef struct {
	ID   string `yaml:"id"`
	Path string `yaml:"path"`
}

SuiteCaseRef points to one benchmark case directory relative to the suite root.

type SuiteDefinition 

type SuiteDefinition struct {
	Version string         `yaml:"version"`
	Suite   SuiteMetadata  `yaml:"suite"`
	Cases   []SuiteCaseRef `yaml:"cases"`
}

SuiteDefinition is the repo-tracked top-level benchmark suite definition.

func LoadSuite 

func LoadSuite(root string) (*SuiteDefinition, error)

LoadSuite loads and validates a benchmark suite rooted at the given directory.

type SuiteMetadata 

type SuiteMetadata struct {
	ID                       string `yaml:"id"`
	Name                     string `yaml:"name"`
	Description              string `yaml:"description"`
	TemplateID               string `yaml:"templateId"`
	LocalOnly                bool   `yaml:"localOnly"`
	ScoringSchemaVersion     string `yaml:"scoringSchemaVersion"`
	ExecutionProtocolVersion string `yaml:"executionProtocolVersion"`
}

SuiteMetadata describes one benchmark suite.

type TemplateScorecard 

type TemplateScorecard struct {
	TemplateKey                string  `json:"templateKey"`
	TemplateID                 string  `json:"templateId"`
	TemplateName               string  `json:"templateName,omitempty"`
	RunCount                   int     `json:"runCount"`
	TotalTaskToolCalls         int     `json:"totalTaskToolCalls"`
	TotalDownstreamToolCalls   int     `json:"totalDownstreamToolCalls"`
	MedianTaskToolCalls        int     `json:"medianTaskToolCalls"`
	MedianDownstreamToolCalls  int     `json:"medianDownstreamToolCalls"`
	TotalCentianErrors         int     `json:"totalCentianErrors"`
	TotalDownstreamToolErrors  int     `json:"totalDownstreamToolErrors"`
	MedianCentianErrors        int     `json:"medianCentianErrors"`
	MedianDownstreamToolErrors int     `json:"medianDownstreamToolErrors"`
	MedianDurationMillis       int64   `json:"medianDurationMillis"`
	SuccessRate                float64 `json:"successRate"`
	FirstPassRate              float64 `json:"firstPassRate"`
}

TemplateScorecard is the generic template-level scorecard built from all persisted task runs.

type TemplateVariant 

type TemplateVariant struct {
	Name      string `json:"name"`
	SourceDir string `json:"sourceDir"`
}

TemplateVariant identifies one template tree used for a benchmark run variant.

func ResolveDefaultTemplateVariants 

func ResolveDefaultTemplateVariants(start string) ([]TemplateVariant, error)

ResolveDefaultTemplateVariants uses task-templates/integrated under the working directory as the implicit current variant, with a repo-root fallback for backwards compatibility.

func ResolveTemplateVariantsFromCentianConfig 

func ResolveTemplateVariantsFromCentianConfig(configPath string) ([]TemplateVariant, error)

ResolveTemplateVariantsFromCentianConfig uses a custom Centian config as the implicit current template variant when it declares a concrete templates path.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.