benchmarks

package
v0.4.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 7, 2026 License: Apache-2.0 Imports: 23 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	// ErrBenchmarkSessionNotFound indicates that the requested benchmark session does not exist.
	ErrBenchmarkSessionNotFound = errors.New("benchmark session not found")
	// ErrBenchmarkRunNotFound indicates that the requested benchmark run does not exist.
	ErrBenchmarkRunNotFound = errors.New("benchmark run not found")
	// ErrBenchmarkComparisonNotFound indicates that no benchmark data matched the requested comparison scope.
	ErrBenchmarkComparisonNotFound = errors.New("benchmark comparison not found")
)

Benchmark read/query errors exposed to higher-level API and UI callers.

Functions

func FindRepoRoot 

func FindRepoRoot(start string) (string, error)

FindRepoRoot walks upward from start until it finds a repo root containing go.mod.

func ResolveDefaultOutputRoot 

func ResolveDefaultOutputRoot(start string) (string, error)

ResolveDefaultOutputRoot uses a local .centian benchmark workspace under the working directory instead of the repo test fixture tree.

func ValidateSuite 

func ValidateSuite(root string, suite *SuiteDefinition) error

ValidateSuite validates a suite and all referenced cases structurally.

Types

type AgentMetadata 

type AgentMetadata struct {
	Format               string                     `json:"format,omitempty"`
	LogPath              string                     `json:"logPath,omitempty"`
	SelectedModel        string                     `json:"selectedModel,omitempty"`
	SessionID            string                     `json:"sessionId,omitempty"`
	ThreadID             string                     `json:"threadId,omitempty"`
	NumTurns             *int                       `json:"numTurns,omitempty"`
	DurationMilliseconds *int64                     `json:"durationMilliseconds,omitempty"`
	TotalCostUSD         *float64                   `json:"totalCostUsd,omitempty"`
	Usage                AgentUsageMetadata         `json:"usage,omitempty"`
	ModelUsage           map[string]AgentModelUsage `json:"modelUsage,omitempty"`
}

AgentMetadata captures parsed usage/session data from one agent run log.

type AgentModelUsage 

type AgentModelUsage struct {
	InputTokens              *int64   `json:"inputTokens,omitempty"`
	OutputTokens             *int64   `json:"outputTokens,omitempty"`
	CacheReadInputTokens     *int64   `json:"cacheReadInputTokens,omitempty"`
	CacheCreationInputTokens *int64   `json:"cacheCreationInputTokens,omitempty"`
	CostUSD                  *float64 `json:"costUsd,omitempty"`
}

AgentModelUsage stores model-specific usage details when the agent exposes them.

type AgentScorecard 

type AgentScorecard struct {
	Agent                      string   `json:"agent"`
	Model                      string   `json:"model,omitempty"`
	Models                     []string `json:"models,omitempty"`
	RunCount                   int      `json:"runCount"`
	TotalTaskToolCalls         int      `json:"totalTaskToolCalls"`
	TotalDownstreamToolCalls   int      `json:"totalDownstreamToolCalls"`
	MedianTaskToolCalls        int      `json:"medianTaskToolCalls"`
	MedianDownstreamToolCalls  int      `json:"medianDownstreamToolCalls"`
	TotalCentianErrors         int      `json:"totalCentianErrors"`
	TotalDownstreamToolErrors  int      `json:"totalDownstreamToolErrors"`
	MedianCentianErrors        int      `json:"medianCentianErrors"`
	MedianDownstreamToolErrors int      `json:"medianDownstreamToolErrors"`
	MedianDurationMillis       int64    `json:"medianDurationMillis"`
	SuccessRate                float64  `json:"successRate"`
	FirstPassRate              float64  `json:"firstPassRate"`
}

AgentScorecard is the agent-level scorecard built from persisted benchmark runs.

type AgentUsageMetadata 

type AgentUsageMetadata struct {
	InputTokens              *int64 `json:"inputTokens,omitempty"`
	OutputTokens             *int64 `json:"outputTokens,omitempty"`
	CachedInputTokens        *int64 `json:"cachedInputTokens,omitempty"`
	CacheCreationInputTokens *int64 `json:"cacheCreationInputTokens,omitempty"`
	CacheReadInputTokens     *int64 `json:"cacheReadInputTokens,omitempty"`
}

AgentUsageMetadata stores normalized token metadata observed in agent logs.

type AggregateSummary 

type AggregateSummary struct {
	Key                             string  `json:"key"`
	SessionPath                     string  `json:"sessionPath,omitempty"`
	CaseID                          string  `json:"caseId,omitempty"`
	Agent                           string  `json:"agent,omitempty"`
	TemplateVariant                 string  `json:"templateVariant,omitempty"`
	RunCount                        int     `json:"runCount"`
	ScoredRunCount                  int     `json:"scoredRunCount"`
	SuccessRate                     float64 `json:"successRate"`
	FirstPassSuccessRate            float64 `json:"firstPassSuccessRate"`
	FinalVerificationPassRate       float64 `json:"finalVerificationPassRate"`
	InvariantViolationRate          float64 `json:"invariantViolationRate"`
	RestartFailTimeoutRate          float64 `json:"restartFailTimeoutRate"`
	MedianWallClockSeconds          float64 `json:"medianWallClockSeconds"`
	MedianTotalToolCalls            float64 `json:"medianTotalToolCalls"`
	MedianInputTokens               float64 `json:"medianInputTokens"`
	MedianOutputTokens              float64 `json:"medianOutputTokens"`
	MedianFailedTaskToolCalls       float64 `json:"medianFailedTaskToolCalls"`
	MedianFailedDownstreamToolCalls float64 `json:"medianFailedDownstreamToolCalls"`
	MedianEditedFilesCount          float64 `json:"medianEditedFilesCount"`
	TotalTaskToolCalls              int     `json:"totalTaskToolCalls"`
	TotalDownstreamToolCalls        int     `json:"totalDownstreamToolCalls"`
}

AggregateSummary stores aggregate comparison metrics for one grouping key.

type BenchmarkComparisonView 

type BenchmarkComparisonView struct {
	SuiteID      string                `json:"suiteId"`
	SuiteName    string                `json:"suiteName,omitempty"`
	TemplateID   string                `json:"templateId,omitempty"`
	TemplateName string                `json:"templateName,omitempty"`
	Filters      BenchmarkRunFilters   `json:"filters"`
	SessionCount int                   `json:"sessionCount"`
	RunCount     int                   `json:"runCount"`
	Sessions     []ComparisonSession   `json:"sessions"`
	Runs         []BenchmarkRunSummary `json:"runs"`
	Aggregates   ComparisonAggregates  `json:"aggregates"`
}

BenchmarkComparisonView is the read-side comparison result used by the API and UI.

type BenchmarkRunDetail 

type BenchmarkRunDetail struct {
	ScorecardID  string        `json:"scorecardId"`
	SessionID    string        `json:"sessionId"`
	SessionPath  string        `json:"sessionPath"`
	SuiteName    string        `json:"suiteName,omitempty"`
	TemplateName string        `json:"templateName,omitempty"`
	CaseName     string        `json:"caseName,omitempty"`
	Scored       bool          `json:"scored"`
	ScoreErrors  []string      `json:"scoreErrors,omitempty"`
	Scorecard    *RunScorecard `json:"scorecard,omitempty"`
}

BenchmarkRunDetail returns the full stored scorecard for one run.

type BenchmarkRunFilters 

type BenchmarkRunFilters struct {
	SuiteID         string `json:"suiteId,omitempty"`
	TemplateID      string `json:"templateId,omitempty"`
	SessionID       string `json:"sessionId,omitempty"`
	CaseID          string `json:"caseId,omitempty"`
	Agent           string `json:"agent,omitempty"`
	TemplateVariant string `json:"templateVariant,omitempty"`
}

BenchmarkRunFilters restrict benchmark read queries to one run subset.

type BenchmarkRunSummary 

type BenchmarkRunSummary struct {
	ScorecardID               string         `json:"scorecardId"`
	SessionID                 string         `json:"sessionId"`
	SessionPath               string         `json:"sessionPath"`
	SuiteID                   string         `json:"suiteId"`
	SuiteName                 string         `json:"suiteName,omitempty"`
	TemplateID                string         `json:"templateId"`
	TemplateName              string         `json:"templateName,omitempty"`
	CaseID                    string         `json:"caseId"`
	CaseName                  string         `json:"caseName,omitempty"`
	Agent                     string         `json:"agent"`
	SelectedModel             string         `json:"selectedModel,omitempty"`
	TemplateVariant           string         `json:"templateVariant"`
	Attempt                   int            `json:"attempt"`
	RawStatus                 string         `json:"rawStatus"`
	Scored                    bool           `json:"scored"`
	LinkedTaskRunIDs          []string       `json:"linkedTaskRunIds,omitempty"`
	CompletedSuccessfully     bool           `json:"completedSuccessfully"`
	FinalVerificationPassed   bool           `json:"finalVerificationPassed"`
	FirstPassSuccess          bool           `json:"firstPassSuccess"`
	InvariantViolation        bool           `json:"invariantViolation"`
	RestartOccurred           bool           `json:"restartOccurred"`
	FailOccurred              bool           `json:"failOccurred"`
	TimeoutOccurred           bool           `json:"timeoutOccurred"`
	WallClockSeconds          float64        `json:"wallClockSeconds"`
	TotalToolCalls            int            `json:"totalToolCalls"`
	TotalTaskToolCalls        int            `json:"totalTaskToolCalls"`
	TotalDownstreamToolCalls  int            `json:"totalDownstreamToolCalls"`
	InputTokens               *int64         `json:"inputTokens,omitempty"`
	OutputTokens              *int64         `json:"outputTokens,omitempty"`
	FailedTaskToolCalls       int            `json:"failedTaskToolCalls"`
	FailedDownstreamToolCalls int            `json:"failedDownstreamToolCalls"`
	EditedFilesCount          int            `json:"editedFilesCount"`
	AgentMetadata             *AgentMetadata `json:"agentMetadata,omitempty"`
	Warnings                  []string       `json:"warnings,omitempty"`
	Errors                    []string       `json:"errors,omitempty"`
}

BenchmarkRunSummary is the compact scorecard-backed row used in list views.

type BenchmarkSessionDetail 

type BenchmarkSessionDetail struct {
	SessionID          string                   `json:"sessionId"`
	SuiteID            string                   `json:"suiteId"`
	SuiteName          string                   `json:"suiteName,omitempty"`
	TemplateID         string                   `json:"templateId,omitempty"`
	TemplateName       string                   `json:"templateName,omitempty"`
	SessionPath        string                   `json:"sessionPath"`
	GeneratedAt        time.Time                `json:"generatedAt"`
	RunCount           int                      `json:"runCount"`
	ScoredRunCount     int                      `json:"scoredRunCount"`
	FailedToScoreCount int                      `json:"failedToScoreCount"`
	Agents             []string                 `json:"agents,omitempty"`
	CaseIDs            []string                 `json:"caseIds,omitempty"`
	CaseNames          []string                 `json:"caseNames,omitempty"`
	TemplateVariants   []string                 `json:"templateVariants,omitempty"`
	Aggregates         SessionSummaryAggregates `json:"aggregates"`
	Runs               []BenchmarkRunSummary    `json:"runs,omitempty"`
}

BenchmarkSessionDetail is the session-level API/UI read model.

type BenchmarkSuiteSummary 

type BenchmarkSuiteSummary struct {
	SuiteID           string    `json:"suiteId"`
	SuiteName         string    `json:"suiteName,omitempty"`
	TemplateID        string    `json:"templateId,omitempty"`
	TemplateName      string    `json:"templateName,omitempty"`
	LatestGeneratedAt time.Time `json:"latestGeneratedAt"`
	SessionCount      int       `json:"sessionCount"`
	RunCount          int       `json:"runCount"`
	Agents            []string  `json:"agents,omitempty"`
	CaseIDs           []string  `json:"caseIds,omitempty"`
	CaseNames         []string  `json:"caseNames,omitempty"`
	TemplateVariants  []string  `json:"templateVariants,omitempty"`
}

BenchmarkSuiteSummary is the suite-level API/UI read model.

type CaseConstraints 

type CaseConstraints struct {
	LockedPaths            []string `yaml:"lockedPaths"`
	AllowedAdditionalPaths []string `yaml:"allowedAdditionalPaths"`
}

CaseConstraints describes path-level restrictions or allowances for the case.

type CaseDefinition 

type CaseDefinition struct {
	Version      string            `yaml:"version"`
	Case         CaseMetadata      `yaml:"case"`
	PromptFile   string            `yaml:"promptFile"`
	Fixture      FixtureDefinition `yaml:"fixture"`
	Expectations CaseExpectations  `yaml:"expectations"`
	Constraints  CaseConstraints   `yaml:"constraints"`
}

CaseDefinition is the repo-tracked definition of one benchmark case.

func LoadCase 

func LoadCase(suiteRoot string, ref SuiteCaseRef) (*CaseDefinition, error)

LoadCase loads and validates one benchmark case referenced by the suite.

type CaseExpectations 

type CaseExpectations struct {
	SelectedCommand       string            `yaml:"selectedCommand"`
	RedSignal             SignalExpectation `yaml:"redSignal"`
	GreenConditionSummary string            `yaml:"greenConditionSummary"`
}

CaseExpectations defines the contract the benchmark runner later verifies.

type CaseMetadata 

type CaseMetadata struct {
	ID          string `yaml:"id"`
	Name        string `yaml:"name"`
	Description string `yaml:"description"`
}

CaseMetadata describes one benchmark case.

type ComparisonAggregates 

type ComparisonAggregates struct {
	BySession          []AggregateSummary `json:"bySession"`
	ByCase             []AggregateSummary `json:"byCase"`
	ByAgent            []AggregateSummary `json:"byAgent"`
	ByTemplateVariant  []AggregateSummary `json:"byTemplateVariant"`
	ByCaseAgentVariant []AggregateSummary `json:"byCaseAgentVariant"`
}

ComparisonAggregates contains grouped aggregates across scored sessions.

type ComparisonSession 

type ComparisonSession struct {
	SessionPath        string    `json:"sessionPath"`
	GeneratedAt        time.Time `json:"generatedAt"`
	RunCount           int       `json:"runCount"`
	ScoredRunCount     int       `json:"scoredRunCount"`
	FailedToScoreCount int       `json:"failedToScoreCount"`
}

ComparisonSession is the compact per-session row preserved in comparison output.

type FixtureDefinition 

type FixtureDefinition struct {
	SeedPath                 string `yaml:"seedPath"`
	ResetMode                string `yaml:"resetMode"`
	StartingRepoStateSummary string `yaml:"startingRepoStateSummary"`
}

FixtureDefinition describes the seeded starting state of a benchmark case.

type PromptDefinition 

type PromptDefinition = common.PromptDefinition

PromptDefinition is the user-style prompt for one benchmark case.

func LoadPrompt 

func LoadPrompt(caseRoot, promptFile string) (*PromptDefinition, error)

LoadPrompt loads and validates the prompt file for one benchmark case.

type QueryService 

type QueryService struct {
	// contains filtered or unexported fields
}

QueryService is the single live benchmark query/scoring service used by the API and UI.

func NewQueryService 

func NewQueryService(store benchmarkQueryStore) *QueryService

NewQueryService builds a live benchmark query service.

func (*QueryService) GetComparison 

func (s *QueryService) GetComparison(ctx context.Context, suiteID string, filters BenchmarkRunFilters) (*BenchmarkComparisonView, error)

GetComparison returns a live comparison view for one suite and filter set.

func (*QueryService) GetRun 

func (s *QueryService) GetRun(ctx context.Context, suiteID, scorecardID string) (*BenchmarkRunDetail, error)

GetRun returns one scored benchmark run detail by suite and run id.

func (*QueryService) GetSession 

func (s *QueryService) GetSession(ctx context.Context, suiteID, sessionID string) (*BenchmarkSessionDetail, error)

GetSession returns one scored benchmark session by suite and session id.

func (*QueryService) ListAgentScorecards 

func (s *QueryService) ListAgentScorecards(ctx context.Context) ([]AgentScorecard, error)

ListAgentScorecards returns agent-level scorecards for all persisted benchmark runs.

func (*QueryService) ListRuns 

func (s *QueryService) ListRuns(ctx context.Context, suiteID string, filters BenchmarkRunFilters) ([]BenchmarkRunSummary, error)

ListRuns returns scored benchmark runs for one suite, filtered by the provided dimensions.

func (*QueryService) ListSessions 

func (s *QueryService) ListSessions(ctx context.Context, suiteID string, filters BenchmarkRunFilters, includeRuns bool) ([]BenchmarkSessionDetail, error)

ListSessions returns suite sessions, optionally including their scored run summaries.

func (*QueryService) ListSuites 

func (s *QueryService) ListSuites(ctx context.Context, filters BenchmarkRunFilters) ([]BenchmarkSuiteSummary, error)

ListSuites returns benchmark suite summaries derived from persisted benchmark sessions and runs.

func (*QueryService) ListTemplateScorecards 

func (s *QueryService) ListTemplateScorecards(ctx context.Context) ([]TemplateScorecard, error)

ListTemplateScorecards returns template-level scorecards for all persisted benchmark runs.

type ReadService 

type ReadService struct {
	// contains filtered or unexported fields
}

ReadService exposes benchmark read models for the API/UI.

func NewReadService 

func NewReadService(store benchmarkQueryStore) *ReadService

NewReadService builds a read-only benchmark service on top of the shared live query layer.

func (*ReadService) GetComparison 

func (s *ReadService) GetComparison(ctx context.Context, suiteID string, filters BenchmarkRunFilters) (*BenchmarkComparisonView, error)

GetComparison derives a filtered comparison view from live benchmark run summaries.

func (*ReadService) GetRun 

func (s *ReadService) GetRun(ctx context.Context, suiteID, scorecardID string) (*BenchmarkRunDetail, error)

GetRun returns one live run detail.

func (*ReadService) GetSession 

func (s *ReadService) GetSession(ctx context.Context, suiteID, sessionID string) (*BenchmarkSessionDetail, error)

GetSession returns one session detail for a suite.

func (*ReadService) ListAgentScorecards 

func (s *ReadService) ListAgentScorecards(ctx context.Context) ([]AgentScorecard, error)

ListAgentScorecards returns agent scorecards over all persisted benchmark runs.

func (*ReadService) ListRuns 

func (s *ReadService) ListRuns(ctx context.Context, suiteID string, filters BenchmarkRunFilters) ([]BenchmarkRunSummary, error)

ListRuns returns live benchmark runs for one suite.

func (*ReadService) ListSessions 

func (s *ReadService) ListSessions(ctx context.Context, suiteID string, filters BenchmarkRunFilters) ([]BenchmarkSessionDetail, error)

ListSessions returns benchmark sessions for one suite.

func (*ReadService) ListSuites 

func (s *ReadService) ListSuites(ctx context.Context, filters BenchmarkRunFilters) ([]BenchmarkSuiteSummary, error)

ListSuites returns suite-level benchmark summaries.

func (*ReadService) ListTemplateScorecards 

func (s *ReadService) ListTemplateScorecards(ctx context.Context) ([]TemplateScorecard, error)

ListTemplateScorecards returns generic template scorecards over all persisted task runs.

type RunArtifactPaths 

type RunArtifactPaths struct {
	RunDir               string `json:"runDir"`
	ProjectDir           string `json:"projectDir"`
	LogsDir              string `json:"logsDir"`
	AgentDir             string `json:"agentDir"`
	ConfigPath           string `json:"configPath"`
	EventStoreMode       string `json:"eventStoreMode,omitempty"`
	EventStorePath       string `json:"eventStorePath"`
	RequestLogPath       string `json:"requestLogPath,omitempty"`
	SelectedTemplatePath string `json:"selectedTemplatePath,omitempty"`
}

RunArtifactPaths lists the raw artifacts captured for one run.

type RunManifest 

type RunManifest struct {
	SuiteID          string           `json:"suiteId"`
	SuiteName        string           `json:"suiteName,omitempty"`
	CaseID           string           `json:"caseId"`
	CaseName         string           `json:"caseName,omitempty"`
	TemplateID       string           `json:"templateId"`
	TemplateName     string           `json:"templateName,omitempty"`
	TemplateVariant  TemplateVariant  `json:"templateVariant"`
	AgentID          string           `json:"agentId"`
	Attempt          int              `json:"attempt"`
	SelectedModel    string           `json:"selectedModel,omitempty"`
	StartedAt        time.Time        `json:"startedAt"`
	EndedAt          time.Time        `json:"endedAt"`
	Status           string           `json:"status"`
	UIPublicURL      string           `json:"uiPublicUrl,omitempty"`
	CentianPID       int              `json:"centianPid,omitempty"`
	LinkedTaskRunIDs []string         `json:"linkedTaskRunIds,omitempty"`
	ArtifactPaths    RunArtifactPaths `json:"artifactPaths"`
	ErrorSummary     string           `json:"errorSummary,omitempty"`
}

RunManifest describes one concrete benchmark run.

type RunOptions 

type RunOptions struct {
	SuitePath         string
	CaseIDs           []string
	Executions        []agentrunner.AgentExecutionOptions
	Repeat            int
	TemplateVariants  []TemplateVariant
	OutputRoot        string
	Timeout           time.Duration
	CentianBinaryPath string
	CentianConfigPath string
	SessionLabel      string
	OnCentianReady    func(*RunManifest)
	AfterRun          func(*RunManifest) error
}

RunOptions configures one benchmark invocation.

type RunScorecard 

type RunScorecard struct {
	SuiteID          string              `json:"suiteId"`
	SuiteName        string              `json:"suiteName,omitempty"`
	CaseID           string              `json:"caseId"`
	CaseName         string              `json:"caseName,omitempty"`
	TemplateID       string              `json:"templateId"`
	TemplateName     string              `json:"templateName,omitempty"`
	TemplateVariant  string              `json:"templateVariant"`
	Agent            string              `json:"agent"`
	SelectedModel    string              `json:"selectedModel,omitempty"`
	Attempt          int                 `json:"attempt"`
	RunManifestPath  string              `json:"runManifestPath"`
	SessionPath      string              `json:"sessionPath"`
	EventStoreMode   string              `json:"eventStoreMode,omitempty"`
	EventStorePath   string              `json:"eventStorePath,omitempty"`
	RequestLogPath   string              `json:"requestLogPath,omitempty"`
	AgentStdoutPath  string              `json:"agentStdoutPath,omitempty"`
	AgentStderrPath  string              `json:"agentStderrPath,omitempty"`
	RawStatus        string              `json:"rawStatus"`
	LinkedTaskRunIDs []string            `json:"linkedTaskRunIds,omitempty"`
	Outcome          ScorecardOutcome    `json:"outcome"`
	Process          ScorecardProcess    `json:"process"`
	Efficiency       ScorecardEfficiency `json:"efficiency"`
	AgentMetadata    *AgentMetadata      `json:"agentMetadata,omitempty"`
	ScoreVersion     string              `json:"scoreVersion"`
	GeneratedAt      time.Time           `json:"generatedAt"`
	Warnings         []string            `json:"warnings,omitempty"`
	Errors           []string            `json:"errors,omitempty"`
}

RunScorecard stores derived metrics for one concrete benchmark run.

type RunSummaryRow 

type RunSummaryRow struct {
	SessionPath               string         `json:"sessionPath,omitempty"`
	CaseID                    string         `json:"caseId"`
	Agent                     string         `json:"agent"`
	SelectedModel             string         `json:"selectedModel,omitempty"`
	TemplateVariant           string         `json:"templateVariant"`
	Attempt                   int            `json:"attempt"`
	EventStoreMode            string         `json:"eventStoreMode,omitempty"`
	EventStorePath            string         `json:"eventStorePath,omitempty"`
	RawStatus                 string         `json:"rawStatus"`
	LinkedTaskRunIDs          []string       `json:"linkedTaskRunIds,omitempty"`
	Scored                    bool           `json:"scored"`
	CompletedSuccessfully     bool           `json:"completedSuccessfully"`
	FinalVerificationPassed   bool           `json:"finalVerificationPassed"`
	FirstPassSuccess          bool           `json:"firstPassSuccess"`
	InvariantViolation        bool           `json:"invariantViolation"`
	RestartOccurred           bool           `json:"restartOccurred"`
	FailOccurred              bool           `json:"failOccurred"`
	TimeoutOccurred           bool           `json:"timeoutOccurred"`
	WallClockSeconds          float64        `json:"wallClockSeconds"`
	TotalToolCalls            int            `json:"totalToolCalls"`
	TotalTaskToolCalls        int            `json:"totalTaskToolCalls"`
	TotalDownstreamToolCalls  int            `json:"totalDownstreamToolCalls"`
	InputTokens               *int64         `json:"inputTokens,omitempty"`
	OutputTokens              *int64         `json:"outputTokens,omitempty"`
	FailedTaskToolCalls       int            `json:"failedTaskToolCalls"`
	FailedDownstreamToolCalls int            `json:"failedDownstreamToolCalls"`
	EditedFilesCount          int            `json:"editedFilesCount"`
	AgentMetadata             *AgentMetadata `json:"agentMetadata,omitempty"`
	Warnings                  []string       `json:"warnings,omitempty"`
	Errors                    []string       `json:"errors,omitempty"`
}

RunSummaryRow is the compact per-run row used by the session summary.

type Runner 

type Runner struct {
	Now                  func() time.Time
	AllocatePort         func() (string, error)
	StartCentian         func(context.Context, StartCentianOptions) (*StartedCentian, error)
	LaunchAgent          func(context.Context, *agentrunner.RunOptions) (*agentrunner.RunResult, error)
	FetchTaskRuns        func(string) ([]persistence.TaskRunSummary, error)
	FetchTaskRunEvents   func(string, string) ([]persistence.TaskRunEvent, error)
	FindLatestRequestLog func(string) (string, error)
	PersistSession       func(context.Context, string, *persistence.BenchmarkSessionRecord) error
	PersistRun           func(context.Context, string, *persistence.BenchmarkRunRecord) error
	PersistRunScore      func(context.Context, string, *persistence.BenchmarkRunScoreRecord) error
}

Runner executes benchmark suites locally.

func NewRunner 

func NewRunner() *Runner

NewRunner returns a benchmark runner with the default local execution hooks.

func (*Runner) RunSuite 

func (r *Runner) RunSuite(ctx context.Context, opts *RunOptions) (*SessionManifest, error)

RunSuite executes the requested benchmark matrix and preserves raw artifacts for every run.

type ScorecardEfficiency 

type ScorecardEfficiency struct {
	WallClockSeconds     float64  `json:"wallClockSeconds"`
	TotalToolCalls       int      `json:"totalToolCalls"`
	InputTokens          *int64   `json:"inputTokens,omitempty"`
	OutputTokens         *int64   `json:"outputTokens,omitempty"`
	EditedFilesCount     int      `json:"editedFilesCount"`
	EditedFiles          []string `json:"editedFiles,omitempty"`
	ObservedCommandCalls int      `json:"observedCommandCalls"`
}

ScorecardEfficiency contains efficiency metrics for one run.

type ScorecardOutcome 

type ScorecardOutcome struct {
	CompletedSuccessfully   bool `json:"completedSuccessfully"`
	FinalVerificationPassed bool `json:"finalVerificationPassed"`
	FirstPassSuccess        bool `json:"firstPassSuccess"`
	RestartOccurred         bool `json:"restartOccurred"`
	FailOccurred            bool `json:"failOccurred"`
	TimeoutOccurred         bool `json:"timeoutOccurred"`
	InvariantViolation      bool `json:"invariantViolation"`
}

ScorecardOutcome contains outcome metrics for one run.

type ScorecardProcess 

type ScorecardProcess struct {
	FailedTaskToolCalls       int `json:"failedTaskToolCalls"`
	FailedDownstreamToolCalls int `json:"failedDownstreamToolCalls"`
	TotalTaskToolCalls        int `json:"totalTaskToolCalls"`
	TotalDownstreamToolCalls  int `json:"totalDownstreamToolCalls"`
	RestartCount              int `json:"restartCount"`
	FailCount                 int `json:"failCount"`
	TimeoutCount              int `json:"timeoutCount"`
}

ScorecardProcess contains process metrics for one run.

type SessionManifest 

type SessionManifest struct {
	SuiteID          string                    `json:"suiteId"`
	SuiteName        string                    `json:"suiteName,omitempty"`
	TemplateID       string                    `json:"templateId"`
	TemplateName     string                    `json:"templateName,omitempty"`
	SuitePath        string                    `json:"suitePath"`
	InvocationDir    string                    `json:"invocationDir"`
	OutputRoot       string                    `json:"outputRoot"`
	StartedAt        time.Time                 `json:"startedAt"`
	EndedAt          time.Time                 `json:"endedAt"`
	Status           string                    `json:"status"`
	Repeat           int                       `json:"repeat"`
	Agents           []string                  `json:"agents"`
	CaseIDs          []string                  `json:"caseIds"`
	TemplateVariants []TemplateVariant         `json:"templateVariants"`
	Runs             []SessionRunManifestEntry `json:"runs"`
}

SessionManifest describes one benchmark invocation and all concrete runs.

type SessionRunManifestEntry 

type SessionRunManifestEntry struct {
	CaseID          string `json:"caseId"`
	CaseName        string `json:"caseName,omitempty"`
	AgentID         string `json:"agentId"`
	TemplateVariant string `json:"templateVariant"`
	Attempt         int    `json:"attempt"`
	RelativeRunDir  string `json:"relativeRunDir"`
	Status          string `json:"status"`
	ErrorSummary    string `json:"errorSummary,omitempty"`
}

SessionRunManifestEntry summarizes one concrete run under a session.

type SessionSummary 

type SessionSummary struct {
	ScoreVersion       string                   `json:"scoreVersion"`
	SessionPath        string                   `json:"sessionPath"`
	SuiteID            string                   `json:"suiteId"`
	GeneratedAt        time.Time                `json:"generatedAt"`
	RunCount           int                      `json:"runCount"`
	ScoredRunCount     int                      `json:"scoredRunCount"`
	FailedToScoreCount int                      `json:"failedToScoreCount"`
	Runs               []RunSummaryRow          `json:"runs"`
	Aggregates         SessionSummaryAggregates `json:"aggregates"`
}

SessionSummary stores comparison-friendly derived metrics for one scored session.

type SessionSummaryAggregates 

type SessionSummaryAggregates struct {
	ByCase             []AggregateSummary `json:"byCase"`
	ByAgent            []AggregateSummary `json:"byAgent"`
	ByTemplateVariant  []AggregateSummary `json:"byTemplateVariant"`
	ByCaseAgentVariant []AggregateSummary `json:"byCaseAgentVariant"`
}

SessionSummaryAggregates contains grouped score summaries.

type SignalExpectation 

type SignalExpectation struct {
	Type  string `yaml:"type"`
	Value string `yaml:"value"`
}

SignalExpectation describes the expected red baseline signal.

type StartCentianOptions 

type StartCentianOptions struct {
	BinaryPath string
	ConfigPath string
	ProjectDir string
	LogsDir    string
	BaseURL    string
	MCPURL     string
}

StartCentianOptions configures one benchmark-local Centian child process.

type StartedCentian 

type StartedCentian struct {
	PID  int
	Stop func() error
}

StartedCentian describes a running child Centian process.

type SuiteCaseRef 

type SuiteCaseRef struct {
	ID   string `yaml:"id"`
	Path string `yaml:"path"`
}

SuiteCaseRef points to one benchmark case directory relative to the suite root.

type SuiteDefinition 

type SuiteDefinition struct {
	Version string         `yaml:"version"`
	Suite   SuiteMetadata  `yaml:"suite"`
	Cases   []SuiteCaseRef `yaml:"cases"`
}

SuiteDefinition is the repo-tracked top-level benchmark suite definition.

func LoadSuite 

func LoadSuite(root string) (*SuiteDefinition, error)

LoadSuite loads and validates a benchmark suite rooted at the given directory.

type SuiteMetadata 

type SuiteMetadata struct {
	ID                       string `yaml:"id"`
	Name                     string `yaml:"name"`
	Description              string `yaml:"description"`
	TemplateID               string `yaml:"templateId"`
	LocalOnly                bool   `yaml:"localOnly"`
	ScoringSchemaVersion     string `yaml:"scoringSchemaVersion"`
	ExecutionProtocolVersion string `yaml:"executionProtocolVersion"`
}

SuiteMetadata describes one benchmark suite.

type TemplateScorecard 

type TemplateScorecard struct {
	TemplateKey                string  `json:"templateKey"`
	TemplateID                 string  `json:"templateId"`
	TemplateName               string  `json:"templateName,omitempty"`
	RunCount                   int     `json:"runCount"`
	TotalTaskToolCalls         int     `json:"totalTaskToolCalls"`
	TotalDownstreamToolCalls   int     `json:"totalDownstreamToolCalls"`
	MedianTaskToolCalls        int     `json:"medianTaskToolCalls"`
	MedianDownstreamToolCalls  int     `json:"medianDownstreamToolCalls"`
	TotalCentianErrors         int     `json:"totalCentianErrors"`
	TotalDownstreamToolErrors  int     `json:"totalDownstreamToolErrors"`
	MedianCentianErrors        int     `json:"medianCentianErrors"`
	MedianDownstreamToolErrors int     `json:"medianDownstreamToolErrors"`
	MedianDurationMillis       int64   `json:"medianDurationMillis"`
	SuccessRate                float64 `json:"successRate"`
	FirstPassRate              float64 `json:"firstPassRate"`
}

TemplateScorecard is the generic template-level scorecard built from all persisted task runs.

type TemplateVariant 

type TemplateVariant struct {
	Name      string `json:"name"`
	SourceDir string `json:"sourceDir"`
}

TemplateVariant identifies one template tree used for a benchmark run variant.

func ResolveDefaultTemplateVariants 

func ResolveDefaultTemplateVariants(start string) ([]TemplateVariant, error)

ResolveDefaultTemplateVariants uses task-templates/integrated under the working directory as the implicit current variant, with a repo-root fallback for backwards compatibility.

func ResolveTemplateVariantsFromCentianConfig 

func ResolveTemplateVariantsFromCentianConfig(configPath string) ([]TemplateVariant, error)

ResolveTemplateVariantsFromCentianConfig uses a custom Centian config as the implicit current template variant when it declares a concrete templates path.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.