eval_analysis

package

Go to main page

Versions in this module

v0

v0.14.3

May 2, 2026

Changes in this version

type ExecutorLangStats

+ APIErrors int

v0.14.2

Apr 26, 2026

Changes in this version

type BenchmarkResult

+ ModelFamily string

type DashboardJSON

+ Harnesses map[string]interface{}

type ModelReliability

+ LanguageAPIErrors map[string]int

type TagAggregate

+ LanguageBreakdown map[string]*TagLangStats

+ type TagLangStats struct

+ Pass int

+ Rate float64

+ Total int

type TierAggregate

+ LanguageStats map[string]*TierLanguageStats

type TierHistoryPoint

+ LanguageStats map[string]*TierLanguageStats

+ type TierLanguageStats struct

+ APIErrors int

+ AvgCostUSD float64

+ Pass int

+ RepairDelta float64

+ Runs int

+ SuccessRate float64

v0.14.1

Apr 21, 2026

Changes in this version

+ func DetectRefusal(code, stderr, stdout string) bool

+ func ExportBenchmarkJSON(matrix *PerformanceMatrix, history []*Baseline, results []*BenchmarkResult, ...) (string, error)

+ func ExportCSV(results []*BenchmarkResult) (string, error)

+ func ExportDocusaurusMDX(matrix *PerformanceMatrix, history []*Baseline) string

+ func ExportHTML(matrix *PerformanceMatrix, history []*Baseline) (string, error)

+ func ExportMarkdown(matrix *PerformanceMatrix, history []*Baseline) string

+ func FormatComparison(report *ComparisonReport, useColor bool) string

+ func FormatJSON(matrix *PerformanceMatrix) (string, error)

+ func FormatJSONL(results []*BenchmarkResult) (string, error)

+ func FormatMatrix(matrix *PerformanceMatrix, useColor bool) string

+ func GenerateReport(matrix *PerformanceMatrix, history []*Baseline) string

+ func ListBaselines() ([]string, error)

+ func LoadBenchmarkTags(dir string) map[string][]string

+ func RefusalPatterns() []string

+ type AILANGWin struct

+ ID string

+ Model string

+ type AILANGWinsReport struct

+ Patterns []string

+ PerBenchmark map[string]int

+ Wins []AILANGWin

+ func DetectAILANGOnlyWins(results []*BenchmarkResult) *AILANGWinsReport

+ type Aggregates struct

+ AvgDurationMs float64

+ FinalSuccess float64

+ RepairSuccessRate float64

+ RepairUsed int

+ TotalCostUSD float64

+ TotalTokens int

+ ZeroShotSuccess float64

+ type Baseline struct

+ FailCount int

+ GitBranch string

+ GitCommit string

+ Languages string

+ MatrixFile string

+ Model string

+ Results []*BenchmarkResult

+ SelfRepair bool

+ SuccessCount int

+ Timestamp time.Time

+ TotalBenchmarks int

+ Version string

+ func GetLatestBaseline() (*Baseline, error)

+ func LoadBaseline(dir string) (*Baseline, error)

+ func LoadBaselineByVersion(version string) (*Baseline, error)

+ func LoadBaselineFromChain(chainID string) (*Baseline, error)

+ type BenchmarkChange struct

+ BaselineError string

+ BaselineStatus bool

+ ID string

+ Lang string

+ Model string

+ NewError string

+ NewStatus bool

+ func FindImprovements(baseline, new []*BenchmarkResult) ([]*BenchmarkChange, error)

+ func FindRegressions(baseline, new []*BenchmarkResult) ([]*BenchmarkChange, error)

+ type BenchmarkResult struct

+ AgentTranscript string

+ AgentTurns int

+ BinaryHash string

+ Caps []string

+ Code string

+ CompileMs int64

+ CompileOk bool

+ Condition string

+ CostUSD float64

+ DurationMs int64

+ ErrCode string

+ ErrorCategory string

+ EvalMode string

+ ExecuteMs int64

+ Executor string

+ FirstAttemptOk bool

+ ID string

+ InputTokens int

+ Lang string

+ Model string

+ OutputTokens int

+ PromptVersion string

+ RefusalDetected bool

+ RepairOk bool

+ RepairTokensIn int

+ RepairTokensOut int

+ RepairUsed bool

+ RuntimeOk bool

+ Seed int64

+ Stderr string

+ StdlibHash string

+ Stdout string

+ StdoutOk bool

+ Timestamp time.Time

+ TotalTokens int

+ func Filter(results []*BenchmarkResult, filter ResultFilter) []*BenchmarkResult

+ func LoadLatestResultsPerModel() ([]*BenchmarkResult, map[string]string, error)

+ func LoadResult(path string) (*BenchmarkResult, error)

+ func LoadResults(dir string) ([]*BenchmarkResult, error)

+ func LoadResultsFromChain(chainID string) ([]*BenchmarkResult, error)

+ func LoadResultsFromLatestEvalChain() ([]*BenchmarkResult, string, error)

+ func (r *BenchmarkResult) ToSummaryEntry() *SummaryEntry

+ type BenchmarkRun struct

+ FirstAttemptOk bool

+ RepairUsed bool

+ Success bool

+ Tokens int

+ type BenchmarkStats struct

+ AvgTokens float64

+ Languages []string

+ SuccessRate float64

+ TotalRuns int

+ type ComparisonReport struct

+ Baseline *Baseline

+ BaselineLabel string

+ BaselineSuccessRate float64

+ Broken []*BenchmarkChange

+ Fixed []*BenchmarkChange

+ New *Baseline

+ NewBenchmarks []*BenchmarkResult

+ NewLabel string

+ NewSuccessRate float64

+ Removed []*BenchmarkResult

+ StillFailing []*BenchmarkResult

+ StillPassing []*BenchmarkResult

+ SuccessRateDelta float64

+ TotalBaselineBench int

+ TotalNewBench int

+ func Compare(baseline, new []*BenchmarkResult, baselineLabel, newLabel string) (*ComparisonReport, error)

+ func CompareBaselines(baseline, new *Baseline) (*ComparisonReport, error)

+ func (r *ComparisonReport) HasImprovements() bool

+ func (r *ComparisonReport) HasRegressions() bool

+ func (r *ComparisonReport) ImprovementPercent() float64

+ func (r *ComparisonReport) NetChange() int

+ func (r *ComparisonReport) Summary() string

+ type DashboardJSON struct

+ AgentModels map[string]interface{}

+ Aggregates map[string]interface{}

+ Benchmarks map[string]interface{}

+ Events []SuiteEvent

+ Executors map[string]interface{}

+ History []HistoryEntry

+ Languages map[string]interface{}

+ Models map[string]interface{}

+ Tags map[string]*TagAggregate

+ Tiers map[string]TierAggregate

+ Timestamp string

+ TotalRuns int

+ Version string

+ func (d *DashboardJSON) Validate() error

+ type ErrorCodeStats struct

+ Code string

+ Count int

+ RepairSuccess float64

+ type ExecutorLangStats struct

+ Cost float64

+ Runs int

+ Success int

+ Tokens int

+ Turns int

+ type ExportFormat string

+ const FormatCSV

+ const FormatHTML

+ const FormatMarkdown

+ type HistoryEntry struct

+ LanguageStats map[string]interface{}

+ Languages string

+ ModelStats map[string]interface{}

+ SuccessCount int

+ SuccessRate float64

+ Tiers map[string]*TierHistoryPoint

+ Timestamp string

+ TotalRuns int

+ Version string

+ type LanguageStats struct

+ AvgTokens float64

+ SuccessRate float64

+ TotalRuns int

+ type ModelDimensionStats struct

+ APIErrorCount int

+ AvgTokens float64

+ RefusalCount int

+ SuccessRate float64

+ TotalRuns int

+ type ModelReliability struct

+ AILANGAPIError int

+ APIErrorCount int

+ APIErrorRate float64

+ PythonAPIError int

+ RefusalCount int

+ RefusalRate float64

+ TotalRuns int

+ type ModelStats struct

+ Aggregates Aggregates

+ BaselineVersion string

+ Benchmarks map[string]*BenchmarkRun

+ Languages map[string]*LanguageStats

+ TotalRuns int

+ type PerformanceMatrix struct

+ Aggregates Aggregates

+ Benchmarks map[string]*BenchmarkStats

+ ErrorCodes []*ErrorCodeStats

+ Languages map[string]*LanguageStats

+ Models map[string]*ModelStats

+ PromptVersions map[string]*PromptStats

+ Timestamp time.Time

+ TotalRuns int

+ Version string

+ func GenerateMatrix(results []*BenchmarkResult, version string) (*PerformanceMatrix, error)

+ func GenerateMatrixWithBaselines(results []*BenchmarkResult, version string, modelBaselines map[string]string) (*PerformanceMatrix, error)

+ type PromptStats struct

+ AvgTokens float64

+ FinalSuccess float64

+ TotalRuns int

+ ZeroShotSuccess float64

+ type ReliabilityCounts struct

+ APIErrorCount int

+ APIErrorRate float64

+ PerModel map[string]*ModelReliability

+ RefusalCount int

+ RefusalRate float64

+ type ResultFilter struct

+ Benchmark string

+ FailuresOnly bool

+ Lang string

+ Model string

+ SuccessOnly bool

+ type SaturatedBenchmark struct

+ BaselinesSeen []string

+ ID string

+ TotalCells int

+ func DetectSaturation(baselines []*Baseline, minBaselines int) []*SaturatedBenchmark

+ type SuiteEvent struct

+ AffectsTiers []string

+ Color string

+ Kind string

+ Label string

+ Version string

+ func LoadSuiteEvents(path string) ([]SuiteEvent, error)

+ type SummaryEntry struct

+ AgentTurns int

+ CompileOk bool

+ Condition string

+ CostUSD float64

+ DurationMs int64

+ ErrCode string

+ ErrorCategory string

+ EvalMode string

+ Executor string

+ FirstAttemptOk bool

+ ID string

+ InputTokens int

+ Lang string

+ Model string

+ OutputTokens int

+ PromptVersion string

+ RepairOk bool

+ RepairUsed bool

+ RuntimeOk bool

+ Seed int64

+ Stderr string

+ StdoutOk bool

+ Timestamp string

+ TotalTokens int

+ func (s *SummaryEntry) MarshalJSON() ([]byte, error)

+ type TagAggregate struct

+ AILANGPass int

+ AILANGTotal int

+ BenchmarkCount int

+ Delta float64

+ ModelStats map[string]map[string]*ModelDimensionStats

+ PythonPass int

+ PythonTotal int

+ Tag string

+ type TagReport struct

+ Aggregates map[string]*TagAggregate

+ Tags []string

+ func GroupByTags(results []*BenchmarkResult, tags map[string][]string) *TagReport

+ type TierAggregate struct

+ AILANGAPIError int

+ AILANGAvgCostUSD float64

+ AILANGRepairDelta float64

+ AILANGRuns int

+ AILANGSuccessRate float64

+ APIErrorCount int

+ BenchmarkCount int

+ ModelStats map[string]map[string]*ModelDimensionStats

+ PythonAPIError int

+ PythonAvgCostUSD float64

+ PythonRepairDelta float64

+ PythonRuns int

+ PythonSuccessRate float64

+ RefusalCount int

+ TotalRuns int

+ type TierHistoryPoint struct

+ AILANGRuns int

+ AILANGSuccessRate float64

+ BenchmarkCount int

+ ModelStats map[string]map[string]*ModelDimensionStats

+ PythonRuns int

+ PythonSuccessRate float64

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL