Versions in this module Expand all Collapse all v1 v1.6.2 Sep 9, 2026 v1.6.1 Sep 8, 2026 v1.6.0 Aug 31, 2026 Changes in this version + const TierCore + const TierDeep + const TierSmoke + var DefaultTierCostCaps = map[string]float64 + var DefaultTierThresholds = map[string]float64 + var KnownConnections = map[string]bool + var KnownGraderNames = map[string]bool + func CompileReplicationMap(path string) (map[string]any, error) + func CompileReplicationYAML(body []byte) (map[string]any, error) + func EnsureEcomParquet() error + func EnsureFreshSlingBin(bin string) (string, error) + func FindSlingBin() (string, error) + func FormatSummary(s SuiteSummary) string + func GenerateEcomParquet(dir string) error + func HostHasClaudeAuth() bool + func HostHasCodexAuth() bool + func HostHasGrokAuth() bool + func HostHasOpenCode2Auth() bool + func LinkHostAuth(home, arm string) error + func LoadFixtureRegistry(path string) (map[string]FixtureDef, error) + func NewRunID() string + func PreflightConns(bin string, names []string, env []string) map[string]bool + func RunTimed(name string, args []string, dir string, envv []string, timeout time.Duration) (stdout string, err error) + func RunTimedStdout(name string, args []string, dir string, envv []string, timeout time.Duration) (stdout string, err error) + func SeedHome(home, fixture string, noSkills bool) error + func SmokeFlipFail(flips []Flip, gating map[string]bool) bool + func TrialPasses(graders []GraderResult) bool + func ValidateCase(c Case, fixtures map[string]FixtureDef) []string + func WeightedScore(graders []GraderResult) float64 + func YAMLMap(path string) (map[string]any, error) + type ArmAvail struct + Binary string + Name string + Skip string + func CheckArm(name string) ArmAvail + type ArmSummary struct + Cases int + CostUSD float64 + Flips string + PassAt1 float64 + PassHatK float64 + type Case struct + Arms []string + Artifact string + BudgetUSD float64 + CaseDir string + Connections []string + EditPath string + Env map[string]string + Fixtures []string + Gating *bool + Graders CaseGraders + ID string + Intention string + Mock MockHints + Path string + ResetSchemas []string + SeedFiles map[string]string + Setup []any + Tags []string + Task string + Teardown []any + Tier string + Timeout string + Trials int + func LoadCases(dir string) ([]Case, error) + func SelectCases(all []Case, f Flags) []Case + func (c Case) DefaultArms() []string + func (c Case) ExpectedPath() string + func (c Case) HasTag(tag string) bool + func (c Case) IsGating() bool + func (c Case) IsNegative() bool + func (c Case) IsSmoke() bool + func (c Case) MutantPaths() []string + func (c Case) TaskID() string + func (c Case) TimeoutDur() time.Duration + func (c Case) TrialCount(override int) int + func (c Case) UsedConnections() []string + type CaseGraders struct + Judge []string + Optional []GraderSpec + Required []GraderSpec + type CaseMeta struct + Gating bool + Tier string + type CaseMetrics struct + Arm string + Case string + Gating bool + MeanCost float64 + MeanDur float64 + MeanScore float64 + PassAtK bool + PassHatK bool + Passed int + SkipReason string + Tier string + Timeouts int + Trials int + func AggregateCase(caseID, arm string, trials []TrialResult, gating bool) CaseMetrics + func AggregateCaseMeta(caseID, arm string, trials []TrialResult, meta CaseMeta) CaseMetrics + type CompareResult struct + Got any + Message string + Pass bool + Path string + Want any + func CompareCompiled(actual, expected map[string]any, policy MatchPolicy) []CompareResult + func CompareCompiledFiles(actualPath, expectedPath string, policy MatchPolicy) ([]CompareResult, error) + type FixtureDef struct + Connection string + Provision []GraderSpec + ReadyCheck ReadyCheck + Requires []string + type FixtureServer struct + URL string + func StartFixtureServer() (*FixtureServer, error) + func (fs *FixtureServer) Close() + type Flags struct + Arms []string + Baseline string + Cases []string + MaxSuiteUSD float64 + Parallel int + ResetFixtures bool + RetryFailed bool + Tags []string + Tiers []string + Trials int + func DefaultFlags() Flags + func ParseFlags(args []string) Flags + type Flip struct + Arm string + Case string + Delta float64 + From string + To string + func PairedDiff(current, baseline []TrialResult, gating map[string]bool) []Flip + type GradeContext struct + APIPageCap int + APITimeout time.Duration + Artifact string + CaseDir string + ConnDown map[string]bool + Env []string + FixtureBaseURL string + FixtureToken string + SkipExecute bool + SlingBin string + Transcript string + WorkDir string + type GraderResult struct + Critique string + Detail string + Judge bool + Name string + Optional bool + Pass bool + Skip bool + func RunGraders(ctx GradeContext, required, optional []GraderSpec) []GraderResult + func RunJudge(questions []string, intention, artifact, transcript string, ...) []GraderResult + type GraderSpec map[string]any + type MatchPolicy struct + MustMatch []string + MustNotExist []string + type MockHints struct + MutantTranscript string + Transcript string + type Provisioner struct + Bin string + Env []string + Logf func(string, ...any) + Registry map[string]FixtureDef + func NewProvisioner(bin string, env []string, logf func(string, ...any)) (*Provisioner, error) + func (p *Provisioner) Ensure(names []string) error + func (p *Provisioner) Ready(name string) bool + func (p *Provisioner) Reset() error + func (p *Provisioner) Skipped(name string) string + type ReadyCheck struct + Connection string + Equals any + SQL string + type ResultsWriter struct + func NewResultsWriter(dir, runID string) (*ResultsWriter, error) + func (w *ResultsWriter) Append(tr TrialResult) error + func (w *ResultsWriter) Path() string + func (w *ResultsWriter) Replace(tr TrialResult) error + func (w *ResultsWriter) RunID() string + func (w *ResultsWriter) Trials() []TrialResult + type Suite struct + Bin string + Flags Flags + Logf func(string, ...any) + Provisioner *Provisioner + Server *FixtureServer + Writer *ResultsWriter + func (s *Suite) RunSuite(cases []Case) (*SuiteSummary, error) + type SuiteSummary struct + Arms map[string]ArmSummary + Binary string + Cases []CaseMetrics + CostByFamily map[string]float64 + CostUSD float64 + Flips []Flip + GatingN int + GatingPass int + GatingRate float64 + Judge string + Run string + SchemaReset string + SkillsDelta string + Threshold float64 + TierRates map[string]float64 + Timeouts int + Verdict string + func AggregateSuite(runID string, trials []TrialResult, gating map[string]bool, threshold float64) SuiteSummary + func AggregateSuiteMeta(runID string, trials []TrialResult, meta map[string]CaseMeta, ...) SuiteSummary + type TrialResult struct + Arm string + Binary string + Case string + CostUSD float64 + DurationS float64 + Error string + Graders []GraderResult + InfraError bool + Invariant string + Model string + Pass bool + PromptMD5 string + Retried bool + Run string + Score float64 + SkillsVersion string + SkipReason string + Timeout bool + Transcript string + Trial int + Turns int + func LoadJSONL(path string) ([]TrialResult, error)
Go to main page