Versions in this module Expand all Collapse all v0 v0.6.1 Nov 3, 2026 v0.6.0 Nov 2, 2026 v0.5.2 Nov 2, 2026 v0.5.1 Nov 1, 2026 v0.5.0 Oct 31, 2026 Changes in this version type FilePairs + AllowPlaintext bool v0.4.1 Oct 30, 2026 v0.4.0 Oct 29, 2026 Changes in this version + func Fence(parts []Part) (prompt, notice string, err error) type HTTPLLM + func NewHTTPLLM(c *JudgeConfig) (*HTTPLLM, error) + func (h *HTTPLLM) CompleteUsage(ctx context.Context, r LLMRequest) (string, TokenUsage, error) + type TokenUsage struct + InputTokens int + OutputTokens int v0.3.0 Oct 24, 2026 Changes in this version + const DefaultAgentQuiet + const FromControl + const FromProduction + const MatchCalls + const MatchTools + const MetricAgentFidelity + const MetricAgentInvalidCall + const MetricAgentPrematureStop + const MetricAgentToolAgreement + const RedactionRulesVersion + const SelectDiverged + const SelectInvalidCall + const SelectJudgeLoss + const SelectPrematureStop + const SelectSample + var DefaultKeepFields = []string + var DefaultSelect = []string + func HistoryIDs(path string) (graded, agent map[string]bool, err error) + func HistoryScores(path string) (map[string]PairScore, error) + func SameCalls(a, b Action) bool + func SameTools(a, b Action) bool + func SessionUnit(session string) string + func WriteStepCases(dir string, cases []StepCase) error + type Action struct + Calls []Call + Malformed bool + Truncated bool + func (a Action) Stopped() bool + type AgentArm struct + Fidelity Rate + InvalidCallRate float64 + InvalidToolRate float64 + PrematureStopRate float64 + SchemaErrorRate float64 + ToolCallRate float64 + type AgentResult struct + ArgAgreement Rate + Auxiliary int + ByDepth []DepthBucket + Candidate AgentArm + Control AgentArm + Diverged float64 + FidelityDelta MetricDelta + FirstDivergence float64 + MultiTurn int + PerStep []StepSample + Referenced int + Retries int + Sessions int + Steps int + ToolAgreement Rate + Truncated int + func AnalyzeAgent(pairs []shadow.Pair, seed uint64) *AgentResult + type Call struct + Input string + Name string + type CaseSummary struct + Expected []string + ExpectedFrom string + ID string + Provenance Provenance + Reasons []string + type DepthBucket struct + N int + Steps string + ToolAgreement float64 + type HarvestOptions struct + Experiment string + JudgeLoss float64 + Now func() time.Time + PerDepth int + Redactor *Redactor + Scores map[string]PairScore + Seed uint64 + Select []string + type HarvestResult struct + ByReason map[string]int + Cases []StepCase + Skipped int + Steps int + func Harvest(ctx context.Context, src PairSource, o HarvestOptions) (*HarvestResult, error) + func (h *HarvestResult) Summary() []CaseSummary type OnlineOptions + AgentQuiet time.Duration + AgentSkip map[string]bool type OnlineResult + Agent *AgentResult + AgentPairIDs []string + AgentSteps []StepSample type PairScore + Session string + type Provenance struct + Depth int + Experiment string + HarvestedAt time.Time + PairID string + RedactionRules string + Session string + type Rate struct + CIHi float64 + CILo float64 + Mean float64 + N int + type Redactor struct + func NewRedactor(patterns, keepFields []string) (*Redactor, error) + func (r *Redactor) Action(a Action) (Action, error) + func (r *Redactor) Request(body []byte) ([]byte, error) + func (r *Redactor) Text(s string, heuristic bool) string + func (r *Redactor) Version() string + type StepCase struct + Alternatives []Action + Expected Action + ExpectedFrom string + ID string + Match string + Protocol string + Provenance Provenance + Reasons []string + Request json.RawMessage + Tools map[string][]string + func LoadStepCases(dir string) ([]StepCase, error) + type StepModel struct + APIKey string + HTTP *http.Client + Model string + URL string + func (m *StepModel) Answer(ctx context.Context, c StepCase) (Action, []byte, error) + type StepResult struct + Actual Action + Depth int + Error string + ID string + JudgeError string + JudgeScore *float64 + Pass bool + PrematureStop bool + SameCalls bool + SameTools bool + Session string + Valid bool + func GradeStep(c StepCase, a Action) StepResult + type StepRunOptions struct + Judge *Judge + Parallel int + Rubric *Rubric + Seed uint64 + type StepSample struct + CandFid float64 + CandInvalid float64 + ControlFidelity float64 + ControlInvalid float64 + HasRef bool + PairID string + Session string + Time time.Time + type StepScorecard struct + Cases int + Errors int + Judge *Rate + JudgeErrors int + Model string + Pass Rate + PrematureStop Rate + Results []StepResult + Rubric string + SameCalls Rate + SameTools Rate + Sessions int + Suite string + Valid Rate + func RunSteps(ctx context.Context, suite string, cases []StepCase, m *StepModel, ...) (*StepScorecard, error) + func (sc *StepScorecard) Table() string v0.2.1 Oct 12, 2026 v0.2.0 Oct 11, 2026 v0.1.1 Oct 4, 2026 v0.1.0 Oct 3, 2026 Changes in this version + const DefaultTimeout + const GateBlock + const GateHold + const GateShip + const GraderCommand + const GraderDiff + const GraderFile + const GraderJudge + const MetricCost + const MetricJudgeDelta + const MetricJudgeErrors + const MetricJudgeScore + const MetricJudgeWinRate + const MetricPairsGraded + const MetricPass + const MetricShadowErrorRate + const MetricShadowLatency + const MetricShadowOutputTokens + const MetricTokens + const MetricToolCalls + const MetricWall + const VerdictBetter + const VerdictNoDiff + const VerdictWorse + const WireAnthropic + const WireOpenAI + var DefaultSandboxProbes = map[string][]string + var ErrJudgeSchema = errors.New("judge reply failed schema validation") + var ErrSandboxUnavailable = errors.New("sandbox unavailable (Landlock)") + func AppendHistory(path string, entries ...HistoryEntry) error + func CheckSecureURL(raw string) error + func Drivers() map[string]Driver + func GradedPairs(path string) (map[string]bool, error) + func OnlineTable(rs []OnlineResult) string + type ArmStats struct + ErrorRate float64 + LatencyMS float64 + OutputTokens float64 + type ClaudeDriver struct + func (ClaudeDriver) Command(t *Task, v Variant) Command + func (ClaudeDriver) Name() string + func (ClaudeDriver) Parse(out []byte) (Usage, error) + func (ClaudeDriver) SettingsPath() string + type CodexDriver struct + func (CodexDriver) Command(t *Task, v Variant) Command + func (CodexDriver) Name() string + func (CodexDriver) Parse(out []byte) (Usage, error) + func (CodexDriver) SettingsPath() string + type Command struct + Args []string + Stdin string + type Comparison struct + CIHi float64 + CILo float64 + Control string + Delta float64 + Deltas map[string]MetricDelta + Gate *Gate + Tasks int + Variant string + Verdict string + type Criterion struct + Description string + Name string + Weight float64 + type DockerRunner struct + Bin string + CPUs string + Image func(Variant) string + Memory string + Network string + PassEnv []string + SandboxProbes map[string][]string + User string + func (d *DockerRunner) Start(ctx context.Context, j Job) (Env, error) + type Driver interface + Command func(t *Task, v Variant) Command + Name func() string + Parse func(stdout []byte) (Usage, error) + SettingsPath func() string + type Env interface + Close func() error + Exec func(ctx context.Context, c Command) (ExecResult, error) + type ExecResult struct + ExitCode int + Stderr []byte + Stdout []byte + type FilePairs struct + Keys [][]byte + Path string + func (f FilePairs) Pairs(ctx context.Context) ([]shadow.Pair, error) + type Gate struct + Flaky []TaskDiff + Reasons []string + Regressed []TaskDiff + Tasks []TaskDiff + Verdict string + type GeminiDriver struct + func (GeminiDriver) Command(t *Task, v Variant) Command + func (GeminiDriver) Name() string + func (GeminiDriver) Parse(out []byte) (Usage, error) + func (GeminiDriver) SettingsPath() string + type Grade struct + Detail string + Error string + Grader string + Pass bool + Rubric string + Score *float64 + type GraderSpec struct + Contains []string + Exists *bool + Matches string + MaxFilesChanged int + MustChange []string + MustNotChange []string + NotContains []string + Path string + Rubric string + Run string + Type string + type HTTPLLM struct + APIKey string + BaseURL string + HTTP *http.Client + Wire string + func (h *HTTPLLM) Complete(ctx context.Context, r LLMRequest) (string, error) + type HistoryEntry struct + Gate *Gate + Kind string + Online *OnlineResult + Suite string + Time time.Time + Variants []VariantScore + func OfflineHistory(sc *Scorecard, at time.Time) HistoryEntry + type Job struct + Settings string + SettingsDest string + Task *Task + Variant Variant + type Judge struct + Cache *JudgeCache + LLM LLM + Model string + func NewJudge(c *JudgeConfig, suiteDir string) (*Judge, error) + func (j *Judge) Grade(ctx context.Context, r *Rubric, parts []Part) (Verdict, error) + type JudgeCache struct + Dir string + type JudgeConfig struct + APIKeyEnv string + Cache string + Model string + URL string + Wire string + type JudgeInfo struct + Model string + Rubrics []string + Wire string + type LLM interface + Complete func(ctx context.Context, req LLMRequest) (string, error) + type LLMRequest struct + MaxTokens int + Model string + Prompt string + System string + type LocalRunner struct + func (LocalRunner) Start(ctx context.Context, j Job) (Env, error) + type Matrix struct + Baseline string + Harnesses []MatrixHarness + Providers []MatrixProvider + func (m *Matrix) Cells(suiteDir string) []Variant + type MatrixHarness struct + Harness string + Models []string + Settings string + Version string + type MatrixProvider struct + Aliases map[string]string + Name string + type MetricDelta struct + Baseline float64 + CIHi float64 + CILo float64 + Mean float64 + type OTLPExporter struct + HTTP *http.Client + Token string + URL string + func (x *OTLPExporter) Export(ctx context.Context, rs []OnlineResult, start, now time.Time) error + type OnlineOptions struct + ControlNames map[string]string + Experiment string + Judge *Judge + Now func() time.Time + Rubric *Rubric + Sample int + Seed uint64 + Skip map[string]bool + type OnlineResult struct + Candidate float64 + CandidateVariant string + Control float64 + ControlVariant string + Delta MetricDelta + Experiment string + Graded int + JudgeErrors int + JudgeModel string + Ops OpsResult + PairIDs []string + Pairs int + Rubric string + Scores []PairScore + WinRate float64 + func RunOnline(ctx context.Context, src PairSource, o OnlineOptions) ([]OnlineResult, error) + type OpsResult struct + Candidate ArmStats + Control ArmStats + Pairs int + type PairScore struct + Candidate float64 + Control float64 + ID string + type PairSource interface + Pairs func(ctx context.Context) ([]shadow.Pair, error) + type Part struct + Body string + Title string + Untrusted bool + type Rubric struct + Criteria []Criterion + ID string + Instructions string + PassThreshold float64 + Version string + func LoadRubric(path string) (*Rubric, error) + func (r *Rubric) Ref() string + type RunOptions struct + Judge *Judge + Parallel int + type Runner interface + Start func(ctx context.Context, j Job) (Env, error) + type Scorecard struct + Comparisons []Comparison + Control string + Gate *Gate + Judge *JudgeInfo + K int + Seed uint64 + Suite string + Thresholds *Thresholds + Trials []Trial + Variants []VariantScore + func BuildScorecard(s *Suite, trials []Trial, seed uint64) (*Scorecard, error) + func (sc *Scorecard) JSON() ([]byte, error) + func (sc *Scorecard) Markdown() string + func (sc *Scorecard) Table() string + type Suite struct + Control string + Dir string + Gate Thresholds + Graders []GraderSpec + Judge *JudgeConfig + K int + Matrix *Matrix + Name string + Repeats int + Seed uint64 + Tasks []string + Variants []Variant + func LoadSuite(path string) (*Suite, error) + func (s *Suite) LoadTasks(tasksRoot string) ([]*Task, error) + func (s *Suite) Validate() error + func (s *Suite) WithMatrix() (*Suite, error) + type Task struct + BudgetUSD float64 + Check string + Dir string + Graders []GraderSpec + ID string + MaxTurns int + Prompt string + Repo string + Setup []string + Tags []string + Timeout time.Duration + func LoadTask(dir string) (*Task, error) + func (t *Task) RepoDir() (string, error) + func (t *Task) Validate() error + type TaskDiff struct + Baseline float64 + Candidate float64 + Reason string + Status string + Task string + type Thresholds struct + MaxCostIncrease *float64 + MaxPassDrop *float64 + MaxRegressedTasks int + MaxTokensIncrease *float64 + MaxToolCallsIncrease *float64 + MaxWallIncrease *float64 + MinTasks int + TaskRegression float64 + type Trial struct + CostUSD float64 + Error string + GraderError string + Grades []Grade + Pass bool + Repeat int + Seed uint64 + Task string + Tokens int64 + ToolCalls int + ToolErrors int + Turns int + Unavailable string + Variant string + WallMs int64 + func RunSuite(ctx context.Context, s *Suite, tasks []*Task, r Runner, ...) ([]Trial, error) + func RunSuiteWith(ctx context.Context, s *Suite, tasks []*Task, r Runner, ...) ([]Trial, error) + func RunTrial(ctx context.Context, r Runner, d Driver, t *Task, v Variant, repeat int) Trial + type Usage struct + CostUSD float64 + DurationMs int64 + Error string + Failed bool + Tokens int64 + ToolCalls int + ToolErrors int + Turns int + type Variant struct + Dir string + Harness string + Model string + Name string + Provider string + Settings string + Version string + type VariantScore struct + CostPerTask float64 + Errors int + GradeErrors int + Harness string + JudgeScore *float64 + K int + Model string + Name string + Pass1 float64 + Pass1CIHi float64 + Pass1CILo float64 + PassAtK float64 + PassHatK float64 + Provider string + Tokens int64 + ToolCalls int + ToolErrors int + Trials int + Unavailable int + UnavailableReason string + Version string + WallP50Ms int64 + WallP95Ms int64 + type Verdict struct + Cached bool + Rationale string + Score float64 + Scores map[string]float64 + func ParseVerdict(r *Rubric, reply string) (Verdict, error) v0.1.0-rc.2 Oct 2, 2026 v0.1.0-rc.1 Oct 2, 2026
Go to main page