Documentation
¶
Overview ¶
Package gate evaluates loop verification gates.
Index ¶
- Constants
- Variables
- func ParseCommandMetricScore(stdout string) (*float64, error)
- func RenderAgentJudgeRubric(criterion dsl.GateCriterion, contract dsl.Contract, evidence JudgeEvidence) (string, error)
- type BestUpdateIntent
- type BlockingIssue
- type CommandRequest
- type CommandResult
- type CommandRunner
- type CriterionResult
- type DiagnosticWarning
- type Evaluator
- type Gate
- type GateEvaluator
- type GateInput
- type HumanDecision
- type HumanDecisionKind
- type JudgeEvidence
- type JudgeRequest
- type JudgeResponse
- type JudgeRunner
- type JudgeUsageReporter
- type JudgeUsageReporterFunc
- type Option
- type Placement
- type RouteAction
- type RouteDecision
- type ToolCaller
- type Verdict
- type VerdictIntent
- type VerdictOutcome
- type VerdictReader
- type VerdictRecord
Constants ¶
const ( // JudgeMalformedOutputIssueID is the canonical blocker for responsive invalid judge JSON. JudgeMalformedOutputIssueID = "judge_output_malformed" // JudgeEvidenceRequiredIssueID is the ADR-018 blocker for empty approved evidence. JudgeEvidenceRequiredIssueID = "judge_evidence_required" // JudgeTransportFailureIssueID is the canonical blocker for broken judge transport. JudgeTransportFailureIssueID = "judge_transport_failure" // JudgeRubricInvalidIssueID is the canonical blocker for template/render failures. JudgeRubricInvalidIssueID = "judge_rubric_invalid" // MetricScoreInvalidIssueID is the canonical blocker and warning for an unusable metric score. MetricScoreInvalidIssueID = "metric_score_invalid" )
const (
// DefaultBrokenJudgeStreakLimit is the ADR-012 fail-open guard.
DefaultBrokenJudgeStreakLimit = 3
)
Variables ¶
var ( // ErrInvalidGate reports an authored gate that cannot be evaluated. ErrInvalidGate = errors.New("loop gate: invalid gate") // ErrMaxRevisionsCeiling reports a gate above the hard revision ceiling. ErrMaxRevisionsCeiling = errors.New("loop gate: max_revisions exceeds ceiling") // ErrVerdictPolicyRequiresJudge reports a revise policy without a verdict source. ErrVerdictPolicyRequiresJudge = errors.New("loop gate: verdict_policy revise_until_clean requires judge or human") )
var ( // ErrValidation classifies invalid durable verdict intent input. ErrValidation = errors.New("gate verdict intent") )
Functions ¶
func ParseCommandMetricScore ¶
ParseCommandMetricScore parses the exact score-only JSON contract emitted by command criteria.
func RenderAgentJudgeRubric ¶
func RenderAgentJudgeRubric( criterion dsl.GateCriterion, contract dsl.Contract, evidence JudgeEvidence, ) (string, error)
RenderAgentJudgeRubric renders a contract-aware, evidence-required judge prompt.
Types ¶
type BestUpdateIntent ¶
BestUpdateIntent records a score that is eligible to replace the current best generation.
func BestUpdateForVerdict ¶
func BestUpdateForVerdict( gate Gate, verdict Verdict, generation int64, current *BestUpdateIntent, ) (*BestUpdateIntent, error)
BestUpdateForVerdict returns an update only for an approved, finite metric score that strictly improves the baseline by the declared directional delta.
type BlockingIssue ¶
BlockingIssue is the structured blocker ID shape from ADR-022.
type CommandRequest ¶
type CommandRequest struct {
WorkspaceID string
GateID string
CriterionID string
Command string
Expect string
Contains string
}
CommandRequest is one command criterion invocation.
type CommandResult ¶
CommandResult captures the deterministic command result.
type CommandRunner ¶
type CommandRunner interface {
RunCommand(ctx context.Context, req CommandRequest) (CommandResult, error)
}
CommandRunner executes command criteria.
type CriterionResult ¶
type CriterionResult struct {
ID string `json:"id"`
Type dsl.CriterionType `json:"type"`
Outcome VerdictOutcome `json:"outcome"`
Passed bool `json:"passed"`
Broken bool `json:"broken,omitempty"`
ExitCode *int `json:"exit_code,omitempty"`
Stdout string `json:"stdout,omitempty"`
Stderr string `json:"stderr,omitempty"`
Score *float64 `json:"score,omitempty"`
Evidence json.RawMessage `json:"evidence,omitempty"`
BlockingIssues []BlockingIssue `json:"blocking_issues,omitempty"`
Warnings []DiagnosticWarning `json:"warnings,omitempty"`
Payload json.RawMessage `json:"payload,omitempty"`
// contains filtered or unexported fields
}
CriterionResult is the observable per-criterion outcome for SSE/UI consumers.
func ParseJudgeVerdict ¶
func ParseJudgeVerdict(criterionID string, criterionType dsl.CriterionType, raw string) CriterionResult
ParseJudgeVerdict maps raw judge output into the ADR-022 criterion result.
type DiagnosticWarning ¶
DiagnosticWarning is a non-terminal evaluator diagnostic.
type Evaluator ¶
type Evaluator struct {
// contains filtered or unexported fields
}
Evaluator evaluates gate criteria using injected runtime boundaries.
func NewEvaluator ¶
NewEvaluator creates an evaluator with deterministic defaults.
type Gate ¶
type Gate struct {
ID string
Criteria []dsl.GateCriterion
VerdictPolicy dsl.VerdictPolicy
// OnResult maps verdict result keys to route overrides. Recognized keys are
// pass, approval, fail, blocked, error, timeout, and invalid_output.
OnResult map[string]any
MaxRevisions int
}
Gate is the runtime shape of a control gate or definition-of-done gate.
func GateFromContract ¶
GateFromContract adapts contract verification criteria into a DoD gate.
func GateFromGoalJudge ¶
func GateFromGoalJudge(nodeID string, criteria []dsl.GateCriterion) Gate
GateFromGoalJudge adapts Goal judge criteria into the per-turn judge gate. Deterministic-only judges (command/extension) evaluate under fixed_passes.
func GateFromNode ¶
GateFromNode adapts a DSL gate node into the evaluator input.
type GateEvaluator ¶
type GateEvaluator interface {
Evaluate(ctx context.Context, gate Gate, in GateInput) (Verdict, error)
}
GateEvaluator evaluates one gate criteria bundle and returns its verdict and route.
type GateInput ¶
type GateInput struct {
LoopRunID string
Placement Placement
Contract *dsl.Contract
Revision int
BrokenJudgeStreak int
BrokenJudgeStreakLimit int
BestScore *float64
HumanDecisions map[string]HumanDecision
ToolScope tools.Scope
ToolCallCorrelationID string
ToolSensitiveInputFields []string
JudgeRuntime dsl.RuntimeSpec
JudgeUsageReporter JudgeUsageReporter
JudgeEvidence JudgeEvidence
NetworkParticipation *participation.Spec
}
GateInput carries runtime state required to evaluate and route a gate.
type HumanDecision ¶
type HumanDecision struct {
Decision HumanDecisionKind
Actor task.ActorContext
Note string
}
HumanDecision captures one human gate decision made through a trusted actor context.
type HumanDecisionKind ¶
type HumanDecisionKind string
HumanDecisionKind is the closed approval vocabulary for human criteria.
const ( // HumanDecisionApprove resumes the passing path. HumanDecisionApprove HumanDecisionKind = "approve" // HumanDecisionRequestChanges routes to revise or next generation. HumanDecisionRequestChanges HumanDecisionKind = "request_changes" // HumanDecisionReject routes to terminal halt. HumanDecisionReject HumanDecisionKind = "reject" )
type JudgeEvidence ¶
type JudgeEvidence struct {
Text string
Structured json.RawMessage
}
JudgeEvidence is the authoritative completed candidate supplied to an agent judge.
type JudgeRequest ¶
type JudgeRequest struct {
LoopRunID string
GateID string
CriterionID string
Attempt int
CorrelationID string
ProfileID string
WorkspaceID string
Agent string
Runtime dsl.RuntimeSpec
Rubric string
Contract dsl.Contract
NetworkParticipation *participation.Spec
}
JudgeRequest is the rendered agent-judge invocation.
type JudgeResponse ¶
JudgeResponse is the raw judge response payload.
type JudgeRunner ¶
type JudgeRunner interface {
Judge(ctx context.Context, req JudgeRequest) (JudgeResponse, error)
}
JudgeRunner executes agent-judge criteria.
type JudgeUsageReporter ¶
type JudgeUsageReporter interface {
ReportJudgeTokensUsed(int64)
}
JudgeUsageReporter receives one aggregate cumulative token total for a gate evaluation.
type JudgeUsageReporterFunc ¶
type JudgeUsageReporterFunc func(int64)
JudgeUsageReporterFunc adapts a function to JudgeUsageReporter.
func (JudgeUsageReporterFunc) ReportJudgeTokensUsed ¶
func (f JudgeUsageReporterFunc) ReportJudgeTokensUsed(tokens int64)
ReportJudgeTokensUsed implements JudgeUsageReporter.
type Option ¶
type Option func(*Evaluator)
Option configures an Evaluator.
func WithBrokenJudgeStreakLimit ¶
WithBrokenJudgeStreakLimit overrides the ADR-012 broken judge streak limit.
func WithCommandRunner ¶
func WithCommandRunner(runner CommandRunner) Option
WithCommandRunner injects command criterion execution.
func WithJudgeRunner ¶
func WithJudgeRunner(runner JudgeRunner) Option
WithJudgeRunner injects agent-judge criterion execution.
func WithMaxRevisionsCeiling ¶
WithMaxRevisionsCeiling overrides the structural max_revisions ceiling.
func WithToolCaller ¶
func WithToolCaller(caller ToolCaller) Option
WithToolCaller injects tools/call dispatch for extension criteria.
type RouteAction ¶
type RouteAction string
RouteAction is the closed routing vocabulary produced by the evaluator.
const ( // RouteContinue advances past an in-body gate. RouteContinue RouteAction = "continue" // RouteRevise re-enters the body revision path. RouteRevise RouteAction = "revise" // RouteHalt stops the loop. RouteHalt RouteAction = "halt" // RouteEscalate parks for human approval. RouteEscalate RouteAction = "escalate" // RouteDone marks a definition-of-done gate complete. RouteDone RouteAction = "done" // RouteNextGeneration schedules the next generation. RouteNextGeneration RouteAction = "next_generation" )
type RouteDecision ¶
type RouteDecision struct {
Placement Placement `json:"placement"`
Action RouteAction `json:"action"`
Target string `json:"target,omitempty"`
TerminalStatus string `json:"terminal_status,omitempty"`
ReasonCode string `json:"reason_code,omitempty"`
}
RouteDecision is the routing result for the gate placement.
type ToolCaller ¶
type ToolCaller interface {
Call(ctx context.Context, scope tools.Scope, req tools.CallRequest) (tools.ToolResult, error)
}
ToolCaller is the tools/call subset used by extension gate checks.
type Verdict ¶
type Verdict struct {
Outcome VerdictOutcome `json:"outcome"`
Criteria []CriterionResult `json:"criteria"`
BlockingIssues []BlockingIssue `json:"blocking_issues,omitempty"`
// Payload carries the first aggregate verdict-source evidence or payload.
Payload json.RawMessage `json:"payload,omitempty"`
// Broken marks fail-open transport failure; consumers must use Route for control flow.
Broken bool `json:"broken,omitempty"`
Warnings []DiagnosticWarning `json:"warnings,omitempty"`
Route RouteDecision `json:"route"`
NextBrokenJudgeStreak int `json:"next_broken_judge_streak,omitempty"`
BlockingIssueSignature []string `json:"blocking_issue_signature,omitempty"`
}
Verdict is the aggregate gate verdict.
func SanitizeVerdict ¶
SanitizeVerdict returns the canonical verdict representation used by both the gate output projection and durable verdict intents.
type VerdictIntent ¶
type VerdictIntent struct {
GateID string `json:"gate_id"`
ItemIndex int `json:"item_index"`
Outcome VerdictOutcome `json:"outcome"`
Score *float64 `json:"score,omitempty"`
RouteCauseRank *int `json:"route_cause_rank,omitempty"`
BlockingIssues json.RawMessage `json:"blocking_issues"`
Criteria json.RawMessage `json:"criteria"`
}
VerdictIntent is the sanitized, immutable machine-verdict mutation carried by a completion plan.
func NewVerdictIntent ¶
func NewVerdictIntent( gateID string, itemIndex int, verdict Verdict, routeCauseRank *int, ) (VerdictIntent, error)
NewVerdictIntent projects an aggregate verdict through the single diagnostics sanitization boundary.
type VerdictOutcome ¶
type VerdictOutcome string
VerdictOutcome is the canonical criterion and aggregate verdict taxonomy.
const ( // VerdictOutcomeApproved accepts the gate. VerdictOutcomeApproved VerdictOutcome = "approved" // VerdictOutcomeRejected rejects the gate and requests revision. VerdictOutcomeRejected VerdictOutcome = "rejected" // VerdictOutcomeAwaitingApproval parks the gate until a human decision arrives. VerdictOutcomeAwaitingApproval VerdictOutcome = "awaiting_approval" // VerdictOutcomeBlocked reports an explicit external blocker. VerdictOutcomeBlocked VerdictOutcome = "blocked" // VerdictOutcomeError reports a checker execution error. VerdictOutcomeError VerdictOutcome = "error" // VerdictOutcomeTimeout reports a checker timeout. VerdictOutcomeTimeout VerdictOutcome = "timeout" // VerdictOutcomeInvalidOutput reports a responsive checker with unusable output. VerdictOutcomeInvalidOutput VerdictOutcome = "invalid_output" )
func (VerdictOutcome) Valid ¶
func (o VerdictOutcome) Valid() bool
Valid reports whether the outcome belongs to the closed gate verdict vocabulary.
type VerdictReader ¶
type VerdictReader interface {
ListGateVerdicts(
ctx context.Context,
workspaceID string,
runID string,
generation int64,
) ([]VerdictRecord, error)
ListRouteCausingVerdicts(
ctx context.Context,
workspaceID string,
runID string,
generation int64,
) ([]VerdictRecord, error)
}
VerdictReader provides workspace-scoped machine verdict history.
type VerdictRecord ¶
type VerdictRecord struct {
RunID string
Generation int64
GateID string
ItemIndex int
Outcome VerdictOutcome
Score *float64
RouteCauseRank *int
BlockingIssues json.RawMessage
Criteria json.RawMessage
DecidedAt time.Time
}
VerdictRecord is the durable read model for one machine gate verdict.