specverify

package
v0.38.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 16, 2026 License: MIT Imports: 16 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func ParseSemanticResponse

func ParseSemanticResponse(raw string) (bool, string)

ParseSemanticResponse accepts a compact JSON or plain yes/no judge response.

Types

type CoveredBy

type CoveredBy struct {
	TaskID   string `json:"task_id"`
	TaskName string `json:"task_name,omitempty"`
	TaskPath string `json:"task_path,omitempty"`
	Mode     string `json:"mode"`
	Reason   string `json:"reason"`
}

CoveredBy describes one task that exercises a requirement.

type EngineSemanticMatcher

type EngineSemanticMatcher struct {
	Engine  execution.AgentEngine
	ModelID string
	Timeout time.Duration
}

EngineSemanticMatcher uses the configured judge model to fill deterministic coverage gaps.

func (*EngineSemanticMatcher) Matches

func (m *EngineSemanticMatcher) Matches(ctx context.Context, req Requirement, task TaskRef) (bool, string, error)

Matches asks the judge whether a task exercises a requirement.

type Options

type Options struct {
	SkillPath     string
	EvalPath      string
	Semantic      bool
	JudgeModel    string
	FailThreshold int
	Matcher       SemanticMatcher
}

Options controls verification behavior.

type ParsedSkillSpec

type ParsedSkillSpec struct {
	SkillName    string        `json:"skill_name"`
	SkillPath    string        `json:"skill_path"`
	Requirements []Requirement `json:"requirements"`
}

ParsedSkillSpec contains deterministic requirements extracted from SKILL.md.

func ParseSkillFile

func ParseSkillFile(path string) (*ParsedSkillSpec, error)

ParseSkillFile deterministically extracts requirements from a SKILL.md file.

type Report

type Report struct {
	SkillPath  string                `json:"skill_path"`
	EvalPath   string                `json:"eval_path"`
	Semantic   bool                  `json:"semantic"`
	JudgeModel string                `json:"judge_model,omitempty"`
	Summary    Summary               `json:"summary"`
	Coverage   []RequirementCoverage `json:"coverage"`
}

Report is the full machine-readable output for spec verification.

func Verify

func Verify(ctx context.Context, opts Options) (*Report, error)

Verify parses SKILL.md, loads eval.yaml tasks, and computes requirement coverage.

type Requirement

type Requirement struct {
	ID     string          `json:"id"`
	Kind   RequirementKind `json:"kind"`
	Text   string          `json:"text"`
	Source SourceSpan      `json:"source"`
}

Requirement is a machine-readable SKILL.md promise.

type RequirementCoverage

type RequirementCoverage struct {
	Requirement Requirement `json:"requirement"`
	CoveredBy   []CoveredBy `json:"covered_by"`
}

RequirementCoverage reports task coverage for one requirement.

type RequirementKind

type RequirementKind string

RequirementKind identifies the part of SKILL.md a requirement came from.

const (
	RequirementDescription RequirementKind = "description"
	RequirementUse         RequirementKind = "use"
	RequirementDont        RequirementKind = "dont"
	RequirementParameter   RequirementKind = "parameter"
)

type SemanticMatcher

type SemanticMatcher interface {
	Matches(ctx context.Context, req Requirement, task TaskRef) (bool, string, error)
}

SemanticMatcher can opt in to LLM-assisted coverage after deterministic matching.

type SourceSpan

type SourceSpan struct {
	File      string `json:"file"`
	StartLine int    `json:"start_line"`
	EndLine   int    `json:"end_line"`
}

SourceSpan points back to the SKILL.md source lines for a parsed requirement.

type Summary

type Summary struct {
	TotalRequirements     int `json:"total_requirements"`
	CoveredRequirements   int `json:"covered_requirements"`
	UncoveredRequirements int `json:"uncovered_requirements"`
	FailThreshold         int `json:"fail_threshold,omitempty"`
}

Summary captures aggregate verification results.

type TaskRef

type TaskRef struct {
	ID              string         `json:"id"`
	Name            string         `json:"name,omitempty"`
	Path            string         `json:"path,omitempty"`
	Text            string         `json:"-"`
	ExpectedTrigger *bool          `json:"expected_trigger,omitempty"`
	Metadata        map[string]any `json:"-"`
}

TaskRef captures the task fields needed for coverage reporting.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL