eval

package
v1.1.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 7, 2026 License: Apache-2.0 Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// TaskSchema identifies the task manifest contract.
	TaskSchema = "agentic.eval.task/v1alpha1"
	// ResultSchema identifies the candidate score contract.
	ResultSchema = "agentic.eval.result/v1alpha1"
	// TranscriptSchema identifies the MCP replay input contract.
	TranscriptSchema = "agentic.eval.transcript/v1alpha1"
	// ReplaySchema identifies the MCP replay result contract.
	ReplaySchema = "agentic.eval.replay/v1alpha1"
)
View Source 
const QualificationSchema = "agentic.eval.qualification/v1alpha1"

QualificationSchema identifies the task-qualification result contract.

Variables

This section is empty.

Functions

func Setup 

func Setup(ctx context.Context, task Task, bundle, workspace string) error

Setup clones one fixture base into a new candidate workspace.

func WriteQualification 

func WriteQualification(path string, qualification Qualification) error

WriteQualification atomically writes one qualification result.

func WriteReplayResult 

func WriteReplayResult(path string, result ReplayResult) error

WriteReplayResult atomically writes one replay summary.

func WriteResult 

func WriteResult(path string, result Result) error

WriteResult atomically writes one candidate score.

Types

type BundleRecord 

type BundleRecord struct {
	TaskID         string `json:"task_id"`
	Repository     string `json:"repository"`
	UpstreamBase   string `json:"upstream_base"`
	UpstreamTarget string `json:"upstream_target"`
	FixtureBase    string `json:"fixture_base"`
	FixtureTarget  string `json:"fixture_target"`
	Bundle         string `json:"bundle"`
	SHA256         string `json:"sha256"`
	GitVersion     string `json:"git_version"`
}

BundleRecord identifies one deterministic private fixture bundle.

func Prepare 

func Prepare(ctx context.Context, task Task, source, outputRoot string) (BundleRecord, error)

Prepare builds one deterministic two-commit fixture bundle.

func PrepareAll 

func PrepareAll(ctx context.Context, tasks []Task, sourceRoot, outputRoot string) ([]BundleRecord, error)

PrepareAll prepares deterministic bundles for every supplied task.

type Command 

type Command struct {
	Argv           []string `json:"argv"`
	TimeoutSeconds int      `json:"timeout_seconds"`
}

Command defines one bounded argv-based acceptance command.

type CommandResult 

type CommandResult struct {
	Status       string   `json:"status"`
	StdoutSHA256 string   `json:"stdout_sha256"`
	StderrSHA256 string   `json:"stderr_sha256"`
	Argv         []string `json:"argv"`
	ExitCode     int      `json:"exit_code"`
	DurationMS   int64    `json:"duration_ms"`
	StdoutBytes  int64    `json:"stdout_bytes"`
	StderrBytes  int64    `json:"stderr_bytes"`
	Truncated    bool     `json:"truncated"`
}

CommandResult records one bounded acceptance-command execution.

type Measurements 

type Measurements struct {
	ModelTokens *int  `json:"model_tokens,omitempty"`
	DurationMS  int64 `json:"duration_ms"`
	ToolCalls   int   `json:"tool_calls"`
	ResultBytes int64 `json:"result_bytes"`
}

Measurements records neutral execution and context-cost measurements.

type Operation 

type Operation struct {
	Arguments   map[string]any `json:"arguments"`
	Tool        string         `json:"tool"`
	ExpectError bool           `json:"expect_error"`
}

Operation defines one expected MCP tool call and transport outcome.

type Oracle 

type Oracle struct {
	OverlayPaths []string `json:"overlay_paths"`
}

Oracle defines upstream files overlaid only inside the scoring clone.

type Qualification 

type Qualification struct {
	SchemaVersion string   `json:"schema_version"`
	TaskID        string   `json:"task_id"`
	Status        string   `json:"status"`
	BundleSHA256  string   `json:"bundle_sha256"`
	Failures      []string `json:"failures"`
	NoOp          Result   `json:"no_op"`
	Reference     Result   `json:"reference"`
	ScopeProbe    Result   `json:"scope_probe"`
	DurationMS    int64    `json:"duration_ms"`
}

Qualification records the three states required for a sound task oracle.

func Qualify 

func Qualify(ctx context.Context, task Task, bundle, source string) (Qualification, error)

Qualify proves that behavior and scope checks discriminate independently.

type ReplayCall 

type ReplayCall struct {
	Tool         string `json:"tool"`
	Status       string `json:"status"`
	ResultSHA256 string `json:"result_sha256"`
	Artifact     string `json:"artifact,omitempty"`
	Error        string `json:"error,omitempty"`
	Sequence     int    `json:"sequence"`
	DurationMS   int64  `json:"duration_ms"`
	ResultBytes  int64  `json:"result_bytes"`
}

ReplayCall records one MCP operation and its bounded result artifact.

type ReplayResult 

type ReplayResult struct {
	SchemaVersion string       `json:"schema_version"`
	TaskID        string       `json:"task_id"`
	Status        string       `json:"status"`
	ServerSHA256  string       `json:"server_sha256"`
	Calls         []ReplayCall `json:"calls"`
	Uncertainties []string     `json:"uncertainties"`
	Measurements  Measurements `json:"measurements"`
}

ReplayResult records a complete deterministic transcript replay.

func Replay 

func Replay(ctx context.Context, transcript Transcript, server, workspace, artifacts string) (ReplayResult, error)

Replay runs one checked-in transcript against a stdio MCP server.

type Repository 

type Repository struct {
	Name   string `json:"name"`
	URL    string `json:"url"`
	Base   string `json:"base"`
	Target string `json:"target"`
}

Repository identifies the adjacent upstream commits used by a task.

type Result 

type Result struct {
	SchemaVersion   string          `json:"schema_version"`
	TaskID          string          `json:"task_id"`
	Status          string          `json:"status"`
	ChangedPaths    []string        `json:"changed_paths"`
	UnexpectedPaths []string        `json:"unexpected_paths"`
	Commands        []CommandResult `json:"commands"`
	Uncertainties   []string        `json:"uncertainties"`
	Measurements    Measurements    `json:"measurements"`
}

Result records structural and behavioral scoring for one candidate.

func Score 

func Score(ctx context.Context, task Task, bundle, workspace string) (Result, error)

Score evaluates one candidate without modifying its workspace.

type Scope 

type Scope struct {
	Workspace    string   `json:"workspace"`
	Packages     []string `json:"packages"`
	AllowedPaths []string `json:"allowed_paths"`
}

Scope defines the candidate workspace, packages, and permitted paths.

type Task 

type Task struct {
	SchemaVersion string     `json:"schema_version"`
	ID            string     `json:"id"`
	Repository    Repository `json:"repository"`
	Prompt        string     `json:"prompt"`
	Scope         Scope      `json:"scope"`
	Oracle        Oracle     `json:"oracle"`
	Acceptance    []Command  `json:"acceptance"`
	Invariants    []string   `json:"invariants"`
	Exercises     []string   `json:"exercises"`
	Pilot         bool       `json:"pilot"`
}

Task defines one pinned historical change and its acceptance contract.

func LoadTask 

func LoadTask(path string) (Task, error)

LoadTask strictly decodes and validates one task manifest.

func LoadTasks 

func LoadTasks(root string) ([]Task, error)

LoadTasks validates the complete eight-task corpus and its transcripts.

func (Task) Validate 

func (t Task) Validate() error

Validate checks one task's portable and safety-critical invariants.

type Transcript 

type Transcript struct {
	SchemaVersion string      `json:"schema_version"`
	TaskID        string      `json:"task_id"`
	Operations    []Operation `json:"operations"`
}

Transcript defines a deterministic sequence of MCP tool calls.

func LoadTranscript 

func LoadTranscript(path string) (Transcript, error)

LoadTranscript strictly decodes and validates one replay transcript.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.