agenteval

type AgentRunInput struct {
	TaskID        string
	Prompt        string
	WorkspacePath string
	Model         string
}

type AgentRunResult struct {
	ExitCode int    `json:"exitCode"`
	Stdout   string `json:"stdout,omitempty"`
	Stderr   string `json:"stderr,omitempty"`
	Error    string `json:"error,omitempty"`
	// Truncated is set when captured stdout/stderr exceeded the runner's
	// OutputLimit and some output was dropped.
	Truncated bool `json:"truncated,omitempty"`
}

type AgentRunner interface {
	Run(context.Context, AgentRunInput) AgentRunResult
}

type AgentRunnerFunc func(context.Context, AgentRunInput) AgentRunResult

type BenchmarkInput struct {
	TaskID         string
	WorkRoot       string
	Models         []string
	KeepWorkspaces bool
	// Timeout bounds each task's materialization, agent run, and scoring. A
	// non-positive value leaves the task unbounded.
	Timeout time.Duration
}

type BenchmarkReport struct {
	Contract string                `json:"contract"`
	SuiteID  string                `json:"suiteId"`
	OK       bool                  `json:"ok"`
	Summary  BenchmarkSummary      `json:"summary"`
	Tasks    []BenchmarkTaskReport `json:"tasks"`
}

type BenchmarkSummary struct {
	TotalTasks   int `json:"totalTasks"`
	PassedTasks  int `json:"passedTasks"`
	FailedTasks  int `json:"failedTasks"`
	BlockedTasks int `json:"blockedTasks"`
	ErrorTasks   int `json:"errorTasks"`
}

type BenchmarkTaskReport struct {
	TaskID        string         `json:"taskId"`
	Model         string         `json:"model,omitempty"`
	WorkspacePath string         `json:"workspacePath"`
	FixturePath   string         `json:"fixturePath"`
	Agent         AgentRunResult `json:"agent"`
	Report        Report         `json:"report"`
}

type ChangedFilesFunc func(context.Context, string) ([]string, error)

type Command struct {
	ID      string   `json:"id"`
	Name    string   `json:"name"`
	Command []string `json:"command"`
}

type CommandAgentRunner struct {
	Command []string
	// OutputLimit caps captured stdout/stderr per stream in bytes. Zero applies
	// defaultAgentOutputLimit; a negative value disables the cap.
	OutputLimit int
}

type CommandResult struct {
	ID       string
	ExitCode int
	Stdout   string
	Stderr   string
	Error    string
}

type CommandRunner func(context.Context, string, Command) CommandResult

type ContextCheckResult struct {
	MissingRequiredFiles  []string
	PresentForbiddenFiles []string
}

type ContextChecks struct {
	RequiredFiles  []string `json:"requiredFiles,omitempty"`
	ForbiddenFiles []string `json:"forbiddenFiles,omitempty"`
}

type Harness struct {
	Materializer Materializer
	Agent        AgentRunner
	Runner       Runner
}

type MaterializeInput struct {
	WorkRoot string
}

type Materializer struct{}

type Report struct {
	Contract     string   `json:"contract"`
	SuiteID      string   `json:"suiteId"`
	TaskID       string   `json:"taskId"`
	Status       Status   `json:"status"`
	OK           bool     `json:"ok"`
	Summary      Summary  `json:"summary"`
	ChangedFiles []string `json:"changedFiles"`
	Results      []Result `json:"results"`
	Error        string   `json:"error,omitempty"`
}

type Result struct {
	ID              string     `json:"id"`
	Name            string     `json:"name"`
	Kind            ResultKind `json:"kind"`
	Status          Status     `json:"status"`
	Command         []string   `json:"command,omitempty"`
	ExitCode        *int       `json:"exitCode,omitempty"`
	Stdout          string     `json:"stdout,omitempty"`
	Stderr          string     `json:"stderr,omitempty"`
	Message         string     `json:"message,omitempty"`
	ExpectedFiles   []string   `json:"expectedFiles,omitempty"`
	ActualFiles     []string   `json:"actualFiles,omitempty"`
	MissingFiles    []string   `json:"missingFiles,omitempty"`
	UnexpectedFiles []string   `json:"unexpectedFiles,omitempty"`
	ExpectedEvents  []string   `json:"expectedEvents,omitempty"`
	ActualEvents    []string   `json:"actualEvents,omitempty"`
	MissingEvents   []string   `json:"missingEvents,omitempty"`
}

type ResultKind string

type RunInput struct {
	TaskID        string
	WorkspacePath string
	TraceStdout   string
	// CommandTimeout bounds each verification command. Non-positive applies
	// defaultCommandTimeout.
	CommandTimeout time.Duration
}

type Runner struct {
	RunCommand   CommandRunner
	ChangedFiles ChangedFilesFunc
	// contains filtered or unexported fields
}

type ScoreInput struct {
	TaskID             string
	CommandResults     []CommandResult
	ChangedFiles       []string
	ContextCheckResult *ContextCheckResult
	ContextCheckError  string
	TraceStdout        string
	Blocked            bool
	BlockReason        string
}

type Status string

type Suite struct {
	ID          string `json:"id"`
	Name        string `json:"name"`
	Description string `json:"description,omitempty"`
	Tasks       []Task `json:"tasks"`
}

type Summary struct {
	Total   int `json:"total"`
	Passed  int `json:"passed"`
	Failed  int `json:"failed"`
	Blocked int `json:"blocked"`
	Errors  int `json:"errors"`
}

type Task struct {
	ID                    string        `json:"id"`
	Name                  string        `json:"name"`
	Description           string        `json:"description,omitempty"`
	Tags                  []string      `json:"tags,omitempty"`
	Difficulty            string        `json:"difficulty,omitempty"`
	Prompt                string        `json:"prompt"`
	WorkspaceFixture      string        `json:"workspaceFixture"`
	VerificationCommands  []Command     `json:"verificationCommands"`
	ExpectedChangedFiles  []string      `json:"expectedChangedFiles"`
	ForbiddenChangedFiles []string      `json:"forbiddenChangedFiles,omitempty"`
	RequiredTraceEvents   []string      `json:"requiredTraceEvents,omitempty"`
	ContextChecks         ContextChecks `json:"contextChecks,omitempty"`
}

type ValidationError struct {
	Problems []string
}

type Workspace struct {
	Path        string
	TaskID      string
	FixturePath string
}