agenteval

type EvalSummary struct {
	Total            int
	Passed           int
	TotalTokens      int
	PromptTokens     int
	CompletionTokens int
	TotalToolCalls   int
	TotalDuration    time.Duration
}

type Runner struct {
	// ModelName overrides the default model from settings.yaml when non-empty.
	ModelName string
}

type Task struct {
	// ID is the unique identifier (from frontmatter or derived from filename).
	ID string
	// Title is a short human-readable name shown in output.
	Title string
	// TimeoutSecs is the per-task wall-clock timeout covering agent run + grading.
	TimeoutSecs int
	// FixtureDir is the directory to copy into a temp workspace before the agent runs.
	// Empty means the agent starts in an empty workspace.
	FixtureDir string
	// Prompt is the task description sent to the agent as the user message.
	Prompt string
	// Grader is a shell script executed in the workspace after the agent run.
	// Exit code 0 means pass; non-zero means fail.
	Grader string
}

type TaskResult struct {
	TaskID    string        `json:"task_id"`
	Title     string        `json:"title"`
	Pass      bool          `json:"pass"`
	Duration  time.Duration `json:"duration_ns"`
	Model     string        `json:"model"`
	Timestamp time.Time     `json:"timestamp"`
	// Token counts for the agent run (all LLM calls summed).
	PromptTokens     int    `json:"prompt_tokens"`
	CompletionTokens int    `json:"completion_tokens"`
	TotalTokens      int    `json:"total_tokens"`
	ToolCalls        int    `json:"tool_calls"`
	Reply            string `json:"reply,omitempty"`
	GraderOutput     string `json:"grader_output,omitempty"`
	AgentError       string `json:"agent_error,omitempty"`
	GraderError      string `json:"grader_error,omitempty"`
}