evaluation

type Config struct {
	JudgeModel  provider.Provider // Model for relevance checking (optional)
	Concurrency int               // Number of concurrent runs (0 = number of CPUs)
	TTYFd       int               // File descriptor for terminal size queries (e.g., int(os.Stdout.Fd()))
}

type EvalCriteria struct {
	Relevance  []string `json:"relevance,omitempty"`   // Statements that should be true about the response
	WorkingDir string   `json:"working_dir,omitempty"` // Subdirectory under evals/working_dirs/
	Size       string   `json:"size,omitempty"`        // Expected response size: S, M, L, XL
}

type EvalRun struct {
	Name      string        `json:"name"`
	Timestamp time.Time     `json:"timestamp"`
	Duration  time.Duration `json:"duration"`
	Results   []Result      `json:"results"`
	Summary   Summary       `json:"summary"`
}

type EvalSession struct {
	session.Session
	Evals EvalCriteria `json:"evals"`
}

type Result struct {
	Title             string   `json:"title"`
	Question          string   `json:"question"`
	Response          string   `json:"response"`
	Cost              float64  `json:"cost"`
	OutputTokens      int64    `json:"output_tokens"`
	Size              string   `json:"size"`
	SizeExpected      string   `json:"size_expected"`
	ToolCallsScore    float64  `json:"tool_calls_score"`
	ToolCallsExpected float64  `json:"tool_calls_score_expected"`
	HandoffsMatch     bool     `json:"handoffs"`
	RelevancePassed   float64  `json:"relevance"`
	RelevanceExpected float64  `json:"relevance_expected"`
	FailedRelevance   []string `json:"failed_relevance,omitempty"`
	Error             string   `json:"error,omitempty"`
}

type Runner struct {
	// contains filtered or unexported fields
}

type Summary struct {
	TotalEvals      int     `json:"total_evals"`
	TotalCost       float64 `json:"total_cost"`
	SizesPassed     int     `json:"sizes_passed"`
	SizesTotal      int     `json:"sizes_total"`
	ToolsPassed     float64 `json:"tools_passed"`
	ToolsTotal      float64 `json:"tools_total"`
	HandoffsPassed  int     `json:"handoffs_passed"`
	HandoffsTotal   int     `json:"handoffs_total"`
	RelevancePassed float64 `json:"relevance_passed"`
	RelevanceTotal  float64 `json:"relevance_total"`
}