evaluation

type Baseline struct {
	Name    string
	Summary Summary
	// Passed maps an evaluation's key (see evalKey) to whether it passed.
	Passed map[string]bool
}

type Comparison struct {
	Baseline  Metrics       `json:"baseline"`
	Current   Metrics       `json:"current"`
	Tolerance float64       `json:"tolerance"`
	Deltas    []MetricDelta `json:"deltas"`
	Changes   []EvalChange  `json:"changes"`
	// Regressed is true when any gating metric moved beyond the tolerance, or
	// an evaluation that passed in the baseline now fails.
	Regressed bool `json:"regressed"`
}

type Config struct {
	AgentFilename    string   // Path to the agent configuration file
	EvalsDir         string   // Directory containing evaluation files
	JudgeModel       string   // Model for relevance checking (format: provider/model, optional)
	Concurrency      int      // Number of concurrent runs (0 = number of CPUs)
	TTYFd            int      // File descriptor for terminal size queries (e.g., int(os.Stdout.Fd()))
	Only             []string // Only run evaluations matching these patterns
	BaseImage        string   // Custom base image for running evaluations
	KeepContainers   bool     // If true, don't remove containers after evaluation (skip --rm)
	EnvVars          []string // Environment variables to pass: KEY (value from env) or KEY=VALUE (explicit)
	Repeat           int      // Number of times to repeat each evaluation (default 1)
	ContainerRuntime string   // Docker-compatible container runtime executable (default "docker")
}

type EvalChange struct {
	Eval string `json:"eval"`
	// Was and Now are "pass", "fail", or "absent".
	Was       string `json:"was"`
	Now       string `json:"now"`
	Regressed bool   `json:"regressed"`
}

type EvalRun struct {
	Name      string        `json:"name"`
	Timestamp time.Time     `json:"timestamp"`
	Duration  time.Duration `json:"duration"`
	Config    Config        `json:"-"` // Used to build RunOutput, not serialized directly
	Results   []Result      `json:"results"`
	Summary   Summary       `json:"summary"`
}

type InputSession struct {
	*session.Session

	SourcePath  string // Path to the source eval file (not serialized)
	RepeatIndex int    // Repeat iteration (1-based); 0 means no repeat
}

type Judge struct {
	// contains filtered or unexported fields
}

type MetricDelta struct {
	Name      string  `json:"name"`
	Baseline  float64 `json:"baseline"`
	Current   float64 `json:"current"`
	Delta     float64 `json:"delta"`
	Regressed bool    `json:"regressed"`
	// Informational marks a metric that is reported but never gates, so a
	// reviewer can see it moved without the build failing over it.
	Informational bool `json:"informational,omitempty"`
}

type Metrics struct {
	TotalEvals  int     `json:"total_evals"`
	FailedEvals int     `json:"failed_evals"`
	FailureRate float64 `json:"failure_rate"`

	SizePassRate float64 `json:"size_pass_rate"`
	HasSizes     bool    `json:"has_sizes"`

	ToolsF1Mean float64 `json:"tools_f1_mean"`
	HasTools    bool    `json:"has_tools"`

	RelevanceRate float64 `json:"relevance_rate"`
	HasRelevance  bool    `json:"has_relevance"`

	TotalCost float64 `json:"total_cost"`
}

type RelevanceResult struct {
	Criterion string `json:"criterion"`
	Passed    bool   `json:"passed"`
	Reason    string `json:"reason"`
}

type Result struct {
	InputPath         string            `json:"input_path"`
	Title             string            `json:"title"`
	Question          string            `json:"question"`
	Response          string            `json:"response"`
	Cost              float64           `json:"cost"`
	OutputTokens      int64             `json:"output_tokens"`
	Size              string            `json:"size"`
	SizeExpected      string            `json:"size_expected"`
	ToolCallsScore    float64           `json:"tool_calls_score"`
	ToolCallsExpected float64           `json:"tool_calls_score_expected"`
	RelevancePassed   float64           `json:"relevance"`
	RelevanceExpected float64           `json:"relevance_expected"`
	RelevanceResults  []RelevanceResult `json:"relevance_results,omitempty"`
	Error             string            `json:"error,omitempty"`
	RawOutput         []map[string]any  `json:"raw_output,omitempty"`
	Session           *session.Session  `json:"-"` // Full session for database storage (not in JSON)
}

type RunOutput struct {
	Name      string             `json:"name"`
	Timestamp time.Time          `json:"timestamp"`
	Duration  string             `json:"duration"`
	Config    RunOutputConfig    `json:"config"`
	Summary   Summary            `json:"summary"`
	Sessions  []*session.Session `json:"sessions"`
}

type RunOutputConfig struct {
	Agent            string `json:"agent"`
	JudgeModel       string `json:"judge_model,omitempty"`
	Concurrency      int    `json:"concurrency"`
	EvalsDir         string `json:"evals_dir"`
	BaseImage        string `json:"base_image,omitempty"`
	ContainerRuntime string `json:"container_runtime,omitempty"`
}

type Runner struct {
	Config
	// contains filtered or unexported fields
}

type Summary struct {
	TotalEvals      int     `json:"total_evals"`
	FailedEvals     int     `json:"failed_evals"`
	TotalCost       float64 `json:"total_cost"`
	SizesPassed     int     `json:"sizes_passed"`
	SizesTotal      int     `json:"sizes_total"`
	ToolsF1Sum      float64 `json:"tools_f1_sum"`
	ToolsCount      int     `json:"tools_count"`
	RelevancePassed float64 `json:"relevance_passed"`
	RelevanceTotal  float64 `json:"relevance_total"`
}