eval

type Agreement struct {
	Items  int     `json:"items"`
	Agreed int     `json:"agreed"`
	Rate   float64 `json:"rate"`
}

type Assignment struct {
	A int64 `json:"a"`
	B int64 `json:"b"`
}

type BlindedItem struct {
	ItemID   int64  `json:"item_id"`
	RunID    int64  `json:"run_id"`
	TaskID   int64  `json:"task_id"`
	Prompt   string `json:"prompt"`
	Category string `json:"category"`
	// Notes is the task's free-text "what good looks like". Judge context, not
	// an assertion — nothing parses it.
	Notes string `json:"notes,omitempty"`
	// PinnedHistory is the context the turn ran against, so a verdict can tell
	// a non-sequitur from a correct follow-up.
	PinnedHistory json.RawMessage `json:"pinned_history,omitempty"`
	Status        string          `json:"status"`
	ResponseA     BlindedResponse `json:"response_a"`
	ResponseB     BlindedResponse `json:"response_b"`
}

type BlindedResponse struct {
	Response   string            `json:"response"`
	Rounds     int               `json:"rounds"`
	StopReason string            `json:"stop_reason,omitempty"`
	ToolCalls  []BlindedToolCall `json:"tool_calls"`
}

type BlindedToolCall struct {
	Round     int    `json:"round"`
	Name      string `json:"tool_name"`
	Server    string `json:"server_name,omitempty"`
	Outcome   string `json:"outcome"`
	Arguments string `json:"arguments,omitempty"`
	Result    string `json:"result,omitempty"`
	Error     string `json:"error,omitempty"`
}

type CategoryResult struct {
	Category    string  `json:"category"`
	JudgedPairs int     `json:"judged_pairs"`
	Wins        int     `json:"wins"`
	Losses      int     `json:"losses"`
	Ties        int     `json:"ties"`
	WinRate     float64 `json:"win_rate"`
	// Deltas mirror the three gates, restricted to this category's tasks.
	DeltaRejectedPP float64 `json:"delta_rejected_pp"`
	DeltaRoundsPct  float64 `json:"delta_rounds_pct"`
	DeltaCostPct    float64 `json:"delta_cost_pct"`
	// Regressed is true when this category alone would fail a gate or fall
	// below the win threshold, whatever the aggregate says.
	Regressed bool `json:"regressed"`
}

type Completeness struct {
	SamplesOK       int     `json:"samples_ok"`
	SamplesExpected int     `json:"samples_expected"`
	Ratio           float64 `json:"ratio"`
	Floor           float64 `json:"floor"`
	Conclusive      bool    `json:"conclusive"`
	// Pairs and PairsJudged sit next to the sample figures because a run can be
	// sample-complete and still have holes in the judging grid: a (task, k)
	// whose sample failed on either side yields no pair at all.
	Pairs       int `json:"pairs"`
	PairsJudged int `json:"pairs_judged"`
}

type Config struct {
	MaxConcurrent     int
	MaxCostPerRun     float64
	DefaultK          int
	CompletenessFloor float64
	AuditMode         string
}

type Engine interface {
	// DryRun executes one turn under an execution policy and persists nothing.
	DryRun(ctx context.Context, msg adapter.IncomingMessage, policy agent.ExecPolicy) (*agent.TurnResult, error)
	// LLMRouter exposes the cost tracker (real per-sample spend) and provider
	// registry (overlay validation).
	LLMRouter() *llm.Router
	// Name is the base agent's name, used for the audit pseudo-identity.
	Name() string
}

type EngineSource func(name string) (Engine, bool)

type Gate struct {
	Name     string  `json:"name"`
	Baseline float64 `json:"baseline"`
	Value    float64 `json:"value"`
	Delta    float64 `json:"delta"`
	// Threshold is the largest Delta that still passes, in Unit.
	Threshold float64 `json:"threshold"`
	// Unit is "pp" (percentage points, for rates) or "%" (relative change).
	Unit string `json:"unit"`
	Pass bool   `json:"pass"`
}

type ImportError struct {
	Line int
	Err  error
}

type JSONLTask struct {
	Prompt               string          `json:"prompt"`
	Category             string          `json:"category"`
	PinnedHistory        json.RawMessage `json:"pinned_history,omitempty"`
	Tags                 json.RawMessage `json:"tags,omitempty"`
	Notes                string          `json:"notes,omitempty"`
	SourceConversationID string          `json:"source_conversation_id,omitempty"`
	SourceMessageID      *int64          `json:"source_message_id,omitempty"`
}

type Judgment struct {
	Pairs int `json:"pairs"`
	// JudgedPairs counts pairs whose *both* presentation orders carry a judge
	// verdict. A half-judged pair is not evidence.
	JudgedPairs  int     `json:"judged_pairs"`
	Wins         int     `json:"wins"`
	Losses       int     `json:"losses"`
	Ties         int     `json:"ties"`
	WinRate      float64 `json:"win_rate"`
	WinThreshold float64 `json:"win_threshold"`
	// OperatorAgreement is nil until the operator marks a calibration item.
	OperatorAgreement *Agreement `json:"operator_agreement,omitempty"`
}

type JudgmentItem struct {
	ID                int64     `db:"id"                 json:"id"`
	PairID            int64     `db:"pair_id"            json:"pair_id"`
	PresentationOrder string    `db:"presentation_order" json:"presentation_order"`
	Status            string    `db:"status"             json:"status"`
	CreatedAt         time.Time `db:"created_at"         json:"created_at"`
}

type Overlay struct {
	Model    string `json:"llm_model,omitempty"`
	Provider string `json:"llm_provider,omitempty"`
}

type Pair struct {
	ID      int64 `db:"id"        json:"id"`
	RunID   int64 `db:"run_id"    json:"run_id"`
	TaskID  int64 `db:"task_id"   json:"task_id"`
	KIndex  int   `db:"k_index"   json:"k_index"`
	SampleA int64 `db:"sample_a"  json:"sample_a"`
	SampleB int64 `db:"sample_b"  json:"sample_b"`
	// Assignment is the letter→variant map. It is the unblinding key and must
	// never reach a judge-visible payload.
	Assignment string    `db:"assignment" json:"assignment"`
	CreatedAt  time.Time `db:"created_at" json:"created_at"`
}

type PendingItem struct {
	ItemID   int64  `db:"item_id"  json:"item_id"`
	PairID   int64  `db:"pair_id"  json:"pair_id"`
	RunID    int64  `db:"run_id"   json:"run_id"`
	TaskID   int64  `db:"task_id"  json:"task_id"`
	Category string `db:"category" json:"category"`
	// Prompt is the task's own text, which is identical for both sides and so
	// leaks nothing.
	Prompt string `db:"prompt" json:"prompt"`
}

type ProgressEvent struct {
	RunID        int64   `json:"run_id"`
	Status       string  `json:"status"`
	SamplesDone  int     `json:"samples_done"`
	SamplesTotal int     `json:"samples_total"`
	CostSpent    float64 `json:"cost_spent"`
	CostCap      float64 `json:"cost_cap"`
	ETASeconds   int     `json:"eta_seconds,omitempty"`
}

type Run struct {
	ID         int64      `db:"id"          json:"id"`
	TaskSetID  int64      `db:"task_set_id" json:"task_set_id"`
	BaseAgent  string     `db:"base_agent"  json:"base_agent"`
	Status     string     `db:"status"      json:"status"`
	K          int        `db:"k"           json:"k"`
	CostCap    float64    `db:"cost_cap"    json:"cost_cap"`
	CostSpent  float64    `db:"cost_spent"  json:"cost_spent"`
	AsOf       time.Time  `db:"as_of"       json:"as_of"`
	Error      string     `db:"error"       json:"error,omitempty"`
	CreatedAt  time.Time  `db:"created_at"  json:"created_at"`
	FinishedAt *time.Time `db:"finished_at" json:"finished_at,omitempty"`
}

type Runner struct {

	// OnProgress is called after each sample and at run start/finish. Nil-safe.
	OnProgress func(ProgressEvent)
	// contains filtered or unexported fields
}

type Sample struct {
	ID         int64  `db:"id"          json:"id"`
	RunID      int64  `db:"run_id"      json:"run_id"`
	VariantID  int64  `db:"variant_id"  json:"variant_id"`
	TaskID     int64  `db:"task_id"     json:"task_id"`
	KIndex     int    `db:"k_index"     json:"k_index"`
	Status     string `db:"status"      json:"status"`
	Error      string `db:"error"       json:"error,omitempty"`
	Response   string `db:"response"    json:"response"`
	Trace      string `db:"trace"       json:"trace"`
	Rounds     int    `db:"rounds"      json:"rounds"`
	StopReason string `db:"stop_reason" json:"stop_reason,omitempty"`
	// Upstream is the provider-reported serving upstream (OpenRouter's routed
	// provider), empty for providers without the concept.
	Upstream string `db:"upstream" json:"upstream,omitempty"`
	// Outcome counts are tool-call level, split exactly as
	// agent.ToolCallRecord.Outcome. Cached and suppressed are kept separate
	// from failed on purpose: folding either in would poison the failed-rate
	// gate, and an eval turn suppresses writes routinely.
	OutcomeOK         int       `db:"outcome_ok"         json:"outcome_ok"`
	OutcomeRejected   int       `db:"outcome_rejected"   json:"outcome_rejected"`
	OutcomeFailed     int       `db:"outcome_failed"     json:"outcome_failed"`
	OutcomeDenied     int       `db:"outcome_denied"     json:"outcome_denied"`
	OutcomeCached     int       `db:"outcome_cached"     json:"outcome_cached"`
	OutcomeSuppressed int       `db:"outcome_suppressed" json:"outcome_suppressed"`
	TokensPrompt      int       `db:"tokens_prompt"      json:"tokens_prompt"`
	TokensCompletion  int       `db:"tokens_completion"  json:"tokens_completion"`
	Cost              float64   `db:"cost"               json:"cost"`
	LatencyMs         int64     `db:"latency_ms"         json:"latency_ms"`
	CreatedAt         time.Time `db:"created_at"         json:"created_at"`
}

type Store struct {
	// contains filtered or unexported fields
}

type Summary struct {
	RunID     int64   `json:"run_id"`
	Status    string  `json:"status"`
	BaseAgent string  `json:"base_agent"`
	TaskSet   string  `json:"task_set"`
	K         int     `json:"k"`
	CostCap   float64 `json:"cost_cap"`
	CostSpent float64 `json:"cost_spent"`
	// BaselineVariant names the variant per-task deltas are measured against:
	// the first by creation order. This is a convention (the incumbent is
	// created first), not something the API enforces.
	BaselineVariant string           `json:"baseline_variant"`
	Variants        []VariantMetrics `json:"variants"`
	PerTask         []TaskMetrics    `json:"per_task"`
	Completeness    Completeness     `json:"completeness"`
	// Verdicts holds one decision per non-baseline variant, each with its gate
	// table, judge tally and per-category breakdown. Present even before any
	// judging: the objective half alone can already say "downgrade" or "no
	// regressions detected".
	Verdicts []VariantVerdict `json:"verdicts"`
}

type SummaryOpts struct {
	CompletenessFloor float64
	// WinThreshold is the judge win-rate a candidate must reach to be called an
	// upgrade.
	WinThreshold float64
	// GateRejectedPP is the largest tolerated rise in rejected tool-call rate,
	// in percentage points.
	GateRejectedPP float64
	// GateRoundsPct and GateCostPct are the largest tolerated relative rises in
	// mean rounds and cost per task.
	GateRoundsPct float64
	GateCostPct   float64
}

type Task struct {
	ID       int64  `db:"id"       json:"id"`
	SetID    int64  `db:"set_id"   json:"set_id"`
	Prompt   string `db:"prompt"   json:"prompt"`
	Category string `db:"category" json:"category"`
	// PinnedHistory is a JSON array of {role, content} replayed verbatim as
	// the context preceding the turn. NULL/empty means a fresh turn.
	PinnedHistory        string    `db:"pinned_history"         json:"pinned_history,omitempty"`
	SourceConversationID string    `db:"source_conversation_id" json:"source_conversation_id,omitempty"`
	SourceMessageID      *int64    `db:"source_message_id"      json:"source_message_id,omitempty"`
	Tags                 string    `db:"tags"                   json:"tags"`
	Notes                string    `db:"notes"                  json:"notes"`
	CreatedAt            time.Time `db:"created_at"             json:"created_at"`
}

type TaskMetrics struct {
	TaskID   int64                `json:"task_id"`
	Prompt   string               `json:"prompt"`
	Category string               `json:"category"`
	Variants []TaskVariantMetrics `json:"variants"`
}

type TaskPatch struct {
	Prompt        *string
	Category      *string
	PinnedHistory *string
	Tags          *string
	Notes         *string
}

type TaskSet struct {
	ID          int64     `db:"id"          json:"id"`
	Name        string    `db:"name"        json:"name"`
	Description string    `db:"description" json:"description"`
	CreatedAt   time.Time `db:"created_at"  json:"created_at"`
	// TaskCount is populated by ListTaskSets; it is not a column.
	TaskCount int `db:"task_count" json:"task_count"`
}

type TaskVariantMetrics struct {
	VariantID   int64   `json:"variant_id"`
	Name        string  `json:"name"`
	SamplesOK   int     `json:"samples_ok"`
	MeanCost    float64 `json:"mean_cost"`
	MeanRounds  float64 `json:"mean_rounds"`
	MeanLatency float64 `json:"mean_latency_ms"`
	// Deltas are against the baseline variant and are zero on the baseline
	// row itself.
	DeltaCost    float64 `json:"delta_cost"`
	DeltaRounds  float64 `json:"delta_rounds"`
	DeltaLatency float64 `json:"delta_latency_ms"`
}

type Variant struct {
	ID      int64  `db:"id"      json:"id"`
	RunID   int64  `db:"run_id"  json:"run_id"`
	Name    string `db:"name"    json:"name"`
	Overlay string `db:"overlay" json:"overlay"`
}

type VariantMetrics struct {
	VariantID int64   `json:"variant_id"`
	Name      string  `json:"name"`
	Overlay   Overlay `json:"overlay"`

	// RejectedRate and FailedRate are tool-call level: the denominator is
	// ok+rejected+failed+denied. Cached and suppressed calls are excluded
	// because nothing executed, so counting them would dilute both rates with
	// non-events.
	RejectedRate float64 `json:"rejected_rate"`
	FailedRate   float64 `json:"failed_rate"`
	// ToolCalls is that denominator, so a reader can tell a 0 % rate over 200
	// calls from a 0 % rate over none.
	ToolCalls int `json:"tool_calls"`

	MeanRounds float64 `json:"mean_rounds"`
	// WrapupCount is samples whose loop was cut short by repeated identical
	// calls or by exhausting the round budget — the "flaily" signal.
	WrapupCount     int     `json:"wrapup_count"`
	MeanCostPerTask float64 `json:"mean_cost_per_task"`
	MeanLatencyMs   float64 `json:"mean_latency_ms"`
	TotalCost       float64 `json:"total_cost"`

	SamplesOK     int `json:"samples_ok"`
	SamplesFailed int `json:"samples_failed"`
}

type VariantVerdict struct {
	VariantID int64  `json:"variant_id"`
	Variant   string `json:"variant"`
	Baseline  string `json:"baseline"`
	Verdict   string `json:"verdict"`
	// Reason is the one-line plain-language explanation, e.g. "downgrade: mean
	// rounds regressed +35% against a +20% threshold".
	Reason     string           `json:"reason"`
	Gates      []Gate           `json:"gates"`
	Judgment   Judgment         `json:"judgment"`
	Categories []CategoryResult `json:"categories"`
	// Divergence is set when the aggregate and a category disagree, e.g. "wins
	// overall; regresses on tool_heavy". v1 surfaces this prominently without
	// gating on it.
	Divergence string `json:"divergence,omitempty"`
}

type Verdict struct {
	ID     int64  `db:"id"      json:"id"`
	ItemID int64  `db:"item_id" json:"item_id"`
	Winner string `db:"winner" json:"winner"`
	// Dimensions is a JSON object of dimension → winner (a/b/tie), the same
	// pairwise form as Winner rather than absolute scores: a judge comparing
	// two responses is reliable, a judge scoring one in isolation is not.
	Dimensions string `db:"dimensions" json:"dimensions"`
	Notes      string `db:"notes"      json:"notes"`
	// JudgeIdent names who judged — an API key name, or JudgeOperator for the
	// operator's calibration marks.
	JudgeIdent string    `db:"judge_ident" json:"judge_ident"`
	CreatedAt  time.Time `db:"created_at"  json:"created_at"`
}