eval

type AgentDispatcher interface {
	// DispatchToHarness executes a fan-out batch and returns once all slots
	// complete, error, or time out. See internal/engine.DispatchRequest for
	// the full field contract.
	DispatchToHarness(ctx context.Context, req DispatchRequest) (*DispatchBatchResult, error)
}

type BusEmitter interface {
	// EmitCogBlock emits a serialized CogBlock to the named bus channel.
	// channelName is the raw bus channel id, e.g. "bus_tournament".
	// block is a JSON-serializable payload; the bus layer wraps it in the
	// ADR-084 pointer-envelope (digest → BlobStore, metadata in envelope).
	EmitCogBlock(ctx context.Context, channelName string, block any) error
}

type BusEvent struct {
	V       int                    `json:"v"`
	BusID   string                 `json:"bus_id,omitempty"`
	Seq     int                    `json:"seq,omitempty"`
	Ts      string                 `json:"ts"`
	From    string                 `json:"from"`
	Type    string                 `json:"type"`
	Payload map[string]interface{} `json:"payload,omitempty"`
	Hash    string                 `json:"hash"`
}

type BusReader interface {
	// ReadChannel reads events from the named channel. since is a hint
	// (hash or timestamp) for incremental reads; implementations may ignore it
	// and return all events (design memo Q2 recommends all-time reads).
	ReadChannel(ctx context.Context, channelName string, since string) ([]BusEvent, error)
}

type Case struct {
	// Name is the stable identifier for this case, matching the task variant ID.
	Name string `json:"name"`
	// Prompt is the user-turn text sent to the model.
	Prompt string `json:"prompt"`
	// Rubric holds the scoring constraints.
	Rubric Rubric `json:"rubric"`
	// SystemPrompt, if non-empty, overrides the default system prompt for this case.
	// Set from the trial's system-prompt variant before dispatch.
	SystemPrompt string `json:"system_prompt,omitempty"`
	// Tags are arbitrary labels inherited from the task variant.
	Tags []string `json:"tags,omitempty"`
	// MaxTokens is the per-trial token budget. Default 1024.
	MaxTokens int `json:"max_tokens"`
}

type Delta struct {
	// VariantKey and BaselineKey identify the compared variants.
	VariantKey  string `json:"variant_key"`
	BaselineKey string `json:"baseline_key"`
	// Delta is positive for improvement, negative for regression.
	// math.Inf(-1) when variant has no data.
	Delta float64 `json:"delta"`
	// VariantPassRate and BaselinePassRate are the aggregated pass rates.
	// nil when no data is available.
	VariantPassRate  *float64 `json:"variant_pass_rate,omitempty"`
	BaselinePassRate *float64 `json:"baseline_pass_rate,omitempty"`
	// TaskDeltas maps task_id → per-task delta (nil = missing data).
	TaskDeltas map[string]*float64 `json:"task_deltas,omitempty"`
}

type DispatchBatchResult struct {
	Results          []DispatchResult `json:"results"`
	TotalDurationSec float64          `json:"total_duration_sec"`
	Notes            []string         `json:"notes,omitempty"`
}

type DispatchRequest struct {
	AgentID        string
	Task           string
	Tools          []string
	Model          string // matches DispatchModel string type
	TimeoutSeconds int
	N              int
	SystemPrompt   string
	Thinking       *bool
}

type DispatchResult struct {
	Index       int                       `json:"index"`
	Success     bool                      `json:"success"`
	Content     string                    `json:"content,omitempty"`
	ToolCalls   []DispatchToolCallSummary `json:"tool_calls,omitempty"`
	Error       string                    `json:"error,omitempty"`
	DurationSec float64                   `json:"duration_sec"`
	Turns       int                       `json:"turns"`
	ModelUsed   string                    `json:"model_used,omitempty"`
}

type DispatchScoredResult struct {
	// contains filtered or unexported fields
}

type DispatchToolCallSummary struct {
	Name         string `json:"name"`
	ArgsDigest   string `json:"args_digest,omitempty"`
	ResultDigest string `json:"result_digest,omitempty"`
	Error        string `json:"error,omitempty"`
}

type EvalActionType string

type EvalConfig struct {
	// Experiments is the set of declared experiments, keyed by experiment ID.
	Experiments map[string]*Experiment `json:"experiments"`
	// BaselinePins maps experiment ID → pinned run ID.
	// Populated from .cog/state/eval-baselines.json (design memo Q1).
	BaselinePins map[string]string `json:"baseline_pins,omitempty"`
	// TournamentRoot is the resolved filesystem path of the tournament cogdoc
	// directory, e.g. /Users/.../cog/.cog/mem/semantic/architecture/tournament.
	// Populated by LoadConfig from workspace root + uri.go "mem" projection.
	TournamentRoot string `json:"tournament_root,omitempty"`
}

type EvalLiveState struct {
	// Trials is the flat list of all completed trial records fetched from the bus.
	Trials []TrialRecord `json:"trials"`
	// Scorecards maps experiment ID → computed scorecard over all fetched trials.
	Scorecards map[string]*Scorecard `json:"scorecards"`
	// FetchedAt is the ISO-8601 timestamp when this snapshot was taken.
	FetchedAt string `json:"fetched_at"`
}

type EvalPlanDetail struct {
	// ExperimentID identifies which experiment this action targets.
	ExperimentID string `json:"experiment_id"`
	// EvalAction is the specific eval operation.
	EvalAction EvalActionType `json:"eval_action"`
	// TrialSpecs are the specific trials to run for incremental and retry actions.
	// Empty for full-run actions (expand_matrix is called at ApplyPlan time).
	TrialSpecs []TrialSpec `json:"trial_specs,omitempty"`
	// RegressionCells lists (variant_key, task_id) pairs that regressed.
	// Populated for EvalActionRetryRegression.
	RegressionCells [][2]string `json:"regression_cells,omitempty"`
	// StaleAfter is the ISO-8601 time after which the baseline is considered stale.
	StaleAfter string `json:"stale_after,omitempty"`
}

type EvalProvider struct {
	// contains filtered or unexported fields
}

type EvalProviderState struct {
	// InFlightTrialIDs lists trial IDs currently being dispatched.
	// Checked by ComputePlan to avoid re-planning in-flight work.
	InFlightTrialIDs []string `json:"in_flight_trial_ids,omitempty"`
	// RecentFailureCounts maps experiment ID → consecutive failure count.
	// When > CircuitBreakerThreshold, ComputePlan skips that experiment.
	RecentFailureCounts map[string]int `json:"recent_failure_counts,omitempty"`
	// LastReconcileAt is the ISO-8601 time of the last completed reconcile.
	LastReconcileAt string `json:"last_reconcile_at,omitempty"`
	// CircuitBreakerThreshold is the failure count above which an experiment
	// is suspended until manually reset. Default 3.
	CircuitBreakerThreshold int `json:"circuit_breaker_threshold,omitempty"`
}

type Experiment struct {
	// ID is the stable identifier, e.g. "exp-001-anti-pattern-placement".
	ID string `json:"id"`
	// Title is the human-readable experiment title from frontmatter.
	Title string `json:"title"`
	// BaselineVariant is the composite key for the baseline cell,
	// e.g. "sp-1-production+td-1-current".
	BaselineVariant string `json:"baseline_variant"`
	// VariantAxes maps axis name → list of variant IDs,
	// e.g. {"system_prompt": ["sp-1-production", "sp-3-stripped"]}.
	VariantAxes map[string][]string `json:"variant_axes"`
	// TaskIDs lists the task variant IDs included in this experiment.
	TaskIDs []string `json:"task_ids"`
	// Target names the dispatch target, e.g. "laptop-lms".
	Target string `json:"target"`
	// Tags are arbitrary labels.
	Tags []string `json:"tags,omitempty"`
	// AutoReconcile, when true, allows the metabolic cycle to run this
	// experiment automatically. Defaults false (on-demand only).
	// Wired from the cogdoc frontmatter key "auto_reconcile: true";
	// when false the experiment only runs via explicit cog_run_experiment.
	AutoReconcile bool `json:"auto_reconcile,omitempty"`
	// BaselinePinned is the run ID of the pinned baseline, if any.
	// Set externally via cog_pin_baseline MCP tool (see design memo Q10).
	BaselinePinned string `json:"baseline_pinned,omitempty"`
}

type FileBusReader struct {
	// contains filtered or unexported fields
}

type HTTPBusReader struct {
	// contains filtered or unexported fields
}

type Rubric struct {
	// ExpectedTools are tool names that MUST appear in the tool-call sequence.
	ExpectedTools []string `json:"expected_tools,omitempty"`
	// ExpectedToolsAnyOf requires at least ONE of these names to appear.
	ExpectedToolsAnyOf []string `json:"expected_tools_any_of,omitempty"`
	// ForbiddenTools are tool names that MUST NOT appear.
	ForbiddenTools []string `json:"forbidden_tools,omitempty"`
	// ContentContains are strings that must appear in the assistant's final content.
	ContentContains []string `json:"content_contains,omitempty"`
	// ContentMustNotContain are strings that must NOT appear in final content.
	ContentMustNotContain []string `json:"content_must_not_contain,omitempty"`
	// ContentContainsCI is the case-insensitive variant of ContentContains.
	// Added during Phase C port to close the task-3 gap from exp-001 runs.
	ContentContainsCI []string `json:"content_contains_ci,omitempty"`
	// ContentMustNotContainCI is the case-insensitive variant of ContentMustNotContain.
	ContentMustNotContainCI []string `json:"content_must_not_contain_ci,omitempty"`
	// FirstToolOneOf constrains the first tool call to one of these names.
	FirstToolOneOf []string `json:"first_tool_one_of,omitempty"`
}

type RunSummary struct {
	// ExperimentID identifies the experiment.
	ExperimentID string `json:"experiment_id"`
	// RunID is the stable identifier for this run, e.g.
	// "exp-001-anti-pattern-placement_run_20260426T010713Z".
	RunID string `json:"run_id"`
	// StartedAt and EndedAt are ISO-8601 timestamps.
	StartedAt string `json:"started_at"`
	EndedAt   string `json:"ended_at"`
	// Total, Passed, Failed are trial counts.
	Total  int `json:"total"`
	Passed int `json:"passed"`
	Failed int `json:"failed"`
	// Target names the dispatch target used in this run.
	Target string `json:"target"`
	// Model is the inference backend used.
	Model string `json:"model,omitempty"`
}

type Scorecard struct {
	// ExperimentID identifies the experiment.
	ExperimentID string `json:"experiment_id"`
	// Cells maps (variant_key, task_id) → pass/fail aggregate.
	// variant_key is "sp-id / td-id"; task_id is the task variant id.
	// nil = no data for this cell.
	Cells map[[2]string]ScorecardCell `json:"cells"`
	// VariantKeys is the sorted list of variant keys (for deterministic output).
	VariantKeys []string `json:"variant_keys"`
	// TaskIDs is the sorted list of task IDs.
	TaskIDs []string `json:"task_ids"`
}

type ScorecardCell = *bool

type ScoredResult interface {
	// Content returns the final assistant text response.
	Content() string
	// ToolCallNames returns the names of tool calls made during the trial,
	// in invocation order.
	ToolCallNames() []string
	// FinishReason returns the reason the model stopped (e.g. "stop", "tool_calls").
	FinishReason() string
}

type ToolCallRecord struct {
	Name         string `json:"name"`
	ArgsDigest   string `json:"args_digest,omitempty"`
	ResultDigest string `json:"result_digest,omitempty"`
	Error        string `json:"error,omitempty"`
}

type TrialRecord struct {
	// TrialID is the stable identifier for this trial.
	TrialID string `json:"trial_id"`
	// ExperimentID links this record to its parent experiment.
	ExperimentID string `json:"experiment_id"`
	// VariantIDs is the axis → variant mapping for this trial.
	VariantIDs map[string]string `json:"variant_ids"`
	// TaskID is the task variant ID.
	TaskID string `json:"task_id"`
	// Target names the dispatch target.
	Target string `json:"target"`
	// Passed reports whether the trial satisfied its rubric.
	Passed bool `json:"passed"`
	// Failures lists rubric violations, empty on pass.
	Failures []string `json:"failures,omitempty"`
	// Notes are informational annotations from the scorer.
	Notes []string `json:"notes,omitempty"`
	// ToolCalls records each tool invocation made during the trial.
	ToolCalls []ToolCallRecord `json:"tool_calls,omitempty"`
	// Content is the final assistant text response.
	Content string `json:"content,omitempty"`
	// Reasoning is the model's reasoning trace, if available.
	Reasoning string `json:"reasoning,omitempty"`
	// DurationSec is the wall-clock time for this trial.
	DurationSec float64 `json:"duration_sec"`
	// Timestamp is the ISO-8601 start time of this trial.
	Timestamp string `json:"timestamp"`
	// Model is the inference backend used.
	Model string `json:"model,omitempty"`
	// TDWired indicates whether tool-description variant overrides were wired.
	// False in Phase 1 (TD axis not yet wired into dispatch).
	TDWired bool `json:"td_wired"`
	// CogBlockHash is the content-addressed hash of the CogBlock emitted for
	// this trial on bus_tournament. Empty if emission failed.
	CogBlockHash string `json:"cogblock_hash,omitempty"`
}

type TrialSpec struct {
	// TrialID is the stable identifier, e.g.
	// "exp-001__sp-3-stripped+td-1-current__task-1-state-probe".
	TrialID string `json:"trial_id"`
	// ExperimentID links this trial to its parent experiment.
	ExperimentID string `json:"experiment_id"`
	// TaskVariant is the resolved task variant.
	TaskVariant Variant `json:"task_variant"`
	// VariantIDs maps axis → variant ID for non-task axes in this trial.
	VariantIDs map[string]string `json:"variant_ids"`
	// SystemPromptVariant is the resolved system-prompt variant, or empty ID if absent.
	SystemPromptVariant *Variant `json:"system_prompt_variant,omitempty"`
	// ToolDescriptionVariant is the resolved tool-description variant, or nil if Phase 1.
	ToolDescriptionVariant *Variant `json:"tool_description_variant,omitempty"`
	// Target names the dispatch target, inherited from the experiment.
	Target string `json:"target"`
}

type Variant struct {
	// ID is the variant identifier from cogdoc frontmatter or the stem of the file.
	ID string `json:"id"`
	// Class identifies whether this variant overrides the system prompt,
	// tool descriptions, or task configuration.
	Class VariantClass `json:"variant_class"`
	// Content is the variant payload:
	//   - system-prompt:    string (body under "## Variant content")
	//   - tool-description: map[string]any (overrides: dict from frontmatter)
	//   - task:             map[string]any (case: dict from frontmatter)
	Content any `json:"content"`
	// BaselineOf links this variant to its baseline counterpart (e.g. "sp-1-production").
	BaselineOf string `json:"baseline_of,omitempty"`
	// Ablation names the specific feature this variant removes.
	Ablation string `json:"ablation,omitempty"`
	// Tags are arbitrary labels for filtering (e.g. ["tournament", "anti-pattern"]).
	Tags []string `json:"tags,omitempty"`
	// SourcePath is the absolute filesystem path from which this variant was loaded.
	SourcePath string `json:"source_path,omitempty"`
}

type VariantClass string

type Verdict struct {
	// Passed is true if all rubric constraints were satisfied.
	Passed bool `json:"passed"`
	// Failures lists each rubric constraint that was not met.
	Failures []string `json:"failures,omitempty"`
	// Notes are informational annotations (e.g. "tool_calls: [cog_read_cogdoc]").
	Notes []string `json:"notes,omitempty"`
}