eval

type AgentAnswer struct {
	Agent       string             `json:"agent"`
	Chosen      int                `json:"chosen"` // option index, -1 if unparsed
	Label       string             `json:"label"`
	Confidences map[string]float64 `json:"confidences,omitempty"`
	Reasoning   string             `json:"reasoning,omitempty"`
	Correct     bool               `json:"correct"`
	Err         string             `json:"error,omitempty"`
}

type AgentMetrics struct {
	Agent          string  `json:"agent"`
	Total          int     `json:"total"`
	Answered       int     `json:"answered"`
	Correct        int     `json:"correct"`
	Accuracy       float64 `json:"accuracy"`
	Coverage       float64 `json:"coverage"`
	MeanConfidence float64 `json:"mean_confidence"`
	Brier          float64 `json:"brier"`
	ECE            float64 `json:"ece"`
	MacroF1        float64 `json:"macro_f1,omitempty"`
}

type AggregateMetrics struct {
	Total    int     `json:"total"`
	Correct  int     `json:"correct"`
	Accuracy float64 `json:"accuracy"`
}

type Analysis struct {
	Mode           string                         `json:"mode"`
	TotalQuestions int                            `json:"total_questions"`
	Agents         []*AgentMetrics                `json:"agents"`
	PerChapter     map[int]*AggregateMetrics      `json:"per_chapter,omitempty"`
	PerDifficulty  map[string]*AggregateMetrics   `json:"per_difficulty,omitempty"`
	Confusion      map[string]map[int]map[int]int `json:"confusion,omitempty"`
	Calibration    map[string][]Bin               `json:"calibration,omitempty"`
	Debate         *DebateAnalysis                `json:"debate,omitempty"`
}

type Bin struct {
	Lo            float64 `json:"lo"`
	Hi            float64 `json:"hi"`
	Count         int     `json:"count"`
	Accuracy      float64 `json:"accuracy"`
	AvgConfidence float64 `json:"avg_confidence"`
}

type DebateAnalysis struct {
	Engaged        int            `json:"engaged"`
	Correct        int            `json:"correct"`
	Accuracy       float64        `json:"accuracy"`
	Quadrants      map[string]int `json:"quadrants,omitempty"`
	AvgDriftAgents float64        `json:"avg_drift_agents,omitempty"`
	DriftRate      float64        `json:"drift_rate,omitempty"`
}

type Mode string

type QuestionResult struct {
	Question      qa.Question             `json:"question"`
	Answers       map[string]*AgentAnswer `json:"answers,omitempty"`
	Debate        *core.FullDebateSummary `json:"debate,omitempty"`
	DebateWinner  int                     `json:"debate_winner"`
	DebateCorrect bool                    `json:"debate_correct"`
	DebateEngaged bool                    `json:"debate_engaged"`
}

type QuestionResultDetail struct {
	ID            string
	Source        string
	Chapter       int
	ChapterTitle  string
	Question      string
	Options       []string
	Answer        int
	AnswerLabel   string
	AgentAnswers  map[string]*AgentAnswer
	DebateWinner  int
	DebateCorrect bool
	DebateEngaged bool
}

type RunConfig struct {
	// Mode selects the evaluation mode. Default ModeAnswerOnly.
	Mode Mode
	// Targets are the agents to test. Required.
	Targets []Target
	// SupervisorModel powers debate mode. Required for ModeDebate/ModeBoth.
	SupervisorModel model.BaseModel[*schema.Message]
	// EngineOptions are applied to the gaslit engine used in debate mode.
	EngineOptions []core.Option
	// Verbose enables engine progress logging in debate mode.
	Verbose bool
	// Logger receives progress output. Optional.
	Logger *slog.Logger
}

type RunResult struct {
	Mode       Mode             `json:"mode"`
	Bank       *qa.QuestionBank `json:"bank"`
	Results    []QuestionResult `json:"results"`
	StartedAt  time.Time        `json:"started_at"`
	FinishedAt time.Time        `json:"finished_at"`
}

type Target struct {
	Name  string
	Agent adk.Agent
}