agentbench

type Case struct {
	ID    string
	Suite string
	// RequiresExa skips the case when EXA_API_KEY is empty (not a failure).
	RequiresExa bool
	// RestoreSession: after the first N-1 turns, rebuild agent from session for the last turn.
	RestoreSession bool
	Turns          []string
	// InterruptChoiceTitle is matched against ask_user_choice options (case-insensitive contains).
	// When empty, selectionIdx 0 is used.
	InterruptChoiceTitle string
	// InterruptSelectionIdx used when title is empty or unmatched.
	InterruptSelectionIdx int
	Seed                  SeedWorld
	Expect                Expect
}

type CaseResult struct {
	ID      string   `json:"id"`
	Suite   string   `json:"suite"`
	Skipped bool     `json:"skipped,omitempty"`
	SkipWhy string   `json:"skip_why,omitempty"`
	Success bool     `json:"success"`
	Notes   []string `json:"notes,omitempty"`
	// Scores are named 0/1 or fractions for aggregation.
	Scores map[string]float64 `json:"scores,omitempty"`
	Turns  []TurnTrace        `json:"-"` // omit heavy traces from default JSON unless verbose
}

type Config struct {
	Suites      []string
	CaseFilter  string // optional exact case id
	ModelURL    string
	ModelAPIKey string
	ModelName   string
	// EmbedURL defaults to ModelURL. EmbedAPIKey defaults to ModelAPIKey.
	// EmbedModel defaults to DefaultEmbedModel / OPENAI_EMBEDDING_MODEL.
	EmbedURL    string
	EmbedAPIKey string
	EmbedModel  string
	// LexicalOnly disables the dense channel (no WithEmbedder).
	LexicalOnly bool
	ExaAPIKey   string
	Timeout     time.Duration // per case
	DryRun      bool
}

type Expect struct {
	// FinalContainsAny: final assistant text must contain at least one (case-insensitive).
	FinalContainsAny []string
	// FinalContainsAll: every string must appear.
	FinalContainsAll []string
	// MustTools: each inner list is an OR group; every group must match some tool name used.
	MustTools [][]string
	// MustNotTools: none of these tool names may appear.
	MustNotTools []string
	// MustInterrupt: at least one ask_user_choice (or any interrupt) must have fired.
	MustInterrupt bool
	// BrainKindContains: after the case, FindObjects or list kinds via search for this kind
	// with query substring must return ≥1 hit (empty Query → any of that kind via find with kind filter + broad query).
	BrainKind  string
	BrainQuery string
	// BrainTitleContains: some object of BrainKind (or any if empty) has title containing this.
	BrainTitleContains string
	// GoldEvidenceIDs: at least one must appear in tool args or results text (evidence hit).
	GoldEvidenceIDs []uuid.UUID
}

type OpenAIEmbedder struct {
	BaseURL    string
	APIKey     string
	Model      string
	HTTPClient *http.Client
}

type Report struct {
	Model      string                 `json:"model"`
	EmbedModel string                 `json:"embed_model,omitempty"`
	StartedAt  time.Time              `json:"started_at"`
	Duration   time.Duration          `json:"duration_ms"` // wall; JSON via custom if needed
	Suites     map[string]SuiteResult `json:"suites"`
	GatesOK    bool                   `json:"gates_ok"`
	GateNotes  []string               `json:"gate_notes,omitempty"`
}

type SeedEdge struct {
	From, To uuid.UUID
	Relation string
	Note     string
}

type SeedObject struct {
	ID       uuid.UUID
	Kind     string
	Title    string
	Summary  string
	Content  string
	ParentID *uuid.UUID
	Position *int
	Props    map[string]any
}

type SeedWorld struct {
	// Objects with fixed IDs (for multihop evidence gold). Empty ID → generated on seed.
	Objects []SeedObject
	Edges   []SeedEdge
}

type SuiteResult struct {
	Suite       string       `json:"suite"`
	N           int          `json:"n"`
	Skipped     int          `json:"skipped"`
	Passed      int          `json:"passed"`
	Failed      int          `json:"failed"`
	SuccessRate float64      `json:"success_rate"` // among non-skipped
	IllegalRate float64      `json:"illegal_rate"`
	Cases       []CaseResult `json:"cases"`
}

type ToolCallRecord struct {
	Name      string
	Arguments string
	Result    string // best-effort from tool_result content
}

type TurnTrace struct {
	Prompt       string
	Assistant    string
	Tools        []ToolCallRecord
	Interrupts   int
	Error        string
	Duration     time.Duration
	RestoredSess bool
}