runners

type AnswerDetails struct {
	// Title is a descriptive header for the response produced by the target AI model.
	Title string
	// Explanation of the answer produced by the target AI model.
	Explanation []string
	// ActualAnswer is the raw answer from the target AI model split into lines.
	ActualAnswer []string
	// ExpectedAnswer is a set of all acceptable correct answers, each being an array of lines.
	ExpectedAnswer [][]string
	// Usage contains token usage statistics for generating the answer.
	Usage TokenUsage
	// ToolUsage contains aggregated execution statistics for any tools invoked while
	// producing the answer.
	ToolUsage map[string]ToolUsage `json:"ToolUsage,omitempty"`
	// ToolCalls contains a log of every individual invocation attempt made while producing
	// the answer, including attempts that never actually ran. Tracked separately from
	// ToolUsage, which only reflects invocations that actually ran.
	ToolCalls []ToolCallSummary `json:"ToolCalls,omitempty"`
}

type AsyncResultSet interface {
	// GetResults returns the task results for each provider.
	// The call will block until the run is finished.
	GetResults() Results
	// ProgressEvents returns a channel that emits run progress as a value between 0 and 1.
	// The channel will be closed when the run is finished.
	ProgressEvents() <-chan float32
	// MessageEvents returns a channel that emits run log messages.
	// The channel will be closed when the run is finished.
	MessageEvents() <-chan string
	// Cancel stops the ongoing run execution.
	Cancel()
}

type Details struct {
	// Answer contains details about the AI model's response and reasoning process.
	Answer AnswerDetails
	// Validation contains details about the answer verification and assessment.
	Validation ValidationDetails
	// Error contains details about any errors that occurred during task execution.
	Error ErrorDetails
}

type EmittingLogger struct {
	// contains filtered or unexported fields
}

type ErrorDetails struct {
	// Title provides a summary description of the error.
	Title string
	// Message contains the primary error message.
	Message string
	// Details contains any additional error information in a generic structure.
	Details map[string][]string
	// Usage contains token usage statistics if available even in error scenarios.
	// This is typically populated if the error occurs when parsing the generated response.
	Usage TokenUsage
	// ToolUsage contains aggregated execution statistics for any tools invoked prior to
	// the error.
	ToolUsage map[string]ToolUsage `json:"ToolUsage,omitempty"`
	// ToolCalls contains a log of every individual invocation attempt made prior to the
	// error, including attempts that never actually ran. Tracked separately from ToolUsage,
	// which only reflects invocations that actually ran.
	ToolCalls []ToolCallSummary `json:"ToolCalls,omitempty"`
	// Transient indicates whether the error appears temporary/external (true), appears
	// permanent/hard (false), or is unknown (nil). This is a best-effort classification,
	// not a complete error taxonomy.
	Transient *bool `json:"Transient,omitempty"`
}

type MergeStats struct {
	// Runs maps provider name → run name → merge statistics for that run.
	Runs map[string]map[string]RunMergeStats
}

type ResultKind int

type ResultSet interface {
	// GetResults returns the task results for each provider.
	GetResults() Results
}

type Results map[string][]RunResult

type RunMergeStats struct {
	// Total is the number of results for this run after merging.
	Total int
	// Updated is the number of unique tasks whose results were replaced by a later input.
	Updated int
}

type RunResult struct {
	// TraceID is a globally unique identifier for this specific task result, used for tracing and correlation.
	TraceID string
	// Kind indicates the result status.
	Kind ResultKind
	// Task is the name of the executed task.
	Task string
	// Provider is the name of the AI provider that executed the task.
	Provider string
	// Run is the name of the provider's run configuration used.
	Run string
	// Got is the actual answer received from the AI model.
	// For plain text response format, this should be a string that follows the format instruction precisely.
	// For structured schema-based response format, this will be any object that conforms to the schema.
	Got interface{}
	// Want are the accepted valid answer(s) for the task.
	// For plain text response format: contains string values that should follow the format instruction precisely.
	// For structured schema-based response format: contains object values that conform to the schema.
	Want utils.ValueSet
	// TaskMetadata carries optional descriptive labels copied from the originating task.
	TaskMetadata TaskMetadata
	// Details contains comprehensive information about the generated response and validation assessment.
	Details Details
	// Duration is the cumulative time the AI model itself spent generating a response,
	// summed across every conversation turn's model request (network + inference).
	// It excludes local tool execution time (see ToolCalls/ToolUsage) and any subsequent
	// validation time, so it is not the total wall-clock time spent processing the task.
	Duration time.Duration
}

type Runner interface {
	// Run executes all given tasks against all run configurations and returns when done.
	Run(ctx context.Context, tasks []config.Task) (ResultSet, error)
	// Start executes all given tasks against all run configurations asynchronously.
	// It returns immediately and the execution continues in the background,
	// offering progress updates and messages through the returned result set.
	Start(ctx context.Context, tasks []config.Task) (AsyncResultSet, error)
	// Close releases resources when the runner is no longer needed.
	Close(ctx context.Context)
}

type TaskMetadata struct {
	// Suite is an optional grouping label for organizing related tasks (e.g. a benchmark suite name).
	Suite string
	// Category is an optional classification label for the task (e.g. "math", "coding").
	Category string
	// Difficulty is an optional free-form difficulty label for the task (e.g. "easy", "hard").
	Difficulty string
	// Tags is an optional set of free-form labels for filtering and grouping tasks.
	Tags []string
}

type TokenUsage struct {
	// InputTokens is the number of tokens consumed by the prompt/input.
	InputTokens *int64 `json:"InputTokens,omitempty"`
	// OutputTokens is the number of tokens generated in the completion/output.
	OutputTokens *int64 `json:"OutputTokens,omitempty"`
}

type ToolCallOutput struct {
	// Bytes is the total size of the output stream, regardless of Truncated.
	Bytes int64
	// Preview is a truncated prefix of the output stream, or nil if not captured or empty.
	Preview *string
	// Truncated indicates whether Preview was cut short of the full output.
	Truncated bool
}

type ToolCallSummary struct {
	// Tool is the name of the tool this call invoked.
	Tool string
	// CallID identifies this call, letting a specific invocation be correlated between this
	// summary, the corresponding tool-call log lines (which include the same ID in their
	// prefix), and - when the calling provider's API assigns its own tool-call ID and that
	// ID was reused here - the provider's own API error messages. Falls back to a
	// generated ID when the calling provider does not supply one, so this is never empty,
	// but its shape/format is not guaranteed to be consistent across providers.
	CallID string
	// ConversationTurn is the 1-based conversation turn this call was made during, or 0 if
	// unknown.
	ConversationTurn int
	// StartedAt is when this call began (start of setup, before the underlying process runs).
	StartedAt time.Time
	// CompletedAt is when this call finished, successfully or not.
	CompletedAt time.Time
	// Duration is the wall-clock duration of the underlying process's runtime, not including
	// setup/teardown overhead. It is nil when no process ever ran (e.g. an infrastructure_error).
	Duration *time.Duration
	// WallTime is the wall-clock duration of the entire call attempt, from setup through
	// output retrieval - i.e. Duration plus setup/teardown overhead. Unlike Duration, this is
	// always set, even for calls whose underlying process never ran.
	WallTime time.Duration
	// ExitCode is the underlying process's exit code, or nil if no exit code is known.
	ExitCode *int64
	// TimedOut indicates the call was aborted due to exceeding its configured timeout.
	TimedOut bool
	// Status is one of: "success", "nonzero_exit", "empty_output", "timeout",
	// "invalid_arguments", "infrastructure_error".
	Status string
	// Stdout is a size-limited capture of the call's standard output, or nil if no output was
	// ever captured.
	Stdout *ToolCallOutput
	// Stderr is a size-limited capture of the call's standard error, or nil if no output was
	// ever captured.
	Stderr *ToolCallOutput
	// ErrorMessage is a short explanation of the failure when Status is not "success".
	ErrorMessage string
}

type ToolUsage struct {
	// CallCount is the number of times the tool's underlying process actually ran.
	CallCount *int64 `json:"CallCount,omitempty"`
	// TotalDuration is the cumulative execution time for the tool's underlying process.
	TotalDuration *time.Duration `json:"TotalDuration,omitempty"`
}

type ValidationDetails struct {
	// Title identifies the type of validation assessment performed.
	Title string
	// Explanation contains detailed analysis of why the validation succeeded or failed.
	Explanation []string
	// Usage contains token usage statistics for the response validation step.
	// This is typically populated when using an LLM judge validator.
	Usage TokenUsage
	// ToolUsage contains aggregated execution statistics for any tools invoked during
	// validation.
	ToolUsage map[string]ToolUsage `json:"ToolUsage,omitempty"`
	// ToolCalls contains a log of every individual invocation attempt made during
	// validation, including attempts that never actually ran. Tracked separately from
	// ToolUsage, which only reflects invocations that actually ran.
	ToolCalls []ToolCallSummary `json:"ToolCalls,omitempty"`
}