runners

type AnswerDetails struct {
	// Title is a descriptive header for the response produced by the target AI model.
	Title string
	// Explanation of the answer produced by the target AI model.
	Explanation []string
	// ActualAnswer is the raw answer from the target AI model split into lines.
	ActualAnswer []string
	// ExpectedAnswer is a set of all acceptable correct answers, each being an array of lines.
	ExpectedAnswer [][]string
	// Usage contains token usage statistics for generating the answer.
	Usage TokenUsage
	// ToolUsage contains aggregated execution statistics for any tools invoked while
	// producing the answer.
	ToolUsage map[string]ToolUsage `json:"ToolUsage,omitempty"`
	// ToolCalls contains a log of every individual invocation attempt made while producing
	// the answer, including attempts that never actually ran. Tracked separately from
	// ToolUsage, which only reflects invocations that actually ran.
	ToolCalls []ToolCallSummary `json:"ToolCalls,omitempty"`
}

type AsyncResultSet interface {
	// GetResults returns the task results for each provider.
	// The call will block until the run is finished.
	GetResults() Results
	// ProgressEvents returns a channel that emits run progress as a value between 0 and 1.
	// The channel will be closed when the run is finished.
	ProgressEvents() <-chan float32
	// MessageEvents returns a channel that emits run log messages.
	// The channel will be closed when the run is finished.
	MessageEvents() <-chan string
	// Cancel stops the ongoing run execution.
	Cancel()
}

type Details struct {
	// Answer contains details about the AI model's response and reasoning process.
	Answer AnswerDetails
	// Validation contains details about the answer verification and assessment.
	Validation ValidationDetails
	// Error contains details about any errors that occurred during task execution.
	Error ErrorDetails
}

type EmittingLogger struct {
	// contains filtered or unexported fields
}

type ErrorDetails struct {
	// Title provides a summary description of the error.
	Title string
	// Message contains the primary error message.
	Message string
	// Details contains any additional error information in a generic structure.
	Details map[string][]string
	// Usage contains token usage statistics if available even in error scenarios.
	// This is typically populated if the error occurs when parsing the generated response.
	Usage TokenUsage
	// ToolUsage contains aggregated execution statistics for any tools invoked prior to
	// the error.
	ToolUsage map[string]ToolUsage `json:"ToolUsage,omitempty"`
	// ToolCalls contains a log of every individual invocation attempt made prior to the
	// error, including attempts that never actually ran. Tracked separately from ToolUsage,
	// which only reflects invocations that actually ran.
	ToolCalls []ToolCallSummary `json:"ToolCalls,omitempty"`
	// ResponseParsing is true when a model response was received but MindTrial could not
	// parse or unmarshal it into the expected result structure; nil otherwise. When
	// Transient is also nil, this kind of failure should generally be treated as
	// non-transient (retrying with the same input is unlikely to help).
	ResponseParsing *bool `json:"ResponseParsing,omitempty"`
	// Transient indicates whether the error appears temporary/external (true), appears
	// permanent/hard (false), or is unknown (nil). This is a best-effort classification,
	// not a complete error taxonomy.
	Transient *bool `json:"Transient,omitempty"`
	// FromValidation is true when Usage/ToolUsage/ToolCalls above belong to the judge's
	// validation attempt rather than the candidate response (i.e. validation itself
	// failed, after a candidate answer was already produced), so aggregate consumers
	// (e.g. the stats package, the HTML report's dynamic summary) must not attribute them
	// to the candidate. False (the default) covers every other error, whose usage, if any,
	// is the candidate's own.
	FromValidation bool `json:"FromValidation,omitempty"`
}

type EvaluationMetadata struct {
	// Seed is shared by every task attempt of the evaluation; passing it to
	// --evaluation-seed reproduces the seed-derived inputs of that evaluation.
	Seed string
}

type InputTokenAccounting string

type MergeStats struct {
	// Runs maps provider name → run name → merge statistics for that run.
	Runs map[string]map[string]RunMergeStats
}

type OutputTokenAccounting string

type Pricing struct {
	Currency             string
	InputPerMillion      *float64
	OutputPerMillion     *float64
	CacheReadPerMillion  *float64
	CacheWritePerMillion *float64
	ReasoningPerMillion  *float64
}

type ResultKind int

type ResultSet interface {
	// GetResults returns the task results for each provider.
	GetResults() Results
}

type Results map[string][]RunResult

type RetryPolicy struct {
	MaxRetryAttempts    uint
	InitialDelaySeconds int
}

type RunConfigSnapshot struct {
	Name                    string
	Model                   string
	MaxRequestsPerMinute    int
	TextOnly                bool
	DisableStructuredOutput bool
	ModelParameters         map[string]interface{}
	RetryPolicy             RetryPolicy
	Pricing                 *Pricing
}

type RunMergeStats struct {
	// Total is the number of results for this run after merging.
	Total int
	// Updated is the number of unique tasks whose results were replaced by a later input.
	Updated int
}

type RunResult struct {
	// TraceID is a globally unique identifier for this specific task result, used for tracing and correlation.
	TraceID string
	// Kind indicates the result status.
	Kind ResultKind
	// Task is the name of the executed task.
	Task string
	// Provider is the name of the AI provider that executed the task.
	Provider string
	// Run is the name of the provider's run configuration used.
	Run string
	// RunConfig contains the effective run configuration used to produce this result,
	// with any API keys or other secrets omitted.
	RunConfig RunConfigSnapshot
	// Evaluation describes the evaluation invocation that produced this result.
	Evaluation EvaluationMetadata
	// Got is the actual answer received from the AI model.
	// For plain text response format, this should be a string that follows the format instruction precisely.
	// For structured schema-based response format, this will be any object that conforms to the schema.
	Got interface{}
	// Want are the accepted valid answer(s) for the task.
	// For plain text response format: contains string values that should follow the format instruction precisely.
	// For structured schema-based response format: contains object values that conform to the schema.
	Want utils.ValueSet
	// TaskMetadata carries optional descriptive labels copied from the originating task.
	TaskMetadata TaskMetadata
	// Details contains comprehensive information about the generated response and validation assessment.
	Details Details
	// Duration is the cumulative time the AI model itself spent generating a response,
	// summed across every conversation turn's model request (network + inference).
	// It excludes local tool execution time (see ToolCalls/ToolUsage) and any subsequent
	// validation time, so it is not the total wall-clock time spent processing the task.
	Duration time.Duration
}

type Runner interface {
	// Run executes all given tasks against all run configurations and returns when done.
	Run(ctx context.Context, tasks []config.Task) (ResultSet, error)
	// Start executes all given tasks against all run configurations asynchronously.
	// It returns immediately and the execution continues in the background,
	// offering progress updates and messages through the returned result set.
	Start(ctx context.Context, tasks []config.Task) (AsyncResultSet, error)
	// Close releases resources when the runner is no longer needed.
	Close(ctx context.Context)
}

type SemanticValidationDetails struct {
	// Verdict contains the raw verdict produced by the judge.
	Verdict interface{}
	// JudgeName identifies the judge configuration used.
	JudgeName string
	// Provider is the name of the AI provider that executed the judge task.
	Provider string
	// Variant is the name of the judge's configuration variant used.
	Variant string
	// VariantConfig is the effective variant configuration used by the judge, with any
	// API keys or other secrets omitted.
	VariantConfig RunConfigSnapshot
}

type TaskMetadata struct {
	// Suite is an optional grouping label for organizing related tasks (e.g. a benchmark suite name).
	Suite string
	// Category is an optional classification label for the task (e.g. "math", "coding").
	Category string
	// Difficulty is an optional free-form difficulty label for the task (e.g. "easy", "hard").
	Difficulty string
	// Tags is an optional set of free-form labels for filtering and grouping tasks.
	Tags []string
}

type TaskRuntimeSettings struct {
	// Services lists the task-scoped services available to tools.
	Services []config.ServiceConfig
	// CustomValidators lists the Docker-backed validators available to tasks.
	CustomValidators []config.ValidatorConfig
	// EvaluationSeed fixes the seed shared by all task attempts of every evaluation.
	// If empty, each evaluation generates its own random seed.
	EvaluationSeed string
}

type TokenUsage struct {
	// InputTokens is the input token count reported by the provider.
	InputTokens *int64 `` /* 199-byte string literal not displayed */

	// OutputTokens is the number of generated output tokens.
	OutputTokens *int64 `` /* 200-byte string literal not displayed */

	// ReasoningTokens is the number of reasoning (thinking) tokens reported by the provider.
	ReasoningTokens *int64 `` /* 265-byte string literal not displayed */

	// InputCacheWriteTokens is the number of input tokens written
	// into a provider prompt cache.
	InputCacheWriteTokens *int64 `` /* 172-byte string literal not displayed */

	// InputCacheReadTokens is the number of input tokens read from a
	// provider prompt cache.
	InputCacheReadTokens *int64 `` /* 167-byte string literal not displayed */

	// InputTokenAccounting defines how the cache token counts relate to InputTokens.
	InputTokenAccounting InputTokenAccounting `` /* 567-byte string literal not displayed */

	// OutputTokenAccounting defines how ReasoningTokens relates to OutputTokens.
	OutputTokenAccounting OutputTokenAccounting `` /* 559-byte string literal not displayed */
}

type ToolCallOutput struct {
	// Bytes is the total size of the output stream, regardless of Truncated.
	Bytes int64
	// Preview is a truncated prefix of the output stream, or nil if not captured or empty.
	Preview *string
	// Truncated indicates whether Preview was cut short of the full output.
	Truncated bool
}

type ToolCallSummary struct {
	// Tool is the name of the tool this call invoked.
	Tool string
	// CallID identifies this call, letting a specific invocation be correlated between this
	// summary, the corresponding tool-call log lines (which include the same ID in their
	// prefix), and - when the calling provider's API assigns its own tool-call ID and that
	// ID was reused here - the provider's own API error messages. Falls back to a
	// generated ID when the calling provider does not supply one, so this is never empty,
	// but its shape/format is not guaranteed to be consistent across providers.
	CallID string
	// ConversationTurn is the 1-based conversation turn this call was made during, or 0 if
	// unknown.
	ConversationTurn int
	// StartedAt is when this call began (start of setup, before the underlying process runs).
	StartedAt time.Time
	// CompletedAt is when this call finished, successfully or not.
	CompletedAt time.Time
	// Duration is the wall-clock duration of the underlying process's runtime, not including
	// setup/teardown overhead. It is nil when no process ever ran (e.g. an infrastructure_error).
	Duration *time.Duration
	// WallTime is the wall-clock duration of the entire call attempt, from setup through
	// output retrieval - i.e. Duration plus setup/teardown overhead. Unlike Duration, this is
	// always set, even for calls whose underlying process never ran.
	WallTime time.Duration
	// ExitCode is the underlying process's exit code, or nil if no exit code is known.
	ExitCode *int64
	// TimedOut indicates the call was aborted due to exceeding its configured timeout.
	TimedOut bool
	// Status is one of: "success", "nonzero_exit", "empty_output", "timeout",
	// "invalid_arguments", "infrastructure_error".
	Status string
	// Stdout is a size-limited capture of the call's standard output, or nil if no output was
	// ever captured.
	Stdout *ToolCallOutput
	// Stderr is a size-limited capture of the call's standard error, or nil if no output was
	// ever captured.
	Stderr *ToolCallOutput
	// ErrorMessage is a short explanation of the failure when Status is not "success".
	ErrorMessage string
}

type ToolUsage struct {
	// CallCount is the number of times the tool's underlying process actually ran.
	CallCount *int64 `json:"CallCount,omitempty"`
	// TotalDuration is the cumulative execution time for the tool's underlying process.
	TotalDuration *time.Duration `json:"TotalDuration,omitempty"`
}

type ValidationDetails struct {
	// Title identifies the type of validation assessment performed.
	Title string
	// Explanation contains detailed analysis of why the validation succeeded or failed.
	Explanation []string
	// Usage contains token usage statistics for the response validation step.
	// This is typically populated when using an LLM judge validator.
	Usage TokenUsage
	// ToolUsage contains aggregated execution statistics for any tools invoked during
	// validation.
	ToolUsage map[string]ToolUsage `json:"ToolUsage,omitempty"`
	// ToolCalls contains a log of every individual invocation attempt made during
	// validation, including attempts that never actually ran. Tracked separately from
	// ToolUsage, which only reflects invocations that actually ran.
	ToolCalls []ToolCallSummary `json:"ToolCalls,omitempty"`
	// Semantic contains the judge's raw verdict and variant provenance, populated
	// when validation was performed by an LLM judge; nil otherwise.
	Semantic *SemanticValidationDetails `json:"Semantic,omitempty"`
	// Method identifies which validation strategy was used.
	Method ValidationMethod `json:"Method,omitempty"`
}

type ValidationMethod string