runners

type AnswerDetails struct {
	// Title is a descriptive header for the response produced by the target AI model.
	Title string
	// Explanation of the answer produced by the target AI model.
	Explanation []string
	// ActualAnswer is the raw answer from the target AI model split into lines.
	ActualAnswer []string
	// ExpectedAnswer is a set of all acceptable correct answers, each being an array of lines.
	ExpectedAnswer [][]string
	// Usage contains token usage statistics for generating the answer.
	Usage TokenUsage
	// ToolUsage contains aggregated statistics for any tools invoked while producing the answer.
	ToolUsage map[string]ToolUsage `json:"ToolUsage,omitempty"`
}

type AsyncResultSet interface {
	// GetResults returns the task results for each provider.
	// The call will block until the run is finished.
	GetResults() Results
	// ProgressEvents returns a channel that emits run progress as a value between 0 and 1.
	// The channel will be closed when the run is finished.
	ProgressEvents() <-chan float32
	// MessageEvents returns a channel that emits run log messages.
	// The channel will be closed when the run is finished.
	MessageEvents() <-chan string
	// Cancel stops the ongoing run execution.
	Cancel()
}

type Details struct {
	// Answer contains details about the AI model's response and reasoning process.
	Answer AnswerDetails
	// Validation contains details about the answer verification and assessment.
	Validation ValidationDetails
	// Error contains details about any errors that occurred during task execution.
	Error ErrorDetails
}

type EmittingLogger struct {
	// contains filtered or unexported fields
}

type ErrorDetails struct {
	// Title provides a summary description of the error.
	Title string
	// Message contains the primary error message.
	Message string
	// Details contains any additional error information in a generic structure.
	Details map[string][]string
	// Usage contains token usage statistics if available even in error scenarios.
	// This is typically populated if the error occurs when parsing the generated response.
	Usage TokenUsage
	// ToolUsage contains aggregated statistics for any tools invoked prior to the error.
	ToolUsage map[string]ToolUsage `json:"ToolUsage,omitempty"`
}

type ResultKind int

type ResultSet interface {
	// GetResults returns the task results for each provider.
	GetResults() Results
}

type Results map[string][]RunResult

type RunResult struct {
	// TraceID is a globally unique identifier for this specific task result, used for tracing and correlation.
	TraceID string
	// Kind indicates the result status.
	Kind ResultKind
	// Task is the name of the executed task.
	Task string
	// Provider is the name of the AI provider that executed the task.
	Provider string
	// Run is the name of the provider's run configuration used.
	Run string
	// Got is the actual answer received from the AI model.
	// For plain text response format, this should be a string that follows the format instruction precisely.
	// For structured schema-based response format, this will be any object that conforms to the schema.
	Got interface{}
	// Want are the accepted valid answer(s) for the task.
	// For plain text response format: contains string values that should follow the format instruction precisely.
	// For structured schema-based response format: contains object values that conform to the schema.
	Want utils.ValueSet
	// Details contains comprehensive information about the generated response and validation assessment.
	Details Details
	// Duration represents the time taken to generate the response.
	Duration time.Duration
}

type Runner interface {
	// Run executes all given tasks against all run configurations and returns when done.
	Run(ctx context.Context, tasks []config.Task) (ResultSet, error)
	// Start executes all given tasks against all run configurations asynchronously.
	// It returns immediately and the execution continues in the background,
	// offering progress updates and messages through the returned result set.
	Start(ctx context.Context, tasks []config.Task) (AsyncResultSet, error)
	// Close releases resources when the runner is no longer needed.
	Close(ctx context.Context)
}

type TokenUsage struct {
	// InputTokens is the number of tokens consumed by the prompt/input.
	InputTokens *int64 `json:"InputTokens,omitempty"`
	// OutputTokens is the number of tokens generated in the completion/output.
	OutputTokens *int64 `json:"OutputTokens,omitempty"`
}

type ToolUsage struct {
	// CallCount is the number of times the tool was invoked.
	CallCount *int64 `json:"CallCount,omitempty"`
	// TotalDuration is the cumulative execution time for the tool.
	TotalDuration *time.Duration `json:"TotalDuration,omitempty"`
}

type ValidationDetails struct {
	// Title identifies the type of validation assessment performed.
	Title string
	// Explanation contains detailed analysis of why the validation succeeded or failed.
	Explanation []string
	// Usage contains token usage statistics for the response validation step.
	// This is typically populated when using an LLM judge validator.
	Usage TokenUsage
	// ToolUsage contains aggregated statistics for any tools invoked during validation.
	ToolUsage map[string]ToolUsage `json:"ToolUsage,omitempty"`
}