config

type AnthropicClientConfig struct {
	// APIKey is the API key for the Anthropic generative models provider.
	APIKey string `yaml:"api-key" validate:"required"`
	// RequestTimeout specifies the timeout for API requests.
	RequestTimeout *time.Duration `yaml:"request-timeout" validate:"omitempty"`
}

type AnthropicModelParams struct {
	// MaxTokens controls the maximum number of tokens available to the model for generating a response.
	// This includes the thinking budget for reasoning models.
	MaxTokens *int64 `yaml:"max-tokens" validate:"omitempty,min=0"`

	// ThinkingBudgetTokens specifies the number of tokens the model can use for its internal reasoning process.
	// It must be at least 1024 and less than `MaxTokens`.
	// If set, this enables enhanced reasoning capabilities for the model.
	ThinkingBudgetTokens *int64 `yaml:"thinking-budget-tokens" validate:"omitempty,min=1024,ltfield=MaxTokens"`

	// Temperature controls the randomness or "creativity" of responses.
	// Values range from 0.0 to 1.0, with lower values making the output more focused.
	// The default value is 1.0.
	// It is generally recommended to alter this or `TopP` but not both.
	Temperature *float64 `yaml:"temperature" validate:"omitempty,min=0,max=1"`

	// TopP controls diversity via nucleus sampling.
	// Values range from 0.0 to 1.0, with lower values making the output more focused.
	// You usually only need to use `Temperature`.
	TopP *float64 `yaml:"top-p" validate:"omitempty,min=0,max=1"`

	// TopK limits response tokens to top K options for each token position.
	// Higher values allow more diverse outputs by considering more token options.
	// You usually only need to use `Temperature`.
	TopK *int64 `yaml:"top-k" validate:"omitempty,min=0"`
}

type AppConfig struct {
	// LogFile specifies path to the log file.
	LogFile string `yaml:"log-file" validate:"omitempty,filepath"`

	// OutputDir specifies directory where results will be saved.
	OutputDir string `yaml:"output-dir" validate:"required"`

	// OutputBaseName specifies base filename for result files.
	OutputBaseName string `yaml:"output-basename" validate:"omitempty,filepath"`

	// TaskSource specifies path to the task definitions file.
	TaskSource string `yaml:"task-source" validate:"required,filepath"`

	// Providers lists configurations for AI providers whose models will be used
	// to execute tasks during the trial run.
	Providers []ProviderConfig `yaml:"providers" validate:"required,dive"`

	// Judges lists LLM configurations for semantic evaluation of open-ended task responses.
	Judges []JudgeConfig `yaml:"judges" validate:"omitempty,unique=Name,dive"`
}

type ClientConfig interface{}

type Config struct {
	// Config contains application-wide settings.
	Config AppConfig `yaml:"config" validate:"required"`
}

type DeepseekClientConfig struct {
	// APIKey is the API key for the Deepseek generative models provider.
	APIKey string `yaml:"api-key" validate:"required"`
	// RequestTimeout specifies the timeout for API requests.
	RequestTimeout *time.Duration `yaml:"request-timeout" validate:"omitempty"`
}

type DeepseekModelParams struct {
	// Temperature controls the randomness or "creativity" of the model's outputs.
	// Values range from 0.0 to 2.0, with lower values making the output more focused.
	// The default value is 1.0.
	// Recommended values by use case:
	// - 0.0: Coding / Math (best for precise, deterministic outputs)
	// - 1.0: Data Cleaning / Data Analysis
	// - 1.3: General Conversation / Translation
	// - 1.5: Creative Writing / Poetry (more varied and creative outputs)
	Temperature *float32 `yaml:"temperature" validate:"omitempty,min=0,max=2"`

	// TopP controls diversity via nucleus sampling.
	// Values range from 0.0 to 1.0, with lower values making the output more focused.
	// You usually only need to use `Temperature`.
	TopP *float32 `yaml:"top-p" validate:"omitempty,min=0,max=1"`

	// PresencePenalty penalizes new tokens based on whether they appear in the text so far.
	// Values range from -2.0 to 2.0, with positive values encouraging the model to use new tokens,
	// increasing the model's likelihood to talk about new topics.
	// The default value is 0.0.
	PresencePenalty *float32 `yaml:"presence-penalty" validate:"omitempty,min=-2,max=2"`

	// FrequencyPenalty penalizes new tokens based on their frequency in the text so far.
	// Values range from -2.0 to 2.0, with positive values encouraging the model to use less frequent tokens,
	// decreasing the model's likelihood to repeat the same line verbatim.
	// The default value is 0.0.
	FrequencyPenalty *float32 `yaml:"frequency-penalty" validate:"omitempty,min=-2,max=2"`
}

type GoogleAIClientConfig struct {
	// APIKey is the API key for the Google AI generative models provider.
	APIKey string `yaml:"api-key" validate:"required"`
}

type GoogleAIModelParams struct {
	// TextResponseFormat indicates whether to use plain-text response format
	// for compatibility with models that do not support JSON.
	TextResponseFormat bool `yaml:"text-response-format" validate:"omitempty"`

	// Temperature controls the randomness or "creativity" of the model's outputs.
	// Values range from 0.0 to 2.0, with lower values making the output more focused and deterministic.
	// The default value is typically around 1.0.
	Temperature *float32 `yaml:"temperature" validate:"omitempty,min=0,max=2"`

	// TopP controls diversity via nucleus sampling.
	// Values range from 0.0 to 1.0, with lower values making the output more focused.
	// The default value is typically around 1.0.
	TopP *float32 `yaml:"top-p" validate:"omitempty,min=0,max=1"`

	// TopK limits response tokens to top K options for each token position.
	// Higher values allow more diverse outputs by considering more token options.
	TopK *int32 `yaml:"top-k" validate:"omitempty,min=0"`
}

type JudgeConfig struct {
	// Name is the unique identifier for this judge configuration.
	Name string `yaml:"name" validate:"required"`

	// Provider encapsulates the provider configuration for the judge.
	Provider ProviderConfig `yaml:"provider" validate:"required"`
}

type JudgeSelector struct {
	// Enabled determines whether judge evaluation is enabled.
	Enabled *bool `yaml:"enabled" validate:"omitempty"`

	// Name specifies the name of the judge configuration to use.
	Name *string `yaml:"name" validate:"omitempty"`

	// Variant specifies the run variant name from the judge's provider configuration.
	Variant *string `yaml:"variant" validate:"omitempty"`
}

type MistralAIClientConfig struct {
	// APIKey is the API key for the Mistral AI generative models provider.
	APIKey string `yaml:"api-key" validate:"required"`
}

type MistralAIModelParams struct {
	// Temperature controls the randomness or "creativity" of the model's outputs.
	// Values range from 0.0 to 1.5, with lower values making the output more focused and deterministic.
	// The default value varies depending on the model.
	// It is generally recommended to alter this or `TopP` but not both.
	Temperature *float32 `yaml:"temperature" validate:"omitempty,min=0,max=1.5"`

	// TopP controls diversity via nucleus sampling.
	// Values range from 0.0 to 1.0, with lower values making the output more focused.
	// The default value is 1.0.
	// It is generally recommended to alter this or `Temperature` but not both.
	TopP *float32 `yaml:"top-p" validate:"omitempty,min=0,max=1"`

	// MaxTokens controls the maximum number of tokens to generate in the completion.
	// The token count of the prompt plus max_tokens cannot exceed the model's context length.
	MaxTokens *int32 `yaml:"max-tokens" validate:"omitempty,min=0"`

	// PresencePenalty penalizes new tokens based on whether they appear in the text so far.
	// Values range from -2.0 to 2.0, with positive values encouraging the model to use new tokens,
	// increasing the model's likelihood to talk about new topics.
	// The default value is 0.0.
	PresencePenalty *float32 `yaml:"presence-penalty" validate:"omitempty,min=-2,max=2"`

	// FrequencyPenalty penalizes new tokens based on their frequency in the text so far.
	// Values range from -2.0 to 2.0, with positive values encouraging the model to use less frequent tokens,
	// decreasing the model's likelihood to repeat the same line verbatim.
	// The default value is 0.0.
	FrequencyPenalty *float32 `yaml:"frequency-penalty" validate:"omitempty,min=-2,max=2"`

	// RandomSeed provides the seed to use for random sampling.
	// If set, requests will generate deterministic results.
	RandomSeed *int32 `yaml:"random-seed" validate:"omitempty"`

	// PromptMode sets the prompt mode for the request.
	// When set to "reasoning", a system prompt will be used to instruct the model to reason if supported.
	PromptMode *string `yaml:"prompt-mode" validate:"omitempty,oneof=reasoning"`

	// SafePrompt controls whether to inject a safety prompt before all conversations.
	SafePrompt *bool `yaml:"safe-prompt" validate:"omitempty"`
}

type ModelParams interface{}

type OpenAIClientConfig struct {
	// APIKey is the API key for the OpenAI provider.
	APIKey string `yaml:"api-key" validate:"required"`
}

type OpenAIModelParams struct {
	// ReasoningEffort controls effort level on reasoning for reasoning models.
	// Valid values are: "low", "medium", "high".
	ReasoningEffort *string `yaml:"reasoning-effort" validate:"omitempty,oneof=low medium high"`

	// TextResponseFormat indicates whether to use plain-text response format
	// for compatibility with models that do not support JSON.
	TextResponseFormat bool `yaml:"text-response-format" validate:"omitempty"`

	// Temperature controls the randomness or "creativity" of the model's outputs.
	// Values range from 0.0 to 2.0, with lower values making the output more focused and deterministic.
	// The default value is 1.0.
	// It is generally recommended to alter this or `TopP` but not both.
	Temperature *float32 `yaml:"temperature" validate:"omitempty,min=0,max=2"`

	// TopP controls diversity via nucleus sampling.
	// Values range from 0.0 to 1.0, with lower values making the output more focused.
	// The default value is 1.0.
	// It is generally recommended to alter this or `Temperature` but not both.
	TopP *float32 `yaml:"top-p" validate:"omitempty,min=0,max=1"`

	// PresencePenalty penalizes new tokens based on whether they appear in the text so far.
	// Values range from -2.0 to 2.0, with positive values encouraging the model to use new tokens,
	// increasing the model's likelihood to talk about new topics.
	// The default value is 0.0.
	PresencePenalty *float32 `yaml:"presence-penalty" validate:"omitempty,min=-2,max=2"`

	// FrequencyPenalty penalizes new tokens based on their frequency in the text so far.
	// Values range from -2.0 to 2.0, with positive values encouraging the model to use less frequent tokens,
	// decreasing the model's likelihood to repeat the same line verbatim.
	// The default value is 0.0.
	FrequencyPenalty *float32 `yaml:"frequency-penalty" validate:"omitempty,min=-2,max=2"`
}

type ProviderConfig struct {
	// Name specifies unique identifier of the provider.
	Name string `yaml:"name" validate:"required,oneof=openai google anthropic deepseek mistralai xai"`

	// ClientConfig holds provider-specific client settings.
	ClientConfig ClientConfig `yaml:"client-config" validate:"required"`

	// Runs lists run configurations for this provider.
	Runs []RunConfig `yaml:"runs" validate:"required,unique=Name,dive"`

	// Disabled indicates if all runs should be disabled by default.
	Disabled bool `yaml:"disabled" validate:"omitempty"`

	// RetryPolicy specifies default retry behavior for all runs in this provider.
	RetryPolicy RetryPolicy `yaml:"retry-policy" validate:"omitempty"`
}

type RetryPolicy struct {
	// MaxRetryAttempts specifies the maximum number of retry attempts.
	// Value of 0 means no retry attempts will be made.
	MaxRetryAttempts uint `yaml:"max-retry-attempts" validate:"omitempty,min=0"`

	// InitialDelaySeconds specifies the initial delay in seconds before the first retry attempt.
	InitialDelaySeconds int `yaml:"initial-delay-seconds" validate:"omitempty,gt=0"`
}

type RunConfig struct {
	// Name is a display-friendly identifier shown in results.
	Name string `yaml:"name" validate:"required"`

	// Model specifies target model's identifier.
	Model string `yaml:"model" validate:"required"`

	// MaxRequestsPerMinute limits the number of API requests per minute sent to this specific model.
	// Value of 0 means no rate limiting will be applied.
	MaxRequestsPerMinute int `yaml:"max-requests-per-minute" validate:"omitempty,numeric,min=0"`

	// Disabled indicates if this run configuration should be skipped.
	// If set, overrides the parent ProviderConfig.Disabled value.
	Disabled *bool `yaml:"disabled" validate:"omitempty"`

	// ModelParams holds any model-specific configuration parameters.
	ModelParams ModelParams `yaml:"model-parameters" validate:"omitempty"`

	// RetryPolicy specifies retry behavior on transient errors.
	// If set, overrides the parent ProviderConfig.RetryPolicy value.
	RetryPolicy *RetryPolicy `yaml:"retry-policy" validate:"omitempty"`
}

type Task struct {
	// Name is a display-friendly identifier shown in results.
	Name string `yaml:"name" validate:"required"`

	// Prompt that will be sent to the AI model.
	Prompt string `yaml:"prompt" validate:"required"`

	// ResponseResultFormat specifies how the AI should format the final answer to the prompt.
	ResponseResultFormat string `yaml:"response-result-format" validate:"required"`

	// ExpectedResult is the set of accepted valid answers for the prompt.
	// All values must follow the ResponseResultFormat precisely.
	// Only one needs to match for the response to be considered correct.
	ExpectedResult utils.StringSet `yaml:"expected-result" validate:"required"`

	// Disabled indicates whether this specific task should be skipped.
	// If set, overrides the global TaskConfig.Disabled value.
	Disabled *bool `yaml:"disabled" validate:"omitempty"`

	// ValidationRules are validation settings for this specific task.
	// If set, overrides the global TaskConfig.ValidationRules values.
	ValidationRules *ValidationRules `yaml:"validation-rules" validate:"omitempty"`

	// Files is a list of files to be included with the prompt.
	// This is primarily used for images but can support other file types
	// depending on the provider's capabilities.
	Files []TaskFile `yaml:"files" validate:"omitempty,unique=Name,dive"`
}

type TaskConfig struct {
	// Tasks is a list of tasks to be executed.
	Tasks []Task `yaml:"tasks" validate:"required,unique=Name,dive"`

	// Disabled indicates whether all tasks should be disabled by default.
	// Individual tasks can override this setting.
	Disabled bool `yaml:"disabled" validate:"omitempty"`

	// ValidationRules are default validation settings for all tasks.
	// Individual tasks can override these settings.
	ValidationRules ValidationRules `yaml:"validation-rules" validate:"omitempty"`
}

type TaskFile struct {
	// Name is a unique identifier for the file, used to reference it in prompts.
	Name string `yaml:"name" validate:"required"`

	// URI is the path or URL to the file.
	URI URI `yaml:"uri" validate:"required"`

	// Type is the MIME type of the file.
	// If not provided, it will be inferred from the file extension or content.
	Type string `yaml:"type" validate:"omitempty"`
	// contains filtered or unexported fields
}

type Tasks struct {
	// TaskConfig contains all task definitions and settings.
	TaskConfig TaskConfig `yaml:"task-config" validate:"required"`
}

type URI struct {
	// contains filtered or unexported fields
}

type ValidationRules struct {
	// CaseSensitive determines whether string comparison should be case-sensitive.
	CaseSensitive *bool `yaml:"case-sensitive" validate:"omitempty"`

	// IgnoreWhitespace determines whether all whitespace should be ignored during comparison.
	// When true, all whitespace characters (spaces, tabs, newlines) are removed before comparison.
	IgnoreWhitespace *bool `yaml:"ignore-whitespace" validate:"omitempty"`

	// TrimLines determines whether to trim leading and trailing whitespace of each line
	// before comparison.
	TrimLines *bool `yaml:"trim-lines" validate:"omitempty"`

	// Judge specifies the judge configuration to use for evaluation.
	// When enabled, an LLM will be used to evaluate the correctness of the response
	// instead of simple string matching.
	Judge JudgeSelector `yaml:"judge" validate:"omitempty"`
}

type XAIClientConfig struct {
	// APIKey is the API key for the xAI provider.
	APIKey string `yaml:"api-key" validate:"required"`
}

type XAIModelParams struct {
	// Temperature controls the randomness or "creativity" of the model's outputs.
	// Notes: Higher values (e.g. 0.8) make outputs more random; lower values
	// (e.g. 0.2) make outputs more focused and deterministic.
	// Valid range: 0.0 — 2.0. Default: 1.0.
	Temperature *float32 `yaml:"temperature" validate:"omitempty,min=0,max=2"`

	// TopP controls diversity via nucleus sampling (probability mass cutoff).
	// Notes: Use either Temperature or TopP, not both, for sampling control.
	// Valid range: (0.0, 1.0]. Default: 1.0.
	TopP *float32 `yaml:"top-p" validate:"omitempty,min=0,max=1"`

	// MaxCompletionTokens controls the maximum number of tokens to generate in the completion.
	MaxCompletionTokens *int32 `yaml:"max-completion-tokens" validate:"omitempty,min=0"`

	// PresencePenalty penalizes new tokens based on whether they appear in the text so far.
	// Notes: Positive values encourage the model to introduce new topics.
	// Valid range: -2.0 — 2.0. Default: 0.0.
	PresencePenalty *float32 `yaml:"presence-penalty" validate:"omitempty,min=-2,max=2"`

	// FrequencyPenalty penalizes new tokens based on their frequency in the text so far.
	// Notes: Positive values discourage repetition.
	// Valid range: -2.0 — 2.0. Default: 0.0.
	FrequencyPenalty *float32 `yaml:"frequency-penalty" validate:"omitempty,min=-2,max=2"`

	// ReasoningEffort constrains how much "reasoning" budget to spend for reasoning-capable models.
	// Notes: Not all reasoning models support this option.
	// Valid values: "low", "high".
	ReasoningEffort *string `yaml:"reasoning-effort" validate:"omitempty,oneof=low high"`

	// Seed requests deterministic sampling when possible.
	// No guaranteed determinism — xAI makes a best-effort to return
	// repeatable outputs for identical inputs when `seed` and other parameters are the same.
	Seed *int32 `yaml:"seed" validate:"omitempty"`
}