eval

type Assertion struct {
	Type  string      `yaml:"type"`
	Value interface{} `yaml:"value"`
}

type Context struct {
	// Layers specifies which config layers to include.
	// Options: "team", "personal", "project", or "merged" (default).
	Layers []string `yaml:"layers,omitempty"`

	// Languages specifies which language configs to include.
	Languages []string `yaml:"languages,omitempty"`
}

type Eval struct {
	Name        string   `yaml:"name"`
	Description string   `yaml:"description"`
	Tags        []string `yaml:"tags,omitempty"`
	Context     Context  `yaml:"context,omitempty"`
	Provider    Provider `yaml:"provider,omitempty"`
	Tests       []Test   `yaml:"tests"`

	// Metadata (not from YAML)
	Source   Source `yaml:"-"`
	FilePath string `yaml:"-"`
}

type EvalTemplate struct {
	Name        string // Template identifier (e.g., "security")
	Description string // Human-readable description
	Content     string // YAML template with Go template placeholders
}

type FailedTest struct {
	Eval  string
	Test  string
	Error string
}

type Formatter struct {
	Writer io.Writer
	Format OutputFormat
	Debug  bool // Show full LLM responses for failures
}

type GenerateOptions struct {
	// OutputDir is the directory to write the Promptfoo config to.
	OutputDir string

	// ResultsPath is the path for Promptfoo to write results to.
	ResultsPath string
}

type JSONEvalResult struct {
	Eval     string           `json:"eval"`
	Total    int              `json:"total"`
	Passed   int              `json:"passed"`
	Failed   int              `json:"failed"`
	Duration string           `json:"duration"`
	Tests    []JSONTestResult `json:"tests"`
}

type JSONOutput struct {
	Summary struct {
		Total    int     `json:"total"`
		Passed   int     `json:"passed"`
		Failed   int     `json:"failed"`
		PassRate float64 `json:"passRate"`
	} `json:"summary"`
	Results []JSONEvalResult `json:"results"`
}

type JSONTestResult struct {
	Name     string `json:"name"`
	Passed   bool   `json:"passed"`
	Duration string `json:"duration,omitempty"`
	Error    string `json:"error,omitempty"`
	Output   string `json:"output,omitempty"`
}

type OutputFormat string

type PromptfooAssertion struct {
	Type  string      `yaml:"type"`
	Value interface{} `yaml:"value,omitempty"`
}

type PromptfooAssertionResult struct {
	Pass   bool   `json:"pass"`
	Reason string `json:"reason"`
}

type PromptfooConfig struct {
	Description string          `yaml:"description,omitempty"`
	Prompts     []string        `yaml:"prompts"`
	Providers   []string        `yaml:"providers"`
	Tests       []PromptfooTest `yaml:"tests"`
	OutputPath  string          `yaml:"outputPath,omitempty"`
}

type PromptfooOutput struct {
	Results []PromptfooResult `json:"results"`
	Stats   struct {
		Successes int `json:"successes"`
		Failures  int `json:"failures"`
	} `json:"stats"`
}

type PromptfooResult struct {
	Description      string                     `json:"description"`
	Success          bool                       `json:"success"`
	Response         string                     `json:"response"`
	AssertionResults []PromptfooAssertionResult `json:"gradingResult"`
}

type PromptfooTest struct {
	Description string                 `yaml:"description,omitempty"`
	Vars        map[string]interface{} `yaml:"vars"`
	Assert      []PromptfooAssertion   `yaml:"assert"`
}

type Provider struct {
	// Model is the model identifier, supports env var expansion.
	// Default: ${STAGHORN_EVAL_MODEL:-claude-sonnet-4-20250514}
	Model string `yaml:"model,omitempty"`
}

type RunConfig struct {
	// Eval is the eval definition to run.
	Eval *Eval

	// ClaudeConfig is the merged CLAUDE.md content to use as system prompt.
	ClaudeConfig string
}

type RunResult struct {
	EvalName   string
	TotalTests int
	Passed     int
	Failed     int
	Duration   time.Duration
	Results    []TestResult
	RawOutput  string
	DebugDir   string // Set when debug mode is enabled
}

type Runner struct {
	// WorkDir is the working directory for Promptfoo.
	WorkDir string

	// Verbose enables verbose output.
	Verbose bool

	// Debug enables debug mode (preserves temp files, shows full responses).
	Debug bool

	// Timeout is the timeout for eval execution.
	Timeout time.Duration
}

type Source string

type Summary struct {
	TotalEvals  int
	TotalTests  int
	Passed      int
	Failed      int
	PassRate    float64
	Duration    float64
	FailedTests []FailedTest
}

type TemplateVars struct {
	Name        string
	Description string
	Tags        []string
}

type Test struct {
	Name        string            `yaml:"name"`
	Description string            `yaml:"description,omitempty"`
	Vars        map[string]string `yaml:"vars,omitempty"`
	Prompt      string            `yaml:"prompt"`
	Assert      []Assertion       `yaml:"assert"`
}

type TestResult struct {
	Name        string
	Description string
	Passed      bool
	Duration    time.Duration
	Error       string
	Output      string
}

type ValidationError struct {
	Field   string          // e.g., "tests[0].assert[0].type"
	Message string          // Human-readable error message
	Level   ValidationLevel // error or warning
}

type ValidationLevel string