evals

type AccuracyEval struct {
	EvalName string
	Judge    model.Provider
	Rubric   string
}

type CaseSpec struct {
	// Eval selects the evaluator: "exact_match", "contains", or "accuracy".
	Eval string `yaml:"eval"`
	// Name is an optional label; it defaults to the Eval type when empty.
	Name     string `yaml:"name"`
	Input    string `yaml:"input"`
	Expected string `yaml:"expected"`
}

type ContainsEval struct {
	EvalName string
}

type Eval interface {
	// Name returns a human-readable name for this eval.
	Name() string
	// Run executes the evaluation and returns a result.
	Run(ctx context.Context, input string, expected string) EvalResult
}

type EvalCase struct {
	Eval     Eval
	Input    string
	Expected string
}

type EvalResult struct {
	Name       string        `json:"name"`
	Score      float64       `json:"score"`
	Passed     bool          `json:"passed"`
	Details    string        `json:"details"`
	Latency    time.Duration `json:"latency"`
	TokensUsed int           `json:"tokens_used,omitempty"`
	Error      string        `json:"error,omitempty"`
}

type ExactMatchEval struct {
	EvalName string
}

type PerformanceBaseline struct {
	MaxLatency      time.Duration
	MaxTotalTokens  int
	MaxPromptTokens int
}

type PerformanceEval struct {
	EvalName string
	RunFunc  func(ctx context.Context) (latency time.Duration, promptTokens int, completionTokens int, err error)
	Baseline *PerformanceBaseline
}

type ReliabilityEval struct {
	EvalName      string
	ActualCalls   []ToolCallExpectation
	ExpectedCalls []ToolCallExpectation
}

type Suite struct {
	Name  string
	Evals []EvalCase
}

type SuiteResult struct {
	SuiteName    string        `json:"suite_name"`
	Results      []EvalResult  `json:"results"`
	TotalEvals   int           `json:"total_evals"`
	Passed       int           `json:"passed"`
	Failed       int           `json:"failed"`
	Errors       int           `json:"errors"`
	AvgScore     float64       `json:"avg_score"`
	TotalLatency time.Duration `json:"total_latency"`
}

type SuiteSpec struct {
	Name  string     `yaml:"name"`
	Cases []CaseSpec `yaml:"cases"`
}

type ToolCallExpectation struct {
	Name string
	Args map[string]any
}