eval

type Aggregate struct {
	// ScorerName is the name of the scorer.
	ScorerName string `json:"scorer_name"`
	// Mean is the average score across all cases (excluding errored cases).
	Mean float64 `json:"mean"`
	// PassRate is the fraction of cases with score >= 0.5 (by default).
	PassRate float64 `json:"pass_rate"`
	// NumCases is the total number of cases evaluated.
	NumCases int `json:"num_cases"`
	// NumErrors is the number of cases that errored during scoring.
	NumErrors int `json:"num_errors"`
}

type Case struct {
	// ID is the unique identifier for this case.
	ID string `json:"id"`
	// Input is the input to the loop (arbitrary JSON-serializable value).
	Input any `json:"input"`
	// Expect is the expected output (arbitrary JSON-serializable value).
	Expect any `json:"expect"`
	// Tags are optional labels for filtering.
	Tags []string `json:"tags,omitempty"`
}

type CaseResult struct {
	// CaseID is the case identifier.
	CaseID string `json:"case_id"`
	// Scores maps scorer name to score value.
	Scores map[string]Score `json:"scores"`
	// Error is non-empty if the case run failed.
	Error string `json:"error,omitempty"`
}

type Dataset []Case

type ExactMatchScorer struct{}

type Gate struct {
	// MinMean is the minimum acceptable mean score per scorer (0..1).
	// If any scorer's mean falls below this threshold, the test fails.
	// Zero means no minimum.
	MinMean float64
	// MaxRegression is the maximum allowed mean score drop vs baseline (0..1).
	// If any scorer regresses by more than this amount, the test fails.
	// Zero means any regression fails.
	MaxRegression float64
	// Baseline is a path to a baseline report JSON file (from a previous run).
	// If empty, regression checking is disabled.
	Baseline string
}

type JudgeRunFunc func(ctx context.Context, rubric string, c Case, run ScoredRun) (Score, error)

type LLMJudgeScorer struct {
	// contains filtered or unexported fields
}

type RegressionCase struct {
	CaseID     string  `json:"case_id"`
	ScorerName string  `json:"scorer_name"`
	OldScore   float64 `json:"old_score"`
	NewScore   float64 `json:"new_score"`
	Delta      float64 `json:"delta"` // new - old (negative = regression)
}

type RegressionReport struct {
	// Regressions is the list of cases that regressed (new score < old score).
	Regressions []RegressionCase `json:"regressions"`
	// Improvements is the list of cases that improved (new score > old score).
	Improvements []RegressionCase `json:"improvements"`
	// MeanDelta maps scorer name to mean score delta (new - old).
	MeanDelta map[string]float64 `json:"mean_delta"`
}

type Report struct {
	// SuiteName is the suite's name.
	SuiteName string `json:"suite_name"`
	// Cases contains per-case results, sorted by CaseID.
	Cases []CaseResult `json:"cases"`
	// Aggregates contains per-scorer aggregate statistics.
	Aggregates []Aggregate `json:"aggregates"`
	// TotalDuration is how long the suite took.
	TotalDuration time.Duration `json:"total_duration_ns"`
	// GeneratedAt is the timestamp of this report.
	GeneratedAt time.Time `json:"generated_at"`
}

type RubricFunc func(c Case, run ScoredRun) Score

type RunFunc func(ctx context.Context, c Case) (output any, err error)

type Score struct {
	// Value is in [0, 1].
	Value float64
	// Label is a human-readable description of the score.
	Label string
	// Details is optional additional detail.
	Details string
}

type ScoredRun struct {
	// CaseID is the case that was run.
	CaseID string
	// ActualOutput is what the loop returned (JSON-serializable).
	ActualOutput any
	// ActualJSON is the JSON-encoded actual output.
	ActualJSON json.RawMessage
	// Error is non-nil if the run itself failed (not a scoring failure).
	Error error
}

type Scorer interface {
	// Name returns the scorer's identifier.
	Name() string
	// Score evaluates one case. Returns Score and any error.
	// A scorer error causes the case to receive score 0 with error details.
	Score(ctx context.Context, c Case, run ScoredRun) (Score, error)
}

type StructuralScorer struct{}

type Suite struct {
	// Name is the suite's identifier.
	Name string
	// Dataset is the set of cases to evaluate.
	Dataset Dataset
	// Run is the function that runs each case.
	Run RunFunc
	// Scorers is the set of scorers to apply to each case's output.
	Scorers []Scorer
	// Parallelism controls how many cases run concurrently. 0 or 1 = sequential.
	Parallelism int
}