eval

type Case struct {
	ID       string `json:"id"`
	Log      string `json:"log"`
	Expected string `json:"expected_category"`
	Note     string `json:"note"`
}

type CategoryScore struct {
	Category  verdict.Category `json:"category"`
	Support   int              `json:"support"`   // labelled instances of this category
	Predicted int              `json:"predicted"` // times the classifier chose it
	TruePos   int              `json:"true_positives"`
	Precision float64          `json:"precision"`
	Recall    float64          `json:"recall"`
	F1        float64          `json:"f1"`
}

type Corpus struct {
	Description string `json:"description"`
	LabelPolicy string `json:"label_policy"`
	Cases       []Case `json:"cases"`
	// contains filtered or unexported fields
}

type Outcome struct {
	CaseID       string           `json:"case_id"`
	Expected     verdict.Category `json:"expected"`
	Predicted    verdict.Category `json:"predicted"`
	Confidence   float64          `json:"confidence"`
	Correct      bool             `json:"correct"`
	Hallucinated int              `json:"hallucinated"`
	Error        string           `json:"error,omitempty"`
}

type Scores struct {
	Provider string `json:"provider"`
	Model    string `json:"model"`
	// Baseline marks results from the rule-based provider so no reader can
	// mistake them for a language model's accuracy.
	Baseline bool `json:"baseline"`

	Total    int     `json:"total"`
	Correct  int     `json:"correct"`
	Errored  int     `json:"errored"`
	Accuracy float64 `json:"accuracy"`
	// MacroF1 averages per-category F1 without weighting by support, so a large
	// easy category cannot hide poor performance on a small hard one.
	MacroF1 float64 `json:"macro_f1"`

	PerCategory []CategoryScore `json:"per_category"`
	// Confusion[actual][predicted] counts.
	Confusion map[verdict.Category]map[verdict.Category]int `json:"confusion"`

	MeanConfidenceCorrect float64 `json:"mean_confidence_correct"`
	MeanConfidenceWrong   float64 `json:"mean_confidence_wrong"`
	HallucinatedCitations int     `json:"hallucinated_citations"`
}