evaluation

type Case struct {
	ID                  string   `json:"id"`
	Category            string   `json:"category"`
	SupportedAssertions []string `json:"supported_assertions"`
	RequiredAssertions  []string `json:"required_assertions"`
	ExpectedConflicts   []string `json:"expected_conflicts"`
}

type CategorySummary struct {
	Category string  `json:"category"`
	Summary  Summary `json:"summary"`
}

type Dataset struct {
	Schema  string `json:"schema"`
	Version string `json:"version"`
	Cases   []Case `json:"cases"`
}

type Observation struct {
	ID                 string   `json:"id"`
	Outcome            string   `json:"outcome"`
	RequestDigest      string   `json:"request_digest"`
	ResponseDigest     string   `json:"response_digest,omitempty"`
	ProposedAssertions []string `json:"proposed_assertions"`
	DetectedConflicts  []string `json:"detected_conflicts"`
}

type Rate struct {
	Numerator   int      `json:"numerator"`
	Denominator int      `json:"denominator"`
	Value       *float64 `json:"value"`
}

type Report struct {
	Schema         string            `json:"schema"`
	DatasetVersion string            `json:"dataset_version"`
	DatasetDigest  string            `json:"dataset_digest"`
	RunDigest      string            `json:"run_digest"`
	ProfileDigest  string            `json:"profile_digest"`
	ModelIdentity  string            `json:"model_identity"`
	Total          Summary           `json:"total"`
	Categories     []CategorySummary `json:"categories"`
}

type Run struct {
	Schema        string        `json:"schema"`
	DatasetDigest string        `json:"dataset_digest"`
	ProfileDigest string        `json:"profile_digest"`
	ModelIdentity string        `json:"model_identity"`
	Cases         []Observation `json:"cases"`
}

type Summary struct {
	Cases                 int  `json:"cases"`
	FailedCases           int  `json:"failed_cases"`
	UnsupportedAssertions Rate `json:"unsupported_assertion_rate"`
	MissedConflicts       Rate `json:"missed_conflict_rate"`
	MissedAssertions      Rate `json:"missed_required_assertion_rate"`
	FalseConflicts        Rate `json:"false_conflict_rate"`
}