eval

type Anchor struct {
	// Text selects the one line containing it.
	Text string `json:"text,omitempty"`
	// Section selects every line under the heading with this exact title, up
	// to the next heading of the same or a higher level.
	Section string `json:"section,omitempty"`
}

type Base struct {
	ID string `json:"id"`
	// Profile is the SpecCritic profile the spec is reviewed under.
	Profile string `json:"profile"`
	// Provenance says where the spec and its labels came from, so a reader
	// knows how far to trust them.
	Provenance string     `json:"provenance"`
	Mutations  []Mutation `json:"mutations"`
	// Text is the clean spec.
	Text string `json:"-"`
}

type Case struct {
	// ID is unique in a run and safe to use as a file name.
	ID      string  `json:"id"`
	Base    string  `json:"base"`
	Variant Variant `json:"variant"`
	Profile string  `json:"profile"`
	// Spec is the text to review.
	Spec string `json:"-"`
	// Defects are the seeded defects. A clean case has none.
	Defects []Defect `json:"defects"`
}

type CaseVerdicts struct {
	Case     string                 `json:"case"`
	Verdicts map[schema.Verdict]int `json:"verdicts"`
	// Agreement is the share of runs that returned the most common verdict.
	Agreement float64 `json:"agreement"`
}

type CategoryStats struct {
	Category schema.Category `json:"category"`
	// Recall is seeded defects of this category that were found.
	Recall Proportion `json:"recall"`
	// Precision is model issues reported under this category that matched a
	// seeded defect. An issue matching no label counts against it even if it
	// is a real defect the labels do not list, so it is a lower bound.
	Precision Proportion `json:"precision"`
}

type Config struct {
	// Runs is how many times each case is reviewed. Verdict agreement needs
	// at least two.
	Runs int
	// Concurrency is how many reviews run at once.
	Concurrency int
	// Tolerance is how many lines a finding may be off a defect's evidence
	// and still be counted as pointing at it.
	Tolerance int
	// Timeout bounds one review. A review that exceeds it is recorded as an
	// error, never as a review that found nothing.
	Timeout time.Duration
	// OutDir receives the specs reviewed, every report, and the results.
	OutDir string
	// Logf, when set, receives one line per finished review.
	Logf func(format string, args ...any)
}

type Defect struct {
	ID         string            `json:"id"`
	Note       string            `json:"note"`
	Categories []schema.Category `json:"categories"`
	Severity   schema.Severity   `json:"severity"`
	Keywords   []string          `json:"keywords"`
	// Ranges are the lines a finding about this defect is expected to cite.
	Ranges []LineRange `json:"ranges"`
}

type DefectGrade struct {
	ID string `json:"id"`
	// Category is the defect's primary category.
	Category schema.Category `json:"category"`
	Expected schema.Severity `json:"expected_severity"`
	// Found is true when at least one finding matched the defect.
	Found bool `json:"found"`
	// FoundByModel is true when a finding from the model, not preflight,
	// matched.
	FoundByModel bool `json:"found_by_model"`
	// CategoryOK is true when a matching issue carried an accepted category.
	CategoryOK bool `json:"category_ok"`
	// Reported is the highest severity among the matching findings.
	Reported schema.Severity `json:"reported_severity,omitempty"`
}

type DefectStats struct {
	Base       string          `json:"base"`
	Defect     string          `json:"defect"`
	Category   schema.Category `json:"category"`
	Expected   schema.Severity `json:"expected_severity"`
	Runs       int             `json:"runs"`
	Found      int             `json:"found"`
	ByModel    int             `json:"found_by_model"`
	CategoryOK int             `json:"category_ok"`
	SeverityOK int             `json:"severity_ok"`
}

type Edit struct {
	// Find is replaced by Replace.
	Find    string `json:"find,omitempty"`
	Replace string `json:"replace,omitempty"`
	// DeleteLine removes the whole line containing this text.
	DeleteLine string `json:"delete_line,omitempty"`
	// ReplaceLine replaces the whole line containing this text with With.
	ReplaceLine string `json:"replace_line,omitempty"`
	// InsertAfterLine adds With as a new line after the line containing this
	// text.
	InsertAfterLine string `json:"insert_after_line,omitempty"`
	With            string `json:"with,omitempty"`
}

type Finding struct {
	// ID is the report's ID for the issue or question.
	ID       string          `json:"id"`
	Question bool            `json:"question,omitempty"`
	Severity schema.Severity `json:"severity"`
	// Category is empty for a question.
	Category schema.Category `json:"category,omitempty"`
	Title    string          `json:"title"`
	Ranges   []LineRange     `json:"ranges"`
	// Preflight is true for a finding the preflight rules produced.
	Preflight bool `json:"preflight,omitempty"`
	// Defect is the ID of the seeded defect this finding was matched to, or
	// empty when it matches none.
	Defect string `json:"defect,omitempty"`
	// CategoryOK is true when Category is one the matched defect accepts.
	CategoryOK bool `json:"category_ok,omitempty"`
	// contains filtered or unexported fields
}

type Grade struct {
	Defects  []DefectGrade `json:"defects"`
	Findings []Finding     `json:"findings"`
}

type LineRange struct {
	Start int `json:"start"`
	End   int `json:"end"`
}

type Mutation struct {
	ID string `json:"id"`
	// Note explains, for a human reader, what is wrong after the mutation.
	Note string `json:"note"`
	// Categories are the defect categories that correctly describe the
	// defect. The first is the primary one, used to group results.
	Categories []schema.Category `json:"categories"`
	// Severity is the severity a reviewer should assign.
	Severity schema.Severity `json:"severity"`
	// Keywords are lowercase fragments, any one of which marks a finding as
	// being about this defect when its category differs from Categories.
	Keywords []string `json:"keywords"`
	Edits    []Edit   `json:"edits"`
	// Evidence names where in the mutated spec a finding about this defect
	// is expected to point.
	Evidence []Anchor `json:"evidence"`
}

type Proportion struct {
	Hits  int     `json:"hits"`
	Total int     `json:"total"`
	Rate  float64 `json:"rate"`
	Low   float64 `json:"low"`
	High  float64 `json:"high"`
}

type Reviewer func(ctx context.Context, c Case, specPath string) (*schema.Report, error)

type RunError struct {
	Case   string `json:"case"`
	Rep    int    `json:"rep"`
	Error  string `json:"error"`
	WallMS int64  `json:"wall_ms"`
}

type RunResult struct {
	Case    string  `json:"case"`
	Base    string  `json:"base"`
	Variant Variant `json:"variant"`
	// Rep numbers the repeats of a case from 0.
	Rep int `json:"rep"`
	// Model is the model that answered, as the provider reported it.
	Model   string         `json:"model"`
	Verdict schema.Verdict `json:"verdict"`
	Score   int            `json:"score"`
	Grade   Grade          `json:"grade"`
	// Usage is what the review's LLM calls used.
	Usage           *schema.UsageMeta `json:"usage,omitempty"`
	DroppedFindings int               `json:"dropped_findings"`
	// WallMS is how long the whole review took, preflight included.
	WallMS int64 `json:"wall_ms"`
}

type Summary struct {
	Models []string `json:"models"`
	// Runs counts graded reviews. Errors counts reviews with nothing to grade.
	Runs   int `json:"runs"`
	Errors int `json:"errors"`

	// Recall is seeded defects found, over every seeded and isolated run.
	Recall Proportion `json:"recall"`
	// ModelRecall counts only defects a model finding matched, leaving out
	// those caught solely by the deterministic preflight rules.
	ModelRecall Proportion `json:"model_recall"`
	// CriticalRecall is defects labeled CRITICAL that were found and reported
	// as CRITICAL. A miss here lets a blocking defect through the gate.
	CriticalRecall Proportion `json:"critical_recall"`
	// CategoryAgreement is found defects reported under an accepted category.
	CategoryAgreement Proportion `json:"category_agreement"`
	// SeverityAgreement is found defects reported at the labeled severity.
	SeverityAgreement Proportion `json:"severity_agreement"`
	// Precision is model findings on seeded specs that matched a label. See
	// CategoryStats.Precision for why it is a lower bound.
	Precision  Proportion      `json:"precision"`
	Categories []CategoryStats `json:"categories"`

	// FalseCritical is clean-spec runs that reported at least one CRITICAL.
	FalseCritical Proportion `json:"false_critical"`
	// CleanCriticalMean is the mean number of CRITICAL findings per clean run.
	CleanCriticalMean float64 `json:"clean_critical_mean"`

	// VerdictAgreement is the mean, over cases, of CaseVerdicts.Agreement.
	VerdictAgreement float64 `json:"verdict_agreement"`
	// VerdictExpected is runs whose verdict fits the case: INVALID for a spec
	// seeded with a CRITICAL defect, anything else for a clean one.
	VerdictExpected Proportion     `json:"verdict_expected"`
	Verdicts        []CaseVerdicts `json:"verdicts"`

	Usage     UsageStats         `json:"usage"`
	Defects   []DefectStats      `json:"defects"`
	Unmatched []UnmatchedFinding `json:"unmatched"`
}

type UnmatchedFinding struct {
	Case     string          `json:"case"`
	Rep      int             `json:"rep"`
	Severity schema.Severity `json:"severity"`
	Category schema.Category `json:"category,omitempty"`
	Title    string          `json:"title"`
	Ranges   []LineRange     `json:"ranges"`
}

type UsageStats struct {
	Calls             int   `json:"calls"`
	RepairCalls       int   `json:"repair_calls"`
	ContinuationCalls int   `json:"continuation_calls"`
	InputTokens       int   `json:"input_tokens"`
	OutputTokens      int   `json:"output_tokens"`
	CacheReadTokens   int   `json:"cache_read_tokens"`
	CacheWriteTokens  int   `json:"cache_write_tokens"`
	DroppedFindings   int   `json:"dropped_findings"`
	WallMS            int64 `json:"wall_ms"`
}

type Variant string