eval

type Calibration struct {
	// OnExpected counts graded findings about an expected problem, and
	// OnExpectedConfirmed those the grader confirmed: agreement.
	OnExpected          int `json:"on_expected"`
	OnExpectedConfirmed int `json:"on_expected_confirmed"`
	// OnClean counts graded findings on clean cases, and OnCleanConfirmed
	// those the grader confirmed: disagreement.
	OnClean          int `json:"on_clean"`
	OnCleanConfirmed int `json:"on_clean_confirmed"`
	// MissedOnClean counts problems the grader said the reviews missed on
	// clean cases: disagreement.
	MissedOnClean int `json:"missed_on_clean"`
	// ExpectedMissed counts expected problems no finding was about, and
	// ExpectedFound those the grader found itself: agreement.
	ExpectedMissed int `json:"expected_missed"`
	ExpectedFound  int `json:"expected_found"`
}

type Case struct {
	Name string `json:"name"`
	// Revision is the commit the patch applies to.
	Revision string `json:"revision"`
	// Request is the change the patch answers, which reviewers are given as
	// the specification, as factory gives them the developer's prompt.
	Request string `json:"request"`
	// Patch is the change, a path to a git diff relative to the suite
	// directory.
	Patch string `json:"patch"`
	// Source says where the case came from, such as a pull request or a
	// mutation, for whoever reads the results.
	Source string `json:"source,omitempty"`
	// Expect are the problems a review should find, when they are known.
	Expect []Expected `json:"expect,omitempty"`
	// Clean labels the change as sound: it should come back with no high or
	// medium findings, each of which would send it back for nothing. A case
	// with neither Clean nor Expect is unlabeled, and only the grader scores
	// it.
	Clean bool `json:"clean,omitempty"`
	// contains filtered or unexported fields
}

type CaseResult struct {
	Case string `json:"case"`
	Run  int    `json:"run"`
	// Clean is whether the case is labeled clean, and Labeled whether it is
	// labeled at all: only labeled cases count towards recall and precision
	// against labels.
	Clean   bool `json:"clean"`
	Labeled bool `json:"labeled"`
	// Errors are the reviews that could not run. A case with any is left out
	// of the summary rather than scored on the reviews that did, which would
	// make a broken prompt look like a quiet one.
	Errors []PromptError `json:"errors,omitempty"`
	// Problems are what the blocking findings were about, each once, with
	// every finding about it: an expected problem one caught, or one the case
	// does not list. An unlisted problem on a clean case sends sound work
	// back. On another it may be real, so it is worth reading rather than
	// counting blindly.
	Problems []Problem `json:"problems,omitempty"`
	// Missed are the expected problems no blocking finding was about, in the
	// case's order.
	Missed []Expected `json:"missed,omitempty"`
	// Findings is everything the reviews reported, blocking or not.
	Findings []review.Finding `json:"findings"`
	// Grade is the grader's judgment of the run, and GradeError why there is
	// none when grading was asked for and failed.
	Grade      *Grade `json:"grade,omitempty"`
	GradeError string `json:"grade_error,omitempty"`
}

type Expected struct {
	// File is the repository-relative path the problem is in.
	File string `json:"file"`
	// Lines is the range of new-file lines the problem is on, first and last.
	// Empty means anywhere in the file.
	Lines []int `json:"lines,omitempty"`
	// About says what the problem is, for whoever reads the results.
	About string `json:"about"`
}

type Grade struct {
	// Findings are the blocking findings, each with its verdict, in the
	// order the grader was shown them.
	Findings []GradedFinding `json:"findings"`
	// Missed are problems the grader found that no blocking finding was
	// about.
	Missed  []Missed `json:"missed,omitempty"`
	Summary string   `json:"summary,omitempty"`
	// Ungraded counts blocking findings the grader gave no verdict.
	Ungraded int     `json:"ungraded,omitempty"`
	CostUSD  float64 `json:"cost_usd"`
}

type GradeOptions struct {
	Credential claudecode.Credential
	BaseURL    string
	Model      string        // optional; claude's default when empty
	Timeout    time.Duration // DefaultGradeTimeout when zero
}

type GradedFinding struct {
	Finding review.Finding `json:"finding"`
	Verdict string         `json:"verdict"`
	// DuplicateOf is the index in Grade.Findings of an earlier finding about
	// the same problem, or -1.
	DuplicateOf int    `json:"duplicate_of"`
	Reason      string `json:"reason,omitempty"`
}

type GradedSummary struct {
	// Runs counts graded runs, and Errors scored runs whose grading failed,
	// which the other counts leave out.
	Runs   int `json:"runs"`
	Errors int `json:"errors"`
	// Problems counts the distinct problems reported, by verdict.
	Problems    int `json:"problems"`
	Confirmed   int `json:"confirmed"`
	NotBlocking int `json:"not_blocking"`
	Wrong       int `json:"wrong"`
	// Duplicates counts findings the grader judged about a problem an
	// earlier one reported, and Ungraded those it gave no verdict.
	Duplicates int `json:"duplicates"`
	Ungraded   int `json:"ungraded"`
	// Missed counts problems the grader found that no finding was about.
	Missed int `json:"missed"`
	// Precision is Confirmed over the problems with a verdict, and Coverage
	// Confirmed over Confirmed and Missed: coverage of what the grader could
	// find, not of every problem there is.
	Precision *float64 `json:"precision,omitempty"`
	Coverage  *float64 `json:"coverage,omitempty"`
	CostUSD   float64  `json:"cost_usd"`
	// Calibration checks the grader against the labeled cases.
	Calibration Calibration `json:"calibration"`
}

type Missed struct {
	File     string `json:"file"`
	Line     int    `json:"line,omitempty"`
	Severity string `json:"severity"`
	About    string `json:"about"`
}

type Options struct {
	// Root is the local repository the suite's revisions are in.
	Root    string
	Suite   Suite
	Prompts []review.Prompt
	// Runs is how many times each case is reviewed; one when zero. Reviews
	// vary from run to run, so comparing prompts on one run each compares
	// luck as much as prompts.
	Runs int
	// Entries are the sidecars the reviews run on. Every case is put on all
	// of them, and its reviews share them; one per prompt runs them all at
	// once.
	Entries []*sidecar.PoolEntry
	Exec    sidecar.Execer
	// Sync mirrors the local directory dir into entry's workspace, leaving
	// the workspace's .git alone, as a pool's sync does.
	Sync   func(ctx context.Context, entry *sidecar.PoolEntry, dir string) error
	Review review.Options
	// Grade, when set, has the grader judge every case run whose reviews all
	// finished. Nil, cases are scored on their labels alone.
	Grade  *GradeOptions
	Status iostream.StatusFunc
	// OnCase is called with each case's result as it is scored.
	OnCase func(CaseResult)
}

type Problem struct {
	// Expected is the case's problem this is; nil when the case does not list
	// it, or lists none.
	Expected *Expected `json:"expected,omitempty"`
	// Findings are the blocking findings about it: for an expected problem,
	// the one paired with it first, and otherwise the first first. More than
	// one is the same problem reported again, by the same prompt or another.
	Findings []review.Finding `json:"findings"`
}

type PromptError struct {
	Prompt string `json:"prompt"`
	Error  string `json:"error"`
}

type PromptSummary struct {
	// Blocking counts its high and medium findings.
	Blocking int `json:"blocking"`
	// Caught counts the expected problems it reported, and Unexpected the
	// unlisted ones on labeled cases.
	Caught     int `json:"caught"`
	Unexpected int `json:"unexpected"`
	// OnClean counts the problems it reported on clean cases, each of which
	// would have sent sound work back.
	OnClean int `json:"on_clean"`
	// Shared counts the problems it reported that another prompt reported
	// too.
	Shared int `json:"shared"`
	// Confirmed, NotBlocking and Wrong count its graded findings by the
	// grader's verdict, duplicates included.
	Confirmed   int `json:"confirmed"`
	NotBlocking int `json:"not_blocking"`
	Wrong       int `json:"wrong"`
	Errors      int `json:"errors"`
}

type Suite struct {
	Dir   string
	Cases []Case
}

type Summary struct {
	// Runs counts case runs, Scored those every review finished, and Errored
	// the rest, which the other counts leave out.
	Runs    int `json:"runs"`
	Scored  int `json:"scored"`
	Errored int `json:"errored"`

	// Blocking counts high and medium findings in scored runs, and Reported
	// the problems they were about, each once, by file and nearby lines.
	// Repeats are the findings beyond the first about each problem.
	Blocking int `json:"blocking"`
	Reported int `json:"reported"`
	Repeats  int `json:"repeats"`
	// Shared counts reported problems more than one prompt reported: where
	// the prompts overlap.
	Shared int `json:"shared"`

	// Labeled counts scored runs of labeled cases; the counts below up to
	// Graded are over those alone.
	Labeled int `json:"labeled"`
	// Expected counts the problems the cases list, and Caught those a
	// blocking finding was about.
	Expected int `json:"expected"`
	Caught   int `json:"caught"`
	// Recall is Caught over Expected; absent when no scored run expected a
	// problem.
	Recall *float64 `json:"recall,omitempty"`
	// ReportedLabeled counts the problems reported on labeled cases, and
	// Unexpected those the cases do not list.
	ReportedLabeled int `json:"reported_labeled"`
	Unexpected      int `json:"unexpected"`
	// Precision is the share of problems reported on labeled cases that the
	// cases list; absent when nothing was reported. It is a floor: an
	// unexpected problem may be a real one the suite does not list.
	Precision *float64 `json:"precision,omitempty"`
	// Clean counts scored runs of clean cases, and CleanPassed those with no
	// blocking finding at all: sound work factory would have let through.
	Clean       int `json:"clean"`
	CleanPassed int `json:"clean_passed"`

	// Graded is the grader's view of the scored runs; nil when none was
	// graded or grading was not asked for.
	Graded *GradedSummary `json:"graded,omitempty"`

	// Prompts breaks the counts down by review prompt, so a noisy, a blind or
	// a redundant one can be told from the rest.
	Prompts map[string]*PromptSummary `json:"prompts"`
}