eval

type Case struct {
	Name string `json:"name"`
	// Revision is the commit the patch applies to.
	Revision string `json:"revision"`
	// Request is the change the patch answers, which reviewers are given as
	// the specification, as factory gives them the developer's prompt.
	Request string `json:"request"`
	// Patch is the change, a path to a git diff relative to the suite
	// directory.
	Patch string `json:"patch"`
	// Source says where the case came from, such as a pull request or a
	// mutation, for whoever reads the results.
	Source string `json:"source,omitempty"`
	// Expect are the problems a review should find. Empty means the change is
	// sound and should come back with no high or medium findings.
	Expect []Expected `json:"expect,omitempty"`
	// contains filtered or unexported fields
}

type CaseResult struct {
	Case string `json:"case"`
	Run  int    `json:"run"`
	// Clean is whether the case expects no problems.
	Clean bool `json:"clean"`
	// Errors are the reviews that could not run. A case with any is left out
	// of the summary rather than scored on the reviews that did, which would
	// make a broken prompt look like a quiet one.
	Errors []PromptError `json:"errors,omitempty"`
	// Problems are what the blocking findings were about, each once, with
	// every finding about it: an expected problem one caught, or one the case
	// does not list. An unlisted problem on a clean case sends sound work
	// back. On another it may be real, so it is worth reading rather than
	// counting blindly.
	Problems []Problem `json:"problems,omitempty"`
	// Missed are the expected problems no blocking finding was about, in the
	// case's order.
	Missed []Expected `json:"missed,omitempty"`
	// Findings is everything the reviews reported, blocking or not.
	Findings []review.Finding `json:"findings"`
}

type Expected struct {
	// File is the repository-relative path the problem is in.
	File string `json:"file"`
	// Lines is the range of new-file lines the problem is on, first and last.
	// Empty means anywhere in the file.
	Lines []int `json:"lines,omitempty"`
	// About says what the problem is, for whoever reads the results.
	About string `json:"about"`
}

type Options struct {
	// Root is the local repository the suite's revisions are in.
	Root    string
	Suite   Suite
	Prompts []review.Prompt
	// Runs is how many times each case is reviewed; one when zero. Reviews
	// vary from run to run, so comparing prompts on one run each compares
	// luck as much as prompts.
	Runs int
	// Entries are the sidecars the reviews run on. Every case is put on all
	// of them, and its reviews share them; one per prompt runs them all at
	// once.
	Entries []*sidecar.PoolEntry
	Exec    sidecar.Execer
	// Sync mirrors the local directory dir into entry's workspace, leaving
	// the workspace's .git alone, as a pool's sync does.
	Sync   func(ctx context.Context, entry *sidecar.PoolEntry, dir string) error
	Review review.Options
	Status iostream.StatusFunc
	// OnCase is called with each case's result as it is scored.
	OnCase func(CaseResult)
}

type Problem struct {
	// Expected is the case's problem this is; nil when the case does not list
	// it.
	Expected *Expected `json:"expected,omitempty"`
	// Findings are the blocking findings about it, the first first. More than
	// one is the same problem reported again, by the same prompt or another.
	Findings []review.Finding `json:"findings"`
}

type PromptError struct {
	Prompt string `json:"prompt"`
	Error  string `json:"error"`
}

type PromptSummary struct {
	// Blocking counts its high and medium findings.
	Blocking int `json:"blocking"`
	// Caught counts the expected problems it reported, and Unexpected the
	// unlisted ones.
	Caught     int `json:"caught"`
	Unexpected int `json:"unexpected"`
	// OnClean counts the problems it reported on clean cases, each of which
	// would have sent sound work back.
	OnClean int `json:"on_clean"`
	// Shared counts the problems it reported that another prompt reported
	// too.
	Shared int `json:"shared"`
	Errors int `json:"errors"`
}

type Suite struct {
	Dir   string
	Cases []Case
}

type Summary struct {
	// Runs counts case runs, Scored those every review finished, and Errored
	// the rest, which the other counts leave out.
	Runs    int `json:"runs"`
	Scored  int `json:"scored"`
	Errored int `json:"errored"`

	// Expected counts the problems in scored runs, and Caught those a
	// blocking finding was about.
	Expected int `json:"expected"`
	Caught   int `json:"caught"`
	// Recall is Caught over Expected; absent when no scored run expected a
	// problem.
	Recall *float64 `json:"recall,omitempty"`

	// Blocking counts high and medium findings in scored runs, and Reported
	// the problems they were about, each once. Unexpected are the reported
	// problems the suite does not list, and Repeats the findings beyond the
	// first about each problem.
	Blocking   int `json:"blocking"`
	Reported   int `json:"reported"`
	Unexpected int `json:"unexpected"`
	Repeats    int `json:"repeats"`
	// Shared counts reported problems more than one prompt reported: where
	// the prompts overlap.
	Shared int `json:"shared"`
	// Precision is the share of reported problems the suite lists; absent
	// when nothing was reported. It is a floor: an unexpected problem may be
	// a real one the suite does not list.
	Precision *float64 `json:"precision,omitempty"`

	// Clean counts scored runs of clean cases, and CleanPassed those with no
	// blocking finding at all: sound work factory would have let through.
	Clean       int `json:"clean"`
	CleanPassed int `json:"clean_passed"`

	// Prompts breaks the counts down by review prompt, so a noisy, a blind or
	// a redundant one can be told from the rest.
	Prompts map[string]*PromptSummary `json:"prompts"`
}