modelscore

type Answer string

type ArgumentVerdict struct {
	// Name is the argument as the action's input schema spells it.
	Name string
	// Truth is where its right value came from.
	Truth TruthKind
	// Required is whether a completed attempt had to send it.
	Required bool
	// Compared is whether the value was put to a truth at all. An authored
	// value never is, and neither is an optional argument the model left out.
	Compared bool
	// Matched is whether it matched, meaningful only when Compared.
	Matched bool
	// Sent is what the model sent, rendered as the string the comparison was
	// made on. It is empty when the argument was absent.
	Sent string
	// Want is the value the truth resolved to, for a reader of a failure. A
	// fact with several accepted spellings names them all.
	Want string
	// Reason says why a compared argument did not match, or why one could not
	// be compared at all: a truncated answer, a field the answer does not
	// carry, a fact the recipe did not produce.
	Reason string
}

type Attempt struct {
	// Run is the run line of the shard this attempt was written in.
	Run modelrecord.Run
	// Session is the server configuration this attempt talked to.
	Session modelrecord.Session
	// Line is the attempt's own line: the case, the model, the stimulus as
	// sent, the facts it was rendered with, and how it ended.
	Line modelrecord.Attempt
	// Turns are its provider requests, in the order they were written.
	Turns []modelrecord.Turn
	// Calls are the tools/call lines it made, in the order they were written.
	Calls []modelrecord.Call
	// Verifies are the recipe's own checks against GitLab afterwards.
	Verifies []modelrecord.Verify
}

type Confirmation string

type Decline string

type Event struct {
	// Index is the call's position among the calls of the attempt, from 1.
	Index int
	// Turn is the provider turn it was made in.
	Turn int
	// Tool is the tool name the model named.
	Tool string
	// Arguments are the arguments it sent, as the model sent them.
	Arguments json.RawMessage
	// Requested is the canonical action this call names, read here against the
	// whole catalog at the row's tier. It is empty when the call names no
	// catalog action, which is what a discovery call, a standalone tool and an
	// invented tool name all look like.
	Requested string
	// Dispatched is the canonical action the server's span said it ran, after
	// every alias rewrite. It is empty for a discovery call, for a call the
	// server refused before dispatching, and on a surface where nothing
	// records a dispatch.
	Dispatched string
	// DispatchedTool is the tool the span reported, which a discovery call
	// carries without an action.
	DispatchedTool string
	// RefusalReason is the server's own reason for declining, in the server's
	// vocabulary: unknown_action, invalid_params, needs_confirmation,
	// safe_mode.
	RefusalReason string
	// Outcome is the class the answer fell into, in the record's vocabulary.
	Outcome string
	// Result is the structured content the call answered with, which is what a
	// later step's Produced argument is bound against.
	Result json.RawMessage
	// ResultTruncated is whether the record's cap bit on that result. A
	// binding into a truncated result fails naming the truncation rather than
	// missing silently.
	ResultTruncated bool
	// Text is the rendered answer the model read, which is where every refusal
	// this server makes puts its reason.
	Text string
	// Discovery is whether this call was a catalog search rather than an
	// action. It is counted as overhead and never matched against a step.
	Discovery bool
	// Observed is whether the server's span for this call arrived at all.
	Observed bool
}

type MatchKind string

type Outcome string

type Overhead struct {
	// Discovery is how many catalog searches were made.
	Discovery int
	// InvalidParams is how many calls this server refused for their arguments.
	InvalidParams int
	// Steps is how many steps were reached, which is what the cost is per.
	Steps int
}

type Ratio struct {
	Numerator   int
	Denominator int
}

type StepVerdict struct {
	// Position is the step's 1-based place in the key.
	Position int
	// Name is the canonical action the step declares, or the name of the tool
	// when the step names one registered outside the catalog.
	Name string
	// Standalone is whether Name is a tool rather than an action.
	Standalone bool
	// Optional is whether a completed attempt had to reach it.
	Optional bool
	// Destructive is whether the server refuses it without a confirmation.
	Destructive bool
	// Mutating is whether it changes anything, which is what a protective mode
	// withholds or previews.
	Mutating bool
	// Known is whether the catalog at this tier has it at all. A step that
	// names nothing is a corpus defect rather than a model failure, and it is
	// reported as one.
	Known bool
	// Reached is whether a call named it.
	Reached bool
	// MatchedBy is which reading matched: the server's dispatch, the model's
	// request, or the name of a standalone tool.
	MatchedBy MatchKind
	// Requested is the action the reaching call named.
	Requested string
	// Dispatched is the action the server said it ran for that call.
	Dispatched string
	// Rewritten is whether the two differ, which is the alias case: the model
	// named one action and the dispatcher ran another.
	Rewritten bool
	// Call is the index of the reaching call among the attempt's calls, zero
	// when no call reached the step.
	Call int
	// CallOutcome is that call's own outcome, in the record's vocabulary.
	CallOutcome string
	// Answer is what that outcome was worth under the mode.
	Answer Answer
	// Decline says how a mutating step was correctly declined, which is only
	// ever set in read-only mode.
	Decline Decline
	// Confirmation says how a destructive step came to carry its approval.
	Confirmation Confirmation
	// Arguments are the declared arguments and what became of each.
	Arguments []ArgumentVerdict
	// AcceptedFirstTime is whether the first call about this step was the one
	// that reached it and was accepted.
	AcceptedFirstTime bool
	// Retries is how many calls about this step came before the reaching one.
	Retries int
	// Observed is whether the verdict rests on a call whose span arrived. A
	// step decided by an unobserved call is a claim about what was asked.
	Observed bool
	// Complete is whether this step went the way the case and the mode expect.
	Complete bool
	// Reason says why it did not, empty when it did.
	Reason string
}

type Totals struct {
	// Attempts is how many attempts were scored into this row, the five
	// outcomes counted apart below excluded.
	Attempts int
	// Skipped is how many attempts never ran because the instance did not meet
	// the case's needs.
	Skipped int
	// Unobserved is how many attempts rest on a call whose dispatch was never
	// observed. They are counted here and nowhere else: a verdict about a call
	// the server's span never described is a claim about what was asked.
	Unobserved int
	// ProviderErrors is how many attempts the provider would not answer, and
	// HarnessErrors how many this side broke. Neither is the model's, which is
	// what the record says of them in so many words, so neither is in any
	// column.
	ProviderErrors int
	HarnessErrors  int
	// GitLabRefused is how many attempts were dispatched as the case declares,
	// with the arguments it declares, and refused by GitLab. What that reports
	// is the instance or the fixture, so it is counted here rather than folded
	// into a column a model is read by.
	GitLabRefused int
	// Outcomes is how many attempts ended each way, the five counts above
	// included, so a reader can see the shape of a row and not only its rates.
	Outcomes map[Outcome]int
	// Declines is how many steps were correctly declined each way, published
	// apart because declining in text is what a read-only deployment wants and
	// being refused the action is only the conversation ending correctly.
	Declines map[Decline]int
	// Confirmations is how many destructive steps carried their approval each
	// way.
	Confirmations map[Confirmation]int

	// Reached is steps reached or correctly declined over non-optional steps
	// declared.
	Reached Ratio
	// AcceptedFirstTime is steps whose first call about them was the one that
	// reached them, over steps reached.
	AcceptedFirstTime Ratio
	// ArgumentFidelity is arguments whose value matched their truth over
	// arguments declared comparable. An authored value is in neither.
	ArgumentFidelity Ratio
	// Confirmation is destructive steps whose reaching call carried the
	// confirmation over destructive steps declared.
	Confirmation Ratio
	// Unaided is attempts completed with no refusal of ours anywhere over
	// attempts run.
	Unaided Ratio
	// Completion is attempts completed over attempts run.
	Completion Ratio
	// Clean is attempts that went right end to end with no help, over attempts
	// run. It is the headline figure and is a conjunction of the others rather
	// than an average of them: see [Verdict.Clean] for why there is no weighted
	// score here.
	Clean Ratio
	// Overhead is what the model spent getting there.
	Overhead Overhead
}

type TruthKind string

type Verdict struct {
	// Case is the corpus case identifier.
	Case string
	// Model is the provider specification that was asked.
	Model string
	// Surface, Mode and Tier are the row this attempt belongs to.
	Surface string
	Mode    string
	Tier    string
	// Session is the session label that served it.
	Session string
	// Repeat is which run of this attempt it was, from 1.
	Repeat int
	// Outcome is what became of it.
	Outcome Outcome
	// Reason says why, for every outcome but completed.
	Reason string
	// Steps are the declared steps and what became of each.
	Steps []StepVerdict
	// Discovery is how many catalog searches the model made. It is the
	// declared cost of the dynamic surface and is never a failure.
	Discovery int
	// InvalidParams is how many calls this server refused for their arguments,
	// which is a model learning a parameter name from a refusal. A refusal
	// that was itself a correct decline is not counted here: on the meta
	// surface a withheld action is refused by schema validation, and counting
	// that as a retry would charge a model for declining correctly.
	InvalidParams int
	// Unaided is whether the attempt completed with no refusal of ours
	// anywhere: no rejected arguments and no refused confirmation.
	Unaided bool
	// Verified is whether every recipe check against GitLab passed. An attempt
	// with no check recorded is verified by default, since there was nothing
	// to fail.
	Verified bool
}