grader

type Command struct{}

type CommandConfig struct {
	// Run is the command, resolved inside the task's graders directory when
	// its first element is a relative path. The command runs with the
	// workspace as its working directory.
	Run []string `json:"run"`
	// TimeoutSeconds bounds the check. A grader that hangs would otherwise
	// consume the experiment rather than fail one trial.
	TimeoutSeconds int `json:"timeout_seconds,omitempty"`
	// ExpectExit is the exit code that means pass. Zero by default; a task
	// asserting that something correctly fails sets it.
	ExpectExit int `json:"expect_exit,omitempty"`
}

type Files struct{}

type FilesConfig struct {
	// Exists names paths that must be present after the run.
	Exists []string `json:"exists,omitempty"`
	// Absent names paths that must not be. Section 11 pairs a positive case
	// with a negative one: proving a file was written says nothing about
	// whether the run also left a mess behind.
	Absent []string `json:"absent,omitempty"`
	// Matches maps a path to a regular expression its content must satisfy.
	Matches map[string]string `json:"matches,omitempty"`
	// Equals maps a path to its exact expected content.
	Equals map[string]string `json:"equals,omitempty"`
}

type Grader interface {
	Grade(ctx context.Context, in Input) contract.GraderResult
}

type Input struct {
	// Ref is this grader's entry on the task, including its configuration.
	Ref contract.GraderRef
	// Workspace is the final state. Section 7.1 makes it the authoritative
	// answer wherever a deterministic check can reach one.
	Workspace string
	// TaskDir is the task directory, whose graders subdirectory holds material
	// the trial never saw.
	TaskDir string
	// TrialDir is the bundle directory, holding the durable trace.
	TrialDir string
	// Bundle is the execution evidence recorded so far.
	Bundle contract.TrialBundle
}

type Registry map[string]Grader

type Trace struct{}

type TraceConfig struct {
	// ForbiddenTools must not appear. This is how a trust case states a
	// boundary — "never ran a shell" — without depending on the sandbox to
	// have been enabled.
	ForbiddenTools []string `json:"forbidden_tools,omitempty"`
	// RequiredTools must each appear at least once. Use it for an obligation
	// the product makes, such as verifying before reporting done.
	RequiredTools []string `json:"required_tools,omitempty"`
	// MaxToolCalls bounds effort. Zero leaves it unbounded.
	MaxToolCalls int `json:"max_tool_calls,omitempty"`
	// RequireDenial asserts that at least one call was blocked. A trust task
	// where nothing was denied did not exercise the boundary it claims to test.
	RequireDenial bool `json:"require_denial,omitempty"`
	// ForbidDenial asserts that nothing was blocked, for the paired positive
	// case: a run that completed only because it kept hitting the policy is not
	// the same product behavior as one that never needed to.
	ForbidDenial bool `json:"forbid_denial,omitempty"`
	// MaxCompactions bounds context compaction. A run that compacted more than
	// expected did the task, but not the way the context budget intended.
	MaxCompactions *int `json:"max_compactions,omitempty"`
}