e2e

type Cell struct {
	Feature        string  `json:"feature"`
	Harness        string  `json:"harness"`
	HarnessVersion string  `json:"harness_version,omitempty"`
	Model          string  `json:"model"`
	Expected       Support `json:"expected"`
	Status         Status  `json:"status"`
	// Passes and Runs give the pass rate of the cell (R-006).
	Passes int `json:"passes"`
	Runs   int `json:"runs"`
	// Reason tells why the cell did not pass, or why it was not tested.
	Reason string `json:"reason,omitempty"`
}

type Change struct {
	Cell Cell
	// Previous is the status in the previous run, empty when the cell is new.
	Previous Status
}

type Changes struct {
	// Regressions are cells that passed in the previous run and fail now.
	Regressions []Change
	// Improvements are cells marked "Unknown" that pass now and did not pass
	// in the previous run, so the team can update the table.
	Improvements []Change
}

type Check struct {
	Scenario string
	Verify   func(s *Session) error
}

type Claude struct {
	// Bin is the claude executable. Empty means "claude" from PATH.
	Bin string
}

type Codex struct {
	// Bin is the codex executable. Empty means "codex" from PATH.
	Bin string
}

type Config struct {
	// Harnesses are keyed by the column name in the feature table.
	Harnesses map[string]HarnessConfig `json:"harnesses"`
	// ModelDriven is the default repeat of a model-driven feature (R-006).
	ModelDriven Repeat `json:"modelDriven"`
	// Features override the repeat of a feature, keyed by feature name.
	Features        map[string]Repeat `json:"features"`
	ScenarioTimeout Duration          `json:"scenarioTimeout"`
	Parallel        int               `json:"parallel"`
}

type Duration time.Duration

type Expected struct {
	// Features are the feature names in the order of the README.
	Features []string
	// Harnesses are the harness column names in the order of the README.
	Harnesses []string
	// contains filtered or unexported fields
}

type Feature struct {
	// ControlPlane is true for a feature that needs a control plane. The suite
	// runs in local storage mode only, so its cells are "not covered" (D-007).
	ControlPlane bool
	// NotCovered is the reason why the suite cannot see the feature at all,
	// for example because neither the user nor the evidence can observe it.
	// Its cells are "not covered".
	NotCovered string
	// ModelDriven is true for a feature that depends on the model following an
	// instruction. Its cells run more than once (R-006).
	ModelDriven bool
	// Checks must all pass for a run of the feature to pass.
	Checks []Check
}

type Filter struct {
	Harnesses []string
	Models    []string
	Features  []string
}

type Harness interface {
	// Column is the column name of the harness in the feature table.
	Column() string
	// TraceFlag is the flag of `chainloop trace init` that selects the
	// harness, for example "--claude".
	TraceFlag() string
	// Version returns the installed version of the harness.
	Version(ctx context.Context) (string, error)
	// Run runs one headless session and returns its output. An error means
	// that the session could not run, not that a feature failed.
	Run(ctx context.Context, req RunRequest) (*RunOutput, error)
	// HookConfig describes the hook configuration file of the harness.
	HookConfig() HookConfig
	// DisableSessionStart removes the session-start hook that `trace init`
	// installed in the repository, so that a session starts with no
	// session-start hook.
	DisableSessionStart(repo string) error
	// SkillPath returns the file of a project skill, relative to the
	// repository root.
	SkillPath(name string) string
}

type HarnessConfig struct {
	// Version is the pinned harness version (D-013).
	Version string `json:"version"`
	// Models are the models that the suite runs the harness with (D-009).
	Models []string `json:"models"`
}

type HookConfig struct {
	// Path is the file of the Chainloop hooks, relative to the repository
	// root.
	Path string
	// ChainloopMarker is a text that only a Chainloop hook in the file holds.
	ChainloopMarker string
	// UserPath is the file of the hook of the user, relative to the
	// repository root. Empty means Path: the hooks share one file.
	UserPath string
	// UserConfig is the content of the user file with one hook of the user
	// and no Chainloop hook.
	UserConfig string
	// UserMarker is a text that only the hook of the user holds.
	UserMarker string
}

type HookMessage struct {
	// Event is the hook event, for example SessionStart.
	Event string
	// UserMessage is the text that the harness shows to the user.
	UserMessage string
	// ModelContext is the text that the harness gives to the model.
	ModelContext string
}

type OpenCode struct {
	// Major is the major version of OpenCode that the column covers.
	Major string
	// Bin is the opencode executable. Empty means "opencode" from PATH.
	Bin string
}

type Pi struct {
	// Bin is the pi executable. Empty means "pi" from PATH.
	Bin string
}

type Repeat struct {
	Runs      int `json:"runs"`
	Threshold int `json:"threshold"`
}

type Result struct {
	SchemaVersion int       `json:"schema_version"`
	StartedAt     time.Time `json:"started_at"`
	CLIVersion    string    `json:"cli_version"`
	// Full is true for a run with no filter. Only a full run becomes the
	// previous result of the next comparison (R-009).
	Full  bool   `json:"full"`
	Cells []Cell `json:"cells"`
}

type ResultStore struct {
	Dir string
}

type RunOutput struct {
	// Raw is the full output of the harness process.
	Raw []byte
	// Hooks are the hook messages that the harness reported, in order. A
	// harness that does not report them leaves it empty.
	Hooks []HookMessage
	// HooksReported is true when the harness reports hook messages in its
	// headless output. When it is false, a check on a hook message is "not
	// covered" (R-004).
	HooksReported bool
}

type RunRequest struct {
	// Dir is the repository the session runs in.
	Dir string
	// Model is the model of the session, as the harness names it.
	Model string
	// Prompt is the only user prompt of the session.
	Prompt string
	// Env is the environment of the harness process.
	Env []string
	// Wrapper is a command that runs the harness, for example
	// `chainloop trace run --local --claude --`. Empty runs the harness
	// directly.
	Wrapper []string
	// Images are PNG images that the user pastes with the prompt.
	Images [][]byte
}

type Runner struct {
	Expected  *Expected
	Config    *Config
	Harnesses []Harness
	Filter    Filter
	// ChainloopBin is the chainloop CLI under test.
	ChainloopBin string
	CLIVersion   string
	// WorkDir holds the workspaces and the harness output of each session.
	WorkDir string
	Logf    func(format string, args ...any)
}

type Scenario struct {
	Name string
	// Seed are the files of the repository before the session, by path.
	Seed map[string]string
	// Prompt is the only user prompt of the session.
	Prompt string
	// Checkout is the checkout where the agent commits and pushes, relative
	// to the workspace base. Empty means the repository of the workspace.
	Checkout string
	// Images are PNG images that the user pastes with the prompt.
	Images [][]byte
	// Skills are the project skills of the repository, by name. The harness
	// gives the path of each skill file.
	Skills map[string]string
	// NoAgent runs no harness session. The checks run commands in the
	// workspace themselves.
	NoAgent bool
	// NoInit skips `chainloop trace init` in the workspace.
	NoInit bool
	// WrapTraceRun runs the harness with `chainloop trace run --local`.
	WrapTraceRun bool
	// SeedUserHooks writes a hook configuration of the harness with one hook
	// of the user before `chainloop trace init`.
	SeedUserHooks bool
	// DisableSessionStart removes the session-start hook after
	// `chainloop trace init`.
	DisableSessionStart bool
}

type Session struct {
	Scenario  string
	Model     string
	Harness   Harness
	Workspace *Workspace
	// Checkout is the checkout where the agent committed and pushed.
	Checkout string
	Output   *RunOutput
	// Stored is the session that the push stored in local storage mode.
	Stored *StoredSession
	// HeadMessage is the message of the commit that the agent made.
	HeadMessage string
}

type Status string

type StoredSession struct {
	// Dir is the directory of the stored session.
	Dir string
	// Evidence is the AI coding session evidence.
	Evidence aicodingsession.Evidence
}

type Support string

type Workspace struct {
	// Dir is the repository.
	Dir string
	// Env is the environment of every command in the workspace, the harness
	// included: the chainloop binary under test first in PATH, and an
	// isolated git configuration.
	Env []string
	// Chainloop is the chainloop CLI under test.
	Chainloop string
}