evaluator

package
v2.2.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 12, 2026 License: MIT Imports: 45 Imported by: 0

Documentation

Overview

Package evaluator runs model evaluations against GitLab MCP tool surfaces.

The package owns provider execution, MCP bridge handlers, fixture preparation, validation, reporting, and publication. Static case definitions live in the cases subpackage so new evaluator tasks can be added without sorting through runtime code.

Package evaluator implements the eval_mcp_surfaces command workflow.

Index

Constants

This section is empty.

Variables

View Source 
var (
	MergeableMergeRequestFixture = CaseFixtureSpec{
		Name:                "mergeable_merge_request",
		Scope:               FixtureScopeAttempt,
		Timeout:             8 * time.Minute,
		Retries:             1,
		Outputs:             []string{"project_id", "project_path", "default_branch", "merge_request_iid"},
		IdempotencyKeyParts: []string{"mergeable_merge_request"},
		Ensure:              ensureMergeableMergeRequestFixture,
		Validate:            validateLiveCaseFixtureOutput,
		Cleanup:             noopCaseFixtureCleanup,
	}
	JobTokenScopeProjectFixture = CaseFixtureSpec{
		Name:                "job_token_scope_project",
		Scope:               FixtureScopeAttempt,
		Timeout:             2 * time.Minute,
		Retries:             2,
		Outputs:             []string{"project_id", "project_path", "target_project_id"},
		IdempotencyKeyParts: []string{"job_token_scope_project"},
		Ensure:              ensureJobTokenScopeProjectFixture,
		Validate:            validateLiveCaseFixtureOutput,
		Cleanup:             noopCaseFixtureCleanup,
	}
	FailedJobArtifactFixture = CaseFixtureSpec{
		Name:                "failed_job_artifact",
		Scope:               FixtureScopeAttempt,
		Timeout:             4 * time.Minute,
		Retries:             1,
		Outputs:             []string{"project_id", "project_path", "job_id", "artifact_path"},
		IdempotencyKeyParts: []string{"failed_job_artifact"},
		Ensure:              ensureFailedJobArtifactFixture,
		Validate:            validateLiveCaseFixtureOutput,
		Cleanup:             noopCaseFixtureCleanup,
	}
	MergeRequestAwardEmojiFixture = CaseFixtureSpec{
		Name:                "merge_request_award_emoji",
		Scope:               FixtureScopeAttempt,
		Timeout:             2 * time.Minute,
		Retries:             1,
		Outputs:             []string{"project_id", "project_path", "merge_request_iid", "award_id"},
		IdempotencyKeyParts: []string{"merge_request_award_emoji"},
		Ensure:              ensureMergeRequestAwardEmojiFixture,
		Validate:            validateLiveCaseFixtureOutput,
		Cleanup:             noopCaseFixtureCleanup,
	}
	IssueAwardEmojiFixture = CaseFixtureSpec{
		Name:                "issue_award_emoji",
		Scope:               FixtureScopeAttempt,
		Timeout:             2 * time.Minute,
		Retries:             1,
		Outputs:             []string{"project_id", "project_path", "issue_iid", "award_id"},
		IdempotencyKeyParts: []string{"issue_award_emoji"},
		Ensure:              ensureIssueAwardEmojiFixture,
		Validate:            validateLiveCaseFixtureOutput,
		Cleanup:             noopCaseFixtureCleanup,
	}
	GroupDeleteFixture = destructiveAttemptFixture("group_delete", []string{"group_id", "group_path"}, nil, func(ctx context.Context, env FixtureContext, preparer *liveFixturePreparer, output FixtureOutput) (FixtureOutput, error) {
		attemptOutput := attemptNameFixtureOutput(env)
		groupPath := preparer.state.GroupPath + "/" + attemptOutput["subgroup_path"]
		group, err := preparer.ensureGroup(ctx, attemptOutput["subgroup_name"], groupPath, preparer.state.GroupID)
		if err != nil {
			return nil, err
		}
		output["group_id"] = strconv.FormatInt(group.ID, 10)
		output["group_path"] = group.FullPath
		output["group_name"] = group.Name
		return output, nil
	})
	IssueDeleteFixture = destructiveAttemptFixture("issue_delete", []string{"project_id", "project_path", "issue_iid"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensureCoreIssues(ctx)
	}, func(_ context.Context, _ FixtureContext, preparer *liveFixturePreparer, output FixtureOutput) (FixtureOutput, error) {
		output["issue_iid"] = strconv.FormatInt(preparer.state.IssueDeleteIID, 10)
		output["issue_title"] = "Fixture issue safe to delete"
		return output, nil
	})
	ProjectCIVariableDeleteFixture = destructiveAttemptFixture("project_ci_variable_delete", []string{"project_id", "project_path", "ci_variable_key"}, nil, ensureProjectCIVariableDeleteFixture)
	RepositoryFileDeleteFixture    = destructiveAttemptFixture("repository_file_delete", []string{"project_id", "project_path", "branch_name", "file_path"}, nil, ensureRepositoryFileDeleteFixture)
	MilestoneDeleteFixture         = destructiveAttemptFixture("milestone_delete", []string{"project_id", "project_path", "milestone_iid"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensureMilestone(ctx)
	}, func(_ context.Context, _ FixtureContext, preparer *liveFixturePreparer, output FixtureOutput) (FixtureOutput, error) {
		output["milestone_iid"] = strconv.FormatInt(preparer.state.MilestoneDeleteIID, 10)
		return output, nil
	})
	ReleaseDeleteFixture            = destructiveAttemptFixture("release_delete", []string{"project_id", "project_path", "release_tag_name", "release_name"}, nil, ensureReleaseDeleteFixture)
	ProjectAccessTokenRevokeFixture = destructiveAttemptFixture("project_access_token_revoke", []string{"project_id", "project_path", "token_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensureProjectAccessToken(ctx)
	}, func(_ context.Context, _ FixtureContext, preparer *liveFixturePreparer, output FixtureOutput) (FixtureOutput, error) {
		output["token_id"] = strconv.FormatInt(preparer.state.ProjectTokenID, 10)
		return output, nil
	})
	ProjectArchiveFixture = destructiveAttemptFixture("project_archive", []string{"project_id", "project_path"}, nil, ensureProjectArchiveFixture)
	PackageDeleteFixture  = destructiveAttemptFixture("package_delete", []string{"project_id", "project_path", "package_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensurePackage(ctx)
	}, nil)
	PipelineDeleteFixture = destructiveAttemptFixture("pipeline_delete", []string{"project_id", "project_path", "pipeline_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensurePipeline(ctx)
	}, nil)
	PipelineTriggerDeleteFixture = destructiveAttemptFixture("pipeline_trigger_delete", []string{"project_id", "project_path", "pipeline_trigger_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensurePipelineTriggers(ctx)
	}, nil)
	PipelineScheduleDeleteFixture = destructiveAttemptFixture("pipeline_schedule_delete", []string{"project_id", "project_path", "pipeline_schedule_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensurePipelineSchedules(ctx)
	}, nil)
	RunnerRemoveFixture = destructiveAttemptFixture("runner_remove", []string{"runner_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensureDisposableRunner(ctx)
	}, nil)
	EnvironmentStopFixture          = destructiveAttemptFixture("environment_stop", []string{"project_id", "project_path", "environment_id", "environment_name"}, nil, ensureEnvironmentStopFixture)
	SnippetDeleteFixture            = destructiveAttemptFixture("snippet_delete", []string{"snippet_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensureSnippet(ctx) }, nil)
	BroadcastMessageDeleteFixture   = destructiveAttemptFixture("broadcast_message_delete", []string{"id"}, nil, ensureBroadcastMessageDeleteFixture)
	ProjectHookDeleteFixture        = destructiveAttemptFixture("project_hook_delete", []string{"project_id", "project_path", "hook_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensureHooks(ctx) }, nil)
	ProjectBadgeDeleteFixture       = destructiveAttemptFixture("project_badge_delete", []string{"project_id", "project_path", "badge_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensureBadge(ctx) }, nil)
	DraftNotePublishAllFixture      = destructiveAttemptFixture("draft_note_publish_all", []string{"project_id", "project_path", "merge_request_iid"}, ensureDraftNotePublishAllFixture, nil)
	InstanceCIVariableDeleteFixture = destructiveAttemptFixture("instance_ci_variable_delete", []string{"instance_ci_variable_key"}, nil, ensureInstanceCIVariableDeleteFixture)
	BranchDeleteFixture             = destructiveAttemptFixture("branch_delete", []string{"project_id", "project_path", "branch_name"}, nil, ensureBranchDeleteFixture)
	TagDeleteFixture                = destructiveAttemptFixture("tag_delete", []string{"project_id", "project_path", "tag_name"}, nil, ensureTagDeleteFixture)
	UserBlockFixture                = destructiveAttemptFixture("user_block", []string{"user_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensureDisposableUser(ctx)
	}, nil)
	FeatureFlagDeleteFixture          = destructiveAttemptFixture("feature_flag_delete", []string{"project_id", "project_path", "feature_flag_name"}, nil, ensureFeatureFlagDeleteFixture)
	WikiDeleteFixture                 = destructiveAttemptFixture("wiki_delete", []string{"project_id", "project_path", "wiki_slug"}, nil, ensureWikiDeleteFixture)
	DeployKeyLifecycleFixture         = destructiveAttemptFixture("deploy_key_lifecycle", []string{"project_id", "project_path", "deploy_key_title", "deploy_key_updated_title", "deploy_key_key"}, nil, ensureDeployKeyLifecycleFixture)
	DeployKeyDeleteFixture            = destructiveAttemptFixture("deploy_key_delete", []string{"project_id", "project_path", "deploy_key_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensureDeployKey(ctx) }, nil)
	DeployTokenDeleteFixture          = destructiveAttemptFixture("deploy_token_delete", []string{"project_id", "project_path", "deploy_token_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensureDeployToken(ctx) }, nil)
	CommitDiscussionDeleteNoteFixture = destructiveAttemptFixture("commit_discussion_delete_note", []string{"project_id", "project_path", "commit_sha", "discussion_id", "note_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensureCommitDiscussion(ctx)
	}, func(_ context.Context, _ FixtureContext, preparer *liveFixturePreparer, output FixtureOutput) (FixtureOutput, error) {
		output["commit_sha"] = preparer.state.CommitSHA
		output["discussion_id"] = preparer.state.CommitDiscussionID
		output["note_id"] = strconv.FormatInt(preparer.state.CommitDiscussionNoteID, 10)
		return output, nil
	})
	BranchProtectionLifecycleFixture = CaseFixtureSpec{
		Name:                "branch_protection_lifecycle",
		Scope:               FixtureScopeAttempt,
		Timeout:             2 * time.Minute,
		Retries:             1,
		Outputs:             []string{"project_id", "project_path", "default_branch", "branch_name"},
		IdempotencyKeyParts: []string{"branch_protection_lifecycle"},
		Ensure:              ensureBranchProtectionLifecycleFixture,
		Validate:            validateLiveCaseFixtureOutput,
		Cleanup:             noopCaseFixtureCleanup,
	}
)
View Source 
var (
	AttemptNamesFixture = CaseFixtureSpec{
		Name:                "attempt_names",
		Scope:               FixtureScopeAttempt,
		Timeout:             5 * time.Second,
		Retries:             0,
		Outputs:             attemptNameFixtureOutputs,
		IdempotencyKeyParts: []string{"attempt_names"},
		Ensure: func(_ context.Context, env FixtureContext) (FixtureOutput, error) {
			return attemptNameFixtureOutput(env), nil
		},
		Validate: validateAttemptNameFixtureOutput,
		Cleanup:  noopCaseFixtureCleanup,
	}
	BranchFixture       = liveCaseFixture("branch", FixtureScopeCase, []string{"project_id", "default_branch", "feature_branch"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensureBranches(ctx) }, "branch")
	FileFixture         = liveCaseFixture("file", FixtureScopeCase, []string{"project_id", "feature_branch", "file_path"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensureBranches(ctx) }, "file")
	IssueFixture        = liveCaseFixture("issue", FixtureScopeCase, []string{"project_id", "issue_iid"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensureCoreIssues(ctx) }, "issue")
	MergeRequestFixture = liveCaseFixture("merge_request", FixtureScopeCase, []string{"project_id", "merge_request_iid", "feature_branch"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensureMergeRequests(ctx)
	}, "merge_request")
	MergeRequestDiscussionFixture = liveCaseFixture("merge_request_discussion", FixtureScopeAttempt, []string{"project_id", "project_path", "merge_request_iid", "discussion_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		if err := preparer.ensureMergeRequests(ctx); err != nil {
			return err
		}
		return preparer.ensureDiscussions(ctx)
	}, "merge_request_discussion")
	PipelineJobFixture = liveCaseFixture("pipeline_job", FixtureScopeAttempt, []string{"project_id", "project_path", "pipeline_id", "job_id", "failed_job_id", "manual_job_id", "runner_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensurePipeline(ctx)
	}, "pipeline_job")
	ReleaseFixture = liveCaseFixture("release", FixtureScopeCase, []string{"project_id", "release_summary_tag"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensureCleanupRelease(ctx)
	}, "release")
	TagFixture = liveCaseFixture("tag", FixtureScopeCase, []string{"project_id", "tag_name"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensureTag(ctx, liveFixtureElicitationTag, preparer.defaultRef())
	}, "tag")
	CIVariableFixture      = liveCaseFixture("ci_variable", FixtureScopeCase, []string{"project_id", "group_id", "ci_variable_key", "group_ci_variable_key", "instance_ci_variable_key"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensureCIVariables(ctx) }, "ci_variable")
	HookFixture            = liveCaseFixture("hook", FixtureScopeCase, []string{"project_id", "hook_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensureHooks(ctx) }, "hook")
	BadgeFixture           = liveCaseFixture("badge", FixtureScopeCase, []string{"project_id", "badge_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensureBadge(ctx) }, "badge")
	WikiFixture            = liveCaseFixture("wiki", FixtureScopeCase, []string{"project_id", "wiki_slug"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensureWiki(ctx) }, "wiki")
	SnippetFixture         = liveCaseFixture("snippet", FixtureScopeCase, []string{"snippet_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensureSnippet(ctx) }, "snippet")
	FeatureFlagFixture     = liveCaseFixture("feature_flag", FixtureScopeCase, []string{"project_id", "feature_flag_name"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensureFeatureFlag(ctx) }, "feature_flag")
	DeployTokenFixture     = liveCaseFixture("deploy_token", FixtureScopeCase, []string{"project_id", "deploy_token_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensureDeployToken(ctx) }, "deploy_token")
	DeployKeyFixture       = liveCaseFixture("deploy_key", FixtureScopeCase, []string{"project_id", "deploy_key_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensureDeployKey(ctx) }, "deploy_key")
	PackageFixture         = liveCaseFixture("package", FixtureScopeCase, []string{"project_id", "package_id", "package_name", "package_file"}, func(ctx context.Context, preparer *liveFixturePreparer) error { return preparer.ensurePackage(ctx) }, "package")
	PackageReleaseFixture  = liveCaseFixture("package_release", FixtureScopeCase, []string{"project_id", "package_release_name", "package_release_version", "package_release_tag", "package_release_dir", "package_release_files"}, ensurePackageReleaseFixture, "package_release")
	PipelineTriggerFixture = liveCaseFixture("pipeline_trigger", FixtureScopeCase, []string{"project_id", "pipeline_trigger_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensurePipelineTriggers(ctx)
	}, "pipeline_trigger")
	PipelineScheduleFixture = liveCaseFixture("pipeline_schedule", FixtureScopeCase, []string{"project_id", "pipeline_schedule_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensurePipelineSchedules(ctx)
	}, "pipeline_schedule")
	MemberFixture = liveCaseFixture("member", FixtureScopeCase, []string{"project_id", "user_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
		return preparer.ensureDisposableUser(ctx)
	}, "member")
	MergeRequestSourceFixture = CaseFixtureSpec{
		Name:                "merge_request_source",
		Scope:               FixtureScopeAttempt,
		Timeout:             2 * time.Minute,
		Retries:             2,
		Outputs:             []string{"project_id", "project_path", "default_branch", "mr_source_branch", "mr_title"},
		IdempotencyKeyParts: []string{"merge_request_source"},
		Ensure: func(ctx context.Context, env FixtureContext) (FixtureOutput, error) {
			return liveFixtureOutputs.ensure(env.IdempotencyKey, func() (FixtureOutput, error) {
				preparer, err := newLiveCaseFixturePreparer(ctx, env)
				if err != nil {
					return nil, err
				}
				attemptOutput := attemptNameFixtureOutput(env)
				sourceBranch := attemptOutput["mr_source_branch"]
				if branchErr := preparer.ensureBranch(ctx, sourceBranch, preparer.defaultRef()); branchErr != nil {
					return nil, branchErr
				}
				filePath := "tmp/eval-mr-" + safeFixturePathPart(sourceBranch) + ".txt"
				if fileErr := preparer.ensureFile(ctx, filePath, sourceBranch, "evaluation merge request fixture\n", "Seed evaluation merge request fixture"); fileErr != nil {
					return nil, fileErr
				}
				if closeErr := preparer.closeOpenMergeRequestsForBranch(ctx, sourceBranch); closeErr != nil {
					return nil, closeErr
				}
				output := fixtureOutputFromLiveState(preparer.state)
				maps.Copy(output, attemptOutput)
				return output, nil
			})
		},
		Validate: validateLiveCaseFixtureOutput,
		Cleanup:  noopCaseFixtureCleanup,
	}
	ReleaseCreateSourceFixture = CaseFixtureSpec{
		Name:                "release_create_source",
		Scope:               FixtureScopeAttempt,
		Timeout:             2 * time.Minute,
		Retries:             2,
		Outputs:             []string{"project_id", "project_path", "default_branch", "release_tag_name", "release_name"},
		IdempotencyKeyParts: []string{"release_create_source"},
		Ensure: func(ctx context.Context, env FixtureContext) (FixtureOutput, error) {
			return liveFixtureOutputs.ensure(env.IdempotencyKey, func() (FixtureOutput, error) {
				preparer, err := newLiveCaseFixturePreparer(ctx, env)
				if err != nil {
					return nil, err
				}
				attemptOutput := attemptNameFixtureOutput(env)
				attemptOutput["release_tag_name"] = suffixEvaluationValue(liveFixtureElicitationTag, attemptOutput["attempt_suffix"])
				attemptOutput["release_name"] = suffixEvaluationValue("Evaluation elicitation release", attemptOutput["attempt_suffix"])
				if tagErr := preparer.ensureTag(ctx, attemptOutput["release_tag_name"], preparer.defaultRef()); tagErr != nil {
					return nil, tagErr
				}
				output := fixtureOutputFromLiveState(preparer.state)
				maps.Copy(output, attemptOutput)
				return output, nil
			})
		},
		Validate: validateLiveCaseFixtureOutput,
		Cleanup:  noopCaseFixtureCleanup,
	}
)
View Source 
var BootstrapProjectFixture = liveCaseFixture("bootstrap_project", FixtureScopeBootstrap, []string{"project_id", "project_path", "default_branch"}, nil, "project")

BootstrapProjectFixture describes the shared project state reused by typed Docker evaluation cases.

View Source 
var ProjectServiceAccountFixture = enterpriseLiveCaseFixture("project_service_account", FixtureScopeCase, []string{"project_id", "project_path", "project_service_account_id"}, func(ctx context.Context, preparer *liveFixturePreparer) error {
	return preparer.ensureProjectServiceAccount(ctx)
}, "project_service_account")

Functions

func RenderCasePrompt 

func RenderCasePrompt(evalCase EvalCase, output FixtureOutput) (string, error)

RenderCasePrompt renders a typed case prompt from fixture outputs.

func Run 

func Run() (runErr error)

Run executes the eval_mcp_surfaces command workflow.

func ValidateEvalCaseRegistry 

func ValidateEvalCaseRegistry(routes map[string]toolutil.ActionMap) []string

ValidateEvalCaseRegistry validates the migrated typed evaluation registry.

Types

type CaseAssertion 

type CaseAssertion struct {
	Type        CaseAssertionType
	Step        int
	Name        string
	Description string
	Required    bool
	Inputs      []string
	Expected    map[string]string
}

CaseAssertion describes a post-call assertion for a case.

type CaseAssertionResult 

type CaseAssertionResult struct {
	Type    CaseAssertionType
	Step    int
	Name    string
	Passed  bool
	Message string
}

CaseAssertionResult records the outcome of one typed case assertion.

type CaseAssertionType 

type CaseAssertionType string

CaseAssertionType identifies the typed validation rule a case assertion uses. 

const (
	// CaseAssertionExpectedAction verifies the model selected the expected tool action.
	CaseAssertionExpectedAction CaseAssertionType = "expected_action"
	// CaseAssertionRequiredParams verifies all required parameters are present.
	CaseAssertionRequiredParams CaseAssertionType = "required_params"
	// CaseAssertionOptionalParams verifies optional parameters are accepted when present.
	CaseAssertionOptionalParams CaseAssertionType = "optional_params"
	// CaseAssertionForbiddenParams verifies forbidden parameters are not present.
	CaseAssertionForbiddenParams CaseAssertionType = "forbidden_params"
	// CaseAssertionDestructiveConfirm verifies destructive calls include confirmation.
	CaseAssertionDestructiveConfirm CaseAssertionType = "destructive_confirm"
	// CaseAssertionOutputContains verifies tool output contains expected evidence.
	CaseAssertionOutputContains CaseAssertionType = "output_contains"
	// CaseAssertionProducedValue verifies a step produced a value used later.
	CaseAssertionProducedValue CaseAssertionType = "produced_value"
	// CaseAssertionNoExtraToolCall verifies the model did not call extra tools.
	CaseAssertionNoExtraToolCall CaseAssertionType = "no_extra_tool_call"
	// CaseAssertionAllowRepair verifies a known repair path is allowed.
	CaseAssertionAllowRepair CaseAssertionType = "allow_repair"
)

type CaseFixtureCleanupFunc 

type CaseFixtureCleanupFunc func(context.Context, FixtureContext, FixtureOutput) error

CaseFixtureCleanupFunc removes fixture state owned by a case attempt.

type CaseFixtureContext 

type CaseFixtureContext = FixtureContext

CaseFixtureContext is kept as a temporary alias for Phase 2 migration code.

type CaseFixtureEnsureFunc 

type CaseFixtureEnsureFunc func(context.Context, FixtureContext) (FixtureOutput, error)

CaseFixtureEnsureFunc creates or repairs fixture state.

type CaseFixtureSpec 

type CaseFixtureSpec struct {
	Name                string
	Scope               FixtureScope
	Timeout             time.Duration
	Retries             int
	RequiredRuntime     EvalCaseEdition
	Ensure              CaseFixtureEnsureFunc
	Validate            CaseFixtureValidateFunc
	Cleanup             CaseFixtureCleanupFunc
	Outputs             []string
	IdempotencyKeyParts []string
}

CaseFixtureSpec declares the GitLab state needed by a case.

func EnterpriseGroupServiceAccountFixture 

func EnterpriseGroupServiceAccountFixture(withPAT bool) CaseFixtureSpec

func EnterprisePushRuleProjectFixture 

func EnterprisePushRuleProjectFixture(seedRule bool) CaseFixtureSpec

type CaseFixtureValidateFunc 

type CaseFixtureValidateFunc func(context.Context, FixtureContext, FixtureOutput) error

CaseFixtureValidateFunc verifies fixture state before prompt rendering.

type CaseMetricsSpec 

type CaseMetricsSpec struct {
	ExpectedModelCalls int
	ExpectedToolCalls  int
	FinalSuccess       bool
}

CaseMetricsSpec customizes how a case contributes to evaluator metrics.

type CasePromptTemplate 

type CasePromptTemplate struct {
	Text      string
	Variables []string
}

CasePromptTemplate captures prompt text plus fixture variables.

type EvalCase 

type EvalCase struct {
	ID               EvalCaseID
	Title            string
	Prompt           string
	PromptTemplate   CasePromptTemplate
	Steps            []ExpectedStep
	Fixtures         []CaseFixtureSpec
	Assertions       []CaseAssertion
	Metrics          CaseMetricsSpec
	Edition          EvalCaseEdition
	Presets          []EvalPreset
	Partition        EvalPartition
	Tags             []string
	Mutating         bool
	Destructive      bool
	CapabilityBridge bool
	SkipReasons      []string
	ReportGroup      string
}

EvalCase is the typed source of truth for one evaluator task.

func AllEvalCases 

func AllEvalCases() []EvalCase

AllEvalCases returns the typed evaluation case registry.

func CaseByID 

func CaseByID(id string) (EvalCase, bool)

CaseByID looks up one typed evaluation case by ID.

func CasesByPreset 

func CasesByPreset(preset string) []EvalCase

CasesByPreset returns typed evaluation cases matching an evaluator preset.

type EvalCaseEdition 

type EvalCaseEdition string

EvalCaseEdition identifies the GitLab edition a case targets.

type EvalCaseID 

type EvalCaseID string

EvalCaseID identifies one model-evaluation case.

type EvalPartition 

type EvalPartition string

EvalPartition identifies the logical evaluation partition for a case.

type EvalPreset 

type EvalPreset string

EvalPreset identifies an evaluator preset that may run a case.

type ExpectedStep 

type ExpectedStep struct {
	ExpectedTool    string
	ExpectedAction  string
	RequiredParams  []string
	OptionalParams  []string
	ForbiddenParams []string
	OptionalStep    bool
	Destructive     bool
	Simulation      string
	AllowedRepairs  []string
	ProducedValues  []string
}

ExpectedStep describes one expected MCP tool or action call.

type FixtureContext 

type FixtureContext struct {
	Client         *gitlabclient.Client
	MCPSession     *mcp.ClientSession
	RuntimeEdition EvalCaseEdition
	ToolSurface    string
	RunSuffix      string
	ModelName      string
	RunIndex       int
	CaseID         EvalCaseID
	FixtureName    string
	IdempotencyKey string
	Logf           func(string, ...any)
}

FixtureContext describes the runtime preparing a case fixture.

type FixtureHealth 

type FixtureHealth struct {
	Name    string
	Ready   bool
	Message string
	Outputs FixtureOutput
}

FixtureHealth records fixture readiness for report and trace diagnostics.

type FixtureOutput 

type FixtureOutput map[string]string

FixtureOutput stores named values emitted by fixture builders.

type FixtureScope 

type FixtureScope string

FixtureScope controls how fixture resources are shared across attempts. 

const (
	// FixtureScopeBootstrap allows a fixture to be reused across unrelated cases.
	FixtureScopeBootstrap FixtureScope = "bootstrap"
	// FixtureScopeRun creates fixture state for one evaluator run suffix.
	FixtureScopeRun FixtureScope = "run"
	// FixtureScopeCase creates fixture state for one case.
	FixtureScopeCase FixtureScope = "case"
	// FixtureScopeAttempt creates fixture state for one model/run attempt.
	FixtureScopeAttempt FixtureScope = "attempt"
)

type PreparedCase 

type PreparedCase struct {
	Case           EvalCase
	Prompt         string
	Steps          []ExpectedStep
	FixtureOutputs FixtureOutput
	FixtureHealth  []FixtureHealth
	Cleanup        []PreparedFixtureCleanup
}

PreparedCase contains a rendered case and the fixture state created for it.

func PrepareCaseAttempt 

func PrepareCaseAttempt(ctx context.Context, env FixtureContext, evalCase EvalCase, model string, runIndex int) (PreparedCase, error)

PrepareCaseAttempt prepares fixtures and renders a typed evaluation case.

type PreparedFixtureCleanup 

type PreparedFixtureCleanup func(context.Context) error

PreparedFixtureCleanup removes fixture state owned by a prepared case.

type PromptAwardData 

type PromptAwardData struct {
	ID   string
	Name string
}

PromptAwardData contains award emoji fixture values.

type PromptBranchData 

type PromptBranchData struct {
	Name    string
	Default string
}

PromptBranchData contains branch fixture values.

type PromptData 

type PromptData struct {
	Project      PromptProjectData
	Group        PromptGroupData
	Issue        PromptIssueData
	MergeRequest PromptMergeRequestData
	Pipeline     PromptPipelineData
	Job          PromptJobData
	Branch       PromptBranchData
	Tag          PromptTagData
	Release      PromptReleaseData
	Package      PromptPackageData
	Environment  PromptEnvironmentData
	Runner       PromptRunnerData
	Token        PromptTokenData
	DeployKey    PromptDeployKeyData
	Award        PromptAwardData
	Dates        PromptDateData
	Values       FixtureOutput
}

PromptData exposes typed fixture outputs to case prompt templates.

type PromptDateData 

type PromptDateData struct {
	Today     string
	Tomorrow  string
	ExpiresAt string
}

PromptDateData contains date fixture values.

type PromptDeployKeyData 

type PromptDeployKeyData struct {
	ID    string
	Title string
}

PromptDeployKeyData contains deploy key fixture values.

type PromptEnvironmentData 

type PromptEnvironmentData struct {
	ID   string
	Name string
}

PromptEnvironmentData contains environment fixture values.

type PromptGroupData 

type PromptGroupData struct {
	ID   string
	Path string
	Name string
}

PromptGroupData contains group fixture values.

type PromptIssueData 

type PromptIssueData struct {
	ID    string
	IID   string
	Title string
}

PromptIssueData contains issue fixture values.

type PromptJobData 

type PromptJobData struct {
	ID     string
	Name   string
	Status string
}

PromptJobData contains job fixture values.

type PromptMergeRequestData 

type PromptMergeRequestData struct {
	ID           string
	IID          string
	Title        string
	SourceBranch string
	TargetBranch string
}

PromptMergeRequestData contains merge request fixture values.

type PromptPackageData 

type PromptPackageData struct {
	ID   string
	Name string
	Type string
}

PromptPackageData contains package fixture values.

type PromptPipelineData 

type PromptPipelineData struct {
	ID  string
	IID string
	Ref string
}

PromptPipelineData contains pipeline fixture values.

type PromptProjectData 

type PromptProjectData struct {
	ID            string
	Path          string
	Name          string
	URL           string
	DefaultBranch string
}

PromptProjectData contains project fixture values.

type PromptReleaseData 

type PromptReleaseData struct {
	TagName string
	Name    string
}

PromptReleaseData contains release fixture values.

type PromptRunnerData 

type PromptRunnerData struct {
	ID string
}

PromptRunnerData contains runner fixture values.

type PromptTagData 

type PromptTagData struct {
	Name string
}

PromptTagData contains tag fixture values.

type PromptTokenData 

type PromptTokenData struct {
	ID   string
	Name string
}

PromptTokenData contains token fixture values.

Source Files

View all Source files

Directories

Path Synopsis
Package cases contains the typed model-evaluation case catalog.
 Package cases contains the typed model-evaluation case catalog.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.