task

type Context struct {
	// Language holds the language for which the task should be evaluated.
	Language language.Language
	// Repository holds the repository which should be evaluated.
	Repository Repository
	// Model holds the model which the task should be evaluated.
	Model model.Model

	// ResultPath holds the directory path where results should be written to.
	ResultPath string

	// Logger holds the logger for this tasks.
	Logger *log.Logger
}

type Identifier string

type Repository interface {
	// Name holds the name of the repository.
	Name() (name string)
	// DataPath holds the absolute path to the repository.
	DataPath() (dataPath string)

	// Configuration returns the configuration of a repository.
	Configuration() *RepositoryConfiguration

	// Validate checks it the repository is well-formed.
	Validate(logger *log.Logger, language language.Language) (err error)

	// Reset resets the repository to its initial state.
	Reset(logger *log.Logger) (err error)
}

type RepositoryConfiguration struct {
	// Tasks holds the tasks supported by the repository.
	Tasks []Identifier `json:"tasks"`
	// IgnorePaths holds the relative paths that should be ignored when searching for cases.
	IgnorePaths []string `json:"ignore,omitempty"`

	// Prompt holds LLM prompt-related configuration.
	Prompt RepositoryConfigurationPrompt `json:"prompt,omitempty"`

	// Validation holds quality gates for evaluation.
	Validation RepositoryConfigurationValidation `json:"validation,omitempty"`

	// MaxScores holds the maximum scores per task type, case and metric for this repository.
	MaxScores map[Identifier]map[string]map[metrics.AssessmentKey]uint64 `json:"scores,omitempty"`
}

type RepositoryConfigurationExecution struct {
	// StdOutRE holds a regular expression that must be part of execution standard output.
	StdOutRE string `json:"stdout,omitempty"`
}

type RepositoryConfigurationPrompt struct {
	// TestFramework overwrites the language-specific test framework to use.
	TestFramework string `json:"test-framework,omitempty"`
}

type RepositoryConfigurationValidation struct {
	// Execution holds execution-related validation.
	Execution RepositoryConfigurationExecution `json:"execution,omitempty"`
}

type Task interface {
	// Identifier returns the task identifier.
	Identifier() (identifier Identifier)

	// Run runs a task in a given repository.
	Run(ctx Context) (assessments map[string]map[Identifier]metrics.Assessments, problems []error, err error)
}