experiment

type Evaluation struct {
	Name  string
	Value any
	Error error

	// Reasoning is a free-form explanation for the evaluation result.
	Reasoning string
	// Assessment is an optional categorical assessment ("pass" / "fail" / "").
	Assessment string
	// Metadata is arbitrary per-evaluation metadata.
	Metadata map[string]any
	// Tags are per-evaluation tags applied in addition to experiment-level tags.
	Tags map[string]string
}

type Evaluator interface {
	Name() string
	Run(ctx context.Context, rec dataset.Record, output any) (any, error)
}

type EvaluatorFunc func(ctx context.Context, rec dataset.Record, output any) (any, error)

type EvaluatorResult struct {
	// Value is the primary evaluation result. Same shape as the bare
	// EvaluatorFunc return: numeric types become score metrics, bool
	// becomes a boolean metric, anything else (string, etc.) becomes
	// categorical.
	Value any
	// Reasoning is a free-form explanation for the evaluation result
	// (e.g. an LLM judge's reasoning paragraph). Empty means unset.
	Reasoning string
	// Assessment is an optional categorical assessment. Conventional
	// values are "pass" and "fail"; empty means unset.
	Assessment string
	// Metadata is arbitrary structured metadata about this specific
	// evaluation. Surfaces as `eval_metric_metadata` on the wire.
	Metadata map[string]any
	// Tags are per-evaluation tags applied in addition to experiment-
	// level tags. Empty means unset.
	Tags map[string]string
}

type Experiment struct {
	Name string
	// contains filtered or unexported fields
}

type ExperimentResult struct {
	// ExperimentName is the name of the experiment as provided to New.
	ExperimentName string
	// DatasetName is the name of the dataset the experiment ran against.
	DatasetName string
	// Runs holds the results for each run iteration, in order. For a single-run
	// experiment this slice has exactly one element.
	Runs []*RunResult
	// Results is kept for single-run backward compatibility and points to Runs[0].Results.
	//
	// Deprecated: Use Runs[0].Results instead.
	Results []*RecordResult
	// SummaryEvaluations is kept for single-run backward compatibility and points to Runs[0].SummaryEvaluations.
	//
	// Deprecated: Use Runs[0].SummaryEvaluations instead.
	SummaryEvaluations []*Evaluation
}

type Option func(cfg *newCfg)

type RecordResult struct {
	// Record is the dataset record containing input, expected output, and metadata.
	Record *dataset.Record
	// Output is the task output for this record.
	Output any
	// Evaluations holds the evaluation results for this record.
	Evaluations []*Evaluation

	// RecordIndex is the index of the record in the dataset.
	RecordIndex int
	// SpanID is the span ID for tracing.
	SpanID string
	// TraceID is the trace ID for tracing.
	TraceID string
	// Timestamp is when the task was executed.
	Timestamp time.Time
	// Error is any error that occurred during task execution.
	Error error
}

type RunInfo struct {
	ID        string // UUID uniquely identifying this run
	Iteration int    // 1-indexed iteration number
}

type RunOption func(cfg *runCfg)

type RunResult struct {
	Run                RunInfo
	Results            []*RecordResult
	SummaryEvaluations []*Evaluation
}

type SummaryEvaluator interface {
	Name() string
	Run(ctx context.Context, results []*RecordResult) (any, error)
}

type SummaryEvaluatorFunc func(ctx context.Context, results []*RecordResult) (any, error)

type Task interface {
	Name() string
	Run(ctx context.Context, rec dataset.Record, experimentCfg map[string]any) (any, error)
}

type TaskFunc func(ctx context.Context, rec dataset.Record, experimentCfg map[string]any) (any, error)