eval

type Case ¶

type Case[T any] struct {
	ID       CaseID
	Subject  T
	Metadata metadata.Map
}

Case gives a stable identity to one evaluation subject.

func (c Case[T]) Validate() error

type CaseID ¶

type CaseID string

CaseID is a stable identity within one Dataset.

func (c CaseID) String() string

func (c CaseID) Validate() error

type CaseResult ¶

type CaseResult struct {
	ID       CaseID
	Metadata metadata.Map
	Report   Report
	Err      error
}

type Comparison ¶

type Comparison struct {
	Baseline       ExperimentSummary
	Candidate      ExperimentSummary
	EvaluatedDelta int
	PassedDelta    int
	FailedDelta    int
	UnjudgedDelta  int
	ErrorDelta     int
	Metrics        []MetricComparison
}

type Component ¶

type Component[T any] struct {
	Evaluator Evaluator[T]
	Weight    float64
	Required  bool
}

Component assigns score weight and pass criticality to one evaluator. A zero Weight selects 1. Required components must pass independently of the aggregate pass policy.

type CompositeConfig ¶

type CompositeConfig[T any] struct {
	Components     []Component[T]
	PassPolicy     PassPolicy
	MinimumPassed  int
	MaxConcurrency int
}

CompositeConfig defines score aggregation, pass semantics, and bounded concurrency. A zero MaxConcurrency selects DefaultMaxConcurrency.

type CompositeEvaluator ¶

type CompositeEvaluator[T any] struct {
	// contains filtered or unexported fields
}

func NewCompositeEvaluator[T any](config CompositeConfig[T]) (*CompositeEvaluator[T], error)

func (c *CompositeEvaluator[T]) Evaluate(ctx context.Context, subject T) (Report, error)

type Dataset ¶

type Dataset[T any] struct {
	// contains filtered or unexported fields
}

Dataset is an immutable ordered set of uniquely identified cases. Evaluators must not mutate subjects; metadata is owned and cloned by the Dataset.

func NewDataset[T any](cases ...Case[T]) (Dataset[T], error)

func (d Dataset[T]) Cases() []Case[T]

func (d Dataset[T]) Len() int

type Direction ¶

type Direction string

Direction describes how a raw measurement relates to quality. It is kept separate from Score, whose direction is always higher-is-better.

const (
	DirectionUnspecified    Direction = ""
	DirectionHigherIsBetter Direction = "higher_is_better"
	DirectionLowerIsBetter  Direction = "lower_is_better"
)

func (d Direction) Validate() error

type Distribution ¶

type Distribution struct {
	Count   int
	Mean    float64
	Minimum float64
	P10     float64
	P50     float64
	P90     float64
	Maximum float64
}

Distribution summarizes one homogeneous numeric signal. Count distinguishes an absent distribution from a real distribution whose values are all zero.

type DistributionDelta ¶

type DistributionDelta struct {
	Present bool
	Mean    float64
}

DistributionDelta is candidate mean minus baseline mean. Present is false when either side has no values, so absence cannot be mistaken for zero.

type ErrorPolicy ¶

type ErrorPolicy string

const (
	ErrorCollect  ErrorPolicy = "collect"
	ErrorFailFast ErrorPolicy = "fail_fast"
)

type Evaluator ¶

type Evaluator[T any] interface {
	// Evaluate inspects one subject without mutating it and returns a valid,
	// owned report for the evaluator's metric. Implementations must honor ctx;
	// a non-nil error means the report must not be consumed.
	Evaluate(ctx context.Context, subject T) (Report, error)
}

Evaluator evaluates one subject and returns a valid report.

type EvaluatorFunc ¶

type EvaluatorFunc[T any] func(context.Context, T) (Report, error)

func (e EvaluatorFunc[T]) Evaluate(ctx context.Context, subject T) (Report, error)

type Experiment ¶

type Experiment[T any] struct {
	// contains filtered or unexported fields
}

Experiment is an immutable plan for evaluating one Dataset. It owns bounded scheduling and error semantics, but no persistence, artifacts, or product identity.

func NewExperiment[T any](config ExperimentConfig[T]) (Experiment[T], error)

func (e Experiment[T]) Run(ctx context.Context) (ExperimentReport, error)

package main

import (
	"context"
	"fmt"

	"github.com/Tangerg/scope/eval"
)

func main() {
	metric, err := eval.NewMetric(eval.MetricConfig{
		Namespace: "example",
		Name:      "non_empty",
	})
	if err != nil {
		panic(err)
	}
	evaluator := eval.EvaluatorFunc[string](func(_ context.Context, subject string) (eval.Report, error) {
		return eval.Report{Metric: metric, Verdict: eval.VerdictPass}, nil
	})
	dataset, err := eval.NewDataset(
		eval.Case[string]{ID: "first", Subject: "answer"},
	)
	if err != nil {
		panic(err)
	}
	experiment, err := eval.NewExperiment(eval.ExperimentConfig[string]{
		Dataset: dataset, Evaluator: evaluator,
	})
	if err != nil {
		panic(err)
	}
	report, err := experiment.Run(context.Background())
	if err != nil {
		panic(err)
	}
	summary := report.Summary()

	fmt.Println(summary.Total, summary.Passed)
}

Output:
1 1

type ExperimentConfig ¶

type ExperimentConfig[T any] struct {
	Dataset        Dataset[T]
	Evaluator      Evaluator[T]
	MaxConcurrency int
	ErrorPolicy    ErrorPolicy
}

type ExperimentReport ¶

type ExperimentReport struct {
	// contains filtered or unexported fields
}

func (e ExperimentReport) Cases() []CaseResult

func (e ExperimentReport) Compare(candidate ExperimentReport) (Comparison, error)

func (e ExperimentReport) Summary() ExperimentSummary

type ExperimentSummary ¶

type ExperimentSummary struct {
	Total     int
	Evaluated int
	Passed    int
	Failed    int
	Unjudged  int
	Errors    int
	Metrics   []MetricSummary
}

type Metric ¶

type Metric struct {
	Namespace  string       `json:"namespace,omitzero"`
	Name       MetricName   `json:"name"`
	Unit       string       `json:"unit,omitzero"`
	Direction  Direction    `json:"direction,omitzero"`
	Parameters metadata.Map `json:"parameters,omitzero"`
}

Metric identifies an evaluation without encoding configuration into a string. Parameters holds owned, structured identity for calculation and decision rules. Unit and Direction describe optional raw measurements; normalized scores are always unitless and higher-is-better.

func NewMetric(config MetricConfig) (Metric, error)

func (m Metric) Clone() Metric

func (m Metric) String() string

func (m Metric) Validate() error

type MetricComparison ¶

type MetricComparison struct {
	Metric           Metric
	Baseline         MetricSummary
	Candidate        MetricSummary
	EvaluatedDelta   int
	PassedDelta      int
	FailedDelta      int
	UnjudgedDelta    int
	ScoreDelta       DistributionDelta
	MeasurementDelta DistributionDelta
}

type MetricConfig ¶

type MetricConfig struct {
	Namespace  string
	Name       MetricName
	Unit       string
	Direction  Direction
	Parameters metadata.Map
}

type MetricName ¶

type MetricName string

MetricName identifies one quality calculation within a namespace.

const MetricNameComposite MetricName = "composite"

const MetricNameSuite MetricName = "suite"

type MetricSummary ¶

type MetricSummary struct {
	Metric       Metric
	Evaluated    int
	Passed       int
	Failed       int
	Unjudged     int
	Scores       Distribution
	Measurements Distribution
}

MetricSummary keeps score and measurement distributions attached to their full Metric identity so unrelated units, directions, and configurations are never aggregated together. Experiment summarizes both top-level reports and their Details.

type PassPolicy ¶

type PassPolicy string

const (
	PassAll     PassPolicy = "all"
	PassAny     PassPolicy = "any"
	PassAtLeast PassPolicy = "at_least"
)

type Projection ¶

type Projection[T, Subject any] func(T) (Subject, error)

type ProjectionEvaluator ¶

type ProjectionEvaluator[T, Subject any] struct {
	// contains filtered or unexported fields
}

ProjectionEvaluator adapts one aggregate case to the narrower subject a domain evaluator consumes.

func NewProjectionEvaluator[T, Subject any](
	evaluator Evaluator[Subject],
	projection Projection[T, Subject],
) (*ProjectionEvaluator[T, Subject], error)

func (p *ProjectionEvaluator[T, Subject]) Evaluate(ctx context.Context, value T) (Report, error)

type Report ¶

type Report struct {
	Metric      Metric       `json:"metric"`
	Verdict     Verdict      `json:"verdict,omitzero"`
	Score       *Score       `json:"score,omitzero"`
	Measurement *float64     `json:"measurement,omitzero"`
	Feedback    string       `json:"feedback,omitzero"`
	Metadata    metadata.Map `json:"metadata,omitzero"`
	Details     []Report     `json:"details,omitzero"`
}

Report is one evaluation result. Verdict, normalized Score, and raw Measurement are independent and optional so measurement-only and qualitative evaluations do not need to invent a pass threshold or quality score. Details contains owned child reports instead of convention-based metadata keys.

func (r Report) Clone() (Report, error)

func (r Report) MarshalJSON() ([]byte, error)

func (r *Report) UnmarshalJSON(data []byte) error

func (r Report) Validate() error

type Score ¶

type Score float64

Score is a normalized quality score in the closed interval [0, 1], where a higher value is always better.

func NewScore(value float64) (Score, error)

func (s Score) Float64() float64

func (s Score) Validate() error

func (s Score) Verdict(threshold Score) (Verdict, error)

type SuiteConfig ¶

type SuiteConfig[T any] struct {
	Evaluators     []Evaluator[T]
	MaxConcurrency int
}

SuiteConfig groups heterogeneous evaluators without collapsing their results into one score. A zero MaxConcurrency selects DefaultMaxConcurrency.

type SuiteEvaluator ¶

type SuiteEvaluator[T any] struct {
	// contains filtered or unexported fields
}

SuiteEvaluator preserves every child report and records the ordered child metrics in its own identity. Its verdict fails when any decided child fails, passes when at least one child passes and none fail, and remains unspecified when every child is measurement-only or qualitative.

func NewSuiteEvaluator[T any](config SuiteConfig[T]) (*SuiteEvaluator[T], error)

func (s *SuiteEvaluator[T]) Evaluate(ctx context.Context, subject T) (Report, error)

type Verdict ¶

type Verdict string

Verdict is an optional categorical judgment. An unspecified verdict is a valid outcome for measurement-only or qualitative evaluations.

const (
	VerdictUnspecified Verdict = ""
	VerdictPass        Verdict = "pass"
	VerdictFail        Verdict = "fail"
)

func (v Verdict) Decided() bool

func (v Verdict) Validate() error