evals

package

Go to main page

Versions in this module

v1

v1.4.0

Mar 8, 2026

v1.3.10

Mar 8, 2026

Changes in this version

+ func FlakinessScore(results []bool) float64

+ func PassAtK(results []bool, k int) float64

+ func PassRate(results []bool) float64

+ func RegisterDefaultAlias(aliasType, targetType string)

type EvalTypeRegistry

+ func (r *EvalTypeRegistry) RegisterAlias(aliasType, targetType string) error

v1.3.9

Mar 4, 2026

v1.3.8

Mar 4, 2026

v1.3.7

Mar 3, 2026

v1.3.6

Mar 3, 2026

v1.3.5

Mar 1, 2026

v1.3.4

Mar 1, 2026

v1.3.3

Mar 1, 2026

Changes in this version

+ func ValidateEvalTypes(defs []EvalDef, registry *EvalTypeRegistry) []string

type EvalResult

+ SessionID string

+ TurnIndex int

type MetricCollectorOption

+ func WithLabels(labels map[string]string) MetricCollectorOption

type MetricDef

+ Labels map[string]string

v1.3.2

Feb 26, 2026

Changes in this version

+ func ShouldRunWhen(when *EvalWhen, toolCalls []ToolCallRecord) (shouldRun bool, reason string)

type EvalContext

+ Extras map[string]any

type EvalDef

+ Message string

+ Threshold *Threshold

+ When *EvalWhen

type EvalDispatcher

+ DispatchConversationEvals func(ctx context.Context, defs []EvalDef, evalCtx *EvalContext) ([]EvalResult, error)

type EvalResult

+ Details map[string]any

+ Message string

+ SkipReason string

+ Skipped bool

+ Violations []EvalViolation

type EvalRunner

+ func (r *EvalRunner) RunConversationEvals(ctx context.Context, defs []EvalDef, evalCtx *EvalContext) []EvalResult

type EvalTrigger

+ const TriggerOnConversationComplete

+ const TriggerOnWorkflowStep

+ type EvalViolation struct

+ Description string

+ Evidence map[string]any

+ TurnIndex int

+ type EvalWhen struct

+ AnyToolCalled bool

+ MinToolCalls int

+ ToolCalled string

+ ToolCalledPattern string

type EventDispatcher

+ func (d *EventDispatcher) DispatchConversationEvals(ctx context.Context, defs []EvalDef, evalCtx *EvalContext) ([]EvalResult, error)

type InProcDispatcher

+ func (d *InProcDispatcher) DispatchConversationEvals(ctx context.Context, defs []EvalDef, evalCtx *EvalContext) ([]EvalResult, error)

type NoOpDispatcher

+ func (d *NoOpDispatcher) DispatchConversationEvals(_ context.Context, _ []EvalDef, _ *EvalContext) ([]EvalResult, error)

+ type Threshold struct

+ MaxScore *float64

+ MinScore *float64

+ Passed *bool

+ func (t *Threshold) Apply(result *EvalResult)

v1.3.1

Feb 23, 2026

v1.2.0

Feb 15, 2026

v1.1.11

Feb 15, 2026

Changes in this version

+ const DefaultEvalTimeout

+ const DefaultSamplePercentage

+ var DefaultBuckets = []float64

+ var ValidMetricTypes = map[MetricType]bool

+ var ValidTriggers = map[EvalTrigger]bool

+ func RegisterDefault(h EvalTypeHandler)

+ func ShouldRun(trigger EvalTrigger, samplePct float64, ctx *TriggerContext) bool

+ func ValidateEvals(defs []EvalDef, scope string) []string

+ type CompositeResultWriter struct

+ func NewCompositeResultWriter(writers ...ResultWriter) *CompositeResultWriter

+ func (w *CompositeResultWriter) WriteResults(ctx context.Context, results []EvalResult) error

+ type EvalContext struct

+ CurrentOutput string

+ Messages []types.Message

+ Metadata map[string]any

+ PromptID string

+ SessionID string

+ ToolCalls []ToolCallRecord

+ TurnIndex int

+ Variables map[string]any

+ type EvalDef struct

+ Description string

+ Enabled *bool

+ ID string

+ Metric *MetricDef

+ Params map[string]any

+ SamplePercentage *float64

+ Trigger EvalTrigger

+ Type string

+ func ResolveEvals(packEvals, promptEvals []EvalDef) []EvalDef

+ func (e *EvalDef) GetSamplePercentage() float64

+ func (e *EvalDef) IsEnabled() bool

+ type EvalDispatcher interface

+ DispatchSessionEvals func(ctx context.Context, defs []EvalDef, evalCtx *EvalContext) ([]EvalResult, error)

+ DispatchTurnEvals func(ctx context.Context, defs []EvalDef, evalCtx *EvalContext) ([]EvalResult, error)

+ type EvalResult struct

+ DurationMs int64

+ Error string

+ EvalID string

+ Explanation string

+ MetricValue *float64

+ Passed bool

+ Score *float64

+ Type string

+ type EvalRunner struct

+ func NewEvalRunner(registry *EvalTypeRegistry, opts ...RunnerOption) *EvalRunner

+ func (r *EvalRunner) RunSessionEvals(ctx context.Context, defs []EvalDef, evalCtx *EvalContext) []EvalResult

+ func (r *EvalRunner) RunTurnEvals(ctx context.Context, defs []EvalDef, evalCtx *EvalContext) []EvalResult

+ type EvalTrigger string

+ const TriggerEveryTurn

+ const TriggerOnSessionComplete

+ const TriggerSampleSessions

+ const TriggerSampleTurns

+ type EvalTypeHandler interface

+ Eval func(ctx context.Context, evalCtx *EvalContext, params map[string]any) (*EvalResult, error)

+ Type func() string

+ type EvalTypeRegistry struct

+ func NewEmptyEvalTypeRegistry() *EvalTypeRegistry

+ func NewEvalTypeRegistry() *EvalTypeRegistry

+ func (r *EvalTypeRegistry) Get(evalType string) (EvalTypeHandler, error)

+ func (r *EvalTypeRegistry) Has(evalType string) bool

+ func (r *EvalTypeRegistry) Register(handler EvalTypeHandler)

+ func (r *EvalTypeRegistry) Types() []string

+ type EvalWorker struct

+ func NewEvalWorker(runner *EvalRunner, subscriber EventSubscriber, resultWriter ResultWriter, ...) *EvalWorker

+ func (w *EvalWorker) Start(ctx context.Context) error

+ type EventBusEvalListener struct

+ func NewEventBusEvalListener(bus *events.EventBus, dispatcher EvalDispatcher, evalLoader PackEvalLoader, ...) *EventBusEvalListener

+ func (l *EventBusEvalListener) Accumulator() *SessionAccumulator

+ func (l *EventBusEvalListener) Close() error

+ func (l *EventBusEvalListener) CloseSession(sessionID string)

+ func (l *EventBusEvalListener) Handle(event *events.Event)

+ type EventBusEvalListenerOption func(*EventBusEvalListener)

+ func WithTTL(ttl time.Duration) EventBusEvalListenerOption

+ type EventDispatcher struct

+ func NewEventDispatcher(publisher EventPublisher) *EventDispatcher

+ func (d *EventDispatcher) DispatchSessionEvals(ctx context.Context, defs []EvalDef, evalCtx *EvalContext) ([]EvalResult, error)

+ func (d *EventDispatcher) DispatchTurnEvals(ctx context.Context, defs []EvalDef, evalCtx *EvalContext) ([]EvalResult, error)

+ type EventPublisher interface

+ Publish func(ctx context.Context, subject string, data []byte) error

+ type EventSubscriber interface

+ Subscribe func(ctx context.Context, subject string, handler func(event []byte) error) error

+ type InProcDispatcher struct

+ func NewInProcDispatcher(runner *EvalRunner, resultWriter ResultWriter) *InProcDispatcher

+ func (d *InProcDispatcher) DispatchSessionEvals(ctx context.Context, defs []EvalDef, evalCtx *EvalContext) ([]EvalResult, error)

+ func (d *InProcDispatcher) DispatchTurnEvals(ctx context.Context, defs []EvalDef, evalCtx *EvalContext) ([]EvalResult, error)

+ type Logger interface

+ Printf func(format string, v ...any)

+ type MetadataResultWriter struct

+ func (w *MetadataResultWriter) WriteResults(_ context.Context, _ []EvalResult) error

+ type MetricCollector struct

+ func NewMetricCollector(opts ...MetricCollectorOption) *MetricCollector

+ func (mc *MetricCollector) Record(result EvalResult, metric *MetricDef) error

+ func (mc *MetricCollector) Reset()

+ func (mc *MetricCollector) WritePrometheus(w io.Writer) error

+ type MetricCollectorOption func(*MetricCollector)

+ func WithBuckets(buckets []float64) MetricCollectorOption

+ func WithNamespace(ns string) MetricCollectorOption

+ type MetricDef struct

+ Extra map[string]any

+ Name string

+ Range *Range

+ Type MetricType

+ func (m *MetricDef) UnmarshalJSON(data []byte) error

+ func (m MetricDef) MarshalJSON() ([]byte, error)

+ type MetricRecorder interface

+ Record func(result EvalResult, metric *MetricDef) error

+ type MetricResultWriter struct

+ func NewMetricResultWriter(recorder MetricRecorder, defs []EvalDef) *MetricResultWriter

+ func (w *MetricResultWriter) WriteResults(_ context.Context, results []EvalResult) error

+ type MetricType string

+ const MetricBoolean

+ const MetricCounter

+ const MetricGauge

+ const MetricHistogram

+ type NoOpDispatcher struct

+ func (d *NoOpDispatcher) DispatchSessionEvals(_ context.Context, _ []EvalDef, _ *EvalContext) ([]EvalResult, error)

+ func (d *NoOpDispatcher) DispatchTurnEvals(_ context.Context, _ []EvalDef, _ *EvalContext) ([]EvalResult, error)

+ type PackEvalLoader interface

+ LoadEvals func(promptID string) ([]EvalDef, error)

+ type Range struct

+ Max *float64

+ Min *float64

+ type ResultWriter interface

+ WriteResults func(ctx context.Context, results []EvalResult) error

+ type RunnerOption func(*EvalRunner)

+ func WithTimeout(d time.Duration) RunnerOption

+ type SessionAccumulator struct

+ func NewSessionAccumulator() *SessionAccumulator

+ func (sa *SessionAccumulator) AddMessage(sessionID, promptID, role, content string)

+ func (sa *SessionAccumulator) BuildEvalContext(sessionID string) *EvalContext

+ func (sa *SessionAccumulator) CleanupBefore(cutoff time.Time) int

+ func (sa *SessionAccumulator) PromptID(sessionID string) string

+ func (sa *SessionAccumulator) Remove(sessionID string)

+ type ToolCallRecord struct

+ Arguments map[string]any

+ Duration time.Duration

+ Error string

+ Result any

+ ToolName string

+ TurnIndex int

+ type TriggerContext struct

+ IsSessionComplete bool

+ SessionID string

+ TurnIndex int

+ type WorkerOption func(*EvalWorker)

+ func WithLogger(l Logger) WorkerOption

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL