evals

package

Go to main page

Versions in this module

v2

v2.13.0

Oct 8, 2026

v2.12.1

Oct 3, 2026

v2.12.0

Oct 3, 2026

v2.11.0

Oct 1, 2026

v2.10.0

Oct 1, 2026

v2.9.1

Sep 30, 2026

v2.9.0

Sep 30, 2026

v2.8.0

Sep 29, 2026

v2.7.0

Sep 28, 2026

v2.6.0

Sep 22, 2026

v2.5.0

Sep 18, 2026

Changes in this version

+ var ErrNoBinding = errors.New("no provider binding configured")

+ var ErrUnboundKey = errors.New("no provider bound to this key")

+ var ErrWrongKind = errors.New("provider bound to this key cannot do what the check needs")

+ func DescribeUnresolved(key string, err error) string

+ func WithProviderBinding(ctx context.Context, b ProviderBinding) context.Context

+ type ProviderBinding interface

+ Classifier func(key string) (classify.Backend, error)

+ LLM func(key string) (providers.Provider, error)

+ func BindingFromContext(ctx context.Context) ProviderBinding

v2.4.0

Sep 18, 2026

Changes in this version

+ const MetaToolsOffered

+ func ExtractToolsOffered(messages []types.Message) []string

type EvalContext

+ ToolsOffered []string

v2.3.0

Sep 16, 2026

v2.2.1

Sep 14, 2026

v2.2.0

Sep 13, 2026

Changes in this version

type AssertionEvalHandler

+ func (h *AssertionEvalHandler) ValidateParams(params map[string]any) error

type GuardrailEvalHandler

+ func (h *GuardrailEvalHandler) ValidateParams(params map[string]any) error

v2.1.0

Sep 12, 2026

v2.0.0

Sep 12, 2026

Changes in this version

+ const ContentScopeCurrent

+ const ContentScopeTranscript

+ const DefaultEvalGroup

+ const DefaultEvalTimeout

+ const DefaultExecEvalHookTimeout

+ const DefaultSamplePercentage

+ const GroupExternal

+ const GroupFastRunning

+ const GroupLongRunning

+ const MetricExpressionKey

+ const TriggerEveryTurn

+ const TriggerOnConversationComplete

+ const TriggerOnSessionComplete

+ const TriggerOnWorkflowStep

+ const TriggerSampleSessions

+ const TriggerSampleTurns

+ const WrapperTypeAssertion

+ const WrapperTypeGuardrail

+ var ParamAliases = map[string]map[string]string

+ var ParamDefaults = map[string]map[string]any

+ var ValidMetricTypes = map[MetricType]bool

+ var ValidTriggers = map[EvalTrigger]bool

+ func ApplyDefaults(evalType string, params map[string]any) map[string]any

+ func DefaultAliases() [][2]string

+ func DefaultGroupsForType(evalType string) []string

+ func EncodeEvalWhen(when *EvalWhen) map[string]any

+ func ExtractValue(result EvalResult, metric *MetricDef) (float64, bool)

+ func ExtractWorkflowExtras(messages []types.Message) map[string]any

+ func FlakinessScore(results []bool) float64

+ func Groups(e *EvalDef) []string

+ func IsEnabled(e *EvalDef) bool

+ func MetricLabels(m *MetricDef) map[string]string

+ func NormalizeParams(evalType string, params map[string]any) map[string]any

+ func PassAtK(results []bool, k int) float64

+ func PassRate(results []bool) float64

+ func RegisterDefault(h EvalTypeHandler)

+ func RegisterDefaultAlias(aliasType, targetType string)

+ func RegisterTypeGroups(evalType string, groups []string)

+ func SamplePercentage(e *EvalDef) float64

+ func SeedBudgetMetadata(metadata map[string]any, messages []types.Message, latencyMs *int64) map[string]any

+ func SetMetricLabels(m *MetricDef, labels map[string]string)

+ func ShouldRun(trigger EvalTrigger, samplePct float64, ctx *TriggerContext) bool

+ func ShouldRunWhen(raw map[string]any, toolCalls []ToolCallRecord) (shouldRun bool, reason string)

+ func StripScoreThresholds(params map[string]any) map[string]any

+ func ValidateEvalTypes(defs []EvalDef, registry *EvalTypeRegistry) []string

+ func ValidateEvalWhen(raw map[string]any) error

+ func ValidateEvals(defs []EvalDef, scope string) []string

+ type AssertionEvalHandler struct

+ func (h *AssertionEvalHandler) Eval(ctx context.Context, evalCtx *EvalContext, params map[string]any) (*EvalResult, error)

+ func (h *AssertionEvalHandler) Type() string

+ type EvalContext struct

+ ContentScope string

+ CurrentOutput string

+ Extras map[string]any

+ Messages []types.Message

+ Metadata map[string]any

+ PriorResults []EvalResult

+ PromptID string

+ SessionID string

+ ToolCalls []ToolCallRecord

+ TurnIndex int

+ Variables map[string]any

+ func BuildEvalContext(messages []types.Message, turnIndex int, sessionID string, promptID string, ...) *EvalContext

+ func BuildGuardrailEvalContext(messages []types.Message, currentOutput string, metadata map[string]any) *EvalContext

+ type EvalDef = packspec.Eval

+ func FilterByGroups(defs []EvalDef, groups []string) []EvalDef

+ func ResolveEvals(packEvals, promptEvals []EvalDef) []EvalDef

+ func Values(in []*EvalDef) []EvalDef

+ type EvalHook interface

+ Name func() string

+ OnEvalResult func(ctx context.Context, def *EvalDef, evalCtx *EvalContext, result *EvalResult)

+ type EvalResult struct

+ Details map[string]any

+ DurationMs int64

+ Error string

+ EvalID string

+ Explanation string

+ Kind events.EvalKind

+ Message string

+ MetricValue *float64

+ Passed *bool

+ Score *float64

+ SessionID string

+ SkipReason string

+ Skipped bool

+ TurnIndex int

+ Type string

+ Value any

+ Violations []EvalViolation

+ type EvalRunner struct

+ func NewEvalRunner(registry *EvalTypeRegistry, opts ...RunnerOption) *EvalRunner

+ func (r *EvalRunner) AddHook(h EvalHook)

+ func (r *EvalRunner) Clone() *EvalRunner

+ func (r *EvalRunner) RunConversationEvals(ctx context.Context, defs []EvalDef, evalCtx *EvalContext) []EvalResult

+ func (r *EvalRunner) RunSessionEvals(ctx context.Context, defs []EvalDef, evalCtx *EvalContext) []EvalResult

+ func (r *EvalRunner) RunTurnEvals(ctx context.Context, defs []EvalDef, evalCtx *EvalContext) []EvalResult

+ func (r *EvalRunner) SetEmitter(e *events.Emitter)

+ type EvalTrigger string

+ type EvalTypeHandler interface

+ Eval func(ctx context.Context, evalCtx *EvalContext, params map[string]any) (*EvalResult, error)

+ Type func() string

+ type EvalTypeRegistry struct

+ func NewEmptyEvalTypeRegistry() *EvalTypeRegistry

+ func NewEvalTypeRegistry() *EvalTypeRegistry

+ func (r *EvalTypeRegistry) Get(evalType string) (EvalTypeHandler, error)

+ func (r *EvalTypeRegistry) Has(evalType string) bool

+ func (r *EvalTypeRegistry) Register(handler EvalTypeHandler)

+ func (r *EvalTypeRegistry) RegisterAlias(aliasType, targetType string) error

+ func (r *EvalTypeRegistry) Types() []string

+ type EvalViolation struct

+ Description string

+ Evidence map[string]any

+ TurnIndex int

+ type EvalWhen struct

+ AnyToolCalled bool

+ MinToolCalls int

+ ToolCalled string

+ ToolCalledPattern string

+ func DecodeEvalWhen(raw map[string]any) *EvalWhen

+ type EvalWorker struct

+ func NewEvalWorker(runner *EvalRunner, subscriber EventSubscriber, resultWriter ResultWriter, ...) *EvalWorker

+ func (w *EvalWorker) Start(ctx context.Context) error

+ type EventSubscriber interface

+ Subscribe func(ctx context.Context, subject string, handler func(event []byte) error) error

+ type ExecEvalHook struct

+ func NewExecEvalHook(cfg *ExecEvalHookConfig) *ExecEvalHook

+ func (h *ExecEvalHook) Name() string

+ func (h *ExecEvalHook) OnEvalResult(ctx context.Context, _ *EvalDef, _ *EvalContext, result *EvalResult)

+ type ExecEvalHookConfig struct

+ Args []string

+ Command string

+ Env []string

+ Name string

+ Sandbox sandbox.Sandbox

+ TimeoutMs int

+ type GuardrailEvalHandler struct

+ func (h *GuardrailEvalHandler) Eval(ctx context.Context, evalCtx *EvalContext, params map[string]any) (*EvalResult, error)

+ func (h *GuardrailEvalHandler) Type() string

+ type Logger interface

+ Printf func(format string, v ...any)

+ type MetricDef = packspec.MetricDef

+ type MetricRecorder interface

+ Record func(result EvalResult, metric *MetricDef) error

+ type MetricResultWriter struct

+ func NewMetricResultWriter(recorder MetricRecorder, defs []EvalDef) *MetricResultWriter

+ func (w *MetricResultWriter) WriteResults(_ context.Context, results []EvalResult) error

+ type MetricType = string

+ const MetricBoolean

+ const MetricCounter

+ const MetricGauge

+ const MetricHistogram

+ type ParamValidator interface

+ ValidateParams func(params map[string]any) error

+ type Range = packspec.MetricDefRange

+ type ResultWriter interface

+ WriteResults func(ctx context.Context, results []EvalResult) error

+ type RunnerOption func(*EvalRunner)

+ func WithEmitter(e *events.Emitter) RunnerOption

+ func WithEvalHook(h EvalHook) RunnerOption

+ func WithTimeout(d time.Duration) RunnerOption

+ type ScoreThresholds struct

+ Max *float64

+ Min *float64

+ func ExtractScoreThresholds(params map[string]any) ScoreThresholds

+ func (t ScoreThresholds) Triggered(result *EvalResult) bool

+ type StreamableEvalHandler interface

+ EvalPartial func(ctx context.Context, content string, params map[string]any) (*EvalResult, error)

+ type Threshold = packspec.EvalThreshold

+ type ToolCallRecord = types.ToolCallRecord

+ func ExtractToolCalls(messages []types.Message) []ToolCallRecord

+ type TriggerContext struct

+ IsSessionComplete bool

+ SessionID string

+ TurnIndex int

+ type WorkerOption func(*EvalWorker)

+ func WithLogger(l Logger) WorkerOption

Other modules containing this package

github.com/AltairaLabs/PromptKit/runtime

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL