handlers

type ContainsAnyHandler ¶

type ContainsAnyHandler struct{}

ContainsAnyHandler checks that at least one assistant message contains at least one of the specified patterns. Params: patterns []string (case-insensitive matching).

func (h *ContainsAnyHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (_ *evals.EvalResult, _ error)

func (h *ContainsAnyHandler) Type() string

type ContainsHandler ¶

type ContainsHandler struct{}

ContainsHandler checks if CurrentOutput contains all specified patterns (case-insensitive). Params: patterns []string.

func (h *ContainsHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *ContainsHandler) Type() string

type ContentExcludesHandler ¶

type ContentExcludesHandler struct{}

ContentExcludesHandler checks that NONE of the assistant messages across the full conversation contain any of the forbidden patterns. Params: patterns []string (case-insensitive matching).

func (h *ContentExcludesHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (_ *evals.EvalResult, _ error)

func (h *ContentExcludesHandler) Type() string

type CosineSimilarityHandler ¶

type CosineSimilarityHandler struct{}

CosineSimilarityHandler computes cosine similarity between embeddings. Params: reference []float64, min_similarity float64. Target embedding comes from Metadata["embedding"].

func (h *CosineSimilarityHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *CosineSimilarityHandler) Type() string

type JSONSchemaHandler ¶

type JSONSchemaHandler struct{}

JSONSchemaHandler validates CurrentOutput against a JSON schema. Params: schema map[string]any.

func (h *JSONSchemaHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *JSONSchemaHandler) Type() string

type JSONValidHandler ¶

type JSONValidHandler struct{}

JSONValidHandler checks if CurrentOutput is valid JSON. No required params.

func (h *JSONValidHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	_ map[string]any,
) (result *evals.EvalResult, err error)

func (h *JSONValidHandler) Type() string

type JudgeOpts ¶

type JudgeOpts struct {
	// Content is the text being evaluated (assistant response or full conversation).
	Content string

	// Criteria describes what the judge should evaluate (e.g. "Is the response helpful?").
	Criteria string

	// Rubric provides detailed scoring guidance (optional).
	Rubric string

	// Model specifies which model to use for judging (optional, provider decides default).
	Model string

	// SystemPrompt overrides the default judge system prompt (optional).
	SystemPrompt string

	// MinScore is the minimum score threshold for passing (optional).
	MinScore *float64

	// Extra holds additional parameters for provider-specific features.
	Extra map[string]any
}

JudgeOpts configures a judge evaluation request.

type JudgeProvider ¶

type JudgeProvider interface {
	// Judge sends the evaluation prompt to an LLM and returns
	// the parsed verdict. Implementations handle provider selection,
	// prompt formatting, and response parsing.
	Judge(ctx context.Context, opts JudgeOpts) (*JudgeResult, error)
}

JudgeProvider abstracts LLM access for judge-based evaluations. Arena, SDK, and eval workers each provide their own implementation wiring their respective provider infrastructure.

type JudgeResult ¶

type JudgeResult struct {
	// Passed indicates whether the content met the evaluation criteria.
	Passed bool

	// Score is the numerical score assigned by the judge (typically 0.0-1.0).
	Score float64

	// Reasoning explains the judge's evaluation.
	Reasoning string

	// Raw is the unprocessed LLM response text.
	Raw string
}

JudgeResult captures the output of an LLM judge evaluation.

type LLMJudgeHandler ¶

type LLMJudgeHandler struct{}

LLMJudgeHandler evaluates a single assistant turn using an LLM judge. The JudgeProvider must be supplied in evalCtx.Metadata["judge_provider"].

Params:

• criteria (string, required): what to evaluate
• rubric (string, optional): detailed scoring guidance
• model (string, optional): model override for the judge
• system_prompt (string, optional): override default system prompt
• min_score (float64, optional): minimum score to pass

func (h *LLMJudgeHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *LLMJudgeHandler) Type() string

type LLMJudgeSessionHandler ¶

type LLMJudgeSessionHandler struct{}

LLMJudgeSessionHandler evaluates an entire conversation using an LLM judge. It concatenates all assistant messages into a single content string for evaluation.

The JudgeProvider must be supplied in evalCtx.Metadata["judge_provider"].

Params:

• criteria (string, required): what to evaluate
• rubric (string, optional): detailed scoring guidance
• model (string, optional): model override for the judge
• system_prompt (string, optional): override default system prompt
• min_score (float64, optional): minimum score to pass

func (h *LLMJudgeSessionHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *LLMJudgeSessionHandler) Type() string

type LatencyBudgetHandler ¶

type LatencyBudgetHandler struct{}

LatencyBudgetHandler checks Metadata["latency_ms"] against a max. Params: max_ms float64.

func (h *LatencyBudgetHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *LatencyBudgetHandler) Type() string

type RegexHandler ¶

type RegexHandler struct{}

RegexHandler checks if CurrentOutput matches a regex pattern. Params: pattern string.

func (h *RegexHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *RegexHandler) Type() string

type ToolArgsExcludedSessionHandler ¶

type ToolArgsExcludedSessionHandler struct{}

ToolArgsExcludedSessionHandler checks that a tool was NOT called with specific argument values across the session. Params: tool_name string, excluded_args map[string]any.

func (h *ToolArgsExcludedSessionHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (_ *evals.EvalResult, _ error)

func (h *ToolArgsExcludedSessionHandler) Type() string

type ToolArgsHandler ¶

type ToolArgsHandler struct{}

ToolArgsHandler checks that a tool was called with specific args. Params: tool_name string, expected_args map[string]any.

func (h *ToolArgsHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *ToolArgsHandler) Type() string

type ToolArgsSessionHandler ¶

type ToolArgsSessionHandler struct{}

ToolArgsSessionHandler checks that a tool was called with specific arguments across the session. Params: tool_name string, expected_args map[string]any.

func (h *ToolArgsSessionHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (_ *evals.EvalResult, _ error)

func (h *ToolArgsSessionHandler) Type() string

type ToolsCalledHandler ¶

type ToolsCalledHandler struct{}

ToolsCalledHandler checks if specific tools were called. Params: tool_names []string, optional min_calls int.

func (h *ToolsCalledHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *ToolsCalledHandler) Type() string

type ToolsCalledSessionHandler ¶

type ToolsCalledSessionHandler struct{}

ToolsCalledSessionHandler checks that specific tools were called across the full session. Params: tool_names []string, min_calls int (optional, default 1).

func (h *ToolsCalledSessionHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (_ *evals.EvalResult, _ error)

func (h *ToolsCalledSessionHandler) Type() string

type ToolsNotCalledHandler ¶

type ToolsNotCalledHandler struct{}

ToolsNotCalledHandler checks that specific tools were NOT called. Params: tool_names []string.

func (h *ToolsNotCalledHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *ToolsNotCalledHandler) Type() string

type ToolsNotCalledSessionHandler ¶

type ToolsNotCalledSessionHandler struct{}

ToolsNotCalledSessionHandler checks that specific tools were NOT called anywhere in the session. Params: tool_names []string.

func (h *ToolsNotCalledSessionHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (_ *evals.EvalResult, _ error)

func (h *ToolsNotCalledSessionHandler) Type() string