handlers

type A2AEvalHandler ¶ added in v1.3.3

type A2AEvalHandler struct{}

A2AEvalHandler evaluates a single assistant turn by sending conversation context to an A2A agent and interpreting the agent's response as a structured eval result.

Params:

• agent_url (string, required): A2A agent endpoint URL
• auth_token (string, optional): auth token, supports ${ENV_VAR}
• timeout (string, optional): request timeout, default 60s
• criteria (string, optional): evaluation criteria
• include_messages (bool, optional): include conversation history, default true
• include_tool_calls (bool, optional): include tool call records, default false
• min_score (float64, optional): minimum score threshold
• extra (map[string]any, optional): arbitrary data forwarded in request

func (h *A2AEvalHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *A2AEvalHandler) Type() string

type A2AEvalSessionHandler ¶ added in v1.3.3

type A2AEvalSessionHandler struct{}

A2AEvalSessionHandler evaluates an entire conversation by sending all assistant messages to an A2A agent.

Params: same as A2AEvalHandler.

func (h *A2AEvalSessionHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *A2AEvalSessionHandler) Type() string

type AgentInvokedHandler ¶ added in v1.3.2

type AgentInvokedHandler struct{}

AgentInvokedHandler checks that expected agents were invoked as tool calls. Params: agents []string — list of agent names that should have been called.

func (h *AgentInvokedHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *AgentInvokedHandler) Type() string

type AgentNotInvokedHandler ¶ added in v1.3.2

type AgentNotInvokedHandler struct{}

AgentNotInvokedHandler checks that forbidden agents were NOT called. Params: agents []string — agent names that should not have been called.

func (h *AgentNotInvokedHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *AgentNotInvokedHandler) Type() string

type AgentResponseContainsHandler ¶ added in v1.3.2

type AgentResponseContainsHandler struct{}

AgentResponseContainsHandler checks that a specific agent's response contains expected text. Agent responses appear as tool-result messages where the tool name matches the agent name. Params: agent string, contains string.

func (h *AgentResponseContainsHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *AgentResponseContainsHandler) Type() string

type AudioDurationHandler ¶ added in v1.3.2

type AudioDurationHandler struct{}

AudioDurationHandler checks that audio duration is within range. Params: min_seconds float64, max_seconds float64.

func (h *AudioDurationHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *AudioDurationHandler) Type() string

type AudioFormatHandler ¶ added in v1.3.2

type AudioFormatHandler struct{}

AudioFormatHandler checks that audio content has allowed formats. Params: formats []string.

func (h *AudioFormatHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *AudioFormatHandler) Type() string

type ContainsAnyHandler ¶

type ContainsAnyHandler struct{}

ContainsAnyHandler checks that at least one assistant message contains at least one of the specified patterns. Params: patterns []string (case-insensitive matching).

func (h *ContainsAnyHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (_ *evals.EvalResult, _ error)

func (h *ContainsAnyHandler) Type() string

type ContainsHandler ¶

type ContainsHandler struct{}

ContainsHandler checks if CurrentOutput contains all specified patterns (case-insensitive). Params: patterns []string.

func (h *ContainsHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *ContainsHandler) Type() string

type ContentExcludesHandler ¶

type ContentExcludesHandler struct{}

ContentExcludesHandler checks that NONE of the assistant messages across the full conversation contain any of the forbidden patterns. Params: patterns []string (case-insensitive matching).

func (h *ContentExcludesHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (_ *evals.EvalResult, _ error)

func (h *ContentExcludesHandler) Type() string

type CosineSimilarityHandler ¶

type CosineSimilarityHandler struct{}

CosineSimilarityHandler computes cosine similarity between embeddings. Params: reference []float64, min_similarity float64. Target embedding comes from Metadata["embedding"].

func (h *CosineSimilarityHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *CosineSimilarityHandler) Type() string

type ExternalEvalRequest ¶ added in v1.3.3

type ExternalEvalRequest struct {
	CurrentOutput string         `json:"current_output"`
	Messages      []messageView  `json:"messages,omitempty"`
	ToolCalls     []toolCallView `json:"tool_calls,omitempty"`
	Criteria      string         `json:"criteria,omitempty"`
	Variables     map[string]any `json:"variables,omitempty"`
	Extra         map[string]any `json:"extra,omitempty"`
}

ExternalEvalRequest is the standard request body sent to external eval endpoints (REST) and formatted as context for A2A eval agents.

type GuardrailTriggeredHandler ¶ added in v1.3.2

type GuardrailTriggeredHandler struct{}

GuardrailTriggeredHandler checks if a specific guardrail validator triggered (or didn't trigger) as expected. Params: validator_type string, should_trigger bool (default true).

func (h *GuardrailTriggeredHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *GuardrailTriggeredHandler) Type() string

type ImageDimensionsHandler ¶ added in v1.3.2

type ImageDimensionsHandler struct{}

ImageDimensionsHandler checks that images meet dimension requirements. Params: min_width, max_width, min_height, max_height, width, height.

func (h *ImageDimensionsHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *ImageDimensionsHandler) Type() string

type ImageFormatHandler ¶ added in v1.3.2

type ImageFormatHandler struct{}

ImageFormatHandler checks that images in assistant messages have allowed formats. Params: formats []string.

func (h *ImageFormatHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *ImageFormatHandler) Type() string

type JSONPathHandler ¶ added in v1.3.2

type JSONPathHandler struct{}

JSONPathHandler validates assistant output as JSON using JMESPath expressions. Params:

• expression string (JMESPath expression)
• expected any (optional: exact match)
• contains []any (optional: array contains check)
• min_results int, max_results int (optional: array length bounds)
• min float64, max float64 (optional: numeric range)
• allow_wrapped bool (optional: extract JSON from code blocks)
• extract_json bool (optional: extract JSON from mixed text)

func (h *JSONPathHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *JSONPathHandler) Type() string

type JSONSchemaHandler ¶

type JSONSchemaHandler struct{}

JSONSchemaHandler validates CurrentOutput against a JSON schema. Params: schema map[string]any.

func (h *JSONSchemaHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *JSONSchemaHandler) Type() string

type JSONValidHandler ¶

type JSONValidHandler struct{}

JSONValidHandler checks if CurrentOutput is valid JSON. No required params.

func (h *JSONValidHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	_ map[string]any,
) (result *evals.EvalResult, err error)

func (h *JSONValidHandler) Type() string

type JudgeOpts ¶

type JudgeOpts struct {
	// Content is the text being evaluated (assistant response or full conversation).
	Content string

	// Criteria describes what the judge should evaluate (e.g. "Is the response helpful?").
	Criteria string

	// Rubric provides detailed scoring guidance (optional).
	Rubric string

	// Model specifies which model to use for judging (optional, provider decides default).
	Model string

	// SystemPrompt overrides the default judge system prompt (optional).
	SystemPrompt string

	// MinScore is the minimum score threshold for passing (optional).
	MinScore *float64

	// Extra holds additional parameters for provider-specific features.
	Extra map[string]any
}

JudgeOpts configures a judge evaluation request.

type JudgeProvider ¶

type JudgeProvider interface {
	// Judge sends the evaluation prompt to an LLM and returns
	// the parsed verdict. Implementations handle provider selection,
	// prompt formatting, and response parsing.
	Judge(ctx context.Context, opts JudgeOpts) (*JudgeResult, error)
}

JudgeProvider abstracts LLM access for judge-based evaluations. Arena, SDK, and eval workers each provide their own implementation wiring their respective provider infrastructure.

type JudgeResult ¶

type JudgeResult struct {
	// Passed indicates whether the content met the evaluation criteria.
	Passed bool

	// Score is the numerical score assigned by the judge (typically 0.0-1.0).
	Score float64

	// Reasoning explains the judge's evaluation.
	Reasoning string

	// Raw is the unprocessed LLM response text.
	Raw string
}

JudgeResult captures the output of an LLM judge evaluation.

type LLMJudgeHandler ¶

type LLMJudgeHandler struct{}

LLMJudgeHandler evaluates a single assistant turn using an LLM judge. The JudgeProvider must be supplied in evalCtx.Metadata["judge_provider"].

Params:

• criteria (string, required): what to evaluate
• rubric (string, optional): detailed scoring guidance
• model (string, optional): model override for the judge
• system_prompt (string, optional): override default system prompt
• min_score (float64, optional): minimum score to pass

func (h *LLMJudgeHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *LLMJudgeHandler) Type() string

type LLMJudgeSessionHandler ¶

type LLMJudgeSessionHandler struct{}

LLMJudgeSessionHandler evaluates an entire conversation using an LLM judge. It concatenates all assistant messages into a single content string for evaluation.

The JudgeProvider must be supplied in evalCtx.Metadata["judge_provider"].

Params:

• criteria (string, required): what to evaluate
• rubric (string, optional): detailed scoring guidance
• model (string, optional): model override for the judge
• system_prompt (string, optional): override default system prompt
• min_score (float64, optional): minimum score to pass

func (h *LLMJudgeSessionHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *LLMJudgeSessionHandler) Type() string

type LLMJudgeToolCallsHandler ¶ added in v1.3.2

type LLMJudgeToolCallsHandler struct{}

LLMJudgeToolCallsHandler evaluates tool call behavior via an LLM judge. Instead of judging the assistant's text response, it feeds tool call data (names, arguments, results) to the judge for evaluation. Params:

• tools []string (optional): filter to specific tool names
• criteria string: what to evaluate
• rubric string (optional): detailed scoring guidance
• model string (optional): model override
• min_score float64 (optional): minimum score to pass

func (h *LLMJudgeToolCallsHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *LLMJudgeToolCallsHandler) Type() string

type LatencyBudgetHandler ¶

type LatencyBudgetHandler struct{}

LatencyBudgetHandler checks Metadata["latency_ms"] against a max. Params: max_ms float64.

func (h *LatencyBudgetHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *LatencyBudgetHandler) Type() string

type NoToolErrorsHandler ¶ added in v1.3.2

type NoToolErrorsHandler struct{}

NoToolErrorsHandler checks that no tool calls returned errors. Params: tools []string (optional) — if set, only checks calls matching those tool names.

func (h *NoToolErrorsHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *NoToolErrorsHandler) Type() string

type RegexHandler ¶

type RegexHandler struct{}

RegexHandler checks if CurrentOutput matches a regex pattern. Params: pattern string.

func (h *RegexHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *RegexHandler) Type() string

type RestEvalHandler ¶ added in v1.3.3

type RestEvalHandler struct{}

RestEvalHandler evaluates a single assistant turn by POSTing conversation context to an external HTTP endpoint and interpreting the structured JSON response.

Params:

• url (string, required): endpoint URL
• method (string, optional): HTTP method, default POST
• headers (map[string]string, optional): request headers, supports ${ENV_VAR}
• timeout (string, optional): request timeout, default 30s
• include_messages (bool, optional): include conversation history, default true
• include_tool_calls (bool, optional): include tool call records, default false
• criteria (string, optional): evaluation criteria forwarded in request
• min_score (float64, optional): minimum score threshold
• extra (map[string]any, optional): arbitrary data forwarded in request

func (h *RestEvalHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *RestEvalHandler) Type() string

type RestEvalSessionHandler ¶ added in v1.3.3

type RestEvalSessionHandler struct{}

RestEvalSessionHandler evaluates an entire conversation by POSTing all assistant messages to an external HTTP endpoint.

Params: same as RestEvalHandler.

func (h *RestEvalSessionHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *RestEvalSessionHandler) Type() string

type SkillActivatedHandler ¶ added in v1.3.2

type SkillActivatedHandler struct{}

SkillActivatedHandler checks that specific skills were activated. Scans evalCtx.ToolCalls for "skill__activate" calls and extracts the "name" argument. Params: skill_names []string, min_calls int (optional, default 1).

func (h *SkillActivatedHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *SkillActivatedHandler) Type() string

type SkillNotActivatedHandler ¶ added in v1.3.2

type SkillNotActivatedHandler struct{}

SkillNotActivatedHandler checks that specific skills were NOT activated. Params: skill_names []string.

func (h *SkillNotActivatedHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *SkillNotActivatedHandler) Type() string

type SpecJudgeProvider ¶ added in v1.3.2

type SpecJudgeProvider struct {
	// contains filtered or unexported fields
}

SpecJudgeProvider implements JudgeProvider by creating a provider from a ProviderSpec. This is the standard implementation used by Arena and any caller that has judge targets as ProviderSpecs.

func NewSpecJudgeProvider(spec *providers.ProviderSpec) *SpecJudgeProvider

func (sp *SpecJudgeProvider) Judge(ctx context.Context, opts JudgeOpts) (*JudgeResult, error)

type ToolArgsExcludedSessionHandler ¶

type ToolArgsExcludedSessionHandler struct{}

ToolArgsExcludedSessionHandler checks that a tool was NOT called with specific argument values across the session. Params: tool_name string, excluded_args map[string]any.

func (h *ToolArgsExcludedSessionHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (_ *evals.EvalResult, _ error)

func (h *ToolArgsExcludedSessionHandler) Type() string

type ToolArgsHandler ¶

type ToolArgsHandler struct{}

ToolArgsHandler checks that a tool was called with specific args. Params: tool_name string, expected_args map[string]any.

func (h *ToolArgsHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *ToolArgsHandler) Type() string

type ToolArgsSessionHandler ¶

type ToolArgsSessionHandler struct{}

ToolArgsSessionHandler checks that a tool was called with specific arguments across the session. Params: tool_name string, expected_args map[string]any.

func (h *ToolArgsSessionHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (_ *evals.EvalResult, _ error)

func (h *ToolArgsSessionHandler) Type() string

type ToolCallChainHandler ¶ added in v1.3.2

type ToolCallChainHandler struct{}

ToolCallChainHandler checks a dependency chain of tool calls with per-step constraints. Params: steps []map — each with tool, result_includes, result_matches, args_match, no_error.

func (h *ToolCallChainHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *ToolCallChainHandler) Type() string

type ToolCallCountHandler ¶ added in v1.3.2

type ToolCallCountHandler struct{}

ToolCallCountHandler checks the count of tool calls within bounds. Params: tool string (optional), min int (optional), max int (optional).

func (h *ToolCallCountHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *ToolCallCountHandler) Type() string

type ToolCallSequenceHandler ¶ added in v1.3.2

type ToolCallSequenceHandler struct{}

ToolCallSequenceHandler checks that tool calls appear in a specified subsequence order. Params: sequence []string — the expected tool names in order.

func (h *ToolCallSequenceHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *ToolCallSequenceHandler) Type() string

type ToolCallsWithArgsHandler ¶ added in v1.3.2

type ToolCallsWithArgsHandler struct{}

ToolCallsWithArgsHandler checks that a tool was called with expected arguments. Supports exact value matching (expected_args), regex pattern matching (args_match), and result-level constraints (result_includes, result_matches, no_error).

func (h *ToolCallsWithArgsHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *ToolCallsWithArgsHandler) Type() string

type ToolResultIncludesHandler ¶ added in v1.3.2

type ToolResultIncludesHandler struct{}

ToolResultIncludesHandler checks that tool results contain expected substrings. Params: tool string, patterns []string, occurrence int (optional, default 1).

func (h *ToolResultIncludesHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *ToolResultIncludesHandler) Type() string

type ToolResultMatchesHandler ¶ added in v1.3.2

type ToolResultMatchesHandler struct{}

ToolResultMatchesHandler checks that tool results match a regex pattern. Params: tool string, pattern string, occurrence int (optional, default 1).

func (h *ToolResultMatchesHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *ToolResultMatchesHandler) Type() string

type ToolsCalledHandler ¶

type ToolsCalledHandler struct{}

ToolsCalledHandler checks if specific tools were called. Params: tool_names []string, optional min_calls int.

func (h *ToolsCalledHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *ToolsCalledHandler) Type() string

type ToolsCalledSessionHandler ¶

type ToolsCalledSessionHandler struct{}

ToolsCalledSessionHandler checks that specific tools were called across the full session. Params: tool_names []string, min_calls int (optional, default 1).

func (h *ToolsCalledSessionHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (_ *evals.EvalResult, _ error)

func (h *ToolsCalledSessionHandler) Type() string

type ToolsNotCalledHandler ¶

type ToolsNotCalledHandler struct{}

ToolsNotCalledHandler checks that specific tools were NOT called. Params: tool_names []string.

func (h *ToolsNotCalledHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (result *evals.EvalResult, err error)

func (h *ToolsNotCalledHandler) Type() string

type ToolsNotCalledSessionHandler ¶

type ToolsNotCalledSessionHandler struct{}

ToolsNotCalledSessionHandler checks that specific tools were NOT called anywhere in the session. Params: tool_names []string.

func (h *ToolsNotCalledSessionHandler) Eval(
	ctx context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (_ *evals.EvalResult, _ error)

func (h *ToolsNotCalledSessionHandler) Type() string

type VideoDurationHandler ¶ added in v1.3.2

type VideoDurationHandler struct{}

VideoDurationHandler checks that video duration is within range. Params: min_seconds float64, max_seconds float64.

func (h *VideoDurationHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *VideoDurationHandler) Type() string

type VideoResolutionHandler ¶ added in v1.3.2

type VideoResolutionHandler struct{}

VideoResolutionHandler checks that video resolution meets requirements. Params: min_width, max_width, min_height, max_height, presets []string.

func (h *VideoResolutionHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *VideoResolutionHandler) Type() string

type WorkflowCompleteHandler ¶ added in v1.3.2

type WorkflowCompleteHandler struct{}

WorkflowCompleteHandler checks that the workflow reached a terminal state. Reads evalCtx.Extras["workflow_complete"] (bool) and ["workflow_current_state"] (string).

func (h *WorkflowCompleteHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	_ map[string]any,
) (*evals.EvalResult, error)

func (h *WorkflowCompleteHandler) Type() string

type WorkflowStateIsHandler ¶ added in v1.3.2

type WorkflowStateIsHandler struct{}

WorkflowStateIsHandler checks that the current workflow state matches an expected value. Params: state string (required).

func (h *WorkflowStateIsHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *WorkflowStateIsHandler) Type() string

type WorkflowTransitionedToHandler ¶ added in v1.3.2

type WorkflowTransitionedToHandler struct{}

WorkflowTransitionedToHandler checks that a transition to a specific state occurred. Params: state string (required).

func (h *WorkflowTransitionedToHandler) Eval(
	_ context.Context,
	evalCtx *evals.EvalContext,
	params map[string]any,
) (*evals.EvalResult, error)

func (h *WorkflowTransitionedToHandler) Type() string