scorer

type BehaviorTriggerScorer struct {
	// BehaviorName is the name of the behavior being tested.
	BehaviorName string
	// ExpectedActions are actions that should appear in the trace.
	ExpectedActions []string
	// TriggerKeywords are keywords that indicate the trigger was detected.
	TriggerKeywords []string
}

type CognitivePhaseScorer struct {
	// ExpectedPhases is the ordered list of cognitive phases expected.
	ExpectedPhases []string
}

type CommunicationStyleScorer struct {
	Client         LLMClient
	Model          string
	ExpectedTone   string
	ExpectedFormat string
}

type ContainsScorer struct {
	Substring       string
	CaseInsensitive bool
}

type CostScorer struct {
	MaxCost float64
}

type CustomScorer struct {
	// contains filtered or unexported fields
}

type Descriptor struct {
	Name        string
	Revision    string // immutable implementation revision; required for frozen evaluations
	Description string
	Dimension   string // the human-like dimension it reports, empty for none
	UsesLLM     bool   // true when scoring makes a model call, which costs money
	// RequiresConfig is true when the scorer cannot be built without
	// config, so a run-level pick with no config would be refused.
	RequiresConfig bool
}

type EmbeddingClient interface {
	Embed(ctx context.Context, texts []string) ([][]float64, error)
}

type ExactScorer struct {
	CaseInsensitive bool
}

type Factory func(config map[string]any) (Scorer, error)

type FactualScorer struct {
	Client LLMClient
	Model  string
}

type Func func(ctx context.Context, input *Input) (*Output, error)

type HallucinationScorer struct {
	Client LLMClient
	Model  string
}

type Input struct {
	Input    string            // User input / prompt
	Expected string            // Expected output (optional)
	Actual   string            // Actual output
	Trace    *evalrun.RunTrace // Agent execution trace (optional, for persona scorers)
	Context  map[string]any    // Additional context
}

type JSONSchemaScorer struct {
	Schema string
}

type JSONValidScorer struct{}

type LLMClient interface {
	Complete(ctx context.Context, model, systemPrompt, userInput string, temperature float64) (string, error)
}

type LLMJudgeScorer struct {
	Client      LLMClient
	Model       string
	Criteria    string
	Temperature float64
}

type LatencyScorer struct {
	MaxMs int
}

type LengthScorer struct {
	MinTokens int
	MaxTokens int
}

type NotContainsScorer struct {
	Substring       string
	CaseInsensitive bool
}

type Output struct {
	Usage     *evalrun.Usage // Optional judge accounting; nil means unknown, not zero.
	Score     float64        // 0.0 to 1.0
	Passed    bool           // Score >= threshold
	Reason    string         // Human-readable explanation
	Dimension string         // Which evaluation dimension (skill, trait, etc.)
	Details   map[string]any // Scorer-specific details
}

type PerceptionFocusScorer struct {
	// RequiredKeywords are terms the output must reference.
	RequiredKeywords []string
	// IgnoredKeywords are distractors the output should not focus on.
	IgnoredKeywords []string
}

type PersonaCoherenceScorer struct {
	Client      LLMClient
	Model       string
	PersonaDesc string
}

type RegexScorer struct {
	Pattern string
	// contains filtered or unexported fields
}

type Registry struct {
	// contains filtered or unexported fields
}

type Scorer interface {
	Name() string
	Score(ctx context.Context, input *Input) (*Output, error)
}

type SemanticScorer struct {
	Client    EmbeddingClient
	Threshold float64
}

type SkillUsageScorer struct {
	ExpectedTools  []string
	ForbiddenTools []string
}

type TraitConsistencyScorer struct {
	Client    LLMClient
	Model     string
	TraitName string
	TraitDesc string
}