Documentation
¶
Overview ¶
Package scenario provides test case generators for persona-aware evaluation scenarios. Each generator creates test cases targeting a specific human-like evaluation dimension.
Index ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type BehaviorTriggerGenerator ¶
type BehaviorTriggerGenerator struct {
// BehaviorName is the name of the behavior.
BehaviorName string
// TriggerCondition describes what should trigger the behavior.
TriggerCondition string
// ExpectedActions are the actions that should be taken.
ExpectedActions []string
}
BehaviorTriggerGenerator creates test cases that test condition-action patterns — does the agent correctly trigger specific behaviors when certain conditions are met?
func NewBehaviorTriggerGenerator ¶
func NewBehaviorTriggerGenerator(name, condition string, actions []string) *BehaviorTriggerGenerator
NewBehaviorTriggerGenerator creates a behavior trigger generator.
func (*BehaviorTriggerGenerator) Generate ¶
func (g *BehaviorTriggerGenerator) Generate(_ context.Context, cfg *GenerateConfig) ([]*testcase.Case, error)
func (*BehaviorTriggerGenerator) Name ¶
func (g *BehaviorTriggerGenerator) Name() string
func (*BehaviorTriggerGenerator) Type ¶
func (g *BehaviorTriggerGenerator) Type() testcase.ScenarioType
type CognitiveStressGenerator ¶
type CognitiveStressGenerator struct {
// ExpectedPhases are the cognitive phases the agent should progress through.
ExpectedPhases []string
}
CognitiveStressGenerator creates test cases that stress-test an agent's thinking strategy transitions under varying complexity.
func NewCognitiveStressGenerator ¶
func NewCognitiveStressGenerator(phases []string) *CognitiveStressGenerator
NewCognitiveStressGenerator creates a cognitive stress generator.
func (*CognitiveStressGenerator) Generate ¶
func (g *CognitiveStressGenerator) Generate(_ context.Context, cfg *GenerateConfig) ([]*testcase.Case, error)
func (*CognitiveStressGenerator) Name ¶
func (g *CognitiveStressGenerator) Name() string
func (*CognitiveStressGenerator) Type ¶
func (g *CognitiveStressGenerator) Type() testcase.ScenarioType
type CommsAdaptationGenerator ¶
type CommsAdaptationGenerator struct {
// Audiences describes target audiences and their expected communication style.
Audiences []Audience
}
CommsAdaptationGenerator creates test cases that evaluate how well the agent adapts its communication style to different audiences.
func NewCommsAdaptationGenerator ¶
func NewCommsAdaptationGenerator(audiences []Audience) *CommsAdaptationGenerator
NewCommsAdaptationGenerator creates a communication adaptation generator.
func (*CommsAdaptationGenerator) Generate ¶
func (g *CommsAdaptationGenerator) Generate(_ context.Context, cfg *GenerateConfig) ([]*testcase.Case, error)
func (*CommsAdaptationGenerator) Name ¶
func (g *CommsAdaptationGenerator) Name() string
func (*CommsAdaptationGenerator) Type ¶
func (g *CommsAdaptationGenerator) Type() testcase.ScenarioType
type GenerateConfig ¶
type GenerateConfig struct {
// SuiteID is the target suite.
SuiteID string
// PersonaDesc is a description of the persona being tested.
PersonaDesc string
// Count is the number of test cases to generate.
Count int
// Difficulty is a hint for test difficulty (easy, medium, hard).
Difficulty string
// Extra holds generator-specific configuration.
Extra map[string]any
}
GenerateConfig provides context for test case generation.
type Generator ¶
type Generator interface {
Name() string
Type() testcase.ScenarioType
Generate(ctx context.Context, cfg *GenerateConfig) ([]*testcase.Case, error)
}
Generator creates test cases for a specific scenario type.
type PerceptionGenerator ¶
type PerceptionGenerator struct {
// RelevantKeywords are terms the agent should attend to.
RelevantKeywords []string
// DistractorKeywords are terms the agent should ignore.
DistractorKeywords []string
}
PerceptionGenerator creates test cases that evaluate the agent's ability to focus on relevant information and ignore distractors.
func NewPerceptionGenerator ¶
func NewPerceptionGenerator(relevant, distractors []string) *PerceptionGenerator
NewPerceptionGenerator creates a perception test generator.
func (*PerceptionGenerator) Generate ¶
func (g *PerceptionGenerator) Generate(_ context.Context, cfg *GenerateConfig) ([]*testcase.Case, error)
func (*PerceptionGenerator) Name ¶
func (g *PerceptionGenerator) Name() string
func (*PerceptionGenerator) Type ¶
func (g *PerceptionGenerator) Type() testcase.ScenarioType
type SkillChallengeGenerator ¶
type SkillChallengeGenerator struct {
// AvailableTools is the list of tools the agent has access to.
AvailableTools []string
}
SkillChallengeGenerator creates test cases that probe an agent's tool selection and usage proficiency.
func NewSkillChallengeGenerator ¶
func NewSkillChallengeGenerator(tools []string) *SkillChallengeGenerator
NewSkillChallengeGenerator creates a skill challenge generator.
func (*SkillChallengeGenerator) Generate ¶
func (g *SkillChallengeGenerator) Generate(_ context.Context, cfg *GenerateConfig) ([]*testcase.Case, error)
func (*SkillChallengeGenerator) Name ¶
func (g *SkillChallengeGenerator) Name() string
func (*SkillChallengeGenerator) Type ¶
func (g *SkillChallengeGenerator) Type() testcase.ScenarioType
type TraitProbeGenerator ¶
type TraitProbeGenerator struct {
// TraitName is the trait being tested.
TraitName string
// TraitDesc describes the expected trait behavior.
TraitDesc string
}
TraitProbeGenerator creates test cases that probe personality trait consistency across different contexts.
func NewTraitProbeGenerator ¶
func NewTraitProbeGenerator(traitName, traitDesc string) *TraitProbeGenerator
NewTraitProbeGenerator creates a trait probe generator.
func (*TraitProbeGenerator) Generate ¶
func (g *TraitProbeGenerator) Generate(_ context.Context, cfg *GenerateConfig) ([]*testcase.Case, error)
func (*TraitProbeGenerator) Name ¶
func (g *TraitProbeGenerator) Name() string
func (*TraitProbeGenerator) Type ¶
func (g *TraitProbeGenerator) Type() testcase.ScenarioType