evaluation

type AccuracyEvaluator struct {
	// contains filtered or unexported fields
}

type CitationEvaluator struct{}

type Dataset struct {
	// Metadata
	Name        string   `yaml:"name"`
	Description string   `yaml:"description"`
	Version     string   `yaml:"version"`
	Author      string   `yaml:"author,omitempty"`
	Tags        []string `yaml:"tags,omitempty"`

	// Configuration
	Config DatasetConfig `yaml:"config"`

	// Test cases
	TestCases []TestCase `yaml:"test_cases"`
}

type DatasetConfig struct {
	// Agent to use (optional - can be overridden at runtime)
	DefaultAgent string `yaml:"default_agent,omitempty"`

	// Collection to query (optional - can be overridden at runtime)
	DefaultCollection string `yaml:"default_collection,omitempty"`

	// Evaluation thresholds
	MinAccuracyScore   float64 `yaml:"min_accuracy_score,omitempty"`
	MinCitationScore   float64 `yaml:"min_citation_score,omitempty"`
	AllowHallucination bool    `yaml:"allow_hallucination,omitempty"`
}

type EvaluationConfig struct {
	AgentPath  string            `json:"agent_path" yaml:"agent_path"`
	Parameters map[string]string `json:"parameters,omitempty" yaml:"parameters,omitempty"`
}

type EvaluationRun struct {
	// Metadata
	ID          string    `json:"id" yaml:"id"`
	Timestamp   time.Time `json:"timestamp" yaml:"timestamp"`
	DatasetName string    `json:"dataset_name" yaml:"dataset_name"`
	AgentName   string    `json:"agent_name" yaml:"agent_name"`
	Collection  string    `json:"collection,omitempty" yaml:"collection,omitempty"`

	// Configuration
	Config EvaluationConfig `json:"config" yaml:"config"`

	// Results
	Results []TestCaseResult `json:"results" yaml:"results"`

	// Summary
	Summary EvaluationSummary `json:"summary" yaml:"summary"`
}

type EvaluationSummary struct {
	TotalTests  int     `json:"total_tests" yaml:"total_tests"`
	PassedTests int     `json:"passed_tests" yaml:"passed_tests"`
	FailedTests int     `json:"failed_tests" yaml:"failed_tests"`
	PassRate    float64 `json:"pass_rate" yaml:"pass_rate"`

	// Average scores
	AvgAccuracy      float64 `json:"avg_accuracy" yaml:"avg_accuracy"`
	AvgCitation      float64 `json:"avg_citation" yaml:"avg_citation"`
	AvgHallucination float64 `json:"avg_hallucination" yaml:"avg_hallucination"`

	// Performance
	TotalTime float64 `json:"total_time_ms" yaml:"total_time_ms"`
	AvgTime   float64 `json:"avg_time_ms" yaml:"avg_time_ms"`

	// Duration
	StartTime time.Time     `json:"start_time" yaml:"start_time"`
	EndTime   time.Time     `json:"end_time" yaml:"end_time"`
	Duration  time.Duration `json:"duration" yaml:"duration"`
}

type Evaluator interface {
	Evaluate(ctx context.Context, testCase *TestCase, actual string, actualCitations []string) (float64, error)
	Name() string
}

type HallucinationDetector struct {
	// contains filtered or unexported fields
}

type Runner struct {
	// contains filtered or unexported fields
}

type TestCase struct {
	// Identification
	ID          string   `yaml:"id"`
	Description string   `yaml:"description,omitempty"`
	Tags        []string `yaml:"tags,omitempty"`

	// Input
	Query      string `yaml:"query"`
	Collection string `yaml:"collection,omitempty"` // Override default collection

	// Expected output
	ExpectedAnswer    string   `yaml:"expected_answer"`
	ExpectedCitations []string `yaml:"expected_citations,omitempty"`
	RequiredConcepts  []string `yaml:"required_concepts,omitempty"`

	// Evaluation criteria
	MinRelevanceScore float64 `yaml:"min_relevance_score,omitempty"`
	MustCite          bool    `yaml:"must_cite,omitempty"`
}

type TestCaseResult struct {
	// Test case info
	TestCaseID string `json:"test_case_id" yaml:"test_case_id"`
	Query      string `json:"query" yaml:"query"`

	// Actual output
	ActualAnswer    string   `json:"actual_answer" yaml:"actual_answer"`
	ActualCitations []string `json:"actual_citations,omitempty" yaml:"actual_citations,omitempty"`
	ResponseTime    float64  `json:"response_time_ms" yaml:"response_time_ms"`

	// Evaluation scores
	AccuracyScore      float64 `json:"accuracy_score" yaml:"accuracy_score"`
	CitationScore      float64 `json:"citation_score" yaml:"citation_score"`
	HallucinationScore float64 `json:"hallucination_score" yaml:"hallucination_score"`

	// Pass/fail
	Passed bool     `json:"passed" yaml:"passed"`
	Errors []string `json:"errors,omitempty" yaml:"errors,omitempty"`

	// Details
	Details map[string]interface{} `json:"details,omitempty" yaml:"details,omitempty"`
}