evaluation

type AccuracyEvaluator struct {
	// contains filtered or unexported fields
}

type CitationEvaluator struct{}

type ContextRelevanceEvaluator struct {
	// contains filtered or unexported fields
}

type CustomEvaluator struct {
	// contains filtered or unexported fields
}

type CustomEvaluatorDef struct {
	Name        string        `yaml:"name"`
	Description string        `yaml:"description"`
	Version     string        `yaml:"version"`
	Prompt      string        `yaml:"prompt"`
	Scoring     ScoringConfig `yaml:"scoring"`
	Model       *ModelConfig  `yaml:"model,omitempty"`
	Tags        []string      `yaml:"tags,omitempty"`
	Author      string        `yaml:"author,omitempty"`
	Required    bool          `yaml:"required"`
}

type CustomEvaluatorLoader struct {
	// contains filtered or unexported fields
}

type Dataset struct {
	// Metadata
	Name        string   `yaml:"name"`
	Description string   `yaml:"description"`
	Version     string   `yaml:"version"`
	Author      string   `yaml:"author,omitempty"`
	Tags        []string `yaml:"tags,omitempty"`

	// Configuration
	Config DatasetConfig `yaml:"config"`

	// Custom evaluators to run (optional)
	CustomEvaluators []string `yaml:"custom_evaluators,omitempty"`

	// Test cases
	TestCases []TestCase `yaml:"test_cases"`
}

type DatasetConfig struct {
	// Agent to use (optional - can be overridden at runtime)
	DefaultAgent string `yaml:"default_agent,omitempty"`

	// Collection to query (optional - can be overridden at runtime)
	DefaultCollection string `yaml:"default_collection,omitempty"`

	// Evaluation thresholds
	MinAccuracyScore   float64 `yaml:"min_accuracy_score,omitempty"`
	MinCitationScore   float64 `yaml:"min_citation_score,omitempty"`
	AllowHallucination bool    `yaml:"allow_hallucination,omitempty"`
}

type EvaluationConfig struct {
	AgentPath  string            `json:"agent_path" yaml:"agent_path"`
	Parameters map[string]string `json:"parameters,omitempty" yaml:"parameters,omitempty"`
}

type EvaluationRun struct {
	// Metadata
	ID          string    `json:"id" yaml:"id"`
	Timestamp   time.Time `json:"timestamp" yaml:"timestamp"`
	DatasetName string    `json:"dataset_name" yaml:"dataset_name"`
	AgentName   string    `json:"agent_name" yaml:"agent_name"`
	Collection  string    `json:"collection,omitempty" yaml:"collection,omitempty"`

	// Configuration
	Config EvaluationConfig `json:"config" yaml:"config"`

	// Results
	Results []TestCaseResult `json:"results" yaml:"results"`

	// Summary
	Summary EvaluationSummary `json:"summary" yaml:"summary"`
}

type EvaluationSummary struct {
	TotalTests  int     `json:"total_tests" yaml:"total_tests"`
	PassedTests int     `json:"passed_tests" yaml:"passed_tests"`
	FailedTests int     `json:"failed_tests" yaml:"failed_tests"`
	PassRate    float64 `json:"pass_rate" yaml:"pass_rate"`

	// Average scores
	AvgAccuracy         float64 `json:"avg_accuracy" yaml:"avg_accuracy"`
	AvgCitation         float64 `json:"avg_citation" yaml:"avg_citation"`
	AvgHallucination    float64 `json:"avg_hallucination" yaml:"avg_hallucination"`
	AvgContextRelevance float64 `json:"avg_context_relevance" yaml:"avg_context_relevance"`
	AvgFaithfulness     float64 `json:"avg_faithfulness" yaml:"avg_faithfulness"`

	// Performance
	TotalTime float64 `json:"total_time_ms" yaml:"total_time_ms"`
	AvgTime   float64 `json:"avg_time_ms" yaml:"avg_time_ms"`

	// Duration
	StartTime time.Time     `json:"start_time" yaml:"start_time"`
	EndTime   time.Time     `json:"end_time" yaml:"end_time"`
	Duration  time.Duration `json:"duration" yaml:"duration"`
}

type Evaluator interface {
	Evaluate(ctx context.Context, testCase *TestCase, actual string, actualCitations []string) (float64, error)
	Name() string
}

type EvaluatorProvider interface {
	// Name returns the provider name (e.g., "local", "opik")
	Name() string

	// GetAccuracyEvaluator returns an evaluator for semantic accuracy
	GetAccuracyEvaluator() Evaluator

	// GetFaithfulnessEvaluator returns an evaluator for faithfulness/groundedness
	GetFaithfulnessEvaluator() Evaluator

	// GetHallucinationEvaluator returns an evaluator for hallucination detection
	GetHallucinationEvaluator() Evaluator

	// GetContextRelevanceEvaluator returns an evaluator for context relevance
	GetContextRelevanceEvaluator() Evaluator

	// IsAvailable checks if the provider is properly configured
	IsAvailable(ctx context.Context) bool
}

type FaithfulnessEvaluator struct {
	// contains filtered or unexported fields
}

type HallucinationDetector struct {
	// contains filtered or unexported fields
}

type LocalProvider struct {
	// contains filtered or unexported fields
}

type ModelConfig struct {
	Provider    string  `yaml:"provider"`    // openai, anthropic
	Name        string  `yaml:"name"`        // model name
	Temperature float64 `yaml:"temperature"` // temperature for generation
}

type OpikAccuracyEvaluator struct {
	// contains filtered or unexported fields
}

type OpikContextRelevanceEvaluator struct {
	// contains filtered or unexported fields
}

type OpikFaithfulnessEvaluator struct {
	// contains filtered or unexported fields
}

type OpikHallucinationEvaluator struct {
	// contains filtered or unexported fields
}

type OpikProvider struct {
	// contains filtered or unexported fields
}

type ProviderType string

type Runner struct {
	// contains filtered or unexported fields
}

type ScoringConfig struct {
	Type      string  `yaml:"type"`              // llm_judge, regex, exact_match, contains
	Threshold float64 `yaml:"threshold"`         // Minimum passing score
	Weight    float64 `yaml:"weight"`            // Weight in overall evaluation (default 1.0)
	Pattern   string  `yaml:"pattern,omitempty"` // For regex type
}

type TestCase struct {
	// Identification
	ID          string   `yaml:"id"`
	Description string   `yaml:"description,omitempty"`
	Tags        []string `yaml:"tags,omitempty"`

	// Input
	Query      string `yaml:"query"`
	Collection string `yaml:"collection,omitempty"` // Override default collection

	// Expected output
	ExpectedAnswer    string   `yaml:"expected_answer"`
	ExpectedCitations []string `yaml:"expected_citations,omitempty"`
	RequiredConcepts  []string `yaml:"required_concepts,omitempty"`

	// Retrieved context (for advanced evaluation)
	RetrievedContext []string `yaml:"retrieved_context,omitempty"`

	// Evaluation criteria
	MinRelevanceScore float64 `yaml:"min_relevance_score,omitempty"`
	MustCite          bool    `yaml:"must_cite,omitempty"`
}

type TestCaseResult struct {
	// Test case info
	TestCaseID string `json:"test_case_id" yaml:"test_case_id"`
	Query      string `json:"query" yaml:"query"`

	// Actual output
	ActualAnswer    string   `json:"actual_answer" yaml:"actual_answer"`
	ActualCitations []string `json:"actual_citations,omitempty" yaml:"actual_citations,omitempty"`
	ResponseTime    float64  `json:"response_time_ms" yaml:"response_time_ms"`

	// Evaluation scores
	AccuracyScore         float64 `json:"accuracy_score" yaml:"accuracy_score"`
	CitationScore         float64 `json:"citation_score" yaml:"citation_score"`
	HallucinationScore    float64 `json:"hallucination_score" yaml:"hallucination_score"`
	ContextRelevanceScore float64 `json:"context_relevance_score" yaml:"context_relevance_score"`
	FaithfulnessScore     float64 `json:"faithfulness_score" yaml:"faithfulness_score"`

	// Custom evaluator scores
	CustomScores map[string]float64 `json:"custom_scores,omitempty" yaml:"custom_scores,omitempty"`

	// Pass/fail
	Passed bool     `json:"passed" yaml:"passed"`
	Errors []string `json:"errors,omitempty" yaml:"errors,omitempty"`

	// Details
	Details map[string]interface{} `json:"details,omitempty" yaml:"details,omitempty"`
}