eval

type AggregateMetrics struct {
	AvgFaithfulness       float64 `json:"avg_faithfulness"`
	AvgRelevance          float64 `json:"avg_relevance"`
	AvgAccuracy           float64 `json:"avg_accuracy"`
	AvgStrictAccuracy     float64 `json:"avg_strict_accuracy"`
	AvgContextRecall      float64 `json:"avg_context_recall"`
	AvgCitationQuality    float64 `json:"avg_citation_quality"`
	AvgConfidence         float64 `json:"avg_confidence"`
	AvgClaimGrounding     float64 `json:"avg_claim_grounding"`
	AvgHallucinationScore float64 `json:"avg_hallucination_score"`

	// Retrieval metrics (populated when ground-truth spans are available)
	AvgRetrievalPrecision map[int]float64 `json:"avg_retrieval_precision,omitempty"` // k -> P@k
	AvgRetrievalRecall    map[int]float64 `json:"avg_retrieval_recall,omitempty"`    // k -> R@k
}

type Dataset struct {
	Name       string     `json:"name"`
	Difficulty string     `json:"difficulty"` // easy, medium, hard, complex, super-hard
	Tests      []TestCase `json:"tests"`
}

type Evaluator struct {
	// contains filtered or unexported fields
}

type FactCheck struct {
	Fact      string `json:"fact"`
	Found     bool   `json:"found"`
	ChunkID   int64  `json:"chunk_id,omitempty"`
	ChunkRank int    `json:"chunk_rank,omitempty"`
	Details   string `json:"details,omitempty"`
}

type FullContextEvaluator struct {
	// contains filtered or unexported fields
}

type GroundTruthCheck struct {
	FactsInDB      []FactCheck `json:"facts_in_db"`
	FactsEmbedded  []FactCheck `json:"facts_embedded"`
	FactsRetrieved []FactCheck `json:"facts_retrieved"`
	FactsInAnswer  []FactCheck `json:"facts_in_answer"`
	Diagnosis      string      `json:"diagnosis"`
}

type GroundTruthSpan struct {
	FilePath string
	Start    int
	End      int
	Text     string
}

type LegalBenchBenchmark struct {
	Tests []LegalBenchTest `json:"tests"`
}

type LegalBenchConfig struct {
	// BenchmarkFiles are paths to benchmark JSON files.
	BenchmarkFiles []string
	// CorpusDir is the path to the corpus directory (for reading snippet text).
	CorpusDir string
	// MaxTestsPerBenchmark caps the number of tests loaded per benchmark file.
	// 0 means no limit (load all). 194 matches the LegalBench-RAG-mini subset.
	MaxTestsPerBenchmark int
}

type LegalBenchSnippet struct {
	FilePath string `json:"file_path"`
	Span     [2]int `json:"span"`   // [start, end] character offsets
	Answer   string `json:"answer"` // pre-extracted snippet text
}

type LegalBenchTest struct {
	Query    string              `json:"query"`
	Snippets []LegalBenchSnippet `json:"snippets"`
	Tags     []string            `json:"tags"`
}

type PDFComplexityResult struct {
	Path            string  `json:"path"`
	ExpectedComplex bool    `json:"expected_complex"`
	DetectedComplex bool    `json:"detected_complex"`
	Score           float64 `json:"score"`
	Correct         bool    `json:"correct"`
	Details         string  `json:"details"`
}

type PDFComplexityTestCase struct {
	Path            string `json:"path"`
	ExpectedComplex bool   `json:"expected_complex"`
	Description     string `json:"description"`
}

type ReasoningStep struct {
	Round     int      `json:"round"`
	Action    string   `json:"action"`
	Prompt    string   `json:"prompt,omitempty"`
	Response  string   `json:"response,omitempty"`
	Tokens    int      `json:"tokens,omitempty"`
	ElapsedMs int64    `json:"elapsed_ms,omitempty"`
	Issues    []string `json:"issues,omitempty"`
}

type Report struct {
	Dataset         string                      `json:"dataset"`
	Difficulty      string                      `json:"difficulty,omitempty"`
	TotalTests      int                         `json:"total_tests"`
	Passed          int                         `json:"passed"`
	Failed          int                         `json:"failed"`
	Metrics         AggregateMetrics            `json:"metrics"`
	CategoryMetrics map[string]AggregateMetrics `json:"category_metrics,omitempty"`
	Results         []TestResult                `json:"results"`
	RunTime         time.Duration               `json:"run_time"`
	TokenUsage      TokenUsage                  `json:"token_usage"`
}

type RetrievalTrace struct {
	VecResults          int      `json:"vec_results"`
	FTSResults          int      `json:"fts_results"`
	GraphResults        int      `json:"graph_results"`
	FusedResults        int      `json:"fused_results"`
	VecWeight           float64  `json:"vec_weight"`
	FTSWeight           float64  `json:"fts_weight"`
	GraphWeight         float64  `json:"graph_weight"`
	IdentifiersDetected bool     `json:"identifiers_detected"`
	FTSQuery            string   `json:"fts_query"`
	GraphEntities       []string `json:"graph_entities"`
	ElapsedMs           int64    `json:"elapsed_ms"`
}

type SourceTrace struct {
	ChunkID    int64    `json:"chunk_id"`
	Heading    string   `json:"heading"`
	Content    string   `json:"content"`
	PageNumber int      `json:"page_number"`
	Score      float64  `json:"score"`
	Methods    []string `json:"methods,omitempty"`
	VecRank    int      `json:"vec_rank,omitempty"`
	FTSRank    int      `json:"fts_rank,omitempty"`
	GraphRank  int      `json:"graph_rank,omitempty"`
}

type TestCase struct {
	Question      string   `json:"question"`
	ExpectedFacts []string `json:"expected_facts"` // Facts that should appear in the answer
	Category      string   `json:"category"`       // single-fact, multi-hop, cross-document, multi-fact, synthesis
	Explanation   string   `json:"explanation"`    // Ground truth reference with page citations
}

type TestResult struct {
	Question           string   `json:"question"`
	ExpectedFacts      []string `json:"expected_facts"`
	Category           string   `json:"category,omitempty"`
	Explanation        string   `json:"explanation,omitempty"`
	Answer             string   `json:"answer"`
	Confidence         float64  `json:"confidence"`
	Faithfulness       float64  `json:"faithfulness"`
	Relevance          float64  `json:"relevance"`
	Accuracy           float64  `json:"accuracy"`
	StrictAccuracy     float64  `json:"strict_accuracy"`
	ContextRecall      float64  `json:"context_recall"`
	CitationQuality    float64  `json:"citation_quality"`
	ClaimGrounding     float64  `json:"claim_grounding"`
	HallucinationScore float64  `json:"hallucination_score"`
	Passed             bool     `json:"passed"`
	Error              string   `json:"error,omitempty"`
	PromptTokens       int      `json:"prompt_tokens"`
	CompletionTokens   int      `json:"completion_tokens"`
	TotalTokens        int      `json:"total_tokens"`

	// Timing
	ElapsedMs int64 `json:"elapsed_ms"`

	// Sources (the chunks the model actually saw)
	Sources []SourceTrace `json:"sources,omitempty"`

	// Retrieval breakdown
	Retrieval *RetrievalTrace `json:"retrieval,omitempty"`

	// Reasoning trace
	ReasoningSteps []ReasoningStep `json:"reasoning_steps,omitempty"`

	// Ground truth diagnosis
	GroundTruth *GroundTruthCheck `json:"ground_truth,omitempty"`

	// Retrieval metrics (populated when ground-truth spans are available)
	RetrievalPrecision map[int]float64 `json:"retrieval_precision,omitempty"` // k -> P@k
	RetrievalRecall    map[int]float64 `json:"retrieval_recall,omitempty"`    // k -> R@k
}

type TokenUsage struct {
	PromptTokens     int `json:"prompt_tokens"`
	CompletionTokens int `json:"completion_tokens"`
	TotalTokens      int `json:"total_tokens"`
}