eval

type Corpus struct {
	Name      string     `json:"name,omitempty"`
	Documents []Document `json:"documents"`
}

type Dataset struct {
	Name    string  `json:"name,omitempty"`
	Queries []Query `json:"queries"`
}

type Document struct {
	Source  string `json:"source"`
	Title   string `json:"title,omitempty"`
	Content string `json:"content"`
	Lang    string `json:"lang,omitempty"`
}

type Query struct {
	// ID is a stable identifier for the query, used to label per-query results
	// and to segment metrics. Optional but recommended.
	ID string `json:"id,omitempty"`
	// Query is the search string submitted to the retriever.
	Query string `json:"query"`
	// Lang is the ISO language code of the query (e.g. "fr", "en", "es"), used
	// to segment metrics by language.
	Lang string `json:"lang,omitempty"`
	// RelevantSources lists the source identifiers expected in the results.
	RelevantSources []string `json:"relevant_sources"`
	// Answers lists the acceptable gold answer strings for the query, used only by
	// the optional generation (reader) evaluation to score answer EM/F1. Empty for
	// pure retrieval datasets; a query may have several equally-correct answers
	// (SQuAD) or a single one (HotpotQA).
	Answers []string `json:"answers,omitempty"`
	// Tags optionally categorise the query (intent, complexity, domain) for
	// segmented reporting.
	Tags []string `json:"tags,omitempty"`
}

type QueryReport struct {
	QueryID string
	Query   string
	Lang    string
	Tags    []string
	// TargetRank is the 1-indexed rank of the first relevant document, or -1 if
	// none was retrieved.
	TargetRank     int
	ReciprocalRank float64
	RecallAtK      map[int]float64
	NDCGAtK        map[int]float64
	PrecisionAtK   map[int]float64
}

type Report struct {
	Dataset    string
	NumQueries int
	Ks         []int
	// MRR is the mean reciprocal rank across all queries.
	MRR float64
	// RecallAtK, NDCGAtK and PrecisionAtK are the dataset-averaged metrics per
	// cut-off.
	RecallAtK    map[int]float64
	NDCGAtK      map[int]float64
	PrecisionAtK map[int]float64
	PerQuery     []QueryReport
}

type Retriever interface {
	Retrieve(ctx context.Context, query string, k int) ([]string, error)
}

type RetrieverFunc func(ctx context.Context, query string, k int) ([]string, error)