benchmarks

type Catalog struct {
	SchemaVersion int           `json:"schema_version"`
	Datasets      []Dataset     `json:"datasets"`
	Runtime       RuntimeStatus `json:"runtime"`
}

type Connection struct {
	Protocol  domain.Protocol `json:"protocol"`
	Model     string          `json:"model"`
	ChannelID string          `json:"channel_id"`
	BaseURL   string          `json:"base_url"`
	APIKey    string          `json:"api_key"`
}

type Dataset struct {
	ID               string `json:"id"`
	Name             string `json:"name"`
	Description      string `json:"description"`
	Category         string `json:"category"`
	SourceURL        string `json:"source_url"`
	DefaultMaxTokens int    `json:"default_max_tokens"`
}

type DatasetDownload struct {
	State       string `json:"state"`
	Bytes       int64  `json:"bytes"`
	Files       int    `json:"files"`
	Samples     int    `json:"samples"`
	InputSHA256 string `json:"input_sha256"`
	CacheSHA256 string `json:"cache_sha256"`
	TaskVersion string `json:"task_version"`
	UpdatedAt   string `json:"updated_at"`
	VerifiedAt  string `json:"verified_at"`
	ErrorCode   string `json:"error_code"`
}

type DatasetInventory struct {
	SchemaVersion int           `json:"schema_version"`
	Datasets      []DatasetItem `json:"datasets"`
	Runtime       RuntimeStatus `json:"runtime"`
}

type DatasetItem struct {
	Dataset  Dataset         `json:"dataset"`
	Download DatasetDownload `json:"download"`
}

type DatasetRun struct {
	ReportID   string      `json:"report_id"`
	Dataset    Dataset     `json:"dataset"`
	Status     string      `json:"status"`
	Stage      string      `json:"stage"`
	Total      int         `json:"total"`
	Completed  int         `json:"completed"`
	Failed     int         `json:"failed"`
	InFlight   int         `json:"in_flight"`
	Metrics    []Metric    `json:"metrics"`
	Provenance *Provenance `json:"provenance"`
	ErrorCode  string      `json:"error_code"`
	StartedAt  string      `json:"started_at"`
	FinishedAt string      `json:"finished_at"`
}

type DatasetRuntime interface {
	DatasetKey(string) string
	DownloadDataset(context.Context, string, string, func(DatasetTransfer) error) error
}

type DatasetTransfer struct {
	SchemaVersion int    `json:"schema_version"`
	DatasetID     string `json:"dataset_id"`
	Kind          string `json:"kind"`
	Bytes         int64  `json:"bytes"`
	Files         int    `json:"files"`
	Samples       int    `json:"samples"`
	InputSHA256   string `json:"input_sha256"`
	TaskVersion   string `json:"task_version"`
}

type Dependencies struct {
	Directory      string
	CacheDirectory string
	CatalogJSON    []byte
	Runtime        Runtime
	Resolve        Resolver
	Changed        func()
}

type Event struct {
	SchemaVersion int         `json:"schema_version"`
	RunID         string      `json:"run_id"`
	ReportID      string      `json:"report_id"`
	Sequence      int         `json:"sequence"`
	Kind          string      `json:"kind"`
	Total         int         `json:"total"`
	Completed     int         `json:"completed"`
	Failed        int         `json:"failed"`
	InFlight      int         `json:"in_flight"`
	Sample        *Sample     `json:"sample"`
	Status        string      `json:"status"`
	Metrics       []Metric    `json:"metrics"`
	Provenance    *Provenance `json:"provenance"`
	Code          string      `json:"code"`
}

type Metric struct {
	Scorer string  `json:"scorer"`
	Name   string  `json:"name"`
	Value  float64 `json:"value"`
}

type Options struct {
	SampleLimit     int      `json:"sample_limit"`
	Epochs          int      `json:"epochs"`
	MaxTokens       int      `json:"max_tokens"`
	Concurrency     int      `json:"concurrency"`
	TimeoutSeconds  int      `json:"timeout_seconds"`
	Temperature     *float64 `json:"temperature"`
	Seed            int      `json:"seed"`
	Stream          bool     `json:"stream"`
	ReasoningEffort string   `json:"reasoning_effort"`
}

type Page struct {
	SchemaVersion int    `json:"schema_version"`
	Runs          []Run  `json:"runs"`
	NextCursor    string `json:"next_cursor"`
}

type Provenance struct {
	InspectAI       string `json:"inspect_ai"`
	InspectEvals    string `json:"inspect_evals"`
	Task            string `json:"task"`
	TaskVersion     string `json:"task_version"`
	DatasetName     string `json:"dataset_name"`
	DatasetLocation string `json:"dataset_location"`
	InputSHA256     string `json:"input_sha256"`
}

type Report struct {
	SchemaVersion int        `json:"schema_version"`
	RunID         string     `json:"run_id"`
	Target        Target     `json:"target"`
	Options       Options    `json:"options"`
	Result        DatasetRun `json:"result"`
	Samples       []Sample   `json:"samples"`
	NextCursor    string     `json:"next_cursor"`
}

type ResolvedConnection struct {
	Target Target
	APIKey []byte
}

type Resolver func(context.Context, Connection) (ResolvedConnection, error)

type Run struct {
	SchemaVersion int          `json:"schema_version"`
	ID            string       `json:"id"`
	Target        Target       `json:"target"`
	Options       Options      `json:"options"`
	Status        string       `json:"status"`
	CreatedAt     string       `json:"created_at"`
	FinishedAt    string       `json:"finished_at"`
	Datasets      []DatasetRun `json:"datasets"`
}

type Runtime interface {
	Ready() bool
	Prepare(context.Context) error
	Execute(context.Context, WorkerRequest, func(Event) error) error
}

type RuntimeStatus struct {
	State     string `json:"state"`
	ErrorCode string `json:"error_code"`
}

type Sample struct {
	ID           string  `json:"id"`
	Epoch        int     `json:"epoch"`
	Status       string  `json:"status"`
	Scores       []Score `json:"scores"`
	Input        string  `json:"input"`
	Target       string  `json:"target"`
	Output       string  `json:"output"`
	InputTokens  int64   `json:"input_tokens"`
	OutputTokens int64   `json:"output_tokens"`
	DurationMS   int64   `json:"duration_ms"`
	ErrorCode    string  `json:"error_code"`
}

type Score struct {
	Name        string `json:"name"`
	Value       string `json:"value"`
	Answer      string `json:"answer"`
	Explanation string `json:"explanation"`
}

type Service struct {
	// contains filtered or unexported fields
}

type StartCommand struct {
	DatasetIDs []string   `json:"dataset_ids"`
	Connection Connection `json:"connection"`
	Options    Options    `json:"options"`
}

type Target struct {
	Protocol    domain.Protocol `json:"protocol"`
	Model       string          `json:"model"`
	ChannelID   string          `json:"channel_id"`
	ChannelName string          `json:"channel_name"`
	BaseURL     string          `json:"base_url"`
}

type WorkerRequest struct {
	SchemaVersion int             `json:"schema_version"`
	RunID         string          `json:"run_id"`
	ReportID      string          `json:"report_id"`
	DatasetID     string          `json:"dataset_id"`
	Protocol      domain.Protocol `json:"protocol"`
	Model         string          `json:"model"`
	BaseURL       string          `json:"base_url"`
	APIKey        string          `json:"api_key"`
	Options       Options         `json:"options"`
	LogDir        string          `json:"log_dir"`
	CacheDir      string          `json:"cache_dir,omitempty"`
}