aicheck

type APIError struct {
	Status int
	Body   string
}

type Cache struct {
	// contains filtered or unexported fields
}

type CacheEntry struct {
	Score  float64 `json:"score"`
	Reason string  `json:"reason"`
	Model  string  `json:"model"`
	At     string  `json:"at"`
}

type Client struct {
	Config JudgeConfig
	APIKey string
	HTTP   *http.Client
	// Backoff is the wait before retry n (1-based); exponential from 1 s when nil.
	Backoff func(n int) time.Duration
	// contains filtered or unexported fields
}

type Evaluator struct {
	Env         map[string]string
	HTTP        *http.Client
	Cache       *Cache
	Concurrency int
	Backoff     func(n int) time.Duration

	Stats Stats
	// contains filtered or unexported fields
}

type File struct {
	Flows map[string]*Flow
}

type Flow struct {
	Name string
	// Iterations is the flow's iterations: (for `stress ci`); 0 when unset.
	Iterations int
	Judge      JudgeConfig // merged and defaulted
	// AutoProvider is true when no provider, model, base_url or api_key_env was set:
	// the judge then follows the keys present (see JudgeFor).
	AutoProvider bool
	Quality      Quality // merged
	// Steps are the steps with an expect: block, in file order.
	Steps []string
	Specs map[string]*Spec
}

type FlowResult struct {
	Runs      []map[string][]Result
	Summaries map[string][]Summary
}

type JudgeConfig = mexpect.JudgeConfig

type JudgeSpec = mexpect.JudgeSpec

type Prompt struct {
	System string
	User   string
}

type Quality struct {
	// FailBelow (0–1): a check passing in a smaller share of runs fails the command. Nil: report only.
	FailBelow *float64
}

type Result struct {
	Kind       string   `json:"kind"`
	Passed     bool     `json:"passed"`
	Score      *float64 `json:"score"`
	Reason     string   `json:"reason"`
	JudgeModel *string  `json:"judge_model"`
	Cached     bool     `json:"cached"`
	Value      *float64 `json:"value"`
	Limit      *float64 `json:"limit"`

	// Skipped: the judge could not judge (no key, provider error). Not a pass or a failure.
	Skipped bool `json:"skipped,omitempty"`
	// Judge accounting: calls made (cache hits: 0) and their tokens.
	Calls        int `json:"judge_calls,omitempty"`
	InputTokens  int `json:"judge_input_tokens,omitempty"`
	OutputTokens int `json:"judge_output_tokens,omitempty"`
}

type Run struct {
	Index int
	Steps map[string]StepRun
}

type Sample struct {
	Score  int
	Reason string
	// Weighted is the logprob-weighted score (G-Eval), when the provider returned logprobs.
	Weighted     *float64
	InputTokens  int
	OutputTokens int
}

type Spec struct {
	Schema       *jsonschema.Schema
	SchemaAt     string
	MaxLatencyMS *float64
	MaxTTFTMS    *float64
	Usage        string
	// UsageSet: the step names its usage path, so there is no fallback to the stream's.
	UsageSet        bool
	MaxInputTokens  *float64
	MaxOutputTokens *float64
	MaxTotalTokens  *float64
	Judge           *JudgeSpec
}

type Stats struct {
	Calls        int // API calls (borderline samples included)
	Judged       int // verdicts that needed calls
	Cached       int // verdicts from the cache or an identical output this run
	Skipped      int
	InputTokens  int
	OutputTokens int
	// Latencies are each verdict's wall time (its calls, samples in parallel).
	Latencies []time.Duration
	// Elapsed is the wall time of the judging phases.
	Elapsed time.Duration
	// NoKey are the reasons judge checks were skipped for a missing key, once each.
	NoKey []string
}

type StepRun struct {
	// Ran: the engine executed it (Success or Failure).
	Ran        bool
	DurationMS *float64
	// Data is the node's output: {"request": {...}, "response": {...}}.
	Data map[string]any
}

type Summary struct {
	Kind              string   `json:"kind"`
	Runs              int      `json:"runs"`
	Passed            int      `json:"passed"`
	Skipped           int      `json:"skipped"`
	PassRate          *float64 `json:"pass_rate"`
	Status            string   `json:"status"`
	FailBelow         *float64 `json:"fail_below"`
	Threshold         *float64 `json:"threshold"`
	MeanScore         *float64 `json:"mean_score"`
	JudgeModel        *string  `json:"judge_model"`
	Criteria          *string  `json:"criteria"`
	Reason            string   `json:"reason"`
	JudgeCalls        int      `json:"judge_calls"`
	JudgeInputTokens  int      `json:"judge_input_tokens"`
	JudgeOutputTokens int      `json:"judge_output_tokens"`
	JudgeCached       int      `json:"-"`

	// Values are the measured latency or token values, for the console's p95.
	Values []float64 `json:"-"`
}

type Verdict struct {
	Score        float64
	Reason       string
	Calls        int
	InputTokens  int
	OutputTokens int
}