livetest

type Case struct {
	Name string
	// Needs returns why the case cannot run against the suite, or "" when
	// it can. Nil runs the case always.
	Needs func() string
	Run   func(t *testing.T)
}

type EmbedderSuite struct {
	// Embedder is the embedder under test. Required.
	Embedder ai.EmbedderArg
	// Dimensions is the length every vector must have. Required.
	Dimensions int
	// Normalized reports whether the provider returns unit-length vectors.
	Normalized bool
}

type Group struct {
	Name  string
	Cases []Case
}

type Suite struct {
	// Model answers the cheap prompts. Every case that needs no special
	// model runs against it. Required.
	Model ai.ModelArg
	// ReasoningModel emits thinking, usually an [ai.ModelRef] carrying the
	// plugin's thinking config. Nil skips the reasoning cases.
	ReasoningModel ai.ModelArg
	// ReasoningContent reports whether the provider returns the thinking
	// content itself. Endpoints that accept the knob but keep the content
	// server-side leave this false and still get the no-error checks.
	ReasoningContent bool
	// StreamOnlyReasoning skips the non-streaming reasoning cases, for
	// providers that only think on streaming calls.
	StreamOnlyReasoning bool
	// VisionModel accepts inline images. Nil falls back to Model when Model
	// claims media support, and skips the vision cases otherwise.
	VisionModel ai.ModelArg
	// AudioModel accepts audio input. Nil skips the audio case, as models
	// that read images often read no audio.
	AudioModel ai.ModelArg
	// DocumentModel accepts PDF input. Nil skips the document case.
	DocumentModel ai.ModelArg
	// LimitConfig is a request config for Model that caps output at a few
	// tokens, to check the length finish reason. Config types differ per
	// plugin, so the suite cannot build one. Nil skips the case.
	LimitConfig any
	// ToolResponseMedia reports whether the provider reads images returned
	// inside a tool response. Providers that cannot carry them drop them with
	// a warning, so the case only runs where they can.
	ToolResponseMedia bool
	// BadKeyPlugin is the plugin configured with an API key the provider
	// rejects, to check that the refusal classifies as UNAUTHENTICATED. Nil
	// skips the case, for plugins that authenticate ambiently.
	BadKeyPlugin api.Plugin
	// Skip maps a case name, as in "generate/tool choice none" or
	// "compat_oai/extra config passthrough", to the reason the plugin or the
	// provider cannot pass it.
	Skip map[string]string
}