Documentation
¶
Overview ¶
Package llm provides a generic abstraction layer for interacting with Large Language Model providers.
Index ¶
- Constants
- Variables
- func ApplyOptions(cfg *Config, opts ...Option)
- func ApplyRequestOptions(req *ChatRequest, opts ...RequestOption)
- func DefaultAPIKeyEnvVar(providerType ProviderType) string
- func DefaultBaseURL(providerType ProviderType) string
- func GetAPIKeyFromEnv(providerType ProviderType) string
- func IsAuthError(err error) bool
- func IsRateLimitError(err error) bool
- func IsRetryable(err error) bool
- func RegisterProvider(providerType ProviderType, factory ProviderFactory)
- func ShouldRetryRequest(ctx context.Context, err error) bool
- func WrapError(provider string, err error) error
- type APIError
- type ChatRequest
- type ChatResponse
- type Config
- type HTTPClient
- type Image
- type LogicalRetryConfig
- type Message
- type OAuthCredential
- type OAuthCredentialProvider
- type Option
- type Provider
- type ProviderFactory
- type ProviderState
- type ProviderType
- type RequestOption
- type Role
- type StreamEvent
- type ThinkingEffort
- type ThinkingRequest
- type Tool
- type ToolCall
- type ToolCallFunction
- type ToolFunction
- type Usage
- type UserLocation
- type WebSearchRequest
Constants ¶
const WebSearchToolName = "web_search"
WebSearchToolName is the canonical explicit tool name for web search.
Variables ¶
var ( // ErrNoAPIKey indicates that no API key was provided. ErrNoAPIKey = errors.New("no API key provided") // ErrInvalidProvider indicates an unknown or invalid provider type. ErrInvalidProvider = errors.New("invalid provider") // ErrRateLimited indicates the API rate limit has been exceeded. ErrRateLimited = errors.New("rate limited") // ErrContextTooLong indicates the input exceeds the model's context limit. ErrContextTooLong = errors.New("context length exceeded") // ErrModelNotFound indicates the requested model does not exist. ErrModelNotFound = errors.New("model not found") // ErrInvalidRequest indicates a malformed request. ErrInvalidRequest = errors.New("invalid request") ErrUnauthorized = errors.New("unauthorized") // ErrServerError indicates a server-side error. ErrServerError = errors.New("server error") // ErrTimeout indicates the request timed out. ErrTimeout = errors.New("request timeout") // ErrStreamClosed indicates the stream was unexpectedly closed. ErrStreamClosed = errors.New("stream closed unexpectedly") )
Sentinel errors for common LLM error conditions.
Functions ¶
func ApplyOptions ¶
ApplyOptions applies the given options to a Config.
func ApplyRequestOptions ¶
func ApplyRequestOptions(req *ChatRequest, opts ...RequestOption)
ApplyRequestOptions applies the given options to a ChatRequest.
func DefaultAPIKeyEnvVar ¶
func DefaultAPIKeyEnvVar(providerType ProviderType) string
DefaultAPIKeyEnvVar returns the standard environment variable name for the API key of a given provider.
func DefaultBaseURL ¶
func DefaultBaseURL(providerType ProviderType) string
DefaultBaseURL returns the default API endpoint URL for a given provider.
func GetAPIKeyFromEnv ¶
func GetAPIKeyFromEnv(providerType ProviderType) string
GetAPIKeyFromEnv attempts to retrieve the API key for a provider from environment variables.
func IsAuthError ¶
IsAuthError returns true if the error is an authentication error.
func IsRateLimitError ¶
IsRateLimitError returns true if the error is a rate limit error.
func IsRetryable ¶
IsRetryable returns true if the error is retryable.
func RegisterProvider ¶
func RegisterProvider(providerType ProviderType, factory ProviderFactory)
RegisterProvider registers a provider factory for a given type. This is typically called in the init() function of each provider package.
func ShouldRetryRequest ¶
ShouldRetryRequest returns true when an LLM request failure is transient and the caller context is still active, making another logical attempt safe.
Types ¶
type APIError ¶
type APIError struct {
// Provider is the name of the provider that returned the error.
Provider string
// StatusCode is the HTTP status code returned.
StatusCode int
// Message is the error message from the API.
Message string
// Retryable indicates whether the request can be retried.
Retryable bool
// Err is the underlying error, if any.
Err error
}
APIError represents an error response from an LLM API.
func NewAPIError ¶
NewAPIError creates a new APIError with the given parameters.
type ChatRequest ¶
type ChatRequest struct {
// Model is the identifier of the model to use.
Model string
// Messages is the session history to send to the model.
Messages []Message
// Temperature controls randomness in the response (0.0 to 2.0).
// Lower values make output more deterministic.
Temperature *float64
// MaxTokens is the maximum number of tokens to generate.
MaxTokens *int
// TopP is the nucleus sampling parameter (0.0 to 1.0).
TopP *float64
// Stop is a list of sequences where the model will stop generating.
Stop []string
// Thinking enables extended thinking/reasoning mode.
// Provider-specific handling in each provider implementation.
Thinking *ThinkingRequest
// Tools is a list of tools available for the model to call.
Tools []Tool
// ToolChoice controls how the model uses tools.
// Values: "auto" (default), "required", "none", or a specific tool name.
ToolChoice string
// WebSearch enables provider-native web search when non-nil and Enabled.
// Provider-specific handling in each provider implementation.
WebSearch *WebSearchRequest
}
ChatRequest contains the input for a chat completion request.
func NewChatRequest ¶
func NewChatRequest(model string, messages []Message, opts ...RequestOption) *ChatRequest
NewChatRequest creates a new ChatRequest with the given model, messages, and options.
func NormalizeChatRequest ¶
func NormalizeChatRequest(req *ChatRequest) *ChatRequest
NormalizeChatRequest applies provider-independent request invariants before a request reaches a concrete provider.
type ChatResponse ¶
type ChatResponse struct {
// Content is the generated text content.
Content string
// ReasoningContent holds the model's internal reasoning/thinking text.
// Must be preserved and echoed back in the next assistant message when
// reasoning is enabled, as required by some providers (e.g. Moonshot/KIMI).
ReasoningContent string
// FinishReason indicates why the model stopped generating.
// Common values: "stop", "length", "content_filter", "tool_calls".
FinishReason string
// Usage contains token usage statistics.
Usage Usage
// ToolCalls contains tool calls requested by the model.
// Only populated when FinishReason is "tool_calls".
ToolCalls []ToolCall
// ProviderState is the provider's record of this turn, set by providers
// that need the turn sent back as received. Callers pass it on the
// assistant message that records the turn.
ProviderState *ProviderState
}
ChatResponse contains the output from a chat completion request.
func ChatWithRetry ¶
func ChatWithRetry(ctx context.Context, provider Provider, req *ChatRequest, cfg LogicalRetryConfig) (*ChatResponse, error)
ChatWithRetry executes provider.Chat with bounded logical retries for transient request failures.
type Config ¶
type Config struct {
// APIKey is the authentication key for the provider.
APIKey string
// AccountID carries provider-specific account metadata when required.
AccountID string
// BaseURL is the API endpoint URL. If empty, the provider's default is used.
BaseURL string
// OAuthCredentialProvider resolves dynamic bearer credentials on demand.
OAuthCredentialProvider OAuthCredentialProvider
// Timeout is the maximum time to wait for a response.
// Default is 60 seconds if not specified unless DisableRequestTimeout is true.
Timeout time.Duration
// DisableRequestTimeout leaves the underlying request client without a
// response timeout. Callers must cancel through the request context.
DisableRequestTimeout bool
// Retry configuration
// MaxRetries is the maximum number of retry attempts for transient errors.
// Default is 3 if not specified.
MaxRetries int
// InitialInterval is the initial backoff interval before the first retry.
// Default is 1 second if not specified.
InitialInterval time.Duration
// MaxInterval is the maximum backoff interval between retries.
// Default is 30 seconds if not specified.
MaxInterval time.Duration
// Multiplier is the factor by which the interval increases after each retry.
// Default is 2.0 if not specified.
Multiplier float64
}
Config contains configuration for an LLM provider.
func DefaultConfig ¶
func DefaultConfig() Config
DefaultConfig returns a Config with sensible defaults.
type HTTPClient ¶
type HTTPClient struct {
// contains filtered or unexported fields
}
HTTPClient performs HTTP requests with retry logic. Uses plain net/http instead of resty to ensure response bodies are properly closed on retries (resty + SetDoNotParseResponse leaks FDs).
func NewHTTPClient ¶
func NewHTTPClient(cfg Config) *HTTPClient
NewHTTPClient creates a new HTTP client with the given configuration. Each client gets its own http.Transport to avoid sharing connection state across unrelated providers.
func (*HTTPClient) Do ¶
func (c *HTTPClient) Do(ctx context.Context, url string, body []byte, headers map[string]string) (io.ReadCloser, error)
Do performs an HTTP POST request with retry logic. Returns the response body as an io.ReadCloser for streaming support. Retries on network errors, rate limiting, and transient server errors.
type Image ¶ added in v2.18.0
type Image struct {
// MediaType is the image's IANA media type, such as "image/png".
MediaType string `json:"media_type"`
// Data holds the encoded image bytes.
Data []byte `json:"data"`
}
Image is an encoded image attached to a message.
type LogicalRetryConfig ¶
type LogicalRetryConfig struct {
MaxAttempts int
InitialInterval time.Duration
MaxInterval time.Duration
Multiplier float64
}
LogicalRetryConfig configures retries for a single logical LLM request. This sits above the provider's own HTTP retries and is intended for transient failures that happen around or after a successful HTTP exchange, such as interrupted body reads or response decode failures.
func DefaultLogicalRetryConfig ¶
func DefaultLogicalRetryConfig() LogicalRetryConfig
DefaultLogicalRetryConfig returns the standard logical request retry budget.
type Message ¶
type Message struct {
// Role identifies who sent the message (system, user, assistant, or tool).
Role Role `json:"role"`
// Content is the text content of the message.
Content string `json:"content"`
// ReasoningContent holds the model's internal reasoning/thinking text from a
// previous assistant turn. Some providers (e.g. Moonshot/KIMI via OpenCode)
// require this to be echoed back in subsequent turns when reasoning is enabled.
ReasoningContent string `json:"reasoning_content,omitempty"`
// Name is an optional identifier for the message sender.
// Useful in multi-agent scenarios or for tool messages.
Name string `json:"name,omitempty"`
// ToolCallID is the ID of the tool call this message is responding to.
// Required when Role is "tool".
ToolCallID string `json:"tool_call_id,omitempty"`
// ToolCalls contains tool calls made by the assistant.
// Only set when Role is "assistant" and the model requests tool calls.
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
// Images are sent with a user message ahead of its text. Other roles
// do not carry images.
Images []Image `json:"images,omitempty"`
// ProviderState, when set on an assistant message, is the provider's own
// record of that turn, taken from ChatResponse.ProviderState.
ProviderState *ProviderState `json:"-"`
}
Message represents a single message in a session.
type OAuthCredential ¶
OAuthCredential contains dynamic bearer-auth details resolved at request time.
type OAuthCredentialProvider ¶
type OAuthCredentialProvider interface {
ResolveCredential(ctx context.Context) (OAuthCredential, error)
}
OAuthCredentialProvider resolves dynamic bearer credentials for providers that cannot safely persist a static API key inside model configuration.
type Option ¶
type Option func(*Config)
Option is a functional option for configuring an LLM provider.
func WithAPIKey ¶
WithAPIKey sets the API key for the provider.
func WithBackoff ¶
WithBackoff sets the backoff configuration for retries.
func WithBaseURL ¶
WithBaseURL sets the base URL for the provider.
func WithMaxRetries ¶
WithMaxRetries sets the maximum number of retry attempts.
func WithTimeout ¶
WithTimeout sets the request timeout.
type Provider ¶
type Provider interface {
// Chat sends messages and returns the complete response.
Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error)
// ChatStream sends messages and streams the response.
// Returns a channel that receives StreamEvents until Done is true or an error occurs.
ChatStream(ctx context.Context, req *ChatRequest) (<-chan StreamEvent, error)
// Name returns the provider name for logging and error messages.
Name() string
}
Provider is the interface that all LLM providers must implement.
func NewProvider ¶
func NewProvider(providerType ProviderType, cfg Config) (Provider, error)
NewProvider creates a new Provider instance based on the provider type and configuration.
func NewProviderFromEnv ¶
func NewProviderFromEnv(providerType ProviderType) (Provider, error)
NewProviderFromEnv creates a new Provider using API key from environment. This is a convenience function for simple use cases.
func NewProviderWithAPIKey ¶
func NewProviderWithAPIKey(providerType ProviderType, apiKey string) (Provider, error)
NewProviderWithAPIKey creates a new Provider with the given API key. This is a convenience function for simple use cases.
type ProviderFactory ¶
ProviderFactory is a function type that creates a new Provider instance.
type ProviderState ¶ added in v2.18.2
type ProviderState struct {
// Provider is the provider that produced the record.
Provider ProviderType
// Data is the record in the provider's own format.
Data json.RawMessage
}
ProviderState is a provider's own record of an assistant turn, such as content that carries signed reasoning. The provider named by Provider sends the turn back from it in place of Content and ToolCalls; other providers ignore it. The record belongs to one conversation and is not persisted.
type ProviderType ¶
type ProviderType string
ProviderType identifies the LLM provider.
const ( // ProviderOpenAI is the OpenAI provider (GPT models). ProviderOpenAI ProviderType = "openai" // ProviderOpenAICodex is the ChatGPT/Codex subscription provider. ProviderOpenAICodex ProviderType = "openai-codex" // ProviderAnthropic is the Anthropic provider (Claude models). ProviderAnthropic ProviderType = "anthropic" // ProviderGemini is the Google Gemini provider. ProviderGemini ProviderType = "gemini" // ProviderOpenRouter is the OpenRouter provider (multi-model gateway). ProviderOpenRouter ProviderType = "openrouter" // ProviderLocal is for local OpenAI-compatible servers (Ollama, vLLM, etc). ProviderLocal ProviderType = "local" // ProviderZAI is the Z.AI provider (GLM models). ProviderZAI ProviderType = "zai" // ProviderOpenCode is the OpenCode provider (Kimi, DeepSeek, GLM, etc. via opencode.ai). ProviderOpenCode ProviderType = "opencode" )
func ParseProviderType ¶
func ParseProviderType(s string) (ProviderType, error)
ParseProviderType converts a string to a ProviderType.
type RequestOption ¶
type RequestOption func(*ChatRequest)
RequestOption is a functional option for configuring a ChatRequest.
func WithMaxTokens ¶
func WithMaxTokens(tokens int) RequestOption
WithMaxTokens sets the maximum tokens for the request.
func WithStop ¶
func WithStop(stop ...string) RequestOption
WithStop sets the stop sequences for the request.
func WithTemperature ¶
func WithTemperature(temp float64) RequestOption
WithTemperature sets the temperature for the request.
func WithTopP ¶
func WithTopP(topP float64) RequestOption
WithTopP sets the top_p (nucleus sampling) parameter.
type Role ¶
type Role string
Role represents the role of a message sender in a session.
const ( // RoleSystem represents a system message that sets behavior/context. RoleSystem Role = "system" // RoleUser represents a message from the user. RoleUser Role = "user" // RoleAssistant represents a message from the AI assistant. RoleAssistant Role = "assistant" // RoleTool represents a tool/function call result. RoleTool Role = "tool" )
type StreamEvent ¶
type StreamEvent struct {
// Delta is the incremental content received in this event.
Delta string
// Done indicates whether the stream has completed.
Done bool
// Error contains any error that occurred during streaming.
Error error
// Usage contains token usage statistics (only set when Done is true).
Usage *Usage
// ToolCalls contains tool calls accumulated during streaming.
// Only populated when Done is true and the model requested tool calls.
ToolCalls []ToolCall
// FinishReason indicates why the model stopped (only set when Done is true).
FinishReason string
}
StreamEvent represents a single event in a streaming response.
type ThinkingEffort ¶
type ThinkingEffort string
ThinkingEffort represents the reasoning depth level for thinking mode. Each provider maps a level it lacks to its nearest supported one.
const ( // ThinkingEffortNone turns thinking off where the provider allows it. ThinkingEffortNone ThinkingEffort = "none" // ThinkingEffortMinimal provides the least reasoning short of none. ThinkingEffortMinimal ThinkingEffort = "minimal" // ThinkingEffortLow provides quick reasoning with minimal token budget. ThinkingEffortLow ThinkingEffort = "low" // ThinkingEffortMedium provides balanced reasoning (default). ThinkingEffortMedium ThinkingEffort = "medium" // ThinkingEffortHigh provides thorough analysis with larger token budget. ThinkingEffortHigh ThinkingEffort = "high" // ThinkingEffortXHigh provides deeper reasoning than high. ThinkingEffortXHigh ThinkingEffort = "xhigh" // ThinkingEffortMax provides maximum reasoning depth. ThinkingEffortMax ThinkingEffort = "max" )
func ParseThinkingEffort ¶
func ParseThinkingEffort(s string) (ThinkingEffort, error)
ParseThinkingEffort validates and returns a ThinkingEffort from a string. Returns empty string for empty input (no effort specified).
type ThinkingRequest ¶
type ThinkingRequest struct {
// Enabled activates thinking mode.
Enabled bool
// Effort controls reasoning depth: none, minimal, low, medium, high,
// xhigh, max.
Effort ThinkingEffort
// BudgetTokens sets explicit token budget (provider-specific).
BudgetTokens *int
// IncludeInOutput includes thinking blocks in the response content.
IncludeInOutput bool
}
ThinkingRequest contains configuration for extended thinking/reasoning mode. Each provider maps this to their native format.
type Tool ¶
type Tool struct {
// Type is always "function" for function tools.
Type string `json:"type"`
// Function contains the function definition.
Function ToolFunction `json:"function"`
}
Tool represents a function/tool available to the LLM.
type ToolCall ¶
type ToolCall struct {
// ID is a unique identifier for this tool call.
ID string `json:"id"`
// Type is always "function" for function calls.
Type string `json:"type"`
// Function contains the function call details.
Function ToolCallFunction `json:"function"`
}
ToolCall represents an LLM's request to call a tool.
type ToolCallFunction ¶
type ToolCallFunction struct {
// Name is the name of the function to call.
Name string `json:"name"`
// Arguments is a JSON string containing the function arguments.
Arguments string `json:"arguments"`
}
ToolCallFunction contains the details of a function call.
type ToolFunction ¶
type ToolFunction struct {
// Name is the function name (must match [a-zA-Z0-9_-]+).
Name string `json:"name"`
// Description explains what the function does.
Description string `json:"description"`
// Parameters is a JSON Schema describing the function parameters.
Parameters map[string]any `json:"parameters"`
}
ToolFunction describes a callable function.
type Usage ¶
type Usage struct {
// PromptTokens is the number of tokens in the prompt.
PromptTokens int `json:"prompt_tokens"`
// CompletionTokens is the number of tokens in the completion.
CompletionTokens int `json:"completion_tokens"`
// TotalTokens is the sum of prompt and completion tokens.
TotalTokens int `json:"total_tokens"`
}
Usage contains token usage information from an LLM API call.
type UserLocation ¶
type UserLocation struct {
// City is the city name.
City string
// Region is the region or state.
Region string
// Country is the ISO country code (e.g., "US").
Country string
// Timezone is the IANA timezone (e.g., "America/New_York").
Timezone string
}
UserLocation provides approximate location for search result localization.
type WebSearchRequest ¶
type WebSearchRequest struct {
// Enabled activates provider-native web search.
Enabled bool
// MaxUses limits search invocations per request.
// Anthropic: maps to max_uses. OpenRouter: maps to max_results.
MaxUses *int
// AllowedDomains restricts results to these domains (Anthropic only).
AllowedDomains []string
// BlockedDomains excludes results from these domains (Anthropic only).
BlockedDomains []string
// UserLocation localizes search results.
UserLocation *UserLocation
}
WebSearchRequest contains configuration for provider-native web search. Each provider maps this to its native format during request building.
Source Files
¶
Directories
¶
| Path | Synopsis |
|---|---|
|
Package allproviders imports all LLM providers to register them.
|
Package allproviders imports all LLM providers to register them. |
|
Package claudemodel reads Claude model IDs as the providers that serve Claude spell them, to tell what a model accepts.
|
Package claudemodel reads Claude model IDs as the providers that serve Claude spell them, to tell what a model accepts. |
|
Package computeruse lets a model operate a desktop through screenshots and pointer and keyboard actions.
|
Package computeruse lets a model operate a desktop through screenshots and pointer and keyboard actions. |
|
Package llmtest provides test doubles for LLM provider APIs.
|
Package llmtest provides test doubles for LLM provider APIs. |
|
providers
|
|
|
anthropic
Package anthropic provides an LLM provider implementation for Anthropic's Claude API.
|
Package anthropic provides an LLM provider implementation for Anthropic's Claude API. |
|
chatcontent
Package chatcontent builds message content for OpenAI-compatible chat completion APIs.
|
Package chatcontent builds message content for OpenAI-compatible chat completion APIs. |
|
gemini
Package gemini provides an LLM provider implementation for Google's Gemini API.
|
Package gemini provides an LLM provider implementation for Google's Gemini API. |
|
local
Package local provides an LLM provider implementation for local OpenAI-compatible servers.
|
Package local provides an LLM provider implementation for local OpenAI-compatible servers. |
|
openai
Package openai provides an LLM provider implementation for OpenAI's API.
|
Package openai provides an LLM provider implementation for OpenAI's API. |
|
openrouter
Package openrouter provides an LLM provider implementation for OpenRouter's API.
|
Package openrouter provides an LLM provider implementation for OpenRouter's API. |
|
zai
Package zai provides an LLM provider implementation for Z.AI's API.
|
Package zai provides an LLM provider implementation for Z.AI's API. |
|
Package toolschema derives LLM function-calling parameter schemas from DAG parameter definitions.
|
Package toolschema derives LLM function-calling parameter schemas from DAG parameter definitions. |