Documentation
¶
Overview ¶
Package model defines pluggable LLM provider interfaces.
Index ¶
- Constants
- func ContextLimit(modelName string, fallback int) int
- type Anthropic
- type AzureConfig
- type AzureOpenAI
- type CachedEmbeddings
- type ChatRequest
- type ChatResponse
- type EmbeddingRequest
- type EmbeddingResponse
- type EmbeddingsProvider
- type EstimatingCounter
- type FallbackProvider
- type FunctionDef
- type Gemini
- type Message
- type Mistral
- type Ollama
- type OllamaEmbeddings
- type OpenAI
- type OpenAICompatible
- func NewAnyscale(apiKey, modelID string) *OpenAICompatible
- func NewDeepSeek(apiKey, modelID string) *OpenAICompatible
- func NewFireworks(apiKey, modelID string) *OpenAICompatible
- func NewGroq(apiKey, modelID string) *OpenAICompatible
- func NewOpenAICompatible(name, baseURL, apiKey, modelID string) *OpenAICompatible
- func NewOpenAICompatibleWithConfig(name string, cfg ProviderConfig) *OpenAICompatible
- func NewOpenRouter(apiKey, modelID string) *OpenAICompatible
- func NewPerplexity(apiKey, modelID string) *OpenAICompatible
- func NewTogether(apiKey, modelID string) *OpenAICompatible
- type OpenAIEmbeddings
- type Provider
- type ProviderConfig
- type StopReason
- type SummarizationConfig
- type SummarizeResult
- type Summarizer
- type TokenCounter
- type ToolCall
- type ToolDefinition
- type Usage
Constants ¶
const ( RoleSystem = "system" RoleUser = "user" RoleAssistant = "assistant" RoleTool = "tool" )
Role constants for chat messages.
Variables ¶
This section is empty.
Functions ¶
func ContextLimit ¶
ContextLimit returns the maximum context window (in tokens) for a model. If the model is unknown, it returns the provided fallback value.
Types ¶
type Anthropic ¶
type Anthropic struct {
// contains filtered or unexported fields
}
Anthropic implements Provider for Claude models via the Anthropic Messages API.
func NewAnthropic ¶
NewAnthropic creates a new Anthropic provider with the given API key.
func NewAnthropicWithConfig ¶
func NewAnthropicWithConfig(cfg ProviderConfig) *Anthropic
NewAnthropicWithConfig creates an Anthropic provider with full configuration.
func (*Anthropic) Chat ¶
func (a *Anthropic) Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error)
func (*Anthropic) StreamChat ¶
func (a *Anthropic) StreamChat(ctx context.Context, req *ChatRequest) (<-chan *ChatResponse, error)
type AzureConfig ¶
type AzureConfig struct {
ProviderConfig
Deployment string `json:"deployment"`
APIVersion string `json:"api_version"`
}
AzureConfig extends ProviderConfig with Azure-specific fields.
type AzureOpenAI ¶
type AzureOpenAI struct {
// contains filtered or unexported fields
}
AzureOpenAI implements Provider for Azure-hosted OpenAI models. Uses the Azure-specific endpoint format: {base}/openai/deployments/{deployment}/chat/completions?api-version={version}
func NewAzureOpenAI ¶
func NewAzureOpenAI(endpoint, apiKey, deployment string) *AzureOpenAI
NewAzureOpenAI creates a new Azure OpenAI provider. endpoint is the Azure resource endpoint (e.g., "https://myresource.openai.azure.com"). apiKey is the Azure API key. deployment is the model deployment name.
func NewAzureOpenAIWithConfig ¶
func NewAzureOpenAIWithConfig(cfg AzureConfig) *AzureOpenAI
NewAzureOpenAIWithConfig creates an Azure OpenAI provider with full configuration.
func (*AzureOpenAI) Chat ¶
func (a *AzureOpenAI) Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error)
func (*AzureOpenAI) Model ¶
func (a *AzureOpenAI) Model() string
func (*AzureOpenAI) Name ¶
func (a *AzureOpenAI) Name() string
func (*AzureOpenAI) StreamChat ¶
func (a *AzureOpenAI) StreamChat(ctx context.Context, req *ChatRequest) (<-chan *ChatResponse, error)
type CachedEmbeddings ¶
type CachedEmbeddings struct {
// contains filtered or unexported fields
}
CachedEmbeddings wraps an EmbeddingsProvider with an in-memory cache.
func NewCachedEmbeddings ¶
func NewCachedEmbeddings(inner EmbeddingsProvider) *CachedEmbeddings
func (*CachedEmbeddings) Embed ¶
func (c *CachedEmbeddings) Embed(ctx context.Context, req *EmbeddingRequest) (*EmbeddingResponse, error)
type ChatRequest ¶
type ChatRequest struct {
Model string `json:"model"`
Messages []Message `json:"messages"`
MaxTokens int `json:"max_tokens,omitempty"`
Temperature float64 `json:"temperature,omitempty"`
TopP float64 `json:"top_p,omitempty"`
Stream bool `json:"stream,omitempty"`
Tools []ToolDefinition `json:"tools,omitempty"`
Stop []string `json:"stop,omitempty"`
// ResponseFormat controls output format. Values:
// "json_object" — model returns valid JSON (no schema enforcement)
// "json_schema" — model returns JSON conforming to Metadata["json_schema"]
ResponseFormat string `json:"response_format,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
ChatRequest is the input to a chat completion.
type ChatResponse ¶
type ChatResponse struct {
ID string `json:"id"`
Content string `json:"content"`
Role string `json:"role"`
Usage Usage `json:"usage"`
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
StopReason StopReason `json:"stop_reason,omitempty"`
// Delta is true when this is a partial streaming response.
Delta bool `json:"delta,omitempty"`
}
ChatResponse is the output of a chat completion.
type EmbeddingRequest ¶
EmbeddingRequest is input for embedding generation.
type EmbeddingResponse ¶
type EmbeddingResponse struct {
Embeddings [][]float32 `json:"embeddings"`
Usage Usage `json:"usage"`
}
EmbeddingResponse contains the generated embeddings.
type EmbeddingsProvider ¶
type EmbeddingsProvider interface {
Embed(ctx context.Context, req *EmbeddingRequest) (*EmbeddingResponse, error)
}
EmbeddingsProvider generates vector embeddings from text.
type EstimatingCounter ¶
type EstimatingCounter struct {
CharsPerToken float64
}
EstimatingCounter uses a character-ratio heuristic (1 token ~ 4 chars) to estimate token counts without external dependencies.
func NewEstimatingCounter ¶
func NewEstimatingCounter() *EstimatingCounter
NewEstimatingCounter returns a counter using the default 4-chars-per-token ratio.
func (*EstimatingCounter) CountString ¶
func (c *EstimatingCounter) CountString(s string) int
func (*EstimatingCounter) CountTokens ¶
func (c *EstimatingCounter) CountTokens(messages []Message) int
type FallbackProvider ¶
type FallbackProvider struct {
// OnFallback is called when a provider fails and the next one is tried.
// It receives the failed provider index, its name, and the error.
OnFallback func(index int, name string, err error)
// contains filtered or unexported fields
}
FallbackProvider wraps multiple providers and tries them in order. If the primary provider fails, it automatically falls back to the next one. This is useful for scenarios like primary-cloud -> cheaper-model or cloud-provider -> local-ollama.
func NewFallbackProvider ¶
func NewFallbackProvider(providers ...Provider) (*FallbackProvider, error)
NewFallbackProvider creates a fallback provider from the given providers. At least one provider is required.
func (*FallbackProvider) Chat ¶
func (f *FallbackProvider) Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error)
Chat tries each provider in order until one succeeds.
func (*FallbackProvider) Model ¶
func (f *FallbackProvider) Model() string
func (*FallbackProvider) Name ¶
func (f *FallbackProvider) Name() string
func (*FallbackProvider) StreamChat ¶
func (f *FallbackProvider) StreamChat(ctx context.Context, req *ChatRequest) (<-chan *ChatResponse, error)
StreamChat tries each provider in order until one succeeds.
type FunctionDef ¶
type FunctionDef struct {
Name string `json:"name"`
Description string `json:"description"`
Parameters map[string]any `json:"parameters"` // JSON Schema
}
FunctionDef describes a callable function for the model.
type Gemini ¶
type Gemini struct {
// contains filtered or unexported fields
}
Gemini implements Provider for Google's Gemini API.
func NewGeminiWithConfig ¶
func NewGeminiWithConfig(cfg ProviderConfig) *Gemini
NewGeminiWithConfig creates a Gemini provider with full configuration.
func (*Gemini) Chat ¶
func (g *Gemini) Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error)
func (*Gemini) StreamChat ¶
func (g *Gemini) StreamChat(ctx context.Context, req *ChatRequest) (<-chan *ChatResponse, error)
type Message ¶
type Message struct {
Role string `json:"role"` // system, user, assistant, tool
Content string `json:"content"`
Name string `json:"name,omitempty"`
ToolCallID string `json:"tool_call_id,omitempty"`
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
}
Message represents a chat message.
type Mistral ¶
type Mistral struct {
// contains filtered or unexported fields
}
Mistral implements Provider for the Mistral AI API. Mistral uses an OpenAI-compatible chat completions format.
func NewMistral ¶
NewMistral creates a new Mistral provider with the given API key.
func NewMistralWithConfig ¶
func NewMistralWithConfig(cfg ProviderConfig) *Mistral
NewMistralWithConfig creates a Mistral provider with full configuration.
func (*Mistral) Chat ¶
func (m *Mistral) Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error)
func (*Mistral) StreamChat ¶
func (m *Mistral) StreamChat(ctx context.Context, req *ChatRequest) (<-chan *ChatResponse, error)
type Ollama ¶
type Ollama struct {
// contains filtered or unexported fields
}
Ollama implements Provider for locally running Ollama models. Ollama exposes an OpenAI-compatible API at /v1/chat/completions.
func NewOllama ¶
NewOllama creates a new Ollama provider pointing at a local instance. host is the Ollama server address (e.g., "http://localhost:11434"). modelName is the model tag (e.g., "llama3.2", "mistral", "codellama").
func NewOllamaWithConfig ¶
func NewOllamaWithConfig(cfg ProviderConfig) *Ollama
NewOllamaWithConfig creates an Ollama provider with full configuration.
func (*Ollama) Chat ¶
func (o *Ollama) Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error)
func (*Ollama) StreamChat ¶
func (o *Ollama) StreamChat(ctx context.Context, req *ChatRequest) (<-chan *ChatResponse, error)
type OllamaEmbeddings ¶
type OllamaEmbeddings struct {
// contains filtered or unexported fields
}
OllamaEmbeddings implements EmbeddingsProvider using a local Ollama server.
func NewOllamaEmbeddings ¶
func NewOllamaEmbeddings(baseURL, modelID string) *OllamaEmbeddings
NewOllamaEmbeddings creates an Ollama embeddings provider.
func (*OllamaEmbeddings) Embed ¶
func (o *OllamaEmbeddings) Embed(ctx context.Context, req *EmbeddingRequest) (*EmbeddingResponse, error)
type OpenAI ¶
type OpenAI struct {
// contains filtered or unexported fields
}
OpenAI implements Provider for OpenAI chat completion endpoints. Works with GPT-4, GPT-4o, GPT-3.5-turbo, o1, o3, and any OpenAI-compatible API.
func NewOpenAIWithConfig ¶
func NewOpenAIWithConfig(cfg ProviderConfig) *OpenAI
NewOpenAIWithConfig creates an OpenAI provider with full configuration.
func (*OpenAI) Chat ¶
func (o *OpenAI) Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error)
func (*OpenAI) StreamChat ¶
func (o *OpenAI) StreamChat(ctx context.Context, req *ChatRequest) (<-chan *ChatResponse, error)
type OpenAICompatible ¶
type OpenAICompatible struct {
// contains filtered or unexported fields
}
OpenAICompatible implements Provider for any OpenAI-compatible API endpoint. Use this for self-hosted models (vLLM, TGI, LiteLLM), cloud providers that expose an OpenAI-compatible interface (Together, Anyscale, Fireworks, Groq, DeepSeek, OpenRouter, Perplexity), or any custom LLM server.
func NewAnyscale ¶
func NewAnyscale(apiKey, modelID string) *OpenAICompatible
NewAnyscale creates a provider for Anyscale Endpoints.
func NewDeepSeek ¶
func NewDeepSeek(apiKey, modelID string) *OpenAICompatible
NewDeepSeek creates a provider for DeepSeek.
func NewFireworks ¶
func NewFireworks(apiKey, modelID string) *OpenAICompatible
NewFireworks creates a provider for Fireworks AI.
func NewGroq ¶
func NewGroq(apiKey, modelID string) *OpenAICompatible
NewGroq creates a provider for Groq.
func NewOpenAICompatible ¶
func NewOpenAICompatible(name, baseURL, apiKey, modelID string) *OpenAICompatible
NewOpenAICompatible creates a provider for any OpenAI-compatible endpoint. name is a human-readable identifier (e.g., "vllm", "together", "deepseek"). baseURL is the API base (e.g., "https://api.together.xyz/v1"). apiKey is the authentication key. modelID is the model identifier (e.g., "meta-llama/Llama-3.1-70B-Instruct").
func NewOpenAICompatibleWithConfig ¶
func NewOpenAICompatibleWithConfig(name string, cfg ProviderConfig) *OpenAICompatible
NewOpenAICompatibleWithConfig creates an OpenAI-compatible provider with full config.
func NewOpenRouter ¶
func NewOpenRouter(apiKey, modelID string) *OpenAICompatible
NewOpenRouter creates a provider for OpenRouter.
func NewPerplexity ¶
func NewPerplexity(apiKey, modelID string) *OpenAICompatible
NewPerplexity creates a provider for Perplexity.
func NewTogether ¶
func NewTogether(apiKey, modelID string) *OpenAICompatible
NewTogether creates a provider for Together AI.
func (*OpenAICompatible) Chat ¶
func (c *OpenAICompatible) Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error)
func (*OpenAICompatible) Model ¶
func (c *OpenAICompatible) Model() string
func (*OpenAICompatible) Name ¶
func (c *OpenAICompatible) Name() string
func (*OpenAICompatible) StreamChat ¶
func (c *OpenAICompatible) StreamChat(ctx context.Context, req *ChatRequest) (<-chan *ChatResponse, error)
type OpenAIEmbeddings ¶
type OpenAIEmbeddings struct {
// contains filtered or unexported fields
}
OpenAIEmbeddings implements EmbeddingsProvider using the OpenAI embeddings API.
func NewOpenAIEmbeddings ¶
func NewOpenAIEmbeddings(apiKey string) *OpenAIEmbeddings
NewOpenAIEmbeddings creates an OpenAI embeddings provider with the given API key.
func NewOpenAIEmbeddingsWithConfig ¶
func NewOpenAIEmbeddingsWithConfig(cfg ProviderConfig) *OpenAIEmbeddings
NewOpenAIEmbeddingsWithConfig creates an OpenAI embeddings provider with full config.
func (*OpenAIEmbeddings) Embed ¶
func (o *OpenAIEmbeddings) Embed(ctx context.Context, req *EmbeddingRequest) (*EmbeddingResponse, error)
type Provider ¶
type Provider interface {
// Chat sends a request and returns a complete response.
Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error)
// StreamChat returns a channel of partial responses for streaming.
// The channel is closed when the response is complete.
StreamChat(ctx context.Context, req *ChatRequest) (<-chan *ChatResponse, error)
// Name returns a human-readable name for this provider.
Name() string
// Model returns the default model ID for this provider.
Model() string
}
Provider is the interface all LLM backends must implement.
type ProviderConfig ¶
type ProviderConfig struct {
APIKey string `json:"api_key"`
BaseURL string `json:"base_url,omitempty"`
Model string `json:"model"`
MaxRetries int `json:"max_retries,omitempty"`
TimeoutSec int `json:"timeout_sec,omitempty"`
OrgID string `json:"org_id,omitempty"`
ContextWindow int `json:"context_window,omitempty"` // override default context window size for the model
}
ProviderConfig holds common configuration shared by all providers.
type StopReason ¶
type StopReason string
StopReason indicates why the model stopped generating.
const ( StopReasonEnd StopReason = "end" StopReasonMaxTokens StopReason = "max_tokens" StopReasonToolCall StopReason = "tool_call" StopReasonFilter StopReason = "content_filter" )
type SummarizationConfig ¶
type SummarizationConfig struct {
// Threshold is the fraction of the context window that triggers summarization (default 0.8).
Threshold float64
// PreserveRecentTurns is the number of recent user/assistant message pairs to keep
// unsummarized so the model retains immediate conversational context (default 5).
PreserveRecentTurns int
// MaxSummaryTokens caps the length of the generated summary (default 1024).
MaxSummaryTokens int
// Prompt overrides the default summarization system prompt.
Prompt string
}
SummarizationConfig controls when and how conversation summarization occurs.
type SummarizeResult ¶
type SummarizeResult struct {
Summary string // the new rolling summary
PreservedMessages []Message // recent messages kept intact
}
SummarizeResult holds the output of a summarization pass.
type Summarizer ¶
type Summarizer struct {
// contains filtered or unexported fields
}
Summarizer produces rolling summaries of conversation history using an LLM.
func NewSummarizer ¶
func NewSummarizer(provider Provider, counter TokenCounter, cfg SummarizationConfig) *Summarizer
NewSummarizer creates a summarizer backed by the given provider and token counter.
func (*Summarizer) NeedsSummarization ¶
func (s *Summarizer) NeedsSummarization(systemTokens int, messages []Message, contextLimit int) bool
NeedsSummarization returns true when the estimated token count of the messages (plus system context) exceeds the configured threshold of the context window.
func (*Summarizer) Summarize ¶
func (s *Summarizer) Summarize(ctx context.Context, priorSummary string, messages []Message) (*SummarizeResult, error)
Summarize compresses older messages into a rolling summary, preserving the most recent turns. If priorSummary is non-empty it is incorporated into the new summary so no context is lost across multiple summarization passes.
type TokenCounter ¶
TokenCounter estimates the token count for a set of messages.
type ToolCall ¶
type ToolCall struct {
ID string `json:"id"`
Name string `json:"name"`
Arguments string `json:"arguments"` // JSON string
}
ToolCall represents a model-requested tool invocation.
type ToolDefinition ¶
type ToolDefinition struct {
Type string `json:"type"` // "function"
Function FunctionDef `json:"function"`
}
ToolDefinition describes a tool the model can call (OpenAI function-calling format).