llm

package
v0.16.4 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 6, 2026 License: Apache-2.0 Imports: 13 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func TokenCost added in v0.15.0

func TokenCost(resp *ChatResponse, reg *pricing.Registry) (float64, string)

TokenCost returns the USD cost for a chat response using the pricing registry. Priority: provider-reported cost > registry lookup > fallback rate > $0. Also returns a pricing source string ("provider"/"registry"/"fallback"/"unknown").

Types

type AgentStats added in v0.12.0

type AgentStats struct {
	Agent        string  `json:"agent"`
	Cost         float64 `json:"cost"`
	InputTokens  int     `json:"input_tokens"`
	OutputTokens int     `json:"output_tokens"`
	Messages     int     `json:"messages"`
	Sessions     int     `json:"sessions"`
}

AgentStats holds aggregated per-agent cost and token data.

type BalanceProvider

type BalanceProvider interface {
	FundsRemaining(ctx context.Context) (float64, error)
}

BalanceProvider is an optional interface implemented by providers that can report their remaining credit balance. Returns -1 if the balance is unlimited.

type ChatRequest

type ChatRequest struct {
	Model       string
	Messages    []Message
	MaxTokens   int
	Temperature *float64
	Tools       []ToolDef
}

type ChatResponse

type ChatResponse struct {
	Content      string
	ToolCalls    []ToolCall
	TokensUsed   TokenUsage
	Model        string
	FinishReason string
	CostUSD      float64 // provider-reported or estimated cost in USD
}

type CostTracker

type CostTracker struct {
	// contains filtered or unexported fields
}

CostTracker tracks token usage and estimated costs per session and globally.

func NewCostTracker

func NewCostTracker(maxPerSession float64) *CostTracker

func (*CostTracker) AgentCosts added in v0.12.0

func (ct *CostTracker) AgentCosts() []AgentStats

AgentCosts returns per-agent aggregated stats. Agent name is extracted from session IDs which have the format "agentname:adapter:externalid".

func (*CostTracker) AllSessionCosts

func (ct *CostTracker) AllSessionCosts() map[string]float64

AllSessionCosts returns a copy of all session costs.

func (*CostTracker) AllSessionStats added in v0.12.0

func (ct *CostTracker) AllSessionStats() map[string]SessionStats

AllSessionStats returns a copy of all session stats.

func (*CostTracker) ExceedsBudget

func (ct *CostTracker) ExceedsBudget(sessionID string) bool

ExceedsBudget checks if a session has exceeded its budget.

func (*CostTracker) GlobalCost

func (ct *CostTracker) GlobalCost() float64

GlobalCost returns the total cost across all sessions.

func (*CostTracker) MaxBudgetPerSession

func (ct *CostTracker) MaxBudgetPerSession() float64

MaxBudgetPerSession returns the per-session cost cap.

func (*CostTracker) Record

func (ct *CostTracker) Record(sessionID string, cost float64) bool

Record adds cost for a session. Returns true if within budget.

func (*CostTracker) RecordWithTokens added in v0.12.0

func (ct *CostTracker) RecordWithTokens(sessionID string, cost float64, inputTokens, outputTokens int, pricingSource ...string) bool

RecordWithTokens adds cost and token usage for a session. Returns true if within budget. The optional pricingSource parameter records which pricing method was used.

func (*CostTracker) SessionCost

func (ct *CostTracker) SessionCost(sessionID string) float64

SessionCost returns the total cost for a session.

type FallbackRule

type FallbackRule struct {
	Trigger    string  // "error" | "rate_limit" | "low_funds"
	Action     string  // "switch_provider" | "switch_model" | "wait_and_retry"
	Provider   string  // provider name — for switch_provider
	Model      string  // model name — for switch_model; optional for switch_provider
	Threshold  float64 // remaining credit threshold in USD — for low_funds
	MaxRetries int     // number of retry attempts — for wait_and_retry
	Backoff    string  // "exponential" | "constant" — for wait_and_retry
}

FallbackRule describes a single fallback step the router will attempt.

type FunctionCall

type FunctionCall struct {
	Name      string `json:"name"`
	Arguments string `json:"arguments"`
}

FunctionCall is the function name and JSON-encoded arguments within a ToolCall.

type FunctionDef

type FunctionDef struct {
	Name        string         `json:"name"`
	Description string         `json:"description"`
	Parameters  map[string]any `json:"parameters"` // JSON Schema object
}

FunctionDef describes the function signature within a ToolDef.

type LLMError

type LLMError struct {
	StatusCode int
	Message    string
}

LLMError is returned by providers for API-level failures. Use errors.As to unwrap from wrapped errors.

func (*LLMError) Error

func (e *LLMError) Error() string

func (*LLMError) Retryable

func (e *LLMError) Retryable() bool

Retryable reports whether the error is worth retrying. Non-retryable: 400 (bad request), 401 (auth), 402 (payment), 404 (not found), 422 (unprocessable). Retryable: 429 (rate limit), 5xx (server errors), and any unrecognised status.

type Message

type Message struct {
	Role       string     `json:"role"`
	Content    string     `json:"content"`
	ToolCalls  []ToolCall `json:"tool_calls,omitempty"`
	ToolCallID string     `json:"tool_call_id,omitempty"`
}

type ModelLister added in v0.15.1

type ModelLister interface {
	ListModels(ctx context.Context) ([]string, error)
}

ModelLister is an optional interface implemented by providers that can enumerate their available models.

type Provider

type Provider interface {
	ChatCompletion(ctx context.Context, req ChatRequest) (*ChatResponse, error)
	Name() string
	HealthCheck(ctx context.Context) error
}

Provider defines the interface for LLM backends.

type ProviderMetadata

type ProviderMetadata struct {
	Name    string
	BaseURL string
	Models  []string
}

type Router

type Router struct {
	// contains filtered or unexported fields
}

Router selects the appropriate LLM provider for a request.

func NewRouter

func NewRouter(defaultProvider, defaultModel string, costTracker *CostTracker) *Router

func (*Router) Complete

func (r *Router) Complete(ctx context.Context, sessionID string, messages []Message) (*ChatResponse, error)

func (*Router) DefaultModel

func (r *Router) DefaultModel() string

DefaultModel returns the router's default model name.

func (*Router) HealthCheck

func (r *Router) HealthCheck(ctx context.Context) error

func (*Router) ListModels added in v0.15.1

func (r *Router) ListModels(ctx context.Context) []string

ListModels queries all registered providers that implement ModelLister and returns a de-duplicated sorted list of available model names.

func (*Router) RegisterProvider

func (r *Router) RegisterProvider(p Provider)

func (*Router) SetDefaultModel added in v0.11.0

func (r *Router) SetDefaultModel(model string)

SetDefaultModel changes the router's default model for subsequent requests.

func (*Router) SetFallbacks

func (r *Router) SetFallbacks(rules []FallbackRule)

SetFallbacks configures the ordered list of fallback rules.

func (*Router) SetPricing added in v0.15.1

func (r *Router) SetPricing(reg *pricing.Registry)

SetPricing configures the model pricing registry used by TokenCost.

func (*Router) SetTools

func (r *Router) SetTools(source func() []ToolDef)

SetTools configures a dynamic tool definition source. The function is called on every LLM request so that tools added at runtime are visible immediately.

type SessionStats added in v0.12.0

type SessionStats struct {
	Cost           float64        `json:"cost"`
	InputTokens    int            `json:"input_tokens"`
	OutputTokens   int            `json:"output_tokens"`
	Messages       int            `json:"messages"`
	PricingSources map[string]int `json:"pricing_sources,omitempty"`
}

SessionStats holds per-session cost and token tracking.

type TokenUsage

type TokenUsage struct {
	Prompt       int
	Completion   int
	CachedPrompt int // tokens served from cache (Anthropic cache_read, OpenAI cached_tokens)
	Total        int
}

type ToolCall

type ToolCall struct {
	ID       string       `json:"id"`
	Type     string       `json:"type"` // "function"
	Function FunctionCall `json:"function"`
}

ToolCall represents a tool invocation requested by the LLM (OpenAI format).

type ToolDef

type ToolDef struct {
	Type     string      `json:"type"` // "function"
	Function FunctionDef `json:"function"`
}

ToolDef describes a tool available for the LLM to call (OpenAI format).

Directories

Path Synopsis
Package anthropic implements the llm.Provider interface against the Anthropic Messages API (https://docs.anthropic.com/en/api/messages).
Package anthropic implements the llm.Provider interface against the Anthropic Messages API (https://docs.anthropic.com/en/api/messages).

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL