Documentation
¶
Index ¶
- Constants
- func NewObservedStream(stream io.ReadCloser, observer UsageObserver, metadata UsageMetadata) io.ReadCloser
- func RegisterClientAdapter(name string, factory ClientAdapterFactory)
- func RegisterLLMAdapter(name string, factory AdapterFactory)
- type AIError
- type AdapterFactory
- type AnthropicAdapter
- func (a *AnthropicAdapter) Chat(_ context.Context, _ *ChatRequest) (*ChatResponse, error)
- func (a *AnthropicAdapter) Name() string
- func (a *AnthropicAdapter) Responses(_ context.Context, _ *ResponsesRequest) (*ResponsesResponse, error)
- func (a *AnthropicAdapter) StreamChat(_ context.Context, _ *ChatRequest) (io.ReadCloser, error)
- func (a *AnthropicAdapter) StreamResponses(_ context.Context, _ *ResponsesRequest) (io.ReadCloser, error)
- type ChatRequest
- type ChatResponse
- type Choice
- type ClientAdapter
- type ClientAdapterFactory
- type CompletionTokensDetails
- type ContentPart
- type ExtraOptions
- type FunctionCall
- type FunctionDesc
- type ImageURL
- type LLMAdapter
- type LLMAdapterOptions
- type Message
- type ObservedStream
- type OpenAIChatAdapter
- func (a *OpenAIChatAdapter) Chat(ctx context.Context, chatReq *ChatRequest) (*ChatResponse, error)
- func (a *OpenAIChatAdapter) Name() string
- func (a *OpenAIChatAdapter) Responses(_ context.Context, _ *ResponsesRequest) (*ResponsesResponse, error)
- func (a *OpenAIChatAdapter) StreamChat(ctx context.Context, chatReq *ChatRequest) (io.ReadCloser, error)
- func (a *OpenAIChatAdapter) StreamResponses(_ context.Context, _ *ResponsesRequest) (io.ReadCloser, error)
- type OpenAIChatClientAdapter
- func (a *OpenAIChatClientAdapter) Name() string
- func (a *OpenAIChatClientAdapter) StreamConverter(stream io.ReadCloser) io.ReadCloser
- func (a *OpenAIChatClientAdapter) ToChatRequest(body []byte) (*ChatRequest, error)
- func (a *OpenAIChatClientAdapter) ToClientChatResponse(resp *ChatResponse) (any, error)
- func (a *OpenAIChatClientAdapter) ToClientError(err *AIError) (any, error)
- func (a *OpenAIChatClientAdapter) ToClientResponsesResponse(resp *ResponsesResponse) (any, error)
- func (a *OpenAIChatClientAdapter) ToResponsesRequest(body []byte) (*ResponsesRequest, error)
- type OpenAIErrorDetail
- type OpenAIErrorResponse
- type PromptTokensDetails
- type Reasoning
- type ResponsesRequest
- type ResponsesResponse
- type StreamChoice
- type StreamChunk
- type StreamDelta
- type StreamOptions
- type Tool
- type ToolCall
- type Usage
- type UsageMetadata
- type UsageObserver
Constants ¶
const ( // ContextKeyAIFamily is the context key for communicating the AI family between middleware and proxy. ContextKeyAIFamily = "ai_family" // ContextKeyChatRequest is the context key for the ChatRequest object. ContextKeyChatRequest = "ai_chat_request" // ContextKeyResponsesRequest is the context key for the ResponsesRequest object. ContextKeyResponsesRequest = "ai_responses_request" // ContextKeyClientAdapter is the context key for the client translator adapter. ContextKeyClientAdapter = "ai_client_adapter" // ContextKeyVirtualModelName is the context key for the original model name from the client. ContextKeyVirtualModelName = "ai_virtual_model_name" // ContextKeyChatResponse is the context key for the ChatResponse object. ContextKeyChatResponse = "ai_chat_response" // ContextKeyResponsesResponse is the context key for the ResponsesResponse object. ContextKeyResponsesResponse = "ai_responses_response" // ContextKeyResponseStream is the context key for the response stream. ContextKeyResponseStream = "ai_response_stream" // FamilyChat is the chat API family identifier. FamilyChat = "chat" // FamilyResponses is the responses API family identifier. FamilyResponses = "responses" )
const (
// TokensPerMillion is the number of tokens in a million.
TokensPerMillion = 1000000.0
)
Variables ¶
This section is empty.
Functions ¶
func NewObservedStream ¶
func NewObservedStream(stream io.ReadCloser, observer UsageObserver, metadata UsageMetadata) io.ReadCloser
NewObservedStream creates a new stream decorator that notifies the observer when usage data is detected in the stream.
func RegisterClientAdapter ¶
func RegisterClientAdapter(name string, factory ClientAdapterFactory)
RegisterClientAdapter registers a new client adapter factory.
func RegisterLLMAdapter ¶
func RegisterLLMAdapter(name string, factory AdapterFactory)
RegisterLLMAdapter registers a new adapter factory with a unique name.
Types ¶
type AIError ¶
type AIError struct {
Type string `json:"type"` // "invalid_request_error", "authentication_error", etc.
Message string `json:"message"` // Human-readable error message
StatusCode int `json:"-"` // HTTP status code
Provider string `json:"provider,omitempty"`
Param optional.Option[string] `json:"param"`
Code optional.Option[string] `json:"code"`
}
AIError defines a standardized error object for the AI Gateway.
type AdapterFactory ¶
type AdapterFactory func(opts LLMAdapterOptions) (LLMAdapter, error)
AdapterFactory is a function type that creates a specific LLMAdapter instance.
type AnthropicAdapter ¶
type AnthropicAdapter struct {
// contains filtered or unexported fields
}
AnthropicAdapter implements LLMAdapter for Anthropic's Messages API.
func NewAnthropicAdapter ¶
func NewAnthropicAdapter(opts LLMAdapterOptions) *AnthropicAdapter
NewAnthropicAdapter creates a new instance of AnthropicAdapter.
func (*AnthropicAdapter) Chat ¶
func (a *AnthropicAdapter) Chat(_ context.Context, _ *ChatRequest) (*ChatResponse, error)
Chat sends a unary chat completion request to Anthropic.
func (*AnthropicAdapter) Name ¶
func (a *AnthropicAdapter) Name() string
Name returns the name of the adapter.
func (*AnthropicAdapter) Responses ¶
func (a *AnthropicAdapter) Responses(_ context.Context, _ *ResponsesRequest) (*ResponsesResponse, error)
Responses sends a batch responses request to Anthropic.
func (*AnthropicAdapter) StreamChat ¶
func (a *AnthropicAdapter) StreamChat(_ context.Context, _ *ChatRequest) (io.ReadCloser, error)
StreamChat sends a streaming chat completion request to Anthropic.
func (*AnthropicAdapter) StreamResponses ¶
func (a *AnthropicAdapter) StreamResponses(_ context.Context, _ *ResponsesRequest) (io.ReadCloser, error)
StreamResponses sends a streaming responses request to Anthropic.
type ChatRequest ¶
type ChatRequest struct {
Model string `json:"model"`
Messages []Message `json:"messages"`
Stream bool `json:"stream,omitempty"`
StreamOptions *StreamOptions `json:"stream_options,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
MaxTokens *int `json:"max_tokens,omitempty"`
Stop []string `json:"stop,omitempty"`
Tools []Tool `json:"tools,omitempty"`
ToolChoice any `json:"tool_choice,omitempty"`
ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"`
Reasoning *Reasoning `json:"reasoning,omitempty"`
ResponseFormat any `json:"response_format,omitempty"`
UnknownFields map[string]any `json:"-"` // Collects unmapped fields for passthrough
Extra *ExtraOptions `json:"extra,omitempty"` // Internal metadata
}
ChatRequest represents a canonical chat completion request, heavily aligned with the OpenAI Chat Completion API.
func (*ChatRequest) MarshalJSON ¶
func (r *ChatRequest) MarshalJSON() ([]byte, error)
MarshalJSON implements custom marshaling to flatten unknown fields.
func (*ChatRequest) UnmarshalJSON ¶
func (r *ChatRequest) UnmarshalJSON(data []byte) error
UnmarshalJSON implements custom unmarshaling to capture unknown fields. IMPORTANT: When adding new fields to ChatRequest, add a corresponding delete(raw, "field_name") below.
type ChatResponse ¶
type ChatResponse struct {
ID string `json:"id"`
Object string `json:"object"` // "chat.completion"
Created int64 `json:"created"`
Model string `json:"model"`
SystemFingerprint string `json:"system_fingerprint,omitempty"`
Choices []Choice `json:"choices"`
Usage Usage `json:"usage"`
}
ChatResponse represents a canonical chat completion response.
type Choice ¶
type Choice struct {
Index int `json:"index"`
Message Message `json:"message"`
FinishReason string `json:"finish_reason"` // "stop", "length", "tool_calls", etc.
Logprobs any `json:"logprobs,omitempty"`
}
Choice represents one potential completion generated by the model.
type ClientAdapter ¶
type ClientAdapter interface {
// Name returns the unique identifier for this client protocol (e.g., "openai-chat", "anthropic").
Name() string
// ToChatRequest translates a raw client JSON body into a canonical ChatRequest.
ToChatRequest(body []byte) (*ChatRequest, error)
// ToResponsesRequest translates a raw client JSON body into a canonical ResponsesRequest.
ToResponsesRequest(body []byte) (*ResponsesRequest, error)
// ToClientChatResponse translates a canonical ChatResponse back into the client's expected format.
ToClientChatResponse(resp *ChatResponse) (any, error)
// ToClientResponsesResponse translates a canonical ResponsesResponse back into the client's expected format.
ToClientResponsesResponse(resp *ResponsesResponse) (any, error)
// StreamConverter wraps a canonical OpenAI-format SSE stream with a protocol-specific
// translator to re-encode chunks into the format expected by the client SDK.
// The input stream must contain valid OpenAI chat.completion.chunk SSE lines.
// The returned stream emits client-format SSE lines and must be closed by the caller.
StreamConverter(stream io.ReadCloser) io.ReadCloser
// ToClientError translates a canonical AIError into the client's expected format.
ToClientError(err *AIError) (any, error)
}
ClientAdapter defines the contract for translating between the client's SDK format and Bifrost's canonical internal AI formats.
func GetClientAdapter ¶
func GetClientAdapter(name string) (ClientAdapter, error)
GetClientAdapter retrieves a client adapter by its format name.
type ClientAdapterFactory ¶
type ClientAdapterFactory func() ClientAdapter
ClientAdapterFactory is a function type that creates a specific ClientAdapter instance.
type CompletionTokensDetails ¶
type CompletionTokensDetails struct {
ReasoningTokens int `json:"reasoning_tokens"`
}
CompletionTokensDetails holds extended output token breakdown (reasoning).
type ContentPart ¶
type ContentPart struct {
Type string `json:"type"` // "text" or "image_url"
Text string `json:"text,omitempty"`
ImageURL *ImageURL `json:"image_url,omitempty"`
}
ContentPart represents a single part of a multi-modal message.
type ExtraOptions ¶
type ExtraOptions struct {
AIFamily string `json:"ai_family"` // "chat" or "responses"
}
ExtraOptions carries Bifrost-specific metadata or passthrough fields.
type FunctionCall ¶
type FunctionCall struct {
Name string `json:"name"`
Arguments string `json:"arguments"` // JSON string
}
FunctionCall contains the name and arguments of a tool invocation.
type FunctionDesc ¶
type FunctionDesc struct {
Name string `json:"name"`
Description string `json:"description,omitempty"`
Parameters json.RawMessage `json:"parameters,omitempty"` // JSON Schema
}
FunctionDesc describes a single tool function.
type ImageURL ¶
type ImageURL struct {
URL string `json:"url"`
Detail string `json:"detail,omitempty"` // "auto", "low", "high"
}
ImageURL contains the URL or base64 data for an image.
type LLMAdapter ¶
type LLMAdapter interface {
// Name returns the unique identifier for this adapter (e.g., "openai-chat", "anthropic").
Name() string
// Chat executes a non-streaming chat completion request.
// It handles the translation from canonical ChatRequest to native payload,
// performs the network request, and translates the native response back to ChatResponse.
Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error)
// StreamChat executes a streaming chat completion request.
// It returns an io.ReadCloser that yields a stream of translated canonical SSE chunks.
// The caller is responsible for closing the stream.
StreamChat(ctx context.Context, req *ChatRequest) (io.ReadCloser, error)
// Responses executes a non-streaming OpenAI-compatible Responses API request.
Responses(ctx context.Context, req *ResponsesRequest) (*ResponsesResponse, error)
// StreamResponses executes a streaming OpenAI-compatible Responses API request.
// It returns an io.ReadCloser yielding canonical SSE chunks for the Responses family.
StreamResponses(ctx context.Context, req *ResponsesRequest) (io.ReadCloser, error)
}
LLMAdapter defines the functional contract for interacting with an LLM provider. It is a stateful object that encapsulates the HTTP client, API credentials, and base URL. Each adapter is responsible for bidirectional translation between the canonical format and the provider's native protocol.
func GetAdapter ¶
func GetAdapter(name string, opts LLMAdapterOptions) (LLMAdapter, error)
GetAdapter creates an instance of the requested adapter using the provided options.
type LLMAdapterOptions ¶
LLMAdapterOptions defines dependencies required to initialize an LLMAdapter.
type Message ¶
type Message struct {
Role string `json:"role"`
Content any `json:"content"` // string OR []ContentPart for multi-modal
Name string `json:"name,omitempty"`
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
ToolCallID string `json:"tool_call_id,omitempty"` // Required for "tool" role
}
Message represents a single turn in a conversation.
type ObservedStream ¶
type ObservedStream struct {
io.ReadCloser
// contains filtered or unexported fields
}
ObservedStream is a decorator for io.ReadCloser that intercepts SSE chunks to extract usage data before passing them to the client.
type OpenAIChatAdapter ¶
type OpenAIChatAdapter struct {
// contains filtered or unexported fields
}
OpenAIChatAdapter implements LLMAdapter for OpenAI's Chat Completions API.
func NewOpenAIChatAdapter ¶
func NewOpenAIChatAdapter(opts LLMAdapterOptions) *OpenAIChatAdapter
NewOpenAIChatAdapter creates a new instance of OpenAIChatAdapter.
func (*OpenAIChatAdapter) Chat ¶
func (a *OpenAIChatAdapter) Chat(ctx context.Context, chatReq *ChatRequest) (*ChatResponse, error)
Chat sends a unary chat completion request to OpenAI.
func (*OpenAIChatAdapter) Name ¶
func (a *OpenAIChatAdapter) Name() string
Name returns the name of the adapter.
func (*OpenAIChatAdapter) Responses ¶
func (a *OpenAIChatAdapter) Responses(_ context.Context, _ *ResponsesRequest) (*ResponsesResponse, error)
Responses sends a batch responses request to OpenAI.
func (*OpenAIChatAdapter) StreamChat ¶
func (a *OpenAIChatAdapter) StreamChat(ctx context.Context, chatReq *ChatRequest) (io.ReadCloser, error)
StreamChat sends a streaming chat completion request to OpenAI.
func (*OpenAIChatAdapter) StreamResponses ¶
func (a *OpenAIChatAdapter) StreamResponses(_ context.Context, _ *ResponsesRequest) (io.ReadCloser, error)
StreamResponses sends a streaming responses request to OpenAI.
type OpenAIChatClientAdapter ¶
type OpenAIChatClientAdapter struct{}
OpenAIChatClientAdapter implements ClientAdapter for the OpenAI chat protocol.
func NewOpenAIChatClientAdapter ¶
func NewOpenAIChatClientAdapter() *OpenAIChatClientAdapter
NewOpenAIChatClientAdapter creates a new OpenAIChatClientAdapter instance.
func (*OpenAIChatClientAdapter) Name ¶
func (a *OpenAIChatClientAdapter) Name() string
Name returns the client protocol name.
func (*OpenAIChatClientAdapter) StreamConverter ¶
func (a *OpenAIChatClientAdapter) StreamConverter(stream io.ReadCloser) io.ReadCloser
StreamConverter returns the stream unchanged, since the canonical SSE stream already uses OpenAI's format.
func (*OpenAIChatClientAdapter) ToChatRequest ¶
func (a *OpenAIChatClientAdapter) ToChatRequest(body []byte) (*ChatRequest, error)
ToChatRequest translates raw client JSON body into a canonical ChatRequest.
func (*OpenAIChatClientAdapter) ToClientChatResponse ¶
func (a *OpenAIChatClientAdapter) ToClientChatResponse(resp *ChatResponse) (any, error)
ToClientChatResponse translates canonical ChatResponse back to client format.
func (*OpenAIChatClientAdapter) ToClientError ¶
func (a *OpenAIChatClientAdapter) ToClientError(err *AIError) (any, error)
ToClientError translates a canonical AIError into the client's expected format.
func (*OpenAIChatClientAdapter) ToClientResponsesResponse ¶
func (a *OpenAIChatClientAdapter) ToClientResponsesResponse(resp *ResponsesResponse) (any, error)
ToClientResponsesResponse translates canonical ResponsesResponse back to client format.
func (*OpenAIChatClientAdapter) ToResponsesRequest ¶
func (a *OpenAIChatClientAdapter) ToResponsesRequest(body []byte) (*ResponsesRequest, error)
ToResponsesRequest translates raw client JSON body into a canonical ResponsesRequest.
type OpenAIErrorDetail ¶
type OpenAIErrorDetail struct {
Message string `json:"message"`
Type string `json:"type"`
Param optional.Option[string] `json:"param"`
Code optional.Option[string] `json:"code"`
}
OpenAIErrorDetail represents details of the OpenAI error.
type OpenAIErrorResponse ¶
type OpenAIErrorResponse struct {
Error OpenAIErrorDetail `json:"error"`
}
OpenAIErrorResponse represents the standard error response returned by OpenAI APIs.
type PromptTokensDetails ¶
type PromptTokensDetails struct {
CachedTokens int `json:"cached_tokens"`
}
PromptTokensDetails holds extended input token breakdown (caching).
type Reasoning ¶
type Reasoning struct {
Effort string `json:"effort,omitempty"` // "low", "medium", "high"
}
Reasoning configures reasoning behavior for models that support extended thinking (e.g., o1, o3, DeepSeek).
type ResponsesRequest ¶
type ResponsesRequest struct {
Model string `json:"model"`
Instructions string `json:"instructions,omitempty"`
Input []Message `json:"input"`
}
ResponsesRequest placeholder for stateful Responses API.
type ResponsesResponse ¶
type ResponsesResponse struct {
ID string `json:"id"`
Status string `json:"status"`
Model string `json:"model"`
Usage Usage `json:"usage"`
}
ResponsesResponse placeholder for Responses API response.
type StreamChoice ¶
type StreamChoice struct {
Index int `json:"index"`
Delta StreamDelta `json:"delta"`
FinishReason *string `json:"finish_reason"`
}
StreamChoice represents a delta update in a streaming response.
type StreamChunk ¶
type StreamChunk struct {
ID string `json:"id"`
Object string `json:"object"` // "chat.completion.chunk"
Created int64 `json:"created"`
Model string `json:"model"`
Choices []StreamChoice `json:"choices"`
Usage *Usage `json:"usage,omitempty"`
}
StreamChunk represents a single chunk in a streaming chat response.
type StreamDelta ¶
type StreamDelta struct {
Role string `json:"role,omitempty"`
Content string `json:"content,omitempty"`
ReasoningContent string `json:"reasoning_content,omitempty"` // For reasoning models
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
}
StreamDelta contains the incremental content or tool calls.
type StreamOptions ¶
type StreamOptions struct {
IncludeUsage bool `json:"include_usage,omitempty"`
}
StreamOptions controls streaming behavior options.
type Tool ¶
type Tool struct {
Type string `json:"type"` // Currently only "function"
Function FunctionDesc `json:"function"`
}
Tool represents a functional capability the model can invoke.
type ToolCall ¶
type ToolCall struct {
ID string `json:"id"`
Type string `json:"type"`
Function FunctionCall `json:"function"`
}
ToolCall represents a specific invocation of a tool by the model.
type Usage ¶
type Usage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
PromptTokensDetails *PromptTokensDetails `json:"prompt_tokens_details,omitempty"`
CompletionTokensDetails *CompletionTokensDetails `json:"completion_tokens_details,omitempty"`
InputCost float64 `json:"input_cost,omitempty"`
OutputCost float64 `json:"output_cost,omitempty"`
}
Usage provides token consumption metrics, including extended details for reasoning and caching.
func (*Usage) CalculateCost ¶
func (u *Usage) CalculateCost(p *config.AIPricingOptions)
CalculateCost calculates the input and output costs based on the pricing options.
type UsageMetadata ¶
type UsageMetadata struct {
Model string `json:"model"` // Logical model name (e.g., "gpt-4o")
UserID string `json:"user_id"` // User identifier for billing/quota
RouteID string `json:"route_id"` // Bifrost route ID
Provider string `json:"provider"` // Target provider ID (e.g., "openai-official")
}
UsageMetadata carries business-related context for usage tracking. This allows passing user information, route details, and other metadata to observers without changing interface signatures.
type UsageObserver ¶
type UsageObserver interface {
// OnUsage is called with the usage metrics and associated metadata.
// Context is provided for potential database or remote logging operations.
OnUsage(ctx context.Context, metadata UsageMetadata, usage Usage)
}
UsageObserver defines the contract for components that monitor AI usage. It is triggered whenever usage data (tokens) is successfully captured from either a unary response or a completed stream.