proxy

package
v0.2.52 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 29, 2026 License: MIT Imports: 36 Imported by: 0

Documentation

Overview

Package proxy contains configuration, middleware, routing, and model interaction logic for the language model proxy server.

Index

Constants

View Source 
const (
	// DefaultPort is the TCP port used by the HTTP server when no explicit port is provided.
	DefaultPort = 8080
	// DefaultWorkers is the maximum number of concurrent upstream HTTP operations.
	DefaultWorkers = 4
	// DefaultQueueSize is the number of upstream HTTP operations that may wait for a worker.
	DefaultQueueSize = 100
	// DefaultModel is the model identifier used when the client does not supply one.
	DefaultModel = ModelNameGPT41
	// DefaultProvider is the provider identifier used when the client does not supply one.
	DefaultProvider = ProviderNameOpenAI
	// DefaultDictationProvider is the provider used when /dictate does not supply one.
	DefaultDictationProvider = ProviderNameOpenAI

	// DefaultRequestTimeoutSeconds is the request work budget used when the client omits one.
	DefaultRequestTimeoutSeconds = 360
	// DefaultMaxRequestTimeoutSeconds is the one-hour operator capacity ceiling for a request.
	DefaultMaxRequestTimeoutSeconds = 60 * 60
	// DefaultMaxPromptBytes limits JSON LLM request bodies accepted by POST /.
	DefaultMaxPromptBytes      = 4 * 1024 * 1024
	DefaultDictationModel      = "gpt-4o-mini-transcribe"
	DefaultMaxInputAudioBytes  = 25 * 1024 * 1024
	DefaultManagementJWTIssuer = "tauth"
	// DefaultManagementUsageQueueSize is the number of managed usage events retained for asynchronous persistence.
	DefaultManagementUsageQueueSize = 1024
)
View Source 
const (
	// LogLevelDebug indicates that the application should log debug information.
	LogLevelDebug = "debug"

	// LogLevelInfo indicates that the application should log informational messages.
	LogLevelInfo = "info"
)
View Source 
const (
	ManagementConfigUIFileName = "config-ui.yaml"
	ManagementConfigUIPath     = "/" + ManagementConfigUIFileName
)
View Source 
const (
	// ModelNameGPT4oMini identifies the GPT-4o-mini model.
	ModelNameGPT4oMini = "gpt-4o-mini"
	// ModelNameGPT4o identifies the GPT-4o model.
	ModelNameGPT4o = "gpt-4o"
	// ModelNameGPT41 identifies the GPT-4.1 model.
	ModelNameGPT41 = "gpt-4.1"
	// ModelNameGPT5Mini identifies the GPT-5-mini model.
	ModelNameGPT5Mini = "gpt-5-mini"
	// ModelNameGPT5 identifies the GPT-5 model which does not accept the temperature field.
	ModelNameGPT5 = "gpt-5"
	// ModelNameGPT55 identifies the GPT-5.5 model which does not accept the temperature field.
	ModelNameGPT55 = "gpt-5.5"
	// ModelNameGPT55Pro identifies the GPT-5.5 pro model which does not accept the temperature field.
	ModelNameGPT55Pro = "gpt-5.5-pro"
	// ModelNameGPT56 identifies the GPT-5.6 model which does not accept the temperature field.
	ModelNameGPT56 = "gpt-5.6"
)
View Source 
const (
	// ProviderNameOpenAI identifies the OpenAI provider.
	ProviderNameOpenAI = "openai"
	// ProviderNameDeepSeek identifies the DeepSeek provider.
	ProviderNameDeepSeek = "deepseek"
	// ProviderNameDashScope identifies Alibaba Cloud Model Studio DashScope-compatible routing.
	ProviderNameDashScope = "dashscope"
	// ProviderNameQwenCloud identifies Qwen Cloud Token Plan routing.
	ProviderNameQwenCloud = "qwencloud"
	// ProviderNameMoonshot identifies Moonshot/Kimi routing.
	ProviderNameMoonshot = "moonshot"
	// ProviderNameMiniMax identifies MiniMax routing.
	ProviderNameMiniMax = "minimax"
	// ProviderNameSiliconFlow identifies SiliconFlow routing.
	ProviderNameSiliconFlow = "siliconflow"
	// ProviderNameZhipu identifies Zhipu/GLM routing.
	ProviderNameZhipu = "zhipu"
	// ProviderNameGemini identifies Google Gemini routing.
	ProviderNameGemini = "gemini"
	// ProviderNameAnthropic identifies Anthropic Claude routing.
	ProviderNameAnthropic = "anthropic"
	// ProviderNameMeta identifies Meta Model API routing.
	ProviderNameMeta = "meta"
	// ProviderNameGrok identifies xAI Grok routing.
	ProviderNameGrok = "grok"
)
View Source 
const (
	// ModelNameDeepSeekV4Flash identifies the low-cost DeepSeek V4 flash model.
	ModelNameDeepSeekV4Flash = "deepseek-v4-flash"
	// ModelNameDeepSeekV4Pro identifies the higher-capability DeepSeek V4 pro model.
	ModelNameDeepSeekV4Pro = "deepseek-v4-pro"
	// ModelNameDeepSeekChat identifies the legacy DeepSeek chat model name.
	ModelNameDeepSeekChat = "deepseek-chat"
	// ModelNameDeepSeekReasoner identifies the legacy DeepSeek reasoner model name.
	ModelNameDeepSeekReasoner = "deepseek-reasoner"
	// ModelNameDashScopeQwenPlus identifies DashScope Qwen Plus.
	ModelNameDashScopeQwenPlus = "qwen-plus"
	// ModelNameQwenCloudQwen38MaxPreview identifies Qwen Cloud Token Plan's Qwen 3.8 Max preview.
	ModelNameQwenCloudQwen38MaxPreview = "qwen3.8-max-preview"
	// ModelNameMoonshotKimiK26 identifies Moonshot Kimi K2.6.
	ModelNameMoonshotKimiK26 = "kimi-k2.6"
	// ModelNameMoonshotKimiK3 identifies Moonshot Kimi K3.
	ModelNameMoonshotKimiK3 = "kimi-k3"
	// ModelNameMoonshotKimiK27Code identifies Moonshot Kimi K2.7 Code.
	ModelNameMoonshotKimiK27Code = "kimi-k2.7-code"
	// ModelNameMoonshotKimiK27CodeHighSpeed identifies Moonshot Kimi K2.7 Code Highspeed.
	ModelNameMoonshotKimiK27CodeHighSpeed = "kimi-k2.7-code-highspeed"
	// ModelNameMiniMaxM27 identifies MiniMax M2.7.
	ModelNameMiniMaxM27 = "MiniMax-M2.7"
	// ModelNameSiliconFlowDeepSeek identifies SiliconFlow-hosted DeepSeek R1.
	ModelNameSiliconFlowDeepSeek = "deepseek-ai/DeepSeek-R1"
	// ModelNameZhipuGLM identifies the GLM 5.1 model.
	ModelNameZhipuGLM = "glm-5.1"
	// ModelNameGemini35Flash identifies Gemini 3.5 Flash.
	ModelNameGemini35Flash = "gemini-3.5-flash"
	// ModelNameGemini31FlashLite identifies Gemini 3.1 Flash-Lite.
	ModelNameGemini31FlashLite = "gemini-3.1-flash-lite"
	// ModelNameGemini25Flash identifies Gemini 2.5 Flash.
	ModelNameGemini25Flash = "gemini-2.5-flash"
	// ModelNameGemini25FlashLite identifies Gemini 2.5 Flash-Lite.
	ModelNameGemini25FlashLite = "gemini-2.5-flash-lite"
	// ModelNameGemini25Pro identifies Gemini 2.5 Pro.
	ModelNameGemini25Pro = "gemini-2.5-pro"
	// ModelNameClaudeOpus48 identifies Claude Opus 4.8.
	ModelNameClaudeOpus48 = "claude-opus-4-8"
	// ModelNameClaudeSonnet46 identifies Claude Sonnet 4.6.
	ModelNameClaudeSonnet46 = "claude-sonnet-4-6"
	// ModelNameClaudeHaiku45 identifies Claude Haiku 4.5.
	ModelNameClaudeHaiku45 = "claude-haiku-4-5-20251001"
	// ModelNameClaudeHaiku45Alias identifies the Claude Haiku 4.5 convenience alias.
	ModelNameClaudeHaiku45Alias = "claude-haiku-4-5"
	// ModelNameClaudeSonnet45 identifies Claude Sonnet 4.5.
	ModelNameClaudeSonnet45 = "claude-sonnet-4-5-20250929"
	// ModelNameClaudeSonnet45Alias identifies the Claude Sonnet 4.5 convenience alias.
	ModelNameClaudeSonnet45Alias = "claude-sonnet-4-5"
	// ModelNameClaudeOpus41 identifies Claude Opus 4.1.
	ModelNameClaudeOpus41 = "claude-opus-4-1-20250805"
	// ModelNameClaudeOpus41Alias identifies the Claude Opus 4.1 convenience alias.
	ModelNameClaudeOpus41Alias = "claude-opus-4-1"
	// ModelNameMuseSpark11 identifies Meta Muse Spark 1.1.
	ModelNameMuseSpark11 = "muse-spark-1.1"
	// ModelNameGrok43 identifies the current Grok 4.3 model.
	ModelNameGrok43 = "grok-4.3"
	// ModelNameGrok43Latest identifies the Grok 4.3 latest alias.
	ModelNameGrok43Latest = "grok-4.3-latest"
	// ModelNameGrokLatest identifies the current Grok latest alias.
	ModelNameGrokLatest = "grok-latest"
	// ModelNameGrokBuild01 identifies the Grok Build coding model.
	ModelNameGrokBuild01 = "grok-build-0.1"
	// ModelNameGrokCodeFast identifies the Grok code fast alias.
	ModelNameGrokCodeFast = "grok-code-fast"
	// ModelNameGrokCodeFast1 identifies the Grok code fast 1 alias.
	ModelNameGrokCodeFast1 = "grok-code-fast-1"
	// ModelNameGrokCodeFast10825 identifies the dated Grok code fast 1 model.
	ModelNameGrokCodeFast10825 = "grok-code-fast-1-0825"
)

Variables

View Source 
var (
	// SchemaGPT4oMini defines allowed payload fields for the GPT-4o-mini model.
	SchemaGPT4oMini = ModelPayloadSchema{AllowedRequestFields: []string{keyModel, keyInput, keyMaxOutputTokens, keyBackground, keyStore, keyTemperature}}
	// SchemaGPT4o defines allowed payload fields for the GPT-4o model.
	SchemaGPT4o = ModelPayloadSchema{AllowedRequestFields: []string{keyModel, keyInput, keyMaxOutputTokens, keyBackground, keyStore, keyTemperature, keyTools, keyToolChoice}}
	// SchemaGPT41 defines allowed payload fields for the GPT-4.1 model.
	SchemaGPT41 = ModelPayloadSchema{AllowedRequestFields: []string{keyModel, keyInput, keyMaxOutputTokens, keyBackground, keyStore, keyTemperature, keyTools, keyToolChoice}}
	// SchemaGPT5 defines allowed payload fields for the GPT-5 model.
	SchemaGPT5 = ModelPayloadSchema{AllowedRequestFields: []string{keyModel, keyInput, keyMaxOutputTokens, keyBackground, keyStore, keyTools, keyToolChoice, keyReasoning}}
	// SchemaGPT55 defines allowed payload fields for GPT-5.5 family models.
	SchemaGPT55 = SchemaGPT5
)
View Source 
var (
	// ErrUnknownProvider is returned when a request names a provider that is not registered.
	ErrUnknownProvider = errors.New(errorUnknownProvider)
	// ErrProviderNotConfigured is returned when a registered provider lacks a required server-side credential.
	ErrProviderNotConfigured = errors.New(errorProviderNotConfigured)
	// ErrUnsupportedCapability is returned when a request asks a provider for an unsupported capability.
	ErrUnsupportedCapability = errors.New(errorUnsupportedCapability)
	// ErrUnsupportedEndpoint is returned when a provider does not support the requested endpoint.
	ErrUnsupportedEndpoint = errors.New(errorUnsupportedEndpoint)
	// ErrConflictingModelParameters is returned when query and JSON body model values disagree.
	ErrConflictingModelParameters = errors.New(errorConflictingModelParameters)
	// ErrProviderRateLimited is returned when an upstream provider reports rate limiting.
	ErrProviderRateLimited = errors.New(errorProviderRateLimited)
	// ErrProviderAPI is returned when an upstream provider returns an unsuccessful response.
	ErrProviderAPI = errors.New(errorProviderAPI)
	// ErrInvalidChatMessages is returned when a JSON request body contains invalid chat messages.
	ErrInvalidChatMessages = errors.New(errorInvalidChatMessages)
	// ErrInvalidModelCatalog is returned when configured provider model catalogs are incomplete or inconsistent.
	ErrInvalidModelCatalog = errors.New("invalid_model_catalog")
)
View Source 
var (
	ErrMissingTenants                 = errors.New("tenants must include at least one tenant")
	ErrInvalidTenant                  = errors.New("invalid tenant")
	ErrInvalidManagementConfiguration = errors.New("invalid management configuration")
)
View Source 
var ErrInvalidUpstreamRateLimitConfiguration = errors.New("invalid_upstream_rate_limit_configuration")

ErrInvalidUpstreamRateLimitConfiguration identifies an invalid shared upstream rate-limit rule.

View Source 
var ErrUnknownModel = errors.New(errorUnknownModel)

ErrUnknownModel is returned when a model identifier is not recognized.

Functions

func BuildRequestPayload 

func BuildRequestPayload(modelIdentifier string, rawRequestProfile string, combinedPrompt string, webSearchEnabled bool, maxTokens *int, reasoningEffort string) any

BuildRequestPayload selects the correct OpenAI Responses payload shape for the configured request profile.

func BuildRouter 

func BuildRouter(configuration Configuration, structuredLogger *zap.SugaredLogger) (*gin.Engine, error)

BuildRouter constructs the HTTP router used by the proxy. configuration supplies queue sizes, worker counts, timeout values, API credentials and other settings. structuredLogger records structured log messages during routing.

func RenderManagementConfigUI added in v0.2.19 

func RenderManagementConfigUI(configuration ManagementConfiguration) string

RenderManagementConfigUI renders the browser-facing llm-proxy, MPR UI, and TAuth YAML config.

func Serve 

func Serve(configuration Configuration, structuredLogger *zap.SugaredLogger) error

Serve builds the router from the supplied configuration and structuredLogger and starts the HTTP server on the configured port.

Types

type Configuration 

type Configuration struct {
	Tenants                      []TenantConfiguration
	Management                   ManagementConfiguration
	OpenAIKey                    string
	DeepSeekKey                  string
	DashScopeKey                 string
	QwenCloudKey                 string
	MoonshotKey                  string
	MiniMaxKey                   string
	SiliconFlowKey               string
	ZhipuKey                     string
	GeminiKey                    string
	AnthropicKey                 string
	MetaKey                      string
	GrokKey                      string
	OpenAIBaseURL                string
	OpenAITranscriptionsURL      string
	DeepSeekBaseURL              string
	DashScopeBaseURL             string
	QwenCloudBaseURL             string
	MoonshotBaseURL              string
	MiniMaxBaseURL               string
	SiliconFlowBaseURL           string
	SiliconFlowTranscriptionsURL string
	ZhipuBaseURL                 string
	ZhipuTranscriptionsURL       string
	GeminiBaseURL                string
	AnthropicBaseURL             string
	MetaBaseURL                  string
	GrokBaseURL                  string
	GrokTranscriptionsURL        string
	Port                         int
	LogLevel                     string
	WorkerCount                  int
	QueueSize                    int
	RequestTimeoutSeconds        int
	MaxRequestTimeoutSeconds     int
	MaxPromptBytes               int64
	MaxInputAudioBytes           int64
	UpstreamRateLimits           []UpstreamRateLimitConfiguration
	Endpoints                    *Endpoints
	ProviderModels               ProviderModelCatalogs
	// contains filtered or unexported fields
}

Configuration holds runtime settings.

func NewConfiguration 

func NewConfiguration(configuration Configuration) (Configuration, error)

NewConfiguration returns a normalized runtime configuration after validating startup invariants.

func (*Configuration) ApplyTunables 

func (configuration *Configuration) ApplyTunables()

ApplyTunables ensures tunable configuration values have sensible defaults.

type Endpoints 

type Endpoints struct {
	// contains filtered or unexported fields
}

Endpoints provides concurrency-safe access to OpenAI endpoint URLs.

func NewEndpoints 

func NewEndpoints() *Endpoints

NewEndpoints creates an Endpoints instance initialized with default URLs.

func NewEndpointsForURLs added in v0.2.17 

func NewEndpointsForURLs(rawBaseURL string, rawTranscriptionsURL string) *Endpoints

NewEndpointsForURLs creates an Endpoints instance from configured OpenAI URLs.

func (*Endpoints) GetModelsURL 

func (endpointConfiguration *Endpoints) GetModelsURL() string

GetModelsURL returns the URL used for the OpenAI models endpoint.

func (*Endpoints) GetResponsesURL 

func (endpointConfiguration *Endpoints) GetResponsesURL() string

GetResponsesURL returns the URL used for the OpenAI responses endpoint.

func (*Endpoints) GetTranscriptionsURL 

func (endpointConfiguration *Endpoints) GetTranscriptionsURL() string

GetTranscriptionsURL returns the URL used for the OpenAI audio transcriptions endpoint.

func (*Endpoints) ResetModelsURL 

func (endpointConfiguration *Endpoints) ResetModelsURL()

ResetModelsURL resets the models endpoint to the default.

func (*Endpoints) ResetResponsesURL 

func (endpointConfiguration *Endpoints) ResetResponsesURL()

ResetResponsesURL resets the responses endpoint to the default.

func (*Endpoints) ResetTranscriptionsURL 

func (endpointConfiguration *Endpoints) ResetTranscriptionsURL()

ResetTranscriptionsURL resets the transcriptions endpoint to the default.

func (*Endpoints) SetModelsURL 

func (endpointConfiguration *Endpoints) SetModelsURL(newURL string)

SetModelsURL sets the URL for the OpenAI models endpoint.

func (*Endpoints) SetResponsesURL 

func (endpointConfiguration *Endpoints) SetResponsesURL(newURL string)

SetResponsesURL sets the URL for the OpenAI responses endpoint.

func (*Endpoints) SetTranscriptionsURL 

func (endpointConfiguration *Endpoints) SetTranscriptionsURL(newURL string)

SetTranscriptionsURL sets the URL for the OpenAI audio transcriptions endpoint.

type HTTPDoer 

type HTTPDoer interface {
	Do(httpRequest *http.Request) (*http.Response, error)
}

HTTPDoer executes HTTP requests, allowing the proxy to abstract the underlying HTTP client. 

var (
	// HTTPClient is the default HTTPDoer implementation that delegates to http.DefaultClient.
	HTTPClient HTTPDoer = http.DefaultClient
)

type ManagementConfiguration added in v0.2.19 

type ManagementConfiguration struct {
	Enabled                  bool
	PublicOrigin             string
	UIDescription            string
	UIOrigins                []string
	AdminEmails              []string
	TAuthURL                 string
	TAuthTenantID            string
	GoogleClientID           string
	LoginPath                string
	LogoutPath               string
	NoncePath                string
	SessionPath              string
	JWTSigningKey            string
	JWTIssuer                string
	SessionCookieName        string
	DatabasePath             string
	UsageQueueSize           int
	ProviderKeyEncryptionKey string
	ManagementAPIOrigin      string
	ProxyOrigin              string
	DatabaseDialector        gorm.Dialector
}

ManagementConfiguration holds authenticated browser UI and self-service tenant settings.

func (*ManagementConfiguration) ApplyTunables added in v0.2.19 

func (configuration *ManagementConfiguration) ApplyTunables()

ApplyTunables normalizes optional management settings.

type ModelConfiguration added in v0.2.17 

type ModelConfiguration struct {
	ID               string
	RequestProfile   string
	WebSearch        bool
	OutputTokenLimit int
	ReasoningEffort  *ReasoningEffortCapability
}

ModelConfiguration declares runtime metadata for one configured model.

type ModelEndpointCatalog added in v0.2.17 

type ModelEndpointCatalog struct {
	DefaultModel string
	Models       []ModelConfiguration
}

ModelEndpointCatalog declares allowed models and the endpoint default model.

type ModelPayloadSchema 

type ModelPayloadSchema struct {
	// AllowedRequestFields enumerates JSON fields permitted in the request payload.
	AllowedRequestFields []string
}

ModelPayloadSchema lists request fields allowed by a model.

func ResolveModelPayloadSchema 

func ResolveModelPayloadSchema(requestProfile string) ModelPayloadSchema

ResolveModelPayloadSchema returns the schema for a request profile or an empty schema when unknown.

type OpenAIClient 

type OpenAIClient struct {
	// contains filtered or unexported fields
}

OpenAIClient provides access to the OpenAI responses API with configurable endpoints and tunable parameters.

func NewOpenAIClient 

func NewOpenAIClient(httpClient HTTPDoer, endpoints *Endpoints) *OpenAIClient

NewOpenAIClient constructs an OpenAIClient initialized with the supplied components.

type ProviderModelCatalog added in v0.2.17 

type ProviderModelCatalog struct {
	Text      ModelEndpointCatalog
	Dictation ModelEndpointCatalog
}

ProviderModelCatalog declares text and dictation model support for one provider.

type ProviderModelCatalogs added in v0.2.17 

type ProviderModelCatalogs map[string]ProviderModelCatalog

ProviderModelCatalogs maps canonical provider identifiers to configured model catalogs.

type Reasoning 

type Reasoning struct {
	Effort string `json:"effort"`
}

Reasoning specifies configuration options for reasoning-capable models. Effort indicates the tenant-selected reasoning intensity supported by the configured upstream adapter.

type ReasoningEffortCapability added in v0.2.35 

type ReasoningEffortCapability struct {
	Adapter string
	Efforts []string
}

ReasoningEffortCapability declares the configured upstream mapping for one exact text provider/model route.

type TenantConfiguration added in v0.2.15 

type TenantConfiguration struct {
	ID       string
	Secret   string
	Defaults TenantDefaults
}

TenantConfiguration is the config-file shape for one authenticated tenant.

func DefaultTenantConfiguration added in v0.2.15 

func DefaultTenantConfiguration(identifier string, secret string) TenantConfiguration

DefaultTenantConfiguration returns one tenant using the built-in request defaults.

func SingleTenantConfigurations added in v0.2.15 

func SingleTenantConfigurations(identifier string, secret string) []TenantConfiguration

SingleTenantConfigurations returns a tenant slice for tests and small deployments.

func SingleTenantConfigurationsWithDefaults added in v0.2.15 

func SingleTenantConfigurationsWithDefaults(identifier string, secret string, defaults TenantDefaults) []TenantConfiguration

SingleTenantConfigurationsWithDefaults returns one tenant with caller-specified request defaults.

type TenantDefaults added in v0.2.15 

type TenantDefaults struct {
	Provider          string
	Model             string
	DictationProvider string
	DictationModel    string
	SystemPrompt      string
	ReasoningEffort   string
}

TenantDefaults holds default request values selected by an authenticated tenant.

func DefaultTenantDefaults added in v0.2.15 

func DefaultTenantDefaults() TenantDefaults

DefaultTenantDefaults returns the built-in request defaults for a single tenant.

type Tool 

type Tool struct {
	Type string `json:"type"`
}

Tool represents a tool available to the model.

type UpstreamRateLimitConfiguration added in v0.2.28 

type UpstreamRateLimitConfiguration struct {
	Origin      string
	MaxRequests int
	Interval    string
}

UpstreamRateLimitConfiguration describes one exact-origin rolling-window limit.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.