Documentation
¶
Overview ¶
Package localgateway implements an OpenAI-compatible action gateway for local inference engines (Ollama, LM Studio, vLLM, …). It proxies chat completions to an operator-configured upstream and executes VaultRun sandbox tools for any tool_calls the model returns.
Index ¶
- func ListenAddrExposesNonLoopback(addr string) bool
- type ChatChoice
- type ChatMessage
- type ChatRequest
- type ChatResponse
- type ChatUpstream
- type Config
- type ErrorBody
- type ErrorDetail
- type FunctionCall
- type Gateway
- type GatewayError
- type HTTPUpstream
- type LoopResult
- type MissionAPI
- type ModelObject
- type ModelsResponse
- type SessionStore
- type ToolCall
- type ToolDef
- type ToolFunction
- type Usage
- type VaultRunClient
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func ListenAddrExposesNonLoopback ¶
ListenAddrExposesNonLoopback reports whether the listen address is likely reachable from outside the host (used for a startup warning).
Types ¶
type ChatChoice ¶
type ChatChoice struct {
Index int `json:"index"`
Message ChatMessage `json:"message"`
FinishReason string `json:"finish_reason"`
}
ChatChoice is one completion choice.
type ChatMessage ¶
type ChatMessage struct {
Role string `json:"role"`
Content any `json:"content,omitempty"`
Name string `json:"name,omitempty"`
ToolCallID string `json:"tool_call_id,omitempty"`
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
}
ChatMessage is one role/content turn (content may be string or multipart).
type ChatRequest ¶
type ChatRequest struct {
Model string `json:"model"`
Messages []ChatMessage `json:"messages"`
Tools []ToolDef `json:"tools,omitempty"`
Stream bool `json:"stream,omitempty"`
// Temperature and other sampling params are forwarded as-is via RawExtra
// is not used — we re-encode a controlled subset.
Temperature *float64 `json:"temperature,omitempty"`
MaxTokens *int `json:"max_tokens,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
}
ChatRequest is the OpenAI-compatible chat completions request body.
type ChatResponse ¶
type ChatResponse struct {
ID string `json:"id"`
Object string `json:"object"`
Created int64 `json:"created"`
Model string `json:"model"`
Choices []ChatChoice `json:"choices"`
Usage *Usage `json:"usage,omitempty"`
}
ChatResponse is the OpenAI-compatible response.
type ChatUpstream ¶
type ChatUpstream interface {
ChatCompletions(ctx context.Context, req ChatRequest) (*ChatResponse, error)
ListModels(ctx context.Context) (*ModelsResponse, error)
}
ChatUpstream talks to an OpenAI-compatible inference server.
type Config ¶
type Config struct {
// ListenAddr is the HTTP bind address (e.g. ":8091").
ListenAddr string
// AuthToken is the required Bearer token for /v1/* (required, min 16 chars).
AuthToken string
// UpstreamURL is the OpenAI-compatible base URL (e.g. http://127.0.0.1:11434).
// Only operator-configured; never taken from request bodies.
UpstreamURL string
// UpstreamTimeout bounds a single upstream chat request.
UpstreamTimeout time.Duration
// VaultRunBaseURL and VaultRunAPIKey authenticate to the VaultRun API.
VaultRunBaseURL string
VaultRunAPIKey string
// DefaultImage is used when auto-creating sessions.
DefaultImage string
// NetworkEnabled controls whether auto-created sessions get network.
NetworkEnabled bool
// MaxToolLoops caps model→tool→model iterations per request.
MaxToolLoops int
// MaxBodyBytes limits inbound JSON body size.
MaxBodyBytes int64
// MaxToolResultBytes truncates tool stdout/stderr fed back to the model.
MaxToolResultBytes int
// MaxFileBytes caps write_file content size.
MaxFileBytes int
// RateLimitPerMin is the per-IP request budget (fixed 1-minute window).
RateLimitPerMin int
// DefaultModel is injected when the client omits "model".
DefaultModel string
// RunTimeoutSeconds default for vaultrun_run_command when the model omits it.
RunTimeoutSeconds int
// MaxRunTimeoutSeconds hard cap for per-run timeouts.
MaxRunTimeoutSeconds int
// CaptureMissions saves successful VaultRun tool sequences as missions.
CaptureMissions bool
}
Config holds runtime settings for the local inference gateway.
func LoadConfigFromEnv ¶
LoadConfigFromEnv reads configuration from environment variables.
type ErrorBody ¶
type ErrorBody struct {
Error ErrorDetail `json:"error"`
}
ErrorBody is returned for gateway-level errors.
type ErrorDetail ¶
type ErrorDetail struct {
Message string `json:"message"`
Type string `json:"type"`
Code string `json:"code,omitempty"`
}
ErrorDetail mirrors the OpenAI error object shape.
type FunctionCall ¶
FunctionCall holds the name and JSON-encoded arguments.
type Gateway ¶
type Gateway struct {
// contains filtered or unexported fields
}
Gateway is the local inference action plane.
func New ¶
func New(cfg Config, vr VaultRunClient, upstream ChatUpstream) *Gateway
New constructs a Gateway.
func NewFromConfig ¶
NewFromConfig builds VaultRun + upstream clients from config.
func (*Gateway) Handler ¶
Handler returns the HTTP handler for the gateway (for ListenAndServe or tests).
func (*Gateway) RunChatLoop ¶
func (g *Gateway) RunChatLoop(ctx context.Context, req ChatRequest, conversationKey, explicitSessionID string) (*LoopResult, error)
RunChatLoop proxies the request to the upstream model, executes VaultRun tools for each tool_calls round, and returns the final assistant message. Upstream tool rounds are always non-streaming; req.Stream only affects how the HTTP handler delivers the final answer (JSON vs SSE).
func (*Gateway) WithMissions ¶
func (g *Gateway) WithMissions(m MissionAPI) *Gateway
WithMissions attaches a mission capture backend (optional).
type GatewayError ¶
GatewayError is an HTTP-mapped error.
func (*GatewayError) Error ¶
func (e *GatewayError) Error() string
type HTTPUpstream ¶
type HTTPUpstream struct {
// contains filtered or unexported fields
}
HTTPUpstream is the default HTTP client for Ollama / LM Studio / vLLM.
func NewHTTPUpstream ¶
func NewHTTPUpstream(baseURL string, timeout time.Duration) *HTTPUpstream
NewHTTPUpstream builds an upstream client. The baseURL is operator-configured only (validated at config load). Private/loopback destinations are allowed.
func (*HTTPUpstream) ChatCompletions ¶
func (u *HTTPUpstream) ChatCompletions(ctx context.Context, req ChatRequest) (*ChatResponse, error)
ChatCompletions POSTs /v1/chat/completions on the upstream.
func (*HTTPUpstream) ListModels ¶
func (u *HTTPUpstream) ListModels(ctx context.Context) (*ModelsResponse, error)
ListModels GETs /v1/models when the upstream supports it.
type LoopResult ¶
type LoopResult struct {
Response *ChatResponse
SessionID string
Stream bool
}
LoopResult is the final chat response plus the bound session id.
type MissionAPI ¶
type MissionAPI interface {
CreateMission(ctx context.Context, req createMissionRequest) (string, error)
RecordMissionRun(ctx context.Context, missionID, sessionID string) error
}
MissionAPI persists successful tool sequences as VaultRun missions.
type ModelObject ¶
type ModelObject struct {
ID string `json:"id"`
Object string `json:"object"`
OwnedBy string `json:"owned_by"`
}
ModelObject is a single model list entry.
type ModelsResponse ¶
type ModelsResponse struct {
Object string `json:"object"`
Data []ModelObject `json:"data"`
}
ModelsResponse is returned by GET /v1/models (proxy/list helper).
type SessionStore ¶
type SessionStore struct {
// contains filtered or unexported fields
}
SessionStore maps conversation keys to VaultRun session IDs.
func NewSessionStore ¶
func NewSessionStore() *SessionStore
NewSessionStore creates an empty in-memory conversation→session map.
func (*SessionStore) Delete ¶
func (s *SessionStore) Delete(conversationKey string)
Delete removes a binding.
func (*SessionStore) Get ¶
func (s *SessionStore) Get(conversationKey string) (string, bool)
Get returns a previously bound session ID, if any.
func (*SessionStore) Put ¶
func (s *SessionStore) Put(conversationKey, sessionID string)
Put binds a conversation key to a session ID.
type ToolCall ¶
type ToolCall struct {
ID string `json:"id"`
Type string `json:"type"`
Function FunctionCall `json:"function"`
}
ToolCall is a model-requested function invocation.
type ToolDef ¶
type ToolDef struct {
Type string `json:"type"`
Function ToolFunction `json:"function"`
}
ToolDef is an OpenAI function tool definition.
type ToolFunction ¶
type ToolFunction struct {
Name string `json:"name"`
Description string `json:"description,omitempty"`
Parameters json.RawMessage `json:"parameters,omitempty"`
}
ToolFunction describes a callable function.
type Usage ¶
type Usage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
}
Usage tracks token counts when the upstream provides them.
type VaultRunClient ¶
type VaultRunClient interface {
CreateSession(ctx context.Context, opts vaultrun.CreateSessionOptions) (*vaultrun.Session, error)
GetSession(ctx context.Context, sessionID string) (*vaultrun.Session, error)
DeleteSession(ctx context.Context, sessionID string) error
Run(ctx context.Context, sessionID string, opts vaultrun.RunOptions) (*vaultrun.Run, error)
UploadFile(ctx context.Context, sessionID, remotePath string, content io.Reader) (*vaultrun.File, error)
DownloadFile(ctx context.Context, sessionID, remotePath string) (io.ReadCloser, error)
ListFiles(ctx context.Context, sessionID string) ([]*vaultrun.File, error)
}
VaultRunClient is the subset of the Go SDK used by the gateway.