Documentation
¶
Overview ¶
Package llmengine adapts llmcore to Obot's services and administrative API. It owns model catalog refresh, legacy imports, and admin writes. Inference, routing, and runtime configuration reconciliation belong to llmcore; the catalog and plugins are injected so llmcore does not depend on Obot's service lifecycle.
Index ¶
- Constants
- func ApplyProviderDefaults(provider map[string]any)
- func CostFromMetadata(metadata map[string]string) (types.ModelCost, bool)
- func DialectForModel(provider, modelID string) llmtypes.Dialect
- func FillDefaultModelAliases(ctx context.Context, c kclient.Client) error
- func HasSecretValue(raw []byte) bool
- func IsMaskedSecret(raw []byte) bool
- func IsSecretReference(raw []byte) bool
- func LegacyImportDone(mp *v1.ModelProvider) bool
- func LegacyProvider(ctx context.Context) string
- func MigrateLegacyModels(ctx context.Context, c kclient.Client, provider *v1.ModelProvider) error
- func ModelProviderResource() *v1.ModelProvider
- func PreferredModels(alias types.DefaultModelAliasType) []string
- func ScrubEngineName(message string) string
- func ScrubEngineNameBytes(body []byte) []byte
- func SecretValue(raw []byte) string
- func TriggerModelSync(ctx context.Context, c kclient.Client) error
- func TrimProviderBaseURLVersion(provider map[string]any, providerType string) bool
- func ValidateAzureKey(body []byte) error
- func WithLegacySource(ctx context.Context, source string) context.Context
- type Engine
- func (e *Engine) AnthropicIsDirect(ctx context.Context) bool
- func (e *Engine) ApplyEgressProxy(ctx context.Context) bool
- func (e *Engine) ChatCompletion(ctx context.Context, model string, messages []schemas.ChatMessage) (*schemas.BifrostChatResponse, error)
- func (e *Engine) ClaudeSubscriptionModels(ctx context.Context) []ListedModel
- func (e *Engine) Client() *bifrostcore.Bifrost
- func (e *Engine) ConfiguredProviders(ctx context.Context) ([]string, error)
- func (e *Engine) CreateKey(ctx context.Context, provider string, body json.RawMessage) (Key, error)
- func (e *Engine) CreatePricingOverride(ctx context.Context, body json.RawMessage) (tables.TablePricingOverride, error)
- func (e *Engine) CreateProvider(ctx context.Context, body json.RawMessage) (ProviderResponse, error)
- func (e *Engine) CreateRoutingRule(ctx context.Context, body json.RawMessage) (tables.TableRoutingRule, error)
- func (e *Engine) DeleteKey(ctx context.Context, provider, id string) error
- func (e *Engine) DeletePricingOverride(ctx context.Context, id string) (tables.TablePricingOverride, error)
- func (e *Engine) DeleteProvider(ctx context.Context, provider string) error
- func (e *Engine) DeleteRoutingRule(ctx context.Context, id string) error
- func (e *Engine) Err() error
- func (e *Engine) Fail(err error)
- func (e *Engine) GetClientConfig(ctx context.Context) (map[string]any, error)
- func (e *Engine) GetKey(ctx context.Context, provider, id string) (Key, error)
- func (e *Engine) GetProvider(ctx context.Context, provider string) (ProviderResponse, error)
- func (e *Engine) GetProxyConfig(ctx context.Context) (map[string]any, error)
- func (e *Engine) GetRoutingRule(ctx context.Context, id string) (tables.TableRoutingRule, error)
- func (e *Engine) GuardProviderChange(ctx context.Context, method, path string, body []byte, removeKeys bool) (ProviderChange, error)
- func (e *Engine) HasUpstreamProviders(ctx context.Context) bool
- func (e *Engine) ImportLegacyModelProviders(ctx context.Context, c kclient.Client, gatewayClient *gateway.Client, ...) error
- func (e *Engine) LLMPluginOrder() []string
- func (e *Engine) LegacyKey(ctx context.Context, target string) (string, error)
- func (e *Engine) ListCatalogModels(ctx context.Context, provider, query string) ([]map[string]any, error)
- func (e *Engine) ListKeys(ctx context.Context) ([]Key, error)
- func (e *Engine) ListModels(ctx context.Context) (*openai.ModelsList, error)
- func (e *Engine) ListPricingOverrides(ctx context.Context) ([]tables.TablePricingOverride, error)
- func (e *Engine) ListProviderKeys(ctx context.Context, provider string) ([]Key, error)
- func (e *Engine) ListProviders(ctx context.Context) ([]ProviderResponse, error)
- func (e *Engine) ListRoutingRules(ctx context.Context) ([]tables.TableRoutingRule, error)
- func (e *Engine) MaxRequestBodyBytes() int64
- func (e *Engine) Ready() bool
- func (e *Engine) RefreshAzureDeployments(ctx context.Context) error
- func (e *Engine) RefreshPricing(ctx context.Context) error
- func (e *Engine) RefreshProviderModels(ctx context.Context, name string) (pending bool, err error)
- func (e *Engine) RunStartupTasks(ctx context.Context, c kclient.Client)
- func (e *Engine) ServeHTTP(w http.ResponseWriter, r *http.Request)
- func (e *Engine) ServeMetrics(w http.ResponseWriter, r *http.Request)
- func (e *Engine) Shutdown(ctx context.Context) error
- func (e *Engine) Start(ctx context.Context, opts Options) error
- func (e *Engine) Sync(ctx context.Context) (bool, error)
- func (e *Engine) UnavailableModels(ctx context.Context, ids []string) (map[string]bool, error)
- func (e *Engine) UpdateClientConfig(ctx context.Context, body json.RawMessage) (map[string]any, error)
- func (e *Engine) UpdateKey(ctx context.Context, provider, id string, body json.RawMessage) (Key, error)
- func (e *Engine) UpdatePricingOverride(ctx context.Context, id string, body json.RawMessage) (tables.TablePricingOverride, error)
- func (e *Engine) UpdateProvider(ctx context.Context, provider string, body json.RawMessage, removeKeys bool) (ProviderResponse, error)
- func (e *Engine) UpdateProxyConfig(ctx context.Context, body json.RawMessage) error
- func (e *Engine) UpdateRoutingRule(ctx context.Context, id string, body json.RawMessage) (tables.TableRoutingRule, error)
- func (e *Engine) Wait(ctx context.Context) error
- type Key
- type ListedModel
- type Options
- type ProviderChange
- type ProviderResponse
Constants ¶
const ( MetadataProvider = "bifrostProvider" MetadataInputCost = "bifrostInputCostPerToken" MetadataOutputCost = "bifrostOutputCostPerToken" MetadataCacheReadCost = "bifrostCacheReadCostPerToken" MetadataCacheWriteCost = "bifrostCacheWriteCostPerToken" MetadataContextLength = "bifrostContextLength" MetadataMaxOutputTokens = "bifrostMaxOutputTokens" )
Metadata keys set on listed models. The provider back-population controller reads "displayName", "usage" and "dialect"; the remaining keys carry Bifrost pricing so the caller can derive ModelInfo costs.
const (
Version = "obot-core"
)
Variables ¶
This section is empty.
Functions ¶
func ApplyProviderDefaults ¶
ApplyProviderDefaults fills in the network settings a new provider leaves unset: retries with backoff, which also let requests fail over between the provider's keys, and the request timeout.
func CostFromMetadata ¶
CostFromMetadata converts Bifrost per-token pricing into Obot's per-million-token cost.
func DialectForModel ¶
DialectForModel picks the API dialect Obot's agents should use for a model. Bifrost translates every dialect to every provider, so this only selects the most faithful one: Anthropic models speak Messages natively, everything else speaks Responses.
func FillDefaultModelAliases ¶
FillDefaultModelAliases sets unset default model aliases to the most preferred gateway model that is available, so a new installation works as soon as a provider is added.
func HasSecretValue ¶
HasSecretValue reports whether a JSON value the engine accepts as an optional secret, such as a CA certificate, holds a value. JSON null leaves the setting unset.
func IsMaskedSecret ¶
IsMaskedSecret reports whether a JSON value is a secret as the engine lists it, redacted, which the engine takes to mean "keep the stored value" when it is sent back in an update.
func IsSecretReference ¶
IsSecretReference reports whether the engine would read a JSON value from the host's environment or a vault, rather than take it as given, wherever it accepts a secret. It decodes the value with the engine's own decoder, so it can't disagree with the engine on spelling, such as key names in another case.
func LegacyImportDone ¶
func LegacyImportDone(mp *v1.ModelProvider) bool
LegacyImportDone reports whether the credentials of legacy model providers have been imported into the gateway, whose ModelProvider is mp.
func LegacyProvider ¶ added in v0.28.1
func MigrateLegacyModels ¶
MigrateLegacyModels rewrites default aliases and Profile references after normal discovery publishes gateway models. It never changes source or destination models.
func ModelProviderResource ¶
func ModelProviderResource() *v1.ModelProvider
ModelProviderResource is the ModelProvider resource that represents the embedded gateway. Unlike registry providers it has no daemon and no credential: providers and keys are configured in Bifrost itself.
func PreferredModels ¶
func PreferredModels(alias types.DefaultModelAliasType) []string
PreferredModels returns the gateway models (provider/model) preferred for a default model alias, most preferred first.
func ScrubEngineName ¶
ScrubEngineName replaces the engine's name in an error message with the product's own, so errors read as coming from Obot's LLM gateway.
func ScrubEngineNameBytes ¶
ScrubEngineNameBytes is ScrubEngineName for a response body.
func SecretValue ¶
SecretValue returns the value the engine reads from a JSON value it accepts as a secret, such as a region, which may be a string or an object holding one.
func TriggerModelSync ¶
TriggerModelSync flips the sync annotation on the gateway's ModelProvider so the provider controller re-lists models. The provider controller updates the same resource while it syncs, so a conflicting write is retried against the latest version.
func TrimProviderBaseURLVersion ¶
TrimProviderBaseURLVersion removes a trailing "/v1" from the base URL of a provider of type providerType (the provider's own name for a built-in provider, or the type a custom provider is built on) when the engine adds the version to every path itself. OpenAI-compatible servers usually document their base URL with "/v1", which the engine would otherwise turn into "/v1/v1/chat/completions". It reports whether it changed the base URL.
func ValidateAzureKey ¶
ValidateAzureKey restricts ambient Azure identity to Azure AI endpoints and scopes. Call it on the effective key after merging partial updates and restoring masked secrets.
Types ¶
type Engine ¶
type Engine struct {
// contains filtered or unexported fields
}
func (*Engine) AnthropicIsDirect ¶
AnthropicIsDirect reports whether the engine's Anthropic provider sends requests straight to Anthropic over verified TLS: no other base URL, and no setting that would let another server present a certificate for Anthropic's host, whether in the provider's network or proxy settings or in the gateway's global proxy. A caller's Claude subscription token is only forwarded when it does, so an administrator can't redirect users' tokens to a server of their own. A proxy without such settings, such as the environment's egress proxy, only relays the TLS connection to Anthropic, so it doesn't stop the token being forwarded.
func (*Engine) ApplyEgressProxy ¶
ApplyEgressProxy routes providers through the HTTP(S) proxy configured in Obot's environment. Bifrost's HTTP client does not read HTTPS_PROXY/NO_PROXY itself, so without this an Obot deployed behind an egress proxy could not reach any provider. Providers that already have a proxy configured, and those whose endpoint NO_PROXY excludes (such as in-cluster vLLM or Ollama servers), are left alone. This only persists desired settings; startup and model controllers apply them through Sync.
func (*Engine) ChatCompletion ¶
func (e *Engine) ChatCompletion(ctx context.Context, model string, messages []schemas.ChatMessage) (*schemas.BifrostChatResponse, error)
ChatCompletion runs trusted message-policy inference through the configured core.
func (*Engine) ClaudeSubscriptionModels ¶
func (e *Engine) ClaudeSubscriptionModels(ctx context.Context) []ListedModel
ClaudeSubscriptionModels returns the Anthropic models listed from the model catalog while the Anthropic provider is used only for Claude subscriptions, which Obot's model list has too (see claudeSubscriptionModels). The gateway's own /v1/models endpoint lists a provider's models with its keys, so it lists none of them.
func (*Engine) Client ¶
func (e *Engine) Client() *bifrostcore.Bifrost
func (*Engine) ConfiguredProviders ¶
ConfiguredProviders returns the names of the upstream providers configured in the gateway.
func (*Engine) CreatePricingOverride ¶
func (e *Engine) CreatePricingOverride(ctx context.Context, body json.RawMessage) (tables.TablePricingOverride, error)
func (*Engine) CreateProvider ¶
func (e *Engine) CreateProvider(ctx context.Context, body json.RawMessage) (ProviderResponse, error)
func (*Engine) CreateRoutingRule ¶
func (e *Engine) CreateRoutingRule(ctx context.Context, body json.RawMessage) (tables.TableRoutingRule, error)
func (*Engine) DeletePricingOverride ¶
func (*Engine) DeleteProvider ¶
func (*Engine) DeleteRoutingRule ¶
func (*Engine) GetClientConfig ¶
func (*Engine) GetProvider ¶
func (*Engine) GetProxyConfig ¶
func (*Engine) GetRoutingRule ¶
func (*Engine) GuardProviderChange ¶
func (e *Engine) GuardProviderChange(ctx context.Context, method, path string, body []byte, removeKeys bool) (ProviderChange, error)
GuardProviderChange checks a change to a provider or provider key made through the admin API, where path is the engine API path ("providers", "providers/<name>", "providers/<name>/keys" or "providers/<name>/keys/<id>"):
- Every address the provider connects to (its base URL and its keys' server URLs) may be a loopback or private network address only when the provider allows private network addresses. Link-local and cloud metadata addresses are never allowed. The engine itself lets loopback addresses through, and doesn't know every cloud metadata address.
- A provider's base URL can't change while it has keys, so stored keys aren't sent to a new address. With removeKeys, the keys are removed instead (see ProviderChange).
- A key's server URL can't change without its secrets being entered again.
- An update that leaves out a custom provider's custom_provider_config keeps it. The engine treats a missing custom_provider_config as removing it, which it refuses.
func (*Engine) HasUpstreamProviders ¶
HasUpstreamProviders reports whether Bifrost has at least one usable provider. A failed check reports the last known answer, so a transient error doesn't unconfigure the gateway.
func (*Engine) ImportLegacyModelProviders ¶
func (e *Engine) ImportLegacyModelProviders(ctx context.Context, c kclient.Client, gatewayClient *gateway.Client, allowPrivateNetwork bool) error
ImportLegacyModelProviders imports each provider family once, without changing models or references.
func (*Engine) LLMPluginOrder ¶
func (*Engine) LegacyKey ¶ added in v0.28.1
LegacyKey prefers an eligible imported credential; other requests use the normal key pool.
func (*Engine) ListCatalogModels ¶
func (*Engine) ListModels ¶
ListModels returns every model Bifrost can currently route to, in the shape the provider back-population controller expects.
func (*Engine) ListPricingOverrides ¶
func (*Engine) ListProviderKeys ¶
func (*Engine) ListProviders ¶
func (e *Engine) ListProviders(ctx context.Context) ([]ProviderResponse, error)
func (*Engine) ListRoutingRules ¶
func (*Engine) MaxRequestBodyBytes ¶
func (*Engine) RefreshAzureDeployments ¶
RefreshAzureDeployments is called only by the leader-elected model-sync controller.
func (*Engine) RefreshProviderModels ¶
RefreshProviderModels queues Azure discovery on the leader; other providers refresh synchronously.
func (*Engine) RunStartupTasks ¶
RunStartupTasks configures environment keys and periodically asks Obot to refresh models.
func (*Engine) ServeMetrics ¶
func (e *Engine) ServeMetrics(w http.ResponseWriter, r *http.Request)
ServeMetrics exposes this replica's gateway inference metrics.
func (*Engine) Shutdown ¶
Shutdown bounds the caller's wait while core cleanup continues before the catalog stops.
func (*Engine) UnavailableModels ¶
UnavailableModels identifies models that resource retention must not restore. Unknown models remain eligible for retention during upstream listing failures.
func (*Engine) UpdateClientConfig ¶
func (*Engine) UpdatePricingOverride ¶
func (e *Engine) UpdatePricingOverride(ctx context.Context, id string, body json.RawMessage) (tables.TablePricingOverride, error)
func (*Engine) UpdateProvider ¶
func (e *Engine) UpdateProvider(ctx context.Context, provider string, body json.RawMessage, removeKeys bool) (ProviderResponse, error)
func (*Engine) UpdateProxyConfig ¶
func (*Engine) UpdateRoutingRule ¶
func (e *Engine) UpdateRoutingRule(ctx context.Context, id string, body json.RawMessage) (tables.TableRoutingRule, error)
type Key ¶
type Key struct {
schemas.Key
llmcoreclient.AzureAccount
}
ProviderResponse adds Obot's admin identifiers to Bifrost's redacted configuration.
type ListedModel ¶
type ListedModel struct {
// ID is the model's provider-qualified ID, such as "anthropic/claude-sonnet-4-5".
ID string
Name string
}
ListedModel is a model the gateway's model listing endpoint doesn't list itself.
type ProviderChange ¶
type ProviderChange struct {
// Body is the validated partial patch, with custom provider identity preserved.
Body []byte
// RemoveKeys are the IDs of the provider's keys to remove before the change: its base URL
// changes, and the caller asked for its keys to be removed rather than sent to the new
// address.
RemoveKeys []string
}
ProviderChange is a provider change made through the admin API, after GuardProviderChange checked it.
type ProviderResponse ¶
type ProviderResponse struct {
Keys []Key `json:"keys"`
*configstore.ProviderConfig
Name schemas.ModelProvider `json:"name"`
Provider schemas.ModelProvider `json:"provider"`
ProviderStatus string `json:"provider_status"`
}