Documentation
¶
Overview ¶
Package router contains eyrie's request-routing building blocks: a CircuitBreaker for failing providers and a ToolFilter for constraining the tools exposed to a given request.
Index ¶
- func BackoffDelay(attempt int, cfg RetryConfig) time.Duration
- func IsTransient(err error) bool
- func RoutingPreviewJSON(requested string, compiled *catalog.CompiledCatalogV1, policy RoutingPolicy) (string, error)
- func ShouldTryNextDeployment(err error) bool
- type CircuitBreaker
- type CircuitBreakerConfig
- type CircuitState
- type DeploymentAdapter
- type DeploymentChoice
- type DeploymentRouter
- func (r *DeploymentRouter) Chat(ctx context.Context, messages []client.EyrieMessage, opts client.ChatOptions) (*client.EyrieResponse, error)
- func (r *DeploymentRouter) Name() string
- func (r *DeploymentRouter) Ping(ctx context.Context) error
- func (r *DeploymentRouter) RoutingStagesFor(canonicalModelID string) []RoutingStage
- func (r *DeploymentRouter) Stats() map[string]int64
- func (r *DeploymentRouter) StreamChat(ctx context.Context, messages []client.EyrieMessage, opts client.ChatOptions) (*client.StreamResult, error)
- type DeploymentRouterOptions
- type Option
- type RetryConfig
- type RetryEvent
- type RouteEntry
- type Router
- func (r *Router) Chat(ctx context.Context, messages []client.EyrieMessage, opts client.ChatOptions) (*client.EyrieResponse, error)
- func (r *Router) Name() string
- func (r *Router) Ping(ctx context.Context) error
- func (r *Router) Stats() map[string]int64
- func (r *Router) StreamChat(ctx context.Context, messages []client.EyrieMessage, opts client.ChatOptions) (*client.StreamResult, error)
- type RoutingPolicy
- type RoutingResolution
- type RoutingStage
- type Strategy
- type ToolFilter
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func BackoffDelay ¶
func BackoffDelay(attempt int, cfg RetryConfig) time.Duration
BackoffDelay calculates exponential backoff delay with jitter using the shared implementation.
func IsTransient ¶
func RoutingPreviewJSON ¶
func RoutingPreviewJSON(requested string, compiled *catalog.CompiledCatalogV1, policy RoutingPolicy) (string, error)
RoutingPreviewJSON returns indented JSON for CLI / config UI.
func ShouldTryNextDeployment ¶
ShouldTryNextDeployment reports billing/credit errors where another deployment may succeed.
Types ¶
type CircuitBreaker ¶
type CircuitBreaker struct {
// contains filtered or unexported fields
}
CircuitBreaker tracks failure rates for a single deployment and opens when failures exceed a threshold.
func NewCircuitBreaker ¶
func NewCircuitBreaker(threshold int, cooldown time.Duration) *CircuitBreaker
NewCircuitBreaker creates a circuit breaker that opens after `threshold` consecutive failures and stays open for `cooldown` duration before transitioning to half-open.
func (*CircuitBreaker) Allow ¶
func (cb *CircuitBreaker) Allow() bool
Allow returns true if the request should be attempted.
func (*CircuitBreaker) Failure ¶
func (cb *CircuitBreaker) Failure()
Failure records a failed call. If the threshold is reached, the circuit opens. In HalfOpen state a single failure immediately re-opens the circuit so the next probe only occurs after another full cooldown.
func (*CircuitBreaker) Reset ¶
func (cb *CircuitBreaker) Reset()
Reset forces the circuit back to closed state.
func (*CircuitBreaker) State ¶
func (cb *CircuitBreaker) State() CircuitState
State returns the current circuit state.
func (*CircuitBreaker) Success ¶
func (cb *CircuitBreaker) Success()
Success records a successful call and resets the circuit.
type CircuitBreakerConfig ¶
type CircuitBreakerConfig struct {
Threshold int `json:"threshold"` // failures before opening (default 5)
Cooldown time.Duration `json:"cooldown"` // time before half-open (default 30s)
}
CircuitBreakerConfig holds tunable circuit breaker parameters.
func DefaultCircuitBreakerConfig ¶
func DefaultCircuitBreakerConfig() CircuitBreakerConfig
DefaultCircuitBreakerConfig returns production defaults.
type CircuitState ¶
type CircuitState int
CircuitState represents the state of a circuit breaker.
const ( CircuitClosed CircuitState = iota // Normal operation CircuitOpen // Failing — requests are rejected immediately CircuitHalfOpen // Testing if the deployment has recovered )
type DeploymentAdapter ¶
type DeploymentChoice ¶
type DeploymentRouter ¶
type DeploymentRouter struct {
// contains filtered or unexported fields
}
func NewDeploymentRouter ¶
func NewDeploymentRouter(opts DeploymentRouterOptions) (*DeploymentRouter, error)
func (*DeploymentRouter) Chat ¶
func (r *DeploymentRouter) Chat(ctx context.Context, messages []client.EyrieMessage, opts client.ChatOptions) (*client.EyrieResponse, error)
func (*DeploymentRouter) Name ¶
func (r *DeploymentRouter) Name() string
func (*DeploymentRouter) RoutingStagesFor ¶
func (r *DeploymentRouter) RoutingStagesFor(canonicalModelID string) []RoutingStage
RoutingStagesFor exposes the active route for a canonical model on a live router.
func (*DeploymentRouter) Stats ¶
func (r *DeploymentRouter) Stats() map[string]int64
func (*DeploymentRouter) StreamChat ¶
func (r *DeploymentRouter) StreamChat(ctx context.Context, messages []client.EyrieMessage, opts client.ChatOptions) (*client.StreamResult, error)
type DeploymentRouterOptions ¶
type DeploymentRouterOptions struct {
Catalog *catalog.CompiledCatalogV1
Deployments map[string]DeploymentAdapter
Routing RoutingPolicy
CircuitBreaker *CircuitBreakerConfig // optional, defaults applied if nil
}
type Option ¶
type Option func(*Router)
Option configures a Router at construction time.
func WithStrategy ¶
WithStrategy sets the load-balancing routing strategy. The default is StrategyWeighted, which preserves the router's original weighted-random behavior.
type RetryConfig ¶
type RetryConfig struct {
types.RetryConfig
OnRetry func(RetryEvent)
}
RetryConfig controls retry behavior at the router level. It embeds types.RetryConfig for the core fields and adds an OnRetry callback for observability.
func DefaultRetryConfig ¶
func DefaultRetryConfig() RetryConfig
func NewRetryConfig ¶
func NewRetryConfig(maxRetries int, baseDelay, maxDelay time.Duration) RetryConfig
NewRetryConfig constructs a router RetryConfig.
type RetryEvent ¶
type RouteEntry ¶
type RouteEntry struct {
Provider client.Provider
Weight int
Retry *RetryConfig
// Cost is an optional relative cost used by StrategyCostBased (e.g. price
// per 1K tokens in some fixed unit). When zero, Weight is used as a proxy.
Cost int
}
type Router ¶
type Router struct {
// contains filtered or unexported fields
}
func New ¶
func New(entries []RouteEntry, fallback []client.Provider, defaultRetry *RetryConfig, opts ...Option) *Router
func (*Router) Chat ¶
func (r *Router) Chat(ctx context.Context, messages []client.EyrieMessage, opts client.ChatOptions) (*client.EyrieResponse, error)
func (*Router) StreamChat ¶
func (r *Router) StreamChat(ctx context.Context, messages []client.EyrieMessage, opts client.ChatOptions) (*client.StreamResult, error)
type RoutingPolicy ¶
type RoutingPolicy struct {
Default []RoutingStage `json:"default,omitempty"`
Providers map[string][]RoutingStage `json:"providers,omitempty"`
Models map[string][]RoutingStage `json:"models,omitempty"`
}
type RoutingResolution ¶
type RoutingResolution struct {
RequestedModel string `json:"requested_model"`
CanonicalModelID string `json:"canonical_model_id"`
Source string `json:"source"` // models, providers, provider_alias, default, automatic, unresolved
Stages []RoutingStage `json:"stages"`
}
RoutingResolution describes which routing policy matched a canonical model.
func ResolveRouting ¶
func ResolveRouting(requested string, compiled *catalog.CompiledCatalogV1, policy RoutingPolicy) RoutingResolution
ResolveRouting previews routing stages without calling provider APIs.
type RoutingStage ¶
type RoutingStage struct {
Deployments []DeploymentChoice `json:"deployments"`
Retries int `json:"retries,omitempty"`
}
type Strategy ¶
type Strategy string
Strategy names a load-balancing routing strategy used to pick an entry from the router's primary entries. StrategyWeighted is the historical default and preserves the original weighted-random behavior.
const ( // StrategyWeighted selects an entry via weighted random over RouteEntry.Weight. // This is the default and matches the router's original behavior. StrategyWeighted Strategy = "weighted" // StrategySimpleShuffle selects uniformly at random among entries, ignoring weight. StrategySimpleShuffle Strategy = "simple-shuffle" // StrategyLeastBusy selects the entry with the fewest in-flight requests. StrategyLeastBusy Strategy = "least-busy" // StrategyLatencyBased selects the entry with the lowest observed EWMA latency. StrategyLatencyBased Strategy = "latency-based" // StrategyCostBased selects the entry with the lowest cost (RouteEntry.Cost, // falling back to Weight as a proxy when no cost is configured). StrategyCostBased Strategy = "cost-based" // StrategyUsageBased selects the entry with the least cumulative token usage // in the current window. StrategyUsageBased Strategy = "usage-based" )
type ToolFilter ¶
type ToolFilter struct {
// contains filtered or unexported fields
}
func NewToolFilter ¶
func NewToolFilter(modelTools map[string][]string) *ToolFilter