Documentation
¶
Overview ¶
Package heartbeat provides proactive background monitoring for provider health. It implements continuous health checks, quota monitoring, and auto-discovery to prevent failures before they impact users.
Index ¶
- type APIHealthChecker
- type ClaudeCLIHealthChecker
- func (c *ClaudeCLIHealthChecker) Check(ctx context.Context) (*HealthStatus, error)
- func (c *ClaudeCLIHealthChecker) GetCheckInterval() time.Duration
- func (c *ClaudeCLIHealthChecker) GetName() string
- func (c *ClaudeCLIHealthChecker) SupportsAutoDiscovery() bool
- func (c *ClaudeCLIHealthChecker) SupportsQuotaMonitoring() bool
- type DiscoveredModel
- type GeminiCLIHealthChecker
- func (c *GeminiCLIHealthChecker) Check(ctx context.Context) (*HealthStatus, error)
- func (c *GeminiCLIHealthChecker) GetCheckInterval() time.Duration
- func (c *GeminiCLIHealthChecker) GetName() string
- func (c *GeminiCLIHealthChecker) SupportsAutoDiscovery() bool
- func (c *GeminiCLIHealthChecker) SupportsQuotaMonitoring() bool
- type HealthStatus
- type HeartbeatConfig
- type HeartbeatEvent
- type HeartbeatEventHandler
- type HeartbeatEventType
- type HeartbeatMonitor
- type HeartbeatMonitorImpl
- func (hm *HeartbeatMonitorImpl) AddEventHandler(handler HeartbeatEventHandler)
- func (hm *HeartbeatMonitorImpl) CheckAll(ctx context.Context) error
- func (hm *HeartbeatMonitorImpl) CheckProvider(ctx context.Context, provider string) (*HealthStatus, error)
- func (hm *HeartbeatMonitorImpl) GetAllStatuses() map[string]*HealthStatus
- func (hm *HeartbeatMonitorImpl) GetConfig() *HeartbeatConfig
- func (hm *HeartbeatMonitorImpl) GetInterval() time.Duration
- func (hm *HeartbeatMonitorImpl) GetStats() *HeartbeatStats
- func (hm *HeartbeatMonitorImpl) GetStatus(provider string) (*HealthStatus, error)
- func (hm *HeartbeatMonitorImpl) IsRunning() bool
- func (hm *HeartbeatMonitorImpl) RegisterChecker(checker ProviderHealthChecker) error
- func (hm *HeartbeatMonitorImpl) RemoveEventHandler(handler HeartbeatEventHandler)
- func (hm *HeartbeatMonitorImpl) SetInterval(interval time.Duration)
- func (hm *HeartbeatMonitorImpl) Start(ctx context.Context) error
- func (hm *HeartbeatMonitorImpl) Stop() error
- func (hm *HeartbeatMonitorImpl) UnregisterChecker(provider string) error
- type HeartbeatStats
- type LoggingEventHandler
- func (h *LoggingEventHandler) Clear()
- func (h *LoggingEventHandler) GetEvents() []HeartbeatEvent
- func (h *LoggingEventHandler) GetEventsByType(eventType HeartbeatEventType) []HeartbeatEvent
- func (h *LoggingEventHandler) GetEventsForProvider(provider string) []HeartbeatEvent
- func (h *LoggingEventHandler) HandleEvent(event *HeartbeatEvent) error
- type ModelDiscovery
- type OllamaHealthChecker
- func (c *OllamaHealthChecker) Check(ctx context.Context) (*HealthStatus, error)
- func (c *OllamaHealthChecker) GetCheckInterval() time.Duration
- func (c *OllamaHealthChecker) GetName() string
- func (c *OllamaHealthChecker) SupportsAutoDiscovery() bool
- func (c *OllamaHealthChecker) SupportsQuotaMonitoring() bool
- type ProviderHealthChecker
- func NewClaudeAPIHealthChecker(apiKey string) ProviderHealthChecker
- func NewGeminiAPIHealthChecker(apiKey string) ProviderHealthChecker
- func NewGroqHealthChecker(apiKey, baseURL string) ProviderHealthChecker
- func NewOpenAICompatibilityHealthChecker(name, baseURL, apiKey string) ProviderHealthChecker
- func NewOpenAIHealthChecker(apiKey string) ProviderHealthChecker
- func NewSwitchAIHealthChecker(apiKey, baseURL string) ProviderHealthChecker
- type ProviderStatus
- type QuotaInfo
- type QuotaStatusType
- type RecoveryAction
- type RecoveryActionType
- type RecoveryConfig
- type RecoveryEventHandler
- func (h *RecoveryEventHandler) GetRecoveryActions() []RecoveryAction
- func (h *RecoveryEventHandler) GetRecoveryActionsForProvider(provider string) []RecoveryAction
- func (h *RecoveryEventHandler) GetRecoveryManager() *RecoveryManager
- func (h *RecoveryEventHandler) GetRecoveryStats() *RecoveryStats
- func (h *RecoveryEventHandler) HandleEvent(event *HeartbeatEvent) error
- func (h *RecoveryEventHandler) IsFallbackEnabled(provider string) bool
- func (h *RecoveryEventHandler) IsProviderDisabled(provider string) bool
- type RecoveryManager
- func (rm *RecoveryManager) GetActions() []RecoveryAction
- func (rm *RecoveryManager) GetActionsForProvider(provider string) []RecoveryAction
- func (rm *RecoveryManager) GetRecoveryAttempts(provider string) int
- func (rm *RecoveryManager) GetStats() *RecoveryStats
- func (rm *RecoveryManager) HandleProviderDegraded(ctx context.Context, provider string, status *HealthStatus) error
- func (rm *RecoveryManager) HandleProviderHealthy(ctx context.Context, provider string, status *HealthStatus) error
- func (rm *RecoveryManager) HandleProviderUnavailable(ctx context.Context, provider string, status *HealthStatus) error
- func (rm *RecoveryManager) IsFallbackEnabled(provider string) bool
- func (rm *RecoveryManager) IsProviderDisabled(provider string) bool
- func (rm *RecoveryManager) ResetRecoveryAttempts(provider string)
- type RecoveryStats
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type APIHealthChecker ¶
type APIHealthChecker struct {
// contains filtered or unexported fields
}
func (*APIHealthChecker) Check ¶
func (c *APIHealthChecker) Check(ctx context.Context) (*HealthStatus, error)
func (*APIHealthChecker) GetCheckInterval ¶
func (c *APIHealthChecker) GetCheckInterval() time.Duration
func (*APIHealthChecker) GetName ¶
func (c *APIHealthChecker) GetName() string
func (*APIHealthChecker) SupportsAutoDiscovery ¶
func (c *APIHealthChecker) SupportsAutoDiscovery() bool
func (*APIHealthChecker) SupportsQuotaMonitoring ¶
func (c *APIHealthChecker) SupportsQuotaMonitoring() bool
type ClaudeCLIHealthChecker ¶
type ClaudeCLIHealthChecker struct {
// contains filtered or unexported fields
}
func NewClaudeCLIHealthChecker ¶
func NewClaudeCLIHealthChecker(cliPath string) *ClaudeCLIHealthChecker
func (*ClaudeCLIHealthChecker) Check ¶
func (c *ClaudeCLIHealthChecker) Check(ctx context.Context) (*HealthStatus, error)
func (*ClaudeCLIHealthChecker) GetCheckInterval ¶
func (c *ClaudeCLIHealthChecker) GetCheckInterval() time.Duration
func (*ClaudeCLIHealthChecker) GetName ¶
func (c *ClaudeCLIHealthChecker) GetName() string
func (*ClaudeCLIHealthChecker) SupportsAutoDiscovery ¶
func (c *ClaudeCLIHealthChecker) SupportsAutoDiscovery() bool
func (*ClaudeCLIHealthChecker) SupportsQuotaMonitoring ¶
func (c *ClaudeCLIHealthChecker) SupportsQuotaMonitoring() bool
type DiscoveredModel ¶
type DiscoveredModel struct {
Provider string `json:"provider"`
ModelID string `json:"model_id"`
DisplayName string `json:"display_name"`
Size int64 `json:"size"`
Discovered time.Time `json:"discovered"`
}
DiscoveredModel represents a model found during discovery.
type GeminiCLIHealthChecker ¶
type GeminiCLIHealthChecker struct {
// contains filtered or unexported fields
}
func NewGeminiCLIHealthChecker ¶
func NewGeminiCLIHealthChecker(cliPath string) *GeminiCLIHealthChecker
func (*GeminiCLIHealthChecker) Check ¶
func (c *GeminiCLIHealthChecker) Check(ctx context.Context) (*HealthStatus, error)
func (*GeminiCLIHealthChecker) GetCheckInterval ¶
func (c *GeminiCLIHealthChecker) GetCheckInterval() time.Duration
func (*GeminiCLIHealthChecker) GetName ¶
func (c *GeminiCLIHealthChecker) GetName() string
func (*GeminiCLIHealthChecker) SupportsAutoDiscovery ¶
func (c *GeminiCLIHealthChecker) SupportsAutoDiscovery() bool
func (*GeminiCLIHealthChecker) SupportsQuotaMonitoring ¶
func (c *GeminiCLIHealthChecker) SupportsQuotaMonitoring() bool
type HealthStatus ¶
type HealthStatus struct {
// Provider is the name of the provider being monitored
Provider string `json:"provider"`
// Status is the current health status
Status ProviderStatus `json:"status"`
// LastCheck is when this status was last updated
LastCheck time.Time `json:"last_check"`
// ResponseTime is the time taken for the health check
ResponseTime time.Duration `json:"response_time"`
// ModelsCount is the number of available models (if applicable)
ModelsCount int `json:"models_count"`
// QuotaUsed is the current quota usage (0.0 to 1.0)
QuotaUsed float64 `json:"quota_used"`
// QuotaLimit is the quota limit (requests, tokens, etc.)
QuotaLimit float64 `json:"quota_limit"`
// ErrorMessage contains error details if status is not healthy
ErrorMessage string `json:"error_message,omitempty"`
// Metadata contains provider-specific health information
Metadata map[string]interface{} `json:"metadata,omitempty"`
}
HealthStatus contains comprehensive health information for a provider.
type HeartbeatConfig ¶
type HeartbeatConfig struct {
// Enabled controls whether heartbeat monitoring is active
Enabled bool `yaml:"enabled" json:"enabled"`
// Interval is the time between heartbeat cycles
Interval time.Duration `yaml:"interval" json:"interval"`
// Timeout is the maximum time to wait for a single provider check
Timeout time.Duration `yaml:"timeout" json:"timeout"`
// AutoDiscovery enables automatic model discovery
AutoDiscovery bool `yaml:"auto-discovery" json:"auto-discovery"`
// QuotaWarningThreshold triggers warnings when quota usage exceeds this (0.0-1.0)
QuotaWarningThreshold float64 `yaml:"quota-warning-threshold" json:"quota-warning-threshold"`
// QuotaCriticalThreshold triggers critical alerts when quota usage exceeds this (0.0-1.0)
QuotaCriticalThreshold float64 `yaml:"quota-critical-threshold" json:"quota-critical-threshold"`
// MaxConcurrentChecks limits the number of simultaneous health checks
MaxConcurrentChecks int `yaml:"max-concurrent-checks" json:"max-concurrent-checks"`
// RetryAttempts is the number of retries for failed health checks
RetryAttempts int `yaml:"retry-attempts" json:"retry-attempts"`
// RetryDelay is the delay between retry attempts
RetryDelay time.Duration `yaml:"retry-delay" json:"retry-delay"`
}
HeartbeatConfig contains configuration for the heartbeat monitor.
func DefaultHeartbeatConfig ¶
func DefaultHeartbeatConfig() *HeartbeatConfig
DefaultHeartbeatConfig returns the default heartbeat configuration.
type HeartbeatEvent ¶
type HeartbeatEvent struct {
// Type is the event type
Type HeartbeatEventType `json:"type"`
// Provider is the provider that triggered the event
Provider string `json:"provider"`
// Timestamp is when the event occurred
Timestamp time.Time `json:"timestamp"`
// Status is the current health status
Status *HealthStatus `json:"status,omitempty"`
// PreviousStatus is the previous health status (for status change events)
PreviousStatus *HealthStatus `json:"previous_status,omitempty"`
// Data contains event-specific data
Data map[string]interface{} `json:"data,omitempty"`
}
HeartbeatEvent represents events that can be triggered by the heartbeat monitor.
type HeartbeatEventHandler ¶
type HeartbeatEventHandler interface {
// HandleEvent processes a heartbeat event
HandleEvent(event *HeartbeatEvent) error
}
HeartbeatEventHandler defines the interface for handling heartbeat events.
type HeartbeatEventType ¶
type HeartbeatEventType string
HeartbeatEventType represents the type of heartbeat event.
const ( // EventProviderHealthy indicates a provider became healthy EventProviderHealthy HeartbeatEventType = "provider_healthy" // EventProviderDegraded indicates a provider became degraded EventProviderDegraded HeartbeatEventType = "provider_degraded" EventProviderUnavailable HeartbeatEventType = "provider_unavailable" // EventQuotaWarning indicates quota usage exceeded warning threshold EventQuotaWarning HeartbeatEventType = "quota_warning" // EventQuotaCritical indicates quota usage exceeded critical threshold EventQuotaCritical HeartbeatEventType = "quota_critical" // EventModelDiscovered indicates new models were discovered EventModelDiscovered HeartbeatEventType = "model_discovered" // EventHealthCheckFailed indicates a health check failed EventHealthCheckFailed HeartbeatEventType = "health_check_failed" // EventHeartbeatStarted indicates the heartbeat monitor started EventHeartbeatStarted HeartbeatEventType = "heartbeat_started" // EventHeartbeatStopped indicates the heartbeat monitor stopped EventHeartbeatStopped HeartbeatEventType = "heartbeat_stopped" )
type HeartbeatMonitor ¶
type HeartbeatMonitor interface {
// Start begins the heartbeat monitoring loop
Start(ctx context.Context) error
// Stop gracefully shuts down the monitor
Stop() error
// CheckAll performs health checks on all registered providers
CheckAll(ctx context.Context) error
// CheckProvider performs a health check on a specific provider
CheckProvider(ctx context.Context, provider string) (*HealthStatus, error)
// GetStatus retrieves the last known status for a provider
GetStatus(provider string) (*HealthStatus, error)
// GetAllStatuses retrieves the last known status for all providers
GetAllStatuses() map[string]*HealthStatus
// RegisterChecker registers a new provider health checker
RegisterChecker(checker ProviderHealthChecker) error
// UnregisterChecker removes a provider health checker
UnregisterChecker(provider string) error
// SetInterval updates the heartbeat check interval
SetInterval(interval time.Duration)
// GetInterval returns the current heartbeat check interval
GetInterval() time.Duration
// AddEventHandler registers an event handler for heartbeat events
AddEventHandler(handler HeartbeatEventHandler)
// RemoveEventHandler removes an event handler
RemoveEventHandler(handler HeartbeatEventHandler)
}
HeartbeatMonitor defines the interface for the heartbeat monitoring service.
func NewHeartbeatMonitor ¶
func NewHeartbeatMonitor(config *HeartbeatConfig) HeartbeatMonitor
NewHeartbeatMonitor creates a new heartbeat monitor with the given configuration.
type HeartbeatMonitorImpl ¶
type HeartbeatMonitorImpl struct {
// contains filtered or unexported fields
}
HeartbeatMonitorImpl implements the HeartbeatMonitor interface. It provides background monitoring of provider health with configurable intervals.
func (*HeartbeatMonitorImpl) AddEventHandler ¶
func (hm *HeartbeatMonitorImpl) AddEventHandler(handler HeartbeatEventHandler)
AddEventHandler registers an event handler for heartbeat events.
func (*HeartbeatMonitorImpl) CheckAll ¶
func (hm *HeartbeatMonitorImpl) CheckAll(ctx context.Context) error
CheckAll performs health checks on all registered providers.
func (*HeartbeatMonitorImpl) CheckProvider ¶
func (hm *HeartbeatMonitorImpl) CheckProvider(ctx context.Context, provider string) (*HealthStatus, error)
CheckProvider performs a health check on a specific provider.
func (*HeartbeatMonitorImpl) GetAllStatuses ¶
func (hm *HeartbeatMonitorImpl) GetAllStatuses() map[string]*HealthStatus
GetAllStatuses retrieves the last known status for all providers.
func (*HeartbeatMonitorImpl) GetConfig ¶
func (hm *HeartbeatMonitorImpl) GetConfig() *HeartbeatConfig
GetConfig returns the current heartbeat configuration.
func (*HeartbeatMonitorImpl) GetInterval ¶
func (hm *HeartbeatMonitorImpl) GetInterval() time.Duration
GetInterval returns the current heartbeat check interval.
func (*HeartbeatMonitorImpl) GetStats ¶
func (hm *HeartbeatMonitorImpl) GetStats() *HeartbeatStats
GetStats returns current heartbeat monitor statistics.
func (*HeartbeatMonitorImpl) GetStatus ¶
func (hm *HeartbeatMonitorImpl) GetStatus(provider string) (*HealthStatus, error)
GetStatus retrieves the last known status for a provider.
func (*HeartbeatMonitorImpl) IsRunning ¶
func (hm *HeartbeatMonitorImpl) IsRunning() bool
IsRunning returns true if the heartbeat monitor is currently running.
func (*HeartbeatMonitorImpl) RegisterChecker ¶
func (hm *HeartbeatMonitorImpl) RegisterChecker(checker ProviderHealthChecker) error
RegisterChecker registers a new provider health checker.
func (*HeartbeatMonitorImpl) RemoveEventHandler ¶
func (hm *HeartbeatMonitorImpl) RemoveEventHandler(handler HeartbeatEventHandler)
RemoveEventHandler removes an event handler.
func (*HeartbeatMonitorImpl) SetInterval ¶
func (hm *HeartbeatMonitorImpl) SetInterval(interval time.Duration)
SetInterval updates the heartbeat check interval.
func (*HeartbeatMonitorImpl) Start ¶
func (hm *HeartbeatMonitorImpl) Start(ctx context.Context) error
Start begins the heartbeat monitoring loop.
func (*HeartbeatMonitorImpl) Stop ¶
func (hm *HeartbeatMonitorImpl) Stop() error
Stop gracefully shuts down the monitor.
func (*HeartbeatMonitorImpl) UnregisterChecker ¶
func (hm *HeartbeatMonitorImpl) UnregisterChecker(provider string) error
UnregisterChecker removes a provider health checker.
type HeartbeatStats ¶
type HeartbeatStats struct {
// StartTime is when the monitor was started
StartTime time.Time `json:"start_time"`
// LastCycleTime is when the last full cycle completed
LastCycleTime time.Time `json:"last_cycle_time"`
// TotalCycles is the number of completed heartbeat cycles
TotalCycles int64 `json:"total_cycles"`
// TotalChecks is the total number of provider checks performed
TotalChecks int64 `json:"total_checks"`
// SuccessfulChecks is the number of successful checks
SuccessfulChecks int64 `json:"successful_checks"`
// FailedChecks is the number of failed checks
FailedChecks int64 `json:"failed_checks"`
// AverageCycleTime is the average time per heartbeat cycle
AverageCycleTime time.Duration `json:"average_cycle_time"`
// ProvidersMonitored is the number of providers being monitored
ProvidersMonitored int `json:"providers_monitored"`
// HealthyProviders is the number of currently healthy providers
HealthyProviders int `json:"healthy_providers"`
// DegradedProviders is the number of currently degraded providers
DegradedProviders int `json:"degraded_providers"`
UnavailableProviders int `json:"unavailable_providers"`
}
HeartbeatStats contains statistics about the heartbeat monitor.
type LoggingEventHandler ¶
type LoggingEventHandler struct {
// contains filtered or unexported fields
}
LoggingEventHandler implements HeartbeatEventHandler to log events.
func NewLoggingEventHandler ¶
func NewLoggingEventHandler() *LoggingEventHandler
NewLoggingEventHandler creates a new logging event handler.
func (*LoggingEventHandler) Clear ¶
func (h *LoggingEventHandler) Clear()
Clear clears all logged events.
func (*LoggingEventHandler) GetEvents ¶
func (h *LoggingEventHandler) GetEvents() []HeartbeatEvent
GetEvents returns all logged events.
func (*LoggingEventHandler) GetEventsByType ¶
func (h *LoggingEventHandler) GetEventsByType(eventType HeartbeatEventType) []HeartbeatEvent
GetEventsByType returns events of a specific type.
func (*LoggingEventHandler) GetEventsForProvider ¶
func (h *LoggingEventHandler) GetEventsForProvider(provider string) []HeartbeatEvent
GetEventsForProvider returns events for a specific provider.
func (*LoggingEventHandler) HandleEvent ¶
func (h *LoggingEventHandler) HandleEvent(event *HeartbeatEvent) error
HandleEvent logs heartbeat events.
type ModelDiscovery ¶
type ModelDiscovery struct {
// contains filtered or unexported fields
}
ModelDiscovery handles automatic discovery of models from providers.
func NewModelDiscovery ¶
func NewModelDiscovery(memory memory.MemoryManager) *ModelDiscovery
NewModelDiscovery creates a new model discovery service.
func (*ModelDiscovery) DiscoverOllamaModels ¶
func (d *ModelDiscovery) DiscoverOllamaModels(baseURL string) ([]*DiscoveredModel, error)
DiscoverOllamaModels discovers models from an Ollama instance.
func (*ModelDiscovery) SetEventCallback ¶
func (d *ModelDiscovery) SetEventCallback(callback func(event *HeartbeatEvent))
SetEventCallback sets the callback for emitting discovery events.
type OllamaHealthChecker ¶
type OllamaHealthChecker struct {
// contains filtered or unexported fields
}
func NewOllamaHealthChecker ¶
func NewOllamaHealthChecker(baseURL string) *OllamaHealthChecker
func (*OllamaHealthChecker) Check ¶
func (c *OllamaHealthChecker) Check(ctx context.Context) (*HealthStatus, error)
func (*OllamaHealthChecker) GetCheckInterval ¶
func (c *OllamaHealthChecker) GetCheckInterval() time.Duration
func (*OllamaHealthChecker) GetName ¶
func (c *OllamaHealthChecker) GetName() string
func (*OllamaHealthChecker) SupportsAutoDiscovery ¶
func (c *OllamaHealthChecker) SupportsAutoDiscovery() bool
func (*OllamaHealthChecker) SupportsQuotaMonitoring ¶
func (c *OllamaHealthChecker) SupportsQuotaMonitoring() bool
type ProviderHealthChecker ¶
type ProviderHealthChecker interface {
// Check performs a health check and returns the current status
Check(ctx context.Context) (*HealthStatus, error)
// GetName returns the provider name this checker handles
GetName() string
// GetCheckInterval returns the preferred check interval for this provider
GetCheckInterval() time.Duration
// SupportsQuotaMonitoring returns true if this provider supports quota monitoring
SupportsQuotaMonitoring() bool
// SupportsAutoDiscovery returns true if this provider supports model auto-discovery
SupportsAutoDiscovery() bool
}
ProviderHealthChecker defines the interface for provider-specific health checkers.
func NewClaudeAPIHealthChecker ¶
func NewClaudeAPIHealthChecker(apiKey string) ProviderHealthChecker
func NewGeminiAPIHealthChecker ¶
func NewGeminiAPIHealthChecker(apiKey string) ProviderHealthChecker
func NewGroqHealthChecker ¶
func NewGroqHealthChecker(apiKey, baseURL string) ProviderHealthChecker
func NewOpenAICompatibilityHealthChecker ¶
func NewOpenAICompatibilityHealthChecker(name, baseURL, apiKey string) ProviderHealthChecker
func NewOpenAIHealthChecker ¶
func NewOpenAIHealthChecker(apiKey string) ProviderHealthChecker
func NewSwitchAIHealthChecker ¶
func NewSwitchAIHealthChecker(apiKey, baseURL string) ProviderHealthChecker
type ProviderStatus ¶
type ProviderStatus string
ProviderStatus represents the health status of a provider.
const ( // StatusHealthy indicates the provider is fully operational StatusHealthy ProviderStatus = "healthy" // StatusDegraded indicates the provider is operational but with issues StatusDegraded ProviderStatus = "degraded" StatusUnavailable ProviderStatus = "unavailable" )
type QuotaStatusType ¶
type QuotaStatusType string
QuotaStatusType represents the status of a quota.
const ( QuotaOK QuotaStatusType = "ok" QuotaWarning QuotaStatusType = "warning" // > 80% QuotaCritical QuotaStatusType = "critical" // > 95% QuotaExceeded QuotaStatusType = "exceeded" // 100% )
func CalculateQuotaStatus ¶
func CalculateQuotaStatus(used, limit float64, warningThreshold, criticalThreshold float64) QuotaStatusType
CalculateQuotaStatus determines the status based on usage and limit.
type RecoveryAction ¶
type RecoveryAction struct {
// Timestamp is when the action was taken
Timestamp time.Time `json:"timestamp"`
// Provider is the provider this action was taken for
Provider string `json:"provider"`
// ActionType describes the type of recovery action
ActionType RecoveryActionType `json:"action_type"`
// Description provides details about the action
Description string `json:"description"`
// Success indicates whether the action succeeded
Success bool `json:"success"`
// Error contains error details if the action failed
Error string `json:"error,omitempty"`
// Metadata contains action-specific data
Metadata map[string]interface{} `json:"metadata,omitempty"`
}
RecoveryAction represents a recovery action taken in response to a health issue.
type RecoveryActionType ¶
type RecoveryActionType string
RecoveryActionType categorizes recovery actions.
const ( // ActionRestartConnection attempts to restart the provider connection ActionRestartConnection RecoveryActionType = "restart_connection" // ActionEnableFallback enables fallback routing to alternative providers ActionEnableFallback RecoveryActionType = "enable_fallback" // ActionDisableProvider temporarily disables the provider ActionDisableProvider RecoveryActionType = "disable_provider" // ActionEnableProvider re-enables a previously disabled provider ActionEnableProvider RecoveryActionType = "enable_provider" // ActionNotifyAdmin sends a notification to administrators ActionNotifyAdmin RecoveryActionType = "notify_admin" // ActionLogWarning logs a warning about the provider status ActionLogWarning RecoveryActionType = "log_warning" )
type RecoveryConfig ¶
type RecoveryConfig struct {
// Enabled controls whether automatic recovery is active
Enabled bool
// MaxRecoveryAttempts is the maximum number of recovery attempts per provider
MaxRecoveryAttempts int
// RecoveryBackoff is the minimum time between recovery attempts
RecoveryBackoff time.Duration
// AutoDisableThreshold is the number of consecutive failures before auto-disable
AutoDisableThreshold int
// AutoEnableDelay is how long to wait before re-enabling a disabled provider
AutoEnableDelay time.Duration
// EnableFallbackRouting controls whether fallback routing is enabled
EnableFallbackRouting bool
// NotifyAdminOnFailure controls whether to notify admins on provider failures
NotifyAdminOnFailure bool
}
RecoveryConfig holds configuration for the recovery manager.
func DefaultRecoveryConfig ¶
func DefaultRecoveryConfig() *RecoveryConfig
DefaultRecoveryConfig returns the default recovery configuration.
type RecoveryEventHandler ¶
type RecoveryEventHandler struct {
// contains filtered or unexported fields
}
RecoveryEventHandler implements HeartbeatEventHandler to trigger recovery actions.
func NewRecoveryEventHandler ¶
func NewRecoveryEventHandler(ctx context.Context, config *RecoveryConfig) *RecoveryEventHandler
NewRecoveryEventHandler creates a new recovery event handler.
func (*RecoveryEventHandler) GetRecoveryActions ¶
func (h *RecoveryEventHandler) GetRecoveryActions() []RecoveryAction
GetRecoveryActions returns all recovery actions.
func (*RecoveryEventHandler) GetRecoveryActionsForProvider ¶
func (h *RecoveryEventHandler) GetRecoveryActionsForProvider(provider string) []RecoveryAction
GetRecoveryActionsForProvider returns recovery actions for a specific provider.
func (*RecoveryEventHandler) GetRecoveryManager ¶
func (h *RecoveryEventHandler) GetRecoveryManager() *RecoveryManager
GetRecoveryManager returns the recovery manager.
func (*RecoveryEventHandler) GetRecoveryStats ¶
func (h *RecoveryEventHandler) GetRecoveryStats() *RecoveryStats
GetRecoveryStats returns recovery statistics.
func (*RecoveryEventHandler) HandleEvent ¶
func (h *RecoveryEventHandler) HandleEvent(event *HeartbeatEvent) error
HandleEvent processes heartbeat events and triggers recovery actions.
func (*RecoveryEventHandler) IsFallbackEnabled ¶
func (h *RecoveryEventHandler) IsFallbackEnabled(provider string) bool
IsFallbackEnabled returns whether fallback routing is enabled for a provider.
func (*RecoveryEventHandler) IsProviderDisabled ¶
func (h *RecoveryEventHandler) IsProviderDisabled(provider string) bool
IsProviderDisabled returns whether a provider is currently disabled.
type RecoveryManager ¶
type RecoveryManager struct {
// contains filtered or unexported fields
}
RecoveryManager handles automatic recovery actions for unhealthy providers.
func NewRecoveryManager ¶
func NewRecoveryManager(config *RecoveryConfig) *RecoveryManager
NewRecoveryManager creates a new recovery manager.
func (*RecoveryManager) GetActions ¶
func (rm *RecoveryManager) GetActions() []RecoveryAction
GetActions returns all recorded recovery actions.
func (*RecoveryManager) GetActionsForProvider ¶
func (rm *RecoveryManager) GetActionsForProvider(provider string) []RecoveryAction
GetActionsForProvider returns recovery actions for a specific provider.
func (*RecoveryManager) GetRecoveryAttempts ¶
func (rm *RecoveryManager) GetRecoveryAttempts(provider string) int
GetRecoveryAttempts returns the number of recovery attempts for a provider.
func (*RecoveryManager) GetStats ¶
func (rm *RecoveryManager) GetStats() *RecoveryStats
GetStats returns recovery statistics.
func (*RecoveryManager) HandleProviderDegraded ¶
func (rm *RecoveryManager) HandleProviderDegraded(ctx context.Context, provider string, status *HealthStatus) error
HandleProviderDegraded handles a provider becoming degraded.
func (*RecoveryManager) HandleProviderHealthy ¶
func (rm *RecoveryManager) HandleProviderHealthy(ctx context.Context, provider string, status *HealthStatus) error
HandleProviderHealthy handles a provider becoming healthy.
func (*RecoveryManager) HandleProviderUnavailable ¶
func (rm *RecoveryManager) HandleProviderUnavailable(ctx context.Context, provider string, status *HealthStatus) error
HandleProviderUnavailable handles a provider becoming unavailable.
func (*RecoveryManager) IsFallbackEnabled ¶
func (rm *RecoveryManager) IsFallbackEnabled(provider string) bool
IsFallbackEnabled returns whether fallback routing is enabled for a provider.
func (*RecoveryManager) IsProviderDisabled ¶
func (rm *RecoveryManager) IsProviderDisabled(provider string) bool
IsProviderDisabled returns whether a provider is currently disabled.
func (*RecoveryManager) ResetRecoveryAttempts ¶
func (rm *RecoveryManager) ResetRecoveryAttempts(provider string)
ResetRecoveryAttempts resets the recovery attempt counter for a provider.
type RecoveryStats ¶
type RecoveryStats struct {
// TotalActions is the total number of recovery actions taken
TotalActions int `json:"total_actions"`
// SuccessfulActions is the number of successful actions
SuccessfulActions int `json:"successful_actions"`
// FailedActions is the number of failed actions
FailedActions int `json:"failed_actions"`
// DisabledProviders is the number of currently disabled providers
DisabledProviders int `json:"disabled_providers"`
// ActionsByType breaks down actions by type
ActionsByType map[RecoveryActionType]int `json:"actions_by_type"`
// ActionsByProvider breaks down actions by provider
ActionsByProvider map[string]int `json:"actions_by_provider"`
}
RecoveryStats contains statistics about recovery actions.