Documentation
¶
Overview ¶
Package middleware provides middleware components for the Multi-Provider LLM Integration Framework.
Package middleware provides middleware components for the Multi-Provider LLM Integration Framework.
Package middleware provides middleware components for the Multi-Provider LLM Integration Framework.
Package middleware provides middleware components for the Multi-Provider LLM Integration Framework.
Package middleware provides middleware components for the Multi-Provider LLM Integration Framework.
Package middleware provides middleware components for the Multi-Provider LLM Integration Framework.
Index ¶
- type CircuitBreakerConfig
- type CircuitBreakerMiddleware
- func (cb *CircuitBreakerMiddleware) Execute(ctx context.Context, fn func(ctx context.Context) (interface{}, error)) (interface{}, error)
- func (cb *CircuitBreakerMiddleware) GetState() CircuitBreakerState
- func (cb *CircuitBreakerMiddleware) Reset()
- func (cb *CircuitBreakerMiddleware) UpdateConfig(config CircuitBreakerConfig)
- type CircuitBreakerState
- type LogEntry
- type LogHandler
- type LogLevel
- type LoggingMiddleware
- func (m *LoggingMiddleware) AddHandler(level LogLevel, handler LogHandler)
- func (m *LoggingMiddleware) AddRedactPattern(pattern string, replacement string) error
- func (m *LoggingMiddleware) ClearRedactPatterns()
- func (m *LoggingMiddleware) GetMinLevel() LogLevel
- func (m *LoggingMiddleware) IsRedactingPII() bool
- func (m *LoggingMiddleware) Log(entry *LogEntry)
- func (m *LoggingMiddleware) LogMiddleware(ctx context.Context, providerType core.ProviderType, operation string, ...) (interface{}, error)
- func (m *LoggingMiddleware) LogRequest(ctx context.Context, providerType core.ProviderType, operation string, ...) string
- func (m *LoggingMiddleware) LogResponse(ctx context.Context, providerType core.ProviderType, operation string, ...)
- func (m *LoggingMiddleware) RemoveHandlers(level LogLevel)
- func (m *LoggingMiddleware) SetMinLevel(level LogLevel)
- func (m *LoggingMiddleware) SetRedactPII(redact bool)
- type ProviderRateLimiter
- func (p *ProviderRateLimiter) DisableAllRateLimiters()
- func (p *ProviderRateLimiter) DisableRateLimiter(providerKey string)
- func (p *ProviderRateLimiter) EnableAllRateLimiters()
- func (p *ProviderRateLimiter) EnableRateLimiter(providerKey string)
- func (p *ProviderRateLimiter) GetAllRateLimiters() map[string]*RateLimiter
- func (p *ProviderRateLimiter) GetRateLimiter(providerKey string, ...) *RateLimiter
- func (p *ProviderRateLimiter) RemoveRateLimiter(providerKey string)
- func (p *ProviderRateLimiter) UpdateRateLimiter(providerKey string, ...)
- type QueuedRequest
- type QueuedResult
- type RateLimiter
- func (l *RateLimiter) Disable()
- func (l *RateLimiter) Enable()
- func (l *RateLimiter) GetLimits() (requestsPerMinute, tokensPerMinute, maxConcurrentRequests, burstSize int)
- func (l *RateLimiter) IsEnabled() bool
- func (l *RateLimiter) Release()
- func (l *RateLimiter) UpdateLimits(requestsPerMinute, tokensPerMinute, maxConcurrentRequests, burstSize int)
- func (l *RateLimiter) Wait(ctx context.Context) error
- func (l *RateLimiter) WaitForTokens(ctx context.Context, tokens int) error
- type RateLimiterMiddleware
- type RequestQueueConfig
- type RequestQueueMiddleware
- func (rq *RequestQueueMiddleware) Execute(ctx context.Context, priority int, ...) (interface{}, error)
- func (rq *RequestQueueMiddleware) GetQueueStats() map[string]interface{}
- func (rq *RequestQueueMiddleware) Start()
- func (rq *RequestQueueMiddleware) Stop()
- func (rq *RequestQueueMiddleware) UpdateConfig(config RequestQueueConfig)
- type RetryMiddleware
- type UsageMetrics
- type UsageTracker
- func (ut *UsageTracker) GetAllMetrics() map[string]*UsageMetrics
- func (ut *UsageTracker) GetMetrics(modelID string) *UsageMetrics
- func (ut *UsageTracker) ResetMetrics()
- func (ut *UsageTracker) SetResetInterval(resetInterval time.Duration)
- func (ut *UsageTracker) TrackRequest(modelID string, tokens int64, duration time.Duration, err error)
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type CircuitBreakerConfig ¶
type CircuitBreakerConfig struct {
// FailureThreshold is the number of consecutive failures that will open the circuit
FailureThreshold int
// ResetTimeout is the time to wait before trying again after the circuit is opened
ResetTimeout time.Duration
// HalfOpenSuccessThreshold is the number of consecutive successes that will close the circuit
HalfOpenSuccessThreshold int
}
CircuitBreakerConfig represents the configuration for the circuit breaker
type CircuitBreakerMiddleware ¶
type CircuitBreakerMiddleware struct {
// contains filtered or unexported fields
}
CircuitBreakerMiddleware provides circuit breaking functionality
func NewCircuitBreaker ¶
func NewCircuitBreaker(config CircuitBreakerConfig) *CircuitBreakerMiddleware
NewCircuitBreaker is an alias for NewCircuitBreakerMiddleware for backward compatibility
func NewCircuitBreakerMiddleware ¶
func NewCircuitBreakerMiddleware(config CircuitBreakerConfig) *CircuitBreakerMiddleware
NewCircuitBreakerMiddleware creates a new circuit breaker middleware
func (*CircuitBreakerMiddleware) Execute ¶
func (cb *CircuitBreakerMiddleware) Execute(ctx context.Context, fn func(ctx context.Context) (interface{}, error)) (interface{}, error)
Execute executes a function with circuit breaking
func (*CircuitBreakerMiddleware) GetState ¶
func (cb *CircuitBreakerMiddleware) GetState() CircuitBreakerState
GetState returns the current state of the circuit breaker
func (*CircuitBreakerMiddleware) Reset ¶
func (cb *CircuitBreakerMiddleware) Reset()
Reset resets the circuit breaker to its initial state
func (*CircuitBreakerMiddleware) UpdateConfig ¶
func (cb *CircuitBreakerMiddleware) UpdateConfig(config CircuitBreakerConfig)
UpdateConfig updates the circuit breaker configuration
type CircuitBreakerState ¶
type CircuitBreakerState int
CircuitBreakerState represents the state of the circuit breaker
const ( // CircuitBreakerStateClosed indicates that the circuit is closed and requests are allowed CircuitBreakerStateClosed CircuitBreakerState = iota // CircuitBreakerStateOpen indicates that the circuit is open and requests are not allowed CircuitBreakerStateOpen // CircuitBreakerStateHalfOpen indicates that the circuit is half-open and a limited number of requests are allowed CircuitBreakerStateHalfOpen )
type LogEntry ¶
type LogEntry struct {
// Timestamp is the timestamp of the log entry
Timestamp time.Time `json:"timestamp"`
// Level is the log level
Level LogLevel `json:"level"`
// ProviderType is the type of provider
ProviderType core.ProviderType `json:"provider_type"`
// Operation is the operation being performed
Operation string `json:"operation"`
// RequestID is the ID of the request
RequestID string `json:"request_id"`
// Request is the request data
Request interface{} `json:"request,omitempty"`
// Response is the response data
Response interface{} `json:"response,omitempty"`
// Error is the error message
Error string `json:"error,omitempty"`
// Duration is the duration of the operation
Duration time.Duration `json:"duration,omitempty"`
// AdditionalInfo is additional information
AdditionalInfo map[string]interface{} `json:"additional_info,omitempty"`
}
LogEntry represents a log entry
type LogHandler ¶
type LogHandler func(entry *LogEntry)
LogHandler is a function that handles log entries
func ConsoleLogHandler ¶
func ConsoleLogHandler() LogHandler
ConsoleLogHandler returns a log handler that logs to the console
func FileLogHandler ¶
func FileLogHandler(filePath string) (LogHandler, error)
FileLogHandler returns a log handler that logs to a file
func JSONLogHandler ¶
func JSONLogHandler(filePath string) (LogHandler, error)
JSONLogHandler returns a log handler that logs to a JSON file
type LoggingMiddleware ¶
type LoggingMiddleware struct {
// contains filtered or unexported fields
}
LoggingMiddleware provides logging functionality
func NewLoggingMiddleware ¶
func NewLoggingMiddleware(minLevel LogLevel, redactPII bool) *LoggingMiddleware
NewLoggingMiddleware creates a new logging middleware
func (*LoggingMiddleware) AddHandler ¶
func (m *LoggingMiddleware) AddHandler(level LogLevel, handler LogHandler)
AddHandler adds a log handler for a specific level
func (*LoggingMiddleware) AddRedactPattern ¶
func (m *LoggingMiddleware) AddRedactPattern(pattern string, replacement string) error
AddRedactPattern adds a pattern to redact
func (*LoggingMiddleware) ClearRedactPatterns ¶
func (m *LoggingMiddleware) ClearRedactPatterns()
ClearRedactPatterns clears all redact patterns
func (*LoggingMiddleware) GetMinLevel ¶
func (m *LoggingMiddleware) GetMinLevel() LogLevel
GetMinLevel returns the minimum log level
func (*LoggingMiddleware) IsRedactingPII ¶
func (m *LoggingMiddleware) IsRedactingPII() bool
IsRedactingPII returns whether PII is being redacted
func (*LoggingMiddleware) LogMiddleware ¶
func (m *LoggingMiddleware) LogMiddleware(ctx context.Context, providerType core.ProviderType, operation string, request interface{}, fn func(ctx context.Context) (interface{}, error)) (interface{}, error)
LogMiddleware is middleware that logs requests and responses
func (*LoggingMiddleware) LogRequest ¶
func (m *LoggingMiddleware) LogRequest(ctx context.Context, providerType core.ProviderType, operation string, request interface{}, additionalInfo map[string]interface{}) string
LogRequest logs a request
func (*LoggingMiddleware) LogResponse ¶
func (m *LoggingMiddleware) LogResponse(ctx context.Context, providerType core.ProviderType, operation string, requestID string, request interface{}, response interface{}, err error, duration time.Duration, additionalInfo map[string]interface{})
LogResponse logs a response
func (*LoggingMiddleware) RemoveHandlers ¶
func (m *LoggingMiddleware) RemoveHandlers(level LogLevel)
RemoveHandlers removes all handlers for a specific level
func (*LoggingMiddleware) SetMinLevel ¶
func (m *LoggingMiddleware) SetMinLevel(level LogLevel)
SetMinLevel sets the minimum log level
func (*LoggingMiddleware) SetRedactPII ¶
func (m *LoggingMiddleware) SetRedactPII(redact bool)
SetRedactPII sets whether to redact PII
type ProviderRateLimiter ¶
type ProviderRateLimiter struct {
// contains filtered or unexported fields
}
ProviderRateLimiter manages rate limiters for multiple providers
func NewProviderRateLimiter ¶
func NewProviderRateLimiter() *ProviderRateLimiter
NewProviderRateLimiter creates a new provider rate limiter
func (*ProviderRateLimiter) DisableAllRateLimiters ¶
func (p *ProviderRateLimiter) DisableAllRateLimiters()
DisableAllRateLimiters disables all rate limiters
func (*ProviderRateLimiter) DisableRateLimiter ¶
func (p *ProviderRateLimiter) DisableRateLimiter(providerKey string)
DisableRateLimiter disables a rate limiter for a provider
func (*ProviderRateLimiter) EnableAllRateLimiters ¶
func (p *ProviderRateLimiter) EnableAllRateLimiters()
EnableAllRateLimiters enables all rate limiters
func (*ProviderRateLimiter) EnableRateLimiter ¶
func (p *ProviderRateLimiter) EnableRateLimiter(providerKey string)
EnableRateLimiter enables a rate limiter for a provider
func (*ProviderRateLimiter) GetAllRateLimiters ¶
func (p *ProviderRateLimiter) GetAllRateLimiters() map[string]*RateLimiter
GetAllRateLimiters returns all rate limiters
func (*ProviderRateLimiter) GetRateLimiter ¶
func (p *ProviderRateLimiter) GetRateLimiter(providerKey string, requestsPerMinute, tokensPerMinute, maxConcurrentRequests, burstSize int) *RateLimiter
GetRateLimiter gets or creates a rate limiter for a provider
func (*ProviderRateLimiter) RemoveRateLimiter ¶
func (p *ProviderRateLimiter) RemoveRateLimiter(providerKey string)
RemoveRateLimiter removes a rate limiter for a provider
func (*ProviderRateLimiter) UpdateRateLimiter ¶
func (p *ProviderRateLimiter) UpdateRateLimiter(providerKey string, requestsPerMinute, tokensPerMinute, maxConcurrentRequests, burstSize int)
UpdateRateLimiter updates a rate limiter for a provider
type QueuedRequest ¶
type QueuedRequest struct {
// Priority is the priority of the request (lower is higher priority)
Priority int
// EnqueueTime is the time the request was enqueued
EnqueueTime time.Time
// Context is the context of the request
Context context.Context
// Function is the function to execute
Function func(ctx context.Context) (interface{}, error)
// ResultChan is the channel to send the result to
ResultChan chan *QueuedResult
}
QueuedRequest represents a request in the queue
type QueuedResult ¶
type QueuedResult struct {
// Result is the result of the request
Result interface{}
// Error is the error from the request
Error error
}
QueuedResult represents the result of a queued request
type RateLimiter ¶
type RateLimiter struct {
// contains filtered or unexported fields
}
RateLimiter provides rate limiting functionality for API requests
func NewRateLimiter ¶
func NewRateLimiter(requestsPerMinute, tokensPerMinute, maxConcurrentRequests, burstSize int) *RateLimiter
NewRateLimiter creates a new rate limiter
func (*RateLimiter) GetLimits ¶
func (l *RateLimiter) GetLimits() (requestsPerMinute, tokensPerMinute, maxConcurrentRequests, burstSize int)
GetLimits returns the current rate limits
func (*RateLimiter) IsEnabled ¶
func (l *RateLimiter) IsEnabled() bool
IsEnabled returns whether rate limiting is enabled
func (*RateLimiter) Release ¶
func (l *RateLimiter) Release()
Release releases the concurrency limiter
func (*RateLimiter) UpdateLimits ¶
func (l *RateLimiter) UpdateLimits(requestsPerMinute, tokensPerMinute, maxConcurrentRequests, burstSize int)
UpdateLimits updates the rate limits
func (*RateLimiter) Wait ¶
func (l *RateLimiter) Wait(ctx context.Context) error
Wait waits for rate limiting to allow a request
func (*RateLimiter) WaitForTokens ¶
func (l *RateLimiter) WaitForTokens(ctx context.Context, tokens int) error
WaitForTokens waits for rate limiting to allow tokens
type RateLimiterMiddleware ¶
type RateLimiterMiddleware struct {
// contains filtered or unexported fields
}
RateLimiterMiddleware is middleware that applies rate limiting to a provider
func NewRateLimiterMiddleware ¶
func NewRateLimiterMiddleware(requestsPerMinute, tokensPerMinute, maxConcurrentRequests, burstSize int) *RateLimiterMiddleware
NewRateLimiterMiddleware creates a new rate limiter middleware
func (*RateLimiterMiddleware) Execute ¶
func (m *RateLimiterMiddleware) Execute(ctx context.Context, tokens int, fn func(ctx context.Context) error) error
Execute executes a function with rate limiting
func (*RateLimiterMiddleware) GetRateLimiter ¶
func (m *RateLimiterMiddleware) GetRateLimiter() *RateLimiter
GetRateLimiter returns the rate limiter
type RequestQueueConfig ¶
type RequestQueueConfig struct {
// MaxQueueSize is the maximum number of requests that can be queued
MaxQueueSize int
// MaxWaitTime is the maximum time a request can wait in the queue
MaxWaitTime time.Duration
// PriorityLevels is the number of priority levels
PriorityLevels int
}
RequestQueueConfig represents the configuration for the request queue
type RequestQueueMiddleware ¶
type RequestQueueMiddleware struct {
// contains filtered or unexported fields
}
RequestQueueMiddleware provides request queuing functionality
func NewRequestQueueMiddleware ¶
func NewRequestQueueMiddleware(config RequestQueueConfig, workerCount int) *RequestQueueMiddleware
NewRequestQueueMiddleware creates a new request queue middleware
func (*RequestQueueMiddleware) Execute ¶
func (rq *RequestQueueMiddleware) Execute(ctx context.Context, priority int, fn func(ctx context.Context) (interface{}, error)) (interface{}, error)
Execute executes a function with request queuing
func (*RequestQueueMiddleware) GetQueueStats ¶
func (rq *RequestQueueMiddleware) GetQueueStats() map[string]interface{}
GetQueueStats returns statistics about the queue
func (*RequestQueueMiddleware) Start ¶
func (rq *RequestQueueMiddleware) Start()
Start starts the request queue workers
func (*RequestQueueMiddleware) Stop ¶
func (rq *RequestQueueMiddleware) Stop()
Stop stops the request queue workers
func (*RequestQueueMiddleware) UpdateConfig ¶
func (rq *RequestQueueMiddleware) UpdateConfig(config RequestQueueConfig)
UpdateConfig updates the request queue configuration
type RetryMiddleware ¶
type RetryMiddleware struct {
// contains filtered or unexported fields
}
RetryMiddleware provides retry functionality for providers
func NewRetryMiddleware ¶
func NewRetryMiddleware(config *core.RetryConfig) *RetryMiddleware
func (*RetryMiddleware) Execute ¶
func (m *RetryMiddleware) Execute(ctx context.Context, fn func(ctx context.Context) (interface{}, error)) (interface{}, error)
Execute executes a function with retries
func (*RetryMiddleware) GetConfig ¶
func (m *RetryMiddleware) GetConfig() *core.RetryConfig
GetConfig returns the retry configuration
func (*RetryMiddleware) UpdateConfig ¶
func (m *RetryMiddleware) UpdateConfig(config *core.RetryConfig)
UpdateConfig updates the retry configuration
type UsageMetrics ¶
type UsageMetrics struct {
// Requests is the number of requests made
Requests int64
// Tokens is the number of tokens used
Tokens int64
// Errors is the number of errors encountered
Errors int64
// LastRequestTime is the time of the last request
LastRequestTime time.Time
// TotalRequestDuration is the total duration of all requests
TotalRequestDuration time.Duration
}
UsageMetrics represents the usage metrics for a provider
type UsageTracker ¶
type UsageTracker struct {
// contains filtered or unexported fields
}
UsageTracker provides usage tracking functionality
func NewUsageTracker ¶
func NewUsageTracker(resetInterval time.Duration) *UsageTracker
NewUsageTracker creates a new usage tracker
func (*UsageTracker) GetAllMetrics ¶
func (ut *UsageTracker) GetAllMetrics() map[string]*UsageMetrics
GetAllMetrics returns the usage metrics for all models
func (*UsageTracker) GetMetrics ¶
func (ut *UsageTracker) GetMetrics(modelID string) *UsageMetrics
GetMetrics returns the usage metrics for a model
func (*UsageTracker) ResetMetrics ¶
func (ut *UsageTracker) ResetMetrics()
ResetMetrics resets the usage metrics
func (*UsageTracker) SetResetInterval ¶
func (ut *UsageTracker) SetResetInterval(resetInterval time.Duration)
SetResetInterval sets the reset interval
func (*UsageTracker) TrackRequest ¶
func (ut *UsageTracker) TrackRequest(modelID string, tokens int64, duration time.Duration, err error)
TrackRequest tracks a request