core

package
v0.6.2-security-analysis Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 3, 2025 License: MIT Imports: 9 Imported by: 0

Documentation

Overview

Package core provides the core interfaces and types for the Multi-Provider LLM Integration Framework.

Package core provides the core interfaces and types for the Multi-Provider LLM Integration Framework.

Package core provides the core interfaces and types for the Multi-Provider LLM Integration Framework.

Package core provides the core interfaces and types for the Multi-Provider LLM Integration Framework.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type BaseProvider

type BaseProvider struct {
	// contains filtered or unexported fields
}

BaseProvider is a base implementation of the Provider interface that can be embedded in specific provider implementations

func NewBaseProvider

func NewBaseProvider(providerType ProviderType, config *ProviderConfig) *BaseProvider

NewBaseProvider creates a new base provider

func (*BaseProvider) AddCapability

func (p *BaseProvider) AddCapability(capability ModelCapability)

AddCapability adds a capability to the provider

func (*BaseProvider) ChatCompletion

func (p *BaseProvider) ChatCompletion(ctx context.Context, request *ChatCompletionRequest) (*ChatCompletionResponse, error)

ChatCompletion generates a chat completion This method should be overridden by specific provider implementations

func (*BaseProvider) Close

func (p *BaseProvider) Close() error

Close closes the provider and releases any resources

func (*BaseProvider) CountTokens

func (p *BaseProvider) CountTokens(ctx context.Context, text string, modelID string) (int, error)

CountTokens counts the number of tokens in a text This method should be overridden by specific provider implementations

func (*BaseProvider) CreateEmbedding

func (p *BaseProvider) CreateEmbedding(ctx context.Context, request *EmbeddingRequest) (*EmbeddingResponse, error)

CreateEmbedding creates an embedding This method should be overridden by specific provider implementations

func (*BaseProvider) GetAllUsageMetrics

func (p *BaseProvider) GetAllUsageMetrics() (map[string]*UsageMetrics, error)

GetAllUsageMetrics returns the usage metrics for all models

func (*BaseProvider) GetCapabilities

func (p *BaseProvider) GetCapabilities() []ModelCapability

GetCapabilities returns the capabilities supported by the provider

func (*BaseProvider) GetConfig

func (p *BaseProvider) GetConfig() *ProviderConfig

GetConfig returns the configuration for the provider

func (*BaseProvider) GetModelInfo

func (p *BaseProvider) GetModelInfo(ctx context.Context, modelID string) (*ModelInfo, error)

GetModelInfo returns information about a specific model

func (*BaseProvider) GetModels

func (p *BaseProvider) GetModels(ctx context.Context) ([]ModelInfo, error)

GetModels returns a list of available models

func (*BaseProvider) GetRateLimitConfig

func (p *BaseProvider) GetRateLimitConfig() *RateLimitConfig

GetRateLimitConfig returns the rate limit configuration

func (*BaseProvider) GetRetryConfig

func (p *BaseProvider) GetRetryConfig() *RetryConfig

GetRetryConfig returns the retry configuration

func (*BaseProvider) GetType

func (p *BaseProvider) GetType() ProviderType

GetType returns the type of provider

func (*BaseProvider) GetUsageMetrics

func (p *BaseProvider) GetUsageMetrics(modelID string) (*UsageMetrics, error)

GetUsageMetrics returns the usage metrics for a specific model

func (*BaseProvider) RemoveCapability

func (p *BaseProvider) RemoveCapability(capability ModelCapability)

RemoveCapability removes a capability from the provider

func (*BaseProvider) ResetUsageMetrics

func (p *BaseProvider) ResetUsageMetrics() error

ResetUsageMetrics resets the usage metrics

func (*BaseProvider) SetConfig

func (p *BaseProvider) SetConfig(config *ProviderConfig) error

SetConfig sets the provider configuration

func (*BaseProvider) SetModels

func (p *BaseProvider) SetModels(models []ModelInfo)

SetModels sets the available models

func (*BaseProvider) SetModelsCacheTTL

func (p *BaseProvider) SetModelsCacheTTL(ttl time.Duration)

SetModelsCacheTTL sets the TTL for the models cache

func (*BaseProvider) StreamingChatCompletion

func (p *BaseProvider) StreamingChatCompletion(ctx context.Context, request *ChatCompletionRequest, callback func(response *ChatCompletionResponse) error) error

StreamingChatCompletion generates a streaming chat completion This method should be overridden by specific provider implementations

func (*BaseProvider) SupportsCapability

func (p *BaseProvider) SupportsCapability(ctx context.Context, capability ModelCapability) bool

SupportsCapability returns whether the provider supports a specific capability

func (*BaseProvider) SupportsModel

func (p *BaseProvider) SupportsModel(ctx context.Context, modelID string) bool

SupportsModel returns whether the provider supports a specific model

func (*BaseProvider) TextCompletion

func (p *BaseProvider) TextCompletion(ctx context.Context, request *TextCompletionRequest) (*TextCompletionResponse, error)

TextCompletion generates a text completion This method should be overridden by specific provider implementations

func (*BaseProvider) UpdateConfig

func (p *BaseProvider) UpdateConfig(updates *ProviderConfig) error

UpdateConfig updates the provider configuration

func (*BaseProvider) UpdateRateLimitConfig

func (p *BaseProvider) UpdateRateLimitConfig(config *RateLimitConfig) error

UpdateRateLimitConfig updates the rate limit configuration

func (*BaseProvider) UpdateRetryConfig

func (p *BaseProvider) UpdateRetryConfig(config *RetryConfig) error

UpdateRetryConfig updates the retry configuration

func (*BaseProvider) UpdateUsageMetrics

func (p *BaseProvider) UpdateUsageMetrics(modelID string, tokens int64, duration time.Duration, err error)

UpdateUsageMetrics updates the usage metrics for a model

func (*BaseProvider) ValidateConfig

func (p *BaseProvider) ValidateConfig() error

ValidateConfig validates the provider configuration

type ChatCompletionChoice

type ChatCompletionChoice struct {
	// Index is the index of the choice
	Index int `json:"index"`
	// Message is the message
	Message Message `json:"message"`
	// FinishReason is the reason the completion finished
	FinishReason string `json:"finish_reason"`
}

ChatCompletionChoice represents a choice in a chat completion response

type ChatCompletionRequest

type ChatCompletionRequest struct {
	// Messages is the list of messages in the conversation
	Messages []Message `json:"messages"`
	// MaxTokens is the maximum number of tokens to generate
	MaxTokens int `json:"max_tokens,omitempty"`
	// Temperature controls randomness (0.0 to 2.0)
	Temperature float64 `json:"temperature,omitempty"`
	// TopP controls diversity via nucleus sampling (0.0 to 1.0)
	TopP float64 `json:"top_p,omitempty"`
	// N is the number of completions to generate
	N int `json:"n,omitempty"`
	// Stop is a list of tokens at which to stop generation
	Stop []string `json:"stop,omitempty"`
	// Stream indicates whether to stream the response
	Stream bool `json:"stream,omitempty"`
	// PresencePenalty penalizes new tokens based on presence in text so far (0.0 to 2.0)
	PresencePenalty float64 `json:"presence_penalty,omitempty"`
	// FrequencyPenalty penalizes new tokens based on frequency in text so far (0.0 to 2.0)
	FrequencyPenalty float64 `json:"frequency_penalty,omitempty"`
	// LogitBias is a map from token IDs to bias values (-100 to 100)
	LogitBias map[string]float64 `json:"logit_bias,omitempty"`
	// User is an optional user identifier
	User string `json:"user,omitempty"`
	// Functions is a list of function definitions
	Functions []Function `json:"functions,omitempty"`
	// FunctionCall controls function calling behavior
	FunctionCall interface{} `json:"function_call,omitempty"`
	// Tools is a list of tool definitions
	Tools []Tool `json:"tools,omitempty"`
	// ToolChoice controls tool choice behavior
	ToolChoice interface{} `json:"tool_choice,omitempty"`
	// ResponseFormat specifies the format of the response
	ResponseFormat map[string]string `json:"response_format,omitempty"`
	// Model is the model to use
	Model string `json:"model,omitempty"`
	// Metadata is optional metadata for the request
	Metadata map[string]interface{} `json:"metadata,omitempty"`
}

ChatCompletionRequest represents a request for chat completion

type ChatCompletionResponse

type ChatCompletionResponse struct {
	// ID is the ID of the completion
	ID string `json:"id"`
	// Object is the object type
	Object string `json:"object"`
	// Created is the Unix timestamp of when the completion was created
	Created int64 `json:"created"`
	// Model is the model used
	Model string `json:"model"`
	// Choices is the list of completion choices
	Choices []ChatCompletionChoice `json:"choices"`
	// Usage is the token usage information
	Usage *TokenUsage `json:"usage,omitempty"`
}

ChatCompletionResponse represents a response from chat completion

type ConnectionPoolConfig added in v0.2.0

type ConnectionPoolConfig struct {
	// Pool sizing
	MaxIdleConns        int `json:"max_idle_conns"`
	MaxIdleConnsPerHost int `json:"max_idle_conns_per_host"`
	MaxConnsPerHost     int `json:"max_conns_per_host"`

	// Timeouts
	IdleConnTimeout       time.Duration `json:"idle_conn_timeout"`
	TLSHandshakeTimeout   time.Duration `json:"tls_handshake_timeout"`
	ExpectContinueTimeout time.Duration `json:"expect_continue_timeout"`
	ResponseHeaderTimeout time.Duration `json:"response_header_timeout"`

	// Keep-alive settings
	KeepAlive          time.Duration `json:"keep_alive"`
	DisableKeepAlives  bool          `json:"disable_keep_alives"`
	DisableCompression bool          `json:"disable_compression"`

	// TLS settings
	InsecureSkipVerify bool `json:"insecure_skip_verify"`

	// Health check settings
	HealthCheckInterval time.Duration `json:"health_check_interval"`
	HealthCheckTimeout  time.Duration `json:"health_check_timeout"`

	// Provider-specific settings
	ProviderType ProviderType `json:"provider_type"`
	BaseURL      string       `json:"base_url"`
}

ConnectionPoolConfig defines configuration for HTTP connection pools

func DefaultConnectionPoolConfig added in v0.2.0

func DefaultConnectionPoolConfig() ConnectionPoolConfig

DefaultConnectionPoolConfig returns default configuration

type ConnectionPoolManager added in v0.2.0

type ConnectionPoolManager struct {
	// contains filtered or unexported fields
}

ConnectionPoolManager manages HTTP connection pools for LLM providers

func NewConnectionPoolManager added in v0.2.0

func NewConnectionPoolManager(config ConnectionPoolConfig, logger Logger) *ConnectionPoolManager

NewConnectionPoolManager creates a new connection pool manager

func (*ConnectionPoolManager) CreatePool added in v0.2.0

CreatePool creates a connection pool for a specific provider

func (*ConnectionPoolManager) GetClient added in v0.2.0

func (m *ConnectionPoolManager) GetClient(providerType ProviderType) (*http.Client, error)

GetClient returns an HTTP client for a provider with connection pooling

func (*ConnectionPoolManager) GetMetrics added in v0.2.0

GetMetrics returns current connection pool metrics

func (*ConnectionPoolManager) GetPool added in v0.2.0

func (m *ConnectionPoolManager) GetPool(providerType ProviderType) (*ProviderConnectionPool, error)

GetPool returns the connection pool for a provider

func (*ConnectionPoolManager) Start added in v0.2.0

func (m *ConnectionPoolManager) Start() error

Start starts the connection pool manager and health checkers

func (*ConnectionPoolManager) Stop added in v0.2.0

func (m *ConnectionPoolManager) Stop() error

Stop stops the connection pool manager

type ConnectionPoolMetrics added in v0.2.0

type ConnectionPoolMetrics struct {
	TotalPools         int64 `json:"total_pools"`
	ActiveConnections  int64 `json:"active_connections"`
	IdleConnections    int64 `json:"idle_connections"`
	ConnectionsCreated int64 `json:"connections_created"`
	ConnectionsReused  int64 `json:"connections_reused"`
	ConnectionErrors   int64 `json:"connection_errors"`
	HealthChecksPassed int64 `json:"health_checks_passed"`
	HealthChecksFailed int64 `json:"health_checks_failed"`

	// Per-provider metrics
	ProviderMetrics map[ProviderType]*ProviderPoolMetrics `json:"provider_metrics"`
}

ConnectionPoolMetrics tracks connection pool performance

func NewConnectionPoolMetrics added in v0.2.0

func NewConnectionPoolMetrics() *ConnectionPoolMetrics

NewConnectionPoolMetrics creates new connection pool metrics

type DefaultLogger added in v0.2.0

type DefaultLogger struct {
	// contains filtered or unexported fields
}

DefaultLogger provides a simple logger implementation

func NewDefaultLogger added in v0.2.0

func NewDefaultLogger() *DefaultLogger

NewDefaultLogger creates a new default logger

func (*DefaultLogger) Debug added in v0.2.0

func (l *DefaultLogger) Debug(msg string, args ...interface{})

func (*DefaultLogger) Error added in v0.2.0

func (l *DefaultLogger) Error(msg string, args ...interface{})

func (*DefaultLogger) Info added in v0.2.0

func (l *DefaultLogger) Info(msg string, args ...interface{})

func (*DefaultLogger) Warn added in v0.2.0

func (l *DefaultLogger) Warn(msg string, args ...interface{})

type DefaultModelRegistry

type DefaultModelRegistry struct {
	// contains filtered or unexported fields
}

DefaultModelRegistry is the default implementation of the ModelRegistry interface

func NewDefaultModelRegistry

func NewDefaultModelRegistry() *DefaultModelRegistry

NewDefaultModelRegistry creates a new default model registry

func (*DefaultModelRegistry) GetAllModels

func (r *DefaultModelRegistry) GetAllModels() []*ModelInfo

GetAllModels returns all registered models

func (*DefaultModelRegistry) GetModel

func (r *DefaultModelRegistry) GetModel(modelID string) (*ModelInfo, error)

GetModel returns a model by ID

func (*DefaultModelRegistry) GetModelsByCapability

func (r *DefaultModelRegistry) GetModelsByCapability(capability ModelCapability) ([]*ModelInfo, error)

GetModelsByCapability returns models by capability

func (*DefaultModelRegistry) GetModelsByProvider

func (r *DefaultModelRegistry) GetModelsByProvider(providerType ProviderType) ([]*ModelInfo, error)

GetModelsByProvider returns models by provider

func (*DefaultModelRegistry) GetModelsByType

func (r *DefaultModelRegistry) GetModelsByType(modelType ModelType) ([]*ModelInfo, error)

GetModelsByType returns models by type

func (*DefaultModelRegistry) RegisterModel

func (r *DefaultModelRegistry) RegisterModel(model *ModelInfo) error

RegisterModel registers a model

type DefaultProviderFactory

type DefaultProviderFactory struct {
	// contains filtered or unexported fields
}

DefaultProviderFactory is the default implementation of the ProviderFactory interface

func NewDefaultProviderFactory

func NewDefaultProviderFactory() *DefaultProviderFactory

NewDefaultProviderFactory creates a new default provider factory

func (*DefaultProviderFactory) CreateProvider

func (f *DefaultProviderFactory) CreateProvider(config *ProviderConfig) (Provider, error)

CreateProvider creates a provider with the given configuration

func (*DefaultProviderFactory) GetSupportedProviderTypes

func (f *DefaultProviderFactory) GetSupportedProviderTypes() []ProviderType

GetSupportedProviderTypes returns the provider types supported by this factory

func (*DefaultProviderFactory) RegisterProviderCreator

func (f *DefaultProviderFactory) RegisterProviderCreator(providerType ProviderType, creator ProviderCreator)

RegisterProviderCreator registers a provider creator function for a provider type

type DefaultProviderRegistry

type DefaultProviderRegistry struct {
	// contains filtered or unexported fields
}

DefaultProviderRegistry is the default implementation of the ProviderRegistry interface

func NewDefaultProviderRegistry

func NewDefaultProviderRegistry() *DefaultProviderRegistry

NewDefaultProviderRegistry creates a new default provider registry

func (*DefaultProviderRegistry) GetAllProviderTypes

func (r *DefaultProviderRegistry) GetAllProviderTypes() []ProviderType

GetAllProviderTypes returns all registered provider types

func (*DefaultProviderRegistry) GetAllProviders

func (r *DefaultProviderRegistry) GetAllProviders() []Provider

GetAllProviders returns all registered providers

func (*DefaultProviderRegistry) GetProvider

func (r *DefaultProviderRegistry) GetProvider(providerType ProviderType) (Provider, error)

GetProvider returns a provider by type

func (*DefaultProviderRegistry) GetProviderByCapability

func (r *DefaultProviderRegistry) GetProviderByCapability(capability ModelCapability) (Provider, error)

GetProviderByCapability returns a provider that supports a specific capability

func (*DefaultProviderRegistry) GetProviderByModel

func (r *DefaultProviderRegistry) GetProviderByModel(modelID string) (Provider, error)

GetProviderByModel returns a provider that supports a specific model

func (*DefaultProviderRegistry) RegisterProvider

func (r *DefaultProviderRegistry) RegisterProvider(provider Provider) error

RegisterProvider registers a provider

func (*DefaultProviderRegistry) RegisterProviderFactory

func (r *DefaultProviderRegistry) RegisterProviderFactory(factory ProviderFactory) error

RegisterProviderFactory registers a provider factory

type Embedding

type Embedding struct {
	// Object is the object type
	Object string `json:"object"`
	// Embedding is the embedding vector
	Embedding []float64 `json:"embedding"`
	// Index is the index of the embedding
	Index int `json:"index"`
}

Embedding represents an embedding

type EmbeddingRequest

type EmbeddingRequest struct {
	// Input is the input to embed
	Input interface{} `json:"input"`
	// Model is the model to use
	Model string `json:"model"`
	// EncodingFormat is the encoding format
	EncodingFormat string `json:"encoding_format,omitempty"`
	// User is an optional user identifier
	User string `json:"user,omitempty"`
	// Dimensions is the number of dimensions for the embeddings
	Dimensions int `json:"dimensions,omitempty"`
}

EmbeddingRequest represents a request for embeddings

type EmbeddingResponse

type EmbeddingResponse struct {
	// Object is the object type
	Object string `json:"object"`
	// Data is the list of embeddings
	Data []Embedding `json:"data"`
	// Model is the model used
	Model string `json:"model"`
	// Usage is the token usage information
	Usage *TokenUsage `json:"usage,omitempty"`
}

EmbeddingResponse represents a response from embedding

type Function

type Function struct {
	// Name is the name of the function
	Name string `json:"name"`
	// Description is a description of the function
	Description string `json:"description"`
	// Parameters is a JSON schema of parameters
	Parameters map[string]interface{} `json:"parameters"`
}

Function represents a function definition

type FunctionCall

type FunctionCall struct {
	// Name is the name of the function
	Name string `json:"name"`
	// Arguments is a JSON string of arguments
	Arguments string `json:"arguments"`
}

FunctionCall represents a function call

type HealthChecker added in v0.2.0

type HealthChecker struct {
	// contains filtered or unexported fields
}

HealthChecker performs periodic health checks on connections

type LogProbs

type LogProbs struct {
	// Tokens is the list of tokens
	Tokens []string `json:"tokens"`
	// TokenLogProbs is the list of log probabilities for tokens
	TokenLogProbs []float64 `json:"token_logprobs"`
	// TopLogProbs is a list of maps from tokens to log probabilities
	TopLogProbs []map[string]float64 `json:"top_logprobs"`
	// TextOffset is a list of offsets in the text
	TextOffset []int `json:"text_offset"`
}

LogProbs represents log probabilities

type Logger added in v0.2.0

type Logger interface {
	Info(msg string, args ...interface{})
	Warn(msg string, args ...interface{})
	Error(msg string, args ...interface{})
	Debug(msg string, args ...interface{})
}

Logger interface for connection pool logging

type Message

type Message struct {
	// Role is the role of the message sender (e.g., "system", "user", "assistant")
	Role string `json:"role"`
	// Content is the content of the message
	Content string `json:"content"`
	// Name is an optional name for the message sender
	Name string `json:"name,omitempty"`
	// FunctionCall is an optional function call
	FunctionCall *FunctionCall `json:"function_call,omitempty"`
	// ToolCalls is an optional list of tool calls
	ToolCalls []*ToolCall `json:"tool_calls,omitempty"`
	// Timestamp is the timestamp of the message
	Timestamp time.Time `json:"timestamp,omitempty"`
}

Message represents a message in a conversation

type ModelCapability

type ModelCapability string

ModelCapability represents a capability of a model

const (
	// TextCompletionCapability represents the capability to generate text completions
	TextCompletionCapability ModelCapability = "text-completion"
	// ChatCompletionCapability represents the capability to generate chat completions
	ChatCompletionCapability ModelCapability = "chat-completion"
	// EmbeddingCapability represents the capability to generate embeddings
	EmbeddingCapability ModelCapability = "embedding"
	// ImageGenerationCapability represents the capability to generate images
	ImageGenerationCapability ModelCapability = "image-generation"
	// ImageAnalysisCapability represents the capability to analyze images
	ImageAnalysisCapability ModelCapability = "image-analysis"
	// StreamingCapability represents the capability to stream responses
	StreamingCapability ModelCapability = "streaming"
	// FunctionCallingCapability represents the capability to call functions
	FunctionCallingCapability ModelCapability = "function-calling"
	// ToolUseCapability represents the capability to use tools
	ToolUseCapability ModelCapability = "tool-use"
	// JSONModeCapability represents the capability to output JSON
	JSONModeCapability ModelCapability = "json-mode"
)

type ModelInfo

type ModelInfo struct {
	// ID is the ID of the model
	ID string `json:"id"`
	// Provider is the provider of the model
	Provider ProviderType `json:"provider"`
	// Type is the type of model
	Type ModelType `json:"type"`
	// Capabilities is a list of capabilities of the model
	Capabilities []ModelCapability `json:"capabilities"`
	// MaxTokens is the maximum number of tokens the model can process
	MaxTokens int `json:"max_tokens"`
	// TrainingCutoff is the training cutoff date of the model
	TrainingCutoff time.Time `json:"training_cutoff,omitempty"`
	// Version is the version of the model
	Version string `json:"version,omitempty"`
	// Description is a description of the model
	Description string `json:"description,omitempty"`
	// PricingInfo is information about pricing for the model
	PricingInfo *ModelPricingInfo `json:"pricing_info,omitempty"`
}

ModelInfo represents information about a model

type ModelPricingInfo

type ModelPricingInfo struct {
	// InputPricePerToken is the price per input token
	InputPricePerToken float64 `json:"input_price_per_token"`
	// OutputPricePerToken is the price per output token
	OutputPricePerToken float64 `json:"output_price_per_token"`
	// Currency is the currency of the prices
	Currency string `json:"currency"`
}

ModelPricingInfo represents pricing information for a model

type ModelRegistry

type ModelRegistry interface {
	// RegisterModel registers a model
	RegisterModel(model *ModelInfo) error

	// GetModel returns a model by ID
	GetModel(modelID string) (*ModelInfo, error)

	// GetModelsByProvider returns models by provider
	GetModelsByProvider(providerType ProviderType) ([]*ModelInfo, error)

	// GetModelsByType returns models by type
	GetModelsByType(modelType ModelType) ([]*ModelInfo, error)

	// GetModelsByCapability returns models by capability
	GetModelsByCapability(capability ModelCapability) ([]*ModelInfo, error)

	// GetAllModels returns all registered models
	GetAllModels() []*ModelInfo
}

ModelRegistry is the interface for registering and retrieving models

type ModelType

type ModelType string

ModelType represents the type of LLM model

const (
	// TextCompletionModel represents a text completion model
	TextCompletionModel ModelType = "text-completion"
	// ChatModel represents a chat model
	ChatModel ModelType = "chat"
	// EmbeddingModel represents an embedding model
	EmbeddingModel ModelType = "embedding"
	// ImageGenerationModel represents an image generation model
	ImageGenerationModel ModelType = "image-generation"
	// ImageAnalysisModel represents an image analysis model
	ImageAnalysisModel ModelType = "image-analysis"
)

type Provider

type Provider interface {
	// GetType returns the type of provider
	GetType() ProviderType
	// GetConfig returns the configuration for the provider
	GetConfig() *ProviderConfig
	// GetModels returns a list of available models
	GetModels(ctx context.Context) ([]ModelInfo, error)
	// GetModelInfo returns information about a specific model
	GetModelInfo(ctx context.Context, modelID string) (*ModelInfo, error)
	// TextCompletion generates a text completion
	TextCompletion(ctx context.Context, request *TextCompletionRequest) (*TextCompletionResponse, error)
	// ChatCompletion generates a chat completion
	ChatCompletion(ctx context.Context, request *ChatCompletionRequest) (*ChatCompletionResponse, error)
	// StreamingChatCompletion generates a streaming chat completion
	StreamingChatCompletion(ctx context.Context, request *ChatCompletionRequest, callback func(response *ChatCompletionResponse) error) error
	// CreateEmbedding creates an embedding
	CreateEmbedding(ctx context.Context, request *EmbeddingRequest) (*EmbeddingResponse, error)
	// CountTokens counts the number of tokens in a text
	CountTokens(ctx context.Context, text string, modelID string) (int, error)
	// SupportsModel returns whether the provider supports a specific model
	SupportsModel(ctx context.Context, modelID string) bool
	// SupportsCapability returns whether the provider supports a specific capability
	SupportsCapability(ctx context.Context, capability ModelCapability) bool
	// Close closes the provider and releases any resources
	Close() error

	// GetRateLimitConfig returns the rate limit configuration
	GetRateLimitConfig() *RateLimitConfig
	// UpdateRateLimitConfig updates the rate limit configuration
	UpdateRateLimitConfig(config *RateLimitConfig) error
	// GetRetryConfig returns the retry configuration
	GetRetryConfig() *RetryConfig
	// UpdateRetryConfig updates the retry configuration
	UpdateRetryConfig(config *RetryConfig) error
	// GetUsageMetrics returns the usage metrics for a specific model
	GetUsageMetrics(modelID string) (*UsageMetrics, error)
	// GetAllUsageMetrics returns the usage metrics for all models
	GetAllUsageMetrics() (map[string]*UsageMetrics, error)
	// ResetUsageMetrics resets the usage metrics
	ResetUsageMetrics() error
}

Provider is the interface that all LLM providers must implement

type ProviderConfig

type ProviderConfig struct {
	// Type is the type of provider
	Type ProviderType `json:"type"`
	// APIKey is the API key for the provider
	APIKey string `json:"api_key,omitempty"`
	// OrgID is the organization ID for the provider
	OrgID string `json:"org_id,omitempty"`
	// BaseURL is the base URL for the provider API
	BaseURL string `json:"base_url,omitempty"`
	// Timeout is the timeout for requests to the provider
	Timeout time.Duration `json:"timeout,omitempty"`
	// RetryConfig is the configuration for retries
	RetryConfig *RetryConfig `json:"retry_config,omitempty"`
	// RateLimitConfig is the configuration for rate limiting
	RateLimitConfig *RateLimitConfig `json:"rate_limit_config,omitempty"`
	// DefaultModel is the default model to use
	DefaultModel string `json:"default_model,omitempty"`
	// AdditionalHeaders is a map of additional headers to include in requests
	AdditionalHeaders map[string]string `json:"additional_headers,omitempty"`
	// AdditionalParams is a map of additional parameters to include in requests
	AdditionalParams map[string]interface{} `json:"additional_params,omitempty"`
}

ProviderConfig represents the configuration for a provider

type ProviderConnectionPool added in v0.2.0

type ProviderConnectionPool struct {
	// contains filtered or unexported fields
}

ProviderConnectionPool manages connections for a specific provider

func (*ProviderConnectionPool) GetClient added in v0.2.0

func (p *ProviderConnectionPool) GetClient() *http.Client

GetClient returns the HTTP client for this pool

func (*ProviderConnectionPool) GetMetrics added in v0.2.0

GetMetrics returns provider pool metrics

func (*ProviderConnectionPool) Start added in v0.2.0

func (p *ProviderConnectionPool) Start() error

Start starts the provider connection pool

func (*ProviderConnectionPool) Stop added in v0.2.0

func (p *ProviderConnectionPool) Stop()

Stop stops the provider connection pool

type ProviderCreator

type ProviderCreator func(config *ProviderConfig) (Provider, error)

ProviderCreator is a function that creates a provider with the given configuration

type ProviderError

type ProviderError struct {
	// StatusCode is the HTTP status code
	StatusCode int `json:"status_code,omitempty"`
	// Type is the type of error
	Type string `json:"type,omitempty"`
	// Message is the error message
	Message string `json:"message"`
	// Param is the parameter that caused the error
	Param string `json:"param,omitempty"`
	// Code is the error code
	Code string `json:"code,omitempty"`
	// RawResponse is the raw response from the provider
	RawResponse string `json:"-"`
}

ProviderError represents an error from a provider

func (*ProviderError) Error

func (e *ProviderError) Error() string

Error returns the error message

type ProviderFactory

type ProviderFactory interface {
	// CreateProvider creates a provider with the given configuration
	CreateProvider(config *ProviderConfig) (Provider, error)
	// GetSupportedProviderTypes returns the provider types supported by this factory
	GetSupportedProviderTypes() []ProviderType
}

ProviderFactory is the interface for creating providers

type ProviderPoolMetrics added in v0.2.0

type ProviderPoolMetrics struct {
	ProviderType       ProviderType  `json:"provider_type"`
	ActiveConnections  int64         `json:"active_connections"`
	IdleConnections    int64         `json:"idle_connections"`
	ConnectionsCreated int64         `json:"connections_created"`
	ConnectionsReused  int64         `json:"connections_reused"`
	ConnectionErrors   int64         `json:"connection_errors"`
	AverageLatency     time.Duration `json:"average_latency"`
	LastHealthCheck    time.Time     `json:"last_health_check"`
	HealthStatus       string        `json:"health_status"`
}

ProviderPoolMetrics tracks metrics for a specific provider pool

func NewProviderPoolMetrics added in v0.2.0

func NewProviderPoolMetrics(providerType ProviderType) *ProviderPoolMetrics

NewProviderPoolMetrics creates new provider pool metrics

type ProviderRegistry

type ProviderRegistry interface {
	// RegisterProvider registers a provider
	RegisterProvider(provider Provider) error

	// RegisterProviderFactory registers a provider factory
	RegisterProviderFactory(factory ProviderFactory) error

	// GetProvider returns a provider by type
	GetProvider(providerType ProviderType) (Provider, error)

	// GetProviderByModel returns a provider that supports a specific model
	GetProviderByModel(modelID string) (Provider, error)

	// GetProviderByCapability returns a provider that supports a specific capability
	GetProviderByCapability(capability ModelCapability) (Provider, error)

	// GetAllProviders returns all registered providers
	GetAllProviders() []Provider

	// GetAllProviderTypes returns all registered provider types
	GetAllProviderTypes() []ProviderType
}

ProviderRegistry is the interface for registering and retrieving providers

type ProviderType

type ProviderType string

ProviderType represents the type of LLM provider

const (
	// OpenAIProvider represents the OpenAI provider
	OpenAIProvider ProviderType = "openai"
	// AnthropicProvider represents the Anthropic provider
	AnthropicProvider ProviderType = "anthropic"
	// AzureOpenAIProvider represents the Azure OpenAI provider
	AzureOpenAIProvider ProviderType = "azure-openai"
	// HuggingFaceProvider represents the HuggingFace provider
	HuggingFaceProvider ProviderType = "huggingface"
	// LocalProvider represents a local provider (e.g., running models locally)
	LocalProvider ProviderType = "local"
	// CustomProvider represents a custom provider
	CustomProvider ProviderType = "custom"
)

type RateLimitConfig

type RateLimitConfig struct {
	// RequestsPerMinute is the maximum number of requests per minute
	RequestsPerMinute int `json:"requests_per_minute"`
	// TokensPerMinute is the maximum number of tokens per minute
	TokensPerMinute int `json:"tokens_per_minute"`
	// MaxConcurrentRequests is the maximum number of concurrent requests
	MaxConcurrentRequests int `json:"max_concurrent_requests"`
	// BurstSize is the maximum burst size
	BurstSize int `json:"burst_size"`
}

RateLimitConfig represents the configuration for rate limiting

type RetryConfig

type RetryConfig struct {
	// MaxRetries is the maximum number of retries
	MaxRetries int `json:"max_retries"`
	// InitialBackoff is the initial backoff duration
	InitialBackoff time.Duration `json:"initial_backoff"`
	// MaxBackoff is the maximum backoff duration
	MaxBackoff time.Duration `json:"max_backoff"`
	// BackoffMultiplier is the multiplier for backoff duration
	BackoffMultiplier float64 `json:"backoff_multiplier"`
	// RetryableStatusCodes is a list of HTTP status codes that should be retried
	RetryableStatusCodes []int `json:"retryable_status_codes"`
}

RetryConfig represents the configuration for retries

type TextCompletionChoice

type TextCompletionChoice struct {
	// Text is the completed text
	Text string `json:"text"`
	// Index is the index of the choice
	Index int `json:"index"`
	// LogProbs is the log probabilities
	LogProbs *LogProbs `json:"logprobs,omitempty"`
	// FinishReason is the reason the completion finished
	FinishReason string `json:"finish_reason"`
}

TextCompletionChoice represents a choice in a text completion response

type TextCompletionRequest

type TextCompletionRequest struct {
	// Prompt is the prompt for text completion
	Prompt string `json:"prompt"`
	// MaxTokens is the maximum number of tokens to generate
	MaxTokens int `json:"max_tokens,omitempty"`
	// Temperature controls randomness (0.0 to 2.0)
	Temperature float64 `json:"temperature,omitempty"`
	// TopP controls diversity via nucleus sampling (0.0 to 1.0)
	TopP float64 `json:"top_p,omitempty"`
	// N is the number of completions to generate
	N int `json:"n,omitempty"`
	// Stop is a list of tokens at which to stop generation
	Stop []string `json:"stop,omitempty"`
	// Stream indicates whether to stream the response
	Stream bool `json:"stream,omitempty"`
	// LogProbs is the number of log probabilities to return
	LogProbs int `json:"logprobs,omitempty"`
	// Echo indicates whether to echo the prompt
	Echo bool `json:"echo,omitempty"`
	// PresencePenalty penalizes new tokens based on presence in text so far (0.0 to 2.0)
	PresencePenalty float64 `json:"presence_penalty,omitempty"`
	// FrequencyPenalty penalizes new tokens based on frequency in text so far (0.0 to 2.0)
	FrequencyPenalty float64 `json:"frequency_penalty,omitempty"`
	// User is an optional user identifier
	User string `json:"user,omitempty"`
	// Model is the model to use
	Model string `json:"model,omitempty"`
}

TextCompletionRequest represents a request for text completion

type TextCompletionResponse

type TextCompletionResponse struct {
	// ID is the ID of the completion
	ID string `json:"id"`
	// Object is the object type
	Object string `json:"object"`
	// Created is the Unix timestamp of when the completion was created
	Created int64 `json:"created"`
	// Model is the model used
	Model string `json:"model"`
	// Choices is the list of completion choices
	Choices []TextCompletionChoice `json:"choices"`
	// Usage is the token usage information
	Usage *TokenUsage `json:"usage,omitempty"`
}

TextCompletionResponse represents a response from text completion

type TokenUsage

type TokenUsage struct {
	// PromptTokens is the number of tokens in the prompt
	PromptTokens int `json:"prompt_tokens"`
	// CompletionTokens is the number of tokens in the completion
	CompletionTokens int `json:"completion_tokens"`
	// TotalTokens is the total number of tokens
	TotalTokens int `json:"total_tokens"`
}

TokenUsage represents token usage information

type Tool

type Tool struct {
	// Type is the type of tool (e.g., "function")
	Type string `json:"type"`
	// Function is the function definition
	Function *Function `json:"function"`
}

Tool represents a tool definition

type ToolCall

type ToolCall struct {
	// ID is the ID of the tool call
	ID string `json:"id"`
	// Type is the type of tool call (e.g., "function")
	Type string `json:"type"`
	// Function is the function call
	Function *FunctionCall `json:"function"`
}

ToolCall represents a tool call

type UsageMetrics

type UsageMetrics struct {
	// Requests is the number of requests made
	Requests int64 `json:"requests"`
	// Tokens is the number of tokens used
	Tokens int64 `json:"tokens"`
	// Errors is the number of errors encountered
	Errors int64 `json:"errors"`
	// LastRequestTime is the time of the last request
	LastRequestTime time.Time `json:"last_request_time"`
	// TotalRequestDuration is the total duration of all requests
	TotalRequestDuration time.Duration `json:"total_request_duration"`
	// AverageResponseTime is the average response time
	AverageResponseTime time.Duration `json:"average_response_time"`
	// TokensPerMinute is the average tokens per minute
	TokensPerMinute float64 `json:"tokens_per_minute"`
	// RequestsPerMinute is the average requests per minute
	RequestsPerMinute float64 `json:"requests_per_minute"`
	// ModelID is the ID of the model
	ModelID string `json:"model_id"`
}

UsageMetrics represents the usage metrics for a provider

func NewUsageMetrics

func NewUsageMetrics(modelID string) *UsageMetrics

NewUsageMetrics creates a new usage metrics instance

func (*UsageMetrics) AddRequest

func (m *UsageMetrics) AddRequest(tokens int64, duration time.Duration, err error)

AddRequest adds a request to the usage metrics

func (*UsageMetrics) Merge

func (m *UsageMetrics) Merge(other *UsageMetrics)

Merge merges another usage metrics into this one

func (*UsageMetrics) Reset

func (m *UsageMetrics) Reset()

Reset resets the usage metrics

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL