Documentation
¶
Overview ¶
Package cache provides caching functionality for the Multi-Provider LLM Integration Framework.
Index ¶
- type Cache
- func (c *Cache) Clear()
- func (c *Cache) Delete(key CacheKey)
- func (c *Cache) Get(key CacheKey) (interface{}, bool)
- func (c *Cache) GetMetrics() *CacheMetrics
- func (c *Cache) Keys() []CacheKey
- func (c *Cache) ResetMetrics()
- func (c *Cache) Set(key CacheKey, value interface{}, ttl time.Duration)
- func (c *Cache) Size() int
- type CacheEntry
- type CacheKey
- type CacheMetrics
- type CachingProvider
- func (p *CachingProvider) ChatCompletion(ctx context.Context, request *core.ChatCompletionRequest) (*core.ChatCompletionResponse, error)
- func (p *CachingProvider) Close() error
- func (p *CachingProvider) CountTokens(ctx context.Context, text string, modelID string) (int, error)
- func (p *CachingProvider) CreateEmbedding(ctx context.Context, request *core.EmbeddingRequest) (*core.EmbeddingResponse, error)
- func (p *CachingProvider) GetConfig() *core.ProviderConfig
- func (p *CachingProvider) GetModelInfo(ctx context.Context, modelID string) (*core.ModelInfo, error)
- func (p *CachingProvider) GetModels(ctx context.Context) ([]core.ModelInfo, error)
- func (p *CachingProvider) GetType() core.ProviderType
- func (p *CachingProvider) StreamingChatCompletion(ctx context.Context, request *core.ChatCompletionRequest, ...) error
- func (p *CachingProvider) SupportsCapability(ctx context.Context, capability core.ModelCapability) bool
- func (p *CachingProvider) SupportsModel(ctx context.Context, modelID string) bool
- func (p *CachingProvider) TextCompletion(ctx context.Context, request *core.TextCompletionRequest) (*core.TextCompletionResponse, error)
- type EvictionPolicy
- type ProviderCache
- func (c *ProviderCache) Clear()
- func (c *ProviderCache) Delete(providerType core.ProviderType, operation string, request interface{}) error
- func (c *ProviderCache) Disable()
- func (c *ProviderCache) Enable()
- func (c *ProviderCache) Get(providerType core.ProviderType, operation string, request interface{}) (interface{}, bool)
- func (c *ProviderCache) GetMetrics() *CacheMetrics
- func (c *ProviderCache) IsEnabled() bool
- func (c *ProviderCache) ResetMetrics()
- func (c *ProviderCache) Set(providerType core.ProviderType, operation string, request interface{}, ...) error
- func (c *ProviderCache) Size() int
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Cache ¶
type Cache struct {
// contains filtered or unexported fields
}
Cache is a simple in-memory cache
func NewCache ¶
func NewCache(defaultTTL time.Duration, maxEntries int, evictionPolicy EvictionPolicy) *Cache
NewCache creates a new cache
func (*Cache) GetMetrics ¶
func (c *Cache) GetMetrics() *CacheMetrics
GetMetrics returns the cache metrics
type CacheEntry ¶
type CacheEntry struct {
// Value is the cached value
Value interface{}
// Expiration is the expiration time
Expiration time.Time
}
CacheEntry represents an entry in the cache
type CacheKey ¶
type CacheKey string
CacheKey represents a key for the cache
func GenerateKey ¶
func GenerateKey(providerType core.ProviderType, operation string, request interface{}) (CacheKey, error)
GenerateKey generates a cache key from a request
type CacheMetrics ¶
type CacheMetrics struct {
// Hits is the number of cache hits
Hits int
// Misses is the number of cache misses
Misses int
// Evictions is the number of cache evictions
Evictions int
// contains filtered or unexported fields
}
CacheMetrics tracks metrics for the cache
type CachingProvider ¶
type CachingProvider struct {
// contains filtered or unexported fields
}
CachingProvider wraps a provider with caching
func NewCachingProvider ¶
func NewCachingProvider(provider core.Provider, cache *ProviderCache) *CachingProvider
NewCachingProvider creates a new caching provider
func (*CachingProvider) ChatCompletion ¶
func (p *CachingProvider) ChatCompletion(ctx context.Context, request *core.ChatCompletionRequest) (*core.ChatCompletionResponse, error)
ChatCompletion generates a chat completion
func (*CachingProvider) Close ¶
func (p *CachingProvider) Close() error
Close closes the provider and releases any resources
func (*CachingProvider) CountTokens ¶
func (p *CachingProvider) CountTokens(ctx context.Context, text string, modelID string) (int, error)
CountTokens counts the number of tokens in a text
func (*CachingProvider) CreateEmbedding ¶
func (p *CachingProvider) CreateEmbedding(ctx context.Context, request *core.EmbeddingRequest) (*core.EmbeddingResponse, error)
CreateEmbedding creates an embedding
func (*CachingProvider) GetConfig ¶
func (p *CachingProvider) GetConfig() *core.ProviderConfig
GetConfig returns the configuration for the provider
func (*CachingProvider) GetModelInfo ¶
func (p *CachingProvider) GetModelInfo(ctx context.Context, modelID string) (*core.ModelInfo, error)
GetModelInfo returns information about a specific model
func (*CachingProvider) GetType ¶
func (p *CachingProvider) GetType() core.ProviderType
GetType returns the type of provider
func (*CachingProvider) StreamingChatCompletion ¶
func (p *CachingProvider) StreamingChatCompletion(ctx context.Context, request *core.ChatCompletionRequest, callback func(response *core.ChatCompletionResponse) error) error
StreamingChatCompletion generates a streaming chat completion
func (*CachingProvider) SupportsCapability ¶
func (p *CachingProvider) SupportsCapability(ctx context.Context, capability core.ModelCapability) bool
SupportsCapability returns whether the provider supports a specific capability
func (*CachingProvider) SupportsModel ¶
func (p *CachingProvider) SupportsModel(ctx context.Context, modelID string) bool
SupportsModel returns whether the provider supports a specific model
func (*CachingProvider) TextCompletion ¶
func (p *CachingProvider) TextCompletion(ctx context.Context, request *core.TextCompletionRequest) (*core.TextCompletionResponse, error)
TextCompletion generates a text completion
type EvictionPolicy ¶
type EvictionPolicy string
EvictionPolicy is the policy for evicting entries from the cache
const ( // LRU is the least recently used eviction policy LRU EvictionPolicy = "lru" // LFU is the least frequently used eviction policy LFU EvictionPolicy = "lfu" // FIFO is the first in, first out eviction policy FIFO EvictionPolicy = "fifo" )
type ProviderCache ¶
type ProviderCache struct {
// contains filtered or unexported fields
}
ProviderCache is a cache for provider responses
func NewProviderCache ¶
func NewProviderCache(defaultTTL time.Duration, maxEntries int, evictionPolicy EvictionPolicy) *ProviderCache
NewProviderCache creates a new provider cache
func (*ProviderCache) Delete ¶
func (c *ProviderCache) Delete(providerType core.ProviderType, operation string, request interface{}) error
Delete deletes a value from the cache
func (*ProviderCache) Get ¶
func (c *ProviderCache) Get(providerType core.ProviderType, operation string, request interface{}) (interface{}, bool)
Get gets a value from the cache
func (*ProviderCache) GetMetrics ¶
func (c *ProviderCache) GetMetrics() *CacheMetrics
GetMetrics returns the cache metrics
func (*ProviderCache) IsEnabled ¶
func (c *ProviderCache) IsEnabled() bool
IsEnabled returns whether caching is enabled
func (*ProviderCache) ResetMetrics ¶
func (c *ProviderCache) ResetMetrics()
ResetMetrics resets the cache metrics
func (*ProviderCache) Set ¶
func (c *ProviderCache) Set(providerType core.ProviderType, operation string, request interface{}, response interface{}, ttl time.Duration) error
Set sets a value in the cache
func (*ProviderCache) Size ¶
func (c *ProviderCache) Size() int
Size returns the number of entries in the cache
Source Files
Click to show internal directories.
Click to hide internal directories.