Documentation
¶
Overview ¶
Package cache provides a multi-layer caching system for LLM responses with in-memory and persistent storage backends.
Index ¶
- Variables
- func DefaultPolicies() map[PolicyType]Policy
- type Cache
- type Config
- type DistributedCache
- func (d *DistributedCache) Close() error
- func (d *DistributedCache) Delete(ctx context.Context, key string) error
- func (d *DistributedCache) Exists(ctx context.Context, key string) (bool, error)
- func (d *DistributedCache) Get(ctx context.Context, key string) ([]byte, bool, error)
- func (d *DistributedCache) GetMulti(ctx context.Context, keys []string) (map[string][]byte, error)
- func (d *DistributedCache) Invalidate(ctx context.Context, pattern string) error
- func (d *DistributedCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error
- func (d *DistributedCache) SetMulti(ctx context.Context, items map[string][]byte, ttl time.Duration) error
- func (d *DistributedCache) Stats() Stats
- func (d *DistributedCache) Warm(_ context.Context, _ []string) error
- type HybridCache
- func (h *HybridCache) Clear()
- func (h *HybridCache) Close() error
- func (h *HybridCache) Delete(ctx context.Context, key string) error
- func (h *HybridCache) Exists(ctx context.Context, key string) (bool, error)
- func (h *HybridCache) Get(ctx context.Context, key string) ([]byte, bool, error)
- func (h *HybridCache) GetMulti(ctx context.Context, keys []string) (map[string][]byte, error)
- func (h *HybridCache) Invalidate(_ context.Context, _ string) error
- func (h *HybridCache) InvalidateLocal(key string)
- func (h *HybridCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error
- func (h *HybridCache) SetMulti(ctx context.Context, items map[string][]byte, ttl time.Duration) error
- func (h *HybridCache) Stats() Stats
- func (h *HybridCache) Warm(ctx context.Context, keys []string) error
- type HybridCacheConfig
- type LocalCache
- func (l *LocalCache) Clear()
- func (l *LocalCache) Close() error
- func (l *LocalCache) Delete(_ context.Context, key string) error
- func (l *LocalCache) Exists(_ context.Context, key string) (bool, error)
- func (l *LocalCache) Get(_ context.Context, key string) ([]byte, bool, error)
- func (l *LocalCache) GetMulti(_ context.Context, keys []string) (map[string][]byte, error)
- func (l *LocalCache) Invalidate(_ context.Context, _ string) error
- func (l *LocalCache) Set(_ context.Context, key string, value []byte, ttl time.Duration) error
- func (l *LocalCache) SetMulti(_ context.Context, items map[string][]byte, ttl time.Duration) error
- func (l *LocalCache) Stats() Stats
- func (l *LocalCache) Warm(_ context.Context, _ []string) error
- type Manager
- func (cm *Manager) Close() error
- func (cm *Manager) GetModel(ctx context.Context, modelID string) (any, bool, error)
- func (cm *Manager) GetResponse(ctx context.Context, key string) ([]byte, bool, error)
- func (cm *Manager) InvalidateModels()
- func (cm *Manager) SetModel(ctx context.Context, modelID string, model any) error
- func (cm *Manager) SetResponse(ctx context.Context, key string, response []byte) error
- func (cm *Manager) Stats() map[string]Stats
- type ManagerConfig
- type MemoryPubSub
- type NoopPubSub
- type Policy
- type PolicyType
- type PubSubClient
- type PubSubProvider
- type Stats
Constants ¶
This section is empty.
Variables ¶
var ( // ErrPubSubClosed is returned when operations are attempted on a closed PubSub client ErrPubSubClosed = errors.New("pubsub client is closed") // ErrCacheClosed is returned when operations are attempted on a closed cache ErrCacheClosed = errors.New("cache is closed") // ErrInvalidKey is returned when an invalid key is provided ErrInvalidKey = errors.New("invalid cache key") // ErrInvalidTTL is returned when an invalid TTL is provided ErrInvalidTTL = errors.New("invalid TTL value") )
Common cache errors
Functions ¶
func DefaultPolicies ¶
func DefaultPolicies() map[PolicyType]Policy
DefaultPolicies returns the default caching policies
Types ¶
type Cache ¶
type Cache interface {
// Get retrieves a value from the cache.
// Returns the value and a boolean indicating if it was found.
Get(ctx context.Context, key string) ([]byte, bool, error)
// Set stores a value in the cache with a TTL.
// A zero TTL means the item never expires.
Set(ctx context.Context, key string, value []byte, ttl time.Duration) error
// Delete removes a value from the cache.
Delete(ctx context.Context, key string) error
// Exists checks if a key exists in the cache.
Exists(ctx context.Context, key string) (bool, error)
// GetMulti retrieves multiple values from the cache.
// Returns a map of key to value for found items.
GetMulti(ctx context.Context, keys []string) (map[string][]byte, error)
// SetMulti stores multiple values in the cache with a TTL.
SetMulti(ctx context.Context, items map[string][]byte, ttl time.Duration) error
// Invalidate removes all items matching the pattern.
// Pattern supports wildcards: * matches any sequence of characters.
Invalidate(ctx context.Context, pattern string) error
// Stats returns cache statistics.
Stats() Stats
// Warm pre-loads the cache with frequently accessed data.
Warm(ctx context.Context, keys []string) error
// Close gracefully shuts down the cache.
Close() error
}
Cache defines the interface for the caching system. It provides a multi-layer cache with in-memory (hot) and persistent (cold) storage.
type Config ¶
type Config struct {
// MaxSize is the maximum number of items in the in-memory cache
MaxSize int64 `env:"MAX_SIZE,default=10000"`
// MaxSizeInMB is the maximum memory usage in MB for the in-memory cache
MaxSizeInMB int64 `env:"MAX_SIZE_MB,default=256"`
// DefaultTTL is the default TTL for cached items
DefaultTTL time.Duration `env:"DEFAULT_TTL,default=1h"`
// EnableMetrics enables detailed metrics collection
EnableMetrics bool `env:"ENABLE_METRICS,default=true"`
// WarmupKeys is a list of keys to pre-load on startup
WarmupKeys []string `env:"WARMUP_KEYS"`
}
Config represents cache configuration
type DistributedCache ¶
type DistributedCache struct {
// contains filtered or unexported fields
}
DistributedCache implements a cache using only the distributed KV store
func NewDistributedCache ¶
func NewDistributedCache(store storage.KVStore, prefix string) *DistributedCache
NewDistributedCache creates a new distributed-only cache
func (*DistributedCache) Close ¶
func (d *DistributedCache) Close() error
Close is a no-op for distributed cache
func (*DistributedCache) Delete ¶
func (d *DistributedCache) Delete(ctx context.Context, key string) error
Delete removes a value from the distributed cache
func (*DistributedCache) Invalidate ¶
func (d *DistributedCache) Invalidate(ctx context.Context, pattern string) error
Invalidate removes all items matching the pattern
func (*DistributedCache) Set ¶
func (d *DistributedCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error
Set stores a value in the distributed cache
func (*DistributedCache) SetMulti ¶
func (d *DistributedCache) SetMulti(ctx context.Context, items map[string][]byte, ttl time.Duration) error
SetMulti stores multiple values in the distributed cache
func (*DistributedCache) Stats ¶
func (d *DistributedCache) Stats() Stats
Stats returns empty stats for distributed cache
type HybridCache ¶
type HybridCache struct {
// contains filtered or unexported fields
}
HybridCache implements a two-layer cache with local (Ristretto) and distributed (KV store) layers It supports pub/sub based invalidation for multi-node consistency
func NewHybridCache ¶
func NewHybridCache(config HybridCacheConfig, store storage.KVStore, pubsub PubSubClient) (*HybridCache, error)
NewHybridCache creates a new hybrid cache
func (*HybridCache) Delete ¶
func (h *HybridCache) Delete(ctx context.Context, key string) error
Delete removes a value from both cache layers and publishes invalidation
func (*HybridCache) Invalidate ¶
func (h *HybridCache) Invalidate(_ context.Context, _ string) error
Invalidate removes all items matching the pattern from both caches
func (*HybridCache) InvalidateLocal ¶
func (h *HybridCache) InvalidateLocal(key string)
InvalidateLocal removes a key from local cache only (used by invalidation handler)
func (*HybridCache) SetMulti ¶
func (h *HybridCache) SetMulti(ctx context.Context, items map[string][]byte, ttl time.Duration) error
SetMulti stores multiple values in the cache
func (*HybridCache) Stats ¶
func (h *HybridCache) Stats() Stats
Stats returns local cache statistics
type HybridCacheConfig ¶
type HybridCacheConfig struct {
LocalSizeMB int64
LocalTTL time.Duration
Prefix string
InvalidatePrefix string
}
HybridCacheConfig configures a hybrid cache
type LocalCache ¶
type LocalCache struct {
// contains filtered or unexported fields
}
LocalCache implements a local-only cache using Ristretto
func NewLocalCache ¶
func NewLocalCache(sizeMB int64, ttl time.Duration) (*LocalCache, error)
NewLocalCache creates a new local-only cache
func (*LocalCache) Delete ¶
func (l *LocalCache) Delete(_ context.Context, key string) error
Delete removes a value from the local cache
func (*LocalCache) Invalidate ¶
func (l *LocalCache) Invalidate(_ context.Context, _ string) error
Invalidate removes all items matching the pattern
type Manager ¶
type Manager struct {
// contains filtered or unexported fields
}
Manager manages different cache strategies for different data types
func NewCacheManager ¶
func NewCacheManager(config ManagerConfig, store storage.KVStore) (*Manager, error)
NewCacheManager creates a new cache manager with appropriate strategies for each data type
func (*Manager) GetResponse ¶
GetResponse retrieves a cached LLM response
func (*Manager) InvalidateModels ¶
func (cm *Manager) InvalidateModels()
InvalidateModels clears all model metadata from local cache
func (*Manager) SetResponse ¶
SetResponse caches an LLM response
type ManagerConfig ¶
type ManagerConfig struct {
// LLM Responses configuration
Responses struct {
Strategy string `env:"STRATEGY,default=auto"`
TTL time.Duration `env:"TTL,default=1h"`
MaxItemSizeKB int `env:"MAX_ITEM_SIZE_KB,default=1024"`
LocalSizeMB int64 `env:"LOCAL_SIZE_MB,default=256"`
} `env:",prefix=RESPONSES_"`
// Model Metadata configuration
Models struct {
Strategy string `env:"STRATEGY,default=local"`
TTL time.Duration `env:"TTL,default=6h"`
SizeMB int64 `env:"SIZE_MB,default=16"`
} `env:",prefix=MODELS_"`
}
ManagerConfig configures the cache manager
type MemoryPubSub ¶
type MemoryPubSub struct {
// contains filtered or unexported fields
}
MemoryPubSub is an in-memory pub/sub implementation for testing
func NewMemoryPubSub ¶
func NewMemoryPubSub() *MemoryPubSub
NewMemoryPubSub creates a new in-memory pub/sub client
type NoopPubSub ¶
type NoopPubSub struct{}
NoopPubSub is a no-op implementation for single-node deployments
func (*NoopPubSub) Close ¶
func (n *NoopPubSub) Close() error
Close does nothing in noop implementation
type Policy ¶
type Policy struct {
// TTL is the time-to-live for this type of data
TTL time.Duration
// MaxSize is the maximum size in bytes for cacheable items
MaxSize int64
// Compress indicates whether to compress the data
Compress bool
// SkipCache indicates whether to skip caching entirely
SkipCache bool
}
Policy defines caching policies for different types of data
type PolicyType ¶
type PolicyType string
PolicyType represents different types of cacheable data
const ( // PolicyTypeChatCompletion is for chat completion responses PolicyTypeChatCompletion PolicyType = "chat_completion" // PolicyTypeEmbedding is for embedding responses PolicyTypeEmbedding PolicyType = "embedding" // PolicyTypeModel is for model list responses PolicyTypeModel PolicyType = "model" // PolicyTypeProvider is for provider metadata PolicyTypeProvider PolicyType = "provider" )
type PubSubClient ¶
type PubSubClient = storage.PubSubClient
PubSubClient re-exports storage.PubSubClient for convenience
type PubSubProvider ¶
type PubSubProvider = storage.PubSubProvider
PubSubProvider re-exports storage.PubSubProvider for convenience
type Stats ¶
type Stats struct {
// Hits is the number of cache hits
Hits uint64 `json:"hits"`
// Misses is the number of cache misses
Misses uint64 `json:"misses"`
// Sets is the number of items added to cache
Sets uint64 `json:"sets"`
// Deletes is the number of items deleted from cache
Deletes uint64 `json:"deletes"`
// Evictions is the number of items evicted due to size/TTL
Evictions uint64 `json:"evictions"`
// HitRate is the cache hit rate (hits / (hits + misses))
HitRate float64 `json:"hit_rate"`
// Size is the current number of items in cache
Size int64 `json:"size"`
// SizeInBytes is the approximate memory usage
SizeInBytes int64 `json:"size_in_bytes"`
}
Stats contains cache performance metrics.