cache

package
v1.2.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 11, 2026 License: AGPL-3.0 Imports: 12 Imported by: 0

Documentation

Overview

Package cache provides a multi-layer caching system for LLM responses with in-memory and persistent storage backends.

Index

Constants

This section is empty.

Variables

View Source
var (
	// ErrPubSubClosed is returned when operations are attempted on a closed PubSub client
	ErrPubSubClosed = errors.New("pubsub client is closed")
	// ErrCacheClosed is returned when operations are attempted on a closed cache
	ErrCacheClosed = errors.New("cache is closed")
	// ErrInvalidKey is returned when an invalid key is provided
	ErrInvalidKey = errors.New("invalid cache key")
	// ErrInvalidTTL is returned when an invalid TTL is provided
	ErrInvalidTTL = errors.New("invalid TTL value")
)

Common cache errors

Functions

func DefaultPolicies

func DefaultPolicies() map[PolicyType]Policy

DefaultPolicies returns the default caching policies

Types

type Cache

type Cache interface {
	// Get retrieves a value from the cache.
	// Returns the value and a boolean indicating if it was found.
	Get(ctx context.Context, key string) ([]byte, bool, error)

	// Set stores a value in the cache with a TTL.
	// A zero TTL means the item never expires.
	Set(ctx context.Context, key string, value []byte, ttl time.Duration) error

	// Delete removes a value from the cache.
	Delete(ctx context.Context, key string) error

	// Exists checks if a key exists in the cache.
	Exists(ctx context.Context, key string) (bool, error)

	// GetMulti retrieves multiple values from the cache.
	// Returns a map of key to value for found items.
	GetMulti(ctx context.Context, keys []string) (map[string][]byte, error)

	// SetMulti stores multiple values in the cache with a TTL.
	SetMulti(ctx context.Context, items map[string][]byte, ttl time.Duration) error

	// Invalidate removes all items matching the pattern.
	// Pattern supports wildcards: * matches any sequence of characters.
	Invalidate(ctx context.Context, pattern string) error

	// Stats returns cache statistics.
	Stats() Stats

	// Warm pre-loads the cache with frequently accessed data.
	Warm(ctx context.Context, keys []string) error

	// Close gracefully shuts down the cache.
	Close() error
}

Cache defines the interface for the caching system. It provides a multi-layer cache with in-memory (hot) and persistent (cold) storage.

func New

func New(config Config, kv storage.KVStore) (Cache, error)

New creates a new layered cache instance

type Config

type Config struct {
	// MaxSize is the maximum number of items in the in-memory cache
	MaxSize int64 `env:"MAX_SIZE,default=10000"`
	// MaxSizeInMB is the maximum memory usage in MB for the in-memory cache
	MaxSizeInMB int64 `env:"MAX_SIZE_MB,default=256"`
	// DefaultTTL is the default TTL for cached items
	DefaultTTL time.Duration `env:"DEFAULT_TTL,default=1h"`
	// EnableMetrics enables detailed metrics collection
	EnableMetrics bool `env:"ENABLE_METRICS,default=true"`
	// WarmupKeys is a list of keys to pre-load on startup
	WarmupKeys []string `env:"WARMUP_KEYS"`
}

Config represents cache configuration

type DistributedCache

type DistributedCache struct {
	// contains filtered or unexported fields
}

DistributedCache implements a cache using only the distributed KV store

func NewDistributedCache

func NewDistributedCache(store storage.KVStore, prefix string) *DistributedCache

NewDistributedCache creates a new distributed-only cache

func (*DistributedCache) Close

func (d *DistributedCache) Close() error

Close is a no-op for distributed cache

func (*DistributedCache) Delete

func (d *DistributedCache) Delete(ctx context.Context, key string) error

Delete removes a value from the distributed cache

func (*DistributedCache) Exists

func (d *DistributedCache) Exists(ctx context.Context, key string) (bool, error)

Exists checks if a key exists in the distributed cache

func (*DistributedCache) Get

func (d *DistributedCache) Get(ctx context.Context, key string) ([]byte, bool, error)

Get retrieves a value from the distributed cache

func (*DistributedCache) GetMulti

func (d *DistributedCache) GetMulti(ctx context.Context, keys []string) (map[string][]byte, error)

GetMulti retrieves multiple values from the distributed cache

func (*DistributedCache) Invalidate

func (d *DistributedCache) Invalidate(ctx context.Context, pattern string) error

Invalidate removes all items matching the pattern

func (*DistributedCache) Set

func (d *DistributedCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error

Set stores a value in the distributed cache

func (*DistributedCache) SetMulti

func (d *DistributedCache) SetMulti(ctx context.Context, items map[string][]byte, ttl time.Duration) error

SetMulti stores multiple values in the distributed cache

func (*DistributedCache) Stats

func (d *DistributedCache) Stats() Stats

Stats returns empty stats for distributed cache

func (*DistributedCache) Warm

func (d *DistributedCache) Warm(_ context.Context, _ []string) error

Warm pre-loads data (no-op for distributed cache)

type HybridCache

type HybridCache struct {
	// contains filtered or unexported fields
}

HybridCache implements a two-layer cache with local (Ristretto) and distributed (KV store) layers It supports pub/sub based invalidation for multi-node consistency

func NewHybridCache

func NewHybridCache(config HybridCacheConfig, store storage.KVStore, pubsub PubSubClient) (*HybridCache, error)

NewHybridCache creates a new hybrid cache

func (*HybridCache) Clear

func (h *HybridCache) Clear()

Clear removes all items from local cache

func (*HybridCache) Close

func (h *HybridCache) Close() error

Close closes the local cache

func (*HybridCache) Delete

func (h *HybridCache) Delete(ctx context.Context, key string) error

Delete removes a value from both cache layers and publishes invalidation

func (*HybridCache) Exists

func (h *HybridCache) Exists(ctx context.Context, key string) (bool, error)

Exists checks if a key exists in the cache

func (*HybridCache) Get

func (h *HybridCache) Get(ctx context.Context, key string) ([]byte, bool, error)

Get retrieves a value from the cache

func (*HybridCache) GetMulti

func (h *HybridCache) GetMulti(ctx context.Context, keys []string) (map[string][]byte, error)

GetMulti retrieves multiple values from the cache

func (*HybridCache) Invalidate

func (h *HybridCache) Invalidate(_ context.Context, _ string) error

Invalidate removes all items matching the pattern from both caches

func (*HybridCache) InvalidateLocal

func (h *HybridCache) InvalidateLocal(key string)

InvalidateLocal removes a key from local cache only (used by invalidation handler)

func (*HybridCache) Set

func (h *HybridCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error

Set stores a value in both cache layers

func (*HybridCache) SetMulti

func (h *HybridCache) SetMulti(ctx context.Context, items map[string][]byte, ttl time.Duration) error

SetMulti stores multiple values in the cache

func (*HybridCache) Stats

func (h *HybridCache) Stats() Stats

Stats returns local cache statistics

func (*HybridCache) Warm

func (h *HybridCache) Warm(ctx context.Context, keys []string) error

Warm pre-loads the cache with frequently accessed data

type HybridCacheConfig

type HybridCacheConfig struct {
	LocalSizeMB      int64
	LocalTTL         time.Duration
	Prefix           string
	InvalidatePrefix string
}

HybridCacheConfig configures a hybrid cache

type LocalCache

type LocalCache struct {
	// contains filtered or unexported fields
}

LocalCache implements a local-only cache using Ristretto

func NewLocalCache

func NewLocalCache(sizeMB int64, ttl time.Duration) (*LocalCache, error)

NewLocalCache creates a new local-only cache

func (*LocalCache) Clear

func (l *LocalCache) Clear()

Clear removes all items from the cache

func (*LocalCache) Close

func (l *LocalCache) Close() error

Close closes the cache

func (*LocalCache) Delete

func (l *LocalCache) Delete(_ context.Context, key string) error

Delete removes a value from the local cache

func (*LocalCache) Exists

func (l *LocalCache) Exists(_ context.Context, key string) (bool, error)

Exists checks if a key exists in the local cache

func (*LocalCache) Get

func (l *LocalCache) Get(_ context.Context, key string) ([]byte, bool, error)

Get retrieves a value from the local cache

func (*LocalCache) GetMulti

func (l *LocalCache) GetMulti(_ context.Context, keys []string) (map[string][]byte, error)

GetMulti retrieves multiple values from the local cache

func (*LocalCache) Invalidate

func (l *LocalCache) Invalidate(_ context.Context, _ string) error

Invalidate removes all items matching the pattern

func (*LocalCache) Set

func (l *LocalCache) Set(_ context.Context, key string, value []byte, ttl time.Duration) error

Set stores a value in the local cache

func (*LocalCache) SetMulti

func (l *LocalCache) SetMulti(_ context.Context, items map[string][]byte, ttl time.Duration) error

SetMulti stores multiple values in the local cache

func (*LocalCache) Stats

func (l *LocalCache) Stats() Stats

Stats returns cache statistics

func (*LocalCache) Warm

func (l *LocalCache) Warm(_ context.Context, _ []string) error

Warm pre-loads the cache (no-op for local cache)

type Manager

type Manager struct {
	// contains filtered or unexported fields
}

Manager manages different cache strategies for different data types

func NewCacheManager

func NewCacheManager(config ManagerConfig, store storage.KVStore) (*Manager, error)

NewCacheManager creates a new cache manager with appropriate strategies for each data type

func (*Manager) Close

func (cm *Manager) Close() error

Close gracefully shuts down the cache manager

func (*Manager) GetModel

func (cm *Manager) GetModel(ctx context.Context, modelID string) (any, bool, error)

GetModel retrieves model metadata (local cache only)

func (*Manager) GetResponse

func (cm *Manager) GetResponse(ctx context.Context, key string) ([]byte, bool, error)

GetResponse retrieves a cached LLM response

func (*Manager) InvalidateModels

func (cm *Manager) InvalidateModels()

InvalidateModels clears all model metadata from local cache

func (*Manager) SetModel

func (cm *Manager) SetModel(ctx context.Context, modelID string, model any) error

SetModel caches model metadata (local cache only)

func (*Manager) SetResponse

func (cm *Manager) SetResponse(ctx context.Context, key string, response []byte) error

SetResponse caches an LLM response

func (*Manager) Stats

func (cm *Manager) Stats() map[string]Stats

Stats returns aggregated cache statistics.

type ManagerConfig

type ManagerConfig struct {
	// LLM Responses configuration
	Responses struct {
		Strategy      string        `env:"STRATEGY,default=auto"`
		TTL           time.Duration `env:"TTL,default=1h"`
		MaxItemSizeKB int           `env:"MAX_ITEM_SIZE_KB,default=1024"`
		LocalSizeMB   int64         `env:"LOCAL_SIZE_MB,default=256"`
	} `env:",prefix=RESPONSES_"`

	// Model Metadata configuration
	Models struct {
		Strategy string        `env:"STRATEGY,default=local"`
		TTL      time.Duration `env:"TTL,default=6h"`
		SizeMB   int64         `env:"SIZE_MB,default=16"`
	} `env:",prefix=MODELS_"`
}

ManagerConfig configures the cache manager

type MemoryPubSub

type MemoryPubSub struct {
	// contains filtered or unexported fields
}

MemoryPubSub is an in-memory pub/sub implementation for testing

func NewMemoryPubSub

func NewMemoryPubSub() *MemoryPubSub

NewMemoryPubSub creates a new in-memory pub/sub client

func (*MemoryPubSub) Close

func (m *MemoryPubSub) Close() error

Close closes the pub/sub client

func (*MemoryPubSub) Publish

func (m *MemoryPubSub) Publish(_ context.Context, channel string, message string) error

Publish sends a message to all matching subscribers

func (*MemoryPubSub) Subscribe

func (m *MemoryPubSub) Subscribe(pattern string, handler func(channel, message string)) error

Subscribe adds a handler for a pattern

type NoopPubSub

type NoopPubSub struct{}

NoopPubSub is a no-op implementation for single-node deployments

func (*NoopPubSub) Close

func (n *NoopPubSub) Close() error

Close does nothing in noop implementation

func (*NoopPubSub) Publish

func (n *NoopPubSub) Publish(_ context.Context, channel string, _ string) error

Publish does nothing in noop implementation

func (*NoopPubSub) Subscribe

func (n *NoopPubSub) Subscribe(pattern string, _ func(channel, message string)) error

Subscribe does nothing in noop implementation

type Policy

type Policy struct {
	// TTL is the time-to-live for this type of data
	TTL time.Duration
	// MaxSize is the maximum size in bytes for cacheable items
	MaxSize int64
	// Compress indicates whether to compress the data
	Compress bool
	// SkipCache indicates whether to skip caching entirely
	SkipCache bool
}

Policy defines caching policies for different types of data

type PolicyType

type PolicyType string

PolicyType represents different types of cacheable data

const (
	// PolicyTypeChatCompletion is for chat completion responses
	PolicyTypeChatCompletion PolicyType = "chat_completion"
	// PolicyTypeEmbedding is for embedding responses
	PolicyTypeEmbedding PolicyType = "embedding"
	// PolicyTypeModel is for model list responses
	PolicyTypeModel PolicyType = "model"
	// PolicyTypeProvider is for provider metadata
	PolicyTypeProvider PolicyType = "provider"
)

type PubSubClient

type PubSubClient = storage.PubSubClient

PubSubClient re-exports storage.PubSubClient for convenience

type PubSubProvider

type PubSubProvider = storage.PubSubProvider

PubSubProvider re-exports storage.PubSubProvider for convenience

type Stats

type Stats struct {
	// Hits is the number of cache hits
	Hits uint64 `json:"hits"`
	// Misses is the number of cache misses
	Misses uint64 `json:"misses"`
	// Sets is the number of items added to cache
	Sets uint64 `json:"sets"`
	// Deletes is the number of items deleted from cache
	Deletes uint64 `json:"deletes"`
	// Evictions is the number of items evicted due to size/TTL
	Evictions uint64 `json:"evictions"`
	// HitRate is the cache hit rate (hits / (hits + misses))
	HitRate float64 `json:"hit_rate"`
	// Size is the current number of items in cache
	Size int64 `json:"size"`
	// SizeInBytes is the approximate memory usage
	SizeInBytes int64 `json:"size_in_bytes"`
}

Stats contains cache performance metrics.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL