cache

package
v0.0.13 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 7, 2025 License: MIT Imports: 18 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func ClearGlobalCache 

func ClearGlobalCache(ctx context.Context) error

ClearGlobalCache clears the global cache used by all LLMs.

func IsEnabled 

func IsEnabled(config CacheConfig) bool

IsEnabled checks if caching is enabled based on configuration.

func SetGlobalCacheEnabled 

func SetGlobalCacheEnabled(enabled bool)

This affects all new LLM instances created after this call.

func WithCacheContext 

func WithCacheContext(ctx context.Context, cache Cache, ttl time.Duration) context.Context

WithCacheContext adds cache configuration to context.

func WrapWithCache 

func WrapWithCache(llm core.LLM, fileConfig *config.CachingConfig) core.LLM

WrapWithCache wraps an LLM with caching using the global cache configuration. This is called automatically by the LLM factory if caching is enabled.

Types

type Cache 

type Cache interface {
	// Get retrieves a cached value by key.
	Get(ctx context.Context, key string) ([]byte, bool, error)

	// Set stores a value with the given key and TTL.
	Set(ctx context.Context, key string, value []byte, ttl time.Duration) error

	// Delete removes a cached value by key.
	Delete(ctx context.Context, key string) error

	// Clear removes all cached values.
	Clear(ctx context.Context) error

	// Stats returns cache statistics.
	Stats() CacheStats

	// Close releases any resources held by the cache.
	Close() error
}

Cache defines the interface for caching LLM responses.

func NewCache 

func NewCache(config CacheConfig) (Cache, error)

NewCache creates a new cache instance based on the configuration.

type CacheConfig 

type CacheConfig struct {
	// Type of cache: "memory" or "sqlite"
	Type string `json:"type" yaml:"type"`

	// Maximum cache size in bytes (0 = unlimited)
	MaxSize int64 `json:"max_size" yaml:"max_size"`

	// Default TTL for cache entries (0 = no expiration)
	DefaultTTL time.Duration `json:"default_ttl" yaml:"default_ttl"`

	// SQLite specific configuration
	SQLiteConfig SQLiteConfig `json:"sqlite_config,omitempty" yaml:"sqlite_config,omitempty"`

	// Memory cache specific configuration
	MemoryConfig MemoryConfig `json:"memory_config,omitempty" yaml:"memory_config,omitempty"`
}

CacheConfig holds cache configuration.

func GetDefaultCacheConfig 

func GetDefaultCacheConfig() CacheConfig

GetDefaultCacheConfig returns the default cache configuration with environment overrides.

func LoadCacheConfig 

func LoadCacheConfig(fileConfig *config.CachingConfig) CacheConfig

3. Smart defaults (lowest priority).

type CacheEntry 

type CacheEntry struct {
	Key        string    `json:"key"`
	Value      []byte    `json:"value"`
	ExpiresAt  time.Time `json:"expires_at"`
	CreatedAt  time.Time `json:"created_at"`
	AccessedAt time.Time `json:"accessed_at"`
	Size       int64     `json:"size"`
}

CacheEntry represents a cached item.

func (*CacheEntry) IsExpired 

func (e *CacheEntry) IsExpired() bool

IsExpired checks if the cache entry has expired.

type CacheStats 

type CacheStats struct {
	Hits       int64     `json:"hits"`
	Misses     int64     `json:"misses"`
	Sets       int64     `json:"sets"`
	Deletes    int64     `json:"deletes"`
	Size       int64     `json:"size"`
	MaxSize    int64     `json:"max_size"`
	LastAccess time.Time `json:"last_access"`
}

CacheStats contains cache performance statistics.

func GetGlobalCacheStats 

func GetGlobalCacheStats() CacheStats

GetGlobalCacheStats returns statistics for the global cache.

func (*CacheStats) HitRate 

func (s *CacheStats) HitRate() float64

HitRate returns the cache hit rate (0.0 to 1.0). Returns 0.0 if no hits or misses have occurred.

func (*CacheStats) MissRate 

func (s *CacheStats) MissRate() float64

MissRate returns the cache miss rate (0.0 to 1.0). Returns 0.0 if no hits or misses have occurred.

type CachedLLM 

type CachedLLM struct {
	core.LLM
	// contains filtered or unexported fields
}

CachedLLM wraps an LLM with transparent caching functionality. It implements core.LLM interface and can be used as a drop-in replacement.

func (*CachedLLM) CacheStats 

func (c *CachedLLM) CacheStats() CacheStats

CacheStats returns cache statistics.

func (*CachedLLM) ClearCache 

func (c *CachedLLM) ClearCache(ctx context.Context) error

ClearCache clears the cache.

func (*CachedLLM) CreateEmbedding 

func (c *CachedLLM) CreateEmbedding(ctx context.Context, input string, options ...core.EmbeddingOption) (*core.EmbeddingResult, error)

Embedding methods are passed through (could be cached in the future).

func (*CachedLLM) CreateEmbeddings 

func (c *CachedLLM) CreateEmbeddings(ctx context.Context, inputs []string, options ...core.EmbeddingOption) (*core.BatchEmbeddingResult, error)

func (*CachedLLM) Generate 

func (c *CachedLLM) Generate(ctx context.Context, prompt string, options ...core.GenerateOption) (*core.LLMResponse, error)

Generate implements core.LLM with caching.

func (*CachedLLM) GenerateWithContent 

func (c *CachedLLM) GenerateWithContent(ctx context.Context, content []core.ContentBlock, options ...core.GenerateOption) (*core.LLMResponse, error)

GenerateWithContent implements core.LLM with caching for multimodal content.

func (*CachedLLM) GenerateWithFunctions 

func (c *CachedLLM) GenerateWithFunctions(ctx context.Context, prompt string, functions []map[string]interface{}, options ...core.GenerateOption) (map[string]interface{}, error)

GenerateWithFunctions implements core.LLM with caching for function calls.

func (*CachedLLM) GenerateWithJSON 

func (c *CachedLLM) GenerateWithJSON(ctx context.Context, prompt string, options ...core.GenerateOption) (map[string]interface{}, error)

GenerateWithJSON implements core.LLM with caching for JSON responses.

func (*CachedLLM) IsCacheEnabled 

func (c *CachedLLM) IsCacheEnabled() bool

IsCacheEnabled returns whether caching is enabled for this LLM instance.

func (*CachedLLM) SetCacheEnabled 

func (c *CachedLLM) SetCacheEnabled(enabled bool)

SetCacheEnabled enables/disables caching for this LLM instance.

func (*CachedLLM) StreamGenerate 

func (c *CachedLLM) StreamGenerate(ctx context.Context, prompt string, options ...core.GenerateOption) (*core.StreamResponse, error)

Streaming methods are not cached - pass through directly.

func (*CachedLLM) StreamGenerateWithContent 

func (c *CachedLLM) StreamGenerateWithContent(ctx context.Context, content []core.ContentBlock, options ...core.GenerateOption) (*core.StreamResponse, error)

func (*CachedLLM) Unwrap 

func (c *CachedLLM) Unwrap() core.LLM

Unwrap returns the underlying LLM instance.

type Content 

type Content struct {
	Type string
	Data string
}

Content represents content for cache key generation.

type KeyGenerator 

type KeyGenerator struct {
	// contains filtered or unexported fields
}

KeyGenerator generates cache keys for LLM requests.

func NewKeyGenerator 

func NewKeyGenerator(prefix string) *KeyGenerator

NewKeyGenerator creates a new cache key generator.

func (*KeyGenerator) GenerateContentKey 

func (g *KeyGenerator) GenerateContentKey(modelID string, contents []Content, options []core.GenerateOption) string

GenerateContentKey creates a cache key for multimodal content requests.

func (*KeyGenerator) GenerateJSONKey 

func (g *KeyGenerator) GenerateJSONKey(modelID string, prompt string, schema interface{}, options []core.GenerateOption) string

GenerateJSONKey creates a cache key for JSON-structured requests.

func (*KeyGenerator) GenerateKey 

func (g *KeyGenerator) GenerateKey(modelID string, prompt string, options []core.GenerateOption) string

GenerateKey creates a deterministic cache key from LLM request parameters.

func (*KeyGenerator) InvalidatePattern 

func (g *KeyGenerator) InvalidatePattern(modelID string) string

InvalidatePattern generates a pattern for invalidating cache entries. This can be used to clear cache entries matching certain criteria.

type MemoryCache 

type MemoryCache struct {
	// contains filtered or unexported fields
}

MemoryCache implements an in-memory cache with LRU eviction.

func NewMemoryCache 

func NewMemoryCache(config CacheConfig) (*MemoryCache, error)

NewMemoryCache creates a new in-memory cache.

func (*MemoryCache) Clear 

func (c *MemoryCache) Clear(ctx context.Context) error

func (*MemoryCache) Close 

func (c *MemoryCache) Close() error

func (*MemoryCache) Delete 

func (c *MemoryCache) Delete(ctx context.Context, key string) error

func (*MemoryCache) Export 

func (c *MemoryCache) Export(ctx context.Context, writer func(entry CacheEntry) error) error

Export exports cache entries for backup/migration.

func (*MemoryCache) Get 

func (c *MemoryCache) Get(ctx context.Context, key string) ([]byte, bool, error)

func (*MemoryCache) Import 

func (c *MemoryCache) Import(ctx context.Context, entries []CacheEntry) error

Import imports cache entries from a source.

func (*MemoryCache) Set 

func (c *MemoryCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error

func (*MemoryCache) Stats 

func (c *MemoryCache) Stats() CacheStats

type MemoryConfig 

type MemoryConfig struct {
	// Eviction policy: "lru", "lfu", "fifo"
	EvictionPolicy string `json:"eviction_policy" yaml:"eviction_policy"`

	// Cleanup interval for expired entries
	CleanupInterval time.Duration `json:"cleanup_interval" yaml:"cleanup_interval"`

	// Number of shards for concurrent access
	ShardCount int `json:"shard_count" yaml:"shard_count"`
}

MemoryConfig holds memory cache specific configuration.

type Middleware 

type Middleware struct {
	// contains filtered or unexported fields
}

Middleware provides caching functionality for LLM requests. It can be embedded into LLM providers or used as a standalone component.

func CacheFromContext 

func CacheFromContext(ctx context.Context) *Middleware

CacheFromContext retrieves cache middleware from context.

func NewMiddleware 

func NewMiddleware(cache Cache, ttl time.Duration) *Middleware

NewMiddleware creates a new cache middleware.

func (*Middleware) Clear 

func (m *Middleware) Clear(ctx context.Context) error

Clear clears all cached entries.

func (*Middleware) Close 

func (m *Middleware) Close() error

Close closes the cache.

func (*Middleware) GenerateCacheKey 

func (m *Middleware) GenerateCacheKey(modelID string, prompt string, options []core.GenerateOption) string

GenerateCacheKey creates a cache key for a standard generation request.

func (*Middleware) GenerateContentCacheKey 

func (m *Middleware) GenerateContentCacheKey(modelID string, contents []Content, options []core.GenerateOption) string

GenerateContentCacheKey creates a cache key for multimodal content.

func (*Middleware) GenerateJSONCacheKey 

func (m *Middleware) GenerateJSONCacheKey(modelID string, prompt string, schema interface{}, options []core.GenerateOption) string

GenerateJSONCacheKey creates a cache key for JSON-structured generation.

func (*Middleware) IsEnabled 

func (m *Middleware) IsEnabled() bool

IsEnabled returns whether caching is enabled.

func (*Middleware) SetEnabled 

func (m *Middleware) SetEnabled(enabled bool)

SetEnabled enables or disables caching.

func (*Middleware) Stats 

func (m *Middleware) Stats() CacheStats

Stats returns cache statistics.

func (*Middleware) WithCache 

func (m *Middleware) WithCache(
	ctx context.Context,
	cacheKey string,
	ttl time.Duration,
	fn func() (*core.LLMResponse, error),
) (*core.LLMResponse, error)

WithCache wraps an LLM request with caching logic. This is meant to be called by LLM providers internally.

type Option 

type Option func(*Middleware)

Option is a functional option for configuring cache middleware.

func WithEnabled 

func WithEnabled(enabled bool) Option

WithEnabled sets the initial enabled state.

func WithKeyPrefix 

func WithKeyPrefix(prefix string) Option

WithKeyPrefix sets a custom key prefix.

func WithTTL 

func WithTTL(ttl time.Duration) Option

WithTTL sets the default TTL for cache entries.

type ProviderCache 

type ProviderCache struct {
	// contains filtered or unexported fields
}

ProviderCache is a helper struct that can be embedded in LLM providers to add caching functionality in a Go-idiomatic way.

func NewProviderCache 

func NewProviderCache(cacheConfig *CacheConfig) (*ProviderCache, error)

NewProviderCache creates a new provider cache helper.

func (*ProviderCache) CacheGenerate 

func (pc *ProviderCache) CacheGenerate(
	ctx context.Context,
	modelID string,
	prompt string,
	options []core.GenerateOption,
	generateFn func() (*core.LLMResponse, error),
) (*core.LLMResponse, error)

CacheGenerate wraps a Generate call with caching.

func (*ProviderCache) CacheGenerateContent 

func (pc *ProviderCache) CacheGenerateContent(
	ctx context.Context,
	modelID string,
	contents []Content,
	options []core.GenerateOption,
	generateFn func() (*core.LLMResponse, error),
) (*core.LLMResponse, error)

CacheGenerateContent wraps a GenerateWithContent call with caching.

func (*ProviderCache) CacheGenerateJSON 

func (pc *ProviderCache) CacheGenerateJSON(
	ctx context.Context,
	modelID string,
	prompt string,
	schema interface{},
	options []core.GenerateOption,
	generateFn func() (*core.LLMResponse, error),
) (*core.LLMResponse, error)

CacheGenerateJSON wraps a GenerateWithJSON call with caching.

func (*ProviderCache) CacheStats 

func (pc *ProviderCache) CacheStats() CacheStats

CacheStats returns cache statistics.

func (*ProviderCache) ClearCache 

func (pc *ProviderCache) ClearCache(ctx context.Context) error

ClearCache clears the cache.

func (*ProviderCache) Close 

func (pc *ProviderCache) Close() error

Close closes the cache.

func (*ProviderCache) SetCacheEnabled 

func (pc *ProviderCache) SetCacheEnabled(enabled bool)

SetCacheEnabled enables or disables caching.

type ProviderOption 

type ProviderOption func(*ProviderCache)

ProviderOption is a functional option for configuring providers with caching.

func WithCache 

func WithCache(config CacheConfig) ProviderOption

WithCache enables caching with the given configuration.

func WithCacheMiddleware 

func WithCacheMiddleware(middleware *Middleware) ProviderOption

WithCacheMiddleware uses an existing cache middleware.

type SQLiteCache 

type SQLiteCache struct {
	// contains filtered or unexported fields
}

SQLiteCache implements Cache interface using SQLite as storage.

func NewSQLiteCache 

func NewSQLiteCache(config CacheConfig) (*SQLiteCache, error)

NewSQLiteCache creates a new SQLite-based cache.

func (*SQLiteCache) Clear 

func (c *SQLiteCache) Clear(ctx context.Context) error

func (*SQLiteCache) Close 

func (c *SQLiteCache) Close() error

func (*SQLiteCache) Delete 

func (c *SQLiteCache) Delete(ctx context.Context, key string) error

func (*SQLiteCache) Export 

func (c *SQLiteCache) Export(ctx context.Context, writer func(entry CacheEntry) error) error

Export exports cache entries to JSON for backup/migration.

func (*SQLiteCache) Get 

func (c *SQLiteCache) Get(ctx context.Context, key string) ([]byte, bool, error)

func (*SQLiteCache) Import 

func (c *SQLiteCache) Import(ctx context.Context, entries []CacheEntry) error

Import imports cache entries from a source.

func (*SQLiteCache) Set 

func (c *SQLiteCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error

func (*SQLiteCache) Stats 

func (c *SQLiteCache) Stats() CacheStats

type SQLiteConfig 

type SQLiteConfig struct {
	// Path to the SQLite database file
	Path string `json:"path" yaml:"path"`

	// Enable WAL mode for better concurrent performance
	EnableWAL bool `json:"enable_wal" yaml:"enable_wal"`

	// Vacuum interval for database maintenance
	VacuumInterval time.Duration `json:"vacuum_interval" yaml:"vacuum_interval"`

	// Maximum number of connections
	MaxConnections int `json:"max_connections" yaml:"max_connections"`
}

SQLiteConfig holds SQLite-specific configuration.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.