Documentation
¶
Overview ¶
Package cache implements ephemeral prompt caching for improved performance.
This package provides caching mechanisms for AI prompts to reduce costs and improve response times through Anthropic's prompt caching feature.
Features:
- Cache control headers for system prompts
- Cache control headers for large context
- Automatic cache key generation
- Cache hit/miss metrics tracking
- Configurable cache behavior
- TTL-based cache expiration
Prompt caching works by marking portions of prompts as cacheable, allowing the AI provider to reuse processed prompts across requests.
Example usage:
import "github.com/AINative-studio/ainative-code/internal/cache"
// Create cache manager
manager := cache.NewManager(cache.Config{
Enabled: true,
MinPromptLength: 1024,
SystemPromptCache: true,
})
// Mark content as cacheable
cacheControl := manager.ShouldCache(content)
if cacheControl != nil {
// Add cache control to API request
}
// Track cache metrics
manager.RecordCacheHit("system_prompt")
stats := manager.GetStats()
Index ¶
- type CacheControl
- type CacheKeyMetrics
- type CacheMetrics
- type CacheStatus
- type CacheableContent
- type Config
- type Manager
- func (m *Manager) ClearCache()
- func (m *Manager) FormatStats() string
- func (m *Manager) GetCacheKeys() []string
- func (m *Manager) GetCacheSize() int64
- func (m *Manager) GetCacheStatus(cacheKey string) *CacheStatus
- func (m *Manager) GetConfig() Config
- func (m *Manager) GetStats() *CacheMetrics
- func (m *Manager) InvalidateCache(cacheKey string)
- func (m *Manager) IsCached(cacheKey string) bool
- func (m *Manager) RecordBytesSaved(bytes int64)
- func (m *Manager) RecordCacheHit(cacheKey string)
- func (m *Manager) RecordCacheMiss(cacheKey string)
- func (m *Manager) RecordCached(cacheKey string, bytesSize int64)
- func (m *Manager) ResetMetrics()
- func (m *Manager) ShouldCache(content *CacheableContent) *CacheControl
- func (m *Manager) Stop()
- func (m *Manager) UpdateConfig(config Config)
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type CacheControl ¶
type CacheControl struct {
Type string `json:"type"` // "ephemeral" for Anthropic prompt caching
Enabled bool `json:"enabled"` // Whether caching is enabled for this segment
CacheKey string `json:"cache_key"` // Unique identifier for this cached content
TTL int `json:"ttl"` // Time-to-live in seconds (optional)
Breakpoint bool `json:"breakpoint"` // Whether this is a cache breakpoint
}
CacheControl represents cache control directives for a prompt segment
type CacheKeyMetrics ¶
type CacheKeyMetrics struct {
Key string `json:"key"`
Hits int64 `json:"hits"`
Misses int64 `json:"misses"`
BytesCached int64 `json:"bytes_cached"`
LastHit time.Time `json:"last_hit"`
Created time.Time `json:"created"`
}
CacheKeyMetrics tracks metrics for a specific cache key
type CacheMetrics ¶
type CacheMetrics struct {
TotalRequests int64 `json:"total_requests"`
CacheHits int64 `json:"cache_hits"`
CacheMisses int64 `json:"cache_misses"`
BytesCached int64 `json:"bytes_cached"`
BytesSaved int64 `json:"bytes_saved"`
AverageHitRate float64 `json:"average_hit_rate"`
LastReset time.Time `json:"last_reset"`
CacheByKey map[string]*CacheKeyMetrics `json:"cache_by_key"`
}
CacheMetrics tracks cache performance metrics
type CacheStatus ¶
type CacheStatus struct {
Key string `json:"key"`
Cached bool `json:"cached"`
ExpiresAt time.Time `json:"expires_at"`
BytesSize int64 `json:"bytes_size"`
HitCount int64 `json:"hit_count"`
LastAccess time.Time `json:"last_access"`
}
CacheStatus represents the current status of a cache entry
type CacheableContent ¶
type CacheableContent struct {
Content string `json:"content"`
Type string `json:"type"` // "system", "context", "tools"
Length int `json:"length"`
CacheKey string `json:"cache_key"`
Priority int `json:"priority"` // Higher priority cached first
Breakpoint bool `json:"breakpoint"` // Mark as cache breakpoint
}
CacheableContent represents content that can be cached
type Config ¶
type Config struct {
Enabled bool `json:"enabled"`
MinPromptLength int `json:"min_prompt_length"` // Minimum length to cache (default: 1024)
MaxPromptLength int `json:"max_prompt_length"` // Maximum length to cache (default: 0 = unlimited)
SystemPromptCache bool `json:"system_prompt_cache"` // Cache system prompts
ContextCache bool `json:"context_cache"` // Cache conversation context
TTL time.Duration `json:"ttl"` // Default TTL (0 = use provider default)
AutoCleanup bool `json:"auto_cleanup"` // Automatically cleanup expired cache
CleanupInterval time.Duration `json:"cleanup_interval"` // How often to run cleanup
}
Config represents caching configuration
func DefaultConfig ¶
func DefaultConfig() Config
DefaultConfig returns the default caching configuration
type Manager ¶
type Manager struct {
// contains filtered or unexported fields
}
Manager manages prompt caching
func (*Manager) FormatStats ¶
FormatStats formats cache statistics as a human-readable string
func (*Manager) GetCacheKeys ¶
GetCacheKeys returns all currently cached keys
func (*Manager) GetCacheSize ¶
GetCacheSize returns the total size of cached content in bytes
func (*Manager) GetCacheStatus ¶
func (m *Manager) GetCacheStatus(cacheKey string) *CacheStatus
GetCacheStatus returns the status of a specific cache key
func (*Manager) GetStats ¶
func (m *Manager) GetStats() *CacheMetrics
GetStats returns current cache statistics
func (*Manager) InvalidateCache ¶
InvalidateCache invalidates a specific cache key
func (*Manager) RecordBytesSaved ¶
RecordBytesSaved records bytes saved due to caching
func (*Manager) RecordCacheHit ¶
RecordCacheHit records a cache hit for metrics
func (*Manager) RecordCacheMiss ¶
RecordCacheMiss records a cache miss for metrics
func (*Manager) RecordCached ¶
RecordCached records that content was cached
func (*Manager) ResetMetrics ¶
func (m *Manager) ResetMetrics()
ResetMetrics resets all cache metrics
func (*Manager) ShouldCache ¶
func (m *Manager) ShouldCache(content *CacheableContent) *CacheControl
ShouldCache determines if content should be cached and returns cache control
func (*Manager) UpdateConfig ¶
UpdateConfig updates the cache configuration
Source Files