cache

package
v0.9.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 22, 2026 License: MIT Imports: 18 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	DefaultSemanticTTL          = 24 * time.Hour
	DefaultEvictionInterval     = 1 * time.Hour
	DefaultStatsPersistInterval = 30 * time.Second
	SemanticSimilarityThreshold = 0.92
)

Variables

This section is empty.

Functions

func CalculateTokenSavingsCost 

func CalculateTokenSavingsCost(model string, tokens int64) float64

func DefaultSemanticStatsPath 

func DefaultSemanticStatsPath() string

DefaultSemanticStatsPath returns the stats file location used by both the server (writer) and the CLI (reader): ~/.leanproxy/cache/semantic-stats.json

func ProcessResponse 

func ProcessResponse(result json.RawMessage)

func ProcessResponseFor 

func ProcessResponseFor(provider Provider, result json.RawMessage)

func SetGlobalSemanticCache 

func SetGlobalSemanticCache(sc *SemanticCache)

func SupportedModelList 

func SupportedModelList() string

Types

type BreakpointInjector 

type BreakpointInjector struct {
	// contains filtered or unexported fields
}

func NewBreakpointInjector 

func NewBreakpointInjector(opts ...BreakpointInjectorOption) *BreakpointInjector

func (*BreakpointInjector) Inject 

func (b *BreakpointInjector) Inject(body []byte) ([]byte, error)

func (*BreakpointInjector) Strategy 

func (b *BreakpointInjector) Strategy() InjectStrategy

type BreakpointInjectorOption 

type BreakpointInjectorOption func(*BreakpointInjector)

func WithInjectLogger 

func WithInjectLogger(logger *slog.Logger) BreakpointInjectorOption

func WithStrategy 

func WithStrategy(strategy InjectStrategy) BreakpointInjectorOption

type CacheStats 

type CacheStats struct {
	TotalRequests     int64 `json:"total_requests"`
	AnthropicRequests int64 `json:"anthropic_requests"`
	CacheableRequests int64 `json:"cacheable_requests"`
	CacheHits         int64 `json:"cache_hits"`
	CacheMisses       int64 `json:"cache_misses"`
	InputTokens       int64 `json:"input_tokens"`
	CachedInputTokens int64 `json:"cached_input_tokens"`
	TokensSaved       int64 `json:"tokens_saved"`
}

func (*CacheStats) EstimatedDollarSavings 

func (s *CacheStats) EstimatedDollarSavings(model string) float64

func (*CacheStats) FormatJSON 

func (s *CacheStats) FormatJSON() string

func (*CacheStats) FormatMarkdown 

func (s *CacheStats) FormatMarkdown(model string) string

func (*CacheStats) HasTraffic 

func (s *CacheStats) HasTraffic() bool

func (*CacheStats) HitRate 

func (s *CacheStats) HitRate() float64

type CacheStatsTracker 

type CacheStatsTracker struct {
	// contains filtered or unexported fields
}

func GlobalCacheStatsTracker 

func GlobalCacheStatsTracker() *CacheStatsTracker

func NewCacheStatsTracker 

func NewCacheStatsTracker() *CacheStatsTracker

func (*CacheStatsTracker) GetStats 

func (t *CacheStatsTracker) GetStats() CacheStats

func (*CacheStatsTracker) RecordCacheHit 

func (t *CacheStatsTracker) RecordCacheHit(tokensSaved int64)

func (*CacheStatsTracker) RecordCacheMiss 

func (t *CacheStatsTracker) RecordCacheMiss()

func (*CacheStatsTracker) RecordRequest 

func (t *CacheStatsTracker) RecordRequest(provider Provider, hasBreakpoint bool, inputTokenEstimate int64)

func (*CacheStatsTracker) Reset 

func (t *CacheStatsTracker) Reset()

type DetectorConfig 

type DetectorConfig struct {
	Providers []ProviderConfig `yaml:"providers"`
}

type HitType 

type HitType int
const (
	HitMiss HitType = iota
	HitExact
	HitSemantic
)

func (HitType) String 

func (h HitType) String() string

type InjectStrategy 

type InjectStrategy string
const (
	StrategyOff        InjectStrategy = "off"
	StrategyAggressive InjectStrategy = "aggressive"
	StrategyBalanced   InjectStrategy = "balanced"
)

type ModelPricing 

type ModelPricing struct {
	ModelName              string
	InputCostPerMTok       float64
	CachedInputCostPerMTok float64
	OutputCostPerMTok      float64
}

func ModelCost 

func ModelCost(model string) (ModelPricing, bool)

type Provider 

type Provider string
const (
	ProviderAnthropic Provider = "anthropic"
	ProviderOther     Provider = "other"
)

type ProviderConfig 

type ProviderConfig struct {
	Name     string   `yaml:"name"`
	Patterns []string `yaml:"patterns"`
}

type ProviderDetector 

type ProviderDetector struct {
	// contains filtered or unexported fields
}

func NewProviderDetector 

func NewProviderDetector(opts ...ProviderDetectorOption) *ProviderDetector

func (*ProviderDetector) Detect 

func (d *ProviderDetector) Detect(rawURL string) Provider

func (*ProviderDetector) Load 

func (d *ProviderDetector) Load() error

func (*ProviderDetector) LoadReader 

func (d *ProviderDetector) LoadReader(r io.Reader) (err error)

func (*ProviderDetector) Reload 

func (d *ProviderDetector) Reload() (err error)

type ProviderDetectorOption 

type ProviderDetectorOption func(*ProviderDetector)

func WithConfigPath 

func WithConfigPath(path string) ProviderDetectorOption

func WithConfigReader 

func WithConfigReader(fn func(string) (io.ReadCloser, error)) ProviderDetectorOption

func WithLogger 

func WithLogger(logger *slog.Logger) ProviderDetectorOption

type SemanticCache 

type SemanticCache struct {
	// contains filtered or unexported fields
}

SemanticCache is a tool-scoped prompt cache with exact-match and vector-similarity (semantic) lookup, TTL eviction, and periodic stats persistence. It is safe for concurrent use.

Cache-aside contract: Get/Set never fail the caller's operation. Vector store errors are logged and surfaced as return values where useful, but a degraded (or absent) vector store simply means exact-match-only behavior.

func GlobalSemanticCache 

func GlobalSemanticCache() *SemanticCache

func NewSemanticCache 

func NewSemanticCache(vectorDB vectordb.Store, logger *slog.Logger, ttl time.Duration, opts ...SemanticCacheOption) *SemanticCache

func (*SemanticCache) Get 

func (sc *SemanticCache) Get(ctx context.Context, prompt, toolName string, embedding []float32) (*SemanticCacheResult, error)

Get looks up a cached response for (toolName, prompt). It checks the exact key first, then falls back to vector similarity when an embedding is available. Lookups never fail the caller: errors degrade to a miss.

func (*SemanticCache) Len 

func (sc *SemanticCache) Len() int

func (*SemanticCache) PurgeAll 

func (sc *SemanticCache) PurgeAll() int

func (*SemanticCache) PurgeTool 

func (sc *SemanticCache) PurgeTool(toolName string) int

func (*SemanticCache) Set 

func (sc *SemanticCache) Set(ctx context.Context, prompt string, response json.RawMessage, toolName string, embedding []float32) error

Set stores a response under the tool-scoped key. An empty response is rejected. A vector upsert failure is returned as an error but the in-memory entry is still stored (exact-match remains available).

func (*SemanticCache) Start 

func (sc *SemanticCache) Start(ctx context.Context)

Start launches the background eviction/persistence loop. It is idempotent: calling Start more than once is a no-op.

func (*SemanticCache) Stats 

func (sc *SemanticCache) Stats() SemanticCacheStats

func (*SemanticCache) Stop 

func (sc *SemanticCache) Stop()

Stop shuts the cache down: it blocks new background work, stops the loop, waits for in-flight vector deletes, and writes a final stats snapshot. Get/Set remain usable after Stop (the loop simply no longer runs).

type SemanticCacheEntry 

type SemanticCacheEntry struct {
	Key        string
	Prompt     string
	ToolName   string
	Response   json.RawMessage
	CreatedAt  time.Time
	AccessedAt time.Time
}

type SemanticCacheOption 

type SemanticCacheOption func(*SemanticCache)

func WithEvictionInterval 

func WithEvictionInterval(d time.Duration) SemanticCacheOption

func WithStatsPersistInterval 

func WithStatsPersistInterval(d time.Duration) SemanticCacheOption

func WithStatsPersistPath 

func WithStatsPersistPath(path string) SemanticCacheOption

type SemanticCacheResult 

type SemanticCacheResult struct {
	Response   json.RawMessage
	HitType    HitType
	Similarity float64
}

type SemanticCacheStats 

type SemanticCacheStats struct {
	TotalRequests  int64   `json:"total_requests"`
	ExactHits      int64   `json:"exact_hits"`
	SemanticHits   int64   `json:"semantic_hits"`
	Misses         int64   `json:"misses"`
	AvgSimilarity  float64 `json:"avg_similarity"`
	EvictedEntries int64   `json:"evicted_entries"`
}

func (SemanticCacheStats) FormatJSON 

func (s SemanticCacheStats) FormatJSON() string

func (SemanticCacheStats) FormatMarkdown 

func (s SemanticCacheStats) FormatMarkdown() string

func (SemanticCacheStats) HitRate 

func (s SemanticCacheStats) HitRate() float64

type SemanticStatsSnapshot 

type SemanticStatsSnapshot struct {
	Version   int                `json:"version"`
	UpdatedAt time.Time          `json:"updated_at"`
	Stats     SemanticCacheStats `json:"stats"`
}

SemanticStatsSnapshot is the on-disk representation of semantic cache statistics, written periodically by the running server so that separate CLI processes can render the dashboard.

func LoadSemanticStatsSnapshot 

func LoadSemanticStatsSnapshot(path string) (*SemanticStatsSnapshot, error)

LoadSemanticStatsSnapshot reads a stats snapshot written by the server. A missing file is reported as an error so callers can render an "unavailable" message.

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.