Documentation
¶
Overview ¶
Package llm provides OpenTelemetry and Prometheus instrumentation for LLM calls.
Index ¶
Examples ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type CachedTokenCostProvider ¶
type CachedTokenCostProvider interface {
TokenCostProvider
CachedPromptTokens() int
}
CachedTokenCostProvider is implemented by modern LLM responses supporting prompt caching (DeepSeek V3, Claude 3.5, GPT-4o).
type CostCalculator ¶
CostCalculator calculates basic cost in USD.
type DetailedCostCalculator ¶
type DetailedCostCalculator func(model string, promptTokens, cachedTokens, compTokens int) (float64, bool)
DetailedCostCalculator calculates exact cost factoring in prompt cache discounts.
type Hook ¶ added in v0.3.0
type Hook struct {
// contains filtered or unexported fields
}
Example ¶
package main
import (
"context"
"log/slog"
"github.com/nexssp/kernel/action"
"github.com/nexssp/observability/llm"
)
type MyResponse struct {
model string
prompt int
comp int
}
func (r MyResponse) Model() string { return r.model }
func (r MyResponse) PromptTokens() int { return r.prompt }
func (r MyResponse) CompletionTokens() int { return r.comp }
func main() {
costCalc := func(_ string, prompt, comp int) (float64, bool) {
// Example pricing: $0.01 per 1k tokens for both.
return (float64(prompt) + float64(comp)) * 0.00001, true
}
hook := llm.NewLLMHook(llm.HookOptions{
CostCalculator: costCalc,
Logger: slog.Default(),
})
ctx := context.Background()
meta := &action.Meta{Name: "my_action"}
res := MyResponse{model: "gpt-4", prompt: 100, comp: 50}
hook.AsAnyHook().After(ctx, nil, res, nil, meta)
}
Output:
func NewLLMHook ¶
func NewLLMHook(opts HookOptions) *Hook
type HookOptions ¶ added in v0.3.0
type HookOptions struct {
Metrics *Metrics
CostCalculator CostCalculator
DetailedCostCalculator DetailedCostCalculator
Logger *slog.Logger
}
type Metrics ¶
type Metrics struct {
PromptTokensTotal *prometheus.CounterVec
CachedPromptTokensTotal *prometheus.CounterVec
CompletionTokensTotal *prometheus.CounterVec
CostUSDTotal *prometheus.CounterVec
LLMDurationSeconds *prometheus.HistogramVec
}
func NewMetrics ¶
func NewMetrics(reg prometheus.Registerer, namespace, subsystem string) *Metrics
type TokenCostProvider ¶
TokenCostProvider is implemented by standard LLM responses.
Source Files
Click to show internal directories.
Click to hide internal directories.