Documentation
¶
Index ¶
- func CountLines(content string) int
- func CountTokensInContent(content string) int
- type CompactionImpact
- type CompactionResult
- type CompactionStrategy
- type Config
- type ContextBudget
- type Engine
- func (e *Engine) AutoCompact(ctx context.Context, systemPrompt string, messages []types.Message, ...) (CompactionResult, error)
- func (e *Engine) AutoCompactThresholdTokens(model types.ModelIdentifier) int
- func (e *Engine) BlockingLimitTokens(model types.ModelIdentifier) int
- func (e *Engine) Budget(model types.ModelIdentifier) ContextBudget
- func (e *Engine) CalculateCompactionImpact(messages []types.Message) CompactionImpact
- func (e *Engine) CalculateUsage(systemPrompt string, messages []types.Message) int
- func (e *Engine) CheckBlockingLimit(systemPrompt string, messages []types.Message, model types.ModelIdentifier, ...) error
- func (e *Engine) Compact(ctx context.Context, systemPrompt string, messages []types.Message, ...) (CompactionResult, error)
- func (e *Engine) EffectiveContextWindow(model types.ModelIdentifier) int
- func (e *Engine) EmergencyTrim(systemPrompt string, messages []types.Message, model types.ModelIdentifier, ...) (CompactionResult, bool)
- func (e *Engine) ShouldCompact(systemPrompt string, messages []types.Message, model types.ModelIdentifier, ...) bool
- func (e *Engine) TargetTokens(model types.ModelIdentifier) int
- func (e *Engine) UsedTokens(systemPrompt string, messages []types.Message, tracking *TrackingState) int
- type MessagePreparer
- func (p *MessagePreparer) CalculateMessageTokens(msg types.Message) int
- func (p *MessagePreparer) CalculateMessagesTokens(messages []types.Message) int
- func (p *MessagePreparer) CreateToolUseSummary(toolUses []types.ToolUseContent) string
- func (p *MessagePreparer) CreateTurnSummary(turnNumber int, userMessage types.Message, assistantMessage types.Message, ...) string
- func (p *MessagePreparer) GetCompactionTarget(messages []types.Message, contextWindow types.ContextWindow, ...) int
- func (p *MessagePreparer) PrepareMessageForAPI(msg types.Message) types.Message
- func (p *MessagePreparer) PrepareMessages(messages []types.Message, maxTurns int) []types.Message
- func (p *MessagePreparer) PrepareToolResult(result types.ToolResultContent) types.ToolResultContent
- func (p *MessagePreparer) PrepareToolUse(toolUse types.ToolUseContent) types.ToolUseContent
- func (p *MessagePreparer) ShouldCompact(messages []types.Message, contextWindow types.ContextWindow, threshold float64) bool
- func (p *MessagePreparer) SummarizeToolUses(msg types.Message) string
- type MicroCompactor
- func (m *MicroCompactor) CalculateToolResultSummary(result types.ToolResultContent) string
- func (m *MicroCompactor) CalculateTrimSize(content string) int
- func (m *MicroCompactor) EstimateToolResultTokens(result string) int
- func (m *MicroCompactor) SetMaxToolResultSize(size int)
- func (m *MicroCompactor) SetTrimStrategy(strategy TrimStrategy)
- func (m *MicroCompactor) ShouldTrim(content string) bool
- func (m *MicroCompactor) TrimContentWithMeta(content string) TrimmedContent
- func (m *MicroCompactor) TrimToolResult(result types.ToolResultContent) types.ToolResultContent
- func (m *MicroCompactor) TrimToolResults(results []types.ToolResultContent) []types.ToolResultContent
- type SummaryModelRule
- type TrackingState
- type TrimStrategy
- type TrimmedContent
- type UsageAnchor
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func CountLines ¶
CountLines counts the number of lines in content
func CountTokensInContent ¶
CountTokensInContent estimates tokens in content
Types ¶
type CompactionImpact ¶
type CompactionImpact struct {
OriginalTokens int `json:"original_tokens"`
MicroCompactTokens int `json:"micro_compact_tokens"`
EstimatedFullCompactTokens int `json:"estimated_full_compact_tokens"`
MicroCompactSavings int `json:"micro_compact_savings"`
EstimatedFullCompactSavings int `json:"estimated_full_compact_savings"`
}
CompactionImpact represents the impact of compaction.
type CompactionResult ¶
type CompactionResult struct {
Messages []types.Message `json:"messages"`
DidCompact bool `json:"did_compact"`
UsedMicroCompact bool `json:"used_micro_compact"`
UsedSummaryCompact bool `json:"used_summary_compact"`
PreCompactTokens int `json:"pre_compact_tokens"`
PostCompactTokens int `json:"post_compact_tokens"`
TargetTokens int `json:"target_tokens"`
Summary string `json:"summary,omitempty"`
ConsecutiveFailures int `json:"consecutive_failures"`
RuntimeMetadata *types.CompactionMetadata `json:"runtime_metadata,omitempty"`
}
CompactionResult is the canonical compaction outcome.
type CompactionStrategy ¶
type CompactionStrategy interface {
// AutoCompact compacts messages if the token budget is exceeded.
// It respects the circuit breaker via TrackingState.ConsecutiveFailures.
// When no compaction is needed it returns the original messages unchanged
// with DidCompact == false.
AutoCompact(
ctx context.Context,
systemPrompt string,
messages []types.Message,
model types.ModelIdentifier,
sessionID types.SessionID,
turnID types.TurnID,
tracking *TrackingState,
) (CompactionResult, error)
// CheckBlockingLimit reports whether systemPrompt+messages fit within the
// hard blocking limit for model (BlockingLimitTokens), returning a typed
// error (ErrCodeCompactFull) if not - even after AutoCompact has already
// tried. Called right after AutoCompact so an oversized request fails
// locally with a clear reason instead of at the provider (M1 in TODO.md).
CheckBlockingLimit(systemPrompt string, messages []types.Message, model types.ModelIdentifier, tracking *TrackingState) error
}
CompactionStrategy abstracts the compaction algorithm used by the query loop. Callers pass the current conversation state; the strategy decides whether and how to compact, returning a canonical CompactionResult.
The Engine type is the default implementation. Swap the strategy in tests or to add new algorithms (e.g., importance-weighted, semantic chunking) without touching the loop.
type Config ¶
type Config struct {
AutoCompactThreshold float64 `json:"auto_compact_threshold"`
CompactTargetPercentage float64 `json:"compact_target_percentage"`
MaxSummaryTokens int `json:"max_summary_tokens"`
AutoCompactBufferTokens int `json:"auto_compact_buffer_tokens"`
ManualCompactBufferTokens int `json:"manual_compact_buffer_tokens"`
MaxConsecutiveFailures int `json:"max_consecutive_failures"`
// SummaryModel overrides which model generates the compaction summary.
// The zero value means "use the session's own model" (Engine.generateSummary
// falls back to the model passed into Compact/AutoCompact), so a session on
// any provider can compact without needing a key for a different one.
SummaryModel types.ModelIdentifier `json:"summary_model,omitempty"`
// SummaryModelRules choose the model that writes the summary by the provider and model of the session, first match
// wins. They apply when SummaryModel is not set, and before the built-in rules (see resolveSummaryModel).
SummaryModelRules []SummaryModelRule `json:"summary_model_rules,omitempty"`
// NoDefaultSummaryModels switches off the built-in rules, so that a session without a matching rule of its own
// summarizes with its own model.
NoDefaultSummaryModels bool `json:"no_default_summary_models,omitempty"`
}
Config represents compaction configuration.
func DefaultConfig ¶
func DefaultConfig() *Config
DefaultConfig returns default compaction configuration.
type ContextBudget ¶ added in v1.10.1
type ContextBudget struct {
WindowTokens int `json:"window_tokens"`
AutoCompactAtTokens int `json:"auto_compact_at_tokens"`
BlockingLimitTokens int `json:"blocking_limit_tokens"`
}
ContextBudget is where a model's window stands against the compaction thresholds.
type Engine ¶
type Engine struct {
// contains filtered or unexported fields
}
Engine performs runtime-driven conversation compaction.
func (*Engine) AutoCompact ¶
func (e *Engine) AutoCompact( ctx context.Context, systemPrompt string, messages []types.Message, model types.ModelIdentifier, sessionID types.SessionID, turnID types.TurnID, tracking *TrackingState, ) (CompactionResult, error)
AutoCompact performs compaction if needed and respects the circuit breaker.
func (*Engine) AutoCompactThresholdTokens ¶
func (e *Engine) AutoCompactThresholdTokens(model types.ModelIdentifier) int
AutoCompactThresholdTokens returns the absolute token threshold for auto-compaction.
func (*Engine) BlockingLimitTokens ¶
func (e *Engine) BlockingLimitTokens(model types.ModelIdentifier) int
BlockingLimitTokens returns the hard blocking limit for the current model.
func (*Engine) Budget ¶ added in v1.10.1
func (e *Engine) Budget(model types.ModelIdentifier) ContextBudget
Budget reports the window of a model and the two thresholds that matter to a host showing a context gauge. A nil engine (a host that wires none) reports a zero budget.
func (*Engine) CalculateCompactionImpact ¶
func (e *Engine) CalculateCompactionImpact(messages []types.Message) CompactionImpact
CalculateCompactionImpact calculates the impact of compaction.
func (*Engine) CalculateUsage ¶
CalculateUsage estimates the tokens of a system prompt and messages. It ignores the tool definitions and anything the provider reported; UsedTokens is the figure to decide on.
func (*Engine) CheckBlockingLimit ¶
func (e *Engine) CheckBlockingLimit(systemPrompt string, messages []types.Message, model types.ModelIdentifier, tracking *TrackingState) error
CheckBlockingLimit implements CompactionStrategy.CheckBlockingLimit.
func (*Engine) Compact ¶
func (e *Engine) Compact( ctx context.Context, systemPrompt string, messages []types.Message, model types.ModelIdentifier, sessionID types.SessionID, turnID types.TurnID, tracking *TrackingState, ) (CompactionResult, error)
Compact performs compaction and returns a canonical result.
func (*Engine) EffectiveContextWindow ¶
func (e *Engine) EffectiveContextWindow(model types.ModelIdentifier) int
EffectiveContextWindow returns the effective context window after reserving summary output budget.
func (*Engine) EmergencyTrim ¶ added in v1.10.2
func (e *Engine) EmergencyTrim(systemPrompt string, messages []types.Message, model types.ModelIdentifier, tracking *TrackingState) (CompactionResult, bool)
EmergencyTrim shrinks a conversation without asking a model, for the moment a summary could not be written (the model returns nothing, the provider is down, the circuit breaker is open) and the request is over the hard limit. Before this, such a conversation ended the turn with an error, and once the breaker had opened nothing ever shrank it.
It trims the large tool results, and if that is not enough it drops the oldest messages, keeping the most recent ones up to the compaction target, behind a marker that says so. The dropped messages are archived by the session store like those of any compaction. It reports false when it changed nothing: the conversation is already within the target, or there is nothing older than the last message to drop.
func (*Engine) ShouldCompact ¶
func (e *Engine) ShouldCompact(systemPrompt string, messages []types.Message, model types.ModelIdentifier, tracking *TrackingState) bool
ShouldCompact returns true if compaction should be attempted.
func (*Engine) TargetTokens ¶
func (e *Engine) TargetTokens(model types.ModelIdentifier) int
TargetTokens returns the post-compaction target token count.
func (*Engine) UsedTokens ¶ added in v1.10.1
func (e *Engine) UsedTokens(systemPrompt string, messages []types.Message, tracking *TrackingState) int
UsedTokens is the size of the request about to be sent. With a provider-reported anchor it is that figure plus an estimate of the messages added since; without one it is the estimate plus the tool definitions.
type MessagePreparer ¶
type MessagePreparer struct {
// contains filtered or unexported fields
}
MessagePreparer prepares messages for compaction
func NewMessagePreparer ¶
func NewMessagePreparer() *MessagePreparer
NewMessagePreparer creates a new message preparer
func (*MessagePreparer) CalculateMessageTokens ¶
func (p *MessagePreparer) CalculateMessageTokens(msg types.Message) int
CalculateMessageTokens estimates tokens for a message
func (*MessagePreparer) CalculateMessagesTokens ¶
func (p *MessagePreparer) CalculateMessagesTokens(messages []types.Message) int
CalculateMessagesTokens calculates total tokens for messages
func (*MessagePreparer) CreateToolUseSummary ¶
func (p *MessagePreparer) CreateToolUseSummary(toolUses []types.ToolUseContent) string
CreateToolUseSummary creates a summary of tool uses
func (*MessagePreparer) CreateTurnSummary ¶
func (p *MessagePreparer) CreateTurnSummary( turnNumber int, userMessage types.Message, assistantMessage types.Message, toolResults []types.ToolResultContent, ) string
CreateTurnSummary creates a summary of a turn
func (*MessagePreparer) GetCompactionTarget ¶
func (p *MessagePreparer) GetCompactionTarget( messages []types.Message, contextWindow types.ContextWindow, targetPercentage float64, ) int
GetCompactionTarget calculates how many tokens need to be removed
func (*MessagePreparer) PrepareMessageForAPI ¶
func (p *MessagePreparer) PrepareMessageForAPI(msg types.Message) types.Message
PrepareMessageForAPI prepares a single message for the API
func (*MessagePreparer) PrepareMessages ¶
func (p *MessagePreparer) PrepareMessages( messages []types.Message, maxTurns int, ) []types.Message
PrepareMessages prepares messages for the API request
func (*MessagePreparer) PrepareToolResult ¶
func (p *MessagePreparer) PrepareToolResult(result types.ToolResultContent) types.ToolResultContent
PrepareToolResult prepares a tool result for storage
func (*MessagePreparer) PrepareToolUse ¶
func (p *MessagePreparer) PrepareToolUse(toolUse types.ToolUseContent) types.ToolUseContent
PrepareToolUse prepares a tool use for execution
func (*MessagePreparer) ShouldCompact ¶
func (p *MessagePreparer) ShouldCompact( messages []types.Message, contextWindow types.ContextWindow, threshold float64, ) bool
ShouldCompact returns true if messages should be compacted
func (*MessagePreparer) SummarizeToolUses ¶
func (p *MessagePreparer) SummarizeToolUses(msg types.Message) string
SummarizeToolUses summarizes tool uses in a message
type MicroCompactor ¶
type MicroCompactor struct {
// contains filtered or unexported fields
}
MicroCompactor performs micro-compaction of tool results
func NewMicroCompactor ¶
func NewMicroCompactor() *MicroCompactor
NewMicroCompactor creates a new micro compactor
func (*MicroCompactor) CalculateToolResultSummary ¶
func (m *MicroCompactor) CalculateToolResultSummary(result types.ToolResultContent) string
CalculateToolResultSummary generates a summary of a tool result
func (*MicroCompactor) CalculateTrimSize ¶
func (m *MicroCompactor) CalculateTrimSize(content string) int
CalculateTrimSize calculates how much would be trimmed
func (*MicroCompactor) EstimateToolResultTokens ¶
func (m *MicroCompactor) EstimateToolResultTokens(result string) int
EstimateToolResultTokens estimates tokens in a tool result
func (*MicroCompactor) SetMaxToolResultSize ¶
func (m *MicroCompactor) SetMaxToolResultSize(size int)
SetMaxToolResultSize sets the maximum tool result size
func (*MicroCompactor) SetTrimStrategy ¶
func (m *MicroCompactor) SetTrimStrategy(strategy TrimStrategy)
SetTrimStrategy sets the trim strategy
func (*MicroCompactor) ShouldTrim ¶
func (m *MicroCompactor) ShouldTrim(content string) bool
ShouldTrim returns true if content should be trimmed
func (*MicroCompactor) TrimContentWithMeta ¶
func (m *MicroCompactor) TrimContentWithMeta(content string) TrimmedContent
TrimContentWithMeta trims content and returns metadata
func (*MicroCompactor) TrimToolResult ¶
func (m *MicroCompactor) TrimToolResult(result types.ToolResultContent) types.ToolResultContent
TrimToolResult trims a tool result if it's too large
func (*MicroCompactor) TrimToolResults ¶
func (m *MicroCompactor) TrimToolResults(results []types.ToolResultContent) []types.ToolResultContent
TrimToolResults trims multiple tool results
type SummaryModelRule ¶ added in v1.10.2
type SummaryModelRule struct {
// Provider is the provider of the session this rule applies to. Empty applies to any provider.
Provider types.APIProvider `json:"provider,omitempty"`
// Model is the session model this rule applies to: an exact name, or a name ending in "*" for a prefix
// ("glm-*"). Empty applies to any model of the provider.
Model string `json:"model,omitempty"`
// Use is the model that writes the summary. An empty Use.Provider means the session's own provider: a rule written
// without a provider stays at the vendor the session already trusts with its conversation, and so with its key.
Use types.ModelIdentifier `json:"use"`
}
SummaryModelRule says which model writes the summary for a session on a given provider and model.
type TrackingState ¶
type TrackingState struct {
ConsecutiveFailures int `json:"consecutive_failures"`
// ExtraTokens is the size of the request outside the system prompt and the messages: the tool definitions, which
// cost context on every request (about 36,000 tokens for the whole builtin registry).
ExtraTokens int `json:"extra_tokens,omitempty"`
// Manual marks a compaction the user asked for. The automatic rules assume the window is nearly full: they keep
// half of it as it is, and skip the summary when trimming tool results is enough. A manual compaction is asked for
// whatever the size, so it always summarizes and keeps only the most recent fifth of the conversation.
Manual bool `json:"manual,omitempty"`
// Anchor is the last size the provider reported. When set, the size of the request is that figure plus an estimate
// of the messages added since, which is exact for everything the provider counted (tools included).
Anchor *UsageAnchor `json:"anchor,omitempty"`
}
TrackingState carries circuit-breaker state across loop iterations, and what the loop knows about the size of the request beyond the messages it hands over.
type TrimStrategy ¶
type TrimStrategy string
TrimStrategy represents how to trim tool results
const ( TrimStrategyTruncate TrimStrategy = "truncate" TrimStrategySnip TrimStrategy = "snip" TrimStrategyPreview TrimStrategy = "preview" )
type TrimmedContent ¶
type TrimmedContent struct {
// Content is the trimmed content
Content string `json:"content"`
// OriginalSize is the original size
OriginalSize int `json:"original_size"`
// TrimmedSize is the trimmed size
TrimmedSize int `json:"trimmed_size"`
// Strategy is the trim strategy used
Strategy TrimStrategy `json:"strategy"`
// Preview is a preview of the original content
Preview string `json:"preview,omitempty"`
}
TrimmedContent represents trimmed content with metadata
type UsageAnchor ¶ added in v1.10.1
UsageAnchor is the size of the conversation as the provider counted it, and how many messages that covered.