Why Go
- Case Studies
  
  Common problems companies solve with Go
- Use Cases
  
  Stories about how and why companies use Go
- Security
  
  How Go can help keep you secure by default
Learn
Docs
- Effective Go
  
  Tips for writing clear, performant, and idiomatic Go code
- Go User Manual
  
  A complete introduction to building software with Go
- Standard library
  
  Reference documentation for Go's standard library
- Release Notes
  
  Learn what's new in each Go release
Packages
Community
- Recorded Talks
  
  Videos from prior events
- Meetups
  
  Meet other local Go developers
- Conferences
  
  Learn and network with Go developers from around the world
- Go blog
  
  The Go project's official blog.
- Go project
  
  Get help and stay informed from Go
- Get connected

tok

package module

Go to main page

Versions in this module

v0

v0.5.1

May 12, 2026

v0.2.0

May 16, 2026

Changes in this version

+ const CompactionSystemPrompt

+ var Version = strings.TrimSpace(versionFile)

+ func BuildCompactionPrompt(context string, maxChars int) string

+ func ClassifyContent(text string) string

+ func EstimateCompression(content string, ratio float64) string

+ func FormatResult(result *OptimizationResult) string

+ func FormatUsageBar(pct float64, width int) string

+ func IsHighEntropy(s string, threshold float64) bool

+ func OptimalBudget(contentType string, importance float64) int

+ func RecommendMode(contentType string, budget int) string

+ func ShannonEntropy(s string) float64

+ func SuggestBudget(blocks []ContentBlock) int

+ func WastedTokens(events []CompressionEvent) int

+ type Alert struct

+ Level string

+ Message string

+ Threshold float64

+ Timestamp time.Time

+ type CompactionSchema struct

+ ContextToPreserve []string

+ CurrentState string

+ ImportantDiscoveries []string

+ NextSteps []string

+ TaskOverview string

+ func ParseCompactionResponse(response string) (*CompactionSchema, error)

+ func (s *CompactionSchema) ToPrompt() string

+ type CompressionAdvisor struct

+ History []CompressionEvent

+ Strategies map[string]*StrategyStats

+ func NewCompressionAdvisor() *CompressionAdvisor

+ func (ca *CompressionAdvisor) Analyze() []Recommendation

+ func (ca *CompressionAdvisor) FormatAdvice() string

+ func (ca *CompressionAdvisor) Record(event CompressionEvent)

+ type CompressionEvent struct

+ ContentType string

+ Input string

+ InputTokens int

+ Mode string

+ Output string

+ OutputTokens int

+ Savings float64

+ Timestamp time.Time

+ type ContentBlock struct

+ Category string

+ CompressedContent string

+ Compressible bool

+ Content string

+ ID string

+ Priority float64

+ Tokens int

+ func CompressBlock(block ContentBlock, targetTokens int) ContentBlock

+ type ContextOptimizer struct

+ Budget int

+ Strategy string

+ func NewContextOptimizer(budget int) *ContextOptimizer

+ func (co *ContextOptimizer) Optimize(blocks []ContentBlock) *OptimizationResult

+ type OptimizationResult struct

+ BudgetUsed float64

+ Compressed []ContentBlock

+ Dropped []ContentBlock

+ Kept []ContentBlock

+ Savings int

+ TotalTokens int

+ func BalancedOptimize(blocks []ContentBlock, budget int) *OptimizationResult

+ func GreedyOptimize(blocks []ContentBlock, budget int) *OptimizationResult

+ func PriorityOptimize(blocks []ContentBlock, budget int) *OptimizationResult

+ type Recommendation struct

+ ContentType string

+ ExpectedSavings float64

+ Priority int

+ Reason string

+ Strategy string

+ type SecretDetector struct

+ func DefaultSecretDetector() *SecretDetector

+ func NewSecretDetector() *SecretDetector

+ func (sd *SecretDetector) DetectAndRedactWithEntropy(text string, entropyThreshold float64) string

+ func (sd *SecretDetector) DetectSecrets(text string) []SecretMatch

+ func (sd *SecretDetector) RedactSecrets(text string) string

+ type SecretMatch struct

+ EndPos int

+ Masked string

+ StartPos int

+ Type string

+ Value string

+ type StrategyStats struct

+ AvgQuality float64

+ AvgSavings float64

+ BestSavings float64

+ Name string

+ TotalEvents int

+ WorstSavings float64

+ type UsageEntry struct

+ CostUSD float64

+ Model string

+ Provider string

+ Timestamp time.Time

+ Tokens int

+ type UsageSummary struct

+ CostRemaining float64

+ DailyCostUSD float64

+ DailyPct float64

+ DailyRemaining int

+ DailyTokens int

+ HourlyPct float64

+ HourlyRemaining int

+ HourlyTokens int

+ SessionRemaining int

+ SessionTokens int

+ type UsageTracker struct

+ Alerts []Alert

+ CostLimitUSD float64

+ DailyLimit int

+ HourlyLimit int

+ SessionLimit int

+ func NewUsageTracker() *UsageTracker

+ func (u *UsageTracker) CanProceed() (bool, string)

+ func (u *UsageTracker) CheckThresholds()

+ func (u *UsageTracker) EstimateRemaining(tokensPerRequest int) int

+ func (u *UsageTracker) FormatSummary() string

+ func (u *UsageTracker) GetUsage() UsageSummary

+ func (u *UsageTracker) PruneOld()

+ func (u *UsageTracker) Record(tokens int, costUSD float64, provider, model string)

+ func (u *UsageTracker) Reset()

v0.1.0

May 12, 2026

Changes in this version

+ const DefaultMinChunkSize

+ func CompressJSON(text string, maxItems int) string

+ func CompressLog(text string) string

+ func DetectLanguageByExtension(path string) string

+ func EstimateTokens(text string) int

+ func EstimateTokensPrecise(text string) int

+ func GetLanguagePatterns(lang string) []string

+ func RegisterChunker(ext string, fn ChunkerFunc)

+ func RegisterLanguagePatterns(lang string, patterns []string)

+ func WarmupTokenizer()

+ type ChunkOptions struct

+ KeepSeparator SeparatorKeep

+ Language string

+ MaxTokens int

+ MinChunkSize int

+ MinTokens int

+ Overlap int

+ func DefaultChunkOptions() ChunkOptions

+ type ChunkerFunc func(path, content string) (language string, chunks []CodeChunk)

+ type CodeChunk struct

+ Content string

+ EndLine int

+ StartLine int

+ Symbol string

+ Tokens int

+ func ChunkCode(source string, opts ChunkOptions) []CodeChunk

+ func ChunkCodePath(path, source string, opts ChunkOptions) []CodeChunk

+ type Compressor struct

+ func NewCompressor(opts ...Option) *Compressor

+ func (c *Compressor) Compress(text string) (string, Stats)

+ type LayerStat struct

+ DurationMs int64

+ TokensSaved int

+ type Mode string

+ const ModeAggressive

+ const ModeMinimal

+ type Option interface

+ var Adaptive Option = WithTier(TierAdaptive)

+ var Aggressive Option = WithMode(ModeAggressive)

+ var Code Option = WithTier(TierCode)

+ var Log Option = WithTier(TierLog)

+ var Minimal Option = WithMode(ModeMinimal)

+ var Surface Option = WithTier(TierSurface)

+ func WithBudget(tokens int) Option

+ func WithMode(m Mode) Option

+ func WithQuery(intent string) Option

+ func WithTier(t Tier) Option

+ type SeparatorKeep int

+ const SepDiscard

+ const SepLeft

+ const SepRight

+ type Stats struct

+ FinalTokens int

+ Layers map[string]LayerStat

+ OriginalTokens int

+ ReductionPercent float64

+ TokensSaved int

+ func Compress(text string, opts ...Option) (string, Stats)

+ type StreamCompressor struct

+ func NewStreamCompressor(threshold int, opts ...Option) *StreamCompressor

+ func (sc *StreamCompressor) Append(content string)

+ func (sc *StreamCompressor) Close()

+ func (sc *StreamCompressor) Raw() string

+ func (sc *StreamCompressor) Snapshot() (string, Stats)

+ func (sc *StreamCompressor) TokenCount() int

+ type Tier string

+ const TierAdaptive

+ const TierCode

+ const TierCore

+ const TierExtract

+ const TierLog

+ const TierSurface

+ const TierThread

+ const TierTrim