tok

package module

Go to main page

Versions in this module

v0

v0.5.1

May 12, 2026

v0.3.0

May 26, 2026

v0.2.0

May 16, 2026

Changes in this version

type SecretMatch

+ EndPos int

+ Masked string

+ StartPos int

+ Type string

+ Value string

v0.1.2

Jul 4, 2026

v0.1.1

Jun 25, 2026

Changes in this version

+ const CompactionSystemPrompt

+ const DefaultBudgetPct

+ const DefaultMaxBudget

+ const PriorityRead

+ const PrioritySkill

+ const PriorityWriteEdit

+ var ErrCacheOpen = errors.New("failed to open cache database")

+ var ErrChunkFailed = errors.New("chunk processing failed")

+ var ErrCommandUnsafe = errors.New("command contains unsafe shell meta-characters")

+ var ErrConfigInvalid = errors.New("config validation failed")

+ var ErrHashNotFound = errors.New("hash not found in reversible map")

+ var ErrInputTooLarge = errors.New("input exceeds max context tokens")

+ var ErrNoReversibleEntries = errors.New("no reversible entries found")

+ var ModelToEncoding = map[string]Encoding

+ var Version = strings.TrimSpace(versionFile)

+ func BuildCompactionPrompt(context string, maxChars int) string

+ func BuiltinProfiles() map[string]*Profile

+ func ClassifyContent(text string) string

+ func EstimateCompression(content string, ratio float64) string

+ func EstimateCostSavings(stats Stats, model string) float64

+ func EstimateTokensFast(text string) int

+ func EstimateTokensForModel(text string, model string) int

+ func EstimateTokensWithEncoding(text string, encoding string) int

+ func FormatCostSavings(stats Stats, model string) string

+ func FormatResult(result *OptimizationResult) string

+ func FormatStats(stats Stats) string

+ func FormatUsageBar(pct float64, width int) string

+ func IsHighEntropy(s string, threshold float64) bool

+ func IsSensitiveBasename(name string) bool

+ func IsSensitiveFilename(path string) (bool, secrets.FilenameMatch)

+ func ListModels() []string

+ func LoadProfiles(dir string) (map[string]*Profile, error)

+ func MaskSecret(value, secretType string) string

+ func OptimalBudget(contentType string, importance float64) int

+ func RecommendMode(contentType string, budget int) string

+ func RegisterModelPricing(model string, inputPricePer1K, outputPricePer1K float64)

+ func ShannonEntropy(s string) float64

+ func SuggestBudget(blocks []ContentBlock) int

+ func WastedTokens(events []CompressionEvent) int

+ type Alert struct

+ Level string

+ Message string

+ Threshold float64

+ Timestamp time.Time

+ type CompactionSchema struct

+ ContextToPreserve []string

+ CurrentState string

+ ImportantDiscoveries []string

+ NextSteps []string

+ TaskOverview string

+ func ParseCompactionResponse(response string) (*CompactionSchema, error)

+ func (s *CompactionSchema) ToPrompt() string

+ type CompressionAdvisor struct

+ History []CompressionEvent

+ Strategies map[string]*StrategyStats

+ func NewCompressionAdvisor() *CompressionAdvisor

+ func (ca *CompressionAdvisor) Analyze() []Recommendation

+ func (ca *CompressionAdvisor) FormatAdvice() string

+ func (ca *CompressionAdvisor) Record(event CompressionEvent)

+ type CompressionEvent struct

+ ContentType string

+ Input string

+ InputTokens int

+ Mode string

+ Output string

+ OutputTokens int

+ Savings float64

+ Timestamp time.Time

+ type ContentBlock struct

+ Category string

+ CompressedContent string

+ Compressible bool

+ Content string

+ ID string

+ Priority float64

+ Tokens int

+ func CompressBlock(block ContentBlock, targetTokens int) ContentBlock

+ type ContextOptimizer struct

+ Budget int

+ Strategy string

+ func NewContextOptimizer(budget int) *ContextOptimizer

+ func (co *ContextOptimizer) Optimize(blocks []ContentBlock) *OptimizationResult

+ type CustomFilter struct

+ func NewCustomFilter(rules []FilterRule) *CustomFilter

+ func (f *CustomFilter) Apply(text string) string

+ func (f *CustomFilter) ApplyTo(text, contentType string) string

+ func (f *CustomFilter) Len() int

+ type Encoding string

+ const Cl100kBase

+ const ClaudeBase

+ const GeminiBase

+ const O200kBase

+ const P50kBase

+ const R50kBase

+ func GetEncodingForModel(model string) Encoding

+ type FilterRule struct

+ AppliesTo string

+ Enabled bool

+ Name string

+ Pattern string

+ Priority int

+ Replacement string

+ func LoadFilterRules(path string) ([]FilterRule, error)

+ type HeuristicPerplexityScorer struct

+ LengthWeight float64

+ RarityWeight float64

+ StructureWeight float64

+ func NewHeuristicPerplexityScorer() *HeuristicPerplexityScorer

+ func (h *HeuristicPerplexityScorer) Score(_ context.Context, tokens []string) ([]float64, error)

+ type Intensity = compress.Intensity

+ const IntensityFull

+ const IntensityLite

+ const IntensityUltra

+ type JSONExtract = extract.Result

+ func ExtractAllJSON(text string) []JSONExtract

+ func ExtractJSON(text string) JSONExtract

+ func ExtractJSONArray(text string) JSONExtract

+ type ModelPricing struct

+ InputPricePer1K float64

+ OutputPricePer1K float64

+ func GetModelPricing(model string) (ModelPricing, bool)

+ type OptimizationResult struct

+ BudgetUsed float64

+ Compressed []ContentBlock

+ Dropped []ContentBlock

+ Kept []ContentBlock

+ Savings int

+ TotalTokens int

+ func BalancedOptimize(blocks []ContentBlock, budget int) *OptimizationResult

+ func GreedyOptimize(blocks []ContentBlock, budget int) *OptimizationResult

+ func PriorityOptimize(blocks []ContentBlock, budget int) *OptimizationResult

type Option

+ func WithCodeAware(language string) Option

+ func WithCustomFilters(rules []FilterRule) Option

+ func WithLLMLinguaProse() Option

+ func WithModel(model string) Option

+ func WithModelEncoding(encoding Encoding) Option

+ func WithPerplexityGuided(scorer PerplexityScorer, dropRatio float64) Option

+ func WithSymbolProvider(p SymbolProvider) Option

+ type PerplexityScorer interface

+ Score func(ctx context.Context, tokens []string) ([]float64, error)

+ type Profile struct

+ Budget int

+ FilterRulesPath string

+ Mode Mode

+ Name string

+ Notes string

+ Tier Tier

+ Version string

+ func BuiltinProfile(name string) (*Profile, bool)

+ func LoadProfile(path string) (*Profile, error)

+ func (p *Profile) Options() []Option

+ func (p *Profile) Validate() error

+ type PromptStats = compress.Stats

+ func PromptCompress(text string, intensity Intensity) (string, PromptStats)

+ type Recommendation struct

+ ContentType string

+ ExpectedSavings float64

+ Priority int

+ Reason string

+ Strategy string

+ type RegexSymbolProvider struct

+ func (RegexSymbolProvider) Symbols(code, lang string) []Span

+ type RestorationEntry struct

+ Content string

+ OpType string

+ Path string

+ Priority int

+ Timestamp time.Time

+ Tokens int

+ type RestorationTracker struct

+ func NewRestorationTracker() *RestorationTracker

+ func (rt *RestorationTracker) Clear()

+ func (rt *RestorationTracker) ComputeBudget(usedTokens, reserveTokens int) int

+ func (rt *RestorationTracker) Entries() []RestorationEntry

+ func (rt *RestorationTracker) GetRestorations(budget int) []RestorationEntry

+ func (rt *RestorationTracker) Len() int

+ func (rt *RestorationTracker) Track(path, content, opType string)

+ func (rt *RestorationTracker) WithBudgetPct(pct float64) *RestorationTracker

+ func (rt *RestorationTracker) WithContextWindow(tokens int) *RestorationTracker

+ func (rt *RestorationTracker) WithMaxBudget(tokens int) *RestorationTracker

+ type ScorerFunc func(ctx context.Context, tokens []string) ([]float64, error)

+ func (f ScorerFunc) Score(ctx context.Context, tokens []string) ([]float64, error)

+ type SecretDetector struct

+ func DefaultSecretDetector() *SecretDetector

+ func NewSecretDetector() *SecretDetector

+ func (sd *SecretDetector) DetectAndRedactWithEntropy(text string, entropyThreshold float64) string

+ func (sd *SecretDetector) DetectSecrets(text string) []SecretMatch

+ func (sd *SecretDetector) RedactSecrets(text string) string

+ type SecretMatch = secrets.SecretMatch

+ type Span struct

+ EndLine int

+ Lines []string

+ StartLine int

+ Symbol string

type Stats

+ CostSavings float64

+ Model string

+ func CompressPerplexityGuided(text string, scorer PerplexityScorer, dropRatio float64) (string, Stats)

+ type StrategyStats struct

+ AvgQuality float64

+ AvgSavings float64

+ BestSavings float64

+ Name string

+ TotalEvents int

+ WorstSavings float64

type StreamCompressor

+ func (sc *StreamCompressor) Reset()

+ type SymbolProvider interface

+ Symbols func(code, lang string) []Span

+ var DefaultSymbolProvider SymbolProvider = RegexSymbolProvider{}

+ type Tracker struct

+ func NewTracker(ctx context.Context) (*Tracker, error)

+ func NewTrackerAt(ctx context.Context, path string) (*Tracker, error)

+ func (t *Tracker) Aggregate(ctx context.Context, days int) (TrackerAggregate, error)

+ func (t *Tracker) Close() error

+ func (t *Tracker) Prune(ctx context.Context) (int64, error)

+ func (t *Tracker) Recent(ctx context.Context, n int) ([]TrackerEvent, error)

+ func (t *Tracker) Record(ctx context.Context, ev TrackerEvent) error

+ func (t *Tracker) WithRetention(days int) *Tracker

+ type TrackerAggregate = tracking.Aggregate

+ type TrackerEvent = tracking.Event

+ type UsageEntry struct

+ CostUSD float64

+ Model string

+ Provider string

+ Timestamp time.Time

+ Tokens int

+ type UsageSummary struct

+ CostRemaining float64

+ DailyCostUSD float64

+ DailyPct float64

+ DailyRemaining int

+ DailyTokens int

+ HourlyPct float64

+ HourlyRemaining int

+ HourlyTokens int

+ SessionRemaining int

+ SessionTokens int

+ type UsageTracker struct

+ Alerts []Alert

+ CostLimitUSD float64

+ DailyLimit int

+ HourlyLimit int

+ SessionLimit int

+ func NewUsageTracker() *UsageTracker

+ func (u *UsageTracker) CanProceed() (bool, string)

+ func (u *UsageTracker) CheckThresholds()

+ func (u *UsageTracker) EstimateRemaining(tokensPerRequest int) int

+ func (u *UsageTracker) FormatSummary() string

+ func (u *UsageTracker) GetUsage() UsageSummary

+ func (u *UsageTracker) PruneOld()

+ func (u *UsageTracker) Record(tokens int, costUSD float64, provider, model string)

+ func (u *UsageTracker) Reset()

v0.1.0

May 12, 2026

Changes in this version

+ const DefaultMinChunkSize

+ func CompressJSON(text string, maxItems int) string

+ func CompressLog(text string) string

+ func DetectLanguageByExtension(path string) string

+ func EstimateTokens(text string) int

+ func EstimateTokensPrecise(text string) int

+ func GetLanguagePatterns(lang string) []string

+ func RegisterChunker(ext string, fn ChunkerFunc)

+ func RegisterLanguagePatterns(lang string, patterns []string)

+ func WarmupTokenizer()

+ type ChunkOptions struct

+ KeepSeparator SeparatorKeep

+ Language string

+ MaxTokens int

+ MinChunkSize int

+ MinTokens int

+ Overlap int

+ func DefaultChunkOptions() ChunkOptions

+ type ChunkerFunc func(path, content string) (language string, chunks []CodeChunk)

+ type CodeChunk struct

+ Content string

+ EndLine int

+ StartLine int

+ Symbol string

+ Tokens int

+ func ChunkCode(source string, opts ChunkOptions) []CodeChunk

+ func ChunkCodePath(path, source string, opts ChunkOptions) []CodeChunk

+ type Compressor struct

+ func NewCompressor(opts ...Option) *Compressor

+ func (c *Compressor) Compress(text string) (string, Stats)

+ type LayerStat struct

+ DurationMs int64

+ TokensSaved int

+ type Mode string

+ const ModeAggressive

+ const ModeMinimal

+ type Option interface

+ var Adaptive Option = WithTier(TierAdaptive)

+ var Aggressive Option = WithMode(ModeAggressive)

+ var Code Option = WithTier(TierCode)

+ var Log Option = WithTier(TierLog)

+ var Minimal Option = WithMode(ModeMinimal)

+ var Surface Option = WithTier(TierSurface)

+ func WithBudget(tokens int) Option

+ func WithMode(m Mode) Option

+ func WithQuery(intent string) Option

+ func WithTier(t Tier) Option

+ type SeparatorKeep int

+ const SepDiscard

+ const SepLeft

+ const SepRight

+ type Stats struct

+ FinalTokens int

+ Layers map[string]LayerStat

+ OriginalTokens int

+ ReductionPercent float64

+ TokensSaved int

+ func Compress(text string, opts ...Option) (string, Stats)

+ type StreamCompressor struct

+ func NewStreamCompressor(threshold int, opts ...Option) *StreamCompressor

+ func (sc *StreamCompressor) Append(content string)

+ func (sc *StreamCompressor) Close()

+ func (sc *StreamCompressor) Raw() string

+ func (sc *StreamCompressor) Snapshot() (string, Stats)

+ func (sc *StreamCompressor) TokenCount() int

+ type Tier string

+ const TierAdaptive

+ const TierCode

+ const TierCore

+ const TierExtract

+ const TierLog

+ const TierSurface

+ const TierThread

+ const TierTrim

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL