tokenstrip

type DropThinkingMode int

type Options struct {
	// EnableSynonymSub turns on phrase→synonym substitution inside assistant
	// <thinking> blocks. Off by default even when tokenstrip itself is on,
	// because the table is opinionated and can produce awkward reasoning
	// text; callers should opt in explicitly.
	EnableSynonymSub bool

	// DropThinkingMode controls whether <thinking>...</thinking> blocks in
	// assistant content are reduced more aggressively than the default
	// stop-word strip. Default is DropThinkingNone (preserve current
	// behavior — full block kept, with stop-word removal applied to its
	// contents).
	//
	// On long Sonnet/Opus sessions, thinking blocks can be 30–50% of
	// total assistant prose. The summary schema (title, key_actions,
	// chapter_titles, aha_moments) doesn't require them, but the chain
	// of reasoning sometimes contains the framing for an aha_moment, so
	// we offer a conservative middle ground:
	//
	//   - DropThinkingNone: keep the entire block (current behavior).
	//   - DropThinkingFirstSentence: keep only the first sentence of
	//     each block (typically the framing — "Let me work out X" or
	//     "I need to figure out Y") and elide the body. Preserves the
	//     hint of where the reasoning was going without the deliberation
	//     cost.
	//   - DropThinkingAll: drop the block entirely. Maximum savings,
	//     maximum risk.
	//
	// Recommended rollout: DropThinkingFirstSentence behind an env-var
	// flag for a few weeks, A/B against EventSummarization quality_score
	// distribution, flip to default if there's no measurable quality
	// drop on the cohort that has it enabled.
	DropThinkingMode DropThinkingMode

	// SynonymTable overrides the default substitution table. Keys are
	// matched case-insensitively as whole words. A nil map falls back to
	// DefaultSynonymTable().
	SynonymTable map[string]string

	// StopWordLanguage is an ISO-639-1 language code (e.g. "en", "fr").
	// Empty string defaults to "en".
	StopWordLanguage string
}

type Stats struct {
	EntriesIn  int
	EntriesOut int
	BytesIn    int64
	BytesOut   int64

	NFCNormalized           int // entries where NFC normalization changed content
	ZeroWidthStripped       int // entries where zero-width / unusual whitespace was removed
	WhitespaceCanonicalized int // entries where whitespace collapse changed content
	StopWordsRemoved        int // <thinking> blocks where stop words were removed
	SynonymsSubstituted     int // <thinking> blocks where synonym substitution fired

	// ThinkingBlocksTrimmed counts <thinking> blocks reduced under
	// DropThinkingFirstSentence (kept first sentence only).
	ThinkingBlocksTrimmed int
	// ThinkingBlocksDropped counts <thinking> blocks removed entirely
	// under DropThinkingAll.
	ThinkingBlocksDropped int

	// Token estimates use a ~4 chars/token heuristic (Anthropic's rule of
	// thumb). They exist so callers can log a rough token-reduction number
	// without pulling in a BPE-heavy tokenizer. When a real tokenizer is
	// wired in later, swap the estimator in transforms.go and these fields
	// will reflect actual counts.
	TokensInEstimate  int64
	TokensOutEstimate int64
}