fusionpipeline

package

v1.5.0 Latest Latest Go to latest Published: Apr 2, 2026 License: MIT Imports: 1 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/GrayCodeAI/tokman

Links

Open Source Insights

Documentation ¶

Index ¶

type AbbrevStage
- func NewAbbrevStage() *AbbrevStage
- func (s *AbbrevStage) Apply(input string) (string, int)
- func (s *AbbrevStage) Name() string
- func (s *AbbrevStage) ShouldApply(input string) bool
type CortexStage
- func NewCortexStage() *CortexStage
- func (s *CortexStage) Apply(input string) (string, int)
- func (s *CortexStage) Name() string
- func (s *CortexStage) ShouldApply(input string) bool
type DiffCrunchStage
- func NewDiffCrunchStage() *DiffCrunchStage
- func (s *DiffCrunchStage) Apply(input string) (string, int)
- func (s *DiffCrunchStage) Name() string
- func (s *DiffCrunchStage) ShouldApply(input string) bool
type FusionEngine
- func NewFusionEngine() *FusionEngine
- func (e *FusionEngine) Compress(input string) (string, *FusionResult)
type FusionResult
type FusionStage
type IonizerStage
- func NewIonizerStage() *IonizerStage
- func (s *IonizerStage) Apply(input string) (string, int)
- func (s *IonizerStage) Name() string
- func (s *IonizerStage) ShouldApply(input string) bool
type LogCrunchStage
- func NewLogCrunchStage() *LogCrunchStage
- func (s *LogCrunchStage) Apply(input string) (string, int)
- func (s *LogCrunchStage) Name() string
- func (s *LogCrunchStage) ShouldApply(input string) bool
type NeurosyntaxStage
- func NewNeurosyntaxStage() *NeurosyntaxStage
- func (s *NeurosyntaxStage) Apply(input string) (string, int)
- func (s *NeurosyntaxStage) Name() string
- func (s *NeurosyntaxStage) ShouldApply(input string) bool
type NexusStage
- func NewNexusStage() *NexusStage
- func (s *NexusStage) Apply(input string) (string, int)
- func (s *NexusStage) Name() string
- func (s *NexusStage) ShouldApply(input string) bool
type PhotonStage
- func NewPhotonStage() *PhotonStage
- func (s *PhotonStage) Apply(input string) (string, int)
- func (s *PhotonStage) Name() string
- func (s *PhotonStage) ShouldApply(input string) bool
type QuantumLockStage
- func NewQuantumLockStage() *QuantumLockStage
- func (s *QuantumLockStage) Apply(input string) (string, int)
- func (s *QuantumLockStage) Name() string
- func (s *QuantumLockStage) ShouldApply(input string) bool
type RLEStage
- func NewRLEStage() *RLEStage
- func (s *RLEStage) Apply(input string) (string, int)
- func (s *RLEStage) Name() string
- func (s *RLEStage) ShouldApply(input string) bool
type SearchCrunchStage
- func NewSearchCrunchStage() *SearchCrunchStage
- func (s *SearchCrunchStage) Apply(input string) (string, int)
- func (s *SearchCrunchStage) Name() string
- func (s *SearchCrunchStage) ShouldApply(input string) bool
type SemanticDedupStage
- func NewSemanticDedupStage() *SemanticDedupStage
- func (s *SemanticDedupStage) Apply(input string) (string, int)
- func (s *SemanticDedupStage) Name() string
- func (s *SemanticDedupStage) ShouldApply(input string) bool
type StructuralCollapseStage
- func NewStructuralCollapseStage() *StructuralCollapseStage
- func (s *StructuralCollapseStage) Apply(input string) (string, int)
- func (s *StructuralCollapseStage) Name() string
- func (s *StructuralCollapseStage) ShouldApply(input string) bool
type TokenOptStage
- func NewTokenOptStage() *TokenOptStage
- func (s *TokenOptStage) Apply(input string) (string, int)
- func (s *TokenOptStage) Name() string
- func (s *TokenOptStage) ShouldApply(input string) bool

Constants ¶

This section is empty.

Variables ¶

This section is empty.

Functions ¶

This section is empty.

Types ¶

type AbbrevStage ¶

type AbbrevStage struct{}

func NewAbbrevStage ¶

func NewAbbrevStage() *AbbrevStage

func (*AbbrevStage) Apply ¶

func (s *AbbrevStage) Apply(input string) (string, int)

func (*AbbrevStage) Name ¶

func (s *AbbrevStage) Name() string

func (*AbbrevStage) ShouldApply ¶

func (s *AbbrevStage) ShouldApply(input string) bool

type CortexStage ¶

type CortexStage struct{}

func NewCortexStage ¶

func NewCortexStage() *CortexStage

func (*CortexStage) Apply ¶

func (s *CortexStage) Apply(input string) (string, int)

func (*CortexStage) Name ¶

func (s *CortexStage) Name() string

func (*CortexStage) ShouldApply ¶

func (s *CortexStage) ShouldApply(input string) bool

type DiffCrunchStage ¶

type DiffCrunchStage struct{}

func NewDiffCrunchStage ¶

func NewDiffCrunchStage() *DiffCrunchStage

func (*DiffCrunchStage) Apply ¶

func (s *DiffCrunchStage) Apply(input string) (string, int)

func (*DiffCrunchStage) Name ¶

func (s *DiffCrunchStage) Name() string

func (*DiffCrunchStage) ShouldApply ¶

func (s *DiffCrunchStage) ShouldApply(input string) bool

type FusionEngine ¶

type FusionEngine struct {
	// contains filtered or unexported fields
}

func NewFusionEngine ¶

func NewFusionEngine() *FusionEngine

func (*FusionEngine) Compress ¶

func (e *FusionEngine) Compress(input string) (string, *FusionResult)

type FusionResult ¶

type FusionResult struct {
	OriginalTokens   int `json:"original_tokens"`
	CompressedTokens int `json:"compressed_tokens"`
	Savings          int `json:"savings"`
	StagesRun        int `json:"stages_run"`
	StagesSkipped    int `json:"stages_skipped"`
}

type FusionStage ¶

type FusionStage interface {
	Name() string
	ShouldApply(input string) bool
	Apply(input string) (string, int)
}

type IonizerStage ¶

type IonizerStage struct{}

func NewIonizerStage ¶

func NewIonizerStage() *IonizerStage

func (*IonizerStage) Apply ¶

func (s *IonizerStage) Apply(input string) (string, int)

func (*IonizerStage) Name ¶

func (s *IonizerStage) Name() string

func (*IonizerStage) ShouldApply ¶

func (s *IonizerStage) ShouldApply(input string) bool

type LogCrunchStage ¶

type LogCrunchStage struct{}

func NewLogCrunchStage ¶

func NewLogCrunchStage() *LogCrunchStage

func (*LogCrunchStage) Apply ¶

func (s *LogCrunchStage) Apply(input string) (string, int)

func (*LogCrunchStage) Name ¶

func (s *LogCrunchStage) Name() string

func (*LogCrunchStage) ShouldApply ¶

func (s *LogCrunchStage) ShouldApply(input string) bool

type NeurosyntaxStage ¶

type NeurosyntaxStage struct{}

func NewNeurosyntaxStage ¶

func NewNeurosyntaxStage() *NeurosyntaxStage

func (*NeurosyntaxStage) Apply ¶

func (s *NeurosyntaxStage) Apply(input string) (string, int)

func (*NeurosyntaxStage) Name ¶

func (s *NeurosyntaxStage) Name() string

func (*NeurosyntaxStage) ShouldApply ¶

func (s *NeurosyntaxStage) ShouldApply(input string) bool

type NexusStage ¶

type NexusStage struct{}

func NewNexusStage ¶

func NewNexusStage() *NexusStage

func (*NexusStage) Apply ¶

func (s *NexusStage) Apply(input string) (string, int)

func (*NexusStage) Name ¶

func (s *NexusStage) Name() string

func (*NexusStage) ShouldApply ¶

func (s *NexusStage) ShouldApply(input string) bool

type PhotonStage ¶

type PhotonStage struct{}

func NewPhotonStage ¶

func NewPhotonStage() *PhotonStage

func (*PhotonStage) Apply ¶

func (s *PhotonStage) Apply(input string) (string, int)

func (*PhotonStage) Name ¶

func (s *PhotonStage) Name() string

func (*PhotonStage) ShouldApply ¶

func (s *PhotonStage) ShouldApply(input string) bool

type QuantumLockStage ¶

type QuantumLockStage struct{}

func NewQuantumLockStage ¶

func NewQuantumLockStage() *QuantumLockStage

func (*QuantumLockStage) Apply ¶

func (s *QuantumLockStage) Apply(input string) (string, int)

func (*QuantumLockStage) Name ¶

func (s *QuantumLockStage) Name() string

func (*QuantumLockStage) ShouldApply ¶

func (s *QuantumLockStage) ShouldApply(input string) bool

type RLEStage ¶

type RLEStage struct{}

func NewRLEStage ¶

func NewRLEStage() *RLEStage

func (*RLEStage) Apply ¶

func (s *RLEStage) Apply(input string) (string, int)

func (*RLEStage) Name ¶

func (s *RLEStage) Name() string

func (*RLEStage) ShouldApply ¶

func (s *RLEStage) ShouldApply(input string) bool

type SearchCrunchStage ¶

type SearchCrunchStage struct{}

func NewSearchCrunchStage ¶

func NewSearchCrunchStage() *SearchCrunchStage

func (*SearchCrunchStage) Apply ¶

func (s *SearchCrunchStage) Apply(input string) (string, int)

func (*SearchCrunchStage) Name ¶

func (s *SearchCrunchStage) Name() string

func (*SearchCrunchStage) ShouldApply ¶

func (s *SearchCrunchStage) ShouldApply(input string) bool

type SemanticDedupStage ¶

type SemanticDedupStage struct{}

func NewSemanticDedupStage ¶

func NewSemanticDedupStage() *SemanticDedupStage

func (*SemanticDedupStage) Apply ¶

func (s *SemanticDedupStage) Apply(input string) (string, int)

func (*SemanticDedupStage) Name ¶

func (s *SemanticDedupStage) Name() string

func (*SemanticDedupStage) ShouldApply ¶

func (s *SemanticDedupStage) ShouldApply(input string) bool

type StructuralCollapseStage ¶

type StructuralCollapseStage struct{}

func NewStructuralCollapseStage ¶

func NewStructuralCollapseStage() *StructuralCollapseStage

func (*StructuralCollapseStage) Apply ¶

func (s *StructuralCollapseStage) Apply(input string) (string, int)

func (*StructuralCollapseStage) Name ¶

func (s *StructuralCollapseStage) Name() string

func (*StructuralCollapseStage) ShouldApply ¶

func (s *StructuralCollapseStage) ShouldApply(input string) bool

type TokenOptStage ¶

type TokenOptStage struct{}

func NewTokenOptStage ¶

func NewTokenOptStage() *TokenOptStage

func (*TokenOptStage) Apply ¶

func (s *TokenOptStage) Apply(input string) (string, int)

func (*TokenOptStage) Name ¶

func (s *TokenOptStage) Name() string

func (*TokenOptStage) ShouldApply ¶

func (s *TokenOptStage) ShouldApply(input string) bool

Source Files ¶

View all Source files

pipeline.go

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL