compression

package
v0.2.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 24, 2026 License: MIT Imports: 11 Imported by: 0

Documentation

Index

Constants

View Source
const (
	DefaultMinBytes   = 256
	DefaultMinSavings = 16
	HeaderSize        = 4
)
View Source
const (
	DefaultTrainBytes          = 1 << 20
	DefaultTrainDictBytes      = 32 << 10
	DefaultTrainMinRecords     = 64
	DefaultTrainMaxRecordBytes = 64 << 10
	DefaultTrainQueue          = 128
	DefaultTrainDedupWindow    = 16

	// Adaptive gating constants for dict+K refresh.
	MinProfileBytes       = 64 << 20 // 64 MiB
	MinProfileRecords     = 250_000  // records
	MinProfileInterval    = 10 * time.Minute
	ProfileDriftThreshold = 0.07 // 7%
	ProfileImproveThresh  = 0.02 // 2% better to accept

	// More aggressive gating when the stream is degraded (e.g., dict compression
	// is paused due to poor observed savings).
	MinProfileBytesDegraded    = 8 << 20 // 8 MiB
	MinProfileRecordsDegraded  = 50_000  // records
	MinProfileIntervalDegraded = 30 * time.Second
)
View Source
const DefaultMetricsWindowBytes = 4 << 20

Variables

View Source
var ErrCorrupt = errors.New("compression: invalid compressed value")

Functions

This section is empty.

Types

type ActiveProfile

type ActiveProfile struct {
	DictHash         uint64
	DictBytes        int
	Dict             []byte
	HistoryBytes     int
	K                int
	PayloadRatio     float64
	TotalRatio       float64
	DecodeNsEstimate int64
	EncodeNsEstimate int64
	AvgSampleBytes   int
	Samples          int
	Timestamp        time.Time
}

func ChooseKForDict

func ChooseKForDict(dict []byte, samples [][]byte) (profile *ActiveProfile)

func ChooseKForDictOptions

func ChooseKForDictOptions(dict []byte, samples [][]byte, opts ChooseKOptions) (profile *ActiveProfile)

type ChooseKOptions

type ChooseKOptions struct {
	CandidateK        []int
	IoNsPerStoredByte float64
	// Deterministic timing overrides (ns per raw byte).
	EncodeNsPerRawByte float64
	DecodeNsPerRawByte float64
}

type Config

type Config struct {
	Kind       Kind
	MinBytes   int
	MinSavings int
	Level      zstd.EncoderLevel
	ZstdEncs   *sync.Pool
	ZstdDecs   *sync.Pool
	BufferPool *sync.Pool
}

func NormalizeOptions

func NormalizeOptions(opts Options) (Config, error)

func (*Config) CompressRecord

func (c *Config) CompressRecord(key, value []byte) ([]byte, bool, error)

func (*Config) CompressValue

func (c *Config) CompressValue(value []byte) ([]byte, bool, error)

func (*Config) CompressValuePooled

func (c *Config) CompressValuePooled(value []byte) ([]byte, bool, func(), error)

func (*Config) DecompressRecord

func (c *Config) DecompressRecord(encoded []byte) ([]byte, []byte, error)

func (*Config) DecompressValue

func (c *Config) DecompressValue(encoded []byte) ([]byte, error)

type DictUseFlag

type DictUseFlag uint8
const (
	DictUseGlobal DictUseFlag = iota
	DictUseLocal
	DictUseRef
)

type Kind

type Kind uint8
const (
	KindNone Kind = iota
	KindZSTD
)

type Metrics

type Metrics struct {
	Enabled          bool
	LogWindows       bool
	WindowBytes      uint64
	SlabID           uint32
	PauseThreshold   float64
	PauseBytes       uint64
	MinRecords       uint64
	WindowRaw        uint64
	WindowStored     uint64
	WindowRecords    uint64
	WindowCompressed uint64
	WindowFull       uint64
	TotalRaw         uint64
	TotalStored      uint64
	TotalRecords     uint64
	TotalCompressed  uint64
	TotalFull        uint64
	TotalDegraded    uint64
	// contains filtered or unexported fields
}

func NewMetrics

func NewMetrics(opts MetricsOptions) Metrics

func (*Metrics) Add

func (m *Metrics) Add(slabID uint32, rawBytes, storedBytes, records, compressedCount, fullCount int) uint64

func (*Metrics) Finish

func (m *Metrics) Finish(reason string)

func (*Metrics) LogWindow

func (m *Metrics) LogWindow() uint64

func (*Metrics) Reset

func (m *Metrics) Reset(nextSlabID uint32)

func (*Metrics) SetSlab

func (m *Metrics) SetSlab(id uint32)

type MetricsOptions

type MetricsOptions struct {
	MetricsEnabled bool
	AdaptiveRatio  float64
	WindowBytes    int
	MinRecords     int
	PauseBytes     int
}

type Options

type Options struct {
	Kind            Kind
	MinBytes        int
	MinSavingsBytes int
	Level           int
}

type TrainConfig

type TrainConfig struct {
	TrainBytes     int
	DictBytes      int
	MinRecords     int
	MaxRecordBytes int
	SampleStride   int
	DedupWindow    int
	Level          int
	// Deterministic cost modeling (tests/benches only). When >0, overrides
	// encode/decode timing estimates in training evaluation.
	EncodeNsPerRawByte float64
	DecodeNsPerRawByte float64
}

type Trainer

type Trainer struct {
	// contains filtered or unexported fields
}

func NewTrainer

func NewTrainer(opts TrainConfig, cfg Config, readOnly bool, metricsEnabled bool) *Trainer

func (*Trainer) AcceptProfile

func (t *Trainer) AcceptProfile(profile *ActiveProfile)

func (*Trainer) ActiveProfile

func (t *Trainer) ActiveProfile() (*ActiveProfile, bool)

func (*Trainer) Close

func (t *Trainer) Close()

func (*Trainer) Collect

func (t *Trainer) Collect(value []byte)

func (*Trainer) Config

func (t *Trainer) Config() TrainConfig

func (*Trainer) ForceCollecting

func (t *Trainer) ForceCollecting()

func (*Trainer) SetAutotuneCandidates

func (t *Trainer) SetAutotuneCandidates(candidateK, candidateHistoryBytes []int)

func (*Trainer) SetAutotuneIOCost

func (t *Trainer) SetAutotuneIOCost(ioNsPerStoredByte float64)

func (*Trainer) ShouldCollect

func (t *Trainer) ShouldCollect() bool

func (*Trainer) SignalDegraded

func (t *Trainer) SignalDegraded(slabID uint32)

func (*Trainer) Stats

func (t *Trainer) Stats() TrainerStats

type TrainerStats

type TrainerStats struct {
	Enabled              bool
	Collecting           bool
	Training             bool
	QueueLen             int
	QueueCap             int
	Enqueued             uint64
	Dropped              uint64
	MaxQueueLen          uint64
	TrainCount           uint64
	LastTrainRatio       float64
	LastTrainSamples     uint64
	LastTrainDict        uint64
	LastTrainDictHash    uint64
	LastTrainDedupMode   string
	LastTrainDedupFlag   string
	LastTrainDedupRef    uint64
	DictDedupLookups     uint64
	DictDedupHits        uint64
	DictDedupGlobal      uint64
	DictDedupRef         uint64
	DictDedupCache       uint64
	DictDedupBytes       uint64
	DictDedupBytesGlobal uint64
	DictDedupBytesRef    uint64
	DictDedupBytesCache  uint64
	CollectCount         uint64
	CollectNanos         uint64
	CollectMaxNanos      uint64
	ProfileK             int
	ProfileTotalRatio    float64
	ProfilePayloadRatio  float64
	ProfileTimestamp     time.Time
	ProfileAttempts      uint64
	ProfileAccepts       uint64
	ProfileRejects       uint64
	ProfileRejectReason  string
	RollingRatioBaseline float64
	RollingRatioCurrent  float64
	LastAcceptBytes      uint64
	LastAcceptRecords    uint64
	LastAcceptTimestamp  time.Time
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL