Documentation
¶
Index ¶
- Constants
- Variables
- type ActiveProfile
- type ChooseKOptions
- type Config
- func (c *Config) CompressRecord(key, value []byte) ([]byte, bool, error)
- func (c *Config) CompressValue(value []byte) ([]byte, bool, error)
- func (c *Config) CompressValuePooled(value []byte) ([]byte, bool, func(), error)
- func (c *Config) DecompressRecord(encoded []byte) ([]byte, []byte, error)
- func (c *Config) DecompressValue(encoded []byte) ([]byte, error)
- type DictUseFlag
- type Kind
- type Metrics
- type MetricsOptions
- type Options
- type TrainConfig
- type Trainer
- func (t *Trainer) AcceptProfile(profile *ActiveProfile)
- func (t *Trainer) ActiveProfile() (*ActiveProfile, bool)
- func (t *Trainer) Close()
- func (t *Trainer) Collect(value []byte)
- func (t *Trainer) Config() TrainConfig
- func (t *Trainer) ForceCollecting()
- func (t *Trainer) SetAutotuneCandidates(candidateK, candidateHistoryBytes []int)
- func (t *Trainer) SetAutotuneIOCost(ioNsPerStoredByte float64)
- func (t *Trainer) ShouldCollect() bool
- func (t *Trainer) SignalDegraded(slabID uint32)
- func (t *Trainer) Stats() TrainerStats
- type TrainerStats
Constants ¶
View Source
const ( DefaultMinBytes = 256 DefaultMinSavings = 16 HeaderSize = 4 )
View Source
const ( DefaultTrainBytes = 1 << 20 DefaultTrainDictBytes = 32 << 10 DefaultTrainMinRecords = 64 DefaultTrainMaxRecordBytes = 64 << 10 DefaultTrainQueue = 128 DefaultTrainDedupWindow = 16 // Adaptive gating constants for dict+K refresh. MinProfileBytes = 64 << 20 // 64 MiB MinProfileRecords = 250_000 // records MinProfileInterval = 10 * time.Minute ProfileDriftThreshold = 0.07 // 7% ProfileImproveThresh = 0.02 // 2% better to accept // More aggressive gating when the stream is degraded (e.g., dict compression // is paused due to poor observed savings). MinProfileBytesDegraded = 8 << 20 // 8 MiB MinProfileRecordsDegraded = 50_000 // records MinProfileIntervalDegraded = 30 * time.Second )
View Source
const DefaultMetricsWindowBytes = 4 << 20
Variables ¶
View Source
var ErrCorrupt = errors.New("compression: invalid compressed value")
Functions ¶
This section is empty.
Types ¶
type ActiveProfile ¶
type ActiveProfile struct {
DictHash uint64
DictBytes int
Dict []byte
HistoryBytes int
K int
PayloadRatio float64
TotalRatio float64
DecodeNsEstimate int64
EncodeNsEstimate int64
AvgSampleBytes int
Samples int
Timestamp time.Time
}
func ChooseKForDict ¶
func ChooseKForDict(dict []byte, samples [][]byte) (profile *ActiveProfile)
func ChooseKForDictOptions ¶
func ChooseKForDictOptions(dict []byte, samples [][]byte, opts ChooseKOptions) (profile *ActiveProfile)
type ChooseKOptions ¶
type Config ¶
type Config struct {
Kind Kind
MinBytes int
MinSavings int
Level zstd.EncoderLevel
ZstdEncs *sync.Pool
ZstdDecs *sync.Pool
BufferPool *sync.Pool
}
func NormalizeOptions ¶
func (*Config) CompressRecord ¶
func (*Config) CompressValuePooled ¶
func (*Config) DecompressRecord ¶
type DictUseFlag ¶
type DictUseFlag uint8
const ( DictUseGlobal DictUseFlag = iota DictUseLocal DictUseRef )
type Metrics ¶
type Metrics struct {
Enabled bool
LogWindows bool
WindowBytes uint64
SlabID uint32
PauseThreshold float64
PauseBytes uint64
MinRecords uint64
WindowRaw uint64
WindowStored uint64
WindowRecords uint64
WindowCompressed uint64
WindowFull uint64
TotalRaw uint64
TotalStored uint64
TotalRecords uint64
TotalCompressed uint64
TotalFull uint64
TotalDegraded uint64
// contains filtered or unexported fields
}
func NewMetrics ¶
func NewMetrics(opts MetricsOptions) Metrics
type MetricsOptions ¶
type TrainConfig ¶
type TrainConfig struct {
TrainBytes int
DictBytes int
MinRecords int
MaxRecordBytes int
SampleStride int
DedupWindow int
Level int
// Deterministic cost modeling (tests/benches only). When >0, overrides
// encode/decode timing estimates in training evaluation.
EncodeNsPerRawByte float64
DecodeNsPerRawByte float64
}
type Trainer ¶
type Trainer struct {
// contains filtered or unexported fields
}
func NewTrainer ¶
func NewTrainer(opts TrainConfig, cfg Config, readOnly bool, metricsEnabled bool) *Trainer
func (*Trainer) AcceptProfile ¶
func (t *Trainer) AcceptProfile(profile *ActiveProfile)
func (*Trainer) ActiveProfile ¶
func (t *Trainer) ActiveProfile() (*ActiveProfile, bool)
func (*Trainer) Config ¶
func (t *Trainer) Config() TrainConfig
func (*Trainer) ForceCollecting ¶
func (t *Trainer) ForceCollecting()
func (*Trainer) SetAutotuneCandidates ¶
func (*Trainer) SetAutotuneIOCost ¶
func (*Trainer) ShouldCollect ¶
func (*Trainer) SignalDegraded ¶
func (*Trainer) Stats ¶
func (t *Trainer) Stats() TrainerStats
type TrainerStats ¶
type TrainerStats struct {
Enabled bool
Collecting bool
Training bool
QueueLen int
QueueCap int
Enqueued uint64
Dropped uint64
MaxQueueLen uint64
TrainCount uint64
LastTrainRatio float64
LastTrainSamples uint64
LastTrainDict uint64
LastTrainDictHash uint64
LastTrainDedupMode string
LastTrainDedupFlag string
LastTrainDedupRef uint64
DictDedupLookups uint64
DictDedupHits uint64
DictDedupGlobal uint64
DictDedupRef uint64
DictDedupCache uint64
DictDedupBytes uint64
DictDedupBytesGlobal uint64
DictDedupBytesRef uint64
DictDedupBytesCache uint64
CollectCount uint64
CollectNanos uint64
CollectMaxNanos uint64
ProfileK int
ProfileTotalRatio float64
ProfilePayloadRatio float64
ProfileTimestamp time.Time
ProfileAttempts uint64
ProfileAccepts uint64
ProfileRejects uint64
ProfileRejectReason string
RollingRatioBaseline float64
RollingRatioCurrent float64
LastAcceptBytes uint64
LastAcceptRecords uint64
LastAcceptTimestamp time.Time
}
Click to show internal directories.
Click to hide internal directories.