Documentation
¶
Index ¶
Constants ¶
This section is empty.
Variables ¶
var Profiles = [...]*block.CompressionProfile{ { Name: "Snappy", DataBlocks: compression.Snappy, ValueBlocks: compression.Snappy, OtherBlocks: compression.Snappy, MinReductionPercent: 0, }, { Name: "MinLZ1", DataBlocks: compression.MinLZFastest, ValueBlocks: compression.MinLZFastest, OtherBlocks: compression.MinLZFastest, MinReductionPercent: 0, }, { Name: "MinLZ2", DataBlocks: compression.MinLZBalanced, ValueBlocks: compression.MinLZBalanced, OtherBlocks: compression.MinLZBalanced, MinReductionPercent: 0, }, { Name: "Zstd1", DataBlocks: compression.ZstdLevel1, ValueBlocks: compression.ZstdLevel1, OtherBlocks: compression.ZstdLevel1, MinReductionPercent: 0, }, { Name: "Auto1", DataBlocks: compression.ZstdLevel1, ValueBlocks: compression.ZstdLevel1, OtherBlocks: compression.MinLZFastest, AdaptiveReductionCutoffPercent: 30, MinReductionPercent: 0, }, { Name: "Zstd3", DataBlocks: compression.ZstdLevel3, ValueBlocks: compression.ZstdLevel3, OtherBlocks: compression.ZstdLevel3, MinReductionPercent: 0, }, { Name: "Auto3", DataBlocks: compression.ZstdLevel3, ValueBlocks: compression.ZstdLevel3, OtherBlocks: compression.MinLZFastest, AdaptiveReductionCutoffPercent: 30, MinReductionPercent: 0, }, }
Functions ¶
This section is empty.
Types ¶
type BlockAnalyzer ¶
type BlockAnalyzer struct {
// contains filtered or unexported fields
}
BlockAnalyzer is used to evaluate the performance and compressibility of different compression algorithms on sstable and blob file blocks.
func NewBlockAnalyzer ¶
func NewBlockAnalyzer() *BlockAnalyzer
func (*BlockAnalyzer) Block ¶
func (a *BlockAnalyzer) Block(kind block.Kind, block []byte)
Block analyzes a block by measuring its compressibility and the performance of various compression algorithms on it.
func (*BlockAnalyzer) Buckets ¶
func (a *BlockAnalyzer) Buckets() *Buckets
func (*BlockAnalyzer) Close ¶
func (a *BlockAnalyzer) Close()
func (*BlockAnalyzer) ResetCompressors ¶
func (a *BlockAnalyzer) ResetCompressors()
ResetCompressors the compressors. This is useful for adaptive compressors which keep some state; we want to clear that state for each sstable.
type BlockSize ¶
type BlockSize uint8
BlockSize identifies a range of block sizes.
func MakeBlockSize ¶
type Bucket ¶
type Bucket struct {
UncompressedSize Welford
Experiments [numProfiles]PerProfile
}
Bucket aggregates results for blocks of the same kind, size range, and compressibility.
type Compressibility ¶
type Compressibility uint8
Compressibility indicates how compressible a block is. It is determined by applying MinLZFastest and noting the reduction.
const ( Incompressible Compressibility = iota MarginallyCompressible ModeratelyCompressible HighlyCompressible )
func MakeCompressibility ¶
func MakeCompressibility(uncompressedSize, compressedSize int) Compressibility
func (Compressibility) String ¶
func (c Compressibility) String() string
type FileAnalyzer ¶
type FileAnalyzer struct {
// contains filtered or unexported fields
}
FileAnalyzer is used to analyze blocks in sstable files.
TODO(radu): add support for blob files.
func NewFileAnalyzer ¶
func NewFileAnalyzer( readLimiter *tokenbucket.TokenBucket, sstReadOpts sstable.ReaderOptions, ) *FileAnalyzer
func (*FileAnalyzer) Buckets ¶
func (fa *FileAnalyzer) Buckets() *Buckets
func (*FileAnalyzer) Close ¶
func (fa *FileAnalyzer) Close()
func (*FileAnalyzer) SSTable ¶
func (fa *FileAnalyzer) SSTable(ctx context.Context, readable objstorage.Readable) error
SSTable analyzes the blocks in an sstable file and closes the readable (even in error cases).
type PerProfile ¶
type PerProfile struct {
CompressionRatio WeightedWelford
// CPU times are in nanoseconds per uncompressed byte.
CompressionTime WeightedWelford
DecompressionTime WeightedWelford
}
PerProfile holds statistics from experiments on blocks in a bucket with a specific compression.Setting.
type WeightedWelford ¶
type WeightedWelford struct {
// contains filtered or unexported fields
}
WeightedWelford maintains running statistics for mean and variance using an extension of Welford's algorithm which allows for weighted samples; see https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Weighted_incremental_algorithm
func (*WeightedWelford) Add ¶
func (ww *WeightedWelford) Add(x float64, frequency uint64)
Add incorporates a new data point x with the given frequency into the running statistics.
func (*WeightedWelford) Mean ¶
func (ww *WeightedWelford) Mean() float64
Mean returns the current running mean. If no values have been added, returns 0.
func (*WeightedWelford) SampleStandardDeviation ¶
func (ww *WeightedWelford) SampleStandardDeviation() float64
SampleStandardDeviation returns the sample standard deviation.
func (*WeightedWelford) SampleVariance ¶
func (ww *WeightedWelford) SampleVariance() float64
SampleVariance returns the sample variance (M2/(n-1)). Returns 0 if the sum of added frequencies is less than 2.
type Welford ¶
type Welford struct {
// contains filtered or unexported fields
}
Welford maintains running statistics for mean and variance using Welford's algorithm.
func (*Welford) Mean ¶
Mean returns the current running mean. If no values have been added, returns 0.
func (*Welford) SampleStandardDeviation ¶
SampleStandardDeviation returns the sample standard deviation.
func (*Welford) SampleVariance ¶
SampleVariance returns the sample variance (M2/(n-1)). Returns 0 if fewer than 2 values.
Source Files