Documentation
¶
Overview ¶
Package regression provides CI/CD regression detection gates.
Architecture ¶
The regression package implements automated regression detection by comparing current performance against established baselines:
┌─────────────────────────────────────────────────────────────────────────┐ │ REGRESSION FRAMEWORK │ ├─────────────────────────────────────────────────────────────────────────┤ │ │ │ Benchmark Results ──► Detector ──► Gate ──► Decision │ │ │ │ │ │ │ ├──► PASS: Deploy │ │ │ ├──► WARN: Review │ │ ▼ └──► FAIL: Block │ │ ┌─────────┐ │ │ │Baseline │ │ │ │ Store │ │ │ └─────────┘ │ │ │ │ Thresholds: │ │ • Latency: P50 +5%, P99 +10% │ │ • Throughput: -5% │ │ • Memory: +10% │ │ • Errors: +1% │ │ │ └─────────────────────────────────────────────────────────────────────────┘
Components ¶
- Baseline: Stores historical performance data
- Detector: Compares current vs baseline with statistical tests
- Gate: Makes pass/warn/fail decisions for CI/CD
- Alert: Notifies stakeholders of regressions
Usage ¶
Basic regression gate:
baseline := regression.NewFileBaseline("./baselines")
gate := regression.NewGate(baseline,
regression.WithLatencyThreshold(0.10), // 10% increase allowed
regression.WithThroughputThreshold(0.05), // 5% decrease allowed
)
// Check benchmark results
decision, err := gate.Check(ctx, benchResults)
if !decision.Pass {
log.Fatalf("Regression detected: %s", decision.Report)
}
CI/CD Integration ¶
The gate is designed for CI/CD pipelines:
# GitHub Actions example - name: Run benchmarks run: go test -bench=. -benchmem ./... - name: Check regression run: regression-gate check --baseline ./baselines
Thread Safety ¶
All types in this package are safe for concurrent use unless otherwise noted.
Index ¶
- Variables
- type Baseline
- type BaselineBuilder
- func (b *BaselineBuilder) Build() *BaselineData
- func (b *BaselineBuilder) WithErrorRate(rate float64) *BaselineBuilder
- func (b *BaselineBuilder) WithLatency(p50, p95, p99, mean, stdDev time.Duration) *BaselineBuilder
- func (b *BaselineBuilder) WithMemory(allocBytesPerOp, allocsPerOp, heapInUse uint64) *BaselineBuilder
- func (b *BaselineBuilder) WithMetadata(key, value string) *BaselineBuilder
- func (b *BaselineBuilder) WithSampleCount(count int) *BaselineBuilder
- func (b *BaselineBuilder) WithThroughput(opsPerSecond, bytesPerSecond float64) *BaselineBuilder
- type BaselineData
- type CurrentMetrics
- type DetectionResult
- type Detector
- type DetectorConfig
- type ErrorBaseline
- type FileBaselineStore
- func (f *FileBaselineStore) Delete(_ context.Context, component string) error
- func (f *FileBaselineStore) Get(_ context.Context, component string) (*BaselineData, error)
- func (f *FileBaselineStore) List(_ context.Context) ([]string, error)
- func (f *FileBaselineStore) Set(_ context.Context, component string, data *BaselineData) error
- type Gate
- func (g *Gate) Check(ctx context.Context, component string, current *CurrentMetrics) (*GateDecision, error)
- func (g *Gate) CheckAll(ctx context.Context, components map[string]*CurrentMetrics) (map[string]*GateDecision, error)
- func (g *Gate) HealthCheck(ctx context.Context) error
- func (g *Gate) Metrics() []eval.MetricDefinition
- func (g *Gate) Name() string
- func (g *Gate) Properties() []eval.Property
- type GateConfig
- type GateDecision
- type GateOption
- func WithAllowedRegressions(count int) GateOption
- func WithErrorThreshold(threshold float64) GateOption
- func WithFailOnWarnings(fail bool) GateOption
- func WithGateLogger(logger *slog.Logger) GateOption
- func WithLatencyThreshold(threshold float64) GateOption
- func WithMemoryThreshold(threshold float64) GateOption
- func WithRequireBaseline(required bool) GateOption
- func WithThroughputThreshold(threshold float64) GateOption
- func WithUpdateBaseline(enabled bool) GateOption
- type LatencyBaseline
- type MemoryBaseline
- type MemoryBaselineStore
- func (m *MemoryBaselineStore) Delete(_ context.Context, component string) error
- func (m *MemoryBaselineStore) Get(_ context.Context, component string) (*BaselineData, error)
- func (m *MemoryBaselineStore) List(_ context.Context) ([]string, error)
- func (m *MemoryBaselineStore) Set(_ context.Context, component string, data *BaselineData) error
- type Regression
- type RegressionType
- type Severity
- type StatisticalDetector
- type ThroughputBaseline
Constants ¶
This section is empty.
Variables ¶
var ( // ErrBaselineNotFound indicates no baseline exists for the component. ErrBaselineNotFound = errors.New("baseline not found") // ErrInvalidBaseline indicates the baseline data is corrupted. ErrInvalidBaseline = errors.New("invalid baseline data") )
var ( // ErrGateFailed indicates the regression gate did not pass. ErrGateFailed = errors.New("regression gate failed") )
Functions ¶
This section is empty.
Types ¶
type Baseline ¶
type Baseline interface {
// Get retrieves the baseline for a component.
// Returns ErrBaselineNotFound if no baseline exists.
Get(ctx context.Context, component string) (*BaselineData, error)
// Set stores a new baseline for a component.
Set(ctx context.Context, component string, data *BaselineData) error
// List returns all available baseline names.
List(ctx context.Context) ([]string, error)
// Delete removes a baseline.
Delete(ctx context.Context, component string) error
}
Baseline stores and retrieves performance baselines.
Thread Safety: Implementations must be safe for concurrent use.
type BaselineBuilder ¶
type BaselineBuilder struct {
// contains filtered or unexported fields
}
BaselineBuilder helps construct BaselineData from benchmark results.
func NewBaselineBuilder ¶
func NewBaselineBuilder(component, version string) *BaselineBuilder
NewBaselineBuilder creates a new baseline builder.
Inputs:
- component: Component name.
- version: Version string.
Outputs:
- *BaselineBuilder: The new builder. Never nil.
func (*BaselineBuilder) Build ¶
func (b *BaselineBuilder) Build() *BaselineData
Build returns the constructed baseline.
func (*BaselineBuilder) WithErrorRate ¶
func (b *BaselineBuilder) WithErrorRate(rate float64) *BaselineBuilder
WithErrorRate sets error metrics.
func (*BaselineBuilder) WithLatency ¶
func (b *BaselineBuilder) WithLatency(p50, p95, p99, mean, stdDev time.Duration) *BaselineBuilder
WithLatency sets latency metrics.
func (*BaselineBuilder) WithMemory ¶
func (b *BaselineBuilder) WithMemory(allocBytesPerOp, allocsPerOp, heapInUse uint64) *BaselineBuilder
WithMemory sets memory metrics.
func (*BaselineBuilder) WithMetadata ¶
func (b *BaselineBuilder) WithMetadata(key, value string) *BaselineBuilder
WithMetadata adds metadata.
func (*BaselineBuilder) WithSampleCount ¶
func (b *BaselineBuilder) WithSampleCount(count int) *BaselineBuilder
WithSampleCount sets the sample count.
func (*BaselineBuilder) WithThroughput ¶
func (b *BaselineBuilder) WithThroughput(opsPerSecond, bytesPerSecond float64) *BaselineBuilder
WithThroughput sets throughput metrics.
type BaselineData ¶
type BaselineData struct {
// Component is the name of the component.
Component string `json:"component"`
// Version identifies this baseline version.
Version string `json:"version"`
// CreatedAt is when the baseline was created.
CreatedAt time.Time `json:"created_at"`
// UpdatedAt is when the baseline was last updated.
UpdatedAt time.Time `json:"updated_at"`
// Latency holds latency metrics.
Latency LatencyBaseline `json:"latency"`
// Throughput holds throughput metrics.
Throughput ThroughputBaseline `json:"throughput"`
// Memory holds memory metrics.
Memory MemoryBaseline `json:"memory"`
// Error holds error rate metrics.
Error ErrorBaseline `json:"error"`
// SampleCount is the number of samples in this baseline.
SampleCount int `json:"sample_count"`
// Metadata holds arbitrary additional data.
Metadata map[string]string `json:"metadata,omitempty"`
}
BaselineData holds the performance metrics for a baseline.
type CurrentMetrics ¶
type CurrentMetrics struct {
// Latency holds current latency metrics.
Latency LatencyBaseline
// Throughput holds current throughput metrics.
Throughput ThroughputBaseline
// Memory holds current memory metrics.
Memory MemoryBaseline
// ErrorRate is the current error rate.
ErrorRate float64
// SampleCount is the number of samples.
SampleCount int
}
CurrentMetrics holds current performance measurements.
type DetectionResult ¶
type DetectionResult struct {
// Component is the component analyzed.
Component string
// Baseline is the baseline data used.
Baseline *BaselineData
// Regressions contains all detected regressions.
Regressions []Regression
// Warnings contains non-blocking issues.
Warnings []Regression
// Pass is true if no blocking regressions were found.
Pass bool
// MaxSeverity is the highest severity found.
MaxSeverity Severity
// AnalyzedAt is when detection was performed.
AnalyzedAt time.Time
}
DetectionResult holds the results of regression detection.
func (*DetectionResult) HasRegressions ¶
func (r *DetectionResult) HasRegressions() bool
HasRegressions returns true if any regressions were detected.
func (*DetectionResult) HasWarnings ¶
func (r *DetectionResult) HasWarnings() bool
HasWarnings returns true if any warnings were detected.
type Detector ¶
type Detector struct {
// contains filtered or unexported fields
}
Detector compares current metrics against baselines.
Thread Safety: Safe for concurrent use (stateless).
func NewDetector ¶
func NewDetector(config *DetectorConfig) *Detector
NewDetector creates a new regression detector.
Inputs:
- config: Detection configuration. If nil, uses defaults.
Outputs:
- *Detector: The new detector. Never nil.
func (*Detector) Detect ¶
func (d *Detector) Detect(baseline *BaselineData, current *CurrentMetrics) *DetectionResult
Detect compares current metrics against a baseline.
Inputs:
- baseline: The baseline to compare against.
- current: Current performance metrics.
Outputs:
- *DetectionResult: Detection results with any regressions found.
Thread Safety: Safe for concurrent use.
type DetectorConfig ¶
type DetectorConfig struct {
// LatencyP50Threshold is the allowed P50 increase ratio (e.g., 0.05 = 5%).
LatencyP50Threshold float64
// LatencyP95Threshold is the allowed P95 increase ratio.
LatencyP95Threshold float64
// LatencyP99Threshold is the allowed P99 increase ratio.
LatencyP99Threshold float64
// ThroughputThreshold is the allowed throughput decrease ratio.
ThroughputThreshold float64
// MemoryThreshold is the allowed memory increase ratio.
MemoryThreshold float64
// ErrorRateThreshold is the allowed error rate increase (absolute).
ErrorRateThreshold float64
// WarnThresholdRatio is the ratio of threshold at which to warn.
// E.g., 0.8 means warn at 80% of threshold.
WarnThresholdRatio float64
// MinSamples is the minimum samples required for valid comparison.
MinSamples int
}
DetectorConfig configures regression detection.
func DefaultDetectorConfig ¶
func DefaultDetectorConfig() *DetectorConfig
DefaultDetectorConfig returns sensible defaults.
type ErrorBaseline ¶
type ErrorBaseline struct {
// Rate is the error rate (0 to 1).
Rate float64 `json:"rate"`
// Count is the typical error count.
Count int64 `json:"count,omitempty"`
}
ErrorBaseline holds error rate metrics.
type FileBaselineStore ¶
type FileBaselineStore struct {
// contains filtered or unexported fields
}
FileBaselineStore stores baselines in JSON files.
Description:
FileBaselineStore persists baselines to disk as JSON files.
Each component gets its own file: {dir}/{component}.json
Thread Safety: Safe for concurrent use.
func NewFileBaseline ¶
func NewFileBaseline(dir string) (*FileBaselineStore, error)
NewFileBaseline creates a file-backed baseline store.
Inputs:
- dir: Directory to store baseline files. Created if not exists.
Outputs:
- *FileBaselineStore: The new store. Never nil.
- error: Non-nil if directory cannot be created.
func (*FileBaselineStore) Delete ¶
func (f *FileBaselineStore) Delete(_ context.Context, component string) error
Delete implements Baseline.
func (*FileBaselineStore) Get ¶
func (f *FileBaselineStore) Get(_ context.Context, component string) (*BaselineData, error)
Get implements Baseline.
func (*FileBaselineStore) List ¶
func (f *FileBaselineStore) List(_ context.Context) ([]string, error)
List implements Baseline.
func (*FileBaselineStore) Set ¶
func (f *FileBaselineStore) Set(_ context.Context, component string, data *BaselineData) error
Set implements Baseline.
type Gate ¶
type Gate struct {
// contains filtered or unexported fields
}
Gate checks for performance regressions against baselines.
Description:
Gate compares current benchmark results against stored baselines and determines whether to pass or fail the CI/CD pipeline.
Thread Safety: Safe for concurrent use.
func NewGate ¶
func NewGate(baseline Baseline, opts ...GateOption) *Gate
NewGate creates a new regression gate.
Inputs:
- baseline: Baseline store. Must not be nil.
- opts: Configuration options.
Outputs:
- *Gate: The new gate. Never nil.
func (*Gate) Check ¶
func (g *Gate) Check(ctx context.Context, component string, current *CurrentMetrics) (*GateDecision, error)
Check evaluates current metrics against baseline.
Description:
Check retrieves the baseline for the component, runs regression detection, and returns a decision about whether to pass or fail.
Inputs:
- ctx: Context for cancellation. Must not be nil.
- component: Component name.
- current: Current performance metrics.
Outputs:
- *GateDecision: The gate decision. Never nil.
- error: Non-nil only if check could not be performed.
Thread Safety: Safe for concurrent use.
func (*Gate) CheckAll ¶
func (g *Gate) CheckAll(ctx context.Context, components map[string]*CurrentMetrics) (map[string]*GateDecision, error)
CheckAll checks multiple components.
Inputs:
- ctx: Context for cancellation.
- components: Map of component name to current metrics.
Outputs:
- map[string]*GateDecision: Decisions for each component.
- error: Non-nil if any check failed to run.
Thread Safety: Safe for concurrent use.
func (*Gate) HealthCheck ¶
HealthCheck implements eval.Evaluable.
func (*Gate) Metrics ¶
func (g *Gate) Metrics() []eval.MetricDefinition
Metrics implements eval.Evaluable.
func (*Gate) Properties ¶
Properties implements eval.Evaluable.
type GateConfig ¶
type GateConfig struct {
// Detector configuration.
DetectorConfig *DetectorConfig
// UpdateBaselineOnPass updates baseline when check passes.
// Default: false
UpdateBaselineOnPass bool
// RequireBaseline fails if no baseline exists.
// Default: false (missing baseline = pass)
RequireBaseline bool
// AllowedRegressions is the maximum regressions before failing.
// Default: 0 (any regression fails)
AllowedRegressions int
// FailOnWarnings fails the gate on warnings.
// Default: false
FailOnWarnings bool
// Logger for output.
Logger *slog.Logger
}
GateConfig configures the regression gate.
func DefaultGateConfig ¶
func DefaultGateConfig() *GateConfig
DefaultGateConfig returns sensible defaults.
type GateDecision ¶
type GateDecision struct {
// Pass is true if the gate allows deployment.
Pass bool
// Component is the checked component.
Component string
// Regressions contains detected regressions.
Regressions []Regression
// Warnings contains non-blocking warnings.
Warnings []Regression
// BaselineUpdated is true if baseline was updated.
BaselineUpdated bool
// Report is a human-readable summary.
Report string
// Duration is the check duration.
Duration time.Duration
// Timestamp is when the check was performed.
Timestamp time.Time
}
GateDecision contains the gate check result.
type GateOption ¶
type GateOption func(*GateConfig)
GateOption configures the gate.
func WithAllowedRegressions ¶
func WithAllowedRegressions(count int) GateOption
WithAllowedRegressions sets allowed regression count.
func WithErrorThreshold ¶
func WithErrorThreshold(threshold float64) GateOption
WithErrorThreshold sets the error rate threshold.
func WithFailOnWarnings ¶
func WithFailOnWarnings(fail bool) GateOption
WithFailOnWarnings enables failing on warnings.
func WithGateLogger ¶
func WithGateLogger(logger *slog.Logger) GateOption
WithGateLogger sets the logger.
func WithLatencyThreshold ¶
func WithLatencyThreshold(threshold float64) GateOption
WithLatencyThreshold sets all latency thresholds.
func WithMemoryThreshold ¶
func WithMemoryThreshold(threshold float64) GateOption
WithMemoryThreshold sets the memory threshold.
func WithRequireBaseline ¶
func WithRequireBaseline(required bool) GateOption
WithRequireBaseline requires baseline to exist.
func WithThroughputThreshold ¶
func WithThroughputThreshold(threshold float64) GateOption
WithThroughputThreshold sets the throughput threshold.
func WithUpdateBaseline ¶
func WithUpdateBaseline(enabled bool) GateOption
WithUpdateBaseline enables baseline update on pass.
type LatencyBaseline ¶
type LatencyBaseline struct {
// P50 is the 50th percentile latency.
P50 time.Duration `json:"p50"`
// P95 is the 95th percentile latency.
P95 time.Duration `json:"p95"`
// P99 is the 99th percentile latency.
P99 time.Duration `json:"p99"`
// Mean is the average latency.
Mean time.Duration `json:"mean"`
// StdDev is the standard deviation.
StdDev time.Duration `json:"std_dev"`
}
LatencyBaseline holds latency performance metrics.
type MemoryBaseline ¶
type MemoryBaseline struct {
// AllocBytesPerOp is bytes allocated per operation.
AllocBytesPerOp uint64 `json:"alloc_bytes_per_op"`
// AllocsPerOp is number of allocations per operation.
AllocsPerOp uint64 `json:"allocs_per_op"`
// HeapInUse is the typical heap usage.
HeapInUse uint64 `json:"heap_in_use,omitempty"`
}
MemoryBaseline holds memory usage metrics.
type MemoryBaselineStore ¶
type MemoryBaselineStore struct {
// contains filtered or unexported fields
}
MemoryBaselineStore stores baselines in memory.
Description:
MemoryBaselineStore is useful for testing and short-lived processes. Data is lost when the process exits.
Thread Safety: Safe for concurrent use.
func NewMemoryBaseline ¶
func NewMemoryBaseline() *MemoryBaselineStore
NewMemoryBaseline creates a new memory-backed baseline store.
Outputs:
- *MemoryBaselineStore: The new store. Never nil.
func (*MemoryBaselineStore) Delete ¶
func (m *MemoryBaselineStore) Delete(_ context.Context, component string) error
Delete implements Baseline.
func (*MemoryBaselineStore) Get ¶
func (m *MemoryBaselineStore) Get(_ context.Context, component string) (*BaselineData, error)
Get implements Baseline.
func (*MemoryBaselineStore) List ¶
func (m *MemoryBaselineStore) List(_ context.Context) ([]string, error)
List implements Baseline.
func (*MemoryBaselineStore) Set ¶
func (m *MemoryBaselineStore) Set(_ context.Context, component string, data *BaselineData) error
Set implements Baseline.
type Regression ¶
type Regression struct {
// Type identifies the regression type.
Type RegressionType
// Severity is the regression severity.
Severity Severity
// Component is the affected component.
Component string
// BaselineValue is the baseline metric value.
BaselineValue float64
// CurrentValue is the current metric value.
CurrentValue float64
// Change is the relative change (positive = regression).
Change float64
// Threshold is the threshold that was exceeded.
Threshold float64
// Message is a human-readable description.
Message string
}
Regression describes a detected regression.
type RegressionType ¶
type RegressionType int
RegressionType identifies the type of regression.
const ( // RegressionNone indicates no regression. RegressionNone RegressionType = iota // RegressionLatencyP50 indicates P50 latency regression. RegressionLatencyP50 // RegressionLatencyP95 indicates P95 latency regression. RegressionLatencyP95 // RegressionLatencyP99 indicates P99 latency regression. RegressionLatencyP99 // RegressionThroughput indicates throughput regression. RegressionThroughput // RegressionMemory indicates memory usage regression. RegressionMemory // RegressionErrorRate indicates error rate regression. RegressionErrorRate )
func (RegressionType) String ¶
func (r RegressionType) String() string
String returns the string representation.
type StatisticalDetector ¶
type StatisticalDetector struct {
*Detector
// contains filtered or unexported fields
}
StatisticalDetector uses statistical tests for more robust detection.
func NewStatisticalDetector ¶
func NewStatisticalDetector(config *DetectorConfig, pValueThreshold float64) *StatisticalDetector
NewStatisticalDetector creates a detector with statistical significance testing.
Inputs:
- config: Detection configuration.
- pValueThreshold: P-value threshold for significance (e.g., 0.05).
Outputs:
- *StatisticalDetector: The new detector. Never nil.
func (*StatisticalDetector) DetectWithSamples ¶
func (d *StatisticalDetector) DetectWithSamples( baseline *BaselineData, current *CurrentMetrics, baselineSamples, currentSamples []time.Duration, ) *DetectionResult
DetectWithSamples detects regressions using raw samples for statistical testing.
Inputs:
- baseline: The baseline to compare against.
- current: Current performance metrics.
- baselineSamples: Raw latency samples from baseline period.
- currentSamples: Raw latency samples from current period.
Outputs:
- *DetectionResult: Detection results with statistical confidence.
Thread Safety: Safe for concurrent use.
type ThroughputBaseline ¶
type ThroughputBaseline struct {
// OpsPerSecond is operations per second.
OpsPerSecond float64 `json:"ops_per_second"`
// BytesPerSecond is throughput in bytes.
BytesPerSecond float64 `json:"bytes_per_second,omitempty"`
}
ThroughputBaseline holds throughput metrics.