Documentation
¶
Overview ¶
Package ml provides anomaly detection using Isolation Forest
Package ml provides neural network-based performance prediction ¶
Package ml provides predictive resource allocation using LSTM models ¶
Production Data Collector for ML Training Collects and processes production metrics for machine learning models
Index ¶
- type AdaptiveThresholds
- func (at *AdaptiveThresholds) GetBaseline(metric string) float64
- func (at *AdaptiveThresholds) GetThreshold(metric string) float64
- func (at *AdaptiveThresholds) SetThreshold(metric string, value float64, confidence float64)
- func (at *AdaptiveThresholds) UpdateWithFeedback(metric string, wasFalsePositive bool)
- type Alert
- type AlertManager
- type AlertSubscriber
- type AnomalyConfig
- type AnomalyDetector
- func (ad *AnomalyDetector) DetectAnomaly(metrics VMMetrics) (*AnomalyScore, error)
- func (ad *AnomalyDetector) GetAlerts(limit int) []Alert
- func (ad *AnomalyDetector) GetAnomalyScores() map[string]*AnomalyScore
- func (ad *AnomalyDetector) MarkFalsePositive(vmID string, timestamp time.Time)
- func (ad *AnomalyDetector) Start(ctx context.Context) error
- func (ad *AnomalyDetector) Train(trainingData []VMMetrics) error
- type AnomalyScore
- type Bottleneck
- type CapacityChange
- type CapacityConstraint
- type CapacityPlan
- type CapacityPlanner
- type CollectorConfig
- type CompressedGradient
- type CompressedTensor
- type CompressionErrorMetrics
- type CompressionPipeline
- func (cp *CompressionPipeline) CompressBatch(ctx context.Context, gradients []*GradientTensor, tensors []*ModelTensor) ([]*CompressionResult, error)
- func (cp *CompressionPipeline) CompressGradient(ctx context.Context, gradient *GradientTensor) (*CompressionResult, error)
- func (cp *CompressionPipeline) CompressTensor(ctx context.Context, tensor *ModelTensor) (*CompressionResult, error)
- func (cp *CompressionPipeline) EstimateCompressionBenefit(gradientSizes []int64, tensorSizes []int64) map[string]interface{}
- func (cp *CompressionPipeline) GetCompressionRecommendations(profile WorkloadProfile) map[string]interface{}
- func (cp *CompressionPipeline) GetStatistics() *PipelineStatistics
- func (cp *CompressionPipeline) OptimizeConfiguration(workloadProfile WorkloadProfile) CompressionPipelineConfig
- func (cp *CompressionPipeline) UpdateConfig(config CompressionPipelineConfig)
- type CompressionPipelineConfig
- type CompressionResult
- type CompressionThresholds
- type DemandProjection
- type ErrorToleranceMode
- type Feature
- type FeatureConfig
- type FeatureEngine
- type GradientCompressionAlgorithm
- type GradientCompressionConfig
- type GradientCompressor
- func (gc *GradientCompressor) CompressGradient(gradient *GradientTensor) (*CompressedGradient, error)
- func (gc *GradientCompressor) DecompressGradient(compressed *CompressedGradient) (*GradientTensor, error)
- func (gc *GradientCompressor) EstimateCompressionRatio(dataSize int, config GradientCompressionConfig) float64
- func (gc *GradientCompressor) GetCompressionStats(compressed *CompressedGradient) map[string]interface{}
- func (gc *GradientCompressor) GetConfig() GradientCompressionConfig
- func (gc *GradientCompressor) UpdateConfig(config GradientCompressionConfig)
- type GradientTensor
- type ImpactPredictor
- type ImportanceMetric
- type IsolationForest
- type IsolationNode
- type IsolationTree
- type LSTMModel
- type Layer
- type MLDataset
- type MetricCollector
- type MetricType
- type MetricsCollector
- type MetricsRingBuffer
- type MigrationCostModel
- type MigrationImpact
- type MigrationOptimizer
- type MigrationSuggestion
- type MigrationWindow
- type ModelTensor
- type NeuralNetwork
- type PatternOccurrence
- type PatternRecognizer
- type PerformanceConfig
- type PerformanceHistory
- type PerformancePrediction
- type PerformancePredictor
- func (pp *PerformancePredictor) GetCapacityPlans() []*CapacityPlan
- func (pp *PerformancePredictor) GetMigrationWindows() []MigrationWindow
- func (pp *PerformancePredictor) GetPredictions() map[string]*PerformancePrediction
- func (pp *PerformancePredictor) PredictPerformance(vmID string, workloadType string) (*PerformancePrediction, error)
- func (pp *PerformancePredictor) Start(ctx context.Context) error
- type PerformanceRecommendation
- type PerformanceRecord
- type PipelineStatistics
- type PredictiveAllocator
- func (pa *PredictiveAllocator) AddMetrics(metrics *ResourceMetrics)
- func (pa *PredictiveAllocator) GetPredictions() map[string]*ResourcePrediction
- func (pa *PredictiveAllocator) PredictResourceUsage(vmID string, horizon time.Duration) (*ResourcePrediction, error)
- func (pa *PredictiveAllocator) Start(ctx context.Context) error
- type PredictorConfig
- type ProductionDataCollector
- func (pdc *ProductionDataCollector) GetAllDatasets() map[string]*MLDataset
- func (pdc *ProductionDataCollector) GetDataset(datasetID string) (*MLDataset, error)
- func (pdc *ProductionDataCollector) GetMetrics() map[string]float64
- func (pdc *ProductionDataCollector) Start(ctx context.Context) error
- func (pdc *ProductionDataCollector) Stop() error
- type PruningSchedule
- type QualitySettings
- type QuantizationBits
- type ResourceMetrics
- type ResourcePrediction
- type ResourceProfile
- type SVDComponents
- type ScalingRecommendation
- type SparseData
- type SparsificationAlgorithm
- type SparsificationConfig
- type SparsificationGranularity
- type SparsificationMetrics
- type Sparsifier
- func (s *Sparsifier) CompareSparsificationMethods(tensor *GradientTensor, layerName string) map[SparsificationAlgorithm]map[string]interface{}
- func (s *Sparsifier) DensifyTensor(sparse *SparseData) (*GradientTensor, error)
- func (s *Sparsifier) EstimateSparsificationBenefit(tensorSize int, config SparsificationConfig) map[string]float64
- func (s *Sparsifier) GetSparsificationStats(sparse *SparseData, metrics *SparsificationMetrics) map[string]interface{}
- func (s *Sparsifier) SparsifyTensor(tensor *GradientTensor, layerName string) (*SparseData, *SparsificationMetrics, error)
- func (s *Sparsifier) UpdateConfig(config SparsificationConfig)
- type SystemCapacity
- type TensorCompressionAlgorithm
- type TensorCompressionConfig
- type TensorCompressionMetrics
- type TensorCompressor
- func (tc *TensorCompressor) CompressTensor(tensor *ModelTensor, algorithm TensorCompressionAlgorithm) (*CompressedTensor, error)
- func (tc *TensorCompressor) DecompressTensor(compressed *CompressedTensor) (*ModelTensor, error)
- func (tc *TensorCompressor) GetTensorCompressionStats(compressed *CompressedTensor) map[string]interface{}
- func (tc *TensorCompressor) UpdateConfig(config GradientCompressionConfig)
- type TensorType
- type ThresholdInfo
- type TimeSeriesBuffer
- type TimeSeriesPoint
- type VMMetrics
- type WorkloadPattern
- type WorkloadProfile
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type AdaptiveThresholds ¶
type AdaptiveThresholds struct {
// contains filtered or unexported fields
}
AdaptiveThresholds manages self-learning threshold adjustment
func NewAdaptiveThresholds ¶
func NewAdaptiveThresholds(learningRate float64) *AdaptiveThresholds
NewAdaptiveThresholds creates adaptive threshold manager
func (*AdaptiveThresholds) GetBaseline ¶
func (at *AdaptiveThresholds) GetBaseline(metric string) float64
func (*AdaptiveThresholds) GetThreshold ¶
func (at *AdaptiveThresholds) GetThreshold(metric string) float64
func (*AdaptiveThresholds) SetThreshold ¶
func (at *AdaptiveThresholds) SetThreshold(metric string, value float64, confidence float64)
func (*AdaptiveThresholds) UpdateWithFeedback ¶
func (at *AdaptiveThresholds) UpdateWithFeedback(metric string, wasFalsePositive bool)
type Alert ¶
type Alert struct {
ID string `json:"id"`
Timestamp time.Time `json:"timestamp"`
VMId string `json:"vm_id"`
Severity string `json:"severity"`
Type string `json:"type"`
Message string `json:"message"`
Score float64 `json:"score"`
Acknowledged bool `json:"acknowledged"`
ResolvedAt *time.Time `json:"resolved_at,omitempty"`
Actions []string `json:"actions"`
}
Alert represents an anomaly alert
type AlertManager ¶
type AlertManager struct {
// contains filtered or unexported fields
}
AlertManager handles anomaly alerts
func NewAlertManager ¶
func NewAlertManager() *AlertManager
NewAlertManager creates a new alert manager
func (*AlertManager) AddAlert ¶
func (am *AlertManager) AddAlert(alert Alert)
func (*AlertManager) GetRecentAlerts ¶
func (am *AlertManager) GetRecentAlerts(limit int) []Alert
type AlertSubscriber ¶
type AlertSubscriber interface {
OnAlert(alert Alert)
}
AlertSubscriber interface for alert notifications
type AnomalyConfig ¶
type AnomalyConfig struct {
NumTrees int `json:"num_trees"` // Number of trees in forest
SampleSize int `json:"sample_size"` // Subsample size
ContaminationRate float64 `json:"contamination_rate"` // Expected anomaly rate
AlertThreshold float64 `json:"alert_threshold"` // Score threshold for alerts
LearningRate float64 `json:"learning_rate"` // Threshold adaptation rate
WindowSize time.Duration `json:"window_size"` // Detection window
UpdateInterval time.Duration `json:"update_interval"` // Model update frequency
EnableAutoLearn bool `json:"enable_auto_learn"` // Auto-adjust thresholds
EnableClustering bool `json:"enable_clustering"` // Group similar anomalies
}
AnomalyConfig configures the anomaly detector
type AnomalyDetector ¶
type AnomalyDetector struct {
// contains filtered or unexported fields
}
AnomalyDetector implements Isolation Forest for VM behavior anomaly detection
func NewAnomalyDetector ¶
func NewAnomalyDetector(logger *logrus.Logger, config AnomalyConfig) *AnomalyDetector
NewAnomalyDetector creates a new anomaly detector
func (*AnomalyDetector) DetectAnomaly ¶
func (ad *AnomalyDetector) DetectAnomaly(metrics VMMetrics) (*AnomalyScore, error)
DetectAnomaly performs real-time anomaly detection
func (*AnomalyDetector) GetAlerts ¶
func (ad *AnomalyDetector) GetAlerts(limit int) []Alert
GetAlerts returns recent alerts
func (*AnomalyDetector) GetAnomalyScores ¶
func (ad *AnomalyDetector) GetAnomalyScores() map[string]*AnomalyScore
GetAnomalyScores returns current anomaly scores
func (*AnomalyDetector) MarkFalsePositive ¶
func (ad *AnomalyDetector) MarkFalsePositive(vmID string, timestamp time.Time)
MarkFalsePositive marks an anomaly as false positive for learning
func (*AnomalyDetector) Start ¶
func (ad *AnomalyDetector) Start(ctx context.Context) error
Start begins the anomaly detection service
func (*AnomalyDetector) Train ¶
func (ad *AnomalyDetector) Train(trainingData []VMMetrics) error
Train trains the Isolation Forest model
type AnomalyScore ¶
type AnomalyScore struct {
Timestamp time.Time `json:"timestamp"`
VMId string `json:"vm_id"`
Score float64 `json:"score"` // 0-1, higher = more anomalous
IsAnomaly bool `json:"is_anomaly"`
Severity string `json:"severity"` // low, medium, high, critical
AnomalyType string `json:"anomaly_type"` // cpu_spike, memory_leak, etc.
Features map[string]float64 `json:"features"`
Deviations map[string]float64 `json:"deviations"` // Feature deviations from normal
ContextualInfo map[string]interface{} `json:"contextual_info"`
Confidence float64 `json:"confidence"`
FalsePositive bool `json:"false_positive"` // Marked by operator feedback
}
AnomalyScore represents anomaly detection results
type Bottleneck ¶
type Bottleneck struct {
Type string `json:"type"` // cpu, memory, io, network
Severity string `json:"severity"` // low, medium, high, critical
Impact float64 `json:"impact"` // Performance impact percentage
TimeWindow time.Time `json:"time_window"` // When bottleneck occurs
Description string `json:"description"`
}
Bottleneck represents a performance bottleneck
type CapacityChange ¶
type CapacityChange struct {
Type string `json:"type"` // add_node, upgrade_cpu, add_memory
Magnitude float64 `json:"magnitude"` // Amount of change
Timing time.Time `json:"timing"` // When to implement
Priority string `json:"priority"`
Justification string `json:"justification"`
}
CapacityChange represents a required capacity change
type CapacityConstraint ¶
type CapacityConstraint struct {
Type string `json:"type"` // budget, hardware, policy
Limit float64 `json:"limit"`
Flexible bool `json:"flexible"`
Description string `json:"description"`
}
CapacityConstraint defines capacity planning constraints
type CapacityPlan ¶
type CapacityPlan struct {
ID string `json:"id"`
CreatedAt time.Time `json:"created_at"`
TimeHorizon time.Duration `json:"time_horizon"`
RequiredChanges []CapacityChange `json:"required_changes"`
EstimatedCost float64 `json:"estimated_cost"`
RiskLevel string `json:"risk_level"`
Benefits []string `json:"benefits"`
}
CapacityPlan represents a capacity planning recommendation
type CapacityPlanner ¶
type CapacityPlanner struct {
// contains filtered or unexported fields
}
CapacityPlanner handles capacity planning
func NewCapacityPlanner ¶
func NewCapacityPlanner() *CapacityPlanner
NewCapacityPlanner creates a capacity planner
func (*CapacityPlanner) CreatePlan ¶
func (cp *CapacityPlanner) CreatePlan(predictions []float64, horizon time.Duration) *CapacityPlan
type CollectorConfig ¶
type CollectorConfig struct {
InfluxURL string `json:"influx_url"`
InfluxToken string `json:"influx_token"`
InfluxOrg string `json:"influx_org"`
InfluxBucket string `json:"influx_bucket"`
CollectionRate time.Duration `json:"collection_rate"`
BufferSize int `json:"buffer_size"`
WindowSize int `json:"window_size"`
StepSize int `json:"step_size"`
AggregationFunc string `json:"aggregation_func"`
FeatureConfig FeatureConfig `json:"feature_config"`
}
CollectorConfig holds configuration for data collector
type CompressedGradient ¶
type CompressedGradient struct {
// OriginalTensor reference to original tensor metadata
OriginalTensor *GradientTensor `json:"original_tensor"`
// CompressedData contains the compressed gradient data
CompressedData []byte `json:"compressed_data"`
// Algorithm used for compression
Algorithm GradientCompressionAlgorithm `json:"algorithm"`
// CompressionRatio achieved
CompressionRatio float64 `json:"compression_ratio"`
// SparsityLevel achieved (for sparse algorithms)
SparsityLevel float64 `json:"sparsity_level"`
// QuantizationLevel used (for quantization algorithms)
QuantizationLevel QuantizationBits `json:"quantization_level"`
// Indices for sparse gradients
Indices []int32 `json:"indices,omitempty"`
// Values for sparse gradients
Values []float32 `json:"values,omitempty"`
// QuantizationScale for dequantization
QuantizationScale float64 `json:"quantization_scale,omitempty"`
// QuantizationZeroPoint for asymmetric quantization
QuantizationZeroPoint int32 `json:"quantization_zero_point,omitempty"`
// ErrorMetrics contains compression error statistics
ErrorMetrics *CompressionErrorMetrics `json:"error_metrics,omitempty"`
}
CompressedGradient represents a compressed gradient with metadata
type CompressedTensor ¶
type CompressedTensor struct {
// OriginalTensor reference to original tensor metadata
OriginalTensor *ModelTensor `json:"original_tensor"`
// CompressedData contains the compressed tensor data
CompressedData []byte `json:"compressed_data"`
// Algorithm used for compression
Algorithm TensorCompressionAlgorithm `json:"algorithm"`
// CompressionRatio achieved
CompressionRatio float64 `json:"compression_ratio"`
// CompressionMetrics contains detailed compression statistics
CompressionMetrics *TensorCompressionMetrics `json:"compression_metrics"`
// ReconstructionData additional data needed for reconstruction
ReconstructionData map[string]interface{} `json:"reconstruction_data,omitempty"`
// PrunedIndices for pruning-based compression
PrunedIndices []int32 `json:"pruned_indices,omitempty"`
// QuantizationClusters for k-means quantization
QuantizationClusters []float32 `json:"quantization_clusters,omitempty"`
// SVDComponents for SVD compression
SVDComponents *SVDComponents `json:"svd_components,omitempty"`
}
CompressedTensor represents a compressed tensor
type CompressionErrorMetrics ¶
type CompressionErrorMetrics struct {
// L1Error absolute error
L1Error float64 `json:"l1_error"`
// L2Error mean squared error
L2Error float64 `json:"l2_error"`
// RelativeError relative error
RelativeError float64 `json:"relative_error"`
// MaxError maximum absolute error
MaxError float64 `json:"max_error"`
// SNR signal-to-noise ratio
SNR float64 `json:"snr"`
}
CompressionErrorMetrics tracks compression error statistics
type CompressionPipeline ¶
type CompressionPipeline struct {
// contains filtered or unexported fields
}
CompressionPipeline manages the entire ML compression pipeline
func NewCompressionPipeline ¶
func NewCompressionPipeline(config CompressionPipelineConfig) *CompressionPipeline
NewCompressionPipeline creates a new compression pipeline
func (*CompressionPipeline) CompressBatch ¶
func (cp *CompressionPipeline) CompressBatch(ctx context.Context, gradients []*GradientTensor, tensors []*ModelTensor) ([]*CompressionResult, error)
CompressBatch compresses a batch of tensors in parallel
func (*CompressionPipeline) CompressGradient ¶
func (cp *CompressionPipeline) CompressGradient(ctx context.Context, gradient *GradientTensor) (*CompressionResult, error)
CompressGradient compresses a gradient tensor through the pipeline
func (*CompressionPipeline) CompressTensor ¶
func (cp *CompressionPipeline) CompressTensor(ctx context.Context, tensor *ModelTensor) (*CompressionResult, error)
CompressTensor compresses a model tensor through the pipeline
func (*CompressionPipeline) EstimateCompressionBenefit ¶
func (cp *CompressionPipeline) EstimateCompressionBenefit(gradientSizes []int64, tensorSizes []int64) map[string]interface{}
EstimateCompressionBenefit estimates the benefit of compressing given data
func (*CompressionPipeline) GetCompressionRecommendations ¶
func (cp *CompressionPipeline) GetCompressionRecommendations(profile WorkloadProfile) map[string]interface{}
GetCompressionRecommendations returns recommendations for optimal compression settings
func (*CompressionPipeline) GetStatistics ¶
func (cp *CompressionPipeline) GetStatistics() *PipelineStatistics
GetStatistics returns current pipeline statistics
func (*CompressionPipeline) OptimizeConfiguration ¶
func (cp *CompressionPipeline) OptimizeConfiguration(workloadProfile WorkloadProfile) CompressionPipelineConfig
OptimizeConfiguration optimizes the compression configuration for given workload characteristics
func (*CompressionPipeline) UpdateConfig ¶
func (cp *CompressionPipeline) UpdateConfig(config CompressionPipelineConfig)
UpdateConfig updates the pipeline configuration
type CompressionPipelineConfig ¶
type CompressionPipelineConfig struct {
// EnableGradientCompression enables gradient compression
EnableGradientCompression bool `json:"enable_gradient_compression"`
// EnableTensorCompression enables model tensor compression
EnableTensorCompression bool `json:"enable_tensor_compression"`
// EnableSparsification enables sparsification algorithms
EnableSparsification bool `json:"enable_sparsification"`
// EnableStorageCompression enables storage-level compression
EnableStorageCompression bool `json:"enable_storage_compression"`
// GradientConfig configuration for gradient compression
GradientConfig GradientCompressionConfig `json:"gradient_config"`
// TensorConfig configuration for tensor compression
TensorConfig TensorCompressionConfig `json:"tensor_config"`
// SparsificationConfig configuration for sparsification
SparsificationConfig SparsificationConfig `json:"sparsification_config"`
// StorageConfig configuration for storage compression
StorageConfig compression.CompressionConfig `json:"storage_config"`
// CompressionThresholds size thresholds for applying compression
CompressionThresholds CompressionThresholds `json:"compression_thresholds"`
// QualitySettings quality vs compression trade-offs
QualitySettings QualitySettings `json:"quality_settings"`
// ParallelProcessing enables parallel compression processing
ParallelProcessing bool `json:"parallel_processing"`
// MaxConcurrency maximum concurrent compression operations
MaxConcurrency int `json:"max_concurrency"`
}
CompressionPipelineConfig holds configuration for the entire compression pipeline
func DefaultCompressionPipelineConfig ¶
func DefaultCompressionPipelineConfig() CompressionPipelineConfig
DefaultCompressionPipelineConfig returns a default pipeline configuration
type CompressionResult ¶
type CompressionResult struct {
// Success whether compression was successful
Success bool `json:"success"`
// CompressedData the compressed data (could be gradient, tensor, or sparse)
CompressedData interface{} `json:"compressed_data"`
// CompressionRatio achieved compression ratio
CompressionRatio float64 `json:"compression_ratio"`
// Method compression method used
Method string `json:"method"`
// ProcessingTimeMs time taken for compression
ProcessingTimeMs int64 `json:"processing_time_ms"`
// ErrorMessage error message if compression failed
ErrorMessage string `json:"error_message,omitempty"`
// Metadata additional metadata about compression
Metadata map[string]interface{} `json:"metadata,omitempty"`
}
CompressionResult contains the result of a compression operation
type CompressionThresholds ¶
type CompressionThresholds struct {
// MinGradientSize minimum size in bytes to apply gradient compression
MinGradientSize int64 `json:"min_gradient_size"`
// MinTensorSize minimum size in bytes to apply tensor compression
MinTensorSize int64 `json:"min_tensor_size"`
// MinSparsificationSize minimum size to apply sparsification
MinSparsificationSize int64 `json:"min_sparsification_size"`
// MinStorageSize minimum size to apply storage compression
MinStorageSize int64 `json:"min_storage_size"`
}
CompressionThresholds defines size thresholds for applying different compression methods
type DemandProjection ¶
type DemandProjection struct {
Timestamp time.Time `json:"timestamp"`
ProjectedCPU int `json:"projected_cpu"`
ProjectedMemory int64 `json:"projected_memory"`
ProjectedIOPS int64 `json:"projected_iops"`
Confidence float64 `json:"confidence"`
Scenario string `json:"scenario"` // best, likely, worst
}
DemandProjection projects future resource demand
type ErrorToleranceMode ¶
type ErrorToleranceMode string
ErrorToleranceMode defines error tolerance levels
const ( // ErrorToleranceStrict strict error bounds ErrorToleranceStrict ErrorToleranceMode = "strict" // ErrorToleranceBalanced balanced error tolerance ErrorToleranceBalanced ErrorToleranceMode = "balanced" // ErrorToleranceRelaxed relaxed error tolerance for maximum compression ErrorToleranceRelaxed ErrorToleranceMode = "relaxed" )
type Feature ¶
type Feature struct {
Name string `json:"name"`
Value float64 `json:"value"`
Type string `json:"type"` // numeric, categorical, temporal
Description string `json:"description"`
Timestamp time.Time `json:"timestamp"`
}
Feature represents an engineered feature for ML
type FeatureConfig ¶
type FeatureConfig struct {
EnableTemporalFeatures bool `json:"enable_temporal_features"`
EnableStatisticalFeatures bool `json:"enable_statistical_features"`
EnableFrequencyFeatures bool `json:"enable_frequency_features"`
EnableLaggedFeatures bool `json:"enable_lagged_features"`
LagPeriods []int `json:"lag_periods"`
RollingWindowSizes []int `json:"rolling_window_sizes"`
QuantileThresholds []float64 `json:"quantile_thresholds"`
}
FeatureConfig defines feature engineering settings
type FeatureEngine ¶
type FeatureEngine interface {
ExtractFeatures(data []TimeSeriesPoint) []Feature
GetFeatureName() string
}
FeatureEngine interface for feature engineering
type GradientCompressionAlgorithm ¶
type GradientCompressionAlgorithm string
GradientCompressionAlgorithm defines the compression algorithm for gradients
const ( // CompressionNone - no compression CompressionNone GradientCompressionAlgorithm = "none" // CompressionTopK - top-k sparsification CompressionTopK GradientCompressionAlgorithm = "topk" // CompressionRandomK - random-k sparsification CompressionRandomK GradientCompressionAlgorithm = "randomk" // CompressionThreshold - threshold-based sparsification CompressionThreshold GradientCompressionAlgorithm = "threshold" // CompressionQuantization - gradient quantization CompressionQuantization GradientCompressionAlgorithm = "quantization" // CompressionHybrid - combination of sparsification and quantization CompressionHybrid GradientCompressionAlgorithm = "hybrid" )
type GradientCompressionConfig ¶
type GradientCompressionConfig struct {
// Algorithm specifies the compression algorithm to use
Algorithm GradientCompressionAlgorithm `json:"algorithm"`
// CompressionRatio target compression ratio (0.0-1.0)
CompressionRatio float64 `json:"compression_ratio"`
// TopKRatio for top-k sparsification (percentage of top gradients to keep)
TopKRatio float64 `json:"topk_ratio"`
// SparsityThreshold for threshold-based sparsification
SparsityThreshold float64 `json:"sparsity_threshold"`
// QuantizationBits for gradient quantization
QuantizationBits QuantizationBits `json:"quantization_bits"`
// EnableAdaptive enables adaptive compression based on gradient distribution
EnableAdaptive bool `json:"enable_adaptive"`
// MinCompressionRatio minimum compression ratio for adaptive mode
MinCompressionRatio float64 `json:"min_compression_ratio"`
// MaxCompressionRatio maximum compression ratio for adaptive mode
MaxCompressionRatio float64 `json:"max_compression_ratio"`
// ErrorBoundCompression enable error-bounded compression
ErrorBoundCompression bool `json:"error_bound_compression"`
// MaxRelativeError maximum relative error for error-bounded compression
MaxRelativeError float64 `json:"max_relative_error"`
}
GradientCompressionConfig holds configuration for gradient compression
func DefaultGradientCompressionConfig ¶
func DefaultGradientCompressionConfig() GradientCompressionConfig
DefaultGradientCompressionConfig returns default configuration for gradient compression
type GradientCompressor ¶
type GradientCompressor struct {
// contains filtered or unexported fields
}
GradientCompressor handles compression and decompression of gradients
func NewGradientCompressor ¶
func NewGradientCompressor(config GradientCompressionConfig) *GradientCompressor
NewGradientCompressor creates a new gradient compressor
func (*GradientCompressor) CompressGradient ¶
func (gc *GradientCompressor) CompressGradient(gradient *GradientTensor) (*CompressedGradient, error)
CompressGradient compresses a gradient tensor using the configured algorithm
func (*GradientCompressor) DecompressGradient ¶
func (gc *GradientCompressor) DecompressGradient(compressed *CompressedGradient) (*GradientTensor, error)
DecompressGradient decompresses a compressed gradient
func (*GradientCompressor) EstimateCompressionRatio ¶
func (gc *GradientCompressor) EstimateCompressionRatio(dataSize int, config GradientCompressionConfig) float64
EstimateCompressionRatio estimates compression ratio for given configuration and data
func (*GradientCompressor) GetCompressionStats ¶
func (gc *GradientCompressor) GetCompressionStats(compressed *CompressedGradient) map[string]interface{}
GetCompressionStats returns statistics about compression performance
func (*GradientCompressor) GetConfig ¶
func (gc *GradientCompressor) GetConfig() GradientCompressionConfig
GetConfig returns the current configuration
func (*GradientCompressor) UpdateConfig ¶
func (gc *GradientCompressor) UpdateConfig(config GradientCompressionConfig)
UpdateConfig updates the compression configuration
type GradientTensor ¶
type GradientTensor struct {
// Shape of the tensor (dimensions)
Shape []int `json:"shape"`
// Data contains the gradient values as float32
Data []float32 `json:"data"`
// LayerName identifies the layer this gradient belongs to
LayerName string `json:"layer_name"`
// Timestamp when the gradient was computed
Timestamp int64 `json:"timestamp"`
}
GradientTensor represents a gradient tensor with shape and data
type ImpactPredictor ¶
type ImpactPredictor struct {
// contains filtered or unexported fields
}
ImpactPredictor predicts migration impact
func NewImpactPredictor ¶
func NewImpactPredictor() *ImpactPredictor
NewImpactPredictor creates an impact predictor
type ImportanceMetric ¶
type ImportanceMetric string
ImportanceMetric defines metrics for measuring parameter importance
const ( // ImportanceMagnitude - absolute magnitude ImportanceMagnitude ImportanceMetric = "magnitude" // ImportanceGradient - gradient magnitude ImportanceGradient ImportanceMetric = "gradient" // ImportanceFisher - Fisher information ImportanceFisher ImportanceMetric = "fisher" // ImportanceHessian - Hessian diagonal ImportanceHessian ImportanceMetric = "hessian" // ImportanceSaliency - gradient * weight ImportanceSaliency ImportanceMetric = "saliency" // ImportanceTaylor - Taylor expansion ImportanceTaylor ImportanceMetric = "taylor" )
type IsolationForest ¶
type IsolationForest struct {
// contains filtered or unexported fields
}
IsolationForest implements the Isolation Forest algorithm
func NewIsolationForest ¶
func NewIsolationForest(numTrees, sampleSize int) *IsolationForest
NewIsolationForest creates a new Isolation Forest
func (*IsolationForest) AnomalyScore ¶
func (iforest *IsolationForest) AnomalyScore(point []float64) float64
AnomalyScore calculates the anomaly score for a data point
func (*IsolationForest) Train ¶
func (iforest *IsolationForest) Train(data [][]float64)
Train builds the isolation forest
type IsolationNode ¶
type IsolationNode struct {
// contains filtered or unexported fields
}
IsolationNode represents a node in an isolation tree
type IsolationTree ¶
type IsolationTree struct {
// contains filtered or unexported fields
}
IsolationTree represents a single tree in the forest
type LSTMModel ¶
type LSTMModel struct {
// contains filtered or unexported fields
}
LSTMModel represents the LSTM neural network
func NewLSTMModel ¶
NewLSTMModel creates a new LSTM model
type Layer ¶
type Layer struct {
// contains filtered or unexported fields
}
Layer represents a neural network layer
type MLDataset ¶
type MLDataset struct {
ID string `json:"id"`
Created time.Time `json:"created"`
Features [][]Feature `json:"features"`
Labels []float64 `json:"labels"`
Metadata map[string]string `json:"metadata"`
WindowSize int `json:"window_size"`
StepSize int `json:"step_size"`
}
MLDataset represents a dataset ready for ML training
type MetricCollector ¶
type MetricCollector interface {
Collect() (PerformanceRecord, error)
}
MetricCollector interface for metric collection
type MetricType ¶
type MetricType string
MetricType defines the type of metric being collected
const ( MetricLatency MetricType = "latency" MetricThroughput MetricType = "throughput" MetricErrorRate MetricType = "error_rate" MetricCPU MetricType = "cpu_usage" MetricMemory MetricType = "memory_usage" MetricNetwork MetricType = "network_io" MetricDisk MetricType = "disk_io" MetricCompression MetricType = "compression_ratio" MetricPrediction MetricType = "prediction_accuracy" MetricConsensus MetricType = "consensus_time" )
type MetricsCollector ¶
type MetricsCollector struct {
// contains filtered or unexported fields
}
MetricsCollector collects performance metrics
func NewMetricsCollector ¶
func NewMetricsCollector() *MetricsCollector
NewMetricsCollector creates a metrics collector
type MetricsRingBuffer ¶
type MetricsRingBuffer struct {
// contains filtered or unexported fields
}
MetricsRingBuffer stores recent metrics for analysis
func NewMetricsRingBuffer ¶
func NewMetricsRingBuffer(capacity int) *MetricsRingBuffer
NewMetricsRingBuffer creates a new ring buffer for metrics
func (*MetricsRingBuffer) Add ¶
func (mrb *MetricsRingBuffer) Add(metrics VMMetrics)
type MigrationCostModel ¶
type MigrationCostModel struct {
// contains filtered or unexported fields
}
MigrationCostModel calculates migration costs
func NewMigrationCostModel ¶
func NewMigrationCostModel() *MigrationCostModel
NewMigrationCostModel creates a migration cost model
type MigrationImpact ¶
type MigrationImpact struct {
Timestamp time.Time `json:"timestamp"`
VMSize int64 `json:"vm_size"`
MigrationTime time.Duration `json:"migration_time"`
Downtime time.Duration `json:"downtime"`
PerformanceImpact float64 `json:"performance_impact"`
Success bool `json:"success"`
}
MigrationImpact records historical migration impacts
type MigrationOptimizer ¶
type MigrationOptimizer struct {
// contains filtered or unexported fields
}
MigrationOptimizer optimizes VM migration timing
func NewMigrationOptimizer ¶
func NewMigrationOptimizer() *MigrationOptimizer
NewMigrationOptimizer creates a migration optimizer
func (*MigrationOptimizer) SuggestMigration ¶
func (mo *MigrationOptimizer) SuggestMigration(vmID string, predictions []float64) *MigrationSuggestion
type MigrationSuggestion ¶
type MigrationSuggestion struct {
VMId string `json:"vm_id"`
SourceNode string `json:"source_node"`
TargetNode string `json:"target_node"`
OptimalWindow MigrationWindow `json:"optimal_window"`
Reason string `json:"reason"`
ExpectedBenefit float64 `json:"expected_benefit"`
EstimatedTime time.Duration `json:"estimated_time"`
Prerequisites []string `json:"prerequisites"`
}
MigrationSuggestion suggests optimal migration timing
type MigrationWindow ¶
type MigrationWindow struct {
Start time.Time `json:"start"`
End time.Time `json:"end"`
Score float64 `json:"score"`
ExpectedImpact float64 `json:"expected_impact"`
ResourceAvailable bool `json:"resource_available"`
RiskLevel string `json:"risk_level"`
}
MigrationWindow represents an optimal migration window
type ModelTensor ¶
type ModelTensor struct {
// Name of the tensor (layer name + parameter type)
Name string `json:"name"`
// Shape of the tensor
Shape []int `json:"shape"`
// Data contains the tensor values
Data []float32 `json:"data"`
// Type of tensor (weight, bias, etc.)
Type TensorType `json:"type"`
// LayerType the layer this tensor belongs to (conv, linear, etc.)
LayerType string `json:"layer_type"`
// Metadata additional information about the tensor
Metadata map[string]interface{} `json:"metadata,omitempty"`
}
ModelTensor represents a tensor from an ML model
type NeuralNetwork ¶
type NeuralNetwork struct {
// contains filtered or unexported fields
}
NeuralNetwork implements a feedforward neural network
func NewNeuralNetwork ¶
func NewNeuralNetwork(layers []int) *NeuralNetwork
NewNeuralNetwork creates a new neural network
func (*NeuralNetwork) Predict ¶
func (nn *NeuralNetwork) Predict(input []float64) []float64
Predict performs forward propagation
type PatternOccurrence ¶
type PatternOccurrence struct {
PatternID string `json:"pattern_id"`
Timestamp time.Time `json:"timestamp"`
VMId string `json:"vm_id"`
Confidence float64 `json:"confidence"`
Duration time.Duration `json:"duration"`
}
PatternOccurrence records when a pattern was detected
type PatternRecognizer ¶
type PatternRecognizer struct {
// contains filtered or unexported fields
}
PatternRecognizer identifies workload patterns
func NewPatternRecognizer ¶
func NewPatternRecognizer() *PatternRecognizer
NewPatternRecognizer creates a pattern recognizer
func (*PatternRecognizer) IdentifyPatterns ¶
func (pr *PatternRecognizer) IdentifyPatterns(history []PerformanceRecord) []*WorkloadPattern
type PerformanceConfig ¶
type PerformanceConfig struct {
ModelLayers []int `json:"model_layers"` // Neural network architecture
LearningRate float64 `json:"learning_rate"` // Training learning rate
PredictionWindow time.Duration `json:"prediction_window"` // How far ahead to predict
UpdateFrequency time.Duration `json:"update_frequency"` // Model update frequency
AccuracyTarget float64 `json:"accuracy_target"` // Target accuracy (85%)
LatencyTarget time.Duration `json:"latency_target"` // Target latency (250ms)
EnableCapacityPlan bool `json:"enable_capacity_plan"` // Enable capacity planning
EnableMigrationOpt bool `json:"enable_migration_opt"` // Enable migration optimization
}
PerformanceConfig configures the performance predictor
type PerformanceHistory ¶
type PerformanceHistory struct {
// contains filtered or unexported fields
}
PerformanceHistory stores historical performance data
func NewPerformanceHistory ¶
func NewPerformanceHistory(capacity int) *PerformanceHistory
NewPerformanceHistory creates a performance history buffer
func (*PerformanceHistory) Add ¶
func (ph *PerformanceHistory) Add(record PerformanceRecord)
func (*PerformanceHistory) GetAll ¶
func (ph *PerformanceHistory) GetAll() []PerformanceRecord
func (*PerformanceHistory) GetRecent ¶
func (ph *PerformanceHistory) GetRecent(vmID string, count int) []PerformanceRecord
type PerformancePrediction ¶
type PerformancePrediction struct {
Timestamp time.Time `json:"timestamp"`
VMId string `json:"vm_id"`
WorkloadType string `json:"workload_type"`
PredictedThroughput float64 `json:"predicted_throughput"`
PredictedLatency time.Duration `json:"predicted_latency"`
PredictedIOPS float64 `json:"predicted_iops"`
PredictedBandwidth float64 `json:"predicted_bandwidth"`
PerformanceScore float64 `json:"performance_score"`
Bottlenecks []Bottleneck `json:"bottlenecks"`
Recommendations []PerformanceRecommendation `json:"recommendations"`
CapacityPlan *CapacityPlan `json:"capacity_plan,omitempty"`
MigrationSuggestion *MigrationSuggestion `json:"migration_suggestion,omitempty"`
Confidence float64 `json:"confidence"`
ModelAccuracy float64 `json:"model_accuracy"`
}
PerformancePrediction contains performance predictions
type PerformancePredictor ¶
type PerformancePredictor struct {
// contains filtered or unexported fields
}
PerformancePredictor implements neural network for workload performance prediction
func NewPerformancePredictor ¶
func NewPerformancePredictor(logger *logrus.Logger, config PerformanceConfig) *PerformancePredictor
NewPerformancePredictor creates a new performance predictor
func (*PerformancePredictor) GetCapacityPlans ¶
func (pp *PerformancePredictor) GetCapacityPlans() []*CapacityPlan
GetCapacityPlans returns current capacity plans
func (*PerformancePredictor) GetMigrationWindows ¶
func (pp *PerformancePredictor) GetMigrationWindows() []MigrationWindow
GetMigrationWindows returns optimal migration windows
func (*PerformancePredictor) GetPredictions ¶
func (pp *PerformancePredictor) GetPredictions() map[string]*PerformancePrediction
GetPredictions returns all current predictions
func (*PerformancePredictor) PredictPerformance ¶
func (pp *PerformancePredictor) PredictPerformance(vmID string, workloadType string) (*PerformancePrediction, error)
PredictPerformance generates performance predictions
type PerformanceRecommendation ¶
type PerformanceRecommendation struct {
Action string `json:"action"`
Priority string `json:"priority"`
ExpectedImprovement float64 `json:"expected_improvement"`
Implementation string `json:"implementation"`
Cost float64 `json:"cost"`
TimeToImplement time.Duration `json:"time_to_implement"`
}
PerformanceRecommendation suggests performance improvements
type PerformanceRecord ¶
type PerformanceRecord struct {
Timestamp time.Time `json:"timestamp"`
VMId string `json:"vm_id"`
Throughput float64 `json:"throughput"`
Latency float64 `json:"latency"`
IOPS float64 `json:"iops"`
Bandwidth float64 `json:"bandwidth"`
CPUUsage float64 `json:"cpu_usage"`
MemoryUsage float64 `json:"memory_usage"`
}
PerformanceRecord represents a historical performance record
type PipelineStatistics ¶
type PipelineStatistics struct {
// TotalOperations total compression operations
TotalOperations int64 `json:"total_operations"`
// GradientOperations gradient compression operations
GradientOperations int64 `json:"gradient_operations"`
// TensorOperations tensor compression operations
TensorOperations int64 `json:"tensor_operations"`
// SparsificationOperations sparsification operations
SparsificationOperations int64 `json:"sparsification_operations"`
// TotalBytesProcessed total bytes processed
TotalBytesProcessed int64 `json:"total_bytes_processed"`
// TotalBytesCompressed total bytes after compression
TotalBytesCompressed int64 `json:"total_bytes_compressed"`
// AverageCompressionRatio average compression ratio achieved
AverageCompressionRatio float64 `json:"average_compression_ratio"`
// TotalCompressionTime total time spent on compression
TotalCompressionTimeMs int64 `json:"total_compression_time_ms"`
// LastUpdated timestamp of last update
LastUpdated time.Time `json:"last_updated"`
}
PipelineStatistics tracks compression pipeline statistics
type PredictiveAllocator ¶
type PredictiveAllocator struct {
// contains filtered or unexported fields
}
PredictiveAllocator implements LSTM-based resource prediction
func NewPredictiveAllocator ¶
func NewPredictiveAllocator(logger *logrus.Logger, config PredictorConfig) *PredictiveAllocator
NewPredictiveAllocator creates a new predictive resource allocator
func (*PredictiveAllocator) AddMetrics ¶
func (pa *PredictiveAllocator) AddMetrics(metrics *ResourceMetrics)
AddMetrics adds new metrics to the buffer
func (*PredictiveAllocator) GetPredictions ¶
func (pa *PredictiveAllocator) GetPredictions() map[string]*ResourcePrediction
GetPredictions returns current predictions for all VMs
func (*PredictiveAllocator) PredictResourceUsage ¶
func (pa *PredictiveAllocator) PredictResourceUsage(vmID string, horizon time.Duration) (*ResourcePrediction, error)
PredictResourceUsage generates resource usage predictions
type PredictorConfig ¶
type PredictorConfig struct {
PredictionHorizon time.Duration `json:"prediction_horizon"` // 15 minutes ahead
UpdateInterval time.Duration `json:"update_interval"` // Model update frequency
BufferSize int `json:"buffer_size"` // Historical data points
AccuracyTarget float64 `json:"accuracy_target"` // 85% accuracy target
LatencyTarget time.Duration `json:"latency_target"` // 250ms latency target
EnableAutoScaling bool `json:"enable_auto_scaling"`
EnablePreemptive bool `json:"enable_preemptive"`
ModelVersion string `json:"model_version"`
}
PredictorConfig configures the predictive allocator
type ProductionDataCollector ¶
type ProductionDataCollector struct {
// contains filtered or unexported fields
}
ProductionDataCollector collects and processes production metrics
func NewProductionDataCollector ¶
func NewProductionDataCollector(config *CollectorConfig) (*ProductionDataCollector, error)
NewProductionDataCollector creates a new production data collector
func (*ProductionDataCollector) GetAllDatasets ¶
func (pdc *ProductionDataCollector) GetAllDatasets() map[string]*MLDataset
GetAllDatasets returns all available datasets
func (*ProductionDataCollector) GetDataset ¶
func (pdc *ProductionDataCollector) GetDataset(datasetID string) (*MLDataset, error)
GetDataset returns a specific dataset by ID
func (*ProductionDataCollector) GetMetrics ¶
func (pdc *ProductionDataCollector) GetMetrics() map[string]float64
GetMetrics returns current collector metrics
func (*ProductionDataCollector) Start ¶
func (pdc *ProductionDataCollector) Start(ctx context.Context) error
Start begins data collection
func (*ProductionDataCollector) Stop ¶
func (pdc *ProductionDataCollector) Stop() error
Stop halts data collection
type PruningSchedule ¶
type PruningSchedule struct {
// InitialSparsity starting sparsity ratio
InitialSparsity float64 `json:"initial_sparsity"`
// FinalSparsity target final sparsity ratio
FinalSparsity float64 `json:"final_sparsity"`
// PruningFrequency how often to apply pruning (in epochs)
PruningFrequency int `json:"pruning_frequency"`
// Schedule type (linear, polynomial, exponential)
Schedule string `json:"schedule"`
// PolynomialPower for polynomial schedule
PolynomialPower float64 `json:"polynomial_power,omitempty"`
}
PruningSchedule defines the schedule for iterative pruning
type QualitySettings ¶
type QualitySettings struct {
// QualityLevel overall quality level (0.0-1.0)
QualityLevel float64 `json:"quality_level"`
// PreserveAccuracy prioritize accuracy over compression ratio
PreserveAccuracy bool `json:"preserve_accuracy"`
// MaxAccuracyLoss maximum acceptable accuracy loss
MaxAccuracyLoss float64 `json:"max_accuracy_loss"`
// AdaptiveCompression adjust compression based on importance
AdaptiveCompression bool `json:"adaptive_compression"`
// ErrorToleranceMode error tolerance for compression
ErrorToleranceMode ErrorToleranceMode `json:"error_tolerance_mode"`
}
QualitySettings defines quality vs compression trade-offs
type QuantizationBits ¶
type QuantizationBits uint8
QuantizationBits defines the bit precision for quantization
const ( // Bits32 - FP32 (no quantization) Bits32 QuantizationBits = 32 // Bits16 - FP16 quantization Bits16 QuantizationBits = 16 // Bits8 - INT8 quantization Bits8 QuantizationBits = 8 // Bits4 - 4-bit quantization (experimental) Bits4 QuantizationBits = 4 )
type ResourceMetrics ¶
type ResourceMetrics struct {
Timestamp time.Time `json:"timestamp"`
VMId string `json:"vm_id"`
CPUUsage float64 `json:"cpu_usage"`
MemoryUsage float64 `json:"memory_usage"`
IOUsage float64 `json:"io_usage"`
NetworkUsage float64 `json:"network_usage"`
}
ResourceMetrics represents current resource usage
type ResourcePrediction ¶
type ResourcePrediction struct {
Timestamp time.Time `json:"timestamp"`
VMId string `json:"vm_id"`
PredictedCPU float64 `json:"predicted_cpu"`
PredictedMemory float64 `json:"predicted_memory"`
PredictedIO float64 `json:"predicted_io"`
PredictedNetwork float64 `json:"predicted_network"`
Confidence float64 `json:"confidence"`
Recommendations []ScalingRecommendation `json:"recommendations"`
PredictionLatency time.Duration `json:"prediction_latency"`
Accuracy float64 `json:"accuracy"`
}
ResourcePrediction contains predicted resource usage
type ResourceProfile ¶
type ResourceProfile struct {
CPUPattern []float64 `json:"cpu_pattern"`
MemoryPattern []float64 `json:"memory_pattern"`
IOPattern []float64 `json:"io_pattern"`
NetworkPattern []float64 `json:"network_pattern"`
}
ResourceProfile defines resource usage patterns
type SVDComponents ¶
type SVDComponents struct {
// U matrix (left singular vectors)
U []float32 `json:"u"`
// S vector (singular values)
S []float32 `json:"s"`
// VT matrix (right singular vectors transposed)
VT []float32 `json:"vt"`
// UShape shape of U matrix
UShape []int `json:"u_shape"`
// VTShape shape of VT matrix
VTShape []int `json:"vt_shape"`
// Rank preserved rank
Rank int `json:"rank"`
}
SVDComponents stores SVD decomposition components
type ScalingRecommendation ¶
type ScalingRecommendation struct {
Action string `json:"action"` // scale_up, scale_down, migrate
Target string `json:"target"` // resource or VM ID
Magnitude float64 `json:"magnitude"` // scaling factor
Urgency string `json:"urgency"` // immediate, scheduled, optional
Reason string `json:"reason"`
EstimatedImpact float64 `json:"estimated_impact"`
TimeWindow time.Duration `json:"time_window"`
}
ScalingRecommendation suggests scaling actions
type SparseData ¶
type SparseData struct {
// Indices of non-zero elements
Indices []int32 `json:"indices"`
// Values of non-zero elements
Values []float32 `json:"values"`
// Shape of the original dense tensor
Shape []int `json:"shape"`
// SparsityLevel achieved sparsity level
SparsityLevel float64 `json:"sparsity_level"`
// Algorithm used for sparsification
Algorithm SparsificationAlgorithm `json:"algorithm"`
// Metadata additional information
Metadata map[string]interface{} `json:"metadata,omitempty"`
}
SparseData represents sparse data structure
type SparsificationAlgorithm ¶
type SparsificationAlgorithm string
SparsificationAlgorithm defines algorithms for creating sparse data structures
const ( // SparsificationMagnitude - magnitude-based pruning SparsificationMagnitude SparsificationAlgorithm = "magnitude" // SparsificationGradient - gradient-based pruning SparsificationGradient SparsificationAlgorithm = "gradient" // SparsificationRandom - random pruning SparsificationRandom SparsificationAlgorithm = "random" // SparsificationStructured - structured pruning SparsificationStructured SparsificationAlgorithm = "structured" // SparsificationSNIP - Single-shot Network Pruning SparsificationSNIP SparsificationAlgorithm = "snip" // SparsificationGraSP - Gradient Signal Preservation SparsificationGraSP SparsificationAlgorithm = "grasp" // SparsificationLottery - Lottery Ticket Hypothesis SparsificationLottery SparsificationAlgorithm = "lottery" // SparsificationAdaptive - adaptive sparsification SparsificationAdaptive SparsificationAlgorithm = "adaptive" )
type SparsificationConfig ¶
type SparsificationConfig struct {
// Algorithm specifies the sparsification algorithm
Algorithm SparsificationAlgorithm `json:"algorithm"`
// SparsityRatio target sparsity ratio (0.0-1.0)
SparsityRatio float64 `json:"sparsity_ratio"`
// Granularity of sparsification
Granularity SparsificationGranularity `json:"granularity"`
// BlockSize for block-wise sparsification
BlockSize []int `json:"block_size,omitempty"`
// PruningSchedule for iterative pruning
PruningSchedule *PruningSchedule `json:"pruning_schedule,omitempty"`
// GradientAccumulation steps for gradient-based pruning
GradientAccumulation int `json:"gradient_accumulation"`
// ImportanceMetric for importance-based pruning
ImportanceMetric ImportanceMetric `json:"importance_metric"`
// AdaptiveThreshold enables adaptive threshold adjustment
AdaptiveThreshold bool `json:"adaptive_threshold"`
// MinSparsity minimum sparsity level
MinSparsity float64 `json:"min_sparsity"`
// MaxSparsity maximum sparsity level
MaxSparsity float64 `json:"max_sparsity"`
// RecoveryEpochs epochs to recover after aggressive pruning
RecoveryEpochs int `json:"recovery_epochs"`
// PreserveStructure whether to maintain structured sparsity
PreserveStructure bool `json:"preserve_structure"`
}
SparsificationConfig holds configuration for sparsification algorithms
func DefaultSparsificationConfig ¶
func DefaultSparsificationConfig() SparsificationConfig
DefaultSparsificationConfig returns a default sparsification configuration
type SparsificationGranularity ¶
type SparsificationGranularity string
SparsificationGranularity defines the granularity of sparsification
const ( // GranularityElement - element-wise sparsification GranularityElement SparsificationGranularity = "element" // GranularityChannel - channel-wise sparsification GranularityChannel SparsificationGranularity = "channel" // GranularityFilter - filter-wise sparsification GranularityFilter SparsificationGranularity = "filter" // GranularityBlock - block-wise sparsification GranularityBlock SparsificationGranularity = "block" // GranularityLayer - layer-wise sparsification GranularityLayer SparsificationGranularity = "layer" )
type SparsificationMetrics ¶
type SparsificationMetrics struct {
// OriginalSize number of parameters before sparsification
OriginalSize int `json:"original_size"`
// NonZeroCount number of non-zero parameters after sparsification
NonZeroCount int `json:"non_zero_count"`
// SparsityLevel achieved sparsity level
SparsityLevel float64 `json:"sparsity_level"`
// CompressionRatio memory compression achieved
CompressionRatio float64 `json:"compression_ratio"`
// FlopsReduction computational reduction
FlopsReduction float64 `json:"flops_reduction"`
// AccuracyDrop estimated accuracy drop
AccuracyDrop float64 `json:"accuracy_drop,omitempty"`
// SparsificationTime time taken to sparsify
SparsificationTimeMs int64 `json:"sparsification_time_ms"`
}
SparsificationMetrics tracks sparsification performance
type Sparsifier ¶
type Sparsifier struct {
// contains filtered or unexported fields
}
Sparsifier implements sparsification algorithms for neural networks
func NewSparsifier ¶
func NewSparsifier(config SparsificationConfig) *Sparsifier
NewSparsifier creates a new sparsifier with the given configuration
func (*Sparsifier) CompareSparsificationMethods ¶
func (s *Sparsifier) CompareSparsificationMethods(tensor *GradientTensor, layerName string) map[SparsificationAlgorithm]map[string]interface{}
CompareSparsificationMethods compares different sparsification methods
func (*Sparsifier) DensifyTensor ¶
func (s *Sparsifier) DensifyTensor(sparse *SparseData) (*GradientTensor, error)
DensifyTensor converts sparse data back to dense tensor
func (*Sparsifier) EstimateSparsificationBenefit ¶
func (s *Sparsifier) EstimateSparsificationBenefit(tensorSize int, config SparsificationConfig) map[string]float64
EstimateSparsificationBenefit estimates the benefit of sparsification
func (*Sparsifier) GetSparsificationStats ¶
func (s *Sparsifier) GetSparsificationStats(sparse *SparseData, metrics *SparsificationMetrics) map[string]interface{}
GetSparsificationStats returns statistics about sparsification
func (*Sparsifier) SparsifyTensor ¶
func (s *Sparsifier) SparsifyTensor(tensor *GradientTensor, layerName string) (*SparseData, *SparsificationMetrics, error)
SparsifyTensor applies sparsification to a tensor
func (*Sparsifier) UpdateConfig ¶
func (s *Sparsifier) UpdateConfig(config SparsificationConfig)
UpdateConfig updates the sparsification configuration
type SystemCapacity ¶
type SystemCapacity struct {
TotalCPU int `json:"total_cpu"`
TotalMemory int64 `json:"total_memory"`
TotalStorage int64 `json:"total_storage"`
TotalNetwork int64 `json:"total_network"`
UsedCPU int `json:"used_cpu"`
UsedMemory int64 `json:"used_memory"`
UsedStorage int64 `json:"used_storage"`
UsedNetwork int64 `json:"used_network"`
VMCount int `json:"vm_count"`
NodeCount int `json:"node_count"`
}
SystemCapacity represents current system capacity
type TensorCompressionAlgorithm ¶
type TensorCompressionAlgorithm string
TensorCompressionAlgorithm defines compression algorithms for ML tensors
const ( // TensorCompressionNone - no compression TensorCompressionNone TensorCompressionAlgorithm = "none" // TensorCompressionSVD - Singular Value Decomposition TensorCompressionSVD TensorCompressionAlgorithm = "svd" // TensorCompressionPruning - Weight pruning TensorCompressionPruning TensorCompressionAlgorithm = "pruning" // TensorCompressionQuantization - Parameter quantization TensorCompressionQuantization TensorCompressionAlgorithm = "quantization" // TensorCompressionHuffman - Huffman coding for weights TensorCompressionHuffman TensorCompressionAlgorithm = "huffman" // TensorCompressionKMeans - K-means clustering quantization TensorCompressionKMeans TensorCompressionAlgorithm = "kmeans" // TensorCompressionLowRank - Low-rank approximation TensorCompressionLowRank TensorCompressionAlgorithm = "lowrank" // TensorCompressionBitpacking - Bit-level packing TensorCompressionBitpacking TensorCompressionAlgorithm = "bitpacking" )
type TensorCompressionConfig ¶
type TensorCompressionConfig struct {
Algorithm TensorCompressionAlgorithm `json:"algorithm"`
CompressionRatio float64 `json:"compression_ratio"`
QuantizationBits QuantizationBits `json:"quantization_bits"`
}
TensorCompressionConfig simplified tensor compression configuration
type TensorCompressionMetrics ¶
type TensorCompressionMetrics struct {
// OriginalSize in bytes
OriginalSize int `json:"original_size"`
// CompressedSize in bytes
CompressedSize int `json:"compressed_size"`
// CompressionRatio actual achieved ratio
CompressionRatio float64 `json:"compression_ratio"`
// ParameterReduction percentage of parameters reduced
ParameterReduction float64 `json:"parameter_reduction"`
// AccuracyLoss estimated accuracy loss from compression
AccuracyLoss float64 `json:"accuracy_loss,omitempty"`
// CompressionTime time taken to compress
CompressionTimeMs int64 `json:"compression_time_ms"`
// DecompressionTime time taken to decompress
DecompressionTimeMs int64 `json:"decompression_time_ms"`
// ErrorMetrics compression error statistics
ErrorMetrics *CompressionErrorMetrics `json:"error_metrics,omitempty"`
}
TensorCompressionMetrics tracks detailed compression statistics
type TensorCompressor ¶
type TensorCompressor struct {
// contains filtered or unexported fields
}
TensorCompressor handles compression and decompression of ML tensors
func NewTensorCompressor ¶
func NewTensorCompressor(config GradientCompressionConfig) *TensorCompressor
NewTensorCompressor creates a new tensor compressor
func (*TensorCompressor) CompressTensor ¶
func (tc *TensorCompressor) CompressTensor(tensor *ModelTensor, algorithm TensorCompressionAlgorithm) (*CompressedTensor, error)
CompressTensor compresses a model tensor using the configured algorithm
func (*TensorCompressor) DecompressTensor ¶
func (tc *TensorCompressor) DecompressTensor(compressed *CompressedTensor) (*ModelTensor, error)
DecompressTensor decompresses a compressed tensor
func (*TensorCompressor) GetTensorCompressionStats ¶
func (tc *TensorCompressor) GetTensorCompressionStats(compressed *CompressedTensor) map[string]interface{}
GetTensorCompressionStats returns compression statistics for a tensor
func (*TensorCompressor) UpdateConfig ¶
func (tc *TensorCompressor) UpdateConfig(config GradientCompressionConfig)
UpdateConfig updates the compression configuration
type TensorType ¶
type TensorType string
TensorType defines the type of tensor being compressed
const ( // TensorTypeWeight - Model weights/parameters TensorTypeWeight TensorType = "weight" // TensorTypeBias - Bias parameters TensorTypeBias TensorType = "bias" // TensorTypeActivation - Activation tensors TensorTypeActivation TensorType = "activation" // TensorTypeEmbedding - Embedding matrices TensorTypeEmbedding TensorType = "embedding" // TensorTypeConvKernel - Convolutional kernels TensorTypeConvKernel TensorType = "conv_kernel" // TensorTypeAttentionWeight - Attention weights TensorTypeAttentionWeight TensorType = "attention_weight" )
type ThresholdInfo ¶
type ThresholdInfo struct {
Current float64 `json:"current"`
Historical []float64 `json:"historical"`
LastUpdated time.Time `json:"last_updated"`
Confidence float64 `json:"confidence"`
TruePositives int `json:"true_positives"`
FalsePositives int `json:"false_positives"`
}
ThresholdInfo stores threshold information for a metric
type TimeSeriesBuffer ¶
type TimeSeriesBuffer struct {
// contains filtered or unexported fields
}
TimeSeriesBuffer stores historical metrics
func NewTimeSeriesBuffer ¶
func NewTimeSeriesBuffer(capacity int) *TimeSeriesBuffer
NewTimeSeriesBuffer creates a new circular buffer for time series data
func (*TimeSeriesBuffer) Add ¶
func (tsb *TimeSeriesBuffer) Add(data []float64, timestamp time.Time)
func (*TimeSeriesBuffer) Cleanup ¶
func (tsb *TimeSeriesBuffer) Cleanup()
func (*TimeSeriesBuffer) GetAll ¶
func (tsb *TimeSeriesBuffer) GetAll() [][]float64
func (*TimeSeriesBuffer) GetSequence ¶
func (tsb *TimeSeriesBuffer) GetSequence(length int) [][]float64
type TimeSeriesPoint ¶
type TimeSeriesPoint struct {
Timestamp time.Time `json:"timestamp"`
Value float64 `json:"value"`
Tags map[string]string `json:"tags"`
Fields map[string]interface{} `json:"fields"`
}
TimeSeriesPoint represents a single data point in time series
type VMMetrics ¶
type VMMetrics struct {
Timestamp time.Time `json:"timestamp"`
VMId string `json:"vm_id"`
CPU float64 `json:"cpu"`
Memory float64 `json:"memory"`
DiskIO float64 `json:"disk_io"`
NetworkIO float64 `json:"network_io"`
Latency float64 `json:"latency"`
ErrorRate float64 `json:"error_rate"`
CustomMetrics map[string]float64 `json:"custom_metrics"`
}
VMMetrics represents VM performance metrics
type WorkloadPattern ¶
type WorkloadPattern struct {
ID string `json:"id"`
Name string `json:"name"`
Type string `json:"type"` // periodic, burst, steady, growing
Characteristics map[string]float64 `json:"characteristics"`
Frequency time.Duration `json:"frequency"`
Duration time.Duration `json:"duration"`
ResourceProfile ResourceProfile `json:"resource_profile"`
LastSeen time.Time `json:"last_seen"`
Occurrences int `json:"occurrences"`
}
WorkloadPattern represents a recognized workload pattern
type WorkloadProfile ¶
type WorkloadProfile struct {
ModelType string `json:"model_type"` // transformer, cnn, mlp, etc.
ModelSize int64 `json:"model_size"` // Model size in parameters
BatchSize int `json:"batch_size"` // Training batch size
MemoryConstrained bool `json:"memory_constrained"` // Whether memory is limited
SpeedCritical bool `json:"speed_critical"` // Whether speed is critical
AccuracyTarget float64 `json:"accuracy_target"` // Target accuracy to maintain
}
WorkloadProfile describes characteristics of the ML workload