ml

package
v0.0.0-...-bd75e23 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 19, 2026 License: MIT Imports: 17 Imported by: 0

README

DWCP ML Pipeline - Quick Reference

Overview

Advanced ML capabilities for distributed VM optimization with AutoML, NAS, HPO, compression, federated learning, and high-performance inference.

Quick Start

import (
    "github.com/yourusername/novacron/backend/core/ml/automl"
    "github.com/yourusername/novacron/backend/core/ml/inference"
)

// 1. Train AutoML model
config := automl.DefaultAutoMLConfig()
engine := automl.NewAutoMLEngine(config)
bestModel, _ := engine.Fit(ctx, X, y, featureNames)

// 2. Deploy for inference
inferenceEngine := inference.NewInferenceEngine(nil)
inferenceEngine.LoadModel("model", "v1", bestModel.Weights, "automl")

// 3. Predict
prediction, _ := inferenceEngine.Predict(ctx, "model", "v1", input)

Components

  1. AutoML (automl/): Automated model selection and training
  2. NAS (nas/): Neural architecture search
  3. HPO (hpo/): Hyperparameter optimization
  4. Compression (compression/): Model compression
  5. Federated (federated/): Federated learning
  6. Registry (registry/): Model versioning
  7. Inference (inference/): High-performance serving
  8. Features (features/): Feature store
  9. Pipeline (pipeline/): Workflow orchestration
  10. Metrics (metrics/): Performance tracking

Performance

  • AutoML: <30 min convergence
  • Inference: <10ms latency
  • Compression: 5-10x with <2% loss
  • Federated: <20% overhead
  • Coverage: >90%

Documentation

See docs/DWCP_ML_PIPELINE.md for complete guide.

Tests

go test -v -cover ./backend/core/ml/...

License

Copyright (c) 2025 NovaCron

Documentation

Overview

Package ml provides anomaly detection using Isolation Forest

Package ml provides neural network-based performance prediction

Package ml provides predictive resource allocation using LSTM models

Production Data Collector for ML Training Collects and processes production metrics for machine learning models

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type AdaptiveThresholds

type AdaptiveThresholds struct {
	// contains filtered or unexported fields
}

AdaptiveThresholds manages self-learning threshold adjustment

func NewAdaptiveThresholds

func NewAdaptiveThresholds(learningRate float64) *AdaptiveThresholds

NewAdaptiveThresholds creates adaptive threshold manager

func (*AdaptiveThresholds) GetBaseline

func (at *AdaptiveThresholds) GetBaseline(metric string) float64

func (*AdaptiveThresholds) GetThreshold

func (at *AdaptiveThresholds) GetThreshold(metric string) float64

func (*AdaptiveThresholds) SetThreshold

func (at *AdaptiveThresholds) SetThreshold(metric string, value float64, confidence float64)

func (*AdaptiveThresholds) UpdateWithFeedback

func (at *AdaptiveThresholds) UpdateWithFeedback(metric string, wasFalsePositive bool)

type Alert

type Alert struct {
	ID           string     `json:"id"`
	Timestamp    time.Time  `json:"timestamp"`
	VMId         string     `json:"vm_id"`
	Severity     string     `json:"severity"`
	Type         string     `json:"type"`
	Message      string     `json:"message"`
	Score        float64    `json:"score"`
	Acknowledged bool       `json:"acknowledged"`
	ResolvedAt   *time.Time `json:"resolved_at,omitempty"`
	Actions      []string   `json:"actions"`
}

Alert represents an anomaly alert

type AlertManager

type AlertManager struct {
	// contains filtered or unexported fields
}

AlertManager handles anomaly alerts

func NewAlertManager

func NewAlertManager() *AlertManager

NewAlertManager creates a new alert manager

func (*AlertManager) AddAlert

func (am *AlertManager) AddAlert(alert Alert)

func (*AlertManager) GetRecentAlerts

func (am *AlertManager) GetRecentAlerts(limit int) []Alert

type AlertSubscriber

type AlertSubscriber interface {
	OnAlert(alert Alert)
}

AlertSubscriber interface for alert notifications

type AnomalyConfig

type AnomalyConfig struct {
	NumTrees          int           `json:"num_trees"`          // Number of trees in forest
	SampleSize        int           `json:"sample_size"`        // Subsample size
	ContaminationRate float64       `json:"contamination_rate"` // Expected anomaly rate
	AlertThreshold    float64       `json:"alert_threshold"`    // Score threshold for alerts
	LearningRate      float64       `json:"learning_rate"`      // Threshold adaptation rate
	WindowSize        time.Duration `json:"window_size"`        // Detection window
	UpdateInterval    time.Duration `json:"update_interval"`    // Model update frequency
	EnableAutoLearn   bool          `json:"enable_auto_learn"`  // Auto-adjust thresholds
	EnableClustering  bool          `json:"enable_clustering"`  // Group similar anomalies
}

AnomalyConfig configures the anomaly detector

type AnomalyDetector

type AnomalyDetector struct {
	// contains filtered or unexported fields
}

AnomalyDetector implements Isolation Forest for VM behavior anomaly detection

func NewAnomalyDetector

func NewAnomalyDetector(logger *logrus.Logger, config AnomalyConfig) *AnomalyDetector

NewAnomalyDetector creates a new anomaly detector

func (*AnomalyDetector) DetectAnomaly

func (ad *AnomalyDetector) DetectAnomaly(metrics VMMetrics) (*AnomalyScore, error)

DetectAnomaly performs real-time anomaly detection

func (*AnomalyDetector) GetAlerts

func (ad *AnomalyDetector) GetAlerts(limit int) []Alert

GetAlerts returns recent alerts

func (*AnomalyDetector) GetAnomalyScores

func (ad *AnomalyDetector) GetAnomalyScores() map[string]*AnomalyScore

GetAnomalyScores returns current anomaly scores

func (*AnomalyDetector) MarkFalsePositive

func (ad *AnomalyDetector) MarkFalsePositive(vmID string, timestamp time.Time)

MarkFalsePositive marks an anomaly as false positive for learning

func (*AnomalyDetector) Start

func (ad *AnomalyDetector) Start(ctx context.Context) error

Start begins the anomaly detection service

func (*AnomalyDetector) Train

func (ad *AnomalyDetector) Train(trainingData []VMMetrics) error

Train trains the Isolation Forest model

type AnomalyScore

type AnomalyScore struct {
	Timestamp      time.Time              `json:"timestamp"`
	VMId           string                 `json:"vm_id"`
	Score          float64                `json:"score"` // 0-1, higher = more anomalous
	IsAnomaly      bool                   `json:"is_anomaly"`
	Severity       string                 `json:"severity"`     // low, medium, high, critical
	AnomalyType    string                 `json:"anomaly_type"` // cpu_spike, memory_leak, etc.
	Features       map[string]float64     `json:"features"`
	Deviations     map[string]float64     `json:"deviations"` // Feature deviations from normal
	ContextualInfo map[string]interface{} `json:"contextual_info"`
	Confidence     float64                `json:"confidence"`
	FalsePositive  bool                   `json:"false_positive"` // Marked by operator feedback
}

AnomalyScore represents anomaly detection results

type Bottleneck

type Bottleneck struct {
	Type        string    `json:"type"`        // cpu, memory, io, network
	Severity    string    `json:"severity"`    // low, medium, high, critical
	Impact      float64   `json:"impact"`      // Performance impact percentage
	TimeWindow  time.Time `json:"time_window"` // When bottleneck occurs
	Description string    `json:"description"`
}

Bottleneck represents a performance bottleneck

type CapacityChange

type CapacityChange struct {
	Type          string    `json:"type"`      // add_node, upgrade_cpu, add_memory
	Magnitude     float64   `json:"magnitude"` // Amount of change
	Timing        time.Time `json:"timing"`    // When to implement
	Priority      string    `json:"priority"`
	Justification string    `json:"justification"`
}

CapacityChange represents a required capacity change

type CapacityConstraint

type CapacityConstraint struct {
	Type        string  `json:"type"` // budget, hardware, policy
	Limit       float64 `json:"limit"`
	Flexible    bool    `json:"flexible"`
	Description string  `json:"description"`
}

CapacityConstraint defines capacity planning constraints

type CapacityPlan

type CapacityPlan struct {
	ID              string           `json:"id"`
	CreatedAt       time.Time        `json:"created_at"`
	TimeHorizon     time.Duration    `json:"time_horizon"`
	RequiredChanges []CapacityChange `json:"required_changes"`
	EstimatedCost   float64          `json:"estimated_cost"`
	RiskLevel       string           `json:"risk_level"`
	Benefits        []string         `json:"benefits"`
}

CapacityPlan represents a capacity planning recommendation

type CapacityPlanner

type CapacityPlanner struct {
	// contains filtered or unexported fields
}

CapacityPlanner handles capacity planning

func NewCapacityPlanner

func NewCapacityPlanner() *CapacityPlanner

NewCapacityPlanner creates a capacity planner

func (*CapacityPlanner) CreatePlan

func (cp *CapacityPlanner) CreatePlan(predictions []float64, horizon time.Duration) *CapacityPlan

type CollectorConfig

type CollectorConfig struct {
	InfluxURL       string        `json:"influx_url"`
	InfluxToken     string        `json:"influx_token"`
	InfluxOrg       string        `json:"influx_org"`
	InfluxBucket    string        `json:"influx_bucket"`
	CollectionRate  time.Duration `json:"collection_rate"`
	BufferSize      int           `json:"buffer_size"`
	WindowSize      int           `json:"window_size"`
	StepSize        int           `json:"step_size"`
	AggregationFunc string        `json:"aggregation_func"`
	FeatureConfig   FeatureConfig `json:"feature_config"`
}

CollectorConfig holds configuration for data collector

type CompressedGradient

type CompressedGradient struct {
	// OriginalTensor reference to original tensor metadata
	OriginalTensor *GradientTensor `json:"original_tensor"`

	// CompressedData contains the compressed gradient data
	CompressedData []byte `json:"compressed_data"`

	// Algorithm used for compression
	Algorithm GradientCompressionAlgorithm `json:"algorithm"`

	// CompressionRatio achieved
	CompressionRatio float64 `json:"compression_ratio"`

	// SparsityLevel achieved (for sparse algorithms)
	SparsityLevel float64 `json:"sparsity_level"`

	// QuantizationLevel used (for quantization algorithms)
	QuantizationLevel QuantizationBits `json:"quantization_level"`

	// Indices for sparse gradients
	Indices []int32 `json:"indices,omitempty"`

	// Values for sparse gradients
	Values []float32 `json:"values,omitempty"`

	// QuantizationScale for dequantization
	QuantizationScale float64 `json:"quantization_scale,omitempty"`

	// QuantizationZeroPoint for asymmetric quantization
	QuantizationZeroPoint int32 `json:"quantization_zero_point,omitempty"`

	// ErrorMetrics contains compression error statistics
	ErrorMetrics *CompressionErrorMetrics `json:"error_metrics,omitempty"`
}

CompressedGradient represents a compressed gradient with metadata

type CompressedTensor

type CompressedTensor struct {
	// OriginalTensor reference to original tensor metadata
	OriginalTensor *ModelTensor `json:"original_tensor"`

	// CompressedData contains the compressed tensor data
	CompressedData []byte `json:"compressed_data"`

	// Algorithm used for compression
	Algorithm TensorCompressionAlgorithm `json:"algorithm"`

	// CompressionRatio achieved
	CompressionRatio float64 `json:"compression_ratio"`

	// CompressionMetrics contains detailed compression statistics
	CompressionMetrics *TensorCompressionMetrics `json:"compression_metrics"`

	// ReconstructionData additional data needed for reconstruction
	ReconstructionData map[string]interface{} `json:"reconstruction_data,omitempty"`

	// PrunedIndices for pruning-based compression
	PrunedIndices []int32 `json:"pruned_indices,omitempty"`

	// QuantizationClusters for k-means quantization
	QuantizationClusters []float32 `json:"quantization_clusters,omitempty"`

	// SVDComponents for SVD compression
	SVDComponents *SVDComponents `json:"svd_components,omitempty"`
}

CompressedTensor represents a compressed tensor

type CompressionErrorMetrics

type CompressionErrorMetrics struct {
	// L1Error absolute error
	L1Error float64 `json:"l1_error"`

	// L2Error mean squared error
	L2Error float64 `json:"l2_error"`

	// RelativeError relative error
	RelativeError float64 `json:"relative_error"`

	// MaxError maximum absolute error
	MaxError float64 `json:"max_error"`

	// SNR signal-to-noise ratio
	SNR float64 `json:"snr"`
}

CompressionErrorMetrics tracks compression error statistics

type CompressionPipeline

type CompressionPipeline struct {
	// contains filtered or unexported fields
}

CompressionPipeline manages the entire ML compression pipeline

func NewCompressionPipeline

func NewCompressionPipeline(config CompressionPipelineConfig) *CompressionPipeline

NewCompressionPipeline creates a new compression pipeline

func (*CompressionPipeline) CompressBatch

func (cp *CompressionPipeline) CompressBatch(ctx context.Context, gradients []*GradientTensor, tensors []*ModelTensor) ([]*CompressionResult, error)

CompressBatch compresses a batch of tensors in parallel

func (*CompressionPipeline) CompressGradient

func (cp *CompressionPipeline) CompressGradient(ctx context.Context, gradient *GradientTensor) (*CompressionResult, error)

CompressGradient compresses a gradient tensor through the pipeline

func (*CompressionPipeline) CompressTensor

func (cp *CompressionPipeline) CompressTensor(ctx context.Context, tensor *ModelTensor) (*CompressionResult, error)

CompressTensor compresses a model tensor through the pipeline

func (*CompressionPipeline) EstimateCompressionBenefit

func (cp *CompressionPipeline) EstimateCompressionBenefit(gradientSizes []int64, tensorSizes []int64) map[string]interface{}

EstimateCompressionBenefit estimates the benefit of compressing given data

func (*CompressionPipeline) GetCompressionRecommendations

func (cp *CompressionPipeline) GetCompressionRecommendations(profile WorkloadProfile) map[string]interface{}

GetCompressionRecommendations returns recommendations for optimal compression settings

func (*CompressionPipeline) GetStatistics

func (cp *CompressionPipeline) GetStatistics() *PipelineStatistics

GetStatistics returns current pipeline statistics

func (*CompressionPipeline) OptimizeConfiguration

func (cp *CompressionPipeline) OptimizeConfiguration(workloadProfile WorkloadProfile) CompressionPipelineConfig

OptimizeConfiguration optimizes the compression configuration for given workload characteristics

func (*CompressionPipeline) UpdateConfig

func (cp *CompressionPipeline) UpdateConfig(config CompressionPipelineConfig)

UpdateConfig updates the pipeline configuration

type CompressionPipelineConfig

type CompressionPipelineConfig struct {
	// EnableGradientCompression enables gradient compression
	EnableGradientCompression bool `json:"enable_gradient_compression"`

	// EnableTensorCompression enables model tensor compression
	EnableTensorCompression bool `json:"enable_tensor_compression"`

	// EnableSparsification enables sparsification algorithms
	EnableSparsification bool `json:"enable_sparsification"`

	// EnableStorageCompression enables storage-level compression
	EnableStorageCompression bool `json:"enable_storage_compression"`

	// GradientConfig configuration for gradient compression
	GradientConfig GradientCompressionConfig `json:"gradient_config"`

	// TensorConfig configuration for tensor compression
	TensorConfig TensorCompressionConfig `json:"tensor_config"`

	// SparsificationConfig configuration for sparsification
	SparsificationConfig SparsificationConfig `json:"sparsification_config"`

	// StorageConfig configuration for storage compression
	StorageConfig compression.CompressionConfig `json:"storage_config"`

	// CompressionThresholds size thresholds for applying compression
	CompressionThresholds CompressionThresholds `json:"compression_thresholds"`

	// QualitySettings quality vs compression trade-offs
	QualitySettings QualitySettings `json:"quality_settings"`

	// ParallelProcessing enables parallel compression processing
	ParallelProcessing bool `json:"parallel_processing"`

	// MaxConcurrency maximum concurrent compression operations
	MaxConcurrency int `json:"max_concurrency"`
}

CompressionPipelineConfig holds configuration for the entire compression pipeline

func DefaultCompressionPipelineConfig

func DefaultCompressionPipelineConfig() CompressionPipelineConfig

DefaultCompressionPipelineConfig returns a default pipeline configuration

type CompressionResult

type CompressionResult struct {
	// Success whether compression was successful
	Success bool `json:"success"`

	// CompressedData the compressed data (could be gradient, tensor, or sparse)
	CompressedData interface{} `json:"compressed_data"`

	// CompressionRatio achieved compression ratio
	CompressionRatio float64 `json:"compression_ratio"`

	// Method compression method used
	Method string `json:"method"`

	// ProcessingTimeMs time taken for compression
	ProcessingTimeMs int64 `json:"processing_time_ms"`

	// ErrorMessage error message if compression failed
	ErrorMessage string `json:"error_message,omitempty"`

	// Metadata additional metadata about compression
	Metadata map[string]interface{} `json:"metadata,omitempty"`
}

CompressionResult contains the result of a compression operation

type CompressionThresholds

type CompressionThresholds struct {
	// MinGradientSize minimum size in bytes to apply gradient compression
	MinGradientSize int64 `json:"min_gradient_size"`

	// MinTensorSize minimum size in bytes to apply tensor compression
	MinTensorSize int64 `json:"min_tensor_size"`

	// MinSparsificationSize minimum size to apply sparsification
	MinSparsificationSize int64 `json:"min_sparsification_size"`

	// MinStorageSize minimum size to apply storage compression
	MinStorageSize int64 `json:"min_storage_size"`
}

CompressionThresholds defines size thresholds for applying different compression methods

type DemandProjection

type DemandProjection struct {
	Timestamp       time.Time `json:"timestamp"`
	ProjectedCPU    int       `json:"projected_cpu"`
	ProjectedMemory int64     `json:"projected_memory"`
	ProjectedIOPS   int64     `json:"projected_iops"`
	Confidence      float64   `json:"confidence"`
	Scenario        string    `json:"scenario"` // best, likely, worst
}

DemandProjection projects future resource demand

type ErrorToleranceMode

type ErrorToleranceMode string

ErrorToleranceMode defines error tolerance levels

const (
	// ErrorToleranceStrict strict error bounds
	ErrorToleranceStrict ErrorToleranceMode = "strict"
	// ErrorToleranceBalanced balanced error tolerance
	ErrorToleranceBalanced ErrorToleranceMode = "balanced"
	// ErrorToleranceRelaxed relaxed error tolerance for maximum compression
	ErrorToleranceRelaxed ErrorToleranceMode = "relaxed"
)

type Feature

type Feature struct {
	Name        string    `json:"name"`
	Value       float64   `json:"value"`
	Type        string    `json:"type"` // numeric, categorical, temporal
	Description string    `json:"description"`
	Timestamp   time.Time `json:"timestamp"`
}

Feature represents an engineered feature for ML

type FeatureConfig

type FeatureConfig struct {
	EnableTemporalFeatures    bool      `json:"enable_temporal_features"`
	EnableStatisticalFeatures bool      `json:"enable_statistical_features"`
	EnableFrequencyFeatures   bool      `json:"enable_frequency_features"`
	EnableLaggedFeatures      bool      `json:"enable_lagged_features"`
	LagPeriods                []int     `json:"lag_periods"`
	RollingWindowSizes        []int     `json:"rolling_window_sizes"`
	QuantileThresholds        []float64 `json:"quantile_thresholds"`
}

FeatureConfig defines feature engineering settings

type FeatureEngine

type FeatureEngine interface {
	ExtractFeatures(data []TimeSeriesPoint) []Feature
	GetFeatureName() string
}

FeatureEngine interface for feature engineering

type GradientCompressionAlgorithm

type GradientCompressionAlgorithm string

GradientCompressionAlgorithm defines the compression algorithm for gradients

const (
	// CompressionNone - no compression
	CompressionNone GradientCompressionAlgorithm = "none"
	// CompressionTopK - top-k sparsification
	CompressionTopK GradientCompressionAlgorithm = "topk"
	// CompressionRandomK - random-k sparsification
	CompressionRandomK GradientCompressionAlgorithm = "randomk"
	// CompressionThreshold - threshold-based sparsification
	CompressionThreshold GradientCompressionAlgorithm = "threshold"
	// CompressionQuantization - gradient quantization
	CompressionQuantization GradientCompressionAlgorithm = "quantization"
	// CompressionHybrid - combination of sparsification and quantization
	CompressionHybrid GradientCompressionAlgorithm = "hybrid"
)

type GradientCompressionConfig

type GradientCompressionConfig struct {
	// Algorithm specifies the compression algorithm to use
	Algorithm GradientCompressionAlgorithm `json:"algorithm"`

	// CompressionRatio target compression ratio (0.0-1.0)
	CompressionRatio float64 `json:"compression_ratio"`

	// TopKRatio for top-k sparsification (percentage of top gradients to keep)
	TopKRatio float64 `json:"topk_ratio"`

	// SparsityThreshold for threshold-based sparsification
	SparsityThreshold float64 `json:"sparsity_threshold"`

	// QuantizationBits for gradient quantization
	QuantizationBits QuantizationBits `json:"quantization_bits"`

	// EnableAdaptive enables adaptive compression based on gradient distribution
	EnableAdaptive bool `json:"enable_adaptive"`

	// MinCompressionRatio minimum compression ratio for adaptive mode
	MinCompressionRatio float64 `json:"min_compression_ratio"`

	// MaxCompressionRatio maximum compression ratio for adaptive mode
	MaxCompressionRatio float64 `json:"max_compression_ratio"`

	// ErrorBoundCompression enable error-bounded compression
	ErrorBoundCompression bool `json:"error_bound_compression"`

	// MaxRelativeError maximum relative error for error-bounded compression
	MaxRelativeError float64 `json:"max_relative_error"`
}

GradientCompressionConfig holds configuration for gradient compression

func DefaultGradientCompressionConfig

func DefaultGradientCompressionConfig() GradientCompressionConfig

DefaultGradientCompressionConfig returns default configuration for gradient compression

type GradientCompressor

type GradientCompressor struct {
	// contains filtered or unexported fields
}

GradientCompressor handles compression and decompression of gradients

func NewGradientCompressor

func NewGradientCompressor(config GradientCompressionConfig) *GradientCompressor

NewGradientCompressor creates a new gradient compressor

func (*GradientCompressor) CompressGradient

func (gc *GradientCompressor) CompressGradient(gradient *GradientTensor) (*CompressedGradient, error)

CompressGradient compresses a gradient tensor using the configured algorithm

func (*GradientCompressor) DecompressGradient

func (gc *GradientCompressor) DecompressGradient(compressed *CompressedGradient) (*GradientTensor, error)

DecompressGradient decompresses a compressed gradient

func (*GradientCompressor) EstimateCompressionRatio

func (gc *GradientCompressor) EstimateCompressionRatio(dataSize int, config GradientCompressionConfig) float64

EstimateCompressionRatio estimates compression ratio for given configuration and data

func (*GradientCompressor) GetCompressionStats

func (gc *GradientCompressor) GetCompressionStats(compressed *CompressedGradient) map[string]interface{}

GetCompressionStats returns statistics about compression performance

func (*GradientCompressor) GetConfig

GetConfig returns the current configuration

func (*GradientCompressor) UpdateConfig

func (gc *GradientCompressor) UpdateConfig(config GradientCompressionConfig)

UpdateConfig updates the compression configuration

type GradientTensor

type GradientTensor struct {
	// Shape of the tensor (dimensions)
	Shape []int `json:"shape"`

	// Data contains the gradient values as float32
	Data []float32 `json:"data"`

	// LayerName identifies the layer this gradient belongs to
	LayerName string `json:"layer_name"`

	// Timestamp when the gradient was computed
	Timestamp int64 `json:"timestamp"`
}

GradientTensor represents a gradient tensor with shape and data

type ImpactPredictor

type ImpactPredictor struct {
	// contains filtered or unexported fields
}

ImpactPredictor predicts migration impact

func NewImpactPredictor

func NewImpactPredictor() *ImpactPredictor

NewImpactPredictor creates an impact predictor

type ImportanceMetric

type ImportanceMetric string

ImportanceMetric defines metrics for measuring parameter importance

const (
	// ImportanceMagnitude - absolute magnitude
	ImportanceMagnitude ImportanceMetric = "magnitude"
	// ImportanceGradient - gradient magnitude
	ImportanceGradient ImportanceMetric = "gradient"
	// ImportanceFisher - Fisher information
	ImportanceFisher ImportanceMetric = "fisher"
	// ImportanceHessian - Hessian diagonal
	ImportanceHessian ImportanceMetric = "hessian"
	// ImportanceSaliency - gradient * weight
	ImportanceSaliency ImportanceMetric = "saliency"
	// ImportanceTaylor - Taylor expansion
	ImportanceTaylor ImportanceMetric = "taylor"
)

type IsolationForest

type IsolationForest struct {
	// contains filtered or unexported fields
}

IsolationForest implements the Isolation Forest algorithm

func NewIsolationForest

func NewIsolationForest(numTrees, sampleSize int) *IsolationForest

NewIsolationForest creates a new Isolation Forest

func (*IsolationForest) AnomalyScore

func (iforest *IsolationForest) AnomalyScore(point []float64) float64

AnomalyScore calculates the anomaly score for a data point

func (*IsolationForest) Train

func (iforest *IsolationForest) Train(data [][]float64)

Train builds the isolation forest

type IsolationNode

type IsolationNode struct {
	// contains filtered or unexported fields
}

IsolationNode represents a node in an isolation tree

type IsolationTree

type IsolationTree struct {
	// contains filtered or unexported fields
}

IsolationTree represents a single tree in the forest

type LSTMModel

type LSTMModel struct {
	// contains filtered or unexported fields
}

LSTMModel represents the LSTM neural network

func NewLSTMModel

func NewLSTMModel(inputSize, hiddenSize, outputSize, sequenceLength int) *LSTMModel

NewLSTMModel creates a new LSTM model

func (*LSTMModel) Predict

func (lstm *LSTMModel) Predict(sequence [][]float64) []float64

Predict runs forward pass through LSTM

func (*LSTMModel) Train

func (lstm *LSTMModel) Train(sequences [][][]float64, labels [][]float64, epochs int, learningRate float64) float64

Train trains the LSTM model

type Layer

type Layer struct {
	// contains filtered or unexported fields
}

Layer represents a neural network layer

type MLDataset

type MLDataset struct {
	ID         string            `json:"id"`
	Created    time.Time         `json:"created"`
	Features   [][]Feature       `json:"features"`
	Labels     []float64         `json:"labels"`
	Metadata   map[string]string `json:"metadata"`
	WindowSize int               `json:"window_size"`
	StepSize   int               `json:"step_size"`
}

MLDataset represents a dataset ready for ML training

type MetricCollector

type MetricCollector interface {
	Collect() (PerformanceRecord, error)
}

MetricCollector interface for metric collection

type MetricType

type MetricType string

MetricType defines the type of metric being collected

const (
	MetricLatency     MetricType = "latency"
	MetricThroughput  MetricType = "throughput"
	MetricErrorRate   MetricType = "error_rate"
	MetricCPU         MetricType = "cpu_usage"
	MetricMemory      MetricType = "memory_usage"
	MetricNetwork     MetricType = "network_io"
	MetricDisk        MetricType = "disk_io"
	MetricCompression MetricType = "compression_ratio"
	MetricPrediction  MetricType = "prediction_accuracy"
	MetricConsensus   MetricType = "consensus_time"
)

type MetricsCollector

type MetricsCollector struct {
	// contains filtered or unexported fields
}

MetricsCollector collects performance metrics

func NewMetricsCollector

func NewMetricsCollector() *MetricsCollector

NewMetricsCollector creates a metrics collector

type MetricsRingBuffer

type MetricsRingBuffer struct {
	// contains filtered or unexported fields
}

MetricsRingBuffer stores recent metrics for analysis

func NewMetricsRingBuffer

func NewMetricsRingBuffer(capacity int) *MetricsRingBuffer

NewMetricsRingBuffer creates a new ring buffer for metrics

func (*MetricsRingBuffer) Add

func (mrb *MetricsRingBuffer) Add(metrics VMMetrics)

func (*MetricsRingBuffer) GetRecent

func (mrb *MetricsRingBuffer) GetRecent(vmID string, count int) []VMMetrics

type MigrationCostModel

type MigrationCostModel struct {
	// contains filtered or unexported fields
}

MigrationCostModel calculates migration costs

func NewMigrationCostModel

func NewMigrationCostModel() *MigrationCostModel

NewMigrationCostModel creates a migration cost model

type MigrationImpact

type MigrationImpact struct {
	Timestamp         time.Time     `json:"timestamp"`
	VMSize            int64         `json:"vm_size"`
	MigrationTime     time.Duration `json:"migration_time"`
	Downtime          time.Duration `json:"downtime"`
	PerformanceImpact float64       `json:"performance_impact"`
	Success           bool          `json:"success"`
}

MigrationImpact records historical migration impacts

type MigrationOptimizer

type MigrationOptimizer struct {
	// contains filtered or unexported fields
}

MigrationOptimizer optimizes VM migration timing

func NewMigrationOptimizer

func NewMigrationOptimizer() *MigrationOptimizer

NewMigrationOptimizer creates a migration optimizer

func (*MigrationOptimizer) SuggestMigration

func (mo *MigrationOptimizer) SuggestMigration(vmID string, predictions []float64) *MigrationSuggestion

type MigrationSuggestion

type MigrationSuggestion struct {
	VMId            string          `json:"vm_id"`
	SourceNode      string          `json:"source_node"`
	TargetNode      string          `json:"target_node"`
	OptimalWindow   MigrationWindow `json:"optimal_window"`
	Reason          string          `json:"reason"`
	ExpectedBenefit float64         `json:"expected_benefit"`
	EstimatedTime   time.Duration   `json:"estimated_time"`
	Prerequisites   []string        `json:"prerequisites"`
}

MigrationSuggestion suggests optimal migration timing

type MigrationWindow

type MigrationWindow struct {
	Start             time.Time `json:"start"`
	End               time.Time `json:"end"`
	Score             float64   `json:"score"`
	ExpectedImpact    float64   `json:"expected_impact"`
	ResourceAvailable bool      `json:"resource_available"`
	RiskLevel         string    `json:"risk_level"`
}

MigrationWindow represents an optimal migration window

type ModelTensor

type ModelTensor struct {
	// Name of the tensor (layer name + parameter type)
	Name string `json:"name"`

	// Shape of the tensor
	Shape []int `json:"shape"`

	// Data contains the tensor values
	Data []float32 `json:"data"`

	// Type of tensor (weight, bias, etc.)
	Type TensorType `json:"type"`

	// LayerType the layer this tensor belongs to (conv, linear, etc.)
	LayerType string `json:"layer_type"`

	// Metadata additional information about the tensor
	Metadata map[string]interface{} `json:"metadata,omitempty"`
}

ModelTensor represents a tensor from an ML model

type NeuralNetwork

type NeuralNetwork struct {
	// contains filtered or unexported fields
}

NeuralNetwork implements a feedforward neural network

func NewNeuralNetwork

func NewNeuralNetwork(layers []int) *NeuralNetwork

NewNeuralNetwork creates a new neural network

func (*NeuralNetwork) Predict

func (nn *NeuralNetwork) Predict(input []float64) []float64

Predict performs forward propagation

func (*NeuralNetwork) Train

func (nn *NeuralNetwork) Train(data [][]float64, labels [][]float64, epochs int, learningRate float64)

Train trains the neural network

type PatternOccurrence

type PatternOccurrence struct {
	PatternID  string        `json:"pattern_id"`
	Timestamp  time.Time     `json:"timestamp"`
	VMId       string        `json:"vm_id"`
	Confidence float64       `json:"confidence"`
	Duration   time.Duration `json:"duration"`
}

PatternOccurrence records when a pattern was detected

type PatternRecognizer

type PatternRecognizer struct {
	// contains filtered or unexported fields
}

PatternRecognizer identifies workload patterns

func NewPatternRecognizer

func NewPatternRecognizer() *PatternRecognizer

NewPatternRecognizer creates a pattern recognizer

func (*PatternRecognizer) IdentifyPatterns

func (pr *PatternRecognizer) IdentifyPatterns(history []PerformanceRecord) []*WorkloadPattern

type PerformanceConfig

type PerformanceConfig struct {
	ModelLayers        []int         `json:"model_layers"`         // Neural network architecture
	LearningRate       float64       `json:"learning_rate"`        // Training learning rate
	PredictionWindow   time.Duration `json:"prediction_window"`    // How far ahead to predict
	UpdateFrequency    time.Duration `json:"update_frequency"`     // Model update frequency
	AccuracyTarget     float64       `json:"accuracy_target"`      // Target accuracy (85%)
	LatencyTarget      time.Duration `json:"latency_target"`       // Target latency (250ms)
	EnableCapacityPlan bool          `json:"enable_capacity_plan"` // Enable capacity planning
	EnableMigrationOpt bool          `json:"enable_migration_opt"` // Enable migration optimization
}

PerformanceConfig configures the performance predictor

type PerformanceHistory

type PerformanceHistory struct {
	// contains filtered or unexported fields
}

PerformanceHistory stores historical performance data

func NewPerformanceHistory

func NewPerformanceHistory(capacity int) *PerformanceHistory

NewPerformanceHistory creates a performance history buffer

func (*PerformanceHistory) Add

func (ph *PerformanceHistory) Add(record PerformanceRecord)

func (*PerformanceHistory) GetAll

func (ph *PerformanceHistory) GetAll() []PerformanceRecord

func (*PerformanceHistory) GetRecent

func (ph *PerformanceHistory) GetRecent(vmID string, count int) []PerformanceRecord

type PerformancePrediction

type PerformancePrediction struct {
	Timestamp           time.Time                   `json:"timestamp"`
	VMId                string                      `json:"vm_id"`
	WorkloadType        string                      `json:"workload_type"`
	PredictedThroughput float64                     `json:"predicted_throughput"`
	PredictedLatency    time.Duration               `json:"predicted_latency"`
	PredictedIOPS       float64                     `json:"predicted_iops"`
	PredictedBandwidth  float64                     `json:"predicted_bandwidth"`
	PerformanceScore    float64                     `json:"performance_score"`
	Bottlenecks         []Bottleneck                `json:"bottlenecks"`
	Recommendations     []PerformanceRecommendation `json:"recommendations"`
	CapacityPlan        *CapacityPlan               `json:"capacity_plan,omitempty"`
	MigrationSuggestion *MigrationSuggestion        `json:"migration_suggestion,omitempty"`
	Confidence          float64                     `json:"confidence"`
	ModelAccuracy       float64                     `json:"model_accuracy"`
}

PerformancePrediction contains performance predictions

type PerformancePredictor

type PerformancePredictor struct {
	// contains filtered or unexported fields
}

PerformancePredictor implements neural network for workload performance prediction

func NewPerformancePredictor

func NewPerformancePredictor(logger *logrus.Logger, config PerformanceConfig) *PerformancePredictor

NewPerformancePredictor creates a new performance predictor

func (*PerformancePredictor) GetCapacityPlans

func (pp *PerformancePredictor) GetCapacityPlans() []*CapacityPlan

GetCapacityPlans returns current capacity plans

func (*PerformancePredictor) GetMigrationWindows

func (pp *PerformancePredictor) GetMigrationWindows() []MigrationWindow

GetMigrationWindows returns optimal migration windows

func (*PerformancePredictor) GetPredictions

func (pp *PerformancePredictor) GetPredictions() map[string]*PerformancePrediction

GetPredictions returns all current predictions

func (*PerformancePredictor) PredictPerformance

func (pp *PerformancePredictor) PredictPerformance(vmID string, workloadType string) (*PerformancePrediction, error)

PredictPerformance generates performance predictions

func (*PerformancePredictor) Start

func (pp *PerformancePredictor) Start(ctx context.Context) error

Start begins the performance prediction service

type PerformanceRecommendation

type PerformanceRecommendation struct {
	Action              string        `json:"action"`
	Priority            string        `json:"priority"`
	ExpectedImprovement float64       `json:"expected_improvement"`
	Implementation      string        `json:"implementation"`
	Cost                float64       `json:"cost"`
	TimeToImplement     time.Duration `json:"time_to_implement"`
}

PerformanceRecommendation suggests performance improvements

type PerformanceRecord

type PerformanceRecord struct {
	Timestamp   time.Time `json:"timestamp"`
	VMId        string    `json:"vm_id"`
	Throughput  float64   `json:"throughput"`
	Latency     float64   `json:"latency"`
	IOPS        float64   `json:"iops"`
	Bandwidth   float64   `json:"bandwidth"`
	CPUUsage    float64   `json:"cpu_usage"`
	MemoryUsage float64   `json:"memory_usage"`
}

PerformanceRecord represents a historical performance record

type PipelineStatistics

type PipelineStatistics struct {
	// TotalOperations total compression operations
	TotalOperations int64 `json:"total_operations"`

	// GradientOperations gradient compression operations
	GradientOperations int64 `json:"gradient_operations"`

	// TensorOperations tensor compression operations
	TensorOperations int64 `json:"tensor_operations"`

	// SparsificationOperations sparsification operations
	SparsificationOperations int64 `json:"sparsification_operations"`

	// TotalBytesProcessed total bytes processed
	TotalBytesProcessed int64 `json:"total_bytes_processed"`

	// TotalBytesCompressed total bytes after compression
	TotalBytesCompressed int64 `json:"total_bytes_compressed"`

	// AverageCompressionRatio average compression ratio achieved
	AverageCompressionRatio float64 `json:"average_compression_ratio"`

	// TotalCompressionTime total time spent on compression
	TotalCompressionTimeMs int64 `json:"total_compression_time_ms"`

	// LastUpdated timestamp of last update
	LastUpdated time.Time `json:"last_updated"`
}

PipelineStatistics tracks compression pipeline statistics

type PredictiveAllocator

type PredictiveAllocator struct {
	// contains filtered or unexported fields
}

PredictiveAllocator implements LSTM-based resource prediction

func NewPredictiveAllocator

func NewPredictiveAllocator(logger *logrus.Logger, config PredictorConfig) *PredictiveAllocator

NewPredictiveAllocator creates a new predictive resource allocator

func (*PredictiveAllocator) AddMetrics

func (pa *PredictiveAllocator) AddMetrics(metrics *ResourceMetrics)

AddMetrics adds new metrics to the buffer

func (*PredictiveAllocator) GetPredictions

func (pa *PredictiveAllocator) GetPredictions() map[string]*ResourcePrediction

GetPredictions returns current predictions for all VMs

func (*PredictiveAllocator) PredictResourceUsage

func (pa *PredictiveAllocator) PredictResourceUsage(vmID string, horizon time.Duration) (*ResourcePrediction, error)

PredictResourceUsage generates resource usage predictions

func (*PredictiveAllocator) Start

func (pa *PredictiveAllocator) Start(ctx context.Context) error

Start begins the predictive allocation service

type PredictorConfig

type PredictorConfig struct {
	PredictionHorizon time.Duration `json:"prediction_horizon"` // 15 minutes ahead
	UpdateInterval    time.Duration `json:"update_interval"`    // Model update frequency
	BufferSize        int           `json:"buffer_size"`        // Historical data points
	AccuracyTarget    float64       `json:"accuracy_target"`    // 85% accuracy target
	LatencyTarget     time.Duration `json:"latency_target"`     // 250ms latency target
	EnableAutoScaling bool          `json:"enable_auto_scaling"`
	EnablePreemptive  bool          `json:"enable_preemptive"`
	ModelVersion      string        `json:"model_version"`
}

PredictorConfig configures the predictive allocator

type ProductionDataCollector

type ProductionDataCollector struct {
	// contains filtered or unexported fields
}

ProductionDataCollector collects and processes production metrics

func NewProductionDataCollector

func NewProductionDataCollector(config *CollectorConfig) (*ProductionDataCollector, error)

NewProductionDataCollector creates a new production data collector

func (*ProductionDataCollector) GetAllDatasets

func (pdc *ProductionDataCollector) GetAllDatasets() map[string]*MLDataset

GetAllDatasets returns all available datasets

func (*ProductionDataCollector) GetDataset

func (pdc *ProductionDataCollector) GetDataset(datasetID string) (*MLDataset, error)

GetDataset returns a specific dataset by ID

func (*ProductionDataCollector) GetMetrics

func (pdc *ProductionDataCollector) GetMetrics() map[string]float64

GetMetrics returns current collector metrics

func (*ProductionDataCollector) Start

func (pdc *ProductionDataCollector) Start(ctx context.Context) error

Start begins data collection

func (*ProductionDataCollector) Stop

func (pdc *ProductionDataCollector) Stop() error

Stop halts data collection

type PruningSchedule

type PruningSchedule struct {
	// InitialSparsity starting sparsity ratio
	InitialSparsity float64 `json:"initial_sparsity"`

	// FinalSparsity target final sparsity ratio
	FinalSparsity float64 `json:"final_sparsity"`

	// PruningFrequency how often to apply pruning (in epochs)
	PruningFrequency int `json:"pruning_frequency"`

	// Schedule type (linear, polynomial, exponential)
	Schedule string `json:"schedule"`

	// PolynomialPower for polynomial schedule
	PolynomialPower float64 `json:"polynomial_power,omitempty"`
}

PruningSchedule defines the schedule for iterative pruning

type QualitySettings

type QualitySettings struct {
	// QualityLevel overall quality level (0.0-1.0)
	QualityLevel float64 `json:"quality_level"`

	// PreserveAccuracy prioritize accuracy over compression ratio
	PreserveAccuracy bool `json:"preserve_accuracy"`

	// MaxAccuracyLoss maximum acceptable accuracy loss
	MaxAccuracyLoss float64 `json:"max_accuracy_loss"`

	// AdaptiveCompression adjust compression based on importance
	AdaptiveCompression bool `json:"adaptive_compression"`

	// ErrorToleranceMode error tolerance for compression
	ErrorToleranceMode ErrorToleranceMode `json:"error_tolerance_mode"`
}

QualitySettings defines quality vs compression trade-offs

type QuantizationBits

type QuantizationBits uint8

QuantizationBits defines the bit precision for quantization

const (
	// Bits32 - FP32 (no quantization)
	Bits32 QuantizationBits = 32
	// Bits16 - FP16 quantization
	Bits16 QuantizationBits = 16
	// Bits8 - INT8 quantization
	Bits8 QuantizationBits = 8
	// Bits4 - 4-bit quantization (experimental)
	Bits4 QuantizationBits = 4
)

type ResourceMetrics

type ResourceMetrics struct {
	Timestamp    time.Time `json:"timestamp"`
	VMId         string    `json:"vm_id"`
	CPUUsage     float64   `json:"cpu_usage"`
	MemoryUsage  float64   `json:"memory_usage"`
	IOUsage      float64   `json:"io_usage"`
	NetworkUsage float64   `json:"network_usage"`
}

ResourceMetrics represents current resource usage

type ResourcePrediction

type ResourcePrediction struct {
	Timestamp         time.Time               `json:"timestamp"`
	VMId              string                  `json:"vm_id"`
	PredictedCPU      float64                 `json:"predicted_cpu"`
	PredictedMemory   float64                 `json:"predicted_memory"`
	PredictedIO       float64                 `json:"predicted_io"`
	PredictedNetwork  float64                 `json:"predicted_network"`
	Confidence        float64                 `json:"confidence"`
	Recommendations   []ScalingRecommendation `json:"recommendations"`
	PredictionLatency time.Duration           `json:"prediction_latency"`
	Accuracy          float64                 `json:"accuracy"`
}

ResourcePrediction contains predicted resource usage

type ResourceProfile

type ResourceProfile struct {
	CPUPattern     []float64 `json:"cpu_pattern"`
	MemoryPattern  []float64 `json:"memory_pattern"`
	IOPattern      []float64 `json:"io_pattern"`
	NetworkPattern []float64 `json:"network_pattern"`
}

ResourceProfile defines resource usage patterns

type SVDComponents

type SVDComponents struct {
	// U matrix (left singular vectors)
	U []float32 `json:"u"`
	// S vector (singular values)
	S []float32 `json:"s"`
	// VT matrix (right singular vectors transposed)
	VT []float32 `json:"vt"`
	// UShape shape of U matrix
	UShape []int `json:"u_shape"`
	// VTShape shape of VT matrix
	VTShape []int `json:"vt_shape"`
	// Rank preserved rank
	Rank int `json:"rank"`
}

SVDComponents stores SVD decomposition components

type ScalingRecommendation

type ScalingRecommendation struct {
	Action          string        `json:"action"`    // scale_up, scale_down, migrate
	Target          string        `json:"target"`    // resource or VM ID
	Magnitude       float64       `json:"magnitude"` // scaling factor
	Urgency         string        `json:"urgency"`   // immediate, scheduled, optional
	Reason          string        `json:"reason"`
	EstimatedImpact float64       `json:"estimated_impact"`
	TimeWindow      time.Duration `json:"time_window"`
}

ScalingRecommendation suggests scaling actions

type SparseData

type SparseData struct {
	// Indices of non-zero elements
	Indices []int32 `json:"indices"`

	// Values of non-zero elements
	Values []float32 `json:"values"`

	// Shape of the original dense tensor
	Shape []int `json:"shape"`

	// SparsityLevel achieved sparsity level
	SparsityLevel float64 `json:"sparsity_level"`

	// Algorithm used for sparsification
	Algorithm SparsificationAlgorithm `json:"algorithm"`

	// Metadata additional information
	Metadata map[string]interface{} `json:"metadata,omitempty"`
}

SparseData represents sparse data structure

type SparsificationAlgorithm

type SparsificationAlgorithm string

SparsificationAlgorithm defines algorithms for creating sparse data structures

const (
	// SparsificationMagnitude - magnitude-based pruning
	SparsificationMagnitude SparsificationAlgorithm = "magnitude"
	// SparsificationGradient - gradient-based pruning
	SparsificationGradient SparsificationAlgorithm = "gradient"
	// SparsificationRandom - random pruning
	SparsificationRandom SparsificationAlgorithm = "random"
	// SparsificationStructured - structured pruning
	SparsificationStructured SparsificationAlgorithm = "structured"
	// SparsificationSNIP - Single-shot Network Pruning
	SparsificationSNIP SparsificationAlgorithm = "snip"
	// SparsificationGraSP - Gradient Signal Preservation
	SparsificationGraSP SparsificationAlgorithm = "grasp"
	// SparsificationLottery - Lottery Ticket Hypothesis
	SparsificationLottery SparsificationAlgorithm = "lottery"
	// SparsificationAdaptive - adaptive sparsification
	SparsificationAdaptive SparsificationAlgorithm = "adaptive"
)

type SparsificationConfig

type SparsificationConfig struct {
	// Algorithm specifies the sparsification algorithm
	Algorithm SparsificationAlgorithm `json:"algorithm"`

	// SparsityRatio target sparsity ratio (0.0-1.0)
	SparsityRatio float64 `json:"sparsity_ratio"`

	// Granularity of sparsification
	Granularity SparsificationGranularity `json:"granularity"`

	// BlockSize for block-wise sparsification
	BlockSize []int `json:"block_size,omitempty"`

	// PruningSchedule for iterative pruning
	PruningSchedule *PruningSchedule `json:"pruning_schedule,omitempty"`

	// GradientAccumulation steps for gradient-based pruning
	GradientAccumulation int `json:"gradient_accumulation"`

	// ImportanceMetric for importance-based pruning
	ImportanceMetric ImportanceMetric `json:"importance_metric"`

	// AdaptiveThreshold enables adaptive threshold adjustment
	AdaptiveThreshold bool `json:"adaptive_threshold"`

	// MinSparsity minimum sparsity level
	MinSparsity float64 `json:"min_sparsity"`

	// MaxSparsity maximum sparsity level
	MaxSparsity float64 `json:"max_sparsity"`

	// RecoveryEpochs epochs to recover after aggressive pruning
	RecoveryEpochs int `json:"recovery_epochs"`

	// PreserveStructure whether to maintain structured sparsity
	PreserveStructure bool `json:"preserve_structure"`
}

SparsificationConfig holds configuration for sparsification algorithms

func DefaultSparsificationConfig

func DefaultSparsificationConfig() SparsificationConfig

DefaultSparsificationConfig returns a default sparsification configuration

type SparsificationGranularity

type SparsificationGranularity string

SparsificationGranularity defines the granularity of sparsification

const (
	// GranularityElement - element-wise sparsification
	GranularityElement SparsificationGranularity = "element"
	// GranularityChannel - channel-wise sparsification
	GranularityChannel SparsificationGranularity = "channel"
	// GranularityFilter - filter-wise sparsification
	GranularityFilter SparsificationGranularity = "filter"
	// GranularityBlock - block-wise sparsification
	GranularityBlock SparsificationGranularity = "block"
	// GranularityLayer - layer-wise sparsification
	GranularityLayer SparsificationGranularity = "layer"
)

type SparsificationMetrics

type SparsificationMetrics struct {
	// OriginalSize number of parameters before sparsification
	OriginalSize int `json:"original_size"`

	// NonZeroCount number of non-zero parameters after sparsification
	NonZeroCount int `json:"non_zero_count"`

	// SparsityLevel achieved sparsity level
	SparsityLevel float64 `json:"sparsity_level"`

	// CompressionRatio memory compression achieved
	CompressionRatio float64 `json:"compression_ratio"`

	// FlopsReduction computational reduction
	FlopsReduction float64 `json:"flops_reduction"`

	// AccuracyDrop estimated accuracy drop
	AccuracyDrop float64 `json:"accuracy_drop,omitempty"`

	// SparsificationTime time taken to sparsify
	SparsificationTimeMs int64 `json:"sparsification_time_ms"`
}

SparsificationMetrics tracks sparsification performance

type Sparsifier

type Sparsifier struct {
	// contains filtered or unexported fields
}

Sparsifier implements sparsification algorithms for neural networks

func NewSparsifier

func NewSparsifier(config SparsificationConfig) *Sparsifier

NewSparsifier creates a new sparsifier with the given configuration

func (*Sparsifier) CompareSparsificationMethods

func (s *Sparsifier) CompareSparsificationMethods(tensor *GradientTensor, layerName string) map[SparsificationAlgorithm]map[string]interface{}

CompareSparsificationMethods compares different sparsification methods

func (*Sparsifier) DensifyTensor

func (s *Sparsifier) DensifyTensor(sparse *SparseData) (*GradientTensor, error)

DensifyTensor converts sparse data back to dense tensor

func (*Sparsifier) EstimateSparsificationBenefit

func (s *Sparsifier) EstimateSparsificationBenefit(tensorSize int, config SparsificationConfig) map[string]float64

EstimateSparsificationBenefit estimates the benefit of sparsification

func (*Sparsifier) GetSparsificationStats

func (s *Sparsifier) GetSparsificationStats(sparse *SparseData, metrics *SparsificationMetrics) map[string]interface{}

GetSparsificationStats returns statistics about sparsification

func (*Sparsifier) SparsifyTensor

func (s *Sparsifier) SparsifyTensor(tensor *GradientTensor, layerName string) (*SparseData, *SparsificationMetrics, error)

SparsifyTensor applies sparsification to a tensor

func (*Sparsifier) UpdateConfig

func (s *Sparsifier) UpdateConfig(config SparsificationConfig)

UpdateConfig updates the sparsification configuration

type SystemCapacity

type SystemCapacity struct {
	TotalCPU     int   `json:"total_cpu"`
	TotalMemory  int64 `json:"total_memory"`
	TotalStorage int64 `json:"total_storage"`
	TotalNetwork int64 `json:"total_network"`
	UsedCPU      int   `json:"used_cpu"`
	UsedMemory   int64 `json:"used_memory"`
	UsedStorage  int64 `json:"used_storage"`
	UsedNetwork  int64 `json:"used_network"`
	VMCount      int   `json:"vm_count"`
	NodeCount    int   `json:"node_count"`
}

SystemCapacity represents current system capacity

type TensorCompressionAlgorithm

type TensorCompressionAlgorithm string

TensorCompressionAlgorithm defines compression algorithms for ML tensors

const (
	// TensorCompressionNone - no compression
	TensorCompressionNone TensorCompressionAlgorithm = "none"
	// TensorCompressionSVD - Singular Value Decomposition
	TensorCompressionSVD TensorCompressionAlgorithm = "svd"
	// TensorCompressionPruning - Weight pruning
	TensorCompressionPruning TensorCompressionAlgorithm = "pruning"
	// TensorCompressionQuantization - Parameter quantization
	TensorCompressionQuantization TensorCompressionAlgorithm = "quantization"
	// TensorCompressionHuffman - Huffman coding for weights
	TensorCompressionHuffman TensorCompressionAlgorithm = "huffman"
	// TensorCompressionKMeans - K-means clustering quantization
	TensorCompressionKMeans TensorCompressionAlgorithm = "kmeans"
	// TensorCompressionLowRank - Low-rank approximation
	TensorCompressionLowRank TensorCompressionAlgorithm = "lowrank"
	// TensorCompressionBitpacking - Bit-level packing
	TensorCompressionBitpacking TensorCompressionAlgorithm = "bitpacking"
)

type TensorCompressionConfig

type TensorCompressionConfig struct {
	Algorithm        TensorCompressionAlgorithm `json:"algorithm"`
	CompressionRatio float64                    `json:"compression_ratio"`
	QuantizationBits QuantizationBits           `json:"quantization_bits"`
}

TensorCompressionConfig simplified tensor compression configuration

type TensorCompressionMetrics

type TensorCompressionMetrics struct {
	// OriginalSize in bytes
	OriginalSize int `json:"original_size"`

	// CompressedSize in bytes
	CompressedSize int `json:"compressed_size"`

	// CompressionRatio actual achieved ratio
	CompressionRatio float64 `json:"compression_ratio"`

	// ParameterReduction percentage of parameters reduced
	ParameterReduction float64 `json:"parameter_reduction"`

	// AccuracyLoss estimated accuracy loss from compression
	AccuracyLoss float64 `json:"accuracy_loss,omitempty"`

	// CompressionTime time taken to compress
	CompressionTimeMs int64 `json:"compression_time_ms"`

	// DecompressionTime time taken to decompress
	DecompressionTimeMs int64 `json:"decompression_time_ms"`

	// ErrorMetrics compression error statistics
	ErrorMetrics *CompressionErrorMetrics `json:"error_metrics,omitempty"`
}

TensorCompressionMetrics tracks detailed compression statistics

type TensorCompressor

type TensorCompressor struct {
	// contains filtered or unexported fields
}

TensorCompressor handles compression and decompression of ML tensors

func NewTensorCompressor

func NewTensorCompressor(config GradientCompressionConfig) *TensorCompressor

NewTensorCompressor creates a new tensor compressor

func (*TensorCompressor) CompressTensor

func (tc *TensorCompressor) CompressTensor(tensor *ModelTensor, algorithm TensorCompressionAlgorithm) (*CompressedTensor, error)

CompressTensor compresses a model tensor using the configured algorithm

func (*TensorCompressor) DecompressTensor

func (tc *TensorCompressor) DecompressTensor(compressed *CompressedTensor) (*ModelTensor, error)

DecompressTensor decompresses a compressed tensor

func (*TensorCompressor) GetTensorCompressionStats

func (tc *TensorCompressor) GetTensorCompressionStats(compressed *CompressedTensor) map[string]interface{}

GetTensorCompressionStats returns compression statistics for a tensor

func (*TensorCompressor) UpdateConfig

func (tc *TensorCompressor) UpdateConfig(config GradientCompressionConfig)

UpdateConfig updates the compression configuration

type TensorType

type TensorType string

TensorType defines the type of tensor being compressed

const (
	// TensorTypeWeight - Model weights/parameters
	TensorTypeWeight TensorType = "weight"
	// TensorTypeBias - Bias parameters
	TensorTypeBias TensorType = "bias"
	// TensorTypeActivation - Activation tensors
	TensorTypeActivation TensorType = "activation"
	// TensorTypeEmbedding - Embedding matrices
	TensorTypeEmbedding TensorType = "embedding"
	// TensorTypeConvKernel - Convolutional kernels
	TensorTypeConvKernel TensorType = "conv_kernel"
	// TensorTypeAttentionWeight - Attention weights
	TensorTypeAttentionWeight TensorType = "attention_weight"
)

type ThresholdInfo

type ThresholdInfo struct {
	Current        float64   `json:"current"`
	Historical     []float64 `json:"historical"`
	LastUpdated    time.Time `json:"last_updated"`
	Confidence     float64   `json:"confidence"`
	TruePositives  int       `json:"true_positives"`
	FalsePositives int       `json:"false_positives"`
}

ThresholdInfo stores threshold information for a metric

type TimeSeriesBuffer

type TimeSeriesBuffer struct {
	// contains filtered or unexported fields
}

TimeSeriesBuffer stores historical metrics

func NewTimeSeriesBuffer

func NewTimeSeriesBuffer(capacity int) *TimeSeriesBuffer

NewTimeSeriesBuffer creates a new circular buffer for time series data

func (*TimeSeriesBuffer) Add

func (tsb *TimeSeriesBuffer) Add(data []float64, timestamp time.Time)

func (*TimeSeriesBuffer) Cleanup

func (tsb *TimeSeriesBuffer) Cleanup()

func (*TimeSeriesBuffer) GetAll

func (tsb *TimeSeriesBuffer) GetAll() [][]float64

func (*TimeSeriesBuffer) GetSequence

func (tsb *TimeSeriesBuffer) GetSequence(length int) [][]float64

type TimeSeriesPoint

type TimeSeriesPoint struct {
	Timestamp time.Time              `json:"timestamp"`
	Value     float64                `json:"value"`
	Tags      map[string]string      `json:"tags"`
	Fields    map[string]interface{} `json:"fields"`
}

TimeSeriesPoint represents a single data point in time series

type VMMetrics

type VMMetrics struct {
	Timestamp     time.Time          `json:"timestamp"`
	VMId          string             `json:"vm_id"`
	CPU           float64            `json:"cpu"`
	Memory        float64            `json:"memory"`
	DiskIO        float64            `json:"disk_io"`
	NetworkIO     float64            `json:"network_io"`
	Latency       float64            `json:"latency"`
	ErrorRate     float64            `json:"error_rate"`
	CustomMetrics map[string]float64 `json:"custom_metrics"`
}

VMMetrics represents VM performance metrics

type WorkloadPattern

type WorkloadPattern struct {
	ID              string             `json:"id"`
	Name            string             `json:"name"`
	Type            string             `json:"type"` // periodic, burst, steady, growing
	Characteristics map[string]float64 `json:"characteristics"`
	Frequency       time.Duration      `json:"frequency"`
	Duration        time.Duration      `json:"duration"`
	ResourceProfile ResourceProfile    `json:"resource_profile"`
	LastSeen        time.Time          `json:"last_seen"`
	Occurrences     int                `json:"occurrences"`
}

WorkloadPattern represents a recognized workload pattern

type WorkloadProfile

type WorkloadProfile struct {
	ModelType         string  `json:"model_type"`         // transformer, cnn, mlp, etc.
	ModelSize         int64   `json:"model_size"`         // Model size in parameters
	BatchSize         int     `json:"batch_size"`         // Training batch size
	MemoryConstrained bool    `json:"memory_constrained"` // Whether memory is limited
	SpeedCritical     bool    `json:"speed_critical"`     // Whether speed is critical
	AccuracyTarget    float64 `json:"accuracy_target"`    // Target accuracy to maintain
}

WorkloadProfile describes characteristics of the ML workload

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL