Documentation
¶
Overview ¶
Package ml provides machine learning capabilities for anomaly detection in WAF traffic. This implementation uses only the Go standard library for pure statistical analysis without external ML dependencies.
The anomaly detection system uses: - Moving averages for baseline establishment - Z-score calculation for statistical anomaly detection - Shannon entropy for content analysis - Time-series analysis for pattern detection
Example usage:
config := ml.Config{
Sensitivity: ml.Medium,
WindowSize: 100,
ZThreshold: 3.0,
}
detector := ml.New(config)
detector.RecordTraffic(ml.TrafficSample{
Timestamp: time.Now(),
Volume: 100,
Size: 1024,
})
anomaly, detected := detector.AnalyzeRequest("GET", "/api/data", 2048)
Index ¶
- Variables
- func RecordBehavioralMetrics(result *BehavioralResult, clientID string, duration time.Duration)
- func RecordContentMetrics(result *AnalysisResult, sensitivity string, duration time.Duration)
- func RecordMLMetrics(metricsType, sensitivity string, duration time.Duration, blocked bool)
- func RecordPromptInjectionMetrics(detection *DetectionResult, sensitivity string, duration time.Duration)
- type AnalysisResult
- type Anomaly
- type AnomalyType
- type Baseline
- type BehavioralAnalyzer
- type BehavioralResult
- type BehavioralStats
- type ClientBehavior
- type CombinedDetector
- type CombinedResult
- type Config
- type ContentAnalysisStats
- type ContentAnalyzer
- type ContentRule
- type ContextDetectionResult
- type ContextManipulationDetector
- type ContextManipulationStats
- type ContextPattern
- type DetectionResult
- type Detector
- func (d *Detector) AnalyzeContent(content []byte) (float64, *Anomaly)
- func (d *Detector) AnalyzePatterns(input string) []Anomaly
- func (d *Detector) AnalyzeRequest(method, path string, size int64) (Anomaly, bool)
- func (d *Detector) ClearAnomalies()
- func (d *Detector) GetAnomalies() []Anomaly
- func (d *Detector) GetBaselineStats() map[string]interface{}
- func (d *Detector) GetMethodsDistribution() map[string]float64
- func (d *Detector) GetRecentAnomalies(since time.Duration) []Anomaly
- func (d *Detector) GetTopPaths(n int) []struct{ ... }
- func (d *Detector) IsReady() bool
- func (d *Detector) RecordTraffic(sample TrafficSample) *Anomaly
- func (d *Detector) Reset()
- type InjectionPattern
- type MetricsExporter
- func (e *MetricsExporter) RecordAnalysisDuration(analysisType string, duration time.Duration)
- func (e *MetricsExporter) RecordAnomaly(anomalyType string, severity int)
- func (e *MetricsExporter) RecordBehavioralAnalysis(result *BehavioralResult, activeClients int)
- func (e *MetricsExporter) RecordContentAnalysis(result *AnalysisResult)
- func (e *MetricsExporter) RecordPromptInjection(detection *DetectionResult, blocked bool)
- func (e *MetricsExporter) RecordRequest(analyzed, blocked bool)
- func (e *MetricsExporter) SetSensitivity(sensitivity string)
- func (e *MetricsExporter) UpdateFromStats(stats map[string]interface{})
- type PromptInjectionDetector
- type PromptInjectionStats
- type Sensitivity
- type Severity
- type TokenDetectionResult
- type TokenPattern
- type TokenSmugglingDetector
- type TokenSmugglingStats
- type TrafficSample
- type UnicodeAttackDetector
- type UnicodeAttackStats
- type UnicodeDetectionResult
- type UnicodePattern
Constants ¶
This section is empty.
Variables ¶
var ( // Middleware metrics MLTotalRequests = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "aegisgate_ml_total_requests_total", Help: "Total number of requests processed by ML middleware", }, []string{"sensitivity"}, ) MLAnalyzedRequests = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "aegisgate_ml_analyzed_requests_total", Help: "Total number of requests analyzed by ML middleware", }, []string{"sensitivity"}, ) MLBlockedRequests = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "aegisgate_ml_blocked_requests_total", Help: "Total number of requests blocked by ML middleware", }, []string{"sensitivity", "reason"}, ) MLAnomalyDetections = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "aegisgate_ml_anomaly_detections_total", Help: "Total number of anomaly detections", }, []string{"type", "severity", "blocked"}, ) // Prompt injection metrics MLPromptInjectionScanned = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "aegisgate_ml_prompt_injection_scanned_total", Help: "Total number of prompts scanned for injection", }, []string{"sensitivity"}, ) MLPromptInjectionDetections = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "aegisgate_ml_prompt_injection_detections_total", Help: "Total number of prompt injection detections", }, []string{"sensitivity", "pattern", "severity"}, ) // Content analysis metrics MLContentAnalyzed = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "aegisgate_ml_content_analyzed_total", Help: "Total number of content items analyzed", }, []string{"sensitivity"}, ) MLContentViolations = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "aegisgate_ml_content_violations_total", Help: "Total number of content policy violations", }, []string{"sensitivity", "type"}, ) // Behavioral metrics MLBehavioralClients = promauto.NewGaugeVec( prometheus.GaugeOpts{ Name: "aegisgate_ml_behavioral_clients_active", Help: "Number of active clients being tracked", }, []string{}, ) MLBehavioralAnomalies = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "aegisgate_ml_behavioral_anomalies_total", Help: "Total number of behavioral anomalies", }, []string{"type", "client"}, ) // Latency metrics MLAnalysisDuration = promauto.NewHistogramVec( prometheus.HistogramOpts{ Name: "aegisgate_ml_analysis_duration_seconds", Help: "Time spent on ML analysis", Buckets: []float64{.0001, .0005, .001, .005, .01, .05, .1, .5, 1}, }, []string{"type"}, ) )
PrometheusMetrics returns all ML-related Prometheus metrics
Functions ¶
func RecordBehavioralMetrics ¶
func RecordBehavioralMetrics(result *BehavioralResult, clientID string, duration time.Duration)
RecordBehavioralMetrics records behavioral analysis metrics
func RecordContentMetrics ¶
func RecordContentMetrics(result *AnalysisResult, sensitivity string, duration time.Duration)
RecordContentMetrics records content analysis metrics
func RecordMLMetrics ¶
RecordMLMetrics records metrics for a single ML analysis
func RecordPromptInjectionMetrics ¶
func RecordPromptInjectionMetrics(detection *DetectionResult, sensitivity string, duration time.Duration)
RecordPromptInjectionMetrics records prompt injection specific metrics
Types ¶
type AnalysisResult ¶
type AnalysisResult struct {
IsViolation bool
Score float64
ViolationTypes []string
RedactedContent string
Severity int
}
AnalysisResult represents the result of content analysis
type Anomaly ¶
type Anomaly struct {
// Type categorizes the anomaly (traffic_spike, entropy_anomaly, etc.)
Type AnomalyType
// Severity indicates the importance level (1-5)
Severity Severity
// Score is the statistical z-score indicating how many
// standard deviations from the mean this value is
Score float64
// Timestamp when the anomaly was detected
Timestamp time.Time
// Evidence contains additional context about the anomaly
Evidence map[string]interface{}
// Description provides a human-readable explanation
Description string
}
Anomaly represents a detected statistical anomaly in traffic patterns. Contains all details needed for evaluation and response.
type AnomalyType ¶
type AnomalyType string
AnomalyType categorizes the type of anomaly detected. These types help in understanding the nature of suspicious activity.
const ( // TrafficSpike indicates an unusual increase in request volume. // May indicate DDoS attack or sudden legitimate traffic surge. TrafficSpike AnomalyType = "traffic_spike" // TrafficDrop indicates an unusual decrease in request volume. // May indicate service disruption or blocked legitimate traffic. TrafficDrop AnomalyType = "traffic_drop" // SizeAnomaly indicates unusual request/response sizes. // May indicate data exfiltration, large uploads, or buffer overflow attempts. SizeAnomaly AnomalyType = "size_anomaly" // ViolationSpike indicates increased security rule violations. // May indicate attack attempts or scanning activity. ViolationSpike AnomalyType = "violation_spike" // EntropyAnomaly indicates unusual entropy in request content. // May indicate encoded payloads, encrypted traffic, or obfuscated attacks. EntropyAnomaly AnomalyType = "entropy_anomaly" // PatternAnomaly indicates unusual request patterns. // May indicate automated tools, bots, or scripted attacks. PatternAnomaly AnomalyType = "pattern_anomaly" // TimeAnomaly indicates unusual timing patterns. // May indicate automated requests or scheduled attack patterns. TimeAnomaly AnomalyType = "time_anomaly" )
type Baseline ¶
type Baseline struct {
// contains filtered or unexported fields
}
Baseline tracks moving averages and standard deviations for metrics. Uses a sliding window approach for real-time adaptation to traffic patterns.
type BehavioralAnalyzer ¶
type BehavioralAnalyzer struct {
// contains filtered or unexported fields
}
BehavioralAnalyzer analyzes behavioral patterns for anomalies
func NewBehavioralAnalyzer ¶
func NewBehavioralAnalyzer() *BehavioralAnalyzer
NewBehavioralAnalyzer creates a new behavioral analyzer
func (*BehavioralAnalyzer) AnalyzeRequest ¶
func (ba *BehavioralAnalyzer) AnalyzeRequest(clientID, method, path string, bytesSent int64) *BehavioralResult
AnalyzeRequest analyzes a request for behavioral anomalies
func (*BehavioralAnalyzer) GetStats ¶
func (ba *BehavioralAnalyzer) GetStats() map[string]interface{}
GetStats returns behavioral analysis statistics
func (*BehavioralAnalyzer) Reset ¶
func (ba *BehavioralAnalyzer) Reset()
Reset clears behavioral analysis statistics
type BehavioralResult ¶
BehavioralResult represents the result of behavioral analysis
type BehavioralStats ¶
type BehavioralStats struct {
TotalClients int64
AnomalousClients int64
TotalAnomalies int64
ByType map[string]int64
// contains filtered or unexported fields
}
BehavioralStats holds behavioral analysis statistics
type ClientBehavior ¶
type ClientBehavior struct {
RequestTimestamps []time.Time
PathFrequencies map[string]int
MethodCounts map[string]int
TotalRequests int
BytesSent int64
FirstSeen time.Time
LastSeen time.Time
AnomalyScore float64
}
ClientBehavior holds behavior data for a client
type CombinedDetector ¶
type CombinedDetector struct {
PromptInjection *PromptInjectionDetector
TokenSmuggling *TokenSmugglingDetector
UnicodeAttack *UnicodeAttackDetector
ContextManipulation *ContextManipulationDetector
}
CombinedDetector provides unified detection across all attack patterns
func NewCombinedDetector ¶
func NewCombinedDetector(sensitivity int) *CombinedDetector
NewCombinedDetector creates a new combined detector with all sub-detectors
func (*CombinedDetector) Detect ¶
func (cd *CombinedDetector) Detect(content string) *CombinedResult
Detect analyzes content across all detection mechanisms
func (*CombinedDetector) GetAllStats ¶
func (cd *CombinedDetector) GetAllStats() map[string]interface{}
GetAllStats returns combined statistics from all detectors
type CombinedResult ¶
type CombinedResult struct {
IsThreat bool
TotalScore float64
PromptInjectionScore float64
TokenSmugglingScore float64
UnicodeAttackScore float64
ContextScore float64
AllMatchedPatterns []string
HighestSeverity int
}
CombinedResult represents the combined detection result
type Config ¶
type Config struct {
// Sensitivity determines the z-score threshold for anomaly detection
Sensitivity Sensitivity
// WindowSize is the number of samples to consider for baseline
WindowSize int
// ZThreshold can override the default threshold based on sensitivity
ZThreshold float64
// MinSamples is the minimum samples needed before detecting anomalies
MinSamples int
// EntropyThreshold for content analysis (0-1, higher = more random)
EntropyThreshold float64
}
Config provides configuration options for the anomaly detector.
type ContentAnalysisStats ¶
type ContentAnalysisStats struct {
TotalAnalyzed int64
ViolationsFound int64
ByType map[string]int64
LastViolation time.Time
// contains filtered or unexported fields
}
ContentAnalysisStats holds analysis statistics
type ContentAnalyzer ¶
type ContentAnalyzer struct {
// contains filtered or unexported fields
}
ContentAnalyzer analyzes LLM responses for policy violations
func NewContentAnalyzer ¶
func NewContentAnalyzer() *ContentAnalyzer
NewContentAnalyzer creates a new content analyzer
func (*ContentAnalyzer) AddRule ¶
func (a *ContentAnalyzer) AddRule(rule ContentRule)
AddRule adds a custom content analysis rule
func (*ContentAnalyzer) Analyze ¶
func (a *ContentAnalyzer) Analyze(content string) *AnalysisResult
Analyze analyzes content for policy violations
func (*ContentAnalyzer) GetStats ¶
func (a *ContentAnalyzer) GetStats() map[string]interface{}
GetStats returns analysis statistics
type ContentRule ¶
type ContentRule struct {
Name string
Pattern *regexp.Regexp
Severity int
Action string // "block", "alert", "redact"
}
ContentRule represents a custom content analysis rule
type ContextDetectionResult ¶
type ContextDetectionResult struct {
IsManipulation bool
Score float64
MatchedTypes []string
Severity int
Description string
}
ContextDetectionResult represents the result of context manipulation detection
type ContextManipulationDetector ¶
type ContextManipulationDetector struct {
// contains filtered or unexported fields
}
ContextManipulationDetector detects attempts to manipulate conversation context
func NewContextManipulationDetector ¶
func NewContextManipulationDetector(sensitivity int) *ContextManipulationDetector
NewContextManipulationDetector creates a new context manipulation detector
func (*ContextManipulationDetector) Detect ¶
func (d *ContextManipulationDetector) Detect(content string) *ContextDetectionResult
Detect analyzes content for context manipulation attempts
func (*ContextManipulationDetector) GetContextStats ¶
func (d *ContextManipulationDetector) GetContextStats() map[string]interface{}
GetContextStats returns detection statistics
type ContextManipulationStats ¶
type ContextManipulationStats struct {
TotalScanned int64
ThreatsDetected int64
BlockedCount int64
ByPattern map[string]int64
LastDetection time.Time
// contains filtered or unexported fields
}
ContextManipulationStats holds detection statistics
type ContextPattern ¶
type ContextPattern struct {
Name string
Regex *regexp.Regexp
Severity int
Weight float64
Description string
}
ContextPattern represents a context manipulation pattern
type DetectionResult ¶
type DetectionResult struct {
IsInjection bool
Score float64 // 0-100
MatchedPatterns []string
Severity int
Explanation string
}
DetectionResult represents the result of prompt injection detection
type Detector ¶
type Detector struct {
// contains filtered or unexported fields
}
Detector is the main anomaly detection engine. Supports concurrent access and real-time analysis.
func New ¶
New creates a new anomaly detector with the specified configuration. Returns a fully initialized Detector ready for use.
func (*Detector) AnalyzeContent ¶
AnalyzeContent calculates entropy and detects anomalies in content. Returns the entropy value and any detected anomaly.
func (*Detector) AnalyzePatterns ¶
AnalyzePatterns performs pattern-based anomaly detection on strings. Detects suspicious patterns like encoding sequences, special char clusters, etc.
func (*Detector) AnalyzeRequest ¶
AnalyzeRequest evaluates a single HTTP request for anomalies. Analyzes method, path, and size against established baselines.
func (*Detector) ClearAnomalies ¶
func (d *Detector) ClearAnomalies()
ClearAnomalies removes all stored anomalies.
func (*Detector) GetAnomalies ¶
GetAnomalies returns all detected anomalies. Thread-safe access to the anomaly history.
func (*Detector) GetBaselineStats ¶
GetBaselineStats returns current baseline statistics for debugging/monitoring.
func (*Detector) GetMethodsDistribution ¶
GetMethodsDistribution returns the current distribution of HTTP methods.
func (*Detector) GetRecentAnomalies ¶
GetRecentAnomalies returns anomalies from the specified duration.
func (*Detector) GetTopPaths ¶
GetTopPaths returns the most frequently accessed paths.
func (*Detector) IsReady ¶
IsReady returns true if the detector has enough samples for anomaly detection.
func (*Detector) RecordTraffic ¶
func (d *Detector) RecordTraffic(sample TrafficSample) *Anomaly
RecordTraffic adds a traffic sample and updates the moving baseline. Returns an anomaly if the sample is statistically anomalous.
type InjectionPattern ¶
type InjectionPattern struct {
Name string
Regex *regexp.Regexp
Severity int // 1-5
Weight float64
}
InjectionPattern represents a known prompt injection pattern
type MetricsExporter ¶
type MetricsExporter struct {
// contains filtered or unexported fields
}
MetricsExporter exports ML metrics to Prometheus
func NewMetricsExporter ¶
func NewMetricsExporter(sensitivity string) *MetricsExporter
NewMetricsExporter creates a new ML metrics exporter
func (*MetricsExporter) RecordAnalysisDuration ¶
func (e *MetricsExporter) RecordAnalysisDuration(analysisType string, duration time.Duration)
RecordAnalysisDuration records the duration of ML analysis
func (*MetricsExporter) RecordAnomaly ¶
func (e *MetricsExporter) RecordAnomaly(anomalyType string, severity int)
RecordAnomaly records an anomaly detection
func (*MetricsExporter) RecordBehavioralAnalysis ¶
func (e *MetricsExporter) RecordBehavioralAnalysis(result *BehavioralResult, activeClients int)
RecordBehavioralAnalysis records behavioral analysis results
func (*MetricsExporter) RecordContentAnalysis ¶
func (e *MetricsExporter) RecordContentAnalysis(result *AnalysisResult)
RecordContentAnalysis records content analysis results
func (*MetricsExporter) RecordPromptInjection ¶
func (e *MetricsExporter) RecordPromptInjection(detection *DetectionResult, blocked bool)
RecordPromptInjection records a prompt injection detection
func (*MetricsExporter) RecordRequest ¶
func (e *MetricsExporter) RecordRequest(analyzed, blocked bool)
RecordRequest records a request processed by ML middleware
func (*MetricsExporter) SetSensitivity ¶
func (e *MetricsExporter) SetSensitivity(sensitivity string)
SetSensitivity updates the sensitivity label
func (*MetricsExporter) UpdateFromStats ¶
func (e *MetricsExporter) UpdateFromStats(stats map[string]interface{})
UpdateFromStats updates metrics from stats
type PromptInjectionDetector ¶
type PromptInjectionDetector struct {
// contains filtered or unexported fields
}
PromptInjectionDetector detects potential prompt injection attacks
func NewPromptInjectionDetector ¶
func NewPromptInjectionDetector(sensitivity int) *PromptInjectionDetector
NewPromptInjectionDetector creates a new prompt injection detector
func (*PromptInjectionDetector) Detect ¶
func (d *PromptInjectionDetector) Detect(content string) *DetectionResult
Detect analyzes content for prompt injection attempts
func (*PromptInjectionDetector) GetStats ¶
func (d *PromptInjectionDetector) GetStats() map[string]interface{}
GetStats returns detection statistics
func (*PromptInjectionDetector) Reset ¶
func (d *PromptInjectionDetector) Reset()
Reset clears detection statistics
func (*PromptInjectionDetector) SetSensitivity ¶
func (d *PromptInjectionDetector) SetSensitivity(sensitivity int)
SetSensitivity updates the detection sensitivity
type PromptInjectionStats ¶
type PromptInjectionStats struct {
TotalScanned int64
ThreatsDetected int64
BlockedCount int64
ByPattern map[string]int64
LastDetection time.Time
// contains filtered or unexported fields
}
PromptInjectionStats holds detection statistics
type Sensitivity ¶
type Sensitivity string
Sensitivity defines how strict the anomaly detection should be. Higher sensitivity means more anomalies will be detected.
const ( // Low sensitivity - only major anomalies (>= 4 standard deviations) Low Sensitivity = "low" // Medium sensitivity - balanced detection (>= 3 standard deviations) Medium Sensitivity = "medium" // High sensitivity - detects subtle anomalies (>= 2 standard deviations) High Sensitivity = "high" // Critical sensitivity - detects very subtle anomalies (>= 1.5 standard deviations) Critical Sensitivity = "critical" )
type Severity ¶
type Severity int
Severity defines the importance level of an anomaly.
const ( // Info severity - log only, no action needed Info Severity = 1 // Low severity - minor deviation from normal LowSev Severity = 2 // Medium severity - notable deviation, may need attention MediumSev Severity = 3 // High severity - significant deviation, likely requires attention HighSev Severity = 4 // Critical severity - extreme deviation, immediate attention required CriticalSev Severity = 5 )
type TokenDetectionResult ¶
type TokenDetectionResult struct {
IsSmuggling bool
Score float64
MatchedTokens []string
Severity int
Description string
}
TokenDetectionResult represents the result of token smuggling detection
type TokenPattern ¶
type TokenPattern struct {
Name string
Regex *regexp.Regexp
Severity int
Weight float64
Description string
}
TokenPattern represents a token manipulation pattern
type TokenSmugglingDetector ¶
type TokenSmugglingDetector struct {
// contains filtered or unexported fields
}
TokenSmugglingDetector detects token-level prompt injection attempts
func NewTokenSmugglingDetector ¶
func NewTokenSmugglingDetector(sensitivity int) *TokenSmugglingDetector
NewTokenSmugglingDetector creates a new token smuggling detector
func (*TokenSmugglingDetector) Detect ¶
func (d *TokenSmugglingDetector) Detect(content string) *TokenDetectionResult
Detect analyzes content for token smuggling attempts
func (*TokenSmugglingDetector) GetTokenStats ¶
func (d *TokenSmugglingDetector) GetTokenStats() map[string]interface{}
GetTokenStats returns detection statistics
type TokenSmugglingStats ¶
type TokenSmugglingStats struct {
TotalScanned int64
ThreatsDetected int64
BlockedCount int64
ByPattern map[string]int64
LastDetection time.Time
// contains filtered or unexported fields
}
TokenSmugglingStats holds detection statistics
type TrafficSample ¶
type TrafficSample struct {
// Timestamp when the sample was recorded
Timestamp time.Time
// Volume is the number of requests in the sample period
Volume int
// Size is the total bytes transferred
Size int64
// Violations count of security rule violations
Violations int
}
TrafficSample represents a data point for traffic analysis. Used to build baselines and detect anomalies over time.
type UnicodeAttackDetector ¶
type UnicodeAttackDetector struct {
// contains filtered or unexported fields
}
UnicodeAttackDetector detects Unicode-based obfuscation attacks
func NewUnicodeAttackDetector ¶
func NewUnicodeAttackDetector(sensitivity int) *UnicodeAttackDetector
NewUnicodeAttackDetector creates a new Unicode attack detector
func (*UnicodeAttackDetector) Detect ¶
func (d *UnicodeAttackDetector) Detect(content string) *UnicodeDetectionResult
Detect analyzes content for Unicode-based attacks
func (*UnicodeAttackDetector) GetUnicodeStats ¶
func (d *UnicodeAttackDetector) GetUnicodeStats() map[string]interface{}
GetUnicodeStats returns detection statistics
type UnicodeAttackStats ¶
type UnicodeAttackStats struct {
TotalScanned int64
ThreatsDetected int64
BlockedCount int64
ByPattern map[string]int64
LastDetection time.Time
// contains filtered or unexported fields
}
UnicodeAttackStats holds detection statistics