ml

package
v1.0.9 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 15, 2026 License: MIT Imports: 9 Imported by: 0

Documentation

Overview

Package ml provides machine learning capabilities for anomaly detection in WAF traffic. This implementation uses only the Go standard library for pure statistical analysis without external ML dependencies.

The anomaly detection system uses: - Moving averages for baseline establishment - Z-score calculation for statistical anomaly detection - Shannon entropy for content analysis - Time-series analysis for pattern detection

Example usage:

config := ml.Config{
    Sensitivity: ml.Medium,
    WindowSize: 100,
    ZThreshold: 3.0,
}
detector := ml.New(config)

detector.RecordTraffic(ml.TrafficSample{
    Timestamp: time.Now(),
    Volume:    100,
    Size:      1024,
})

anomaly, detected := detector.AnalyzeRequest("GET", "/api/data", 2048)

Index

Constants

This section is empty.

Variables

View Source
var (
	// Middleware metrics
	MLTotalRequests = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "aegisgate_ml_total_requests_total",
			Help: "Total number of requests processed by ML middleware",
		},
		[]string{"sensitivity"},
	)

	MLAnalyzedRequests = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "aegisgate_ml_analyzed_requests_total",
			Help: "Total number of requests analyzed by ML middleware",
		},
		[]string{"sensitivity"},
	)

	MLBlockedRequests = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "aegisgate_ml_blocked_requests_total",
			Help: "Total number of requests blocked by ML middleware",
		},
		[]string{"sensitivity", "reason"},
	)

	MLAnomalyDetections = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "aegisgate_ml_anomaly_detections_total",
			Help: "Total number of anomaly detections",
		},
		[]string{"type", "severity", "blocked"},
	)

	// Prompt injection metrics
	MLPromptInjectionScanned = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "aegisgate_ml_prompt_injection_scanned_total",
			Help: "Total number of prompts scanned for injection",
		},
		[]string{"sensitivity"},
	)

	MLPromptInjectionDetections = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "aegisgate_ml_prompt_injection_detections_total",
			Help: "Total number of prompt injection detections",
		},
		[]string{"sensitivity", "pattern", "severity"},
	)

	// Content analysis metrics
	MLContentAnalyzed = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "aegisgate_ml_content_analyzed_total",
			Help: "Total number of content items analyzed",
		},
		[]string{"sensitivity"},
	)

	MLContentViolations = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "aegisgate_ml_content_violations_total",
			Help: "Total number of content policy violations",
		},
		[]string{"sensitivity", "type"},
	)

	// Behavioral metrics
	MLBehavioralClients = promauto.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "aegisgate_ml_behavioral_clients_active",
			Help: "Number of active clients being tracked",
		},
		[]string{},
	)

	MLBehavioralAnomalies = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "aegisgate_ml_behavioral_anomalies_total",
			Help: "Total number of behavioral anomalies",
		},
		[]string{"type", "client"},
	)

	// Latency metrics
	MLAnalysisDuration = promauto.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "aegisgate_ml_analysis_duration_seconds",
			Help:    "Time spent on ML analysis",
			Buckets: []float64{.0001, .0005, .001, .005, .01, .05, .1, .5, 1},
		},
		[]string{"type"},
	)
)

PrometheusMetrics returns all ML-related Prometheus metrics

Functions

func RecordBehavioralMetrics

func RecordBehavioralMetrics(result *BehavioralResult, clientID string, duration time.Duration)

RecordBehavioralMetrics records behavioral analysis metrics

func RecordContentMetrics

func RecordContentMetrics(result *AnalysisResult, sensitivity string, duration time.Duration)

RecordContentMetrics records content analysis metrics

func RecordMLMetrics

func RecordMLMetrics(metricsType, sensitivity string, duration time.Duration, blocked bool)

RecordMLMetrics records metrics for a single ML analysis

func RecordPromptInjectionMetrics

func RecordPromptInjectionMetrics(detection *DetectionResult, sensitivity string, duration time.Duration)

RecordPromptInjectionMetrics records prompt injection specific metrics

Types

type AnalysisResult

type AnalysisResult struct {
	IsViolation     bool
	Score           float64
	ViolationTypes  []string
	RedactedContent string
	Severity        int
}

AnalysisResult represents the result of content analysis

type Anomaly

type Anomaly struct {
	// Type categorizes the anomaly (traffic_spike, entropy_anomaly, etc.)
	Type AnomalyType

	// Severity indicates the importance level (1-5)
	Severity Severity

	// Score is the statistical z-score indicating how many
	// standard deviations from the mean this value is
	Score float64

	// Timestamp when the anomaly was detected
	Timestamp time.Time

	// Evidence contains additional context about the anomaly
	Evidence map[string]interface{}

	// Description provides a human-readable explanation
	Description string
}

Anomaly represents a detected statistical anomaly in traffic patterns. Contains all details needed for evaluation and response.

type AnomalyType

type AnomalyType string

AnomalyType categorizes the type of anomaly detected. These types help in understanding the nature of suspicious activity.

const (
	// TrafficSpike indicates an unusual increase in request volume.
	// May indicate DDoS attack or sudden legitimate traffic surge.
	TrafficSpike AnomalyType = "traffic_spike"

	// TrafficDrop indicates an unusual decrease in request volume.
	// May indicate service disruption or blocked legitimate traffic.
	TrafficDrop AnomalyType = "traffic_drop"

	// SizeAnomaly indicates unusual request/response sizes.
	// May indicate data exfiltration, large uploads, or buffer overflow attempts.
	SizeAnomaly AnomalyType = "size_anomaly"

	// ViolationSpike indicates increased security rule violations.
	// May indicate attack attempts or scanning activity.
	ViolationSpike AnomalyType = "violation_spike"

	// EntropyAnomaly indicates unusual entropy in request content.
	// May indicate encoded payloads, encrypted traffic, or obfuscated attacks.
	EntropyAnomaly AnomalyType = "entropy_anomaly"

	// PatternAnomaly indicates unusual request patterns.
	// May indicate automated tools, bots, or scripted attacks.
	PatternAnomaly AnomalyType = "pattern_anomaly"

	// TimeAnomaly indicates unusual timing patterns.
	// May indicate automated requests or scheduled attack patterns.
	TimeAnomaly AnomalyType = "time_anomaly"
)

type Baseline

type Baseline struct {
	// contains filtered or unexported fields
}

Baseline tracks moving averages and standard deviations for metrics. Uses a sliding window approach for real-time adaptation to traffic patterns.

type BehavioralAnalyzer

type BehavioralAnalyzer struct {
	// contains filtered or unexported fields
}

BehavioralAnalyzer analyzes behavioral patterns for anomalies

func NewBehavioralAnalyzer

func NewBehavioralAnalyzer() *BehavioralAnalyzer

NewBehavioralAnalyzer creates a new behavioral analyzer

func (*BehavioralAnalyzer) AnalyzeRequest

func (ba *BehavioralAnalyzer) AnalyzeRequest(clientID, method, path string, bytesSent int64) *BehavioralResult

AnalyzeRequest analyzes a request for behavioral anomalies

func (*BehavioralAnalyzer) GetStats

func (ba *BehavioralAnalyzer) GetStats() map[string]interface{}

GetStats returns behavioral analysis statistics

func (*BehavioralAnalyzer) Reset

func (ba *BehavioralAnalyzer) Reset()

Reset clears behavioral analysis statistics

type BehavioralResult

type BehavioralResult struct {
	IsAnomaly   bool
	AnomalyType string
	Score       float64
	Description string
}

BehavioralResult represents the result of behavioral analysis

type BehavioralStats

type BehavioralStats struct {
	TotalClients     int64
	AnomalousClients int64
	TotalAnomalies   int64
	ByType           map[string]int64
	// contains filtered or unexported fields
}

BehavioralStats holds behavioral analysis statistics

type ClientBehavior

type ClientBehavior struct {
	RequestTimestamps []time.Time
	PathFrequencies   map[string]int
	MethodCounts      map[string]int
	TotalRequests     int
	BytesSent         int64
	FirstSeen         time.Time
	LastSeen          time.Time
	AnomalyScore      float64
}

ClientBehavior holds behavior data for a client

type CombinedDetector

type CombinedDetector struct {
	PromptInjection     *PromptInjectionDetector
	TokenSmuggling      *TokenSmugglingDetector
	UnicodeAttack       *UnicodeAttackDetector
	ContextManipulation *ContextManipulationDetector
}

CombinedDetector provides unified detection across all attack patterns

func NewCombinedDetector

func NewCombinedDetector(sensitivity int) *CombinedDetector

NewCombinedDetector creates a new combined detector with all sub-detectors

func (*CombinedDetector) Detect

func (cd *CombinedDetector) Detect(content string) *CombinedResult

Detect analyzes content across all detection mechanisms

func (*CombinedDetector) GetAllStats

func (cd *CombinedDetector) GetAllStats() map[string]interface{}

GetAllStats returns combined statistics from all detectors

type CombinedResult

type CombinedResult struct {
	IsThreat             bool
	TotalScore           float64
	PromptInjectionScore float64
	TokenSmugglingScore  float64
	UnicodeAttackScore   float64
	ContextScore         float64
	AllMatchedPatterns   []string
	HighestSeverity      int
}

CombinedResult represents the combined detection result

type Config

type Config struct {
	// Sensitivity determines the z-score threshold for anomaly detection
	Sensitivity Sensitivity

	// WindowSize is the number of samples to consider for baseline
	WindowSize int

	// ZThreshold can override the default threshold based on sensitivity
	ZThreshold float64

	// MinSamples is the minimum samples needed before detecting anomalies
	MinSamples int

	// EntropyThreshold for content analysis (0-1, higher = more random)
	EntropyThreshold float64
}

Config provides configuration options for the anomaly detector.

func (Config) Validate

func (c Config) Validate() error

Validate ensures the detector configuration is valid.

type ContentAnalysisStats

type ContentAnalysisStats struct {
	TotalAnalyzed   int64
	ViolationsFound int64
	ByType          map[string]int64
	LastViolation   time.Time
	// contains filtered or unexported fields
}

ContentAnalysisStats holds analysis statistics

type ContentAnalyzer

type ContentAnalyzer struct {
	// contains filtered or unexported fields
}

ContentAnalyzer analyzes LLM responses for policy violations

func NewContentAnalyzer

func NewContentAnalyzer() *ContentAnalyzer

NewContentAnalyzer creates a new content analyzer

func (*ContentAnalyzer) AddRule

func (a *ContentAnalyzer) AddRule(rule ContentRule)

AddRule adds a custom content analysis rule

func (*ContentAnalyzer) Analyze

func (a *ContentAnalyzer) Analyze(content string) *AnalysisResult

Analyze analyzes content for policy violations

func (*ContentAnalyzer) GetStats

func (a *ContentAnalyzer) GetStats() map[string]interface{}

GetStats returns analysis statistics

func (*ContentAnalyzer) Reset

func (a *ContentAnalyzer) Reset()

Reset clears analysis statistics

type ContentRule

type ContentRule struct {
	Name     string
	Pattern  *regexp.Regexp
	Severity int
	Action   string // "block", "alert", "redact"
}

ContentRule represents a custom content analysis rule

type ContextDetectionResult

type ContextDetectionResult struct {
	IsManipulation bool
	Score          float64
	MatchedTypes   []string
	Severity       int
	Description    string
}

ContextDetectionResult represents the result of context manipulation detection

type ContextManipulationDetector

type ContextManipulationDetector struct {
	// contains filtered or unexported fields
}

ContextManipulationDetector detects attempts to manipulate conversation context

func NewContextManipulationDetector

func NewContextManipulationDetector(sensitivity int) *ContextManipulationDetector

NewContextManipulationDetector creates a new context manipulation detector

func (*ContextManipulationDetector) Detect

Detect analyzes content for context manipulation attempts

func (*ContextManipulationDetector) GetContextStats

func (d *ContextManipulationDetector) GetContextStats() map[string]interface{}

GetContextStats returns detection statistics

type ContextManipulationStats

type ContextManipulationStats struct {
	TotalScanned    int64
	ThreatsDetected int64
	BlockedCount    int64
	ByPattern       map[string]int64
	LastDetection   time.Time
	// contains filtered or unexported fields
}

ContextManipulationStats holds detection statistics

type ContextPattern

type ContextPattern struct {
	Name        string
	Regex       *regexp.Regexp
	Severity    int
	Weight      float64
	Description string
}

ContextPattern represents a context manipulation pattern

type DetectionResult

type DetectionResult struct {
	IsInjection     bool
	Score           float64 // 0-100
	MatchedPatterns []string
	Severity        int
	Explanation     string
}

DetectionResult represents the result of prompt injection detection

type Detector

type Detector struct {
	// contains filtered or unexported fields
}

Detector is the main anomaly detection engine. Supports concurrent access and real-time analysis.

func New

func New(cfg Config) *Detector

New creates a new anomaly detector with the specified configuration. Returns a fully initialized Detector ready for use.

func (*Detector) AnalyzeContent

func (d *Detector) AnalyzeContent(content []byte) (float64, *Anomaly)

AnalyzeContent calculates entropy and detects anomalies in content. Returns the entropy value and any detected anomaly.

func (*Detector) AnalyzePatterns

func (d *Detector) AnalyzePatterns(input string) []Anomaly

AnalyzePatterns performs pattern-based anomaly detection on strings. Detects suspicious patterns like encoding sequences, special char clusters, etc.

func (*Detector) AnalyzeRequest

func (d *Detector) AnalyzeRequest(method, path string, size int64) (Anomaly, bool)

AnalyzeRequest evaluates a single HTTP request for anomalies. Analyzes method, path, and size against established baselines.

func (*Detector) ClearAnomalies

func (d *Detector) ClearAnomalies()

ClearAnomalies removes all stored anomalies.

func (*Detector) GetAnomalies

func (d *Detector) GetAnomalies() []Anomaly

GetAnomalies returns all detected anomalies. Thread-safe access to the anomaly history.

func (*Detector) GetBaselineStats

func (d *Detector) GetBaselineStats() map[string]interface{}

GetBaselineStats returns current baseline statistics for debugging/monitoring.

func (*Detector) GetMethodsDistribution

func (d *Detector) GetMethodsDistribution() map[string]float64

GetMethodsDistribution returns the current distribution of HTTP methods.

func (*Detector) GetRecentAnomalies

func (d *Detector) GetRecentAnomalies(since time.Duration) []Anomaly

GetRecentAnomalies returns anomalies from the specified duration.

func (*Detector) GetTopPaths

func (d *Detector) GetTopPaths(n int) []struct {
	Path  string
	Count int
}

GetTopPaths returns the most frequently accessed paths.

func (*Detector) IsReady

func (d *Detector) IsReady() bool

IsReady returns true if the detector has enough samples for anomaly detection.

func (*Detector) RecordTraffic

func (d *Detector) RecordTraffic(sample TrafficSample) *Anomaly

RecordTraffic adds a traffic sample and updates the moving baseline. Returns an anomaly if the sample is statistically anomalous.

func (*Detector) Reset

func (d *Detector) Reset()

Reset clears all baselines and detected anomalies.

type InjectionPattern

type InjectionPattern struct {
	Name     string
	Regex    *regexp.Regexp
	Severity int // 1-5
	Weight   float64
}

InjectionPattern represents a known prompt injection pattern

type MetricsExporter

type MetricsExporter struct {
	// contains filtered or unexported fields
}

MetricsExporter exports ML metrics to Prometheus

func NewMetricsExporter

func NewMetricsExporter(sensitivity string) *MetricsExporter

NewMetricsExporter creates a new ML metrics exporter

func (*MetricsExporter) RecordAnalysisDuration

func (e *MetricsExporter) RecordAnalysisDuration(analysisType string, duration time.Duration)

RecordAnalysisDuration records the duration of ML analysis

func (*MetricsExporter) RecordAnomaly

func (e *MetricsExporter) RecordAnomaly(anomalyType string, severity int)

RecordAnomaly records an anomaly detection

func (*MetricsExporter) RecordBehavioralAnalysis

func (e *MetricsExporter) RecordBehavioralAnalysis(result *BehavioralResult, activeClients int)

RecordBehavioralAnalysis records behavioral analysis results

func (*MetricsExporter) RecordContentAnalysis

func (e *MetricsExporter) RecordContentAnalysis(result *AnalysisResult)

RecordContentAnalysis records content analysis results

func (*MetricsExporter) RecordPromptInjection

func (e *MetricsExporter) RecordPromptInjection(detection *DetectionResult, blocked bool)

RecordPromptInjection records a prompt injection detection

func (*MetricsExporter) RecordRequest

func (e *MetricsExporter) RecordRequest(analyzed, blocked bool)

RecordRequest records a request processed by ML middleware

func (*MetricsExporter) SetSensitivity

func (e *MetricsExporter) SetSensitivity(sensitivity string)

SetSensitivity updates the sensitivity label

func (*MetricsExporter) UpdateFromStats

func (e *MetricsExporter) UpdateFromStats(stats map[string]interface{})

UpdateFromStats updates metrics from stats

type PromptInjectionDetector

type PromptInjectionDetector struct {
	// contains filtered or unexported fields
}

PromptInjectionDetector detects potential prompt injection attacks

func NewPromptInjectionDetector

func NewPromptInjectionDetector(sensitivity int) *PromptInjectionDetector

NewPromptInjectionDetector creates a new prompt injection detector

func (*PromptInjectionDetector) Detect

func (d *PromptInjectionDetector) Detect(content string) *DetectionResult

Detect analyzes content for prompt injection attempts

func (*PromptInjectionDetector) GetStats

func (d *PromptInjectionDetector) GetStats() map[string]interface{}

GetStats returns detection statistics

func (*PromptInjectionDetector) Reset

func (d *PromptInjectionDetector) Reset()

Reset clears detection statistics

func (*PromptInjectionDetector) SetSensitivity

func (d *PromptInjectionDetector) SetSensitivity(sensitivity int)

SetSensitivity updates the detection sensitivity

type PromptInjectionStats

type PromptInjectionStats struct {
	TotalScanned    int64
	ThreatsDetected int64
	BlockedCount    int64
	ByPattern       map[string]int64
	LastDetection   time.Time
	// contains filtered or unexported fields
}

PromptInjectionStats holds detection statistics

type Sensitivity

type Sensitivity string

Sensitivity defines how strict the anomaly detection should be. Higher sensitivity means more anomalies will be detected.

const (
	// Low sensitivity - only major anomalies (>= 4 standard deviations)
	Low Sensitivity = "low"
	// Medium sensitivity - balanced detection (>= 3 standard deviations)
	Medium Sensitivity = "medium"
	// High sensitivity - detects subtle anomalies (>= 2 standard deviations)
	High Sensitivity = "high"
	// Critical sensitivity - detects very subtle anomalies (>= 1.5 standard deviations)
	Critical Sensitivity = "critical"
)

type Severity

type Severity int

Severity defines the importance level of an anomaly.

const (
	// Info severity - log only, no action needed
	Info Severity = 1
	// Low severity - minor deviation from normal
	LowSev Severity = 2
	// Medium severity - notable deviation, may need attention
	MediumSev Severity = 3
	// High severity - significant deviation, likely requires attention
	HighSev Severity = 4
	// Critical severity - extreme deviation, immediate attention required
	CriticalSev Severity = 5
)

type TokenDetectionResult

type TokenDetectionResult struct {
	IsSmuggling   bool
	Score         float64
	MatchedTokens []string
	Severity      int
	Description   string
}

TokenDetectionResult represents the result of token smuggling detection

type TokenPattern

type TokenPattern struct {
	Name        string
	Regex       *regexp.Regexp
	Severity    int
	Weight      float64
	Description string
}

TokenPattern represents a token manipulation pattern

type TokenSmugglingDetector

type TokenSmugglingDetector struct {
	// contains filtered or unexported fields
}

TokenSmugglingDetector detects token-level prompt injection attempts

func NewTokenSmugglingDetector

func NewTokenSmugglingDetector(sensitivity int) *TokenSmugglingDetector

NewTokenSmugglingDetector creates a new token smuggling detector

func (*TokenSmugglingDetector) Detect

Detect analyzes content for token smuggling attempts

func (*TokenSmugglingDetector) GetTokenStats

func (d *TokenSmugglingDetector) GetTokenStats() map[string]interface{}

GetTokenStats returns detection statistics

type TokenSmugglingStats

type TokenSmugglingStats struct {
	TotalScanned    int64
	ThreatsDetected int64
	BlockedCount    int64
	ByPattern       map[string]int64
	LastDetection   time.Time
	// contains filtered or unexported fields
}

TokenSmugglingStats holds detection statistics

type TrafficSample

type TrafficSample struct {
	// Timestamp when the sample was recorded
	Timestamp time.Time
	// Volume is the number of requests in the sample period
	Volume int
	// Size is the total bytes transferred
	Size int64
	// Violations count of security rule violations
	Violations int
}

TrafficSample represents a data point for traffic analysis. Used to build baselines and detect anomalies over time.

type UnicodeAttackDetector

type UnicodeAttackDetector struct {
	// contains filtered or unexported fields
}

UnicodeAttackDetector detects Unicode-based obfuscation attacks

func NewUnicodeAttackDetector

func NewUnicodeAttackDetector(sensitivity int) *UnicodeAttackDetector

NewUnicodeAttackDetector creates a new Unicode attack detector

func (*UnicodeAttackDetector) Detect

Detect analyzes content for Unicode-based attacks

func (*UnicodeAttackDetector) GetUnicodeStats

func (d *UnicodeAttackDetector) GetUnicodeStats() map[string]interface{}

GetUnicodeStats returns detection statistics

type UnicodeAttackStats

type UnicodeAttackStats struct {
	TotalScanned    int64
	ThreatsDetected int64
	BlockedCount    int64
	ByPattern       map[string]int64
	LastDetection   time.Time
	// contains filtered or unexported fields
}

UnicodeAttackStats holds detection statistics

type UnicodeDetectionResult

type UnicodeDetectionResult struct {
	IsAttack     bool
	Score        float64
	MatchedTypes []string
	Severity     int
	Description  string
}

UnicodeDetectionResult represents the result of Unicode attack detection

type UnicodePattern

type UnicodePattern struct {
	Name        string
	Regex       *regexp.Regexp
	Severity    int
	Weight      float64
	Description string
}

UnicodePattern represents a Unicode manipulation pattern

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL