metrics

package
v0.4.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 12, 2026 License: MIT Imports: 7 Imported by: 0

Documentation

Index

Constants

View Source
const (
	AlertRuleEvaluationOutcomeSuccess               = "success"
	AlertRuleEvaluationOutcomeSetupError            = "setup_error"
	AlertRuleEvaluationOutcomeUserError             = "user_error"
	AlertRuleEvaluationOutcomeServiceError          = "service_error"
	AlertRuleEvaluationOutcomeCancelled             = "cancelled" // Explicit cancellation, not evaluation or parent timeout.
	AlertRuleEvaluationOutcomeNotificationThrottled = "notification_throttled"
)

Variables

View Source
var (
	// LatencyBuckets are the histogram buckets in seconds for data age, from sub-second realtime delivery to a
	// half hour backlog.
	LatencyBuckets = []float64{0.5, 1, 2, 5, 10, 15, 30, 60, 120, 300, 600, 1800}

	// DurationBuckets are the histogram buckets in seconds for request durations.
	DurationBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 300, 600}
)
View Source
var (
	// DebugMetricsEnabled is a flag to enable debug metrics.  If set, additional metrics will be collected that are
	// useful for debugging but may be too verbose for regular production use.
	DebugMetricsEnabled bool

	Namespace = "adxmon"

	// Ingestor metrics
	IngestorHealthCheck = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "health_check",
		Help:      "Gauge indicating if Ingestor is healthy or not",
	}, []string{"region"})

	IngestorHealthStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "health_status",
		Help:      "Gauge indicating current Ingestor health status by unhealthy reason. Value is 1 when healthy and 0 when unhealthy.",
	}, []string{"region", "unhealthy_reason"})

	RequestsReceived = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "requests_received_total",
		Help:      "Counter of requests received from an ingestor instance",
	}, []string{"path", "code"})

	IngestorTransfersReceived = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "transfers_received_total",
		Help:      "Counter of /transfer requests received, by sender (collector, or peer for another ingestor) and response code",
	}, []string{"sender", "code"})

	IngestorPeerTransfers = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "peer_transfers_total",
		Help:      "Counter of transfers sent to the ingestor that owns the table, by outcome: accepted, locked (the owner's segment was busy), table_overloaded, peer_overloaded, too_large, exists, bad_request or error",
	}, []string{"outcome"})

	IngestorPeerTransferSegments = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "peer_transfer_segments",
		Help:      "Number of segments sent in each transfer to the ingestor that owns the table",
		Buckets:   []float64{1, 2, 3, 5, 8, 13, 25, 50, 100},
	})

	RequestsBytesReceived = promauto.NewCounter(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "requests_received_bytes_total",
		Help:      "Counter of bytes received from an ingestor instance",
	})

	IngestorDroppedPrefixes = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "dropped_prefixes_total",
		Help:      "Counter of dropped prefixes for an ingestor instance",
	}, []string{"prefix"})

	IngestorActiveConnections = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "active_connections",
		Help:      "Gauge indicating the number of active connections for an ingestor instance",
	})

	IngestorDroppedConnectionsTotal = promauto.NewCounter(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "dropped_connections_total",
		Help:      "Counter of dropped connections for an ingestor instance",
	})

	IngestorQueueSize = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "queue_size",
		Help:      "Gauge indicating the size of the queue for an ingestor instance",
	}, []string{"queue"})

	MetricsDroppedTotal = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "metrics_dropped_total",
		Help:      "Counter of metrics droopped for an ingestor instance",
	}, []string{"metric"})

	IngestorSegmentsUploadedTotal = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "wal_segments_uploaded_total",
		Help:      "Counter of the number of segments uploaded to Kusto",
	}, []string{"metric"})

	InvalidLogsDropped = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "invalid_logs_dropped",
		Help:      "Counter of the number of invalid logs dropped",
	}, []string{})

	IngestorRealtimeBatches = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "realtime_batches_total",
		Help:      "Counter of realtime batches by outcome: streamed, retried (retry_<reason>) or ingested with queued ingestion (fallback_<reason>)",
	}, []string{"database", "table", "outcome"})

	IngestorRealtimeStreamingRequests = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "realtime_streaming_requests_total",
		Help:      "Counter of streaming ingestion requests",
	}, []string{"database"})

	IngestorRealtimeStreamingDuration = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "realtime_streaming_duration_seconds_total",
		Help:      "Counter of the total duration in seconds of streaming ingestion requests.  Divide by realtime_streaming_requests_total for the average duration",
	}, []string{"database"})

	IngestorRealtimeIngestLatency = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "realtime_ingest_latency_seconds",
		Help:      "Gauge of the age in seconds of the oldest segment of the most recent realtime batch ingested with streaming ingestion",
	}, []string{"database", "table"})

	IngestorUploadLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "upload_latency_seconds",
		Help:      "Age in seconds of the oldest segment in a batch when the batch is uploaded, by upload method (streaming or queued)",
		Buckets:   LatencyBuckets,
	}, []string{"database", "method"})

	IngestorUploadDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "upload_duration_seconds",
		Help:      "Duration in seconds of upload requests to Kusto, by upload method (streaming or queued)",
		Buckets:   DurationBuckets,
	}, []string{"database", "method"})

	IngestorUploadFailures = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "upload_failures_total",
		Help:      "Counter of queued ingestion uploads that failed and will be retried",
	}, []string{"database", "table"})

	IngestorTransferLatency = newTransferLatency("ingestor")

	SampleLatency = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: "ingestor",
		Name:      "sample_latency",
		Help:      "Latency of a sample in seconds as determined by epoch in the filename vs the time the sample was uploaded.",
	}, []string{"database", "table"})

	// Alerting metrics
	AlerterHealthCheck = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: "alerter",
		Name:      "health_check",
		Help:      "Gauge indicating if Alerter is healthy or not",
	}, []string{"location"})

	QueryHealth = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: "alerter",
		Name:      "query_health",
		Help:      "Gauge indicating if a query is healthy or not",
	}, []string{"namespace", "name"})

	QueriesRunTotal = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "alerter",
		Name:      "queries_run_total",
		Help:      "Counter of the number of queries run",
	}, []string{})

	NotificationUnhealthy = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: "alerter",
		Name:      "notification_unhealthy",
		Help:      "Gauge indicating if a notification is healthy or not",
	}, []string{"namespace", "name"})

	AlertRuleEvaluationDurationSeconds = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: Namespace,
		Subsystem: "alerter",
		Name:      "alert_rule_evaluation_duration_seconds",
		Help:      "Duration of AlertRule evaluations in seconds, including query result processing and notification handling.",
		Buckets:   []float64{0.5, 1, 2, 5, 10, 30, 60, 120, 300},
	})

	AlertRuleEvaluationsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "alerter",
		Name:      "alert_rule_evaluations_total",
		Help:      "Counter of AlertRule evaluations by outcome.",
	}, []string{"outcome"})

	AlertsGeneratedTotal = promauto.NewCounter(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "alerter",
		Name:      "alerts_generated_total",
		Help:      "Counter of alerts successfully generated by AlertRule evaluations.",
	})

	// Collector metrics
	CollectorHealthCheck = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: "collector",
		Name:      "health_check",
		Help:      "Gauge indicating if Collector is healthy or not",
	}, []string{"region"})

	LogsProxyReceived = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "collector",
		Name:      "logs_received_total",
		Help:      "Counter of the number of logs received by the proxy",
	}, []string{"database", "table"})

	LogKeys = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: "collector",
		Name:      "logs_keys",
		Help:      "Number of keys found in logs",
	}, []string{"database", "table"})

	LogSize = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: "collector",
		Name:      "logs_size",
		Help:      "Size of logs in bytes",
	}, []string{"database", "table"})

	LogLatency = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: "collector",
		Name:      "logs_latency",
		Help:      "Latency of logs in seconds as determined by the timestamp in a log vs the time the log was recieved.",
	}, []string{"database", "table"})

	LogsCollectorLogsSent = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "collector",
		Name:      "logs_sent",
		Help:      "Counter of the number of logs successfully sent by the collector",
	}, []string{"source", "sink"})

	LogsCollectorLogsDropped = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "collector",
		Name:      "logs_dropped",
		Help:      "Counter of the number of logs dropped due to errors",
	}, []string{"source", "stage"})

	MetricsRequestsReceived = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "collector",
		Name:      "metrics_requests_received_total",
		Help:      "Counter of the number of metrics requests received",
	}, []string{"protocol", "code", "endpoint"})

	CollectorExporterSent = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "collector",
		Name:      "exporter_sent_total",
		Help:      "Counter of the number of telemetry points successfully sent by an exporter",
	}, []string{"exporter", "endpoint"})

	CollectorExporterFailed = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "collector",
		Name:      "exporter_failed_total",
		Help:      "Counter of the number of telemetry points that failed to be sent by an exporter",
	}, []string{"exporter", "endpoint"})

	CollectorTransferLatency = newTransferLatency("collector")

	CollectorScrapeDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: Namespace,
		Subsystem: "collector",
		Name:      "scrape_duration_seconds",
		Help:      "Duration in seconds of scraping a target, including writing its series",
		Buckets:   DurationBuckets,
	}, []string{"database"})

	CollectorScrapesSkipped = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "collector",
		Name:      "scrapes_skipped_total",
		Help:      "Counter of target scrapes skipped because the previous scrape was still running (in_flight) or the collector was unhealthy (unhealthy)",
	}, []string{"reason"})

	SamplesStored = promauto.NewCounter(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: "collector",
		Name:      "samples_stored_total",
		Help:      "Counter of samples stored for an collector instance",
	})
)
View Source
var IngestorDeferredBatches = promauto.NewCounterVec(prometheus.CounterOpts{Namespace: Namespace, Subsystem: "ingestor", Name: "deferred_batches_total", Help: "Batches deferred because a destination reached its per-round dispatch limit."}, []string{"database", "table"})

IngestorDeferredBatches counts batches left on disk because a destination already contributed its share to this dispatch round. They are retried on the next scan; a sustained rate means that destination is producing faster than the pipeline drains it.

View Source
var IngestorUndeliverableSegments = promauto.NewCounterVec(prometheus.CounterOpts{Namespace: Namespace, Subsystem: "ingestor", Name: "undeliverable_segments_total", Help: "Segments that exceed the destination table budget and cannot be split further."}, []string{"database", "table"})

IngestorUndeliverableSegments counts segments a destination can never accept because the payload exceeds its entire budget. This requires an operator to raise the table budget or lower the transfer size; the data is retained.

View Source
var TableQuotaRejectedBytes = promauto.NewCounterVec(prometheus.CounterOpts{Namespace: Namespace, Name: "table_quota_rejected_bytes_total", Help: "Uncompressed encoded bytes intentionally shed at table backlog quotas."}, []string{"component", "kind", "reason"})
View Source
var TableQuotaRejectedRecords = promauto.NewCounterVec(prometheus.CounterOpts{Namespace: Namespace, Name: "table_quota_rejected_records_total", Help: "Records intentionally shed at table backlog quotas."}, []string{"component", "kind", "reason"})

Functions

func NewCollectorSegmentMetrics added in v0.2.0

func NewCollectorSegmentMetrics() (segmentsTotal, segmentsSizeBytes, segmentsMaxAge prometheus.Gauge)

NewCollectorSegmentMetrics creates the collector-specific WAL segment metrics. These are created lazily to ensure ingestor instances don't expose collector metrics. This function is safe to call multiple times - metrics are only created once.

func NewCollectorSegmentSizeByPriorityMetric added in v0.4.0

func NewCollectorSegmentSizeByPriorityMetric() *prometheus.GaugeVec

NewCollectorSegmentSizeByPriorityMetric creates the collector WAL segment size by ingestion priority metric. It is safe to call multiple times.

func NewIngestorSegmentMetrics added in v0.2.0

func NewIngestorSegmentMetrics() (segmentsTotal, segmentsSizeBytes, segmentsMaxAge prometheus.Gauge)

NewIngestorSegmentMetrics creates the ingestor-specific WAL segment metrics. These are created lazily to ensure collector instances don't expose ingestor metrics. This function is safe to call multiple times - metrics are only created once.

func NewIngestorSegmentSizeByPriorityMetric added in v0.4.0

func NewIngestorSegmentSizeByPriorityMetric() *prometheus.GaugeVec

NewIngestorSegmentSizeByPriorityMetric creates the ingestor WAL segment size by ingestion priority metric. It is safe to call multiple times.

func RegisterIntakeMemory added in v0.4.0

func RegisterIntakeMemory(component string, m IntakeMemory) func()

func RegisterTableBudget added in v0.4.0

func RegisterTableBudget(component string, budget *wal.Admission) func()

func RegisterTableWork added in v0.4.0

func RegisterTableWork(pipeline string, read func() map[wal.Destination]TableWork) func()

Types

type HealthReporter

type HealthReporter interface {
	IsHealthy() bool
	TransferQueueSize() int
	UploadQueueSize() int
	SegmentsTotal() int64
	SegmentsSize() int64
	UnhealthyReason() string
	MaxSegmentAge() time.Duration
}

HealthReporter exposes health and queue/segment status used by the per-process metrics collection services in the collector and ingestor.

type IntakeMemory added in v0.4.0

type IntakeMemory interface {
	InFlight() int64
	Limit() int64
}

IntakeMemory reports the bytes buffered by in-flight transfer requests and the configured ceiling, where zero means the ceiling is disabled.

type TableWork added in v0.4.0

type TableWork struct {
	Queued, Running int
	RetryAt         time.Time
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL