Documentation
¶
Index ¶
- Constants
- Variables
- func NewCollectorSegmentMetrics() (segmentsTotal, segmentsSizeBytes, segmentsMaxAge prometheus.Gauge)
- func NewCollectorSegmentSizeByPriorityMetric() *prometheus.GaugeVec
- func NewIngestorSegmentMetrics() (segmentsTotal, segmentsSizeBytes, segmentsMaxAge prometheus.Gauge)
- func NewIngestorSegmentSizeByPriorityMetric() *prometheus.GaugeVec
- func RegisterIntakeMemory(component string, m IntakeMemory) func()
- func RegisterTableBudget(component string, budget *wal.Admission) func()
- func RegisterTableWork(pipeline string, read func() map[wal.Destination]TableWork) func()
- type HealthReporter
- type IntakeMemory
- type TableWork
Constants ¶
const ( AlertRuleEvaluationOutcomeSuccess = "success" AlertRuleEvaluationOutcomeSetupError = "setup_error" AlertRuleEvaluationOutcomeUserError = "user_error" AlertRuleEvaluationOutcomeServiceError = "service_error" AlertRuleEvaluationOutcomeCancelled = "cancelled" // Explicit cancellation, not evaluation or parent timeout. AlertRuleEvaluationOutcomeNotificationThrottled = "notification_throttled" )
Variables ¶
var ( // LatencyBuckets are the histogram buckets in seconds for data age, from sub-second realtime delivery to a // half hour backlog. LatencyBuckets = []float64{0.5, 1, 2, 5, 10, 15, 30, 60, 120, 300, 600, 1800} // DurationBuckets are the histogram buckets in seconds for request durations. DurationBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 300, 600} )
var ( // DebugMetricsEnabled is a flag to enable debug metrics. If set, additional metrics will be collected that are // useful for debugging but may be too verbose for regular production use. DebugMetricsEnabled bool Namespace = "adxmon" // Ingestor metrics IngestorHealthCheck = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "health_check", Help: "Gauge indicating if Ingestor is healthy or not", }, []string{"region"}) IngestorHealthStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "health_status", Help: "Gauge indicating current Ingestor health status by unhealthy reason. Value is 1 when healthy and 0 when unhealthy.", }, []string{"region", "unhealthy_reason"}) RequestsReceived = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "requests_received_total", Help: "Counter of requests received from an ingestor instance", }, []string{"path", "code"}) IngestorTransfersReceived = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "transfers_received_total", Help: "Counter of /transfer requests received, by sender (collector, or peer for another ingestor) and response code", }, []string{"sender", "code"}) IngestorPeerTransfers = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "peer_transfers_total", Help: "Counter of transfers sent to the ingestor that owns the table, by outcome: accepted, locked (the owner's segment was busy), table_overloaded, peer_overloaded, too_large, exists, bad_request or error", }, []string{"outcome"}) IngestorPeerTransferSegments = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "peer_transfer_segments", Help: "Number of segments sent in each transfer to the ingestor that owns the table", Buckets: []float64{1, 2, 3, 5, 8, 13, 25, 50, 100}, }) RequestsBytesReceived = promauto.NewCounter(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "requests_received_bytes_total", Help: "Counter of bytes received from an ingestor instance", }) IngestorDroppedPrefixes = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "dropped_prefixes_total", Help: "Counter of dropped prefixes for an ingestor instance", }, []string{"prefix"}) IngestorActiveConnections = promauto.NewGauge(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "active_connections", Help: "Gauge indicating the number of active connections for an ingestor instance", }) IngestorDroppedConnectionsTotal = promauto.NewCounter(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "dropped_connections_total", Help: "Counter of dropped connections for an ingestor instance", }) IngestorQueueSize = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "queue_size", Help: "Gauge indicating the size of the queue for an ingestor instance", }, []string{"queue"}) MetricsDroppedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "metrics_dropped_total", Help: "Counter of metrics droopped for an ingestor instance", }, []string{"metric"}) IngestorSegmentsUploadedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "wal_segments_uploaded_total", Help: "Counter of the number of segments uploaded to Kusto", }, []string{"metric"}) InvalidLogsDropped = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "invalid_logs_dropped", Help: "Counter of the number of invalid logs dropped", }, []string{}) IngestorRealtimeBatches = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "realtime_batches_total", Help: "Counter of realtime batches by outcome: streamed, retried (retry_<reason>) or ingested with queued ingestion (fallback_<reason>)", }, []string{"database", "table", "outcome"}) IngestorRealtimeStreamingRequests = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "realtime_streaming_requests_total", Help: "Counter of streaming ingestion requests", }, []string{"database"}) IngestorRealtimeStreamingDuration = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "realtime_streaming_duration_seconds_total", Help: "Counter of the total duration in seconds of streaming ingestion requests. Divide by realtime_streaming_requests_total for the average duration", }, []string{"database"}) IngestorRealtimeIngestLatency = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "realtime_ingest_latency_seconds", Help: "Gauge of the age in seconds of the oldest segment of the most recent realtime batch ingested with streaming ingestion", }, []string{"database", "table"}) IngestorUploadLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "upload_latency_seconds", Help: "Age in seconds of the oldest segment in a batch when the batch is uploaded, by upload method (streaming or queued)", Buckets: LatencyBuckets, }, []string{"database", "method"}) IngestorUploadDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "upload_duration_seconds", Help: "Duration in seconds of upload requests to Kusto, by upload method (streaming or queued)", Buckets: DurationBuckets, }, []string{"database", "method"}) IngestorUploadFailures = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "upload_failures_total", Help: "Counter of queued ingestion uploads that failed and will be retried", }, []string{"database", "table"}) IngestorTransferLatency = newTransferLatency("ingestor") SampleLatency = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "ingestor", Name: "sample_latency", Help: "Latency of a sample in seconds as determined by epoch in the filename vs the time the sample was uploaded.", }, []string{"database", "table"}) // Alerting metrics AlerterHealthCheck = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "alerter", Name: "health_check", Help: "Gauge indicating if Alerter is healthy or not", }, []string{"location"}) QueryHealth = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "alerter", Name: "query_health", Help: "Gauge indicating if a query is healthy or not", }, []string{"namespace", "name"}) QueriesRunTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "alerter", Name: "queries_run_total", Help: "Counter of the number of queries run", }, []string{}) NotificationUnhealthy = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "alerter", Name: "notification_unhealthy", Help: "Gauge indicating if a notification is healthy or not", }, []string{"namespace", "name"}) AlertRuleEvaluationDurationSeconds = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: Namespace, Subsystem: "alerter", Name: "alert_rule_evaluation_duration_seconds", Help: "Duration of AlertRule evaluations in seconds, including query result processing and notification handling.", Buckets: []float64{0.5, 1, 2, 5, 10, 30, 60, 120, 300}, }) AlertRuleEvaluationsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "alerter", Name: "alert_rule_evaluations_total", Help: "Counter of AlertRule evaluations by outcome.", }, []string{"outcome"}) AlertsGeneratedTotal = promauto.NewCounter(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "alerter", Name: "alerts_generated_total", Help: "Counter of alerts successfully generated by AlertRule evaluations.", }) // Collector metrics CollectorHealthCheck = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "collector", Name: "health_check", Help: "Gauge indicating if Collector is healthy or not", }, []string{"region"}) LogsProxyReceived = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "collector", Name: "logs_received_total", Help: "Counter of the number of logs received by the proxy", }, []string{"database", "table"}) LogKeys = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "collector", Name: "logs_keys", Help: "Number of keys found in logs", }, []string{"database", "table"}) LogSize = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "collector", Name: "logs_size", Help: "Size of logs in bytes", }, []string{"database", "table"}) LogLatency = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "collector", Name: "logs_latency", Help: "Latency of logs in seconds as determined by the timestamp in a log vs the time the log was recieved.", }, []string{"database", "table"}) LogsCollectorLogsSent = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "collector", Name: "logs_sent", Help: "Counter of the number of logs successfully sent by the collector", }, []string{"source", "sink"}) LogsCollectorLogsDropped = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "collector", Name: "logs_dropped", Help: "Counter of the number of logs dropped due to errors", }, []string{"source", "stage"}) MetricsRequestsReceived = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "collector", Name: "metrics_requests_received_total", Help: "Counter of the number of metrics requests received", }, []string{"protocol", "code", "endpoint"}) CollectorExporterSent = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "collector", Name: "exporter_sent_total", Help: "Counter of the number of telemetry points successfully sent by an exporter", }, []string{"exporter", "endpoint"}) CollectorExporterFailed = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "collector", Name: "exporter_failed_total", Help: "Counter of the number of telemetry points that failed to be sent by an exporter", }, []string{"exporter", "endpoint"}) CollectorTransferLatency = newTransferLatency("collector") CollectorScrapeDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: Namespace, Subsystem: "collector", Name: "scrape_duration_seconds", Help: "Duration in seconds of scraping a target, including writing its series", Buckets: DurationBuckets, }, []string{"database"}) CollectorScrapesSkipped = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "collector", Name: "scrapes_skipped_total", Help: "Counter of target scrapes skipped because the previous scrape was still running (in_flight) or the collector was unhealthy (unhealthy)", }, []string{"reason"}) SamplesStored = promauto.NewCounter(prometheus.CounterOpts{ Namespace: Namespace, Subsystem: "collector", Name: "samples_stored_total", Help: "Counter of samples stored for an collector instance", }) )
var IngestorDeferredBatches = promauto.NewCounterVec(prometheus.CounterOpts{Namespace: Namespace, Subsystem: "ingestor", Name: "deferred_batches_total", Help: "Batches deferred because a destination reached its per-round dispatch limit."}, []string{"database", "table"})
IngestorDeferredBatches counts batches left on disk because a destination already contributed its share to this dispatch round. They are retried on the next scan; a sustained rate means that destination is producing faster than the pipeline drains it.
var IngestorUndeliverableSegments = promauto.NewCounterVec(prometheus.CounterOpts{Namespace: Namespace, Subsystem: "ingestor", Name: "undeliverable_segments_total", Help: "Segments that exceed the destination table budget and cannot be split further."}, []string{"database", "table"})
IngestorUndeliverableSegments counts segments a destination can never accept because the payload exceeds its entire budget. This requires an operator to raise the table budget or lower the transfer size; the data is retained.
var TableQuotaRejectedBytes = promauto.NewCounterVec(prometheus.CounterOpts{Namespace: Namespace, Name: "table_quota_rejected_bytes_total", Help: "Uncompressed encoded bytes intentionally shed at table backlog quotas."}, []string{"component", "kind", "reason"})
var TableQuotaRejectedRecords = promauto.NewCounterVec(prometheus.CounterOpts{Namespace: Namespace, Name: "table_quota_rejected_records_total", Help: "Records intentionally shed at table backlog quotas."}, []string{"component", "kind", "reason"})
Functions ¶
func NewCollectorSegmentMetrics ¶ added in v0.2.0
func NewCollectorSegmentMetrics() (segmentsTotal, segmentsSizeBytes, segmentsMaxAge prometheus.Gauge)
NewCollectorSegmentMetrics creates the collector-specific WAL segment metrics. These are created lazily to ensure ingestor instances don't expose collector metrics. This function is safe to call multiple times - metrics are only created once.
func NewCollectorSegmentSizeByPriorityMetric ¶ added in v0.4.0
func NewCollectorSegmentSizeByPriorityMetric() *prometheus.GaugeVec
NewCollectorSegmentSizeByPriorityMetric creates the collector WAL segment size by ingestion priority metric. It is safe to call multiple times.
func NewIngestorSegmentMetrics ¶ added in v0.2.0
func NewIngestorSegmentMetrics() (segmentsTotal, segmentsSizeBytes, segmentsMaxAge prometheus.Gauge)
NewIngestorSegmentMetrics creates the ingestor-specific WAL segment metrics. These are created lazily to ensure collector instances don't expose ingestor metrics. This function is safe to call multiple times - metrics are only created once.
func NewIngestorSegmentSizeByPriorityMetric ¶ added in v0.4.0
func NewIngestorSegmentSizeByPriorityMetric() *prometheus.GaugeVec
NewIngestorSegmentSizeByPriorityMetric creates the ingestor WAL segment size by ingestion priority metric. It is safe to call multiple times.
func RegisterIntakeMemory ¶ added in v0.4.0
func RegisterIntakeMemory(component string, m IntakeMemory) func()
func RegisterTableBudget ¶ added in v0.4.0
func RegisterTableWork ¶ added in v0.4.0
func RegisterTableWork(pipeline string, read func() map[wal.Destination]TableWork) func()
Types ¶
type HealthReporter ¶
type HealthReporter interface {
IsHealthy() bool
TransferQueueSize() int
UploadQueueSize() int
SegmentsTotal() int64
SegmentsSize() int64
UnhealthyReason() string
MaxSegmentAge() time.Duration
}
HealthReporter exposes health and queue/segment status used by the per-process metrics collection services in the collector and ingestor.
type IntakeMemory ¶ added in v0.4.0
IntakeMemory reports the bytes buffered by in-flight transfer requests and the configured ceiling, where zero means the ceiling is disabled.