metrics

package
v0.22.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 3, 2026 License: AGPL-3.0 Imports: 15 Imported by: 0

Documentation

Index

Constants

View Source
const QueryParamAgentProcessID = "agent_process_id"

QueryParamAgentProcessID is the CONNECT-IP query-string key the agent uses to tell the server its stable per-process ID. Referenced by both the client and the server handler so a rename can't silently break the wire.

View Source
const (
	// StaleResultTimeout is how long a pushed result remains valid.
	// Results older than this are skipped during collection.
	StaleResultTimeout = 60 * time.Second
)

Variables

View Source
var (

	// TunnelAgentInfo is an info metric that exports version labels. Always set to 1.
	TunnelAgentInfo = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "tunnel_agent_info",
			Help: "Agent build information. Always 1.",
		},
		[]string{"version", "build_date", "commit"},
	)
	// TunnelAgentUptimeSeconds reports agent process uptime.
	TunnelAgentUptimeSeconds = prometheus.NewGaugeFunc(
		prometheus.GaugeOpts{
			Name: "tunnel_agent_uptime_seconds",
			Help: "Seconds since the agent process started.",
		},
		func() float64 { return time.Since(startTime).Seconds() },
	)
	// TunnelRelayRTTSeconds reports the latency to each relay the agent holds a
	// live session with: the smoothed RTT QUIC continuously measures on the
	// control connection (falling back to probe/connect time before the first
	// measurement). The series is deleted when the session ends.
	TunnelRelayRTTSeconds = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "tunnel_relay_rtt_seconds",
			Help: "Smoothed round-trip time to the connected relay, as measured by QUIC.",
		},
		[]string{"relay"},
	)
	// TunnelRelayPacketsLost counts packets QUIC declared lost on the control
	// connection, by relay and loss reason ("timeout", "reordering", "other").
	// Loss rate is usually a better "tunnel feels slow" signal than RTT.
	TunnelRelayPacketsLost = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "tunnel_relay_packets_lost_total",
			Help: "Packets declared lost by QUIC on the relay control connection.",
		},
		[]string{"relay", "reason"},
	)
	// TunnelRelayPTOs counts QUIC probe timeouts on the control connection —
	// full stalls where nothing was ACKed for a whole probe interval. A rising
	// rate means the path to that relay is dying, not just congested.
	TunnelRelayPTOs = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "tunnel_relay_ptos_total",
			Help: "QUIC probe timeouts (PTO) on the relay control connection.",
		},
		[]string{"relay"},
	)
	// TunnelConnectionReconnects counts reconnection attempts across all connections.
	TunnelConnectionReconnects = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "tunnel_connection_reconnects_total",
			Help: "Total number of tunnel reconnection attempts.",
		},
	)

	// TunnelServer metrics.
	TunnelPingRequests = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "tunnel_ping_requests_total",
			Help: "Total number of ping requests for latency probing.",
		},
	)
	TunnelConnectionRequests = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "tunnel_connection_requests_total",
			Help: "Total number of connection requests to the tunnel server.",
		},
	)
	TunnelConnectionsActive = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "tunnel_connections_active",
			Help: "Number of currently active tunnel connections.",
		},
	)
	TunnelConnectionFailures = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "tunnel_connection_failures_total",
			Help: "Total number of failed connection attempts.",
		},
		[]string{"reason"},
	)
	TunnelNodesManaged = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "tunnel_nodes_managed_total",
			Help: "Number of currently managed tunnel nodes.",
		},
	)

	// MuxedConn metrics.
	TunnelPacketsSent = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "tunnel_packets_sent_total",
			Help: "Total number of packets sent through the tunnel.",
		},
	)
	TunnelBytesSent = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "tunnel_bytes_sent_total",
			Help: "Total number of bytes sent through the tunnel.",
		},
	)
	// TunnelPacketsSentErrors tracks packet send errors with labels.
	// Common error_type values: "invalid_ip", "no_tunnel", "invalid_connection_type", "write_error"
	TunnelPacketsSentErrors = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "tunnel_packets_sent_errors_total",
			Help: "Total number of packets sent through the tunnel with errors.",
		},
		[]string{"error_type"},
	)
	TunnelPacketsReceived = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "tunnel_packets_received_total",
			Help: "Total number of packets received from the tunnel.",
		},
	)
	TunnelBytesReceived = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "tunnel_bytes_received_total",
			Help: "Total number of bytes received from the tunnel.",
		},
	)
	// TunnelPacketsReceivedErrors tracks packet receive errors with labels.
	// Common error_type values: "read_error", "connection_closed"
	TunnelPacketsReceivedErrors = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "tunnel_packets_received_errors_total",
			Help: "Total number of packets received from the tunnel with errors.",
		},
		[]string{"error_type"},
	)
	// TunnelPacketsDropped tracks packets that were dropped.
	// Common reason values: "channel_full", "channel_closed"
	TunnelPacketsDropped = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "tunnel_packets_dropped_total",
			Help: "Total number of packets dropped by the tunnel.",
		},
		[]string{"reason"},
	)

	// Per-protocol packet and byte counters.
	// Protocol values: "tcp", "udp", "icmp", "other".
	// Direction values: "tx", "rx".
	TunnelPacketsByProtocol = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "tunnel_packets_by_protocol_total",
			Help: "Total packets broken down by IP protocol and direction.",
		},
		[]string{"protocol", "direction"},
	)
	TunnelBytesByProtocol = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "tunnel_bytes_by_protocol_total",
			Help: "Total bytes broken down by IP protocol and direction.",
		},
		[]string{"protocol", "direction"},
	)

	// TunnelConnectIPICMPReturned counts ICMP packets that CONNECT-IP
	// synthesized in response to an outbound-write failure (notably the QUIC
	// DatagramTooLargeError path that emits an ICMPv6 Packet-Too-Big back at
	// the sender for PMTUD). Cross-reference with tunnel_icmp6_packet_too_big_*
	// to detect PTB generation vs. propagation issues.
	// TODO(APO-543): remove once PMTUD-over-tunnel is verified healthy.
	TunnelConnectIPICMPReturned = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "tunnel_connect_ip_icmp_returned_total",
			Help: "ICMP packets synthesized by connect-ip-go on WritePacket failure (e.g. QUIC DatagramTooLargeError → ICMPv6 PTB).",
		},
	)
)

Functions

func AgentProcessID

func AgentProcessID() string

AgentProcessID returns the stable per-process ID for this agent.

func ProtocolFromIPHeader

func ProtocolFromIPHeader(proto byte) string

ProtocolFromIPHeader returns a protocol label from the IP next-header/protocol byte.

func RegisterAgentMetrics

func RegisterAgentMetrics()

RegisterAgentMetrics registers agent-only metrics (info and uptime) with the controller-runtime metrics registry. This must be called explicitly by agent processes — server processes (tunnelproxy) should NOT call this, as they re-export agent metrics via the AgentScraper/ReexportCollector path instead.

Types

type MetricsStore

type MetricsStore struct {
	// contains filtered or unexported fields
}

MetricsStore accepts pushed metrics from tunnel agents and makes them available for re-export via Prometheus. It replaces the old pull-based AgentScraper: instead of scraping agents over the overlay network, agents POST their metrics to the tunnelproxy over the existing HTTP/3 connection.

func NewMetricsStore

func NewMetricsStore() *MetricsStore

NewMetricsStore creates a new MetricsStore.

func (*MetricsStore) ForEachResult

func (s *MetricsStore) ForEachResult(fn func(connID string, result *StoreResult) bool)

ForEachResult iterates over results under a read lock. The callback must not call other MetricsStore methods (deadlock). Keep callbacks short.

func (*MetricsStore) Push

func (s *MetricsStore) Push(connID string, families map[string]*dto.MetricFamily)

Push stores a new set of metric families for the given connection, replacing any previously pushed data. Unknown connection IDs are silently ignored (the push may arrive between Unregister and the next push tick).

func (*MetricsStore) Register

func (s *MetricsStore) Register(target StoreTarget)

Register records a new agent connection. Metadata from the target is attached to every metric when re-exported. The result starts with empty Families until the first push arrives.

func (*MetricsStore) Results

func (s *MetricsStore) Results() map[string]*StoreResult

Results returns a snapshot of all current results.

func (*MetricsStore) Start

func (s *MetricsStore) Start(ctx context.Context) error

Start blocks until ctx is cancelled. It exists for compatibility with callers that run the store in an errgroup alongside other long-running goroutines.

func (*MetricsStore) Unregister

func (s *MetricsStore) Unregister(connID string)

Unregister removes an agent connection and its cached metrics.

type ProtocolCounters

type ProtocolCounters struct {
	Packets prometheus.Counter
	Bytes   prometheus.Counter
}

ProtocolCounters holds pre-resolved counters for a (protocol, direction) pair to avoid per-packet WithLabelValues lookups on the hot path.

func GetProtocolCounters

func GetProtocolCounters(proto, direction string) *ProtocolCounters

GetProtocolCounters returns pre-resolved counters for the given protocol and direction. Returns nil if the protocol string is empty.

type ReexportCollector

type ReexportCollector struct {
	// contains filtered or unexported fields
}

ReexportCollector implements prometheus.Collector by iterating over pushed agent metrics and re-emitting them with tunnel_node, agent, conn_id, agent_process_id, and project_id labels injected. It should be registered with the tunnelproxy's Prometheus registry so agent metrics appear on the tunnelproxy's /metrics endpoint.

func NewReexportCollector

func NewReexportCollector(store *MetricsStore, opts ...ReexportOption) *ReexportCollector

NewReexportCollector creates a new ReexportCollector backed by the given store.

func (*ReexportCollector) Collect

func (c *ReexportCollector) Collect(ch chan<- prometheus.Metric)

Collect implements prometheus.Collector. It takes a snapshot of the store results (briefly holding RLock), then iterates the snapshot without any lock.

func (*ReexportCollector) Describe

func (c *ReexportCollector) Describe(ch chan<- *prometheus.Desc)

Describe implements prometheus.Collector. Because the set of metrics is dynamic (depends on what the agents export), we emit no fixed descriptors and mark this as an unchecked collector.

type ReexportOption

type ReexportOption func(*ReexportCollector)

ReexportOption configures a ReexportCollector.

func WithReexportPrefix

func WithReexportPrefix(prefix string) ReexportOption

WithReexportPrefix sets a prefix added to all re-exported metric names. Defaults to "apoxy_".

type StoreResult

type StoreResult struct {
	Target StoreTarget
	// RegisteredAt is the time the connection was registered with the store.
	// Used to compute per-connection uptime independent of the agent process
	// uptime (which is shared across all conns of the same agent).
	RegisteredAt time.Time
	Families     map[string]*dto.MetricFamily
	PushedAt     time.Time
}

StoreResult holds the parsed metric families from a single agent push.

type StoreTarget

type StoreTarget struct {
	ConnID     string
	TunnelNode string
	// AgentName is the legacy label value used for the "agent" label on
	// re-exported metrics. In the tunnelproxy CONNECT-IP path it is set to
	// ConnID (so "agent" is really per-conn). Retained for dashboard
	// backwards-compatibility; prefer ConnID and AgentProcessID for new queries.
	AgentName string
	// AgentProcessID is a stable UUID for the agent *process* the connection
	// originates from. Multiple conns opened by the same agent process share
	// this value; use it to deduplicate process-scoped metrics like
	// tunnel_agent_uptime_seconds across min-conns > 1 sessions. May be empty
	// if the agent is old enough to not send it.
	AgentProcessID string
	ProjectID      string
}

StoreTarget describes an agent whose metrics are stored.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL