hubmetrics

var (
	TaskStarts = Default.MustRegister(Definition{
		Name:   "cloop_executor_task_starts_total",
		Help:   "Runs dispatched to an executor, by driver kind and isolation level, counted once the executor is chosen whether or not it then launches the run.",
		Type:   TypeCounter,
		Labels: []string{"executor_kind", "isolation"},
	})

	TaskCompletions = Default.MustRegister(Definition{
		Name:   "cloop_executor_task_completions_total",
		Help:   "Dispatched runs whose workload ran to completion and exited zero.",
		Type:   TypeCounter,
		Labels: []string{"executor_kind", "isolation"},
	})

	TaskFailures = Default.MustRegister(Definition{
		Name:   "cloop_executor_task_failures_total",
		Help:   "Dispatched runs that did not complete successfully. reason is one of start (it never began: the executor refused it or could not launch it), run (it launched and failed, or ended in a way its executor could not account for), cancelled (the control plane stopped it), disk_limit (its executor stopped it because its workspace grew past its disk limit).",
		Type:   TypeCounter,
		Labels: []string{"executor_kind", "isolation", "reason"},
	})

	// Buckets span a second to two hours because that is the real range: a
	// run with nothing left to do, or one that crashes on start, is over in
	// seconds; an agent task on a remote sandbox runs for an hour. Linear
	// buckets would put every interesting value in one bin at one end or the
	// other.
	TaskDuration = Default.MustRegister(Definition{
		Name:    "cloop_executor_task_duration_seconds",
		Help:    "Wall-clock duration of dispatched runs that launched and reached a terminal state, as the executor timed them.",
		Type:    TypeHistogram,
		Labels:  []string{"executor_kind", "isolation"},
		Buckets: []float64{1, 5, 15, 60, 300, 900, 1800, 3600, 7200},
	})
)

var (
	ExecutorsByState = Default.MustRegister(Definition{
		Name:   "cloop_executors",
		Help:   "Registered executors by driver kind and scheduling state (ready, degraded, unreachable, cordoned, draining). Exported by the cluster leader only.",
		Type:   TypeGauge,
		Labels: []string{"kind", "state"},
	})

	// The oldest heartbeat per kind rather than one gauge per executor.
	// The alert that matters is "some executor of this kind has gone
	// quiet", and a max answers it in one series instead of one per device
	// — which for an edge fleet is the difference between a bounded metric
	// and a per-device one.
	ExecutorHeartbeatAgeMax = Default.MustRegister(Definition{
		Name:   "cloop_executor_heartbeat_age_seconds_max",
		Help:   "Age of the least-recently-successful liveness probe across executors of this kind. Rises without bound while a node is unreachable. Exported by the cluster leader only.",
		Type:   TypeGauge,
		Labels: []string{"kind"},
	})

	Placements = Default.MustRegister(Definition{
		Name:   "cloop_executor_placements_total",
		Help:   "Placement decisions, by result (placed or failed).",
		Type:   TypeCounter,
		Labels: []string{"result"},
	})

	// constraint is pkg/executor.Constraint (24 values), the headline
	// reason Select reports. The per-candidate Detail string is not a label:
	// it interpolates executor IDs and byte counts.
	PlacementFailures = Default.MustRegister(Definition{
		Name:   "cloop_executor_placement_failures_total",
		Help:   "Placement attempts that found no eligible executor, by the constraint that rejected the most candidates.",
		Type:   TypeCounter,
		Labels: []string{"constraint"},
	})
)

var (
	LeaseEvents = Default.MustRegister(Definition{
		Name:   "cloop_secret_lease_events_total",
		Help:   "Lease lifecycle transitions by credential kind and event (issued, renewed, revoked, expired).",
		Type:   TypeCounter,
		Labels: []string{"kind", "event"},
	})

	// Per process, not per cluster: a lease's material lives with the member
	// that materialised it, so each member reports the leases it holds and
	// the sum is the cluster's.
	LeasesLive = Default.MustRegister(Definition{
		Name:   "cloop_secret_leases_live",
		Help:   "Leases this hub process issued, still holds and has not seen expire, by credential kind. Each cluster member reports its own.",
		Type:   TypeGauge,
		Labels: []string{"kind"},
	})
)

var (
	UnsealFailures = Default.MustRegister(Definition{
		Name:   "cloop_secret_unseal_failures_total",
		Help:   "Envelope opens that failed, by reason (key_unknown, key_retired, key_unavailable, seal_failed). Any non-zero rate means secrets are unreadable.",
		Type:   TypeCounter,
		Labels: []string{"reason"},
	})

	// Both read the rotation history `cloop hub key rotate` writes to the
	// shared database, so a hub cluster exports them from its leader only.
	KEKRotationRecords = Default.MustRegister(Definition{
		Name:   "cloop_secret_kek_rotation_records",
		Help:   "Progress of the current or most recent key-encryption-key rotation, by phase (total, rewrapped, skipped, failed). Exported by the cluster leader only.",
		Type:   TypeGauge,
		Labels: []string{"phase"},
	})

	KEKRotationActive = Default.MustRegister(Definition{
		Name: "cloop_secret_kek_rotation_active",
		Help: "1 while a key-encryption-key rotation is in progress, 0 otherwise. A rotation that stays at 1 across scrapes has stalled with records still wrapped under the old key. Exported by the cluster leader only.",
		Type: TypeGauge,
	})
)

var (
	SessionsCreated = Default.MustRegister(Definition{
		Name: "cloop_sessions_created_total",
		Help: "Browser sessions established after a successful OIDC exchange.",
		Type: TypeCounter,
	})

	// Counted where the session store's delete is the arbiter, so a session
	// two requests both found expired, or a janitor pass racing a sign-out,
	// ends once in this counter as it does in the audit trail.
	SessionsTerminated = Default.MustRegister(Definition{
		Name:   "cloop_sessions_terminated_total",
		Help:   "Sessions ended, by reason (idle_evicted, absolute_expired, admin_revoked, self_logout, idp_revoked, quota_evicted).",
		Type:   TypeCounter,
		Labels: []string{"reason"},
	})

	SessionsLive = Default.MustRegister(Definition{
		Name: "cloop_sessions_live",
		Help: "Sessions currently valid: neither past their absolute expiry nor idle beyond the timeout. Exported by the cluster leader only.",
		Type: TypeGauge,
	})
)

var (
	OIDCLogins = Default.MustRegister(Definition{
		Name:   "cloop_oidc_login_total",
		Help:   "Sign-in attempts that reached a verdict, by outcome (success, discovery_failed, idp_error, invalid_request, invalid_state, exchange_failed, token_invalid, session_error, state_error). A redirect to the provider is not counted until it comes back.",
		Type:   TypeCounter,
		Labels: []string{"outcome"},
	})

	// Unlabelled deliberately. The reason for a discovery failure is in the
	// hub's log and in `cloop hub doctor`, both of which can afford prose;
	// what a scrape needs is the single number an alert fires on, because
	// any non-zero rate means nobody new can sign in.
	OIDCDiscoveryFailures = Default.MustRegister(Definition{
		Name: "cloop_oidc_discovery_failures_total",
		Help: "Failures to resolve the OIDC issuer (discovery or JWKS). Any non-zero rate means no new sign-in can complete; run `cloop hub doctor` for which of the two failed and why.",
		Type: TypeCounter,
	})

	// Silent claim renewals (Task 20359): the prompt=none round trip a
	// signed-in dashboard makes from a hidden frame when the hub holds no
	// refresh token to re-assert the session's claims with. The outcomes are
	// oidcauth.Renew*, a closed set.
	//
	// Separate from OIDCLogins rather than another outcome on it: a renewal
	// runs every few minutes per open tab, so folding it in would swamp the
	// sign-in ratio an operator reads as "can people sign in". The ratio worth
	// watching here is interaction_required against ok — a hub where it climbs
	// is one whose users are being sent back to the provider mid-session, most
	// often because their browser blocks the provider's cookies inside a frame.
	OIDCRenewals = Default.MustRegister(Definition{
		Name:   "cloop_oidc_renewal_total",
		Help:   "Silent claim renewals that reached a verdict, by outcome (ok, interaction_required, no_session, not_enabled, idp_error, discovery_failed, state_error, invalid_state, exchange_failed, token_invalid, subject_mismatch, store_error). A rising interaction_required rate means users are being sent back to the provider mid-session.",
		Type:   TypeCounter,
		Labels: []string{"outcome"},
	})
)

var (
	TokenAuth = Default.MustRegister(Definition{
		Name:   "cloop_apitoken_auth_total",
		Help:   "API token verification attempts, by result (success or failure).",
		Type:   TypeCounter,
		Labels: []string{"result"},
	})

	TokenAuthFailures = Default.MustRegister(Definition{
		Name:   "cloop_apitoken_auth_failures_total",
		Help:   "API token verifications that failed, by reason (malformed, not_found, bad_secret, revoked, expired, no_roles, store_error). A bad_secret spike against many token IDs is credential stuffing; store_error is the hub, not the caller.",
		Type:   TypeCounter,
		Labels: []string{"reason"},
	})
)

var (
	WriteBackBundles = Default.MustRegister(Definition{
		Name:   "cloop_writeback_bundles_total",
		Help:   "Write-back bundles offered by executors, by result (accepted, rejected, unavailable).",
		Type:   TypeCounter,
		Labels: []string{"result"},
	})

	WriteBackRejections = Default.MustRegister(Definition{
		Name:   "cloop_writeback_rejections_total",
		Help:   "Bundles refused on their content, by reason code. Reasons are executor.WriteBackReason*; the prose refusal is in the audit trail, not here, because it interpolates branch names and commit SHAs.",
		Type:   TypeCounter,
		Labels: []string{"reason"},
	})
)

var (
	WriteBackFrameRefusals = Default.MustRegister(Definition{
		Name:   "cloop_writeback_frame_refusals_total",
		Help:   "Result frames from remote executors the hub refused before keeping their bytes, by reason (late: after the handle's final status or its result; not_requested: for a handle dispatched with no such write-back or no project seed; over_cap: past the bundle cap the handle's spec asked for; executor_budget, process_budget: holding it would cross executors.remote.max_pinned_writeback_bytes or max_pinned_writeback_total_bytes). An honest agent sends none of the first three.",
		Type:   TypeCounter,
		Labels: []string{"reason"},
	})

	WriteBackPinnedBytes = Default.MustRegister(Definition{
		Name: "cloop_writeback_pinned_bytes",
		Help: "Returned work this hub process holds in memory waiting to be collected: bundles remote executors are sending or have sent, and seeded runs' project-state documents. Bounded by executors.remote.max_pinned_writeback_total_bytes. Each cluster member reports its own.",
		Type: TypeGauge,
	})

	WriteBackPinnedBytesMax = Default.MustRegister(Definition{
		Name: "cloop_writeback_pinned_bytes_max",
		Help: "The most returned work any single remote executor has this hub process hold, against its executors.remote.max_pinned_writeback_bytes budget. Each cluster member reports its own.",
		Type: TypeGauge,
	})
)

var (
	MergeQueueDepth = Default.MustRegister(Definition{
		Name: "cloop_mergequeue_depth",
		Help: "Merges waiting to be applied. The queue is serial, so sustained depth is the integration path falling behind task completion.",
		Type: TypeGauge,
	})

	MergeOutcomes = Default.MustRegister(Definition{
		Name:   "cloop_mergequeue_merges_total",
		Help:   "Completed merge attempts, by outcome (clean, auto_resolved, unresolved, error).",
		Type:   TypeCounter,
		Labels: []string{"outcome"},
	})
)

var (
	GitProxyPushes = Default.MustRegister(Definition{
		Name:   "cloop_gitproxy_pushes_total",
		Help:   "Pushes evaluated by the interception proxy, by result (allowed or denied).",
		Type:   TypeCounter,
		Labels: []string{"result"},
	})

	GitProxyDenials = Default.MustRegister(Definition{
		Name:   "cloop_gitproxy_push_denials_total",
		Help:   "Pushes refused, by reason (ref_not_allowed, delete_denied, create_denied, update_denied, no_write, too_many_commands, push_cert). A sandbox repeatedly hitting ref_not_allowed is probing the allowlist.",
		Type:   TypeCounter,
		Labels: []string{"reason"},
	})

	// Counted rather than audited: see Proxy.reject.
	GitProxyAnonymous = Default.MustRegister(Definition{
		Name: "cloop_gitproxy_anonymous_requests_total",
		Help: "Requests to the git proxy that presented no credential and were refused before reaching a session. Most are git's own authentication challenge, answered with 401 before it retries with the session credential; a rate far above the proxy's request rate is something other than git reaching the port.",
		Type: TypeCounter,
	})
)

var (
	KubeGuardRequests = Default.MustRegister(Definition{
		Name:   "cloop_kubeguard_requests_total",
		Help:   "Kubernetes API requests evaluated by the access monitor, by result (allowed or denied).",
		Type:   TypeCounter,
		Labels: []string{"result"},
	})

	KubeGuardDenials = Default.MustRegister(Definition{
		Name:   "cloop_kubeguard_denials_total",
		Help:   "Kubernetes API requests refused, by reason (verb_not_allowed, namespace_not_allowed, cluster_scope, resource_not_allowed, dangerous_subresource, non_resource_path, protocol_upgrade, body_too_large, unauthenticated, and for writes that would mint a credential outside the monitor: service_account_token_secret, workload_service_account, host_access, rbac_binding, rbac_role, certificate_signing_request, unreadable_body). Sustained verb_not_allowed is a workload that expects write access it was not granted; dangerous_subresource is an attempt to exec into a pod; any of the credential reasons is a workload reaching for a credential of its own.",
		Type:   TypeCounter,
		Labels: []string{"reason"},
	})
)

var (
	EgressRequests = Default.MustRegister(Definition{
		Name:   "cloop_egress_requests_total",
		Help:   "Requests through the egress proxy that reached a policy verdict, by result (allowed or denied). A request that failed for want of DNS or a route is neither.",
		Type:   TypeCounter,
		Labels: []string{"result"},
	})

	// Every refusal the broker makes, which is more than the denied requests:
	// no_grant, revoked and an expired grant refuse a session before any
	// request exists, and quota_exhausted and expired also cut transfers
	// already under way.
	EgressDenials = Default.MustRegister(Definition{
		Name:   "cloop_egress_denials_total",
		Help:   "Egress refusals, by reason (no_grant, revoked, expired, host_not_allowed, port_not_allowed, method_not_allowed, destination_blocked, quota_exhausted).",
		Type:   TypeCounter,
		Labels: []string{"reason"},
	})

	EgressBytes = Default.MustRegister(Definition{
		Name:   "cloop_egress_bytes_total",
		Help:   "Bytes proxied on behalf of sandboxes, by direction (up or down).",
		Type:   TypeCounter,
		Labels: []string{"direction"},
	})

	EgressSessionsLive = Default.MustRegister(Definition{
		Name: "cloop_egress_sessions_live",
		Help: "Egress proxy sessions this process has issued and not yet closed. An expired session is closed within the proxy's reap interval.",
		Type: TypeGauge,
	})
)

var (
	QuotaEnforcementEnabled = Default.MustRegister(Definition{
		Name: "cloop_quota_enforcement_enabled",
		Help: "Whether a per-identity quota policy is in force. Exported by the cluster leader only.",
		Type: TypeGauge,
	})

	QuotaLimit = Default.MustRegister(Definition{
		Name:      "cloop_quota_limit",
		Help:      "Configured ceiling per identity and resource. Exported by the cluster leader only.",
		Type:      TypeGauge,
		Labels:    []string{"identity", "resource"},
		MaxSeries: quotaIdentityCeiling,
	})

	QuotaUsage = Default.MustRegister(Definition{
		Name:      "cloop_quota_usage",
		Help:      "Live consumption per identity and resource. Exported by the cluster leader only.",
		Type:      TypeGauge,
		Labels:    []string{"identity", "resource"},
		MaxSeries: quotaIdentityCeiling,
	})

	QuotaDenials = Default.MustRegister(Definition{
		Name:   "cloop_quota_denials_total",
		Help:   "Admission refusals since this hub started, by resource.",
		Type:   TypeCounter,
		Labels: []string{"resource"},
	})

	QuotaIdentities = Default.MustRegister(Definition{
		Name: "cloop_quota_identities",
		Help: "Identities the hub is currently accounting. Exported by the cluster leader only.",
		Type: TypeGauge,
	})

	ProjectsRegistered = Default.MustRegister(Definition{
		Name: "cloop_projects_registered",
		Help: "Projects in the hub registry. Exported by the cluster leader only.",
		Type: TypeGauge,
	})
)

var (
	AuditAppendFailures = Default.MustRegister(Definition{
		Name:   "cloop_audit_append_failures_total",
		Help:   "Audit events that could not be appended to a hash chain, by the role of the database handle and the failure class. Every one is later recorded in that chain as part of an audit.gap row, or reported on stderr if the process exits first.",
		Type:   TypeCounter,
		Labels: []string{"chain", "reason"},
	})

	AuditGapEvents = Default.MustRegister(Definition{
		Name:   "cloop_audit_gap_events_total",
		Help:   "Lost audit events that have been written into their chain as part of an audit.gap row, by the role of the database handle. Trails cloop_audit_append_failures_total by what is still unrecorded.",
		Type:   TypeCounter,
		Labels: []string{"chain"},
	})

	AuditCheckpoints = Default.MustRegister(Definition{
		Name:   "cloop_audit_checkpoints_total",
		Help:   "Audit head checkpoints the cluster leader wrote off the database, by outcome: signed under CLOOP_SECRET_KEY, unsigned because the hub has no key, or failed to reach the configured file.",
		Type:   TypeCounter,
		Labels: []string{"outcome"},
	})
)

var (
	CSPViolations = Default.MustRegister(Definition{
		Name:   "cloop_csp_violations_total",
		Help:   "Content-Security-Policy violations browsers reported, by effective directive (script-src-attr, script-src-elem, script-src, style-src-attr, style-src-elem, style-src, img-src, connect-src, frame-src, frame-ancestors, base-uri, form-action, object-src, default-src, worker-src, font-src, media-src, manifest-src, other) and by document (dashboard, glasses, sign-in, other). script-src-attr on the dashboard is an inline handler a conversion missed, or markup somebody injected: a button that does nothing, or an attack the policy stopped.",
		Type:   TypeCounter,
		Labels: []string{"directive", "document"},
	})
)

var (
	CrossOriginRefusals = Default.MustRegister(Definition{
		Name:   "cloop_cross_origin_refusals_total",
		Help:   "Requests refused because a page on another origin made them, by reason (cross_site, same_site, unknown_fetch_site, foreign_origin, media_type, unknown_host). Any sustained rate is a page somebody opened trying to drive the hub, or a proxy that hides the hub's own origin from it.",
		Type:   TypeCounter,
		Labels: []string{"reason"},
	})
)

var Default = New()

var (
	DiskFreeBytes = Default.MustRegister(Definition{
		Name:      "cloop_hub_disk_free_bytes",
		Help:      "Free space this hub process could still write on each volume it writes to, by mount point: its own state volume and its projects'. Runs pause below orchestrator.min_free_disk_mb. Each cluster member reports its own.",
		Type:      TypeGauge,
		Labels:    []string{"volume"},
		MaxSeries: DiskVolumesMax,
	})
)

var (
	SessionRestores = Default.MustRegister(Definition{
		Name:   "cloop_proxy_session_restores_total",
		Help:   "Sessions and GitHub App token slots a hub process adopting a run, or receiving a CI job's call, tried to bring back, by kind (git, kube, egress, app_token, ci) and outcome (restored; refused — its grant or rule was revoked, expired or narrowed while no process held it; expired — it lapsed first; lost — another process restored it first; failed — its credential could not be re-derived for a reason that may pass, and it is retried).",
		Type:   TypeCounter,
		Labels: []string{"kind", "outcome"},
	})
)

var (
	StateDBWALBytes = Default.MustRegister(Definition{
		Name: "cloop_statedb_wal_bytes",
		Help: "Size of the control-plane database's write-ahead log (.cloop/state.db-wal) in bytes. Read-write connections trim it to 64 MiB when they restart it and the leader truncates it while it is larger, so one that stays above that is held open by a reader or written only by builds that predate the limit. Exported by the cluster leader only.",
		Type: TypeGauge,
	})
)