Documentation
¶
Index ¶
- Constants
- Variables
- func AttachForwardingMetadata(ctx context.Context, rm RequestMetadata) context.Context
- func CheckHopCount(hopCount int) error
- func IncrementHopCount(ctx context.Context, localNodeID string) (context.Context, error)
- func IsRetryableError(err error) bool
- func IsRoutingError(err error) bool
- func IsTemporaryError(err error) bool
- func SetResponseMetadata(ctx context.Context, resp ResponseMetadata) error
- func StreamClientEpochInterceptor(epochGetter EpochGetter, onEpochMismatch func(clientEpoch, serverEpoch uint64)) grpc.StreamClientInterceptor
- func StreamServerEpochInterceptor(epochGetter EpochGetter) grpc.StreamServerInterceptor
- func StreamServerRecoveryInterceptor() grpc.StreamServerInterceptor
- func UnaryClientEpochInterceptor(epochGetter EpochGetter, onEpochMismatch func(clientEpoch, serverEpoch uint64)) grpc.UnaryClientInterceptor
- func UnaryServerEpochInterceptor(epochGetter EpochGetter) grpc.UnaryServerInterceptor
- func UnaryServerRecoveryInterceptor() grpc.UnaryServerInterceptor
- type Config
- type ConnectionStats
- type Coordinator
- func (c *Coordinator) ErrorChan() <-chan error
- func (c *Coordinator) GetClusterState(ctx context.Context, req *clusterpb.Empty) (*clusterpb.ClusterState, error)
- func (c *Coordinator) GetClusterTopology(ctx context.Context, req *clusterpb.Empty) (*clusterpb.ClusterTopology, error)
- func (c *Coordinator) GetEpoch() uint64
- func (c *Coordinator) GetLocalNodeID() string
- func (c *Coordinator) GetNodeForKey(key string) (*ring.NodeInfo, error)
- func (c *Coordinator) GetRing() *ring.RingManager
- func (c *Coordinator) GetRouter() *Router
- func (c *Coordinator) IsLocal(key string) bool
- func (c *Coordinator) IsReady() bool
- func (c *Coordinator) MarkReady()
- func (c *Coordinator) Route(key string) (pb.CacheServiceClient, error)
- func (c *Coordinator) Start(ctx context.Context) error
- func (c *Coordinator) Stop() error
- func (c *Coordinator) WaitReady(ctx context.Context) error
- type EpochGetter
- type RequestMetadata
- type ResponseMetadata
- type Ring
- type Router
- func (r *Router) Close() error
- func (r *Router) GetClientForNode(nodeID string) (pb.CacheServiceClient, error)
- func (r *Router) GetConnectionStats() map[string]ConnectionStats
- func (r *Router) IsLocal(key string) bool
- func (r *Router) RefreshConnections()
- func (r *Router) RemoveClient(nodeID string)
- func (r *Router) Route(key string) (pb.CacheServiceClient, error)
- func (r *Router) RouteWithRetry(key string, maxRetries int) (pb.CacheServiceClient, error)
- type RouterConfig
- type RouterError
- func NewCircuitBreakerOpenError(nodeID string) *RouterError
- func NewConnectionFailedError(nodeID string, address string, cause error) *RouterError
- func NewLocalRoutingError(nodeID string, key string) *RouterError
- func NewMaxRetriesExceededError(nodeID string, key string, attempts int, cause error) *RouterError
- func NewNodeNotFoundError(nodeID string, key string) *RouterError
Constants ¶
const ( // MetadataKeyRingEpoch is the key for the ring epoch in gRPC metadata MetadataKeyRingEpoch = "x-ring-epoch" // MetadataKeyHop is the key for the hop count in gRPC metadata MetadataKeyHop = "x-hop" // MetadataKeyForwarded indicates if the request was forwarded MetadataKeyForwarded = "x-forwarded" // MetadataKeyOrigin is the node that originally received the request MetadataKeyOrigin = "x-origin" // MetadataKeyOwner is the canonical owner of the key MetadataKeyOwner = "x-owner" // MetadataKeyForwardedBy is the node that forwarded the request MetadataKeyForwardedBy = "x-forwarded-by" // MaxHops is the maximum number of hops allowed for a request // Prevents infinite forwarding loops when nodes disagree on ownership MaxHops = 3 )
const ( // MaxMessageSize is the maximum message size for gRPC messages MaxMessageSize = 128 * 1024 * 1024 // 128MB )
Variables ¶
var ( // ErrNodeNotFound indicates the target node doesn't exist in the ring ErrNodeNotFound = errors.New("node not found in ring") // ErrCircuitBreakerOpen indicates the circuit breaker is open for a node ErrCircuitBreakerOpen = errors.New("circuit breaker open") // ErrLocalRouting indicates an attempt to route to the local node ErrLocalRouting = errors.New("cannot route to local node") // ErrNoAvailableNode indicates no node is available for the key ErrNoAvailableNode = errors.New("no available node for key") // ErrConnectionFailed indicates failure to establish connection ErrConnectionFailed = errors.New("failed to establish connection") // ErrMaxRetriesExceeded indicates all retry attempts failed ErrMaxRetriesExceeded = errors.New("max retries exceeded") )
Router error types
Functions ¶
func AttachForwardingMetadata ¶
func AttachForwardingMetadata(ctx context.Context, rm RequestMetadata) context.Context
AttachForwardingMetadata attaches forwarding metadata to outgoing context
func CheckHopCount ¶
CheckHopCount validates that the hop count hasn't exceeded MaxHops. Returns an error if the limit is exceeded, nil otherwise. This is the single source of truth for hop count validation.
func IncrementHopCount ¶
IncrementHopCount extracts metadata, increments hop count, and returns new context
func IsRetryableError ¶
IsRetryableError checks if an error is retryable
func IsRoutingError ¶
IsRoutingError checks if an error is a transient routing error that should be retried
func IsTemporaryError ¶
IsTemporaryError checks if an error is temporary and might succeed on retry
func SetResponseMetadata ¶
func SetResponseMetadata(ctx context.Context, resp ResponseMetadata) error
SetResponseMetadata sets response metadata headers
func StreamClientEpochInterceptor ¶
func StreamClientEpochInterceptor(epochGetter EpochGetter, onEpochMismatch func(clientEpoch, serverEpoch uint64)) grpc.StreamClientInterceptor
StreamClientEpochInterceptor creates a gRPC stream client interceptor that: 1. Attaches client epoch to outgoing requests 2. Extracts server epoch from response headers on first message (for cache invalidation)
func StreamServerEpochInterceptor ¶
func StreamServerEpochInterceptor(epochGetter EpochGetter) grpc.StreamServerInterceptor
StreamServerEpochInterceptor creates a gRPC stream interceptor that: 1. Checks hop count and rejects if exceeded 2. Adds epoch to response metadata on first message
func StreamServerRecoveryInterceptor ¶
func StreamServerRecoveryInterceptor() grpc.StreamServerInterceptor
StreamServerRecoveryInterceptor is the streaming counterpart to UnaryServerRecoveryInterceptor. Install it as the outermost stream interceptor.
func UnaryClientEpochInterceptor ¶
func UnaryClientEpochInterceptor(epochGetter EpochGetter, onEpochMismatch func(clientEpoch, serverEpoch uint64)) grpc.UnaryClientInterceptor
UnaryClientEpochInterceptor creates a gRPC unary client interceptor that: 1. Attaches client epoch to outgoing requests 2. Extracts server epoch from responses (for cache invalidation)
func UnaryServerEpochInterceptor ¶
func UnaryServerEpochInterceptor(epochGetter EpochGetter) grpc.UnaryServerInterceptor
UnaryServerEpochInterceptor creates a gRPC unary interceptor that: 1. Checks hop count and rejects if exceeded 2. Adds epoch to response metadata
func UnaryServerRecoveryInterceptor ¶
func UnaryServerRecoveryInterceptor() grpc.UnaryServerInterceptor
UnaryServerRecoveryInterceptor returns a gRPC unary interceptor that recovers from panics in downstream interceptors and handlers, failing only that single RPC with a codes.Internal error instead of letting the panic unwind and crash the whole process. gRPC does not recover handler panics by default, so without this a single poison request (e.g. a nil-deref on a corrupt key) takes down the node. This gives the inter-node gRPC path the same per-request isolation that net/http already provides on the gateway path (issue #150).
Install it as the OUTERMOST interceptor so it also covers panics raised by inner interceptors.
Types ¶
type Config ¶
type Config struct {
Enabled bool // Whether the coordinator is enabled
MyNodeID string // The ID of the node
ClusterAddr string // The address for memberlist gossip (host:port format, e.g., "0.0.0.0:7946")
ListenAddr string // The address the node listens on for client requests (Put/Get/Delete and cluster topology)
AdvertiseAddr string // The address advertised to other nodes for routing (if different from ListenAddr)
Seeds []string // Seed nodes for joining cluster (memberlist addresses of other nodes)
DiskPath string // The path to the disk for persisting ring tokens
// LifecyclerConfig allows advanced ring configuration (optional).
// Mainly used for testing.
LifecyclerConfig ring.LifecyclerConfig
// Router configuration
RouterConfig *RouterConfig
// GRPCDialOptions are additional gRPC dial options for inter-node connections.
// These are passed through to the Router and appended to outgoing connections.
GRPCDialOptions []grpc.DialOption
// Registerer is the prometheus registerer to use. If nil, uses prometheus.DefaultRegisterer.
// This is useful for tests to avoid duplicate registration panics.
Registerer prometheus.Registerer
}
Config contains the configuration for the coordinator
type ConnectionStats ¶
type ConnectionStats struct {
State string
FailureCount int32
CircuitOpen bool
LastFailure time.Time
CircuitOpenTime time.Time
}
ConnectionStats represents statistics for a single connection
type Coordinator ¶
type Coordinator struct {
clusterpb.UnimplementedClusterServiceServer
// contains filtered or unexported fields
}
Coordinator manages cluster membership, request routing, and cluster RPC handling. Uses dskit ring + memberlist for gossip-based membership. Note: GetClusterState/GetClusterTopology RPCs are registered on the main server gRPC service.
func (*Coordinator) ErrorChan ¶
func (c *Coordinator) ErrorChan() <-chan error
ErrorChan returns a channel for receiving fatal coordinator errors
func (*Coordinator) GetClusterState ¶
func (c *Coordinator) GetClusterState(ctx context.Context, req *clusterpb.Empty) (*clusterpb.ClusterState, error)
GetClusterState returns current cluster membership for clients
func (*Coordinator) GetClusterTopology ¶
func (c *Coordinator) GetClusterTopology(ctx context.Context, req *clusterpb.Empty) (*clusterpb.ClusterTopology, error)
GetClusterTopology returns full cluster topology including token assignments for routing
func (*Coordinator) GetEpoch ¶
func (c *Coordinator) GetEpoch() uint64
GetEpoch returns the current ring epoch
func (*Coordinator) GetLocalNodeID ¶
func (c *Coordinator) GetLocalNodeID() string
GetLocalNodeID returns the ID of the local node
func (*Coordinator) GetNodeForKey ¶
func (c *Coordinator) GetNodeForKey(key string) (*ring.NodeInfo, error)
GetNodeForKey returns the node for the given key
func (*Coordinator) GetRing ¶
func (c *Coordinator) GetRing() *ring.RingManager
GetRing returns the ring manager
func (*Coordinator) GetRouter ¶
func (c *Coordinator) GetRouter() *Router
GetRouter returns the router
func (*Coordinator) IsLocal ¶
func (c *Coordinator) IsLocal(key string) bool
IsLocal checks if the key belongs to the local node
func (*Coordinator) IsReady ¶
func (c *Coordinator) IsReady() bool
IsReady returns true if the coordinator is ready to serve requests
func (*Coordinator) MarkReady ¶
func (c *Coordinator) MarkReady()
MarkReady signals that this node can serve requests, allowing it to advertise ACTIVE in the ring. Call it once storage has booted and the gRPC server is listening; until then the node stays JOINING and peers do not route to it (issue #164). Idempotent.
func (*Coordinator) Route ¶
func (c *Coordinator) Route(key string) (pb.CacheServiceClient, error)
Route returns a client for routing requests for the given key
func (*Coordinator) Start ¶
func (c *Coordinator) Start(ctx context.Context) error
Start starts the coordinator and joins the cluster
func (*Coordinator) Stop ¶
func (c *Coordinator) Stop() error
Stop stops the coordinator and cleans up resources
type EpochGetter ¶
type EpochGetter func() uint64
EpochGetter is a function that returns the current ring epoch
type RequestMetadata ¶
RequestMetadata holds metadata extracted from incoming requests
func ExtractRequestMetadata ¶
func ExtractRequestMetadata(ctx context.Context) RequestMetadata
ExtractRequestMetadata extracts routing metadata from the incoming context
type ResponseMetadata ¶
ResponseMetadata holds metadata to add to outgoing responses
type Ring ¶
type Ring interface {
// GetNode returns the node that owns the given key
GetNode(key string) (*ring.NodeInfo, error)
// GetAllNodes returns all nodes in the ring
GetAllNodes() []*ring.NodeInfo
// GetActiveNodes returns all active nodes in the ring
GetActiveNodes() []*ring.NodeInfo
// IsLocal returns true if the local node owns the given key
IsLocal(key string) bool
}
Ring defines the interface for the hash ring used by the router. This interface allows for testing with mock implementations.
type Router ¶
type Router struct {
// contains filtered or unexported fields
}
Router is a router for routing requests to the appropriate node
func NewRouterWithConfig ¶
func NewRouterWithConfig(ring Ring, localID string, config *RouterConfig) *Router
NewRouterWithConfig creates a new router with a custom configuration
func (*Router) GetClientForNode ¶
func (r *Router) GetClientForNode(nodeID string) (pb.CacheServiceClient, error)
GetClientForNode returns a client for a specific node ID This is useful for operations that need to query all nodes (e.g., List)
func (*Router) GetConnectionStats ¶
func (r *Router) GetConnectionStats() map[string]ConnectionStats
GetConnectionStats returns statistics about current connections
func (*Router) RefreshConnections ¶
func (r *Router) RefreshConnections()
RefreshConnections removes connections to inactive nodes
func (*Router) RemoveClient ¶
RemoveClient removes and closes the client connection for a node
func (*Router) Route ¶
func (r *Router) Route(key string) (pb.CacheServiceClient, error)
Route returns a client for routing requests for the given key Returns an error if the key should be handled locally (defensive check)
func (*Router) RouteWithRetry ¶
RouteWithRetry returns a client for routing with configurable retry attempts Returns an error if the key maps to the local node (this should not happen as callers should check IsLocal first, but we check defensively)
type RouterConfig ¶
type RouterConfig struct {
// Connection timeout for establishing new connections
ConnectionTimeout time.Duration
// Maximum message size for sending (in bytes)
MaxSendMsgSize int
// Maximum message size for receiving (in bytes)
MaxRecvMsgSize int
// Number of retry attempts for transient failures
MaxRetries int
// Initial retry backoff duration
InitialRetryBackoff time.Duration
// Maximum retry backoff duration
MaxRetryBackoff time.Duration
// Keepalive parameters
KeepaliveTime time.Duration // Send keepalive ping every this duration
KeepaliveTimeout time.Duration // Wait this long for keepalive response
// Circuit breaker parameters
CircuitBreakerThreshold int // Number of consecutive failures to open circuit
CircuitBreakerTimeout time.Duration // How long to wait before attempting to close circuit
// GRPCDialOptions are additional gRPC dial options applied to all outgoing connections.
// These are appended after the default options (transport credentials, keepalive, message size).
GRPCDialOptions []grpc.DialOption
}
RouterConfig contains configuration for the Router
func DefaultRouterConfig ¶
func DefaultRouterConfig() *RouterConfig
DefaultRouterConfig returns a RouterConfig with sensible defaults
type RouterError ¶
type RouterError struct {
Type error // The base error type
NodeID string // The node that caused the error
Key string // The key being routed (if applicable)
Message string // Additional context
Cause error // The underlying error (if any)
}
RouterError represents a routing error with additional context
func NewCircuitBreakerOpenError ¶
func NewCircuitBreakerOpenError(nodeID string) *RouterError
NewCircuitBreakerOpenError creates a new circuit breaker open error
func NewConnectionFailedError ¶
func NewConnectionFailedError(nodeID string, address string, cause error) *RouterError
NewConnectionFailedError creates a new connection failed error
func NewLocalRoutingError ¶
func NewLocalRoutingError(nodeID string, key string) *RouterError
NewLocalRoutingError creates a new local routing error
func NewMaxRetriesExceededError ¶
func NewMaxRetriesExceededError(nodeID string, key string, attempts int, cause error) *RouterError
NewMaxRetriesExceededError creates a new max retries exceeded error
func NewNodeNotFoundError ¶
func NewNodeNotFoundError(nodeID string, key string) *RouterError
NewNodeNotFoundError creates a new node not found error
func (*RouterError) Error ¶
func (e *RouterError) Error() string
Error implements the error interface
func (*RouterError) Is ¶
func (e *RouterError) Is(target error) bool
Is implements error matching for RouterError
func (*RouterError) Unwrap ¶
func (e *RouterError) Unwrap() error
Unwrap returns the underlying error type for errors.Is