Documentation
¶
Overview ¶
Package streamsvc implements the Streams facade on top of the internal EventLog. It provides publish/subscribe, idempotency, retry backoff, and DLQ behavior consumed by gRPC/HTTP transports.
Example:
svc := streamsvc.New(rt)
// Publish with key and idempotency
id, _ := svc.Publish(ctx, "default", "orders", []byte("payload"), map[string]string{"idempotencyKey":"pub-1"}, "user:123")
_ = svc.Ack(ctx, "default", "orders", "workers", id)
// Subscribe using sink
_ = svc.StreamSubscribe(ctx, "default", "orders", "workers", nil, mySink)
Index ¶
- Constants
- type BackoffType
- type CreateStreamOptions
- type DLQMessageItem
- type MessageDeliveryStatus
- type MessageListItem
- type Metric
- type PartitionStat
- type RetryDLQStats
- type RetryMessageItem
- type RetryPolicy
- type SearchOptions
- type Series
- type Service
- func (s *Service) Ack(ctx context.Context, ns, stream, group string, id []byte) error
- func (s *Service) ActiveSubscribersCount(ns, stream string) int
- func (s *Service) ActiveSubscribersCountGroup(ns, stream, group string) int
- func (s *Service) CreateStream(ctx context.Context, ns, name string, partitions int) error
- func (s *Service) CreateStreamWithOptions(ctx context.Context, opts CreateStreamOptions) error
- func (s *Service) DeleteMessage(ctx context.Context, ns, stream string, messageID []byte) error
- func (s *Service) DiscoverGroupsWithRetryData(ns, stream string) []string
- func (s *Service) EnsureNamespace(ctx context.Context, ns string) (namespace.Meta, error)
- func (s *Service) FlushStream(ctx context.Context, ns, stream string, partition *uint32) (uint64, error)
- func (s *Service) GetMessageByID(ctx context.Context, ns, stream string, id []byte) (MessageListItem, error)
- func (s *Service) GetMessageDeliveryStatus(ctx context.Context, ns, stream string, id []byte) ([]MessageDeliveryStatus, error)
- func (s *Service) GetRetryDLQStats(ctx context.Context, ns, stream, group string) ([]RetryDLQStats, error)
- func (s *Service) GetSubscriptions(ctx context.Context, ns, stream string) ([]SubscriptionInfo, error)
- func (s *Service) GetWithCursorGroups(ctx context.Context, ns, stream string) ([]string, error)
- func (s *Service) GroupsCount(ctx context.Context, ns, stream string) (int, error)
- func (s *Service) LastDeliveredMs(ctx context.Context, ns, stream string) (uint64, error)
- func (s *Service) ListAllStreams(ctx context.Context) ([]StreamInfo, error)
- func (s *Service) ListDLQMessages(ctx context.Context, ns, stream, group string, startToken []byte, limit int, ...) ([]DLQMessageItem, []byte, error)
- func (s *Service) ListMessages(ctx context.Context, ns, stream string, partition uint32, startToken []byte, ...) ([]MessageListItem, []byte, error)
- func (s *Service) ListNamespaces(ctx context.Context) ([]string, error)
- func (s *Service) ListRetryMessages(ctx context.Context, ns, stream, group string, startToken []byte, limit int, ...) ([]RetryMessageItem, []byte, error)
- func (s *Service) ListStreams(ctx context.Context, ns string) ([]string, error)
- func (s *Service) Metrics(ctx context.Context, ns, ch string, metric Metric, ...) ([]Series, error)
- func (s *Service) Nack(ctx context.Context, ns, stream, group string, id []byte, errorMsg string) error
- func (s *Service) OpenLogForRead(ns, stream string, partition uint32) (*eventlog.Log, error)
- func (s *Service) Publish(ctx context.Context, ns, stream string, payload []byte, ...) ([]byte, error)
- func (s *Service) Search(ctx context.Context, ns, stream string, nextToken []byte, opts SearchOptions) ([]MessageListItem, []byte, error)
- func (s *Service) StreamStats(ctx context.Context, ns, stream string) ([]PartitionStat, uint64, error)
- func (s *Service) StreamSubscribe(ctx context.Context, ns, stream, group string, startToken []byte, ...) error
- func (s *Service) StreamTail(ctx context.Context, ns, stream string, startToken []byte, ...) error
- func (s *Service) SystemWideMetrics(ctx context.Context, ns string, metric Metric, startMs, endMs, stepMs int64, ...) (map[string]any, error)
- type StreamInfo
- type StreamMeta
- type SubscribeItem
- type SubscribeOptions
- type SubscribeSink
- type SubscriptionInfo
Constants ¶
const ( Res1m = "1m" Res5m = "5m" Res1h = "1h" )
Metric resolutions we maintain for counters.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type BackoffType ¶
type BackoffType string
const ( BackoffExp BackoffType = "exp" BackoffExpJitter BackoffType = "exp-jitter" BackoffFixed BackoffType = "fixed" BackoffNone BackoffType = "none" )
type CreateStreamOptions ¶
type CreateStreamOptions struct {
Namespace string
Name string
Partitions int
RetentionAgeMs int64
MaxLenBytes int64
}
CreateStreamOptions encapsulates options used when creating/updating a stream's metadata.
type DLQMessageItem ¶
type DLQMessageItem struct {
ID []byte `json:"id"`
Payload []byte `json:"payload"`
Headers map[string]string `json:"headers"`
Partition uint32 `json:"partition"`
Seq uint64 `json:"seq"`
RetryCount uint32 `json:"retry_count"`
MaxRetries uint32 `json:"max_retries"`
FailedAtMs int64 `json:"failed_at_ms"`
LastError string `json:"last_error"`
Group string `json:"group"`
}
DLQMessageItem represents a message in the DLQ
type MessageDeliveryStatus ¶
type MessageDeliveryStatus struct {
Group string `json:"group"`
DeliveredAt int64 `json:"delivered_at_ms"`
Acknowledged bool `json:"acknowledged"`
Nacked bool `json:"nacked"`
RetryCount uint32 `json:"retry_count"`
LastError string `json:"last_error,omitempty"`
}
MessageDeliveryStatus represents the delivery status of a message to a specific group
type MessageListItem ¶
MessageListItem is a single message returned by ListMessages.
type PartitionStat ¶
type PartitionStat struct {
Partition uint32
FirstSeq uint64
LastSeq uint64
Count uint64
Bytes uint64
// LastPublishMs is the header timestamp (ms) of the latest record in the partition.
LastPublishMs uint64
}
PartitionStat summarizes a single partition.
type RetryDLQStats ¶
type RetryDLQStats struct {
Namespace string `json:"namespace"`
Stream string `json:"stream"`
Group string `json:"group"`
RetryCount uint32 `json:"retry_count"`
DLQCount uint32 `json:"dlq_count"`
TotalRetries uint32 `json:"total_retries"`
SuccessRate float64 `json:"success_rate"`
Groups []string `json:"groups,omitempty"`
}
RetryDLQStats represents statistics for retry/DLQ data
type RetryMessageItem ¶
type RetryMessageItem struct {
ID []byte `json:"id"`
Payload []byte `json:"payload"`
Headers map[string]string `json:"headers"`
Partition uint32 `json:"partition"`
Seq uint64 `json:"seq"`
RetryCount uint32 `json:"retry_count"`
MaxRetries uint32 `json:"max_retries"`
NextRetryAtMs int64 `json:"next_retry_at_ms"`
CreatedAtMs int64 `json:"created_at_ms"`
LastError string `json:"last_error"`
Group string `json:"group"`
}
RetryMessageItem represents a message in the retry queue
type RetryPolicy ¶
type SearchOptions ¶
SearchOptions control historical search over a stream.
type Service ¶
type Service struct {
// contains filtered or unexported fields
}
Service provides publish/subscribe operations built on the internal EventLog. It implements idempotent publish, at-least-once delivery with group cursors, basic retry/DLQ, and subscribe with optional batching and concurrency.
Performance tunables (env-driven, read at construction time):
- FLO_SUB_FLUSH_MS: optional subscribe flush window in ms (default 0). When >0, the subscriber writer coalesces sends for up to the window before flushing. Small values (2–5ms) reduce flush overhead at high QPS.
- FLO_SUB_BUF: buffered stream capacity per subscriber (default 1024). Increase for bursty producers or slow transports to avoid backpressure.
func NewWithLogger ¶
NewWithLogger constructs the service with an injected logger. NewWithLogger returns a Service using the provided logger.
func (*Service) ActiveSubscribersCount ¶
ActiveSubscribersCount returns the number of active subscribers for a stream across all groups.
func (*Service) ActiveSubscribersCountGroup ¶
ActiveSubscribersCountGroup returns the number of active subscribers for a specific group on a stream.
func (*Service) CreateStream ¶
func (*Service) CreateStreamWithOptions ¶
func (s *Service) CreateStreamWithOptions(ctx context.Context, opts CreateStreamOptions) error
CreateStreamWithOptions persists partitions and optional retention/max length.
func (*Service) DeleteMessage ¶
DeleteMessage removes a specific message by ID from a stream.
func (*Service) DiscoverGroupsWithRetryData ¶
DiscoverGroupsWithRetryData finds groups that have retry or DLQ data
func (*Service) EnsureNamespace ¶
func (*Service) FlushStream ¶
func (s *Service) FlushStream(ctx context.Context, ns, stream string, partition *uint32) (uint64, error)
FlushStream removes all messages from a stream (or specific partition). If partition is nil, flushes all partitions for the stream. Returns the number of messages deleted.
func (*Service) GetMessageByID ¶
func (s *Service) GetMessageByID(ctx context.Context, ns, stream string, id []byte) (MessageListItem, error)
GetMessageByID retrieves a specific message by its ID from the main stream
func (*Service) GetMessageDeliveryStatus ¶
func (s *Service) GetMessageDeliveryStatus(ctx context.Context, ns, stream string, id []byte) ([]MessageDeliveryStatus, error)
GetMessageDeliveryStatus returns delivery status for a specific message across all groups
func (*Service) GetRetryDLQStats ¶
func (s *Service) GetRetryDLQStats(ctx context.Context, ns, stream, group string) ([]RetryDLQStats, error)
GetRetryDLQStats returns statistics for retry/DLQ data
func (*Service) GetSubscriptions ¶
func (s *Service) GetSubscriptions(ctx context.Context, ns, stream string) ([]SubscriptionInfo, error)
GetSubscriptions returns per-group cursor and lag information for a stream.
func (*Service) GetWithCursorGroups ¶
GetWithCursorGroups returns the names of all consumer groups that have cursor keys for a stream.
func (*Service) GroupsCount ¶
GroupsCount returns the number of distinct consumer groups with cursors for a stream.
func (*Service) LastDeliveredMs ¶
LastDeliveredMs returns the most recent delivery time (ms) for any group on the stream.
func (*Service) ListAllStreams ¶
func (s *Service) ListAllStreams(ctx context.Context) ([]StreamInfo, error)
ListAllStreams returns all streams across all namespaces.
func (*Service) ListDLQMessages ¶
func (s *Service) ListDLQMessages(ctx context.Context, ns, stream, group string, startToken []byte, limit int, reverse bool) ([]DLQMessageItem, []byte, error)
ListDLQMessages returns messages from the DLQ
func (*Service) ListMessages ¶
func (s *Service) ListMessages(ctx context.Context, ns, stream string, partition uint32, startToken []byte, limit int, reverse bool) ([]MessageListItem, []byte, error)
ListMessages returns up to limit messages from a specific partition starting at startToken.
func (*Service) ListNamespaces ¶
ListNamespaces returns all namespaces known to the system.
func (*Service) ListRetryMessages ¶
func (s *Service) ListRetryMessages(ctx context.Context, ns, stream, group string, startToken []byte, limit int, reverse bool) ([]RetryMessageItem, []byte, error)
ListRetryMessages returns messages from the retry queue
func (*Service) ListStreams ¶
ListStreams returns known streams for a namespace from metadata and logs.
func (*Service) Nack ¶
func (s *Service) Nack(ctx context.Context, ns, stream, group string, id []byte, errorMsg string) error
Nack increments attempts, applies exp-jitter backoff, and appends to DLQ when attempts >= defaultMaxAttempts.
func (*Service) OpenLogForRead ¶
func (*Service) Search ¶
func (s *Service) Search(ctx context.Context, ns, stream string, nextToken []byte, opts SearchOptions) ([]MessageListItem, []byte, error)
Search returns up to Limit messages starting at the specified time anchor or from/earliest. Multi-partition fan-in with a simple round-robin scheduler; pagination token encodes per-partition cursors.
func (*Service) StreamStats ¶
func (s *Service) StreamStats(ctx context.Context, ns, stream string) ([]PartitionStat, uint64, error)
StreamStats returns first/last seq and count per partition for a stream.
func (*Service) StreamSubscribe ¶
func (s *Service) StreamSubscribe(ctx context.Context, ns, stream, group string, startToken []byte, opts SubscribeOptions, sink SubscribeSink) error
StreamSubscribe supports from/at start positions.
func (*Service) StreamTail ¶
func (s *Service) StreamTail(ctx context.Context, ns, stream string, startToken []byte, opts SubscribeOptions, sink SubscribeSink) error
StreamTail streams new messages for a stream without using consumer groups, delivery tracking, retries, or acknowledgements. It is intended for UI tailing.
func (*Service) SystemWideMetrics ¶
func (s *Service) SystemWideMetrics(ctx context.Context, ns string, metric Metric, startMs, endMs, stepMs int64, byPartition bool, fill, unit string) (map[string]any, error)
SystemWideMetrics builds aggregated metrics across all streams in a namespace or all namespaces. If ns is empty, it aggregates across all namespaces.
type StreamInfo ¶
StreamInfo represents a stream with its namespace.
type StreamMeta ¶
type StreamMeta struct {
Partitions int `json:"partitions"`
Labels map[string]string `json:"labels,omitempty"`
// RetentionAgeMs trims entries older than this age when >0.
RetentionAgeMs int64 `json:"retention_age_ms,omitempty"`
// MaxLenBytes approximates total bytes cap per partition when >0.
MaxLenBytes int64 `json:"max_len_bytes,omitempty"`
}
StreamMeta stores per-stream overrides and labels (future use).
type SubscribeItem ¶
type SubscribeItem struct {
ID []byte
Payload []byte
Headers map[string]string
Partition uint32
Seq uint64
// WriteTsMs carries the original publish timestamp when available for e2e metrics.
WriteTsMs int64
}
SubscribeItem represents a delivered event for streaming.
type SubscribeOptions ¶
type SubscribeOptions struct {
From string
AtMs int64
// Filter is an optional CEL expression evaluated per message.
// When empty, all messages are delivered.
Filter string
// Limit is the maximum number of messages to deliver before stopping.
// When 0, no limit is applied and the function runs indefinitely.
Limit int
// Optional retry policy override for this subscribe. When provided with a group,
// it is persisted as the group's policy before streaming starts.
Policy *RetryPolicy
// Optional pacing override between multiple due retry deliveries. Persisted when provided.
RetryPace *time.Duration
}
SubscribeOptions controls starting position for subscribe. From: "latest" (default) or "earliest". AtMs: start at first event with header ts >= AtMs.
type SubscribeSink ¶
type SubscribeSink interface {
Send(SubscribeItem) error
Context() context.Context
Flush() error
}
SubscribeSink is implemented by transports to receive streamed items.
type SubscriptionInfo ¶
type SubscriptionInfo struct {
Group string `json:"group"`
ActiveSubscribers int `json:"active_subscribers"`
LastDeliveredMs uint64 `json:"last_delivered_ms"`
TotalLag uint64 `json:"total_lag"`
Partitions []struct {
Partition uint32 `json:"partition"`
CursorSeq uint64 `json:"cursor_seq"`
EndSeq uint64 `json:"end_seq"`
Lag uint64 `json:"lag"`
} `json:"partitions"`
}
SubscriptionInfo summarizes consumer group position and lag.