streamsvc

package
v0.0.0-...-ae17176 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 7, 2025 License: MIT Imports: 20 Imported by: 0

Documentation

Overview

Package streamsvc implements the Streams facade on top of the internal EventLog. It provides publish/subscribe, idempotency, retry backoff, and DLQ behavior consumed by gRPC/HTTP transports.

Example:

svc := streamsvc.New(rt)
// Publish with key and idempotency
id, _ := svc.Publish(ctx, "default", "orders", []byte("payload"), map[string]string{"idempotencyKey":"pub-1"}, "user:123")
_ = svc.Ack(ctx, "default", "orders", "workers", id)
// Subscribe using sink
_ = svc.StreamSubscribe(ctx, "default", "orders", "workers", nil, mySink)

Index

Constants

View Source
const (
	Res1m = "1m"
	Res5m = "5m"
	Res1h = "1h"
)

Metric resolutions we maintain for counters.

Variables

This section is empty.

Functions

This section is empty.

Types

type BackoffType

type BackoffType string
const (
	BackoffExp       BackoffType = "exp"
	BackoffExpJitter BackoffType = "exp-jitter"
	BackoffFixed     BackoffType = "fixed"
	BackoffNone      BackoffType = "none"
)

type CreateStreamOptions

type CreateStreamOptions struct {
	Namespace      string
	Name           string
	Partitions     int
	RetentionAgeMs int64
	MaxLenBytes    int64
}

CreateStreamOptions encapsulates options used when creating/updating a stream's metadata.

type DLQMessageItem

type DLQMessageItem struct {
	ID         []byte            `json:"id"`
	Payload    []byte            `json:"payload"`
	Headers    map[string]string `json:"headers"`
	Partition  uint32            `json:"partition"`
	Seq        uint64            `json:"seq"`
	RetryCount uint32            `json:"retry_count"`
	MaxRetries uint32            `json:"max_retries"`
	FailedAtMs int64             `json:"failed_at_ms"`
	LastError  string            `json:"last_error"`
	Group      string            `json:"group"`
}

DLQMessageItem represents a message in the DLQ

type MessageDeliveryStatus

type MessageDeliveryStatus struct {
	Group        string `json:"group"`
	DeliveredAt  int64  `json:"delivered_at_ms"`
	Acknowledged bool   `json:"acknowledged"`
	Nacked       bool   `json:"nacked"`
	RetryCount   uint32 `json:"retry_count"`
	LastError    string `json:"last_error,omitempty"`
}

MessageDeliveryStatus represents the delivery status of a message to a specific group

type MessageListItem

type MessageListItem struct {
	ID        []byte
	Payload   []byte
	Header    []byte
	Partition uint32
	Seq       uint64
}

MessageListItem is a single message returned by ListMessages.

type Metric

type Metric string
const (
	MetricPublishCount Metric = "publish_count"
	MetricAckCount     Metric = "ack_count"
	MetricPublishRate  Metric = "publish_rate"
	MetricBytesRate    Metric = "bytes_rate"
)

type PartitionStat

type PartitionStat struct {
	Partition uint32
	FirstSeq  uint64
	LastSeq   uint64
	Count     uint64
	Bytes     uint64
	// LastPublishMs is the header timestamp (ms) of the latest record in the partition.
	LastPublishMs uint64
}

PartitionStat summarizes a single partition.

type RetryDLQStats

type RetryDLQStats struct {
	Namespace    string   `json:"namespace"`
	Stream       string   `json:"stream"`
	Group        string   `json:"group"`
	RetryCount   uint32   `json:"retry_count"`
	DLQCount     uint32   `json:"dlq_count"`
	TotalRetries uint32   `json:"total_retries"`
	SuccessRate  float64  `json:"success_rate"`
	Groups       []string `json:"groups,omitempty"`
}

RetryDLQStats represents statistics for retry/DLQ data

type RetryMessageItem

type RetryMessageItem struct {
	ID            []byte            `json:"id"`
	Payload       []byte            `json:"payload"`
	Headers       map[string]string `json:"headers"`
	Partition     uint32            `json:"partition"`
	Seq           uint64            `json:"seq"`
	RetryCount    uint32            `json:"retry_count"`
	MaxRetries    uint32            `json:"max_retries"`
	NextRetryAtMs int64             `json:"next_retry_at_ms"`
	CreatedAtMs   int64             `json:"created_at_ms"`
	LastError     string            `json:"last_error"`
	Group         string            `json:"group"`
}

RetryMessageItem represents a message in the retry queue

type RetryPolicy

type RetryPolicy struct {
	Type        BackoffType
	Base        time.Duration
	Cap         time.Duration
	Factor      float64
	MaxAttempts uint32
}

type SearchOptions

type SearchOptions struct {
	From    string
	AtMs    int64
	Reverse bool
	Limit   int
	Filter  string
}

SearchOptions control historical search over a stream.

type Series

type Series struct {
	Label  string       `json:"label"`
	Points [][2]float64 `json:"points"` // [t_ms, value]
}

type Service

type Service struct {
	// contains filtered or unexported fields
}

Service provides publish/subscribe operations built on the internal EventLog. It implements idempotent publish, at-least-once delivery with group cursors, basic retry/DLQ, and subscribe with optional batching and concurrency.

Performance tunables (env-driven, read at construction time):

  • FLO_SUB_FLUSH_MS: optional subscribe flush window in ms (default 0). When >0, the subscriber writer coalesces sends for up to the window before flushing. Small values (2–5ms) reduce flush overhead at high QPS.
  • FLO_SUB_BUF: buffered stream capacity per subscriber (default 1024). Increase for bursty producers or slow transports to avoid backpressure.

func New

func New(rt *runtime.Runtime) *Service

New returns a Service using a default logger.

func NewWithLogger

func NewWithLogger(rt *runtime.Runtime, logger logpkg.Logger) *Service

NewWithLogger constructs the service with an injected logger. NewWithLogger returns a Service using the provided logger.

func (*Service) Ack

func (s *Service) Ack(ctx context.Context, ns, stream, group string, id []byte) error

Ack marks a message as acknowledged.

func (*Service) ActiveSubscribersCount

func (s *Service) ActiveSubscribersCount(ns, stream string) int

ActiveSubscribersCount returns the number of active subscribers for a stream across all groups.

func (*Service) ActiveSubscribersCountGroup

func (s *Service) ActiveSubscribersCountGroup(ns, stream, group string) int

ActiveSubscribersCountGroup returns the number of active subscribers for a specific group on a stream.

func (*Service) CreateStream

func (s *Service) CreateStream(ctx context.Context, ns, name string, partitions int) error

func (*Service) CreateStreamWithOptions

func (s *Service) CreateStreamWithOptions(ctx context.Context, opts CreateStreamOptions) error

CreateStreamWithOptions persists partitions and optional retention/max length.

func (*Service) DeleteMessage

func (s *Service) DeleteMessage(ctx context.Context, ns, stream string, messageID []byte) error

DeleteMessage removes a specific message by ID from a stream.

func (*Service) DiscoverGroupsWithRetryData

func (s *Service) DiscoverGroupsWithRetryData(ns, stream string) []string

DiscoverGroupsWithRetryData finds groups that have retry or DLQ data

func (*Service) EnsureNamespace

func (s *Service) EnsureNamespace(ctx context.Context, ns string) (namespace.Meta, error)

func (*Service) FlushStream

func (s *Service) FlushStream(ctx context.Context, ns, stream string, partition *uint32) (uint64, error)

FlushStream removes all messages from a stream (or specific partition). If partition is nil, flushes all partitions for the stream. Returns the number of messages deleted.

func (*Service) GetMessageByID

func (s *Service) GetMessageByID(ctx context.Context, ns, stream string, id []byte) (MessageListItem, error)

GetMessageByID retrieves a specific message by its ID from the main stream

func (*Service) GetMessageDeliveryStatus

func (s *Service) GetMessageDeliveryStatus(ctx context.Context, ns, stream string, id []byte) ([]MessageDeliveryStatus, error)

GetMessageDeliveryStatus returns delivery status for a specific message across all groups

func (*Service) GetRetryDLQStats

func (s *Service) GetRetryDLQStats(ctx context.Context, ns, stream, group string) ([]RetryDLQStats, error)

GetRetryDLQStats returns statistics for retry/DLQ data

func (*Service) GetSubscriptions

func (s *Service) GetSubscriptions(ctx context.Context, ns, stream string) ([]SubscriptionInfo, error)

GetSubscriptions returns per-group cursor and lag information for a stream.

func (*Service) GetWithCursorGroups

func (s *Service) GetWithCursorGroups(ctx context.Context, ns, stream string) ([]string, error)

GetWithCursorGroups returns the names of all consumer groups that have cursor keys for a stream.

func (*Service) GroupsCount

func (s *Service) GroupsCount(ctx context.Context, ns, stream string) (int, error)

GroupsCount returns the number of distinct consumer groups with cursors for a stream.

func (*Service) LastDeliveredMs

func (s *Service) LastDeliveredMs(ctx context.Context, ns, stream string) (uint64, error)

LastDeliveredMs returns the most recent delivery time (ms) for any group on the stream.

func (*Service) ListAllStreams

func (s *Service) ListAllStreams(ctx context.Context) ([]StreamInfo, error)

ListAllStreams returns all streams across all namespaces.

func (*Service) ListDLQMessages

func (s *Service) ListDLQMessages(ctx context.Context, ns, stream, group string, startToken []byte, limit int, reverse bool) ([]DLQMessageItem, []byte, error)

ListDLQMessages returns messages from the DLQ

func (*Service) ListMessages

func (s *Service) ListMessages(ctx context.Context, ns, stream string, partition uint32, startToken []byte, limit int, reverse bool) ([]MessageListItem, []byte, error)

ListMessages returns up to limit messages from a specific partition starting at startToken.

func (*Service) ListNamespaces

func (s *Service) ListNamespaces(ctx context.Context) ([]string, error)

ListNamespaces returns all namespaces known to the system.

func (*Service) ListRetryMessages

func (s *Service) ListRetryMessages(ctx context.Context, ns, stream, group string, startToken []byte, limit int, reverse bool) ([]RetryMessageItem, []byte, error)

ListRetryMessages returns messages from the retry queue

func (*Service) ListStreams

func (s *Service) ListStreams(ctx context.Context, ns string) ([]string, error)

ListStreams returns known streams for a namespace from metadata and logs.

func (*Service) Metrics

func (s *Service) Metrics(ctx context.Context, ns, ch string, metric Metric, startMs, endMs, stepMs int64, byPartition bool, fill string) ([]Series, error)

func (*Service) Nack

func (s *Service) Nack(ctx context.Context, ns, stream, group string, id []byte, errorMsg string) error

Nack increments attempts, applies exp-jitter backoff, and appends to DLQ when attempts >= defaultMaxAttempts.

func (*Service) OpenLogForRead

func (s *Service) OpenLogForRead(ns, stream string, partition uint32) (*eventlog.Log, error)

func (*Service) Publish

func (s *Service) Publish(ctx context.Context, ns, stream string, payload []byte, headers map[string]string, key string) ([]byte, error)

func (*Service) Search

func (s *Service) Search(ctx context.Context, ns, stream string, nextToken []byte, opts SearchOptions) ([]MessageListItem, []byte, error)

Search returns up to Limit messages starting at the specified time anchor or from/earliest. Multi-partition fan-in with a simple round-robin scheduler; pagination token encodes per-partition cursors.

func (*Service) StreamStats

func (s *Service) StreamStats(ctx context.Context, ns, stream string) ([]PartitionStat, uint64, error)

StreamStats returns first/last seq and count per partition for a stream.

func (*Service) StreamSubscribe

func (s *Service) StreamSubscribe(ctx context.Context, ns, stream, group string, startToken []byte, opts SubscribeOptions, sink SubscribeSink) error

StreamSubscribe supports from/at start positions.

func (*Service) StreamTail

func (s *Service) StreamTail(ctx context.Context, ns, stream string, startToken []byte, opts SubscribeOptions, sink SubscribeSink) error

StreamTail streams new messages for a stream without using consumer groups, delivery tracking, retries, or acknowledgements. It is intended for UI tailing.

func (*Service) SystemWideMetrics

func (s *Service) SystemWideMetrics(ctx context.Context, ns string, metric Metric, startMs, endMs, stepMs int64, byPartition bool, fill, unit string) (map[string]any, error)

SystemWideMetrics builds aggregated metrics across all streams in a namespace or all namespaces. If ns is empty, it aggregates across all namespaces.

type StreamInfo

type StreamInfo struct {
	Namespace string `json:"namespace"`
	Stream    string `json:"stream"`
}

StreamInfo represents a stream with its namespace.

type StreamMeta

type StreamMeta struct {
	Partitions int               `json:"partitions"`
	Labels     map[string]string `json:"labels,omitempty"`
	// RetentionAgeMs trims entries older than this age when >0.
	RetentionAgeMs int64 `json:"retention_age_ms,omitempty"`
	// MaxLenBytes approximates total bytes cap per partition when >0.
	MaxLenBytes int64 `json:"max_len_bytes,omitempty"`
}

StreamMeta stores per-stream overrides and labels (future use).

type SubscribeItem

type SubscribeItem struct {
	ID        []byte
	Payload   []byte
	Headers   map[string]string
	Partition uint32
	Seq       uint64
	// WriteTsMs carries the original publish timestamp when available for e2e metrics.
	WriteTsMs int64
}

SubscribeItem represents a delivered event for streaming.

type SubscribeOptions

type SubscribeOptions struct {
	From string
	AtMs int64
	// Filter is an optional CEL expression evaluated per message.
	// When empty, all messages are delivered.
	Filter string
	// Limit is the maximum number of messages to deliver before stopping.
	// When 0, no limit is applied and the function runs indefinitely.
	Limit int
	// Optional retry policy override for this subscribe. When provided with a group,
	// it is persisted as the group's policy before streaming starts.
	Policy *RetryPolicy
	// Optional pacing override between multiple due retry deliveries. Persisted when provided.
	RetryPace *time.Duration
}

SubscribeOptions controls starting position for subscribe. From: "latest" (default) or "earliest". AtMs: start at first event with header ts >= AtMs.

type SubscribeSink

type SubscribeSink interface {
	Send(SubscribeItem) error
	Context() context.Context
	Flush() error
}

SubscribeSink is implemented by transports to receive streamed items.

type SubscriptionInfo

type SubscriptionInfo struct {
	Group             string `json:"group"`
	ActiveSubscribers int    `json:"active_subscribers"`
	LastDeliveredMs   uint64 `json:"last_delivered_ms"`
	TotalLag          uint64 `json:"total_lag"`
	Partitions        []struct {
		Partition uint32 `json:"partition"`
		CursorSeq uint64 `json:"cursor_seq"`
		EndSeq    uint64 `json:"end_seq"`
		Lag       uint64 `json:"lag"`
	} `json:"partitions"`
}

SubscriptionInfo summarizes consumer group position and lag.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL