common

package
v0.0.0-...-d038e5a Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 4, 2026 License: MIT Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source
const (
	DefaultNATSPort           = 4222
	DefaultNATSStartupTimeout = 5 * time.Second
	NATSQueueDriver           = "nats"
	NoopDriver                = "noop"

	// ObjectStorageTypeNATS is the ObjectStorageConfig.Type value selecting
	// the NATS ObjectStore backend for oversize/snapshot event storage.
	ObjectStorageTypeNATS = "nats"
	ObjectStorageTypeS3   = "s3"
	ObjectStorageTypeFile = "file"
	ObjectStorageTypeGCS  = "gcs"

	SiphonStreamCreatedHeader = "siphon_stream_created_at"

	// SiphonEventHeader, SnapshotCompleteEvent, and SnapshotCompleteOnTemp
	// identify the internal snapshot-complete merge sentinels. Canonical home
	// here so the producer (pkg/siphon) and the consumer share one source.
	SiphonEventHeader      = "siphon-event"
	SnapshotCompleteEvent  = "snapshot-complete"
	SnapshotCompleteOnTemp = "snapshot-complete-on-temp"

	// SiphonLSNHeader carries the PostgreSQL LSN of the replication event
	// as a decimal string. The producer stamps it on every flush; failover
	// mode requires it to construct lex-sortable object names so drain can
	// replay events in LSN order.
	SiphonLSNHeader = "X-Siphon-LSN"
)

Variables

View Source
var DurationMsBuckets = []float64{1, 3, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000}

Functions

func AlphaNumDashUnderscoreValidator

func AlphaNumDashUnderscoreValidator(fl validator.FieldLevel) bool

func BuildHTTPHandler

func BuildHTTPHandler(port int, mux *http.ServeMux) *http.Server

func CommonPublishOptions

func CommonPublishOptions() []jetstream.PublishOpt

func Compress

func Compress(data []byte) ([]byte, error)

func ConsumerOptions

func ConsumerOptions(
	name string,
	subjects []string,
	ackWaitSeconds int,
) *jetstream.ConsumerConfig

func Decompress

func Decompress(compressed []byte) ([]byte, error)

func EscapeConnStringValue

func EscapeConnStringValue(v string) string

EscapeConnStringValue quotes a value for use in a libpq keyword-value connection string, escaping backslashes and single quotes per the libpq parsing rules.

func IsAlphaNumDashUnderscore

func IsAlphaNumDashUnderscore(s string) bool

IsAlphaNumDashUnderscore applies the alphanumdashunderscore rule to a value that is not a struct field, such as a NATS subject token arriving in a request body.

func IsPostgreSQLBatchSizeError

func IsPostgreSQLBatchSizeError(err error) bool

IsPostgreSQLBatchSizeError reports whether err is a transient size-related failure that should trigger adaptive halving. Connection-level errors (08006/08003) are excluded: halving cannot recover from them.

func NATSConsumerNameValidator

func NATSConsumerNameValidator(fl validator.FieldLevel) bool

func PGIdentifierValidator

func PGIdentifierValidator(fl validator.FieldLevel) bool

func RegisterCommonValidations

func RegisterCommonValidations(validate *validator.Validate) error

func RegisterMetric

func RegisterMetric[T prometheus.Collector](registry prometheus.Registerer, metric T) (T, error)

func RetryIndefinitelyOperation

func RetryIndefinitelyOperation(
	ctx context.Context,
	operation func(context.Context) error,
	failureMsg string,
	logger *zap.SugaredLogger,
) error

func RetryOperation

func RetryOperation(
	ctx context.Context,
	operation func(context.Context) error,
	failureMsg string,
	logger *zap.SugaredLogger,
) error

func RetryWithExponentialBackoffOperation

func RetryWithExponentialBackoffOperation(
	ctx context.Context,
	operation func(context.Context) error,
	failureMsg string,
	logger *zap.SugaredLogger,
) error

func SetupLogger

func SetupLogger(logLevel string) (*zap.SugaredLogger, func(), zap.Config)

func StreamOptions

func StreamOptions(name string, subject string, natsStream NATSConfig) *jetstream.StreamConfig

func ValidateStruct

func ValidateStruct(s any) error

ValidateStruct runs go-playground/validator over s with the common siphon validations registered, returning the first field error (or nil). It centralizes the validator setup + error-unwrapping boilerplate shared by the component config types (producer, consumer, reconciler, retention).

Types

type APIConfig

type APIConfig struct {
	Port int `json:"port,omitempty"`
}

APIConfig configures an optional dedicated listener for the mutating /v1 admin API. An absent `api` block keeps /v1 on the shared HTTP port, so upgrades are a no-op; setting it moves /v1 off the port the cluster scrapes, which is otherwise read-only.

Port has no `validate:"required"` (unlike HTTPConfig.Port) because zero means "not configured" — there is no default to fill.

type ConnectionStringBuilder

type ConnectionStringBuilder struct {
	// contains filtered or unexported fields
}

func NewConnectionStringBuilder

func NewConnectionStringBuilder() *ConnectionStringBuilder

func (*ConnectionStringBuilder) String

func (b *ConnectionStringBuilder) String() string

func (*ConnectionStringBuilder) WithApplicationName

func (b *ConnectionStringBuilder) WithApplicationName(appName string) *ConnectionStringBuilder

func (*ConnectionStringBuilder) WithDatabase

func (b *ConnectionStringBuilder) WithDatabase(database string) *ConnectionStringBuilder

func (*ConnectionStringBuilder) WithHost

func (*ConnectionStringBuilder) WithPassword

func (b *ConnectionStringBuilder) WithPassword(password string) *ConnectionStringBuilder

func (*ConnectionStringBuilder) WithPort

func (*ConnectionStringBuilder) WithReplication

func (b *ConnectionStringBuilder) WithReplication() *ConnectionStringBuilder

func (*ConnectionStringBuilder) WithSSLMode

func (b *ConnectionStringBuilder) WithSSLMode(sslMode string) *ConnectionStringBuilder

func (*ConnectionStringBuilder) WithSessionReplicationRole

func (b *ConnectionStringBuilder) WithSessionReplicationRole(role string) *ConnectionStringBuilder

func (*ConnectionStringBuilder) WithUser

type FailoverStorageConfig

type FailoverStorageConfig struct {
	// Storage is the backend definition. Type must be "gcs", "s3", or "file".
	// When Storage is nil, failover mode is disabled and existing
	// behavior is unchanged: a NATS outage causes the producer to
	// crash-loop until NATS recovers (the pre-failover-mode default).
	// Pointer (vs embedded struct) so the inner `required` tags only
	// fire when the user actually configured failover mode.
	Storage *ObjectStorageConfig `json:"storage,omitempty"`
	// RecoveryPollSeconds is how often to probe NATS while in failover
	// mode. Defaults to 30.
	RecoveryPollSeconds int `json:"recovery_poll_seconds,omitempty"`
	// MaxConcurrentDrainSubjects caps the number of parallel subject-drain
	// goroutines. Defaults to 4.
	MaxConcurrentDrainSubjects int `json:"max_concurrent_drain_subjects,omitempty"`
}

FailoverStorageConfig configures failover-mode buffering behavior: which backend to use, how often to probe NATS for recovery, and how to pace the drain when NATS comes back.

func (*FailoverStorageConfig) Enabled

func (c *FailoverStorageConfig) Enabled() bool

Enabled reports whether failover mode is configured (any non-nil Storage with a non-empty Type counts; full validation is in Validate).

func (*FailoverStorageConfig) Validate

func (c *FailoverStorageConfig) Validate() error

Validate performs strict checks on a configured failover storage: supported type, required fields, and per-type constraints. Called at startup (cmd/main.go) so misconfiguration crashes the producer with a clear message instead of failing the first time failover mode tries to engage. Returns nil when failover mode is disabled.

type HTTPConfig

type HTTPConfig struct {
	Port int `json:"port" validate:"required"`
}

HTTPConfig configures the HTTP server every Siphon component runs. It always serves /metrics, the health probes, and (with --profile) /debug/pprof. It also serves the mutating /v1 API unless APIConfig moves it to its own port.

type NATSConfig

type NATSConfig struct {
	// Replicas is the number of stream replicas in the NATS cluster.
	// Defaults to 1 when 0 or unset.
	Replicas int `json:"replicas,omitempty"`
	// MaxAgeSeconds is the maximum age of messages in the stream, in seconds.
	// Messages older than this are automatically removed. Defaults to 0
	// (unlimited / keep forever) when unset.
	MaxAgeSeconds int `json:"max_age_seconds,omitempty"`
	// OversizeBucketTTLSeconds is the time-to-live for objects in the oversize
	// bucket, in seconds. Objects older than this are automatically removed.
	// Defaults to 0 (unlimited / keep forever) when unset.
	OversizeBucketTTLSeconds int `json:"oversize_bucket_ttl_seconds,omitempty"`
}

NATSConfig holds NATS-specific tuning settings for JetStream streams and the object store. These are separated from the driver-agnostic fields because they only apply when the queueing driver is "nats".

type NATSTLSConfig

type NATSTLSConfig struct {
	CACertPath     *string `json:"ca_cert_path,omitempty"`
	ClientCertPath *string `json:"client_cert_path,omitempty"`
	ClientKeyPath  *string `json:"client_key_path,omitempty"`
}

NATSTLSConfig holds TLS settings for NATS connections. When ClientCertPath and ClientKeyPath are both set, the client presents a certificate during the TLS handshake (mTLS). Both fields must be provided together; setting only one is a configuration error.

type ObjectStorageConfig

type ObjectStorageConfig struct {
	// Identifier is the unique name of this storage backend.
	Identifier string `json:"identifier" validate:"required"`
	// Type selects the backend implementation: "gcs", "s3", "file", or "nats".
	Type string `json:"type" validate:"required,oneof=gcs file nats s3"`
	// BucketName is the GCS/S3 bucket name, the local directory for "file",
	// or the NATS ObjectStore bucket name for "nats".
	BucketName string `json:"bucket_name" validate:"required"`
}

ObjectStorageConfig describes a single named durable-storage backend. It is used both for oversize/snapshot event storage (via QueuingConfig.ObjectStorageConfig) and for failover-mode buffering (via FailoverStorageConfig.Storage).

Supported backends: "gcs", "s3", "file", and "nats".

Multi-tenant safety is provided by ApplicationIdentifier — every object is namespaced under "{appID}/{stream}/..." in the bucket, so multiple producers can share a bucket without colliding as long as their application_identifiers differ.

type PrometheusConfig deprecated

type PrometheusConfig = HTTPConfig

PrometheusConfig is the former name of HTTPConfig.

Deprecated: use HTTPConfig. Retained so the `prometheus` config key and existing field references keep working.

type QueuingConfig

type QueuingConfig struct {
	Driver                        string        `json:"driver" validate:"required"`
	URL                           string        `json:"url,omitempty"`
	TLSConfig                     NATSTLSConfig `json:"tls_config,omitempty"`
	NATS                          NATSConfig    `json:"nats_config,omitempty"`
	Username                      string        `json:"username,omitempty"`
	Password                      string        `json:"password,omitempty"`
	StreamName                    string        `json:"stream_name,omitempty"`
	TempStreamName                string        `json:"temp_stream_name,omitempty"`
	SnapshotStreamName            string        `json:"snapshot_stream_name,omitempty"`
	SnapshotStatusBucketName      string        `json:"snapshot_status_bucket_name,omitempty"`
	SnapshotStatusEnabled         bool          `json:"snapshot_status_enabled,omitempty"`
	SnapshotSubjectName           string        `json:"snapshot_subject_name,omitempty"`
	LowestAckPurgeEnabled         bool          `json:"lowest_ack_purge_enabled,omitempty"`
	LowestAckPurgeScheduleSeconds int           `json:"lowest_ack_purge_schedule_seconds,omitempty"`
	OversizeLimitBytes            int64         `json:"oversize_limit_bytes,omitempty"`
	// ObjectStorageConfig configures the durable storage backend used for
	// oversize and snapshot events. When nil, oversize/snapshot events
	// will fail at runtime if they occur. When non-nil, all inner fields
	// (identifier, type, bucket_name) are required.
	ObjectStorageConfig *ObjectStorageConfig `json:"object_storage_config,omitempty"`
	// FailoverStorage configures durable buffering for the case where
	// NATS becomes unavailable. When unset, failover mode is disabled.
	FailoverStorage FailoverStorageConfig `json:"failover_storage,omitempty"`
	// AckWaitSeconds is the time NATS waits before redelivering an unacknowledged
	// message. Defaults to 30 seconds (NATS default) when 0 or unset.
	AckWaitSeconds int `json:"ack_wait_seconds,omitempty"`
	// SnapshotRequestsStreamName is the stream snapshot requests are published
	// to and consumed from. Defaults to StreamName. A dedicated stream keeps the
	// rarely used request subject out of the busy main stream, where fetching a
	// new request after a quiet week scans every block written since the last
	// one and stalls writes to the stream.
	SnapshotRequestsStreamName string `json:"snapshot_requests_stream_name,omitempty"`
}

func (*QueuingConfig) GetSnapshotRequestsStreamName

func (c *QueuingConfig) GetSnapshotRequestsStreamName() string

func (*QueuingConfig) GetSnapshotStatusBucketName

func (c *QueuingConfig) GetSnapshotStatusBucketName() string

func (*QueuingConfig) GetSnapshotStreamName

func (c *QueuingConfig) GetSnapshotStreamName() string

func (*QueuingConfig) GetTempStreamName

func (c *QueuingConfig) GetTempStreamName() string

func (*QueuingConfig) HasDedicatedSnapshotRequestsStream

func (c *QueuingConfig) HasDedicatedSnapshotRequestsStream() bool

HasDedicatedSnapshotRequestsStream reports whether snapshot requests live on their own stream rather than on the main stream.

func (*QueuingConfig) PopulateObjectStorageDefaults

func (c *QueuingConfig) PopulateObjectStorageDefaults()

PopulateObjectStorageDefaults sets ObjectStorageConfig to a NATS-backed default when it is nil. This preserves backward compatibility for configs that don't specify object_storage_config — they fall back to NATS ObjectStore with the default bucket "repl_events_bucket". Must be called before Validate() so the struct tag validator sees a fully populated config.

type SortedStringSlice

type SortedStringSlice struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewStringSlice

func NewStringSlice() *SortedStringSlice

func (*SortedStringSlice) Any

func (s *SortedStringSlice) Any() bool

func (*SortedStringSlice) Append

func (s *SortedStringSlice) Append(item string)

func (*SortedStringSlice) Get

func (s *SortedStringSlice) Get(index int) (string, bool)

func (*SortedStringSlice) Len

func (s *SortedStringSlice) Len() int

func (*SortedStringSlice) Remove

func (s *SortedStringSlice) Remove(index int) bool

func (*SortedStringSlice) Search

func (s *SortedStringSlice) Search(item string) (int, bool)

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL