Documentation
¶
Index ¶
- Constants
- Variables
- func AlphaNumDashUnderscoreValidator(fl validator.FieldLevel) bool
- func BuildHTTPHandler(port int, mux *http.ServeMux) *http.Server
- func CommonPublishOptions() []jetstream.PublishOpt
- func Compress(data []byte) ([]byte, error)
- func ConsumerOptions(name string, subjects []string, ackWaitSeconds int) *jetstream.ConsumerConfig
- func Decompress(compressed []byte) ([]byte, error)
- func EscapeConnStringValue(v string) string
- func IsAlphaNumDashUnderscore(s string) bool
- func IsPostgreSQLBatchSizeError(err error) bool
- func NATSConsumerNameValidator(fl validator.FieldLevel) bool
- func PGIdentifierValidator(fl validator.FieldLevel) bool
- func RegisterCommonValidations(validate *validator.Validate) error
- func RegisterMetric[T prometheus.Collector](registry prometheus.Registerer, metric T) (T, error)
- func RetryIndefinitelyOperation(ctx context.Context, operation func(context.Context) error, failureMsg string, ...) error
- func RetryOperation(ctx context.Context, operation func(context.Context) error, failureMsg string, ...) error
- func RetryWithExponentialBackoffOperation(ctx context.Context, operation func(context.Context) error, failureMsg string, ...) error
- func SetupLogger(logLevel string) (*zap.SugaredLogger, func(), zap.Config)
- func StreamOptions(name string, subject string, natsStream NATSConfig) *jetstream.StreamConfig
- func ValidateStruct(s any) error
- type APIConfig
- type ConnectionStringBuilder
- func (b *ConnectionStringBuilder) String() string
- func (b *ConnectionStringBuilder) WithApplicationName(appName string) *ConnectionStringBuilder
- func (b *ConnectionStringBuilder) WithDatabase(database string) *ConnectionStringBuilder
- func (b *ConnectionStringBuilder) WithHost(host string) *ConnectionStringBuilder
- func (b *ConnectionStringBuilder) WithPassword(password string) *ConnectionStringBuilder
- func (b *ConnectionStringBuilder) WithPort(port int) *ConnectionStringBuilder
- func (b *ConnectionStringBuilder) WithReplication() *ConnectionStringBuilder
- func (b *ConnectionStringBuilder) WithSSLMode(sslMode string) *ConnectionStringBuilder
- func (b *ConnectionStringBuilder) WithSessionReplicationRole(role string) *ConnectionStringBuilder
- func (b *ConnectionStringBuilder) WithUser(user string) *ConnectionStringBuilder
- type FailoverStorageConfig
- type HTTPConfig
- type NATSConfig
- type NATSTLSConfig
- type ObjectStorageConfig
- type PrometheusConfigdeprecated
- type QueuingConfig
- func (c *QueuingConfig) GetSnapshotRequestsStreamName() string
- func (c *QueuingConfig) GetSnapshotStatusBucketName() string
- func (c *QueuingConfig) GetSnapshotStreamName() string
- func (c *QueuingConfig) GetTempStreamName() string
- func (c *QueuingConfig) HasDedicatedSnapshotRequestsStream() bool
- func (c *QueuingConfig) PopulateObjectStorageDefaults()
- type SortedStringSlice
Constants ¶
const ( DefaultNATSPort = 4222 DefaultNATSStartupTimeout = 5 * time.Second NATSQueueDriver = "nats" NoopDriver = "noop" // ObjectStorageTypeNATS is the ObjectStorageConfig.Type value selecting // the NATS ObjectStore backend for oversize/snapshot event storage. ObjectStorageTypeNATS = "nats" ObjectStorageTypeS3 = "s3" ObjectStorageTypeFile = "file" ObjectStorageTypeGCS = "gcs" SiphonStreamCreatedHeader = "siphon_stream_created_at" // SiphonEventHeader, SnapshotCompleteEvent, and SnapshotCompleteOnTemp // identify the internal snapshot-complete merge sentinels. Canonical home // here so the producer (pkg/siphon) and the consumer share one source. SiphonEventHeader = "siphon-event" SnapshotCompleteEvent = "snapshot-complete" SnapshotCompleteOnTemp = "snapshot-complete-on-temp" // SiphonLSNHeader carries the PostgreSQL LSN of the replication event // as a decimal string. The producer stamps it on every flush; failover // mode requires it to construct lex-sortable object names so drain can // replay events in LSN order. SiphonLSNHeader = "X-Siphon-LSN" )
Variables ¶
var DurationMsBuckets = []float64{1, 3, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000}
Functions ¶
func AlphaNumDashUnderscoreValidator ¶
func AlphaNumDashUnderscoreValidator(fl validator.FieldLevel) bool
func CommonPublishOptions ¶
func CommonPublishOptions() []jetstream.PublishOpt
func ConsumerOptions ¶
func ConsumerOptions( name string, subjects []string, ackWaitSeconds int, ) *jetstream.ConsumerConfig
func Decompress ¶
func EscapeConnStringValue ¶
EscapeConnStringValue quotes a value for use in a libpq keyword-value connection string, escaping backslashes and single quotes per the libpq parsing rules.
func IsAlphaNumDashUnderscore ¶
IsAlphaNumDashUnderscore applies the alphanumdashunderscore rule to a value that is not a struct field, such as a NATS subject token arriving in a request body.
func IsPostgreSQLBatchSizeError ¶
IsPostgreSQLBatchSizeError reports whether err is a transient size-related failure that should trigger adaptive halving. Connection-level errors (08006/08003) are excluded: halving cannot recover from them.
func NATSConsumerNameValidator ¶
func NATSConsumerNameValidator(fl validator.FieldLevel) bool
func PGIdentifierValidator ¶
func PGIdentifierValidator(fl validator.FieldLevel) bool
func RegisterMetric ¶
func RegisterMetric[T prometheus.Collector](registry prometheus.Registerer, metric T) (T, error)
func RetryOperation ¶
func SetupLogger ¶
func SetupLogger(logLevel string) (*zap.SugaredLogger, func(), zap.Config)
func StreamOptions ¶
func StreamOptions(name string, subject string, natsStream NATSConfig) *jetstream.StreamConfig
func ValidateStruct ¶
ValidateStruct runs go-playground/validator over s with the common siphon validations registered, returning the first field error (or nil). It centralizes the validator setup + error-unwrapping boilerplate shared by the component config types (producer, consumer, reconciler, retention).
Types ¶
type APIConfig ¶
type APIConfig struct {
Port int `json:"port,omitempty"`
}
APIConfig configures an optional dedicated listener for the mutating /v1 admin API. An absent `api` block keeps /v1 on the shared HTTP port, so upgrades are a no-op; setting it moves /v1 off the port the cluster scrapes, which is otherwise read-only.
Port has no `validate:"required"` (unlike HTTPConfig.Port) because zero means "not configured" — there is no default to fill.
type ConnectionStringBuilder ¶
type ConnectionStringBuilder struct {
// contains filtered or unexported fields
}
func NewConnectionStringBuilder ¶
func NewConnectionStringBuilder() *ConnectionStringBuilder
func (*ConnectionStringBuilder) String ¶
func (b *ConnectionStringBuilder) String() string
func (*ConnectionStringBuilder) WithApplicationName ¶
func (b *ConnectionStringBuilder) WithApplicationName(appName string) *ConnectionStringBuilder
func (*ConnectionStringBuilder) WithDatabase ¶
func (b *ConnectionStringBuilder) WithDatabase(database string) *ConnectionStringBuilder
func (*ConnectionStringBuilder) WithHost ¶
func (b *ConnectionStringBuilder) WithHost(host string) *ConnectionStringBuilder
func (*ConnectionStringBuilder) WithPassword ¶
func (b *ConnectionStringBuilder) WithPassword(password string) *ConnectionStringBuilder
func (*ConnectionStringBuilder) WithPort ¶
func (b *ConnectionStringBuilder) WithPort(port int) *ConnectionStringBuilder
func (*ConnectionStringBuilder) WithReplication ¶
func (b *ConnectionStringBuilder) WithReplication() *ConnectionStringBuilder
func (*ConnectionStringBuilder) WithSSLMode ¶
func (b *ConnectionStringBuilder) WithSSLMode(sslMode string) *ConnectionStringBuilder
func (*ConnectionStringBuilder) WithSessionReplicationRole ¶
func (b *ConnectionStringBuilder) WithSessionReplicationRole(role string) *ConnectionStringBuilder
func (*ConnectionStringBuilder) WithUser ¶
func (b *ConnectionStringBuilder) WithUser(user string) *ConnectionStringBuilder
type FailoverStorageConfig ¶
type FailoverStorageConfig struct {
// Storage is the backend definition. Type must be "gcs", "s3", or "file".
// When Storage is nil, failover mode is disabled and existing
// behavior is unchanged: a NATS outage causes the producer to
// crash-loop until NATS recovers (the pre-failover-mode default).
// Pointer (vs embedded struct) so the inner `required` tags only
// fire when the user actually configured failover mode.
Storage *ObjectStorageConfig `json:"storage,omitempty"`
// RecoveryPollSeconds is how often to probe NATS while in failover
// mode. Defaults to 30.
RecoveryPollSeconds int `json:"recovery_poll_seconds,omitempty"`
// MaxConcurrentDrainSubjects caps the number of parallel subject-drain
// goroutines. Defaults to 4.
MaxConcurrentDrainSubjects int `json:"max_concurrent_drain_subjects,omitempty"`
}
FailoverStorageConfig configures failover-mode buffering behavior: which backend to use, how often to probe NATS for recovery, and how to pace the drain when NATS comes back.
func (*FailoverStorageConfig) Enabled ¶
func (c *FailoverStorageConfig) Enabled() bool
Enabled reports whether failover mode is configured (any non-nil Storage with a non-empty Type counts; full validation is in Validate).
func (*FailoverStorageConfig) Validate ¶
func (c *FailoverStorageConfig) Validate() error
Validate performs strict checks on a configured failover storage: supported type, required fields, and per-type constraints. Called at startup (cmd/main.go) so misconfiguration crashes the producer with a clear message instead of failing the first time failover mode tries to engage. Returns nil when failover mode is disabled.
type HTTPConfig ¶
type HTTPConfig struct {
Port int `json:"port" validate:"required"`
}
HTTPConfig configures the HTTP server every Siphon component runs. It always serves /metrics, the health probes, and (with --profile) /debug/pprof. It also serves the mutating /v1 API unless APIConfig moves it to its own port.
type NATSConfig ¶
type NATSConfig struct {
// Replicas is the number of stream replicas in the NATS cluster.
// Defaults to 1 when 0 or unset.
Replicas int `json:"replicas,omitempty"`
// MaxAgeSeconds is the maximum age of messages in the stream, in seconds.
// Messages older than this are automatically removed. Defaults to 0
// (unlimited / keep forever) when unset.
MaxAgeSeconds int `json:"max_age_seconds,omitempty"`
// OversizeBucketTTLSeconds is the time-to-live for objects in the oversize
// bucket, in seconds. Objects older than this are automatically removed.
// Defaults to 0 (unlimited / keep forever) when unset.
OversizeBucketTTLSeconds int `json:"oversize_bucket_ttl_seconds,omitempty"`
}
NATSConfig holds NATS-specific tuning settings for JetStream streams and the object store. These are separated from the driver-agnostic fields because they only apply when the queueing driver is "nats".
type NATSTLSConfig ¶
type NATSTLSConfig struct {
CACertPath *string `json:"ca_cert_path,omitempty"`
ClientCertPath *string `json:"client_cert_path,omitempty"`
ClientKeyPath *string `json:"client_key_path,omitempty"`
}
NATSTLSConfig holds TLS settings for NATS connections. When ClientCertPath and ClientKeyPath are both set, the client presents a certificate during the TLS handshake (mTLS). Both fields must be provided together; setting only one is a configuration error.
type ObjectStorageConfig ¶
type ObjectStorageConfig struct {
// Identifier is the unique name of this storage backend.
Identifier string `json:"identifier" validate:"required"`
// Type selects the backend implementation: "gcs", "s3", "file", or "nats".
Type string `json:"type" validate:"required,oneof=gcs file nats s3"`
// BucketName is the GCS/S3 bucket name, the local directory for "file",
// or the NATS ObjectStore bucket name for "nats".
BucketName string `json:"bucket_name" validate:"required"`
}
ObjectStorageConfig describes a single named durable-storage backend. It is used both for oversize/snapshot event storage (via QueuingConfig.ObjectStorageConfig) and for failover-mode buffering (via FailoverStorageConfig.Storage).
Supported backends: "gcs", "s3", "file", and "nats".
Multi-tenant safety is provided by ApplicationIdentifier — every object is namespaced under "{appID}/{stream}/..." in the bucket, so multiple producers can share a bucket without colliding as long as their application_identifiers differ.
type PrometheusConfig
deprecated
type PrometheusConfig = HTTPConfig
PrometheusConfig is the former name of HTTPConfig.
Deprecated: use HTTPConfig. Retained so the `prometheus` config key and existing field references keep working.
type QueuingConfig ¶
type QueuingConfig struct {
Driver string `json:"driver" validate:"required"`
URL string `json:"url,omitempty"`
TLSConfig NATSTLSConfig `json:"tls_config,omitempty"`
NATS NATSConfig `json:"nats_config,omitempty"`
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
StreamName string `json:"stream_name,omitempty"`
TempStreamName string `json:"temp_stream_name,omitempty"`
SnapshotStreamName string `json:"snapshot_stream_name,omitempty"`
SnapshotStatusBucketName string `json:"snapshot_status_bucket_name,omitempty"`
SnapshotStatusEnabled bool `json:"snapshot_status_enabled,omitempty"`
SnapshotSubjectName string `json:"snapshot_subject_name,omitempty"`
LowestAckPurgeEnabled bool `json:"lowest_ack_purge_enabled,omitempty"`
LowestAckPurgeScheduleSeconds int `json:"lowest_ack_purge_schedule_seconds,omitempty"`
OversizeLimitBytes int64 `json:"oversize_limit_bytes,omitempty"`
// ObjectStorageConfig configures the durable storage backend used for
// oversize and snapshot events. When nil, oversize/snapshot events
// will fail at runtime if they occur. When non-nil, all inner fields
// (identifier, type, bucket_name) are required.
ObjectStorageConfig *ObjectStorageConfig `json:"object_storage_config,omitempty"`
// FailoverStorage configures durable buffering for the case where
// NATS becomes unavailable. When unset, failover mode is disabled.
FailoverStorage FailoverStorageConfig `json:"failover_storage,omitempty"`
// AckWaitSeconds is the time NATS waits before redelivering an unacknowledged
// message. Defaults to 30 seconds (NATS default) when 0 or unset.
AckWaitSeconds int `json:"ack_wait_seconds,omitempty"`
// SnapshotRequestsStreamName is the stream snapshot requests are published
// to and consumed from. Defaults to StreamName. A dedicated stream keeps the
// rarely used request subject out of the busy main stream, where fetching a
// new request after a quiet week scans every block written since the last
// one and stalls writes to the stream.
SnapshotRequestsStreamName string `json:"snapshot_requests_stream_name,omitempty"`
}
func (*QueuingConfig) GetSnapshotRequestsStreamName ¶
func (c *QueuingConfig) GetSnapshotRequestsStreamName() string
func (*QueuingConfig) GetSnapshotStatusBucketName ¶
func (c *QueuingConfig) GetSnapshotStatusBucketName() string
func (*QueuingConfig) GetSnapshotStreamName ¶
func (c *QueuingConfig) GetSnapshotStreamName() string
func (*QueuingConfig) GetTempStreamName ¶
func (c *QueuingConfig) GetTempStreamName() string
func (*QueuingConfig) HasDedicatedSnapshotRequestsStream ¶
func (c *QueuingConfig) HasDedicatedSnapshotRequestsStream() bool
HasDedicatedSnapshotRequestsStream reports whether snapshot requests live on their own stream rather than on the main stream.
func (*QueuingConfig) PopulateObjectStorageDefaults ¶
func (c *QueuingConfig) PopulateObjectStorageDefaults()
PopulateObjectStorageDefaults sets ObjectStorageConfig to a NATS-backed default when it is nil. This preserves backward compatibility for configs that don't specify object_storage_config — they fall back to NATS ObjectStore with the default bucket "repl_events_bucket". Must be called before Validate() so the struct tag validator sees a fully populated config.
type SortedStringSlice ¶
func NewStringSlice ¶
func NewStringSlice() *SortedStringSlice
func (*SortedStringSlice) Any ¶
func (s *SortedStringSlice) Any() bool
func (*SortedStringSlice) Append ¶
func (s *SortedStringSlice) Append(item string)
func (*SortedStringSlice) Len ¶
func (s *SortedStringSlice) Len() int
func (*SortedStringSlice) Remove ¶
func (s *SortedStringSlice) Remove(index int) bool