Documentation
¶
Overview ¶
Package kafka provides Kafka producers and subscribers built on segmentio/kafka-go.
A Broker creates producers and subscribers for topics. It supports SASL authentication, optional topic auto-creation, worker-based parallel consumption (ordered per message key) and request-context propagation via a message envelope. Use the builders (NewProducerCfgBuilder, NewSubscriberCfgBuilder, NewTopicCfgBuilder) to configure each component.
Subscribers support two delivery guarantees, selected via SubscriberConfigBuilder.DeliveryGuarantee: AtMostOnce (the default — parallel, fastest, but may drop in-flight messages on shutdown/crash) and AtLeastOnce (sequential, commits only after processing, so nothing is lost on shutdown/crash at the cost of possible redelivery). See DeliveryGuarantee.
Index ¶
- Constants
- Variables
- func Decode[T any](parentCtx context.Context, msg []byte) (T, context.Context, error)
- type Broker
- type BrokerConfig
- type DeliveryGuarantee
- type HandlerFn
- type Message
- type MessageT
- type Producer
- type ProducerConfig
- type ProducerConfigBuilder
- type Sasl
- type SubscriberConfig
- type SubscriberConfigBuilder
- type TopicBuilder
- type TopicConfig
- type TopicParam
Examples ¶
Constants ¶
const ( ErrCodeKafkaFetchMessage = "KF-001" ErrCodeKafkaCommitMessage = "KF-002" ErrCodeKafkaNotInitialized = "KF-003" ErrCodeKafkaInvalidConfig = "KF-004" ErrCodeKafkaMessageContextInvalid = "KF-005" ErrCodeKafkaMessageMarshal = "KF-006" ErrCodeKafkaProducerTopicEmpty = "KF-008" ErrCodeKafkaSubTopicEmpty = "KF-009" ErrCodeKafkaSubNoHandlers = "KF-010" ErrCodeKafkaConnection = "KF-011" ErrCodeKafkaCreateTopics = "KF-012" ErrCodeKafkaMessageWrite = "KF-013" ErrCodeKafkaDecodeMsgUnmarshal = "KF-014" ErrCodeKafkaMsgUnmarshalPayload = "KF-015" ErrCodeKafkaProduceMsg = "KF-016" ErrCodeKafkaSaslNotSupportedType = "KF-017" ErrCodeKafkaSaslGetMechanism = "KF-018" ErrCodeKafkaInvalidDeliveryGuarantee = "KF-019" ErrCodeKafkaHandlerPanic = "KF-020" )
const ( SaslTypePlain = "plain" SaslTypeScramSha256 = "sha256" SaslTypeScramSha512 = "sha512" )
Variables ¶
var ( ErrKafkaFetchMessage = func(cause error) error { return jet.NewAppErrBuilder(ErrCodeKafkaFetchMessage, "").Wrap(cause).Err() } ErrKafkaCommitMessage = func(cause error) error { return jet.NewAppErrBuilder(ErrCodeKafkaCommitMessage, "").Wrap(cause).Err() } ErrKafkaNotInitialized = func(ctx context.Context) error { return jet.NewAppErrBuilder(ErrCodeKafkaNotInitialized, "not initialized").C(ctx).Err() } ErrKafkaProducerTopicEmpty = func(ctx context.Context) error { return jet.NewAppErrBuilder(ErrCodeKafkaProducerTopicEmpty, "topic empty").C(ctx).Err() } ErrKafkaInvalidConfig = func(ctx context.Context) error { return jet.NewAppErrBuilder(ErrCodeKafkaInvalidConfig, "config invalid").C(ctx).Err() } ErrKafkaMessageContextInvalid = func(ctx context.Context, topic string) error { return jet.NewAppErrBuilder(ErrCodeKafkaMessageContextInvalid, "message context invalid").F(jet.KV{"topic": topic}).C(ctx).Err() } ErrKafkaMessageMarshal = func(ctx context.Context, cause error, topic string) error { return jet.NewAppErrBuilder(ErrCodeKafkaMessageMarshal, "").Wrap(cause).F(jet.KV{"topic": topic}).C(ctx).Err() } ErrKafkaMessageWrite = func(ctx context.Context, cause error, topic string) error { return jet.NewAppErrBuilder(ErrCodeKafkaMessageWrite, "").Wrap(cause).F(jet.KV{"topic": topic}).C(ctx).Err() } ErrKafkaConnection = func(ctx context.Context, cause error) error { return jet.NewAppErrBuilder(ErrCodeKafkaConnection, "").Wrap(cause).C(ctx).Err() } ErrKafkaCreateTopics = func(ctx context.Context, cause error) error { return jet.NewAppErrBuilder(ErrCodeKafkaCreateTopics, "").Wrap(cause).C(ctx).Err() } ErrKafkaDecodeMsgUnmarshal = func(ctx context.Context, cause error) error { return jet.NewAppErrBuilder(ErrCodeKafkaDecodeMsgUnmarshal, "").Wrap(cause).C(ctx).Err() } ErrKafkaMsgUnmarshalPayload = func(ctx context.Context, cause error) error { return jet.NewAppErrBuilder(ErrCodeKafkaMsgUnmarshalPayload, "").Wrap(cause).C(ctx).Err() } ErrKafkaSubTopicEmpty = func(ctx context.Context) error { return jet.NewAppErrBuilder(ErrCodeKafkaSubTopicEmpty, "topic empty").C(ctx).Err() } ErrKafkaSubNoHandlers = func(ctx context.Context) error { return jet.NewAppErrBuilder(ErrCodeKafkaSubNoHandlers, "no handlers specified").C(ctx).Err() } ErrKafkaProduceMsg = func(ctx context.Context, cause error) error { return jet.NewAppErrBuilder(ErrCodeKafkaProduceMsg, "").Wrap(cause).C(ctx).Err() } ErrKafkaSaslNotSupportedType = func(ctx context.Context) error { return jet.NewAppErrBuilder(ErrCodeKafkaSaslNotSupportedType, "not supported sasl type").C(ctx).Err() } ErrKafkaSaslGetMechanism = func(ctx context.Context, cause error) error { return jet.NewAppErrBuilder(ErrCodeKafkaSaslGetMechanism, "sasl mechanism").C(ctx).Err() } ErrKafkaInvalidDeliveryGuarantee = func(ctx context.Context, value string) error { return jet.NewAppErrBuilder(ErrCodeKafkaInvalidDeliveryGuarantee, "invalid delivery guarantee").F(jet.KV{"value": value}).C(ctx).Business().Err() } ErrKafkaHandlerPanic = func(cause error) error { return jet.NewAppErrBuilder(ErrCodeKafkaHandlerPanic, "handler panic").Wrap(cause).Err() } )
Functions ¶
Types ¶
type Broker ¶
type Broker interface {
// Init initializes broker
Init(ctx context.Context, cfg *BrokerConfig) error
// AddProducer adds a producer with configuration
AddProducer(ctx context.Context, topic *TopicConfig, cfg *ProducerConfig) (Producer, error)
// AddSubscriber adds a subscriber with configuration
AddSubscriber(ctx context.Context, topic *TopicConfig, cfg *SubscriberConfig, handlers ...HandlerFn) error
// DeclareTopics declares topics in kafka broker
// must be executed after all producer and subscribers added
DeclareTopics(ctx context.Context) error
// Start starts listening
Start(ctx context.Context) error
// Close closes broker
Close(ctx context.Context)
}
Broker kafka
Example ¶
package main
import (
"context"
"github.com/zloevil/jet"
"github.com/zloevil/jet/kafka"
)
func main() {
logFn := func() jet.CLogger { return jet.L(jet.InitLogger(&jet.LogConfig{Level: jet.InfoLevel})) }
ctx := context.Background()
broker := kafka.NewBroker(logFn)
if err := broker.Init(ctx, &kafka.BrokerConfig{Url: "localhost:9092"}); err != nil {
return
}
// subscribe to a topic
_ = broker.AddSubscriber(ctx,
kafka.NewTopicCfgBuilder("orders").Build(),
kafka.NewSubscriberCfgBuilder().GroupId("my-service").Build(),
func(payload []byte) error {
// handle the message
return nil
},
)
_ = broker.Start(ctx)
defer broker.Close(ctx)
}
Output:
Example (AtLeastOnce) ¶
package main
import (
"context"
"github.com/zloevil/jet"
"github.com/zloevil/jet/kafka"
)
func main() {
logFn := func() jet.CLogger { return jet.L(jet.InitLogger(&jet.LogConfig{Level: jet.InfoLevel})) }
ctx := context.Background()
broker := kafka.NewBroker(logFn)
if err := broker.Init(ctx, &kafka.BrokerConfig{Url: "localhost:9092"}); err != nil {
return
}
// subscribe with at-least-once delivery: the offset is committed only after
// the handler returns, so no message is lost on shutdown or crash. A message
// may be redelivered, so the handler must be idempotent.
_ = broker.AddSubscriber(ctx,
kafka.NewTopicCfgBuilder("orders").Build(),
kafka.NewSubscriberCfgBuilder().
GroupId("my-service").
DeliveryGuarantee(kafka.AtLeastOnce).
Build(),
func(payload []byte) error {
// handle the message; the offset is committed once this returns nil
// (returning an error retries the same message)
return nil
},
)
_ = broker.Start(ctx)
defer broker.Close(ctx)
}
Output:
func NewBroker ¶
func NewBroker(logger jet.CLoggerFunc) Broker
type BrokerConfig ¶
type BrokerConfig struct {
ClientId string `mapstructure:"client_id"` // ClientId identifies client
TopicAutoCreation bool `mapstructure:"topic_auto_creation"` // TopicAutoCreation if true topics are created by code (otherwise must be preliminary declared)
Url string // Url comma separated host-port pairs ("localhost:9092,localhost:9093")
Sasl Sasl // Sasl configuration
}
BrokerConfig kafka broker configuration
type DeliveryGuarantee ¶ added in v0.2.0
type DeliveryGuarantee string
DeliveryGuarantee controls when a consumed message's offset is committed relative to handler processing.
const ( // AtMostOnce commits the offset as soon as a message is read, before the // handlers run. It is the default. It keeps the parallel per-key worker pool // and is the fastest option, but messages that are buffered or in-flight when // the process stops (SIGTERM, scale-down) or crashes are lost, because their // offset has already advanced. AtMostOnce DeliveryGuarantee = "at_most_once" // AtLeastOnce commits the offset only after a handler returns nil, so nothing // is lost on shutdown or crash — an uncommitted message is redelivered, and // handlers must therefore be idempotent. A handler that returns an error (or // panics) is retried on the same message until it succeeds; return nil to skip // a message that cannot be processed. Messages are processed sequentially // (Kafka tracks a single offset per partition, so out-of-order commits would // skip unprocessed messages), which lowers throughput compared to AtMostOnce. AtLeastOnce DeliveryGuarantee = "at_least_once" )
type Message ¶
type Message struct {
// Ctx allows passing context from producer to subscriber
Ctx *jet.RequestContext `json:"ctx"`
// Key is a kafka message key which allow to specify target kafka partition
Key string `json:"key"`
// Payload arbitrary data
Payload any `json:"payload"`
}
type MessageT ¶
type MessageT[T any] struct { // Ctx allows passing context from producer to subscriber Ctx *jet.RequestContext `json:"ctx"` // Key is a kafka message key which allow to specify target kafka partition Key string `json:"key"` // Payload arbitrary data Payload T `json:"payload"` }
type Producer ¶
type Producer interface {
// Send sends a message to broker
Send(ctx context.Context, key string, payload interface{}) error
// SendMany sends bulk of messages to broker
SendMany(ctx context.Context, messages ...*Message) error
}
Producer allows sending message to broker
type ProducerConfig ¶
type ProducerConfig struct {
BatchSize *int
BatchTimeout *time.Duration
RequiredAcks *int
MaxAttempts *int
Async bool
RetryTimes *int
RetryTimeout *time.Duration
}
ProducerConfig specifies producer config params use builder rather than manual population
type ProducerConfigBuilder ¶
type ProducerConfigBuilder interface {
// BatchTimeout sets batch timeout value (default: 1s)
BatchTimeout(to time.Duration) ProducerConfigBuilder
// BatchSize sets batch size (default: 100)
BatchSize(size int) ProducerConfigBuilder
// Async if true, WriteMessages call will never block but errors aren't returned (default: false)
Async(v bool) ProducerConfigBuilder
// Retry sets retry params (default: time=3, timeout = 1s)
Retry(time int, timeout time.Duration) ProducerConfigBuilder
// Build builds config
Build() *ProducerConfig
}
func NewProducerCfgBuilder ¶
func NewProducerCfgBuilder() ProducerConfigBuilder
type SubscriberConfig ¶
type SubscriberConfig struct {
GroupId string
BatchTimeout *time.Duration
MaxWait *time.Duration
CommitInterval *time.Duration
Workers *int
MaxAttempts *int
StartOffset *int64
JoinGroupBackoff *time.Duration
Logging bool
DeliveryGuarantee DeliveryGuarantee
}
SubscriberConfig specifies subscriber config params use builder rather than manual population
type SubscriberConfigBuilder ¶
type SubscriberConfigBuilder interface {
// GroupId allows load balancing messages within the same group
GroupId(groupId string) SubscriberConfigBuilder
// BatchTimeout sets timeout of batch fetching from kafka (default: 10s)
BatchTimeout(to time.Duration) SubscriberConfigBuilder
// MaxWait sets maximum amount of time to wait for new data to come when fetching batches (default: 10s)
MaxWait(to time.Duration) SubscriberConfigBuilder
// CommitInterval sets interval to commit to kafka (default: sync). Ignored for AtLeastOnce, which always commits synchronously
CommitInterval(to time.Duration) SubscriberConfigBuilder
// Workers sets number of workers (default: 4). Only used with AtMostOnce delivery (AtLeastOnce is sequential)
Workers(num int) SubscriberConfigBuilder
// StartOffset determines from which offset a new group starts to consume. it must be set to one of FirstOffset = -2 or LastOffset = -1 (Default: FirstOffset)
// Only used when GroupID is set
StartOffset(v int64) SubscriberConfigBuilder
// JoinGroupBackoff optionally sets the length of time to wait between re-joining
JoinGroupBackoff(t time.Duration) SubscriberConfigBuilder
// Logging if true subscriber logging enabled
Logging(v bool) SubscriberConfigBuilder
// DeliveryGuarantee sets delivery semantics: AtMostOnce (default) or
// AtLeastOnce. See the DeliveryGuarantee constants for the tradeoffs.
DeliveryGuarantee(g DeliveryGuarantee) SubscriberConfigBuilder
// Build builds config
Build() *SubscriberConfig
}
func NewSubscriberCfgBuilder ¶
func NewSubscriberCfgBuilder() SubscriberConfigBuilder
type TopicBuilder ¶
type TopicBuilder interface {
// WithPartitionNum setting num of partitions
WithPartitionNum(num int) TopicBuilder
// WithParams setting additional params
WithParams(params ...TopicParam) TopicBuilder
// Build builds config
Build() *TopicConfig
}
TopicBuilder simplifies preparing topic config
func NewTopicCfgBuilder ¶
func NewTopicCfgBuilder(topic string) TopicBuilder
type TopicConfig ¶
type TopicConfig struct {
// Topic name
Topic string
// Partitions number of partitions
Partitions *int
// Configs config params
Configs map[string]string
}
TopicConfig topic config