Documentation
¶
Overview ¶
Package kafka provides Kafka producers and subscribers built on segmentio/kafka-go.
A Broker creates producers and subscribers for topics. It supports SASL authentication, optional topic auto-creation, worker-based parallel consumption (ordered per message key) and request-context propagation via a message envelope. Use the builders (NewProducerCfgBuilder, NewSubscriberCfgBuilder, NewTopicCfgBuilder) to configure each component.
Index ¶
- Constants
- Variables
- func Decode[T any](parentCtx context.Context, msg []byte) (T, context.Context, error)
- type Broker
- type BrokerConfig
- type HandlerFn
- type Message
- type MessageT
- type Producer
- type ProducerConfig
- type ProducerConfigBuilder
- type Sasl
- type SubscriberConfig
- type SubscriberConfigBuilder
- type TopicBuilder
- type TopicConfig
- type TopicParam
Examples ¶
Constants ¶
const ( ErrCodeKafkaFetchMessage = "KF-001" ErrCodeKafkaNotInitialized = "KF-003" ErrCodeKafkaInvalidConfig = "KF-004" ErrCodeKafkaMessageContextInvalid = "KF-005" ErrCodeKafkaMessageMarshal = "KF-006" ErrCodeKafkaProducerTopicEmpty = "KF-008" ErrCodeKafkaSubTopicEmpty = "KF-009" ErrCodeKafkaSubNoHandlers = "KF-010" ErrCodeKafkaConnection = "KF-011" ErrCodeKafkaCreateTopics = "KF-012" ErrCodeKafkaMessageWrite = "KF-013" ErrCodeKafkaDecodeMsgUnmarshal = "KF-014" ErrCodeKafkaMsgUnmarshalPayload = "KF-015" ErrCodeKafkaProduceMsg = "KF-016" ErrCodeKafkaSaslNotSupportedType = "KF-017" ErrCodeKafkaSaslGetMechanism = "KF-018" )
const ( SaslTypePlain = "plain" SaslTypeScramSha256 = "sha256" SaslTypeScramSha512 = "sha512" )
Variables ¶
var ( ErrKafkaFetchMessage = func(cause error) error { return jet.NewAppErrBuilder(ErrCodeKafkaFetchMessage, "").Wrap(cause).Err() } ErrKafkaNotInitialized = func(ctx context.Context) error { return jet.NewAppErrBuilder(ErrCodeKafkaNotInitialized, "not initialized").C(ctx).Err() } ErrKafkaProducerTopicEmpty = func(ctx context.Context) error { return jet.NewAppErrBuilder(ErrCodeKafkaProducerTopicEmpty, "topic empty").C(ctx).Err() } ErrKafkaInvalidConfig = func(ctx context.Context) error { return jet.NewAppErrBuilder(ErrCodeKafkaInvalidConfig, "config invalid").C(ctx).Err() } ErrKafkaMessageContextInvalid = func(ctx context.Context, topic string) error { return jet.NewAppErrBuilder(ErrCodeKafkaMessageContextInvalid, "message context invalid").F(jet.KV{"topic": topic}).C(ctx).Err() } ErrKafkaMessageMarshal = func(ctx context.Context, cause error, topic string) error { return jet.NewAppErrBuilder(ErrCodeKafkaMessageMarshal, "").Wrap(cause).F(jet.KV{"topic": topic}).C(ctx).Err() } ErrKafkaMessageWrite = func(ctx context.Context, cause error, topic string) error { return jet.NewAppErrBuilder(ErrCodeKafkaMessageWrite, "").Wrap(cause).F(jet.KV{"topic": topic}).C(ctx).Err() } ErrKafkaConnection = func(ctx context.Context, cause error) error { return jet.NewAppErrBuilder(ErrCodeKafkaConnection, "").Wrap(cause).C(ctx).Err() } ErrKafkaCreateTopics = func(ctx context.Context, cause error) error { return jet.NewAppErrBuilder(ErrCodeKafkaCreateTopics, "").Wrap(cause).C(ctx).Err() } ErrKafkaDecodeMsgUnmarshal = func(ctx context.Context, cause error) error { return jet.NewAppErrBuilder(ErrCodeKafkaDecodeMsgUnmarshal, "").Wrap(cause).C(ctx).Err() } ErrKafkaMsgUnmarshalPayload = func(ctx context.Context, cause error) error { return jet.NewAppErrBuilder(ErrCodeKafkaMsgUnmarshalPayload, "").Wrap(cause).C(ctx).Err() } ErrKafkaSubTopicEmpty = func(ctx context.Context) error { return jet.NewAppErrBuilder(ErrCodeKafkaSubTopicEmpty, "topic empty").C(ctx).Err() } ErrKafkaSubNoHandlers = func(ctx context.Context) error { return jet.NewAppErrBuilder(ErrCodeKafkaSubNoHandlers, "no handlers specified").C(ctx).Err() } ErrKafkaProduceMsg = func(ctx context.Context, cause error) error { return jet.NewAppErrBuilder(ErrCodeKafkaProduceMsg, "").Wrap(cause).C(ctx).Err() } ErrKafkaSaslNotSupportedType = func(ctx context.Context) error { return jet.NewAppErrBuilder(ErrCodeKafkaSaslNotSupportedType, "not supported sasl type").C(ctx).Err() } ErrKafkaSaslGetMechanism = func(ctx context.Context, cause error) error { return jet.NewAppErrBuilder(ErrCodeKafkaSaslGetMechanism, "sasl mechanism").C(ctx).Err() } )
Functions ¶
Types ¶
type Broker ¶
type Broker interface {
// Init initializes broker
Init(ctx context.Context, cfg *BrokerConfig) error
// AddProducer adds a producer with configuration
AddProducer(ctx context.Context, topic *TopicConfig, cfg *ProducerConfig) (Producer, error)
// AddSubscriber adds a subscriber with configuration
AddSubscriber(ctx context.Context, topic *TopicConfig, cfg *SubscriberConfig, handlers ...HandlerFn) error
// DeclareTopics declares topics in kafka broker
// must be executed after all producer and subscribers added
DeclareTopics(ctx context.Context) error
// Start starts listening
Start(ctx context.Context) error
// Close closes broker
Close(ctx context.Context)
}
Broker kafka
Example ¶
package main
import (
"context"
"github.com/zloevil/jet"
"github.com/zloevil/jet/kafka"
)
func main() {
logFn := func() jet.CLogger { return jet.L(jet.InitLogger(&jet.LogConfig{Level: jet.InfoLevel})) }
ctx := context.Background()
broker := kafka.NewBroker(logFn)
if err := broker.Init(ctx, &kafka.BrokerConfig{Url: "localhost:9092"}); err != nil {
return
}
// subscribe to a topic
_ = broker.AddSubscriber(ctx,
kafka.NewTopicCfgBuilder("orders").Build(),
kafka.NewSubscriberCfgBuilder().GroupId("my-service").Build(),
func(payload []byte) error {
// handle the message
return nil
},
)
_ = broker.Start(ctx)
defer broker.Close(ctx)
}
Output:
func NewBroker ¶
func NewBroker(logger jet.CLoggerFunc) Broker
type BrokerConfig ¶
type BrokerConfig struct {
ClientId string `mapstructure:"client_id"` // ClientId identifies client
TopicAutoCreation bool `mapstructure:"topic_auto_creation"` // TopicAutoCreation if true topics are created by code (otherwise must be preliminary declared)
Url string // Url comma separated host-port pairs ("localhost:9092,localhost:9093")
Sasl Sasl // Sasl configuration
}
BrokerConfig kafka broker configuration
type Message ¶
type Message struct {
// Ctx allows passing context from producer to subscriber
Ctx *jet.RequestContext `json:"ctx"`
// Key is a kafka message key which allow to specify target kafka partition
Key string `json:"key"`
// Payload arbitrary data
Payload any `json:"payload"`
}
type MessageT ¶
type MessageT[T any] struct { // Ctx allows passing context from producer to subscriber Ctx *jet.RequestContext `json:"ctx"` // Key is a kafka message key which allow to specify target kafka partition Key string `json:"key"` // Payload arbitrary data Payload T `json:"payload"` }
type Producer ¶
type Producer interface {
// Send sends a message to broker
Send(ctx context.Context, key string, payload interface{}) error
// SendMany sends bulk of messages to broker
SendMany(ctx context.Context, messages ...*Message) error
}
Producer allows sending message to broker
type ProducerConfig ¶
type ProducerConfig struct {
BatchSize *int
BatchTimeout *time.Duration
RequiredAcks *int
MaxAttempts *int
Async bool
RetryTimes *int
RetryTimeout *time.Duration
}
ProducerConfig specifies producer config params use builder rather than manual population
type ProducerConfigBuilder ¶
type ProducerConfigBuilder interface {
// BatchTimeout sets batch timeout value (default: 1s)
BatchTimeout(to time.Duration) ProducerConfigBuilder
// BatchSize sets batch size (default: 100)
BatchSize(size int) ProducerConfigBuilder
// Async if true, WriteMessages call will never block but errors aren't returned (default: false)
Async(v bool) ProducerConfigBuilder
// Retry sets retry params (default: time=3, timeout = 1s)
Retry(time int, timeout time.Duration) ProducerConfigBuilder
// Build builds config
Build() *ProducerConfig
}
func NewProducerCfgBuilder ¶
func NewProducerCfgBuilder() ProducerConfigBuilder
type SubscriberConfig ¶
type SubscriberConfig struct {
GroupId string
BatchTimeout *time.Duration
MaxWait *time.Duration
CommitInterval *time.Duration
Workers *int
MaxAttempts *int
StartOffset *int64
JoinGroupBackoff *time.Duration
Logging bool
}
SubscriberConfig specifies subscriber config params use builder rather than manual population
type SubscriberConfigBuilder ¶
type SubscriberConfigBuilder interface {
// GroupId allows load balancing messages within the same group
GroupId(groupId string) SubscriberConfigBuilder
// BatchTimeout sets timeout of batch fetching from kafka (default: 10s)
BatchTimeout(to time.Duration) SubscriberConfigBuilder
// MaxWait sets maximum amount of time to wait for new data to come when fetching batches (default: 10s)
MaxWait(to time.Duration) SubscriberConfigBuilder
// CommitInterval sets interval to commit to kafka (default: sync)
CommitInterval(to time.Duration) SubscriberConfigBuilder
// Workers sets number of workers (default: 4)
Workers(num int) SubscriberConfigBuilder
// StartOffset determines from which offset a new group starts to consume. it must be set to one of FirstOffset = -2 or LastOffset = -1 (Default: FirstOffset)
// Only used when GroupID is set
StartOffset(v int64) SubscriberConfigBuilder
// JoinGroupBackoff optionally sets the length of time to wait between re-joining
JoinGroupBackoff(t time.Duration) SubscriberConfigBuilder
// Logging if true subscriber logging enabled
Logging(v bool) SubscriberConfigBuilder
// Build builds config
Build() *SubscriberConfig
}
func NewSubscriberCfgBuilder ¶
func NewSubscriberCfgBuilder() SubscriberConfigBuilder
type TopicBuilder ¶
type TopicBuilder interface {
// WithPartitionNum setting num of partitions
WithPartitionNum(num int) TopicBuilder
// WithParams setting additional params
WithParams(params ...TopicParam) TopicBuilder
// Build builds config
Build() *TopicConfig
}
TopicBuilder simplifies preparing topic config
func NewTopicCfgBuilder ¶
func NewTopicCfgBuilder(topic string) TopicBuilder
type TopicConfig ¶
type TopicConfig struct {
// Topic name
Topic string
// Partitions number of partitions
Partitions *int
// Configs config params
Configs map[string]string
}
TopicConfig topic config