kafka

package
v0.2.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 24, 2026 License: MIT Imports: 16 Imported by: 0

Documentation

Overview

Package kafka provides Kafka producers and subscribers built on segmentio/kafka-go.

A Broker creates producers and subscribers for topics. It supports SASL authentication, optional topic auto-creation, worker-based parallel consumption (ordered per message key) and request-context propagation via a message envelope. Use the builders (NewProducerCfgBuilder, NewSubscriberCfgBuilder, NewTopicCfgBuilder) to configure each component.

Subscribers support two delivery guarantees, selected via SubscriberConfigBuilder.DeliveryGuarantee: AtMostOnce (the default — parallel, fastest, but may drop in-flight messages on shutdown/crash) and AtLeastOnce (sequential, commits only after processing, so nothing is lost on shutdown/crash at the cost of possible redelivery). See DeliveryGuarantee.

Index

Examples

Constants

View Source
const (
	ErrCodeKafkaFetchMessage             = "KF-001"
	ErrCodeKafkaCommitMessage            = "KF-002"
	ErrCodeKafkaNotInitialized           = "KF-003"
	ErrCodeKafkaInvalidConfig            = "KF-004"
	ErrCodeKafkaMessageContextInvalid    = "KF-005"
	ErrCodeKafkaMessageMarshal           = "KF-006"
	ErrCodeKafkaProducerTopicEmpty       = "KF-008"
	ErrCodeKafkaSubTopicEmpty            = "KF-009"
	ErrCodeKafkaSubNoHandlers            = "KF-010"
	ErrCodeKafkaConnection               = "KF-011"
	ErrCodeKafkaCreateTopics             = "KF-012"
	ErrCodeKafkaMessageWrite             = "KF-013"
	ErrCodeKafkaDecodeMsgUnmarshal       = "KF-014"
	ErrCodeKafkaMsgUnmarshalPayload      = "KF-015"
	ErrCodeKafkaProduceMsg               = "KF-016"
	ErrCodeKafkaSaslNotSupportedType     = "KF-017"
	ErrCodeKafkaSaslGetMechanism         = "KF-018"
	ErrCodeKafkaInvalidDeliveryGuarantee = "KF-019"
	ErrCodeKafkaHandlerPanic             = "KF-020"
)
View Source
const (
	SaslTypePlain       = "plain"
	SaslTypeScramSha256 = "sha256"
	SaslTypeScramSha512 = "sha512"
)

Variables

View Source
var (
	ErrKafkaFetchMessage = func(cause error) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaFetchMessage, "").Wrap(cause).Err()
	}
	ErrKafkaCommitMessage = func(cause error) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaCommitMessage, "").Wrap(cause).Err()
	}
	ErrKafkaNotInitialized = func(ctx context.Context) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaNotInitialized, "not initialized").C(ctx).Err()
	}
	ErrKafkaProducerTopicEmpty = func(ctx context.Context) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaProducerTopicEmpty, "topic empty").C(ctx).Err()
	}
	ErrKafkaInvalidConfig = func(ctx context.Context) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaInvalidConfig, "config invalid").C(ctx).Err()
	}
	ErrKafkaMessageContextInvalid = func(ctx context.Context, topic string) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaMessageContextInvalid, "message context invalid").F(jet.KV{"topic": topic}).C(ctx).Err()
	}
	ErrKafkaMessageMarshal = func(ctx context.Context, cause error, topic string) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaMessageMarshal, "").Wrap(cause).F(jet.KV{"topic": topic}).C(ctx).Err()
	}
	ErrKafkaMessageWrite = func(ctx context.Context, cause error, topic string) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaMessageWrite, "").Wrap(cause).F(jet.KV{"topic": topic}).C(ctx).Err()
	}
	ErrKafkaConnection = func(ctx context.Context, cause error) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaConnection, "").Wrap(cause).C(ctx).Err()
	}
	ErrKafkaCreateTopics = func(ctx context.Context, cause error) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaCreateTopics, "").Wrap(cause).C(ctx).Err()
	}
	ErrKafkaDecodeMsgUnmarshal = func(ctx context.Context, cause error) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaDecodeMsgUnmarshal, "").Wrap(cause).C(ctx).Err()
	}
	ErrKafkaMsgUnmarshalPayload = func(ctx context.Context, cause error) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaMsgUnmarshalPayload, "").Wrap(cause).C(ctx).Err()
	}
	ErrKafkaSubTopicEmpty = func(ctx context.Context) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaSubTopicEmpty, "topic empty").C(ctx).Err()
	}
	ErrKafkaSubNoHandlers = func(ctx context.Context) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaSubNoHandlers, "no handlers specified").C(ctx).Err()
	}
	ErrKafkaProduceMsg = func(ctx context.Context, cause error) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaProduceMsg, "").Wrap(cause).C(ctx).Err()
	}
	ErrKafkaSaslNotSupportedType = func(ctx context.Context) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaSaslNotSupportedType, "not supported sasl type").C(ctx).Err()
	}
	ErrKafkaSaslGetMechanism = func(ctx context.Context, cause error) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaSaslGetMechanism, "sasl mechanism").C(ctx).Err()
	}
	ErrKafkaInvalidDeliveryGuarantee = func(ctx context.Context, value string) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaInvalidDeliveryGuarantee, "invalid delivery guarantee").F(jet.KV{"value": value}).C(ctx).Business().Err()
	}
	ErrKafkaHandlerPanic = func(cause error) error {
		return jet.NewAppErrBuilder(ErrCodeKafkaHandlerPanic, "handler panic").Wrap(cause).Err()
	}
)

Functions

func Decode

func Decode[T any](parentCtx context.Context, msg []byte) (T, context.Context, error)

Types

type Broker

type Broker interface {
	// Init initializes broker
	Init(ctx context.Context, cfg *BrokerConfig) error
	// AddProducer adds a producer with configuration
	AddProducer(ctx context.Context, topic *TopicConfig, cfg *ProducerConfig) (Producer, error)
	// AddSubscriber adds a subscriber with configuration
	AddSubscriber(ctx context.Context, topic *TopicConfig, cfg *SubscriberConfig, handlers ...HandlerFn) error
	// DeclareTopics declares topics in kafka broker
	// must be executed after all producer and subscribers added
	DeclareTopics(ctx context.Context) error
	// Start starts listening
	Start(ctx context.Context) error
	// Close closes broker
	Close(ctx context.Context)
}

Broker kafka

Example
package main

import (
	"context"

	"github.com/zloevil/jet"
	"github.com/zloevil/jet/kafka"
)

func main() {
	logFn := func() jet.CLogger { return jet.L(jet.InitLogger(&jet.LogConfig{Level: jet.InfoLevel})) }
	ctx := context.Background()

	broker := kafka.NewBroker(logFn)
	if err := broker.Init(ctx, &kafka.BrokerConfig{Url: "localhost:9092"}); err != nil {
		return
	}

	// subscribe to a topic
	_ = broker.AddSubscriber(ctx,
		kafka.NewTopicCfgBuilder("orders").Build(),
		kafka.NewSubscriberCfgBuilder().GroupId("my-service").Build(),
		func(payload []byte) error {
			// handle the message
			return nil
		},
	)

	_ = broker.Start(ctx)
	defer broker.Close(ctx)
}
Example (AtLeastOnce)
package main

import (
	"context"

	"github.com/zloevil/jet"
	"github.com/zloevil/jet/kafka"
)

func main() {
	logFn := func() jet.CLogger { return jet.L(jet.InitLogger(&jet.LogConfig{Level: jet.InfoLevel})) }
	ctx := context.Background()

	broker := kafka.NewBroker(logFn)
	if err := broker.Init(ctx, &kafka.BrokerConfig{Url: "localhost:9092"}); err != nil {
		return
	}

	// subscribe with at-least-once delivery: the offset is committed only after
	// the handler returns, so no message is lost on shutdown or crash. A message
	// may be redelivered, so the handler must be idempotent.
	_ = broker.AddSubscriber(ctx,
		kafka.NewTopicCfgBuilder("orders").Build(),
		kafka.NewSubscriberCfgBuilder().
			GroupId("my-service").
			DeliveryGuarantee(kafka.AtLeastOnce).
			Build(),
		func(payload []byte) error {
			// handle the message; the offset is committed once this returns nil
			// (returning an error retries the same message)
			return nil
		},
	)

	_ = broker.Start(ctx)
	defer broker.Close(ctx)
}

func NewBroker

func NewBroker(logger jet.CLoggerFunc) Broker

type BrokerConfig

type BrokerConfig struct {
	ClientId          string `mapstructure:"client_id"`           // ClientId identifies client
	TopicAutoCreation bool   `mapstructure:"topic_auto_creation"` // TopicAutoCreation if true topics are created by code (otherwise must be preliminary declared)
	Url               string // Url comma separated host-port pairs ("localhost:9092,localhost:9093")
	Sasl              Sasl   // Sasl configuration
}

BrokerConfig kafka broker configuration

type DeliveryGuarantee added in v0.2.0

type DeliveryGuarantee string

DeliveryGuarantee controls when a consumed message's offset is committed relative to handler processing.

const (
	// AtMostOnce commits the offset as soon as a message is read, before the
	// handlers run. It is the default. It keeps the parallel per-key worker pool
	// and is the fastest option, but messages that are buffered or in-flight when
	// the process stops (SIGTERM, scale-down) or crashes are lost, because their
	// offset has already advanced.
	AtMostOnce DeliveryGuarantee = "at_most_once"
	// AtLeastOnce commits the offset only after a handler returns nil, so nothing
	// is lost on shutdown or crash — an uncommitted message is redelivered, and
	// handlers must therefore be idempotent. A handler that returns an error (or
	// panics) is retried on the same message until it succeeds; return nil to skip
	// a message that cannot be processed. Messages are processed sequentially
	// (Kafka tracks a single offset per partition, so out-of-order commits would
	// skip unprocessed messages), which lowers throughput compared to AtMostOnce.
	AtLeastOnce DeliveryGuarantee = "at_least_once"
)

type HandlerFn

type HandlerFn func(payload []byte) error

HandlerFn handler function

type Message

type Message struct {
	// Ctx allows passing context from producer to subscriber
	Ctx *jet.RequestContext `json:"ctx"`
	// Key is a kafka message key which allow to specify target kafka partition
	Key string `json:"key"`
	// Payload arbitrary data
	Payload any `json:"payload"`
}

type MessageT

type MessageT[T any] struct {
	// Ctx allows passing context from producer to subscriber
	Ctx *jet.RequestContext `json:"ctx"`
	// Key is a kafka message key which allow to specify target kafka partition
	Key string `json:"key"`
	// Payload arbitrary data
	Payload T `json:"payload"`
}

type Producer

type Producer interface {
	// Send sends a message to broker
	Send(ctx context.Context, key string, payload interface{}) error
	// SendMany sends bulk of messages to broker
	SendMany(ctx context.Context, messages ...*Message) error
}

Producer allows sending message to broker

type ProducerConfig

type ProducerConfig struct {
	BatchSize    *int
	BatchTimeout *time.Duration
	RequiredAcks *int
	MaxAttempts  *int
	Async        bool
	RetryTimes   *int
	RetryTimeout *time.Duration
}

ProducerConfig specifies producer config params use builder rather than manual population

type ProducerConfigBuilder

type ProducerConfigBuilder interface {
	// BatchTimeout sets batch timeout value (default: 1s)
	BatchTimeout(to time.Duration) ProducerConfigBuilder
	// BatchSize sets batch size (default: 100)
	BatchSize(size int) ProducerConfigBuilder
	// Async if true, WriteMessages call will never block but errors aren't returned (default: false)
	Async(v bool) ProducerConfigBuilder
	// Retry sets retry params (default: time=3, timeout = 1s)
	Retry(time int, timeout time.Duration) ProducerConfigBuilder
	// Build builds config
	Build() *ProducerConfig
}

func NewProducerCfgBuilder

func NewProducerCfgBuilder() ProducerConfigBuilder

type Sasl

type Sasl struct {
	Enabled  bool   // Enabled true if Sasl must be applied
	Type     string // Type of mechanism (plain, sha256, sha512)
	Username string // Username
	Password string // Password
}

type SubscriberConfig

type SubscriberConfig struct {
	GroupId           string
	BatchTimeout      *time.Duration
	MaxWait           *time.Duration
	CommitInterval    *time.Duration
	Workers           *int
	MaxAttempts       *int
	StartOffset       *int64
	JoinGroupBackoff  *time.Duration
	Logging           bool
	DeliveryGuarantee DeliveryGuarantee
}

SubscriberConfig specifies subscriber config params use builder rather than manual population

type SubscriberConfigBuilder

type SubscriberConfigBuilder interface {
	// GroupId allows load balancing messages within the same group
	GroupId(groupId string) SubscriberConfigBuilder
	// BatchTimeout sets timeout of batch fetching from kafka (default: 10s)
	BatchTimeout(to time.Duration) SubscriberConfigBuilder
	// MaxWait sets maximum amount of time to wait for new data to come when fetching batches (default: 10s)
	MaxWait(to time.Duration) SubscriberConfigBuilder
	// CommitInterval sets interval to commit to kafka (default: sync). Ignored for AtLeastOnce, which always commits synchronously
	CommitInterval(to time.Duration) SubscriberConfigBuilder
	// Workers sets number of workers (default: 4). Only used with AtMostOnce delivery (AtLeastOnce is sequential)
	Workers(num int) SubscriberConfigBuilder
	// StartOffset determines from which offset a new group starts to consume. it must be set to one of FirstOffset = -2 or LastOffset = -1 (Default: FirstOffset)
	// Only used when GroupID is set
	StartOffset(v int64) SubscriberConfigBuilder
	// JoinGroupBackoff optionally sets the length of time to wait between re-joining
	JoinGroupBackoff(t time.Duration) SubscriberConfigBuilder
	// Logging if true subscriber logging enabled
	Logging(v bool) SubscriberConfigBuilder
	// DeliveryGuarantee sets delivery semantics: AtMostOnce (default) or
	// AtLeastOnce. See the DeliveryGuarantee constants for the tradeoffs.
	DeliveryGuarantee(g DeliveryGuarantee) SubscriberConfigBuilder
	// Build builds config
	Build() *SubscriberConfig
}

func NewSubscriberCfgBuilder

func NewSubscriberCfgBuilder() SubscriberConfigBuilder

type TopicBuilder

type TopicBuilder interface {
	// WithPartitionNum setting num of partitions
	WithPartitionNum(num int) TopicBuilder
	// WithParams setting additional params
	WithParams(params ...TopicParam) TopicBuilder
	// Build builds config
	Build() *TopicConfig
}

TopicBuilder simplifies preparing topic config

func NewTopicCfgBuilder

func NewTopicCfgBuilder(topic string) TopicBuilder

type TopicConfig

type TopicConfig struct {
	// Topic name
	Topic string
	// Partitions number of partitions
	Partitions *int
	// Configs config params
	Configs map[string]string
}

TopicConfig topic config

type TopicParam

type TopicParam struct {
	Name  string
	Value string
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL