kafka

package
v1.18.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 30, 2026 License: Apache-2.0 Imports: 42 Imported by: 2

Documentation

Index

Constants

View Source
const (
	// DefaultMaxBulkSubCount is the default max bulk count for kafka pubsub component
	// if the MaxBulkCountKey is not set in the metadata.
	DefaultMaxBulkSubCount = 80
	// DefaultMaxBulkSubAwaitDurationMs is the default max bulk await duration for kafka pubsub component
	// if the MaxBulkAwaitDurationKey is not set in the metadata.
	DefaultMaxBulkSubAwaitDurationMs = 10000
)

Variables

Functions

func GetEventMetadata

func GetEventMetadata(message *sarama.ConsumerMessage, kafka *Kafka) map[string]string

func GetSyncProducer added in v1.15.0

func GetSyncProducer(config sarama.Config, brokers []string, maxMessageBytes int) (sarama.SyncProducer, error)

Types

type AwsClients added in v1.16.0

type AwsClients struct {
	ConsumerGroup sarama.ConsumerGroup
	Producer      sarama.SyncProducer
	// contains filtered or unexported fields
}

func InitAwsClients added in v1.16.0

func InitAwsClients(opts KafkaOptions) *AwsClients

func (*AwsClients) New added in v1.16.0

func (c *AwsClients) New(cfg *aws.Config) error

type BulkEventHandler

type BulkEventHandler func(ctx context.Context, msg *KafkaBulkMessage) ([]pubsub.BulkSubscribeResponseEntry, error)

BulkEventHandler is the handler used to handle the subscribed bulk event.

type EventHandler

type EventHandler func(ctx context.Context, msg *NewEvent) error

EventHandler is the handler used to handle the subscribed event.

type Kafka

type Kafka struct {

	// The default value should be true for kafka pubsub component and false for kafka binding component
	// This default value can be overridden by metadata consumeRetryEnabled
	DefaultConsumeRetryEnabled bool
	// contains filtered or unexported fields
}

Kafka allows reading/writing to a Kafka consumer group.

func NewKafka

func NewKafka(logger logger.Logger) *Kafka

func (*Kafka) BulkPublish

func (k *Kafka) BulkPublish(_ context.Context, topic string, entries []pubsub.BulkMessageEntry, metadata map[string]string) (pubsub.BulkPublishResponse, error)

func (*Kafka) Close

func (k *Kafka) Close() error

func (*Kafka) DeserializeValue

func (k *Kafka) DeserializeValue(message *sarama.ConsumerMessage, config SubscriptionHandlerConfig) ([]byte, error)

func (*Kafka) GetTopicHandlerConfig

func (k *Kafka) GetTopicHandlerConfig(topic string) (SubscriptionHandlerConfig, error)

GetTopicBulkHandler returns the handlerConfig for a topic

func (*Kafka) Init

func (k *Kafka) Init(ctx context.Context, metadata map[string]string) error

Init does metadata parsing and connection establishment.

func (*Kafka) Pause added in v1.17.4

func (k *Kafka) Pause(ctx context.Context) error

Pause stops fetching new messages from the broker for all active subscriptions on this component. The Sarama session and partition assignments are preserved so messages already buffered in claim queues can still be processed by handlers. Idempotent.

func (*Kafka) Publish

func (k *Kafka) Publish(_ context.Context, topic string, data []byte, metadata map[string]string) error

Publish message to Kafka cluster.

func (*Kafka) Resume added in v1.17.4

func (k *Kafka) Resume(ctx context.Context) error

Resume resumes fetching new messages from the broker for all active subscriptions on this component. Idempotent.

func (*Kafka) SerializeValue

func (k *Kafka) SerializeValue(topic string, data []byte, metadata map[string]string) ([]byte, error)

func (*Kafka) Subscribe

func (k *Kafka) Subscribe(ctx context.Context, handlerConfig SubscriptionHandlerConfig, topics ...string)

Subscribe adds a handler and configuration for a topic, and subscribes. Unsubscribes to the topic on context cancel.

func (*Kafka) ValidateAWS added in v1.15.0

func (k *Kafka) ValidateAWS(metadata map[string]string) (awsAuth.Options, error)

type KafkaBulkMessage

type KafkaBulkMessage struct {
	Entries  []KafkaBulkMessageEntry `json:"entries"`
	Topic    string                  `json:"topic"`
	Metadata map[string]string       `json:"metadata"`
}

KafkaBulkMessage is a bulk event arriving from a message bus instance.

type KafkaBulkMessageEntry

type KafkaBulkMessageEntry struct {
	EntryId     string            `json:"entryId"` //nolint:stylecheck
	Event       []byte            `json:"event"`
	ContentType string            `json:"contentType,omitempty"`
	Metadata    map[string]string `json:"metadata"`
}

KafkaBulkMessageEntry is an item contained inside bulk event arriving from a message bus instance.

type KafkaMetadata

type KafkaMetadata struct {
	Brokers string `mapstructure:"brokers"`

	ConsumerGroup string `mapstructure:"consumerGroup"`
	ClientID      string `mapstructure:"clientId"`
	AuthType      string `mapstructure:"authType"`
	SaslUsername  string `mapstructure:"saslUsername"`
	SaslPassword  string `mapstructure:"saslPassword"`
	SaslMechanism string `mapstructure:"saslMechanism"`
	InitialOffset string `mapstructure:"initialOffset"`

	MaxMessageBytes         int    `mapstructure:"maxMessageBytes"`
	OidcTokenEndpoint       string `mapstructure:"oidcTokenEndpoint"`
	OidcClientID            string `mapstructure:"oidcClientID"`
	OidcClientSecret        string `mapstructure:"oidcClientSecret"`
	OidcScopes              string `mapstructure:"oidcScopes"`
	OidcExtensions          string `mapstructure:"oidcExtensions"`
	OidcClientAssertionCert string `mapstructure:"oidcClientAssertionCert"`
	OidcClientAssertionKey  string `mapstructure:"oidcClientAssertionKey"`
	OidcResource            string `mapstructure:"oidcResource"`
	OidcAudience            string `mapstructure:"oidcAudience"`
	OidcKid                 string `mapstructure:"oidcKid"`

	TLSDisable           bool          `mapstructure:"disableTls"`
	TLSSkipVerify        bool          `mapstructure:"skipVerify"`
	TLSCaCert            string        `mapstructure:"caCert"`
	TLSClientCert        string        `mapstructure:"clientCert"`
	TLSClientKey         string        `mapstructure:"clientKey"`
	ConsumeRetryEnabled  bool          `mapstructure:"consumeRetryEnabled"`
	ConsumeRetryInterval time.Duration `mapstructure:"consumeRetryInterval"`
	HeartbeatInterval    time.Duration `mapstructure:"heartbeatInterval"`
	SessionTimeout       time.Duration `mapstructure:"sessionTimeout"`
	Version              string        `mapstructure:"version"`
	EscapeHeaders        bool          `mapstructure:"escapeHeaders"`

	// configs for kafka client
	ClientConnectionTopicMetadataRefreshInterval time.Duration `mapstructure:"clientConnectionTopicMetadataRefreshInterval"`
	ClientConnectionKeepAliveInterval            time.Duration `mapstructure:"clientConnectionKeepAliveInterval"`

	ConsumerGroupRebalanceStrategy string `mapstructure:"consumerGroupRebalanceStrategy"`

	// configs for kafka producer
	Compression string `mapstructure:"compression"`

	// schema registry
	SchemaRegistryURL           string        `mapstructure:"schemaRegistryURL"`
	SchemaRegistryAPIKey        string        `mapstructure:"schemaRegistryAPIKey"`
	SchemaRegistryAPISecret     string        `mapstructure:"schemaRegistryAPISecret"`
	SchemaCachingEnabled        bool          `mapstructure:"schemaCachingEnabled"`
	SchemaLatestVersionCacheTTL time.Duration `mapstructure:"schemaLatestVersionCacheTTL"`
	UseAvroJSON                 bool          `mapstructure:"useAvroJSON"`

	// header from/to metadata excluded keys regex
	ExcludeHeaderMetaRegex string `mapstructure:"excludeHeaderMetaRegex"`
	// contains filtered or unexported fields
}

type KafkaOptions added in v1.16.0

type KafkaOptions struct {
	Config          *sarama.Config
	ConsumerGroup   string
	Brokers         []string
	MaxMessageBytes int
}

type NewEvent

type NewEvent struct {
	Data        []byte            `json:"data"`
	Topic       string            `json:"topic"`
	Metadata    map[string]string `json:"metadata"`
	ContentType *string           `json:"contentType,omitempty"`
}

NewEvent is an event arriving from a message bus instance.

type OAuthTokenSource

type OAuthTokenSource struct {
	CachedToken   oauth2.Token
	Extensions    map[string]string
	TokenEndpoint oauth2.Endpoint
	ClientID      string
	ClientSecret  string
	Scopes        []string
	// contains filtered or unexported fields
}

func (*OAuthTokenSource) Token

func (ts *OAuthTokenSource) Token() (*sarama.AccessToken, error)

type OAuthTokenSourcePrivateKeyJWT added in v1.17.0

type OAuthTokenSourcePrivateKeyJWT struct {
	CachedToken   oauth2.Token
	Extensions    map[string]string
	TokenEndpoint oauth2.Endpoint
	ClientID      string
	ClientSecret  string
	Scopes        []string

	ClientAuthMethod    string
	ClientAssertionCert string
	ClientAssertionKey  string
	Resource            string
	Audience            string
	Kid                 string
	// contains filtered or unexported fields
}

func (*OAuthTokenSourcePrivateKeyJWT) Token added in v1.17.0

At the time of writing this, the oauth2 package does not support the client assertion authentication method. Ref: https://github.com/golang/oauth2/issues/744

type SaramaLogBridge

type SaramaLogBridge struct {
	// contains filtered or unexported fields
}

func (SaramaLogBridge) Print

func (b SaramaLogBridge) Print(v ...interface{})

func (SaramaLogBridge) Printf

func (b SaramaLogBridge) Printf(format string, v ...interface{})

func (SaramaLogBridge) Println

func (b SaramaLogBridge) Println(v ...interface{})

type SchemaCacheEntry

type SchemaCacheEntry struct {
	// contains filtered or unexported fields
}

type SchemaType

type SchemaType int
const (
	None SchemaType = iota
	Avro
)

func GetValueSchemaType

func GetValueSchemaType(metadata map[string]string) (SchemaType, error)

type SubscriptionHandlerConfig

type SubscriptionHandlerConfig struct {
	IsBulkSubscribe bool
	SubscribeConfig pubsub.BulkSubscribeConfig
	BulkHandler     BulkEventHandler
	Handler         EventHandler
	ValueSchemaType SchemaType
}

SubscriptionHandlerConfig is the handler and configuration for subscription.

type TopicHandlerConfig

type TopicHandlerConfig map[string]SubscriptionHandlerConfig

TopicHandlerConfig is the map of topics and sruct containing handler and their config.

func (TopicHandlerConfig) TopicList

func (tbh TopicHandlerConfig) TopicList() []string

TopicList returns the list of topics

type XDGSCRAMClient

type XDGSCRAMClient struct {
	*scram.Client
	*scram.ClientConversation
	scram.HashGeneratorFcn
}

func (*XDGSCRAMClient) Begin

func (x *XDGSCRAMClient) Begin(userName, password, authzID string) (err error)

func (*XDGSCRAMClient) Done

func (x *XDGSCRAMClient) Done() bool

func (*XDGSCRAMClient) Step

func (x *XDGSCRAMClient) Step(challenge string) (response string, err error)

Directories

Path Synopsis
Package mocks is a generated GoMock package.
Package mocks is a generated GoMock package.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL