kafka

package
v0.6.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 5, 2026 License: MIT Imports: 20 Imported by: 0

Documentation

Overview

+marmot:name=Kafka +marmot:description=This plugin discovers Kafka topics from Kafka clusters. +marmot:status=experimental +marmot:features=Assets

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type AuthConfig

type AuthConfig struct {
	Type      string `` /* 142-byte string literal not displayed */
	Username  string `json:"username,omitempty" description:"SASL username"`
	Password  string `json:"password,omitempty" description:"SASL password" sensitive:"true"`
	Mechanism string `` /* 153-byte string literal not displayed */
}

Authentication configuration

type Config

type Config struct {
	plugin.BaseConfig `json:",inline"`

	BootstrapServers string            `json:"bootstrap_servers" description:"Comma-separated list of bootstrap servers" validate:"required"`
	ClientID         string            `json:"client_id" description:"Client ID for the consumer"`
	Authentication   *AuthConfig       `json:"authentication,omitempty" description:"Authentication configuration"`
	ConsumerConfig   map[string]string `json:"consumer_config,omitempty" description:"Additional consumer configuration"`
	ClientTimeout    int               `json:"client_timeout_seconds" description:"Request timeout in seconds" validate:"omitempty,min=1,max=300"`
	TLS              *TLSConfig        `json:"tls,omitempty" description:"TLS configuration"`

	SchemaRegistry *SchemaRegistryConfig `json:"schema_registry,omitempty" description:"Schema Registry configuration"`

	TopicFilter *plugin.Filter `json:"topic_filter,omitempty" description:"Filter configuration for topics"`

	IncludePartitionInfo bool `json:"include_partition_info" description:"Whether to include partition information in metadata" default:"true"`

	IncludeTopicConfig bool `json:"include_topic_config" description:"Whether to include topic configuration in metadata" default:"true"`
}

Config for Kafka plugin +marmot:config

func (*Config) ApplyDefaults added in v0.2.0

func (c *Config) ApplyDefaults()

type ConsumerGroupDetails

type ConsumerGroupDetails struct {
	State        string
	Protocol     string
	ProtocolType string
	Members      []ConsumerGroupMember
}

type ConsumerGroupMember

type ConsumerGroupMember struct {
	ClientID        string
	ClientHost      string
	TopicPartitions map[string][]int
}

type KafkaConsumerGroupFields

type KafkaConsumerGroupFields struct {
	GroupId          string   `json:"group_id" metadata:"group_id" description:"Consumer group ID"`
	State            string   `json:"state" metadata:"state" description:"Current state of the consumer group"`
	Protocol         string   `json:"protocol" metadata:"protocol" description:"Rebalance protocol"`
	ProtocolType     string   `json:"protocol_type" metadata:"protocol_type" description:"Protocol type"`
	SubscribedTopics []string `json:"subscribed_topics" metadata:"subscribed_topics" description:"Topics the group is subscribed to"`
	Members          []string `json:"members" metadata:"members" description:"Members of the consumer group"`
}

KafkaConsumerGroupFields represents Kafka consumer group-specific metadata fields +marmot:metadata

type KafkaTopicFields

type KafkaTopicFields struct {
	TopicName          string `json:"topic_name" metadata:"topic_name" description:"Name of the Kafka topic"`
	PartitionCount     int32  `json:"partition_count" metadata:"partition_count" description:"Number of partitions"`
	ReplicationFactor  int16  `json:"replication_factor" metadata:"replication_factor" description:"Replication factor"`
	RetentionMs        string `json:"retention_ms" metadata:"retention_ms" description:"Message retention period in milliseconds"`
	RetentionBytes     string `json:"retention_bytes" metadata:"retention_bytes" description:"Maximum size of the topic in bytes"`
	CleanupPolicy      string `json:"cleanup_policy" metadata:"cleanup_policy" description:"Topic cleanup policy"`
	MinInsyncReplicas  string `json:"min_insync_replicas" metadata:"min_insync.replicas" description:"Minimum number of in-sync replicas"`
	MaxMessageBytes    string `json:"max_message_bytes" metadata:"max_message.bytes" description:"Maximum message size in bytes"`
	SegmentBytes       string `json:"segment_bytes" metadata:"segment_bytes" description:"Segment file size in bytes"`
	SegmentMs          string `json:"segment_ms" metadata:"segment_ms" description:"Segment file roll time in milliseconds"`
	DeleteRetentionMs  string `json:"delete_retention_ms" metadata:"delete_retention_ms" description:"Time to retain deleted segments in milliseconds"`
	ValueSchemaId      int    `json:"value_schema_id" metadata:"value_schema_id" description:"ID of the value schema in Schema Registry"`
	ValueSchemaVersion int    `json:"value_schema_version" metadata:"value_schema_version" description:"Version of the value schema"`
	ValueSchemaType    string `json:"value_schema_type" metadata:"value_schema_type" description:"Type of the value schema (AVRO, JSON, etc.)"`
	ValueSchema        string `json:"value_schema" metadata:"value_schema" description:"Value schema definition"`
	KeySchemaId        int    `json:"key_schema_id" metadata:"key_schema_id" description:"ID of the key schema in Schema Registry"`
	KeySchemaVersion   int    `json:"key_schema_version" metadata:"key_schema_version" description:"Version of the key schema"`
	KeySchemaType      string `json:"key_schema_type" metadata:"key_schema_type" description:"Type of the key schema (AVRO, JSON, etc.)"`
	KeySchema          string `json:"key_schema" metadata:"key_schema" description:"Key schema definition"`
}

KafkaTopicFields represents Kafka topic-specific metadata fields +marmot:metadata

type SchemaRegistryConfig

type SchemaRegistryConfig struct {
	URL        string            `json:"url" description:"Schema Registry URL" validate:"omitempty,url"`
	Config     map[string]string `json:"config,omitempty" description:"Additional Schema Registry configuration"`
	Enabled    bool              `json:"enabled" description:"Whether to use Schema Registry"`
	SkipVerify bool              `json:"skip_verify,omitempty" description:"Skip TLS certificate verification"`
}

Schema Registry configuration

type Source

type Source struct {
	// contains filtered or unexported fields
}

func (*Source) Discover

func (s *Source) Discover(ctx context.Context, pluginConfig plugin.RawPluginConfig) (*plugin.DiscoveryResult, error)

func (*Source) Validate

func (s *Source) Validate(rawConfig plugin.RawPluginConfig) (plugin.RawPluginConfig, error)

type TLSConfig added in v0.2.0

type TLSConfig struct {
	Enabled    bool   `json:"enabled" description:"Whether to enable TLS"`
	CertPath   string `json:"cert_path,omitempty" description:"Path to TLS certificate file"`
	KeyPath    string `json:"key_path,omitempty" description:"Path to TLS key file"`
	CACertPath string `json:"ca_cert_path,omitempty" description:"Path to TLS CA certificate file"`
	SkipVerify bool   `json:"skip_verify,omitempty" description:"Skip TLS verification"`
}

TLS configuration

type TopicDetails

type TopicDetails struct {
	// contains filtered or unexported fields
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL