ingestor

package
v0.4.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 10, 2026 License: MIT Imports: 30 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Interface added in v0.2.0

type Interface interface {
	Open(ctx context.Context) error
	Close() error
	HandleReady(w http.ResponseWriter, r *http.Request)
	HandleTransfer(w http.ResponseWriter, r *http.Request)
	Shutdown(ctx context.Context) error
	UploadSegments(ctx context.Context) error
	DisableWrites() error
}

type RealtimeOpts added in v0.4.0

type RealtimeOpts struct {
	// Policy assigns ingestion priorities to tables.
	Policy *ingestpolicy.Policy

	// MaxSegmentAge is the max age of realtime WAL segments before they are rotated.
	MaxSegmentAge time.Duration

	// MaxBatchLatency is the max time a closed realtime segment waits to be batched.
	MaxBatchLatency time.Duration

	// MaxBatchBytes is the max size of a realtime batch in WAL bytes.
	MaxBatchBytes int64

	// ReservedDiskBytes is disk space reserved for realtime segments.
	ReservedDiskBytes int64
}

RealtimeOpts configures realtime ingestion in the ingestor.

type Service

type Service struct {
	// contains filtered or unexported fields
}

func NewService

func NewService(opts ServiceOpts) (*Service, error)

func (*Service) Close

func (s *Service) Close() error

func (*Service) DisableWrites

func (s *Service) DisableWrites() error

func (*Service) HandleDebugStore added in v0.2.0

func (s *Service) HandleDebugStore(w http.ResponseWriter, r *http.Request)

func (*Service) HandleReady added in v0.2.0

func (s *Service) HandleReady(w http.ResponseWriter, r *http.Request)

HandleReady handles the readiness probe.

func (*Service) HandleTransfer

func (s *Service) HandleTransfer(w http.ResponseWriter, r *http.Request)

HandleTransfer handles the transfer WAL segments from other nodes in the cluster.

func (*Service) Open

func (s *Service) Open(ctx context.Context) error

func (*Service) Shutdown added in v0.2.0

func (s *Service) Shutdown(ctx context.Context) error

func (*Service) UploadSegments

func (s *Service) UploadSegments(ctx context.Context) error

type ServiceOpts

type ServiceOpts struct {
	StorageDir     string
	Uploader       Uploader
	MaxSegmentSize int64
	MaxSegmentAge  time.Duration

	K8sCli     kubernetes.Interface
	K8sCtrlCli client.Client

	// MetricsKustoCli is the Kusto client connected to the metrics kusto cluster.
	MetricsKustoCli []ingestormetrics.StatementExecutor

	// LogsKustoCli is the Kusto client connected to the logs kusto cluster.
	LogsKustoCli []ingestormetrics.StatementExecutor

	// InsecureSkipVerify disables TLS certificate verification.
	InsecureSkipVerify bool

	// Namespace is the namespace used for peer discovery.
	Namespace string

	// Hostname is the hostname of the current node.
	Hostname string

	// Region is a location identifier
	Region string

	// ForwardTransfers forwards collector transfers for tables owned by another ingestor to the owner as they are
	// received, rather than storing them and transferring them after the segment rotates.
	ForwardTransfers bool

	// DisablePeerTransfer disables peer discovery and prevents transfers of small segments to an owner.
	// Each instance of ingestor will upload received segments directly.
	DisablePeerTransfer bool

	// MaxTransferSize is the minimum size of a segment that will be transferred to another node.  If a segment
	// exceeds this size, it will be uploaded directly by the current node.
	MaxTransferSize int64

	// MaxTransferAge is the maximum age of a segment that will be transferred to another node.  If a segment
	// exceeds this age, it will be uploaded directly by the current node.
	MaxTransferAge time.Duration

	// MaxSegmentCount is the maximum number of segments files allowed on disk before signaling back-pressure.
	MaxSegmentCount int64

	// MaxDiskUsage is the maximum disk usage allowed before signaling back-pressure.
	MaxDiskUsage      int64
	MaxTableDiskUsage int64
	// MaxIntakeMemory bounds the bytes buffered across concurrent transfer
	// requests. Zero disables the ceiling.
	MaxIntakeMemory      int64
	MaxTableSegmentCount int64
	DisableTableLimits   bool
	EnforceTableLimits   bool
	TableLimits          []wal.TableLimit

	// AllowedDatabases is the distinct set of database names that are allowed to be written to.
	AllowedDatabase []string

	// MetricsDatabase is the name of the metrics database.
	MetricsDatabases []string

	// LogsDatabases is a slice of log database names.
	LogsDatabases []string

	// PartitionSize is the max size of the group of nodes forming a partition.  A partition is a set of nodes where
	// keys are distributed.
	PartitionSize int

	// MaxTransferConcurrency is the maximum number of concurrent transfers allowed in flight at the same time.
	MaxTransferConcurrency int

	// ConcurrentUploads is the maximum number of concurrent uploads.
	ConcurrentUploads int

	// EnableWALFsync enables fsync of segments before closing the segment.
	EnableWALFsync bool

	// DropFilePrefixes is a slice of prefixes that will be dropped when importing segments.
	DropFilePrefixes []string

	// MaxBatchSegments is the maximum number of segments to include when transferring segments in a batch.  The segments
	// are merged into a new segment.  A higher number takes longer to combine on the sending side and increases the
	// size of segments on the receiving side.  A lower number creates more batches and high remote transfer calls.  If
	// not specified, the default is 25.
	MaxBatchSegments int

	// SlowRequestThreshold is the threshold for logging slow requests.
	SlowRequestThreshold float64

	ClusterLabels map[string]string

	StorageBackend storage.Backend

	// Realtime configures realtime ingestion.  When nil or without realtime tables, all tables are queued.
	Realtime *RealtimeOpts

	// QueuedReservedWorkersPercent is the percentage of transfer workers reserved for queued batches.
	QueuedReservedWorkersPercent int

	// PeerListeners are called with the ingestor's peer count and rank when the service opens and when they change.
	PeerListeners []func(cluster.PeerInfo)
}

type Uploader added in v0.2.0

type Uploader interface {
	service.Component
	UploadQueue() chan *cluster.Batch
	Database() string
}

Uploader captures the minimal contract required by the ingestor service to interact with storage backends. Both the legacy ADX uploader and the new ClickHouse uploader satisfy this interface.

Directories

Path Synopsis
Package cluster implements rendezvous hashing (a.k.a.
Package cluster implements rendezvous hashing (a.k.a.
runner

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL