collector

package
v0.4.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 12, 2026 License: MIT Imports: 45 Imported by: 0

Documentation

Index

Constants

View Source
const DefaultMaxDiskUsage int64 = 10 * 1024 * 1024 * 1024

DefaultMaxDiskUsage is the max disk usage when ServiceOpts.MaxDiskUsage is not set.

View Source
const DefaultMaxTransferSize = 100 * 1024 * 1024

DefaultMaxTransferSize bounds the size of one transfer made by merging a table's waiting batches. It matches the ingestor's default max-transfer-size.

View Source
const DefaultScrapeConcurrency = 4

DefaultScrapeConcurrency is the default maximum number of targets scraped at once.

View Source
const DefaultTransferBatchSize = 4 * 1024 * 1024

DefaultTransferBatchSize is the size at which the segments of a table are sent to the ingestor as a batch.

Variables

This section is empty.

Functions

func NewGzipReaderWithClose

func NewGzipReaderWithClose(r io.ReadCloser) (*gzipReaderWithClose, error)

NewGzipReaderWithClose creates a new gzipReaderWithClose

Types

type ClientOpts

type ClientOpts struct {
	DialTimeout         time.Duration
	TLSHandshakeTimeout time.Duration
	ScrapeTimeOut       time.Duration
	NodeName            string
}

func (ClientOpts) WithDefaults

func (c ClientOpts) WithDefaults() ClientOpts

type HttpLogCollectorOpts added in v0.2.0

type HttpLogCollectorOpts struct {
	CreateHTTPSvc createHttpFunc
}

HttpLogCollectorOpts is the options for creating a log collector with an HTTP endpoint.

type LogCollectorOpts

type LogCollectorOpts struct {
	Create createFunc
}

LogCollectorOpts is the options for creating a log collector.

type MetricsClient

type MetricsClient struct {
	NodeName string
	// contains filtered or unexported fields
}

func NewMetricsClient

func NewMetricsClient(opts ClientOpts) (*MetricsClient, error)

func (*MetricsClient) Close

func (c *MetricsClient) Close() error

func (*MetricsClient) FetchMetrics

func (c *MetricsClient) FetchMetrics(target string) (map[string]*prom_model.MetricFamily, error)

func (*MetricsClient) FetchMetricsIterator

func (c *MetricsClient) FetchMetricsIterator(ctx context.Context, target string) (*prompb.Iterator, error)

type MetricsHandlerOpts

type MetricsHandlerOpts struct {
	AddLabels map[string]string

	// DropLabels is a map of metric names regexes to label name regexes.  When both match, the label will be dropped.
	DropLabels map[*regexp.Regexp]*regexp.Regexp

	// DropMetrics is a slice of regexes that drops metrics when the metric name matches.  The metric name format
	// should match the Prometheus naming style before the metric is translated to a Kusto table name.
	DropMetrics []*regexp.Regexp

	KeepMetrics []*regexp.Regexp

	KeepMetricsLabelValues map[*regexp.Regexp]*regexp.Regexp

	// DisableMetricsForwarding disables the forwarding of metrics to the remote write endpoint.
	DisableMetricsForwarding bool
	DefaultDropMetrics       bool

	RemoteWriteClients []remote.RemoteWriteClient

	DynamicLabeler metadata.MetricLabeler
}

func (MetricsHandlerOpts) RequestTransformer

func (o MetricsHandlerOpts) RequestTransformer() *transform.RequestTransformer

type OtlpMetricsHandlerOpts

type OtlpMetricsHandlerOpts struct {
	// Optional. Path is the path where the OTLP/HTTP handler will be registered.
	Path string
	// Optional. GrpcPort is the port where the metrics OTLP/GRPC handler will listen.
	GrpcPort int
	// RequestTimeout is the maximum duration for an OTLP metrics request.
	RequestTimeout time.Duration

	MetricOpts MetricsHandlerOpts
}

type PrometheusRemoteWriteHandlerOpts

type PrometheusRemoteWriteHandlerOpts struct {
	// Path is the path where the handler will be registered.
	Path string
	// RequestTimeout is the maximum duration for a remote write request.
	RequestTimeout time.Duration

	MetricOpts MetricsHandlerOpts
}

type RealtimeOpts added in v0.4.0

type RealtimeOpts struct {
	// Policy assigns ingestion priorities to tables.
	Policy *ingestpolicy.Policy

	// MaxSegmentAge is the max age of realtime WAL segments before they are rotated.
	MaxSegmentAge time.Duration

	// MaxBatchLatency is the max time a closed realtime segment waits to be batched before it is transferred.
	MaxBatchLatency time.Duration
	// LogBatchWait is the pod-tail batch wait for realtime destinations.
	LogBatchWait time.Duration

	// MaxBatchBytes is the max size of a realtime transfer batch in WAL bytes.
	MaxBatchBytes int64

	// ReservedDiskBytes is disk space reserved for realtime segments.
	ReservedDiskBytes int64
}

RealtimeOpts configures realtime ingestion in the collector.

type ScrapeTarget

type ScrapeTarget struct {
	Static    bool
	Addr      string
	Namespace string
	Pod       string
	Container string

	// ScrapeInterval overrides the global interval. Zero inherits the default.
	ScrapeInterval time.Duration
}

func (ScrapeTarget) Equals

func (t ScrapeTarget) Equals(other ScrapeTarget) bool

func (ScrapeTarget) String

func (t ScrapeTarget) String() string

type Scraper

type Scraper struct {
	// contains filtered or unexported fields
}

func NewScraper

func NewScraper(opts *ScraperOpts) *Scraper

func (*Scraper) Close

func (s *Scraper) Close() error

func (*Scraper) OnAdd

func (s *Scraper) OnAdd(obj any, isInitialList bool)

func (*Scraper) OnDelete

func (s *Scraper) OnDelete(obj any)

func (*Scraper) OnUpdate

func (s *Scraper) OnUpdate(oldObj, newObj any)

func (*Scraper) Open

func (s *Scraper) Open(ctx context.Context) error

func (*Scraper) Targets

func (s *Scraper) Targets() []ScrapeTarget

type ScraperOpts

type ScraperOpts struct {
	NodeName string

	DefaultDropMetrics bool

	// AddLabels is a map of key/value pairs that will be added to all metrics.
	AddLabels map[string]string

	// DropLabels is a map of metric names regexes to label name regexes.  When both match, the label will be dropped.
	DropLabels map[*regexp.Regexp]*regexp.Regexp

	// DropMetrics is a slice of regexes that drops metrics when the metric name matches.  The metric name format
	// should match the Prometheus naming style before the metric is translated to a Kusto table name.
	DropMetrics []*regexp.Regexp

	KeepMetricsWithLabelValue map[*regexp.Regexp]*regexp.Regexp

	KeepMetrics []*regexp.Regexp

	// Database is the name of the database to write metrics to.
	Database string

	// ScrapeInterval is the interval at which to scrape metrics from the targets.
	ScrapeInterval time.Duration

	// ScrapeTimeout is the timeout for scraping metrics from a target.  It is capped at the target's scrape interval.
	ScrapeTimeout time.Duration

	// ScrapeConcurrency is the maximum number of targets scraped at once.  Defaults to DefaultScrapeConcurrency.
	ScrapeConcurrency int

	// DisableMetricsForwarding disables the forwarding of metrics to the remote write endpoint.
	DisableMetricsForwarding bool

	// DisableDiscovery disables the discovery of scrape targets.
	DisableDiscovery bool

	PodInformer k8s.PodInformerInterface

	// Targets is a list of static scrape targets.
	Targets []ScrapeTarget

	// MaxBatchSize is the maximum number of samples to send in a single batch.
	MaxBatchSize int

	RemoteClients []remote.RemoteWriteClient

	HealthChecker interface{ IsHealthy() bool }

	DynamicLabeler metadata.MetricLabeler
}

func (*ScraperOpts) RequestTransformer

func (s *ScraperOpts) RequestTransformer() *transform.RequestTransformer

type Service

type Service struct {
	// contains filtered or unexported fields
}

func NewService

func NewService(opts *ServiceOpts) (*Service, error)

func (*Service) Close

func (s *Service) Close() error

func (*Service) HandleDebugStore added in v0.2.0

func (s *Service) HandleDebugStore(w stdhttp.ResponseWriter, r *stdhttp.Request)

func (*Service) HandleReady added in v0.2.0

func (s *Service) HandleReady(w stdhttp.ResponseWriter, r *stdhttp.Request)

func (*Service) Open

func (s *Service) Open(ctx context.Context) error

type ServiceOpts

type ServiceOpts struct {
	ListenAddr     string
	NodeName       string
	Endpoint       string
	StorageBackend storage.Backend

	KubeNode *metadata.KubeNode

	// PodInformer is the pod informer used for discovery (optional)
	PodInformer k8s.PodInformerInterface

	// LogCollectionHandlers is the list of log collection handlers
	LogCollectionHandlers []LogCollectorOpts
	// HttpLogCollectionHandlers is the list of log collection handlers with HTTP endpoints
	HttpLogCollectorOpts []HttpLogCollectorOpts
	// PromMetricsHandlers is the list of prom-remote handlers
	PromMetricsHandlers []PrometheusRemoteWriteHandlerOpts
	// OtlpMetricsHandlers is the list of oltp metrics handlers
	OtlpMetricsHandlers []OtlpMetricsHandlerOpts
	// Scraper is the options for the prom scraper
	Scraper *ScraperOpts

	// Labels to lift to columns
	LiftLabels []string

	AddAttributes  map[string]string
	LiftAttributes []string
	LiftResources  []string

	// InsecureSkipVerify skips the verification of the remote write endpoint certificate chain and host name.
	InsecureSkipVerify bool

	TLSCertFile string
	TLSKeyFile  string

	// MaxBatchSize is the maximum number of samples to send in a single batch.
	MaxBatchSize int

	// MaxSegmentAge is the maximum time allowed before a segment is rolled over.
	MaxSegmentAge time.Duration

	// MaxSegmentSize is the maximum size allowed for a segment before it is rolled over.
	MaxSegmentSize int64

	// MaxDiskUsage is the max size in bytes to use for segment store.  If this value is exceeded, writes
	// will be rejected until space is freed.  A value of 0 means no max usage.
	MaxDiskUsage int64

	// MaxSegmentCount is the maximum number of segments files allowed on disk before signaling back-pressure.
	MaxSegmentCount      int64
	MaxTableDiskUsage    int64
	MaxTableSegmentCount int64
	DisableTableLimits   bool
	EnforceTableLimits   bool
	TableLimits          []wal.TableLimit

	// StorageDir is the directory where the WAL will be stored
	StorageDir string

	// EnablePprof enables pprof endpoints.
	EnablePprof bool

	// Region is a location identifier
	Region string

	MaxConnections int

	// WALFlushInterval is the interval at which the WAL segment is flushed to disk.  A higher value results in
	// better disk IO and less CPU usage but has a greater risk of data loss in the event of a crash.  A lower
	// value results in more frequent disk IO and higher CPU usage but less data loss in the event of a crash.
	// The default is 100ms and the value must be greater than 0.
	WALFlushInterval time.Duration

	// MaxTransferConcurrency is the maximum number of concurrent transfers to the ingestor.
	MaxTransferConcurrency int

	// MaxBatchSegments is the maximum number of segments to include when transferring segments in a batch.  The segments
	// are merged into a new segment.  A higher number takes longer to combine on the sending size and increases the
	// size of segments on the receiving size.  A lower number creates more batches and high remote transfer calls.  If
	// not specified, the default is 25.
	MaxBatchSegments int

	// TransferBatchSize is the size at which the segments of a table are sent to the ingestor as a batch.  When zero,
	// it is DefaultTransferBatchSize.
	TransferBatchSize int64

	// MaxTransferSize bounds the bytes of one transfer made by merging a table's batches that wait for a transfer
	// worker.  When zero, it is DefaultMaxTransferSize.
	MaxTransferSize int64

	// DisableGzip disables gzip compression for the transfer endpoint.
	DisableGzip bool

	// Realtime configures realtime ingestion.  When nil or without realtime tables, all tables are queued.
	Realtime *RealtimeOpts

	// QueuedReservedWorkersPercent is the percentage of transfer workers reserved for queued batches.
	QueuedReservedWorkersPercent int
}

type StoreRemoteClient

type StoreRemoteClient struct {
	// contains filtered or unexported fields
}

func (*StoreRemoteClient) CloseIdleConnections

func (s *StoreRemoteClient) CloseIdleConnections()

func (*StoreRemoteClient) Write

type StoreRequestWriter

type StoreRequestWriter struct {
	// contains filtered or unexported fields
}

func (*StoreRequestWriter) CloseIdleConnections

func (s *StoreRequestWriter) CloseIdleConnections()

func (*StoreRequestWriter) Write

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL