Documentation
¶
Index ¶
- Constants
- type Interface
- type RealtimeOpts
- type Service
- func (s *Service) Close() error
- func (s *Service) DisableWrites() error
- func (s *Service) HandleDebugStore(w http.ResponseWriter, r *http.Request)
- func (s *Service) HandleReady(w http.ResponseWriter, r *http.Request)
- func (s *Service) HandleTransfer(w http.ResponseWriter, r *http.Request)
- func (s *Service) Open(ctx context.Context) error
- func (s *Service) Shutdown(ctx context.Context) error
- func (s *Service) UploadSegments(ctx context.Context) error
- type ServiceOpts
- type Uploader
Constants ¶
View Source
const ( // DefaultUnownedSegmentMaxAge is the default max age of segments of tables owned by another ingestor. DefaultUnownedSegmentMaxAge = time.Second // DefaultTransferLinger is the default time a closed segment of a table owned by another ingestor waits to be // batched with others before it is transferred. Transfers waiting for the owner are merged, so it can be shorter // than the upload linger. DefaultTransferLinger = time.Second )
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type RealtimeOpts ¶ added in v0.4.0
type RealtimeOpts struct {
// Policy assigns ingestion priorities to tables.
Policy *ingestpolicy.Policy
// MaxSegmentAge is the max age of realtime WAL segments before they are rotated.
MaxSegmentAge time.Duration
// MaxBatchLatency is the max time a closed realtime segment waits to be batched.
MaxBatchLatency time.Duration
// MaxBatchBytes is the max size of a realtime batch in WAL bytes.
MaxBatchBytes int64
// ReservedDiskBytes is disk space reserved for realtime segments.
ReservedDiskBytes int64
}
RealtimeOpts configures realtime ingestion in the ingestor.
type Service ¶
type Service struct {
// contains filtered or unexported fields
}
func NewService ¶
func NewService(opts ServiceOpts) (*Service, error)
func (*Service) DisableWrites ¶
func (*Service) HandleDebugStore ¶ added in v0.2.0
func (s *Service) HandleDebugStore(w http.ResponseWriter, r *http.Request)
func (*Service) HandleReady ¶ added in v0.2.0
func (s *Service) HandleReady(w http.ResponseWriter, r *http.Request)
HandleReady handles the readiness probe.
func (*Service) HandleTransfer ¶
func (s *Service) HandleTransfer(w http.ResponseWriter, r *http.Request)
HandleTransfer handles the transfer WAL segments from other nodes in the cluster.
type ServiceOpts ¶
type ServiceOpts struct {
StorageDir string
Uploader Uploader
MaxSegmentSize int64
MaxSegmentAge time.Duration
// UnownedSegmentMaxAge, when positive, replaces MaxSegmentAge for queued tables owned by another ingestor while
// peer transfers are enabled, so their data reaches the owner sooner.
UnownedSegmentMaxAge time.Duration
K8sCli kubernetes.Interface
K8sCtrlCli client.Client
// MetricsKustoCli is the Kusto client connected to the metrics kusto cluster.
MetricsKustoCli []ingestormetrics.StatementExecutor
// LogsKustoCli is the Kusto client connected to the logs kusto cluster.
LogsKustoCli []ingestormetrics.StatementExecutor
// InsecureSkipVerify disables TLS certificate verification.
InsecureSkipVerify bool
// Namespace is the namespace used for peer discovery.
Namespace string
// Hostname is the hostname of the current node.
Hostname string
// Region is a location identifier
Region string
// DisablePeerTransfer disables peer discovery and prevents transfers of small segments to an owner.
// Each instance of ingestor will upload received segments directly.
DisablePeerTransfer bool
// MaxTransferSize is the minimum size of a segment that will be transferred to another node. If a segment
// exceeds this size, it will be uploaded directly by the current node.
MaxTransferSize int64
// MaxTransferAge is the maximum age of a segment that will be transferred to another node. If a segment
// exceeds this age, it will be uploaded directly by the current node.
MaxTransferAge time.Duration
// MaxSegmentCount is the maximum number of segments files allowed on disk before signaling back-pressure.
MaxSegmentCount int64
// MaxDiskUsage is the maximum disk usage allowed before signaling back-pressure.
MaxDiskUsage int64
MaxTableDiskUsage int64
// MaxIntakeMemory bounds the bytes buffered across concurrent transfer
// requests. Zero disables the ceiling.
MaxIntakeMemory int64
MaxTableSegmentCount int64
DisableTableLimits bool
EnforceTableLimits bool
TableLimits []wal.TableLimit
// AllowedDatabases is the distinct set of database names that are allowed to be written to.
AllowedDatabase []string
// MetricsDatabase is the name of the metrics database.
MetricsDatabases []string
// LogsDatabases is a slice of log database names.
LogsDatabases []string
// PartitionSize is the max size of the group of nodes forming a partition. A partition is a set of nodes where
// keys are distributed.
PartitionSize int
// MaxTransferConcurrency is the maximum number of concurrent transfers allowed in flight at the same time.
MaxTransferConcurrency int
// MaxTransfersPerTable, when positive, is how many transfers of one table run at once. Batches waiting behind a
// running transfer are merged, so the owner receives fewer, larger transfers. When zero, a table may use every
// transfer worker but the reserved ones.
MaxTransfersPerTable int
// TransferLinger, when positive, is how long a closed segment of a table owned by another ingestor waits to be
// batched with others before it is transferred. When zero, it waits as long as owned segments.
TransferLinger time.Duration
// UploadLinger, when positive, is how long a closed segment of a table this ingestor owns waits to be batched
// with others before it is uploaded. When zero, it waits cluster.DefaultQueuedLinger.
UploadLinger time.Duration
// ConcurrentUploads is the maximum number of concurrent uploads.
ConcurrentUploads int
// EnableWALFsync enables fsync of segments before closing the segment.
EnableWALFsync bool
// DropFilePrefixes is a slice of prefixes that will be dropped when importing segments.
DropFilePrefixes []string
// MaxBatchSegments is the maximum number of segments to include when transferring segments in a batch. The segments
// are merged into a new segment. A higher number takes longer to combine on the sending side and increases the
// size of segments on the receiving side. A lower number creates more batches and high remote transfer calls. If
// not specified, the default is 25.
MaxBatchSegments int
// SlowRequestThreshold is the threshold for logging slow requests.
SlowRequestThreshold float64
ClusterLabels map[string]string
StorageBackend storage.Backend
// Realtime configures realtime ingestion. When nil or without realtime tables, all tables are queued.
Realtime *RealtimeOpts
// QueuedReservedWorkersPercent is the percentage of transfer workers reserved for queued batches.
QueuedReservedWorkersPercent int
// PeerListeners are called with the ingestor's peer count and rank when the service opens and when they change.
PeerListeners []func(cluster.PeerInfo)
}
Click to show internal directories.
Click to hide internal directories.