ingestor

type Interface interface {
	Open(ctx context.Context) error
	Close() error
	HandleReady(w http.ResponseWriter, r *http.Request)
	HandleTransfer(w http.ResponseWriter, r *http.Request)
	Shutdown(ctx context.Context) error
	UploadSegments(ctx context.Context) error
	DisableWrites() error
}

type Service struct {
	// contains filtered or unexported fields
}

type ServiceOpts struct {
	StorageDir     string
	Uploader       Uploader
	MaxSegmentSize int64
	MaxSegmentAge  time.Duration

	K8sCli     kubernetes.Interface
	K8sCtrlCli client.Client

	// MetricsKustoCli is the Kusto client connected to the metrics kusto cluster.
	MetricsKustoCli []metrics.StatementExecutor

	// LogsKustoCli is the Kusto client connected to the logs kusto cluster.
	LogsKustoCli []metrics.StatementExecutor

	// InsecureSkipVerify disables TLS certificate verification.
	InsecureSkipVerify bool

	// Namespace is the namespace used for peer discovery.
	Namespace string

	// Hostname is the hostname of the current node.
	Hostname string

	// Region is a location identifier
	Region string

	// DisablePeerTransfer disables peer discovery and prevents transfers of small segments to an owner.
	// Each instance of ingestor will upload received segments directly.
	DisablePeerTransfer bool

	// MaxTransferSize is the minimum size of a segment that will be transferred to another node.  If a segment
	// exceeds this size, it will be uploaded directly by the current node.
	MaxTransferSize int64

	// MaxTransferAge is the maximum age of a segment that will be transferred to another node.  If a segment
	// exceeds this age, it will be uploaded directly by the current node.
	MaxTransferAge time.Duration

	// MaxSegmentCount is the maximum number of segments files allowed on disk before signaling back-pressure.
	MaxSegmentCount int64

	// MaxDiskUsage is the maximum disk usage allowed before signaling back-pressure.
	MaxDiskUsage int64

	// AllowedDatabases is the distinct set of database names that are allowed to be written to.
	AllowedDatabase []string

	// MetricsDatabase is the name of the metrics database.
	MetricsDatabases []string

	// LogsDatabases is a slice of log database names.
	LogsDatabases []string

	// PartitionSize is the max size of the group of nodes forming a partition.  A partition is a set of nodes where
	// keys are distributed.
	PartitionSize int

	// MaxTransferConcurrency is the maximum number of concurrent transfers allowed in flight at the same time.
	MaxTransferConcurrency int

	// ConcurrentUploads is the maximum number of concurrent uploads.
	ConcurrentUploads int

	// EnableWALFsync enables fsync of segments before closing the segment.
	EnableWALFsync bool

	// DropFilePrefixes is a slice of prefixes that will be dropped when importing segments.
	DropFilePrefixes []string

	// MaxBatchSegments is the maximum number of segments to include when transferring segments in a batch.  The segments
	// are merged into a new segment.  A higher number takes longer to combine on the sending side and increases the
	// size of segments on the receiving side.  A lower number creates more batches and high remote transfer calls.  If
	// not specified, the default is 25.
	MaxBatchSegments int

	// SlowRequestThreshold is the threshold for logging slow requests.
	SlowRequestThreshold float64

	ClusterLabels map[string]string

	StorageBackend storage.Backend
}

type Uploader interface {
	service.Component
	UploadQueue() chan *cluster.Batch
	Database() string
}