core

type ErrorPolicy int

type Handler interface {
	Init(ctx context.Context) error
	Write(msg []byte) error
	Invoke(ctx context.Context) (arrow.Table, error)
}

type Message struct {
	Value     []byte
	Topic     string
	Partition int32
	Offset    int64
}

type MetadataWriter interface {
	WriteMessage(msg Message) error
}

type Metrics struct {
	MessageCount           metric.Int64Counter
	ErrorCount             metric.Int64Counter
	SourceReadLatency      metric.Float64Histogram
	SinkFlushLatency       metric.Float64Histogram
	SinkFlushNumRows       metric.Int64Gauge
	SinkFlushCount         metric.Int64Counter
	BatchProcessingLatency metric.Float64Histogram
}

type PipelineErrorPolicies struct {
	Policy  ErrorPolicy
	DLQSink Sink
}

type Sink interface {
	WriteTable(batch arrow.Table) error
	Flush() error
	Batch() (arrow.Table, error)
}

type Source interface {
	Start() error
	Stream() <-chan []Message
	Commit() error
	Close() error
}

type Stats struct {
	StartTime time.Time
	NumErrors int
	// contains filtered or unexported fields
}

type Turbine struct {
	// contains filtered or unexported fields
}

type TurbineOption func(turbine *Turbine)