Documentation
¶
Index ¶
- Variables
- func NewHandler(ctx context.Context, ready chan bool, consumerGroupID string, ...) *loaderHandler
- func ToExtraMaskSchemaMap(r string) map[string]serializer.ExtraMaskInfo
- func ToExtraMaskSchemaString(m map[string]serializer.ExtraMaskInfo) string
- func ToMaskSchemaMap(r string) map[string]serializer.MaskInfo
- func ToMaskSchemaString(m map[string]serializer.MaskInfo) string
- type Job
- type LoaderConfig
Constants ¶
This section is empty.
Variables ¶
View Source
var ( DefaultMaxWaitSeconds int = 1800 DefaultMaxProcessingTime int32 = 600000 MaxRunningLoaders float64 = 10 ThrottlingBudget int = 10 FirstThrottlingBudget int = 120 )
View Source
var JobAvroSchema string = `` /* 867-byte string literal not displayed */
Functions ¶
func NewHandler ¶
func NewHandler( ctx context.Context, ready chan bool, consumerGroupID string, loaderConfig LoaderConfig, saramaConfig kafka.SaramaConfig, redshifter *redshift.Redshift, redshiftSchema string, redshiftGroup *string, redshiftMetrics bool, prometheusClient prometheus.Client, schemaQueries *model.Vector, ) *loaderHandler
func ToExtraMaskSchemaMap ¶
func ToExtraMaskSchemaMap(r string) map[string]serializer.ExtraMaskInfo
TODO: hack, to release fast, found unwanted complications in using map[string]interface in goavro(will revisit)
func ToExtraMaskSchemaString ¶
func ToExtraMaskSchemaString(m map[string]serializer.ExtraMaskInfo) string
TODO: hack, to release fast, found unwanted complications in using map[string]interface in goavro (may revisit if required)
func ToMaskSchemaMap ¶
func ToMaskSchemaMap(r string) map[string]serializer.MaskInfo
TODO: hack, to release fast, found unwanted complications in using map[string]interface in goavro(will revisit)
func ToMaskSchemaString ¶
func ToMaskSchemaString(m map[string]serializer.MaskInfo) string
TODO: hack, to release fast, found unwanted complications in using map[string]interface in goavro (may revisit if required)
Types ¶
type Job ¶
type Job struct {
UpstreamTopic string `json:"upstreamTopic"` // batcher topic
StartOffset int64 `json:"startOffset"`
EndOffset int64 `json:"endOffset"`
CsvDialect string `json:"csvDialect"`
S3Path string `json:"s3Path"`
SchemaId int `json:"schemaId"` // schema id of debezium event for the value for upstream topic (batcher topic)
SchemaIdKey int `json:"schemaIdKey"` // schema id of debezium event for the key for upstream topic (batcher topic)
MaskSchema map[string]serializer.MaskInfo `json:"maskSchema"`
ExtraMaskSchema map[string]serializer.ExtraMaskInfo `json:"extraMaskSchema"`
SkipMerge bool `json:"skipMerge"` // deprecated in favour of createEvents, updateEvents and deleteEvents
BatchBytes int64 `json:"batchBytes"` // batch bytes store sum of all message bytes in this batch
CreateEvents int64 `json:"createEvents"` // stores count of create events
UpdateEvents int64 `json:"updateEvents"` // stores count of update events
DeleteEvents int64 `json:"deleteEvents"` // stores count of delete events
}
func NewJob ¶
func NewJob( upstreamTopic string, startOffset int64, endOffset int64, csvDialect string, s3Path string, schemaId int, schemaIdKey int, maskSchema map[string]serializer.MaskInfo, extraMaskSchema map[string]serializer.ExtraMaskInfo, skipMerge bool, batchBytes, createEvents, updateEvents, deleteEvents int64) Job
func StringMapToJob ¶
StringMapToUser returns a User from a map representation of the User.
func (Job) ToStringMap ¶
ToStringMap returns a map representation of the Job
type LoaderConfig ¶
type LoaderConfig struct {
// Maximum size of a batch, on exceeding this batch is pushed
// regarless of the wait time.
// Deprecated: in favour of MaxBytesPerBatch
MaxSize int `yaml:"maxSize,omitempty"`
// MaxBytesPerBatch is the maximum bytes per batch. It is not the size
// of kafka message but the size of all the messages that would be
// loaded in the batch. Default is there
// if the user has not specified a default will be applied.
// If this is specified, maxSize specification is not considered.
// Default would be specified after MaxSize is gone
MaxBytesPerBatch *int64 `yaml:"maxBytesPerBatch,omitempty"`
// MaxWaitSeconds after which the bash would be pushed regardless of its size.
MaxWaitSeconds *int `yaml:"maxWaitSeconds,omitempty"`
}
Source Files
Click to show internal directories.
Click to hide internal directories.