Versions in this module Expand all Collapse all v0 v0.2.4 Apr 23, 2022 Changes in this version + var Defaults = struct{ ... } + func GetSqlSliceSnowflakeCopyInto(schemaTableName rdbms.SchemaTable, stageName string, fileName string, ...) []string + func GetSqlSliceSnowflakeMerge(cfg *SnowflakeMergeSqlConfig) ... + func GetSqlSliceSnowflakeSyncTable(cfg *SnowflakeSyncSqlConfig) ... + func NewCartNIterator(sizes []int) *cartNIterator + func NewChannelBridge(i interface{}) (inputChan chan chan stream.Record, outputChan chan stream.Record) + func NewChannelCombiner(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewCopyFilesToS3(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewCqnWithArgs(i interface{}) (mainOutputChan chan stream.Record, mainControlChan chan ControlAction) + func NewCsvFileWriter(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewDateRangeGenerator(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewFieldMapper(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewFilterRows(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewGenerateRows(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewManifestWriter(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewMergeDiff(i interface{}) (chan stream.Record, chan ControlAction) + func NewMergeNChannels(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewNumberRangeGenerator(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewS3BucketList(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewS3ManifestReader(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewSnowflakeLoader(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewSnowflakeMerge(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewSnowflakeSync(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewSqlExec(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewSqlQueryWithArgs(i interface{}) (chan stream.Record, chan ControlAction) + func NewSqlQueryWithInputChan(i interface{}) (chan stream.Record, chan ControlAction) + func NewSqlQueryWithReplace(i interface{}) (chan stream.Record, chan ControlAction) + func NewStdOutPassThrough(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewTableMerge(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + func NewTableSync(i interface{}) (outputChan chan stream.Record, controlChan chan ControlAction) + type Action uint32 + const Pause + const Resume + const Shutdown + type ChannelBridgeConfig struct + Log logger.Logger + Name string + PanicHandlerFn PanicHandlerFunc + StepWatcher *stats.StepWatcher + WaitCounter ComponentWaiter + type ChannelCombinerConfig struct + Chan1 chan stream.Record + Chan2 chan stream.Record + Log logger.Logger + Name string + PanicHandlerFn PanicHandlerFunc + StepWatcher *s.StepWatcher + WaitCounter ComponentWaiter + type ColumnType struct + type ComponentStep struct + Data map[string]string + Type string + type ComponentWaiter interface + Add func() + Done func() + type ControlAction struct + Action Action + ResponseChan chan error + type CopyFilesToS3Config struct + BucketName string + BucketPrefix string + FileNameChanField string + InputChan chan stream.Record + Log logger.Logger + Name string + PanicHandlerFn PanicHandlerFunc + Region string + RemoveInputFiles bool + StepWatcher *stats.StepWatcher + WaitCounter ComponentWaiter + type CqnDownstreamSnowflakeSync struct + BucketName string + BucketPrefix string + CsvFileNamePrefix string + CsvHeaderFields []string + CsvMaxFileBytes int + CsvMaxFileRows int + Region string + StageName string + func (s *CqnDownstreamSnowflakeSync) GetDownstreamSyncFunc(cfg *CqnWithArgsConfig) ... + type CqnDownstreamSyncer interface + GetDownstreamSyncFunc func(cfg *CqnWithArgsConfig) ... + type CqnDownstreamTableSync struct + BatchSize int + func (s *CqnDownstreamTableSync) GetDownstreamSyncFunc(cfg *CqnWithArgsConfig) ... + type CqnWithArgsConfig struct + DownstreamHandler CqnDownstreamSyncer + Log logger.Logger + MergeDiffCompareKeys *om.OrderedMap + MergeDiffJoinKeys *om.OrderedMap + Name string + PanicHandlerFn PanicHandlerFunc + SrcCqnConnection shared.OracleCqnExecutor + SrcDBConnector shared.Connector + SrcRowIdKey string + SrcSqlQuery string + StepWatcher *s.StepWatcher + TgtDBConnector shared.Connector + TgtKeyCols *om.OrderedMap + TgtOtherCols *om.OrderedMap + TgtRowIdKey string + TgtSchema string + TgtSqlQuery string + TgtTable string + WaitCounter ComponentWaiter + type CsvFileWriterConfig struct + FileNameExtension string + FileNamePrefix string + FileNameSuffixAppendCreationStamp bool + FileNameSuffixDateFormat string + HeaderFields []string + InputChan chan stream.Record + Log logger.Logger + MaxFileBytes int + MaxFileRows int + Name string + OutputChanField4FilePath string + OutputDir string + PanicHandlerFn PanicHandlerFunc + StepWatcher *s.StepWatcher + UseGzip bool + WaitCounter ComponentWaiter + type DateRangeGeneratorConfig struct + InputChan chan stream.Record + InputChanFieldName4FromDate string + InputChanFieldName4ToDate string + IntervalSizeSeconds int + Log log.Logger + Name string + OutputChanFieldName4HiDate string + OutputChanFieldName4LowDate string + PanicHandlerFn PanicHandlerFunc + PassInputFieldsToOutput bool + StepWatcher *stats.StepWatcher + ToDateRFC3339orNow string + UseUTC bool + WaitCounter ComponentWaiter + type FieldMapperConfig struct + InputChan chan stream.Record + Log logger.Logger + Name string + PanicHandlerFn PanicHandlerFunc + StepWatcher *stats.StepWatcher + Steps []ComponentStep + WaitCounter ComponentWaiter + type FilterMetadata string + type FilterRowsConfig struct + FilterMetadata FilterMetadata + FilterType FilterType + InputChan chan stream.Record + Log log.Logger + Name string + PanicHandlerFn PanicHandlerFunc + StepWatcher *stats.StepWatcher + WaitCounter ComponentWaiter + type FilterType string + type GenerateRowsConfig struct + FieldName4Sequence string + Log logger.Logger + MapFieldNamesValuesCSV string + Name string + NumRows int + PanicHandlerFn PanicHandlerFunc + SleepIntervalSeconds int + StepWatcher *stats.StepWatcher + WaitCounter ComponentWaiter + type ManifestWriterConfig struct + InputChan chan stream.Record + InputChanField4FilePath string + Log logger.Logger + ManifestFileNameExtension string + ManifestFileNamePrefix string + ManifestFileNameSuffixAppendCreationStamp bool + ManifestFileNameSuffixDateFormat string + Name string + OutputChanField4ManifestDir string + OutputChanField4ManifestFullPath string + OutputChanField4ManifestName string + OutputDir string + PanicHandlerFn PanicHandlerFunc + StepWatcher *s.StepWatcher + WaitCounter ComponentWaiter + type MergeDiffConfig struct + ChanNew chan stream.Record + ChanOld chan stream.Record + CompareKeys *om.OrderedMap + JoinKeys *om.OrderedMap + Log logger.Logger + Name string + OutputIdenticalRows bool + PanicHandlerFn PanicHandlerFunc + ResultFlagKeyName string + StepWatcher *s.StepWatcher + WaitCounter ComponentWaiter + type MergeNChannelsConfig struct + AllowFieldOverwrite bool + InputChannels []chan stream.Record + Log logger.Logger + Name string + PanicHandlerFn PanicHandlerFunc + StepWatcher *stats.StepWatcher + WaitCounter ComponentWaiter + type MockComponentWaiter struct + func (cw *MockComponentWaiter) Add() + func (cw *MockComponentWaiter) Done() + type NumberRangeGeneratorConfig struct + InputChan chan stream.Record + InputChanFieldName4HighNum string + InputChanFieldName4LowNum string + IntervalSize float64 + Log log.Logger + Name string + OutputChanFieldName4HighNum string + OutputChanFieldName4LowNum string + OutputLeftPaddedNumZeros int + PanicHandlerFn PanicHandlerFunc + PassInputFieldsToOutput bool + StepWatcher *stats.StepWatcher + WaitCounter ComponentWaiter + type PanicHandlerFunc func() + type S3BucketListerConfig struct + BucketName string + BucketPrefix string + Log logger.Logger + Name string + ObjectNamePrefix string + ObjectNameRegexp string + OutputField4BucketName string + OutputField4BucketPrefix string + OutputField4BucketRegion string + OutputField4FileName string + OutputField4FileNameWithoutPrefix string + PanicHandlerFn PanicHandlerFunc + Region string + StepWatcher *stats.StepWatcher + WaitCounter ComponentWaiter + type S3ManifestReaderConfig struct + BucketName string + BucketPrefix string + InputChan chan stream.Record + InputChanField4ManifestName string + Log logger.Logger + Name string + OutputChanField4DataFileName string + PanicHandlerFn PanicHandlerFunc + Region string + StepWatcher *stats.StepWatcher + WaitCounter ComponentWaiter + type SnowflakeLoaderConfig struct + CommitSequenceKeyName string + Db shared.Connector + DeleteAll bool + FnGetSnowflakeSqlSlice SnowflakeSqlBuilderFunc + InputChan chan stream.Record + InputChanField4FileName string + Log logger.Logger + Name string + PanicHandlerFn PanicHandlerFunc + StageName string + StepWatcher *stats.StepWatcher + TargetSchemaTableName rdbms.SchemaTable + WaitCounter ComponentWaiter + type SnowflakeMergeConfig struct + CommitSequenceKeyName string + Db shared.Connector + InputChan chan stream.Record + InputChanField4FileName string + Log logger.Logger + Name string + PanicHandlerFn PanicHandlerFunc + StageName string + StepWatcher *stats.StepWatcher + TargetKeyCols *om.OrderedMap + TargetOtherCols *om.OrderedMap + TargetSchemaTableName rdbms.SchemaTable + WaitCounter ComponentWaiter + type SnowflakeMergeSqlConfig struct + TargetKeyCols *om.OrderedMap + TargetOtherCols *om.OrderedMap + type SnowflakeSqlBuilderFunc func(tableName rdbms.SchemaTable, stageName string, fileName string, force bool) []string + type SnowflakeSyncConfig struct + CommitSequenceKeyName string + Db shared.Connector + FlagField string + InputChan chan stream.Record + InputChanField4FileName string + Log logger.Logger + Name string + PanicHandlerFn PanicHandlerFunc + StageName string + StepWatcher *stats.StepWatcher + TargetKeyCols *om.OrderedMap + TargetOtherCols *om.OrderedMap + TargetSchemaTableName rdbms.SchemaTable + WaitCounter ComponentWaiter + type SnowflakeSyncSqlConfig struct + FlagField string + TargetKeyCols *om.OrderedMap + TargetOtherCols *om.OrderedMap + type SqlExecConfig struct + InputChan chan stream.Record + Log logger.Logger + Name string + OutputDb shared.Connector + PanicHandlerFn PanicHandlerFunc + SqlQueryFieldName string + SqlRowsAffectedFieldName string + StepWatcher *s.StepWatcher + WaitCounter ComponentWaiter + type SqlQueryWithArgsConfig struct + Args []interface{} + Db shared.Connector + Log logger.Logger + Name string + PanicHandlerFn PanicHandlerFunc + Sqltext string + StepWatcher *s.StepWatcher + WaitCounter ComponentWaiter + type SqlQueryWithChanConfig struct + Db shared.Connector + InputChan chan stream.Record + InputChanFields []string + Log logger.Logger + Name string + PanicHandlerFn PanicHandlerFunc + Sqltext string + StepWatcher *s.StepWatcher + WaitCounter ComponentWaiter + type SqlQueryWithReplace struct + Args []interface{} + Db shared.Connector + Log logger.Logger + Name string + PanicHandlerFn PanicHandlerFunc + Replacements map[string]string + Sqltext string + StepWatcher *s.StepWatcher + WaitCounter ComponentWaiter + type StdOutPassThroughConfig struct + AbortAfterCount int64 + InputChan chan stream.Record + Log logger.Logger + Name string + OutputFields []string + PanicHandlerFn PanicHandlerFunc + StepWatcher *stats.StepWatcher + WaitCounter ComponentWaiter + Writer io.Writer + type TableMergeConfig struct + CommitBatchSize int + ExecBatchSize int + InputChan chan stream.Record + Log logger.Logger + Name string + OutputDb shared.Connector + PanicHandlerFn PanicHandlerFunc + StepWatcher *s.StepWatcher + WaitCounter ComponentWaiter + type TableSyncConfig struct + CommitBatchSize int + CommitSequenceKeyName string + FlagKeyName string + InputChan chan stream.Record + Log logger.Logger + Name string + OutputDb shared.Connector + PanicHandlerFn PanicHandlerFunc + StepWatcher *s.StepWatcher + TxtBatchNumRows int + WaitCounter ComponentWaiter
Go to main page