Documentation
¶
Index ¶
- func ExportSourceByPKRange(ctx context.Context, db *sql.DB, cfg SourceConfig) ([]string, error)
- func ImportReduceOutputs(ctx context.Context, db *sql.DB, cfg SinkConfig) error
- func OpenForApp(ctx context.Context, cfg DBConfig) (*sql.DB, error)
- func PrepareSyntheticSource(ctx context.Context, db *sql.DB, cfg PrepareConfig) error
- func RunFlow(ctx context.Context, cfg FlowConfig) error
- func RunPipeline(ctx context.Context, cfg PipelineConfig) error
- func ValidateAggregation(ctx context.Context, db *sql.DB, cfg ValidateConfig) error
- type BenchmarkConfig
- type BenchmarkResult
- type DBConfig
- type FlowConfig
- type FlowSinkConfig
- type FlowSourceConfig
- type FlowTransformConfig
- type PipelineConfig
- type PrepareConfig
- type SinkConfig
- type SourceConfig
- type ValidateConfig
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func ExportSourceByPKRange ¶
ExportSourceByPKRange exports source rows into shard text files. Each output line format is: id\tbiz_key\tmetric
func ImportReduceOutputs ¶
ImportReduceOutputs imports mr-out-* files into target table with batch upsert.
func OpenForApp ¶
OpenForApp opens a MySQL connection for advanced/custom flows.
func PrepareSyntheticSource ¶
PrepareSyntheticSource creates a synthetic source table for benchmark.
func RunFlow ¶
func RunFlow(ctx context.Context, cfg FlowConfig) error
RunFlow executes source -> transform -> sink defined by FlowConfig.
func RunPipeline ¶
func RunPipeline(ctx context.Context, cfg PipelineConfig) error
RunPipeline executes MySQL source -> MapReduce -> MySQL sink in-process.
func ValidateAggregation ¶
ValidateAggregation checks source(group by) equals target table data.
Types ¶
type BenchmarkConfig ¶
type BenchmarkConfig struct {
DB DBConfig
Prepare bool
PrepareC PrepareConfig
Pipeline PipelineConfig
Validate ValidateConfig
}
BenchmarkConfig configures benchmark workflow.
type BenchmarkResult ¶
type BenchmarkResult struct {
PrepareDuration time.Duration
PipelineDuration time.Duration
ValidateDuration time.Duration
TotalDuration time.Duration
}
BenchmarkResult captures stage durations.
func RunBenchmark ¶
func RunBenchmark(ctx context.Context, cfg BenchmarkConfig) (BenchmarkResult, error)
type DBConfig ¶
type DBConfig struct {
Host string
Port int
User string
Password string
Database string
Params map[string]string
}
DBConfig defines MySQL connection parameters.
type FlowConfig ¶
type FlowConfig struct {
Source FlowSourceConfig `json:"source"`
Transform FlowTransformConfig `json:"transform"`
Sink FlowSinkConfig `json:"sink"`
}
FlowConfig describes a SeaTunnel-like source/transform/sink pipeline. Current implementation supports: - source.type: "mysql" - transform.type: "mapreduce" - sink.type: "mysql"
type FlowSinkConfig ¶
type FlowSinkConfig struct {
Type string `json:"type"`
DB DBConfig `json:"db"`
Config SinkConfig `json:"config"`
}
type FlowSourceConfig ¶
type FlowSourceConfig struct {
Type string `json:"type"`
DB DBConfig `json:"db"`
Config SourceConfig `json:"config"`
}
type FlowTransformConfig ¶
type PipelineConfig ¶
type PipelineConfig struct {
DB DBConfig // backward compatibility fallback when SourceDB/SinkDB are not set
SourceDB DBConfig
SinkDB DBConfig
Source SourceConfig
Sink SinkConfig
PluginPath string
Reducers int
Workers int
InRAM bool
Port int
}
PipelineConfig describes end-to-end MySQL -> MapReduce -> MySQL job.
type PrepareConfig ¶
PrepareConfig configures synthetic source table generation for benchmarking.
type SinkConfig ¶
type SinkConfig struct {
TargetTable string
KeyColumn string
ValColumn string
InputGlob string
Replace bool
BatchSize int
}
SinkConfig configures reduce output import into MySQL.