mysqlbatch

package
v0.1.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 9, 2026 License: Apache-2.0 Imports: 14 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func ExportSourceByPKRange

func ExportSourceByPKRange(ctx context.Context, db *sql.DB, cfg SourceConfig) ([]string, error)

ExportSourceByPKRange exports source rows into shard text files. Each output line format is: id\tbiz_key\tmetric

func ImportReduceOutputs

func ImportReduceOutputs(ctx context.Context, db *sql.DB, cfg SinkConfig) error

ImportReduceOutputs imports mr-out-* files into target table with batch upsert.

func OpenForApp

func OpenForApp(ctx context.Context, cfg DBConfig) (*sql.DB, error)

OpenForApp opens a MySQL connection for advanced/custom flows.

func PrepareSyntheticSource

func PrepareSyntheticSource(ctx context.Context, db *sql.DB, cfg PrepareConfig) error

PrepareSyntheticSource creates a synthetic source table for benchmark.

func RunFlow

func RunFlow(ctx context.Context, cfg FlowConfig) error

RunFlow executes source -> transform -> sink defined by FlowConfig.

func RunPipeline

func RunPipeline(ctx context.Context, cfg PipelineConfig) error

RunPipeline executes MySQL source -> MapReduce -> MySQL sink in-process.

func ValidateAggregation

func ValidateAggregation(ctx context.Context, db *sql.DB, cfg ValidateConfig) error

ValidateAggregation checks source(group by) equals target table data.

Types

type BenchmarkConfig

type BenchmarkConfig struct {
	DB       DBConfig
	Prepare  bool
	PrepareC PrepareConfig
	Pipeline PipelineConfig
	Validate ValidateConfig
}

BenchmarkConfig configures benchmark workflow.

type BenchmarkResult

type BenchmarkResult struct {
	PrepareDuration  time.Duration
	PipelineDuration time.Duration
	ValidateDuration time.Duration
	TotalDuration    time.Duration
}

BenchmarkResult captures stage durations.

func RunBenchmark

func RunBenchmark(ctx context.Context, cfg BenchmarkConfig) (BenchmarkResult, error)

type DBConfig

type DBConfig struct {
	Host     string
	Port     int
	User     string
	Password string
	Database string
	Params   map[string]string
}

DBConfig defines MySQL connection parameters.

type FlowConfig

type FlowConfig struct {
	Source    FlowSourceConfig    `json:"source"`
	Transform FlowTransformConfig `json:"transform"`
	Sink      FlowSinkConfig      `json:"sink"`
}

FlowConfig describes a SeaTunnel-like source/transform/sink pipeline. Current implementation supports: - source.type: "mysql" - transform.type: "mapreduce" - sink.type: "mysql"

type FlowSinkConfig

type FlowSinkConfig struct {
	Type   string     `json:"type"`
	DB     DBConfig   `json:"db"`
	Config SinkConfig `json:"config"`
}

type FlowSourceConfig

type FlowSourceConfig struct {
	Type   string       `json:"type"`
	DB     DBConfig     `json:"db"`
	Config SourceConfig `json:"config"`
}

type FlowTransformConfig

type FlowTransformConfig struct {
	Type       string            `json:"type"`
	PluginPath string            `json:"plugin_path"`
	Reducers   int               `json:"reducers"`
	Workers    int               `json:"workers"`
	InRAM      bool              `json:"in_ram"`
	Port       int               `json:"port"`
	Params     map[string]string `json:"params"`
}

type PipelineConfig

type PipelineConfig struct {
	DB         DBConfig // backward compatibility fallback when SourceDB/SinkDB are not set
	SourceDB   DBConfig
	SinkDB     DBConfig
	Source     SourceConfig
	Sink       SinkConfig
	PluginPath string
	Reducers   int
	Workers    int
	InRAM      bool
	Port       int
}

PipelineConfig describes end-to-end MySQL -> MapReduce -> MySQL job.

type PrepareConfig

type PrepareConfig struct {
	SourceTable string
	Rows        int64
	KeyMod      int64
}

PrepareConfig configures synthetic source table generation for benchmarking.

type SinkConfig

type SinkConfig struct {
	TargetTable string
	KeyColumn   string
	ValColumn   string
	InputGlob   string
	Replace     bool
	BatchSize   int
}

SinkConfig configures reduce output import into MySQL.

type SourceConfig

type SourceConfig struct {
	Table      string
	PKColumn   string
	KeyColumn  string
	ValColumn  string
	Where      string
	Shards     int
	Parallel   int
	OutputDir  string
	FilePrefix string
}

SourceConfig configures source export from MySQL table to text shards.

type ValidateConfig

type ValidateConfig struct {
	SourceTable string
	SourceKey   string
	SourceVal   string
	TargetTable string
	TargetKey   string
	TargetVal   string
}

ValidateConfig compares source aggregation with target table.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL