mysqlbatch

package
v0.1.2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 9, 2026 License: Apache-2.0 Imports: 16 Imported by: 0

Documentation

Index

Constants

View Source
const FlowVersionV1 = "v1"

Variables

This section is empty.

Functions

func ExportSourceByPKRange

func ExportSourceByPKRange(ctx context.Context, db *sql.DB, cfg SourceConfig) ([]string, error)

ExportSourceByPKRange exports source rows into shard text files. Each output line format is: id\tbiz_key\tmetric

func ImportReduceOutputs

func ImportReduceOutputs(ctx context.Context, db *sql.DB, cfg SinkConfig) error

ImportReduceOutputs imports mr-out-* files into target table with batch upsert.

func OpenForApp

func OpenForApp(ctx context.Context, cfg DBConfig) (*sql.DB, error)

OpenForApp opens a MySQL connection for advanced/custom flows.

func PrepareSyntheticSource

func PrepareSyntheticSource(ctx context.Context, db *sql.DB, cfg PrepareConfig) error

PrepareSyntheticSource creates a synthetic source table for benchmark.

func RunFlow

func RunFlow(ctx context.Context, cfg FlowConfig) error

RunFlow executes source -> transform -> sink defined by FlowConfig.

func RunPipeline

func RunPipeline(ctx context.Context, cfg PipelineConfig) error

RunPipeline executes MySQL source -> MapReduce -> MySQL sink in-process.

func ValidateAggregation

func ValidateAggregation(ctx context.Context, db *sql.DB, cfg ValidateConfig) error

ValidateAggregation checks source(group by) equals target table data.

func ValidateFlowConfig added in v0.1.2

func ValidateFlowConfig(cfg FlowConfig) error

ValidateFlowConfig validates v1 flow schema and required fields.

Types

type BenchmarkConfig

type BenchmarkConfig struct {
	DB       DBConfig
	Prepare  bool
	PrepareC PrepareConfig
	Pipeline PipelineConfig
	Validate ValidateConfig
}

BenchmarkConfig configures benchmark workflow.

type BenchmarkResult

type BenchmarkResult struct {
	PrepareDuration  time.Duration
	PipelineDuration time.Duration
	ValidateDuration time.Duration
	TotalDuration    time.Duration
}

BenchmarkResult captures stage durations.

func RunBenchmark

func RunBenchmark(ctx context.Context, cfg BenchmarkConfig) (BenchmarkResult, error)

type DBConfig

type DBConfig struct {
	Host     string
	Port     int
	User     string
	Password string
	Database string
	Params   map[string]string
}

DBConfig defines MySQL connection parameters.

type FlowConfig

type FlowConfig struct {
	Version   string              `json:"version"`
	Source    FlowSourceConfig    `json:"source"`
	Transform FlowTransformConfig `json:"transform"`
	Sink      FlowSinkConfig      `json:"sink"`
}

FlowConfig describes a SeaTunnel-like source/transform/sink pipeline. Current implementation supports: - source.type: "mysql" - transform.type: "mapreduce" - sink.type: "mysql"

type FlowSinkConfig

type FlowSinkConfig struct {
	Type   string     `json:"type"`
	DB     DBConfig   `json:"db"`
	Config SinkConfig `json:"config"`
}

type FlowSourceConfig

type FlowSourceConfig struct {
	Type   string       `json:"type"`
	DB     DBConfig     `json:"db"`
	Config SourceConfig `json:"config"`
}

type FlowTransformConfig

type FlowTransformConfig struct {
	Type       string            `json:"type"`
	Builtin    string            `json:"builtin"`
	PluginPath string            `json:"plugin_path"`
	Reducers   int               `json:"reducers"`
	Workers    int               `json:"workers"`
	InRAM      bool              `json:"in_ram"`
	Port       int               `json:"port"`
	Params     map[string]string `json:"params"`
}

type PipelineConfig

type PipelineConfig struct {
	DB         DBConfig // backward compatibility fallback when SourceDB/SinkDB are not set
	SourceDB   DBConfig
	SinkDB     DBConfig
	Source     SourceConfig
	Sink       SinkConfig
	PluginPath string
	Reducers   int
	Workers    int
	InRAM      bool
	Port       int
}

PipelineConfig describes end-to-end MySQL -> MapReduce -> MySQL job.

type PrepareConfig

type PrepareConfig struct {
	SourceTable string
	Rows        int64
	KeyMod      int64
}

PrepareConfig configures synthetic source table generation for benchmarking.

type SinkConfig

type SinkConfig struct {
	TargetTable string
	KeyColumn   string
	ValColumn   string
	InputGlob   string
	Replace     bool
	BatchSize   int
}

SinkConfig configures reduce output import into MySQL.

type SourceConfig

type SourceConfig struct {
	Table      string
	PKColumn   string
	KeyColumn  string
	ValColumn  string
	Where      string
	Shards     int
	Parallel   int
	OutputDir  string
	FilePrefix string
}

SourceConfig configures source export from MySQL table to text shards.

type ValidateConfig

type ValidateConfig struct {
	SourceTable string
	SourceKey   string
	SourceVal   string
	TargetTable string
	TargetKey   string
	TargetVal   string
}

ValidateConfig compares source aggregation with target table.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL