Documentation
¶
Index ¶
- Constants
- Variables
- func DeleteParquetFiles(t *testing.T, parquetDB, tableName string) error
- func GetBackfillStreamsFromCDC(cdcStreams []string) []string
- func VerifyIcebergSync(t *testing.T, tableName, icebergDB string, datatypeSchema map[string]string, ...)
- func VerifyParquetSync(t *testing.T, tableName, parquetDB string, datatypeSchema map[string]string, ...)
- type IntegrationTest
- type PerformanceTest
- type SyncSpeed
- type TestConfig
Constants ¶
View Source
const ( SyncTimeout = 10 * time.Minute BenchmarkThreshold = 0.9 )
Variables ¶
View Source
var GlobalTypeMapping = map[string]string{
"tinyint": "int",
"smallint": "int",
"mediumint": "int",
"int": "int",
"integer": "int",
"unsigned int": "int",
"unsigned smallint": "int",
"unsigned tinyint": "int",
"unsigned mediumint": "int",
"int2": "int",
"int4": "int",
"smallserial": "int",
"serial": "int",
"serial2": "int",
"serial4": "int",
"bigint": "bigint",
"int8": "bigint",
"serial8": "bigint",
"bigserial": "bigint",
"year": "bigint",
"float": "float",
"real": "float",
"decimal": "double",
"numeric": "double",
"float4": "float",
"money": "float",
"double": "double",
"float8": "double",
"double precision": "double",
"bool": "boolean",
"boolean": "boolean",
"string": "string",
"char": "string",
"varchar": "string",
"tinytext": "string",
"text": "string",
"mediumtext": "string",
"longtext": "string",
"character": "string",
"character varying": "string",
"longvarchar": "string",
"bpchar": "string",
"name": "string",
"binary": "string",
"varbinary": "string",
"tinyblob": "string",
"blob": "string",
"mediumblob": "string",
"longblob": "string",
"bytea": "string",
"json": "string",
"jsonb": "string",
"xml": "string",
"hstore": "string",
"cidr": "string",
"inet": "string",
"macaddr": "string",
"macaddr8": "string",
"geometry": "string",
"point": "string",
"linestring": "string",
"polygon": "string",
"multipoint": "string",
"multilinestring": "string",
"multipolygon": "string",
"geometrycollection": "string",
"circle": "string",
"path": "string",
"box": "string",
"line": "string",
"lseg": "string",
"tsvector": "string",
"tsquery": "string",
"uuid": "string",
"tsrange": "string",
"tstzrange": "string",
"int4range": "string",
"numrange": "string",
"daterange": "string",
"array": "string",
"ARRAY": "string",
"int2vector": "string",
"enum": "string",
"set": "string",
"date": "timestamp",
"timestamp": "timestamp",
"datetime": "timestamp",
"timestamptz": "timestamp",
"timestamp with time zone": "timestamp",
"timestamp without time zone": "timestamp",
"time": "string",
"timez": "string",
"interval": "string",
"pg_lsn": "string",
"bit varying": "string",
"varbit": "string",
"bit(n)": "string",
"varying(n)": "string",
}
GlobalTypeToDataType maps database-specific types (Postgres/MySQL/MongoDB) to internal standard types
Functions ¶
func DeleteParquetFiles ¶ added in v0.3.5
DeleteParquetFiles deletes only .parquet files directly in the table folder in MinIO
func GetBackfillStreamsFromCDC ¶ added in v0.1.10
to get backfill streams from cdc streams e.g. "demo_cdc" -> "demo"
func VerifyIcebergSync ¶
func VerifyIcebergSync(t *testing.T, tableName, icebergDB string, datatypeSchema map[string]string, schema map[string]interface{}, opSymbol, partitionRegex, driver string)
TODO: Refactor parsing logic into a reusable utility functions verifyIcebergSync verifies that data was correctly synchronized to Iceberg
Types ¶
type IntegrationTest ¶
type IntegrationTest struct {
TestConfig *TestConfig
ExpectedData map[string]interface{}
ExpectedUpdatedData map[string]interface{}
DestinationDataTypeSchema map[string]string
UpdatedDestinationDataTypeSchema map[string]string
Namespace string
ExecuteQuery func(ctx context.Context, t *testing.T, streams []string, operation string, fileConfig bool)
DestinationDB string
CursorField string
PartitionRegex string
}
func (*IntegrationTest) TestIntegration ¶
func (cfg *IntegrationTest) TestIntegration(t *testing.T)
type PerformanceTest ¶ added in v0.1.10
type PerformanceTest struct {
TestConfig *TestConfig
Namespace string
BackfillStreams []string
CDCStreams []string
ExecuteQuery func(ctx context.Context, t *testing.T, streams []string, operation string, fileConfig bool)
}
func (*PerformanceTest) TestPerformance ¶ added in v0.1.10
func (cfg *PerformanceTest) TestPerformance(t *testing.T)
type TestConfig ¶ added in v0.1.10
type TestConfig struct {
Driver string
HostRootPath string
SourcePath string
CatalogPath string
IcebergDestinationPath string
ParquetDestinationPath string
StatePath string
StatsPath string
HostTestDataPath string
HostCatalogPath string
HostTestCatalogPath string
}
func GetTestConfig ¶ added in v0.1.10
func GetTestConfig(driver string) *TestConfig
GetTestConfig returns the test config for the given driver
Source Files
Click to show internal directories.
Click to hide internal directories.