databrickssql

package

v0.0.4 Latest Latest Go to latest Published: Mar 15, 2024 License: MIT Imports: 14 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/pingcap-inc/tidb2dw

Links

Open Source Insights

Documentation ¶

Index ¶

Variables
func GenCreateExternalTableSQL(tableName string, tableColumns []cloudstorage.TableCol, storageUri string, ...) (string, error)
func GenCreateTableSQL(tableName string, tableColumns []cloudstorage.TableCol) (string, error)
func GenDDLViaColumnsDiff(prevColumns []cloudstorage.TableCol, curTableDef cloudstorage.TableDefinition) ([]string, error)
func GenDropTableSQL(sourceTable string) string
func GenMergeIntoSQL(tableDef cloudstorage.TableDefinition, tableName, externalTableName string) string
func GetCredentialNameSet(db *sql.DB) (map[string]interface{}, error)
func GetDatabricksColumnString(column cloudstorage.TableCol) (string, error)
func GetDatabricksTypeString(column cloudstorage.TableCol) (string, error)
func LoadCSVFromS3(db *sql.DB, columns []cloudstorage.TableCol, ...) error
type DataBricksConfig
- func (config *DataBricksConfig) OpenDB() (*sql.DB, error)
type DatabricksConnector
- func NewDatabricksConnector(dbConfig *DataBricksConfig, credential string, storageURI *url.URL) (*DatabricksConnector, error)

Constants ¶

This section is empty.

Variables ¶

View Source

var TiDB2DatabricksTypeMap = map[string]string{
	"text":       "STRING",
	"tinytext":   "STRING",
	"mediumtext": "STRING",
	"longtext":   "STRING",
	"blob":       "STRING",
	"tinyblob":   "STRING",
	"mediumblob": "STRING",
	"longblob":   "STRING",
	"varchar":    "STRING",
	"char":       "STRING",
	"int":        "INT",
	"mediumint":  "INT",
	"tinyint":    "TINYINT",
	"smallint":   "SMALLINT",
	"bigint":     "BIGINT",
	"float":      "FLOAT",
	"double":     "DOUBLE",
	"decimal":    "DECIMAL",
	"numeric":    "NUMERIC",
	"bool":       "BOOLEAN",
	"boolean":    "BOOLEAN",
	"date":       "DATE",
	"datetime":   "TIMESTAMP_NTZ",
	"timestamp":  "TIMESTAMP",
	"time":       "TIMESTAMP_NTZ",
}

TiDB2DatabricksTypeMap is a map from TiDB type to Databricks type. Please refer to https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html Databricks don't support the `BINARY` type in the external table with the CSV file which are `tidb2dw` used. So we drop the `binary` and `varbinary` type in the map.

Functions ¶

func GenCreateExternalTableSQL ¶

func GenCreateExternalTableSQL(tableName string, tableColumns []cloudstorage.TableCol, storageUri string, credential string) (string, error)

func GenCreateTableSQL ¶

func GenCreateTableSQL(tableName string, tableColumns []cloudstorage.TableCol) (string, error)

func GenDDLViaColumnsDiff ¶

func GenDDLViaColumnsDiff(prevColumns []cloudstorage.TableCol, curTableDef cloudstorage.TableDefinition) ([]string, error)

func GenDropTableSQL ¶

func GenDropTableSQL(sourceTable string) string

func GenMergeIntoSQL ¶

func GenMergeIntoSQL(tableDef cloudstorage.TableDefinition, tableName, externalTableName string) string

func GetCredentialNameSet ¶

func GetCredentialNameSet(db *sql.DB) (map[string]interface{}, error)

GetCredentialNameSet returns all storage credential names in the database

func GetDatabricksColumnString ¶

func GetDatabricksColumnString(column cloudstorage.TableCol) (string, error)

GetDatabricksColumnString returns a string describing the column in Databricks, e.g. "id INT NOT NULL DEFAULT '0'" Refer to: https://dev.mysql.com/doc/refman/8.0/en/data-types.html https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html

func GetDatabricksTypeString ¶

func GetDatabricksTypeString(column cloudstorage.TableCol) (string, error)

func LoadCSVFromS3 ¶

func LoadCSVFromS3(db *sql.DB, columns []cloudstorage.TableCol, targetTable, storageUri, filePath string, credential string) error

Types ¶

type DataBricksConfig ¶

type DataBricksConfig struct {
	Host     string
	Port     int
	Token    string
	Endpoint string
	Catalog  string
	Schema   string
}

func (*DataBricksConfig) OpenDB ¶

func (config *DataBricksConfig) OpenDB() (*sql.DB, error)

type DatabricksConnector ¶

type DatabricksConnector struct {
	// contains filtered or unexported fields
}

func NewDatabricksConnector ¶

func NewDatabricksConnector(dbConfig *DataBricksConfig, credential string, storageURI *url.URL) (*DatabricksConnector, error)

func (*DatabricksConnector) Close ¶

func (dc *DatabricksConnector) Close()

func (*DatabricksConnector) CopyTableSchema ¶

func (dc *DatabricksConnector) CopyTableSchema(sourceDatabase string, sourceTable string, sourceTiDBConn *sql.DB) error

func (*DatabricksConnector) ExecDDL ¶

func (dc *DatabricksConnector) ExecDDL(tableDef cloudstorage.TableDefinition) error

func (*DatabricksConnector) InitSchema ¶

func (dc *DatabricksConnector) InitSchema(columns []cloudstorage.TableCol) error

func (*DatabricksConnector) LoadIncrement ¶

func (dc *DatabricksConnector) LoadIncrement(tableDef cloudstorage.TableDefinition, filePath string) error

func (*DatabricksConnector) LoadSnapshot ¶

func (dc *DatabricksConnector) LoadSnapshot(targetTable, filePath string) error

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL