Documentation
¶
Index ¶
- Variables
- func Copy(ctx context.Context, fromTable *Table, partitions []*Partition, ...) error
- func CopyDB(ctx context.Context, fromClient, toClient Client, ...) error
- func CopyOrMerge(ctx context.Context, fromClient, toClient Client, ...) error
- func CopyOrMergeAll(ctx context.Context, fromClient, toClient Client, ...) error
- func CopyOrMergeFromValues(ctx context.Context, fromClient Client, fTable *Table, toClient Client, ...) error
- func CopyOrMergeToSymlink(ctx context.Context, fromClient, toClient Client, ...) error
- func CopyPartition(ctx context.Context, fromClient ReadClient, toClient Client, ...) error
- func Diff(iterA, iterB Collection) (catalog.Differences, error)
- func DiffIterable(iterA, iterB Collection, callbackFn DiffCallbackFn) error
- func ExtractRepoAndBranch(metastoreLocationURI string) (string, string, error)
- func GetSymlinkLocation(location, locationPrefix string) (string, error)
- func HandleDBFSLocation(ctx context.Context, location string, dbfsLocation string) string
- func ImportAll(ctx context.Context, fromClient, toClient Client, ...) error
- func Merge(ctx context.Context, table *Table, partitionIter Collection, ...) error
- func ReplaceBranchName(location, branch string) (string, error)
- func ReplaceExternalToLakeFSImported(location, repo, branch string) (string, error)
- type Client
- type Collection
- type ColumnCollection
- func (c *ColumnCollection) CompareWith(i int, v interface{}, j int) CompareResult
- func (c *ColumnCollection) Len() int
- func (c *ColumnCollection) Less(i, j int) bool
- func (c *ColumnCollection) Name(i int) string
- func (c *ColumnCollection) Swap(i, j int)
- func (c *ColumnCollection) Value(i int) interface{}
- type CompareResult
- type Database
- type DiffCallbackFn
- type FieldSchema
- type MetaDiff
- type Order
- type Partition
- type PartitionCollection
- func (p *PartitionCollection) CompareWith(i int, v interface{}, j int) CompareResult
- func (p *PartitionCollection) Len() int
- func (p *PartitionCollection) Less(i, j int) bool
- func (p *PartitionCollection) Name(i int) string
- func (p *PartitionCollection) Swap(i, j int)
- func (p *PartitionCollection) Value(i int) interface{}
- type ReadClient
- type SerDeInfo
- type SkewedInfo
- type StorageDescriptor
- type Table
- type WriteClient
Constants ¶
This section is empty.
Variables ¶
View Source
var ErrInvalidLocation = errors.New("got empty schema or host while parsing location url, location should be schema://host/path")
Functions ¶
func CopyOrMerge ¶ added in v0.40.3
func CopyOrMergeAll ¶ added in v0.40.3
func CopyOrMergeFromValues ¶ added in v0.40.3
func CopyOrMergeToSymlink ¶ added in v0.40.3
func CopyPartition ¶ added in v0.40.3
func Diff ¶
func Diff(iterA, iterB Collection) (catalog.Differences, error)
func DiffIterable ¶
func DiffIterable(iterA, iterB Collection, callbackFn DiffCallbackFn) error
func ExtractRepoAndBranch ¶ added in v0.61.0
func GetSymlinkLocation ¶
func HandleDBFSLocation ¶ added in v0.45.1
HandleDBFSLocation translates Data Bricks File system path to the S3 path using the dbfsLocation
func ReplaceBranchName ¶
func ReplaceExternalToLakeFSImported ¶ added in v0.45.1
Types ¶
type Client ¶ added in v0.40.3
type Client interface {
ReadClient
WriteClient
}
type Collection ¶
type ColumnCollection ¶ added in v0.40.3
type ColumnCollection struct {
// contains filtered or unexported fields
}
func NewColumnCollection ¶ added in v0.40.3
func NewColumnCollection(columns []*FieldSchema) *ColumnCollection
func (*ColumnCollection) CompareWith ¶ added in v0.40.3
func (c *ColumnCollection) CompareWith(i int, v interface{}, j int) CompareResult
func (*ColumnCollection) Len ¶ added in v0.40.3
func (c *ColumnCollection) Len() int
func (*ColumnCollection) Less ¶ added in v0.40.3
func (c *ColumnCollection) Less(i, j int) bool
func (*ColumnCollection) Name ¶ added in v0.40.3
func (c *ColumnCollection) Name(i int) string
func (*ColumnCollection) Swap ¶ added in v0.40.3
func (c *ColumnCollection) Swap(i, j int)
func (*ColumnCollection) Value ¶ added in v0.40.3
func (c *ColumnCollection) Value(i int) interface{}
type CompareResult ¶
type CompareResult int
const ( ItemLess CompareResult = iota ItemSameKey ItemSame ItemGreater )
type DiffCallbackFn ¶
type DiffCallbackFn func(difference catalog.DifferenceType, iter interface{}, name string) error
type FieldSchema ¶ added in v0.40.3
type MetaDiff ¶
type MetaDiff struct {
PartitionDiff catalog.Differences
ColumnsDiff catalog.Differences
}
type Partition ¶ added in v0.40.3
type PartitionCollection ¶ added in v0.40.3
type PartitionCollection struct {
// contains filtered or unexported fields
}
func NewPartitionCollection ¶ added in v0.40.3
func NewPartitionCollection(partitions []*Partition) *PartitionCollection
func (*PartitionCollection) CompareWith ¶ added in v0.40.3
func (p *PartitionCollection) CompareWith(i int, v interface{}, j int) CompareResult
func (*PartitionCollection) Len ¶ added in v0.40.3
func (p *PartitionCollection) Len() int
func (*PartitionCollection) Less ¶ added in v0.40.3
func (p *PartitionCollection) Less(i, j int) bool
func (*PartitionCollection) Name ¶ added in v0.40.3
func (p *PartitionCollection) Name(i int) string
func (*PartitionCollection) Swap ¶ added in v0.40.3
func (p *PartitionCollection) Swap(i, j int)
func (*PartitionCollection) Value ¶ added in v0.40.3
func (p *PartitionCollection) Value(i int) interface{}
type ReadClient ¶ added in v0.40.3
type ReadClient interface {
GetTable(ctx context.Context, dbName string, tableName string) (r *Table, err error)
HasTable(ctx context.Context, dbName string, tableName string) (hasTable bool, err error)
GetPartitions(ctx context.Context, dbName string, tableName string) (r []*Partition, err error)
GetPartition(ctx context.Context, dbName string, tableName string, values []string) (r *Partition, err error)
GetDatabase(ctx context.Context, name string) (r *Database, err error)
GetDatabases(ctx context.Context, pattern string) (databases []*Database, err error)
GetTables(ctx context.Context, dbName string, pattern string) (tables []*Table, err error)
}
type SkewedInfo ¶ added in v0.40.3
type StorageDescriptor ¶ added in v0.40.3
type StorageDescriptor struct {
Cols []*FieldSchema
Location string
InputFormat string
OutputFormat string
Compressed bool
NumBuckets int
SerdeInfo *SerDeInfo
BucketCols []string
SortCols []*Order
Parameters map[string]string
SkewedInfo *SkewedInfo
StoredAsSubDirectories *bool
AWSSchemaReference interface{}
}
type Table ¶ added in v0.40.3
type Table struct {
TableName string
DBName string
Owner string
CreateTime int64
LastAccessTime int64
Retention int
Sd *StorageDescriptor
PartitionKeys []*FieldSchema
Parameters map[string]string
ViewOriginalText string
ViewExpandedText string
TableType string
Temporary bool
RewriteEnabled *bool
AWSCreatedBy *string
AWSDescription *string
AWSIsRegisteredWithLakeFormation *bool
AWSLastAnalyzedTime *time.Time
AWSTargetTable interface{}
AWSUpdateTime *time.Time
Privileges interface{}
}
type WriteClient ¶ added in v0.40.3
type WriteClient interface {
CreateTable(ctx context.Context, tbl *Table) error
AlterTable(ctx context.Context, dbName string, tableName string, newTable *Table) error
AddPartitions(ctx context.Context, tableName string, dbName string, newParts []*Partition) error
AlterPartitions(ctx context.Context, dbName string, tableName string, newPartitions []*Partition) error
AlterPartition(ctx context.Context, dbName string, tableName string, partition *Partition) error
AddPartition(ctx context.Context, tableName string, dbName string, newPartition *Partition) error
DropPartition(ctx context.Context, dbName string, tableName string, values []string) error
CreateDatabase(ctx context.Context, database *Database) error
NormalizeDBName(name string) string // NormalizeDBName changes the db name to be a valid name for the client
GetDBLocation(dbName string) string // getDBLocation returns the expected locationURI of the database
}
Source Files
¶
Click to show internal directories.
Click to hide internal directories.