data_source

package
v0.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 4, 2024 License: Apache-2.0 Imports: 23 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func GetDataObjectType 

func GetDataObjectType(name string) *ds.DataObjectType

func GetS3MetaData 

func GetS3MetaData() *ds.MetaData

Types

type AwsGlueRepository 

type AwsGlueRepository struct {
	// contains filtered or unexported fields
}

func NewAwsGlueRepository 

func NewAwsGlueRepository(configMap *config.ConfigMap) *AwsGlueRepository

func (*AwsGlueRepository) GetGlueClient 

func (repo *AwsGlueRepository) GetGlueClient(ctx context.Context, region *string) (*glue.Client, error)

func (*AwsGlueRepository) ListDatabases 

func (repo *AwsGlueRepository) ListDatabases(ctx context.Context, accountId string, region string) ([]string, error)

func (*AwsGlueRepository) ListTablesForDatabase 

func (repo *AwsGlueRepository) ListTablesForDatabase(ctx context.Context, accountId string, database string, region string) (map[string]string, error)

type AwsS3Repository 

type AwsS3Repository struct {
	// contains filtered or unexported fields
}

func NewAwsS3Repository 

func NewAwsS3Repository(configMap *config.ConfigMap) *AwsS3Repository

func (*AwsS3Repository) GetFile 

func (repo *AwsS3Repository) GetFile(ctx context.Context, bucket, key string, region string) (io.ReadCloser, error)

func (*AwsS3Repository) GetS3Client 

func (repo *AwsS3Repository) GetS3Client(ctx context.Context, region *string) (*s3.Client, error)

func (*AwsS3Repository) ListBuckets 

func (repo *AwsS3Repository) ListBuckets(ctx context.Context) ([]model.AwsS3Entity, error)

func (*AwsS3Repository) ListFiles 

func (repo *AwsS3Repository) ListFiles(ctx context.Context, bucket string, prefix *string) ([]model.AwsS3Entity, error)

type DataSourceSyncer 

type DataSourceSyncer struct {
	// contains filtered or unexported fields
}

func NewDataSourceSyncer 

func NewDataSourceSyncer() *DataSourceSyncer

func (*DataSourceSyncer) FetchGlueDataObjects 

func (s *DataSourceSyncer) FetchGlueDataObjects(ctx context.Context, dataSourceHandler wrappers.DataSourceObjectHandler, region string) error

func (*DataSourceSyncer) FetchS3DataObjects 

func (s *DataSourceSyncer) FetchS3DataObjects(ctx context.Context, dataSourceHandler wrappers.DataSourceObjectHandler) error

func (*DataSourceSyncer) GetAvailableObjects 

func (s *DataSourceSyncer) GetAvailableObjects(ctx context.Context, cfg *config.ConfigMap) (map[string]interface{}, error)

GetAvailableObjects is used by the data usage component to fetch all available data objects in a map structure for easy lookup of what is available

func (*DataSourceSyncer) GetDataSourceMetaData 

func (s *DataSourceSyncer) GetDataSourceMetaData(ctx context.Context, configParams *config.ConfigMap) (*ds.MetaData, error)

func (*DataSourceSyncer) SyncDataSource 

func (s *DataSourceSyncer) SyncDataSource(ctx context.Context, dataSourceHandler wrappers.DataSourceObjectHandler, cfg *ds.DataSourceSyncConfig) error

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.