data_source

Jun 7, 2024

Changes in this version

+ func ClearMetadata()

type DataSourceSyncer

+ func (s *DataSourceSyncer) GetAvailableObjectTypes(ctx context.Context, cfg *config.ConfigMap) (*trie.Trie[string], error)

+ type MetadataProvider interface

+ AccessProviderTypes func() []*ds.AccessProviderType

+ DataObjectTypes func() []*ds.DataObjectType

+ UsageMetaInfo func() *ds.UsageMetaInput

+ type S3FileMetadataProvider struct

+ func (p S3FileMetadataProvider) AccessProviderTypes() []*ds.AccessProviderType

+ func (p S3FileMetadataProvider) DataObjectTypes() []*ds.DataObjectType

+ func (p S3FileMetadataProvider) UsageMetaInfo() *ds.UsageMetaInput

+ type S3GlueMetadataProvider struct

+ func (p S3GlueMetadataProvider) AccessProviderTypes() []*ds.AccessProviderType

+ func (p S3GlueMetadataProvider) DataObjectTypes() []*ds.DataObjectType

+ func (p S3GlueMetadataProvider) UsageMetaInfo() *ds.UsageMetaInput

Jun 4, 2024

Changes in this version

+ func GetDataObjectType(name string) *ds.DataObjectType

+ func GetS3MetaData() *ds.MetaData

+ type AwsGlueRepository struct

+ func NewAwsGlueRepository(configMap *config.ConfigMap) *AwsGlueRepository

+ func (repo *AwsGlueRepository) GetGlueClient(ctx context.Context, region *string) (*glue.Client, error)

+ func (repo *AwsGlueRepository) ListDatabases(ctx context.Context, accountId string, region string) ([]string, error)

+ func (repo *AwsGlueRepository) ListTablesForDatabase(ctx context.Context, accountId string, database string, region string) (map[string]string, error)

+ type AwsS3Repository struct

+ func NewAwsS3Repository(configMap *config.ConfigMap) *AwsS3Repository

+ func (repo *AwsS3Repository) GetFile(ctx context.Context, bucket, key string, region string) (io.ReadCloser, error)

+ func (repo *AwsS3Repository) GetS3Client(ctx context.Context, region *string) (*s3.Client, error)

+ func (repo *AwsS3Repository) ListBuckets(ctx context.Context) ([]model.AwsS3Entity, error)

+ func (repo *AwsS3Repository) ListFiles(ctx context.Context, bucket string, prefix *string) ([]model.AwsS3Entity, error)

+ type DataSourceSyncer struct

+ func NewDataSourceSyncer() *DataSourceSyncer

+ func (s *DataSourceSyncer) FetchGlueDataObjects(ctx context.Context, dataSourceHandler wrappers.DataSourceObjectHandler, ...) error

+ func (s *DataSourceSyncer) FetchS3DataObjects(ctx context.Context, dataSourceHandler wrappers.DataSourceObjectHandler) error

+ func (s *DataSourceSyncer) GetAvailableObjects(ctx context.Context, cfg *config.ConfigMap) (map[string]interface{}, error)

+ func (s *DataSourceSyncer) GetDataSourceMetaData(ctx context.Context, configParams *config.ConfigMap) (*ds.MetaData, error)

+ func (s *DataSourceSyncer) SyncDataSource(ctx context.Context, dataSourceHandler wrappers.DataSourceObjectHandler, ...) error