Documentation
¶
Overview ¶
+marmot:name=BigQuery +marmot:description=This plugin discovers datasets and tables from Google BigQuery projects. +marmot:status=experimental
Index ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type BigQueryColumnFields ¶
type BigQueryColumnFields struct {
Name string `json:"name" metadata:"name" description:"Column name"`
Type string `json:"type" metadata:"type" description:"Column data type"`
Description string `json:"description" metadata:"description" description:"Column description"`
NestedFields []map[string]interface{} `json:"nested_fields" metadata:"nested_fields" description:"Nested fields for RECORD type columns"`
}
+marmot:metadata
type BigQueryDatasetFields ¶
type BigQueryDatasetFields struct {
ProjectID string `json:"project_id" metadata:"project_id" description:"Google Cloud Project ID"`
DatasetID string `json:"dataset_id" metadata:"dataset_id" description:"Dataset ID"`
Location string `json:"location" metadata:"location" description:"Geographic location of the dataset"`
CreationTime string `json:"creation_time" metadata:"creation_time" description:"Dataset creation timestamp"`
LastModified string `json:"last_modified" metadata:"last_modified" description:"Last modification timestamp"`
Description string `json:"description" metadata:"description" description:"Dataset description"`
DefaultTableExpiration string `json:"default_table_expiration" metadata:"default_table_expiration" description:"Default table expiration duration"`
DefaultPartitionExpiration string `` /* 127-byte string literal not displayed */
Labels map[string]string `json:"labels" metadata:"labels" description:"Dataset labels"`
AccessEntriesCount int `json:"access_entries_count" metadata:"access_entries_count" description:"Number of access control entries"`
}
+marmot:metadata
type BigQueryExternalDataConfig ¶
type BigQueryExternalDataConfig struct {
SourceFormat string `json:"source_format" metadata:"source_format" description:"Source data format (CSV, JSON, AVRO, etc.)"`
SourceURIs []string `json:"source_uris" metadata:"source_uris" description:"Source URIs for external data"`
}
+marmot:metadata
type BigQueryTableFields ¶
type BigQueryTableFields struct {
ProjectID string `json:"project_id" metadata:"project_id" description:"Google Cloud Project ID"`
DatasetID string `json:"dataset_id" metadata:"dataset_id" description:"Dataset ID"`
TableID string `json:"table_id" metadata:"table_id" description:"Table ID"`
TableType string `json:"table_type" metadata:"table_type" description:"Table type (TABLE, VIEW, EXTERNAL)"`
CreationTime string `json:"creation_time" metadata:"creation_time" description:"Table creation timestamp"`
LastModified string `json:"last_modified" metadata:"last_modified" description:"Last modification timestamp"`
Description string `json:"description" metadata:"description" description:"Table description"`
ExpirationTime string `json:"expiration_time" metadata:"expiration_time" description:"Table expiration timestamp"`
Labels map[string]string `json:"labels" metadata:"labels" description:"Table labels"`
NumRows uint64 `json:"num_rows" metadata:"num_rows" description:"Number of rows in the table"`
NumBytes int64 `json:"num_bytes" metadata:"num_bytes" description:"Size of the table in bytes"`
TimePartitioningType string `json:"time_partitioning_type" metadata:"time_partitioning_type" description:"Time partitioning type"`
TimePartitioningField string `json:"time_partitioning_field" metadata:"time_partitioning_field" description:"Time partitioning field"`
PartitionExpiration string `json:"partition_expiration" metadata:"partition_expiration" description:"Partition expiration duration"`
RangePartitioningField string `json:"range_partitioning_field" metadata:"range_partitioning_field" description:"Range partitioning field"`
ClusteringFields []string `json:"clustering_fields" metadata:"clustering_fields" description:"Clustering fields"`
ViewQuery string `json:"view_query" metadata:"view_query" description:"SQL query for views"`
ExternalDataConfig map[string]interface{} `json:"external_data_config" metadata:"external_data_config" description:"External data configuration for external tables"`
}
+marmot:metadata
type Config ¶
type Config struct {
plugin.BaseConfig `json:",inline"`
ProjectID string `json:"project_id" description:"Google Cloud Project ID"`
CredentialsPath string `json:"credentials_path,omitempty" description:"Path to service account credentials JSON file"`
CredentialsJSON string `json:"credentials_json,omitempty" description:"Service account credentials JSON content" sensitive:"true"`
UseDefaultCredentials bool `json:"use_default_credentials" description:"Use default Google Cloud credentials" default:"false"`
IncludeDatasets bool `json:"include_datasets" description:"Whether to discover datasets" default:"true"`
IncludeTableStats bool `json:"include_table_stats" description:"Whether to include table statistics (row count, size)" default:"true"`
IncludeViews bool `json:"include_views" description:"Whether to discover views" default:"true"`
IncludeExternalTables bool `json:"include_external_tables" description:"Whether to discover external tables" default:"true"`
DatasetFilter *plugin.Filter `json:"dataset_filter,omitempty" description:"Filter configuration for datasets"`
TableFilter *plugin.Filter `json:"table_filter,omitempty" description:"Filter configuration for tables"`
ExcludeSystemDatasets bool `json:"exclude_system_datasets" description:"Whether to exclude system datasets (_script, _analytics, etc.)" default:"true"`
MaxConcurrentRequests int `json:"max_concurrent_requests" description:"Maximum number of concurrent API requests" default:"10"`
}
+marmot:config
func (*Config) ApplyDefaults ¶
func (c *Config) ApplyDefaults()
type Source ¶
type Source struct {
// contains filtered or unexported fields
}
func (*Source) Discover ¶
func (s *Source) Discover(ctx context.Context, pluginConfig plugin.RawPluginConfig) (*plugin.DiscoveryResult, error)
func (*Source) Validate ¶
func (s *Source) Validate(rawConfig plugin.RawPluginConfig) (plugin.RawPluginConfig, error)
Click to show internal directories.
Click to hide internal directories.