bigquery

package
v0.4.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 28, 2025 License: MIT Imports: 14 Imported by: 0

Documentation

Overview

+marmot:name=BigQuery +marmot:description=This plugin discovers datasets and tables from Google BigQuery projects. +marmot:status=experimental

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type BigQueryColumnFields

type BigQueryColumnFields struct {
	Name         string                   `json:"name" metadata:"name" description:"Column name"`
	Type         string                   `json:"type" metadata:"type" description:"Column data type"`
	Description  string                   `json:"description" metadata:"description" description:"Column description"`
	NestedFields []map[string]interface{} `json:"nested_fields" metadata:"nested_fields" description:"Nested fields for RECORD type columns"`
}

+marmot:metadata

type BigQueryDatasetFields

type BigQueryDatasetFields struct {
	ProjectID                  string            `json:"project_id" metadata:"project_id" description:"Google Cloud Project ID"`
	DatasetID                  string            `json:"dataset_id" metadata:"dataset_id" description:"Dataset ID"`
	Location                   string            `json:"location" metadata:"location" description:"Geographic location of the dataset"`
	CreationTime               string            `json:"creation_time" metadata:"creation_time" description:"Dataset creation timestamp"`
	LastModified               string            `json:"last_modified" metadata:"last_modified" description:"Last modification timestamp"`
	Description                string            `json:"description" metadata:"description" description:"Dataset description"`
	DefaultTableExpiration     string            `json:"default_table_expiration" metadata:"default_table_expiration" description:"Default table expiration duration"`
	DefaultPartitionExpiration string            `` /* 127-byte string literal not displayed */
	Labels                     map[string]string `json:"labels" metadata:"labels" description:"Dataset labels"`
	AccessEntriesCount         int               `json:"access_entries_count" metadata:"access_entries_count" description:"Number of access control entries"`
}

+marmot:metadata

type BigQueryExternalDataConfig

type BigQueryExternalDataConfig struct {
	SourceFormat string   `json:"source_format" metadata:"source_format" description:"Source data format (CSV, JSON, AVRO, etc.)"`
	SourceURIs   []string `json:"source_uris" metadata:"source_uris" description:"Source URIs for external data"`
}

+marmot:metadata

type BigQueryTableFields

type BigQueryTableFields struct {
	ProjectID              string                 `json:"project_id" metadata:"project_id" description:"Google Cloud Project ID"`
	DatasetID              string                 `json:"dataset_id" metadata:"dataset_id" description:"Dataset ID"`
	TableID                string                 `json:"table_id" metadata:"table_id" description:"Table ID"`
	TableType              string                 `json:"table_type" metadata:"table_type" description:"Table type (TABLE, VIEW, EXTERNAL)"`
	CreationTime           string                 `json:"creation_time" metadata:"creation_time" description:"Table creation timestamp"`
	LastModified           string                 `json:"last_modified" metadata:"last_modified" description:"Last modification timestamp"`
	Description            string                 `json:"description" metadata:"description" description:"Table description"`
	ExpirationTime         string                 `json:"expiration_time" metadata:"expiration_time" description:"Table expiration timestamp"`
	Labels                 map[string]string      `json:"labels" metadata:"labels" description:"Table labels"`
	NumRows                uint64                 `json:"num_rows" metadata:"num_rows" description:"Number of rows in the table"`
	NumBytes               int64                  `json:"num_bytes" metadata:"num_bytes" description:"Size of the table in bytes"`
	TimePartitioningType   string                 `json:"time_partitioning_type" metadata:"time_partitioning_type" description:"Time partitioning type"`
	TimePartitioningField  string                 `json:"time_partitioning_field" metadata:"time_partitioning_field" description:"Time partitioning field"`
	PartitionExpiration    string                 `json:"partition_expiration" metadata:"partition_expiration" description:"Partition expiration duration"`
	RangePartitioningField string                 `json:"range_partitioning_field" metadata:"range_partitioning_field" description:"Range partitioning field"`
	ClusteringFields       []string               `json:"clustering_fields" metadata:"clustering_fields" description:"Clustering fields"`
	ViewQuery              string                 `json:"view_query" metadata:"view_query" description:"SQL query for views"`
	ExternalDataConfig     map[string]interface{} `json:"external_data_config" metadata:"external_data_config" description:"External data configuration for external tables"`
}

+marmot:metadata

type Config

type Config struct {
	plugin.BaseConfig `json:",inline"`

	ProjectID             string `json:"project_id" description:"Google Cloud Project ID"`
	CredentialsPath       string `json:"credentials_path,omitempty" description:"Path to service account credentials JSON file"`
	CredentialsJSON       string `json:"credentials_json,omitempty" description:"Service account credentials JSON content" sensitive:"true"`
	UseDefaultCredentials bool   `json:"use_default_credentials" description:"Use default Google Cloud credentials" default:"false"`

	IncludeDatasets       bool           `json:"include_datasets" description:"Whether to discover datasets" default:"true"`
	IncludeTableStats     bool           `json:"include_table_stats" description:"Whether to include table statistics (row count, size)" default:"true"`
	IncludeViews          bool           `json:"include_views" description:"Whether to discover views" default:"true"`
	IncludeExternalTables bool           `json:"include_external_tables" description:"Whether to discover external tables" default:"true"`
	DatasetFilter         *plugin.Filter `json:"dataset_filter,omitempty" description:"Filter configuration for datasets"`
	TableFilter           *plugin.Filter `json:"table_filter,omitempty" description:"Filter configuration for tables"`
	ExcludeSystemDatasets bool           `json:"exclude_system_datasets" description:"Whether to exclude system datasets (_script, _analytics, etc.)" default:"true"`
	MaxConcurrentRequests int            `json:"max_concurrent_requests" description:"Maximum number of concurrent API requests" default:"10"`
}

+marmot:config

func (*Config) ApplyDefaults

func (c *Config) ApplyDefaults()

type Source

type Source struct {
	// contains filtered or unexported fields
}

func (*Source) Discover

func (s *Source) Discover(ctx context.Context, pluginConfig plugin.RawPluginConfig) (*plugin.DiscoveryResult, error)

func (*Source) Validate

func (s *Source) Validate(rawConfig plugin.RawPluginConfig) (plugin.RawPluginConfig, error)

type TableType

type TableType string
const (
	TableTypeTable    TableType = "TABLE"
	TableTypeView     TableType = "VIEW"
	TableTypeExternal TableType = "EXTERNAL"
)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL