bigquery

package
v0.11.372 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 28, 2025 License: Apache-2.0 Imports: 34 Imported by: 0

Documentation

Index

Constants

View Source 
const CharacterLimit = 10000

Variables

This section is empty.

Functions

func CheckADCCredentialsForPipeline added in v0.11.367 

func CheckADCCredentialsForPipeline(ctx context.Context, p *pipeline.Pipeline, connGetter config.ConnectionGetter) error

CheckADCCredentialsForPipeline checks ADC credentials for all BigQuery connections used in the pipeline before execution starts. This ensures credentials are available before any tasks begin running, avoiding prompts during parallel execution.

func IsSameClustering added in v0.11.129 

func IsSameClustering(meta *bigquery.TableMetadata, asset *pipeline.Asset) bool

func IsSamePartitioning added in v0.11.129 

func IsSamePartitioning(meta *bigquery.TableMetadata, asset *pipeline.Asset) bool

func NewMaterializer added in v0.5.0 

func NewMaterializer(fullRefresh bool) *pipeline.Materializer

Types

type ADCCredentialError added in v0.11.327 

type ADCCredentialError struct {
	ClientType  string // e.g., "BigQuery client" or "Data Transfer client"
	OriginalErr error
}

ADCCredentialError represents an error when Application Default Credentials cannot be found or are invalid.

func (*ADCCredentialError) Error added in v0.11.327 

func (e *ADCCredentialError) Error() string

func (*ADCCredentialError) Unwrap added in v0.11.327 

func (e *ADCCredentialError) Unwrap() error

type AcceptedValuesCheck 

type AcceptedValuesCheck struct {
	// contains filtered or unexported fields
}

func (*AcceptedValuesCheck) Check 

func (c *AcceptedValuesCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error

type BasicOperator 

type BasicOperator struct {
	// contains filtered or unexported fields
}

func NewBasicOperator 

func NewBasicOperator(conn config.ConnectionGetter, extractor query.QueryExtractor, materializer materializer, parser *sqlparser.SQLParser) *BasicOperator

func (BasicOperator) Run 

func (o BasicOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error

func (BasicOperator) RunTask 

func (o BasicOperator) RunTask(ctx context.Context, p *pipeline.Pipeline, t *pipeline.Asset) error

type Client 

type Client struct {
	// contains filtered or unexported fields
}

func NewDB 

func NewDB(c *Config) (*Client, error)

func (*Client) BuildTableExistsQuery added in v0.11.172 

func (d *Client) BuildTableExistsQuery(tableName string) (string, error)

func (*Client) CreateDataSetIfNotExist added in v0.11.129 

func (d *Client) CreateDataSetIfNotExist(asset *pipeline.Asset, ctx context.Context) error

func (*Client) DropTableOnMismatch added in v0.11.132 

func (d *Client) DropTableOnMismatch(ctx context.Context, tableName string, asset *pipeline.Asset) error

func (*Client) GetColumns added in v0.11.247 

func (d *Client) GetColumns(ctx context.Context, databaseName, tableName string) ([]*ansisql.DBColumn, error)

GetColumns retrieves column information for a specific table in a BigQuery dataset. It takes a context, dataset name, and table name as parameters and returns a slice of column information. The method handles errors appropriately and returns an error if the table doesn't exist.

func (*Client) GetDatabaseSummary added in v0.11.242 

func (d *Client) GetDatabaseSummary(ctx context.Context) (*ansisql.DBDatabase, error)

func (*Client) GetDatabases added in v0.11.247 

func (d *Client) GetDatabases(ctx context.Context) ([]string, error)

func (*Client) GetIngestrURI added in v0.7.0 

func (d *Client) GetIngestrURI() (string, error)

func (*Client) GetTableSummary added in v0.11.224 

func (d *Client) GetTableSummary(ctx context.Context, tableName string, schemaOnly bool) (*diff.TableSummaryResult, error)

func (*Client) GetTables added in v0.11.247 

func (d *Client) GetTables(ctx context.Context, databaseName string) ([]string, error)

GetTables retrieves all table names from a BigQuery dataset (database). It takes a context and dataset name as parameters and returns a slice of table names. The method handles errors appropriately and returns an empty slice if the dataset has no tables.

func (*Client) IsMaterializationTypeMismatch added in v0.11.132 

func (d *Client) IsMaterializationTypeMismatch(ctx context.Context, meta *bigquery.TableMetadata, asset *pipeline.Asset) bool

func (*Client) IsPartitioningOrClusteringMismatch added in v0.11.132 

func (d *Client) IsPartitioningOrClusteringMismatch(ctx context.Context, meta *bigquery.TableMetadata, asset *pipeline.Asset) bool

func (*Client) IsValid 

func (d *Client) IsValid(ctx context.Context, query *query.Query) (bool, error)

func (*Client) Location added in v0.11.280 

func (d *Client) Location() string

func (*Client) NewDataTransferClient added in v0.11.280 

func (d *Client) NewDataTransferClient(ctx context.Context) (*datatransfer.Client, error)

func (*Client) Ping added in v0.11.106 

func (d *Client) Ping(ctx context.Context) error

Test runs a simple query (SELECT 1) to validate the connection.

func (*Client) ProjectID added in v0.11.280 

func (d *Client) ProjectID() string

func (*Client) QueryDryRun added in v0.11.281 

func (d *Client) QueryDryRun(ctx context.Context, queryObj *query.Query) (*bigquery.QueryStatistics, error)

func (*Client) RunQueryWithoutResult 

func (d *Client) RunQueryWithoutResult(ctx context.Context, query *query.Query) error

func (*Client) Select 

func (d *Client) Select(ctx context.Context, query *query.Query) ([][]interface{}, error)

func (*Client) SelectWithSchema added in v0.11.53 

func (d *Client) SelectWithSchema(ctx context.Context, queryObj *query.Query) (*query.QueryResult, error)

func (*Client) UpdateTableMetadataIfNotExist added in v0.9.0 

func (d *Client) UpdateTableMetadataIfNotExist(ctx context.Context, asset *pipeline.Asset) error

func (*Client) UsesApplicationDefaultCredentials added in v0.11.367 

func (d *Client) UsesApplicationDefaultCredentials() bool

type ColumnCheckOperator 

type ColumnCheckOperator struct {
	// contains filtered or unexported fields
}

func NewColumnCheckOperator 

func NewColumnCheckOperator(manager config.ConnectionGetter) (*ColumnCheckOperator, error)

func (ColumnCheckOperator) Run 

func (o ColumnCheckOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error

type Config 

type Config struct {
	ProjectID           string `envconfig:"BIGQUERY_PROJECT"`
	CredentialsFilePath string `envconfig:"BIGQUERY_CREDENTIALS_FILE"`
	CredentialsJSON     string
	Credentials         *google.Credentials
	Location            string `envconfig:"BIGQUERY_LOCATION"`
	// Add support for Application Default Credentials
	UseApplicationDefaultCredentials bool `envconfig:"BIGQUERY_USE_ADC"`
}

func (Config) GetConnectionURI added in v0.6.13 

func (c Config) GetConnectionURI() (string, error)

func (Config) GetIngestrURI added in v0.7.0 

func (c Config) GetIngestrURI() (string, error)

func (Config) IsValid 

func (c Config) IsValid() bool

type DB 

type DB interface {
	Querier
	Selector
	MetadataUpdater
	TableManager
	UsesApplicationDefaultCredentials() bool
}

type DryRunner added in v0.11.281 

type DryRunner struct {
	ConnectionGetter connectionGetter
	QueryExtractor   queryExtractor
}

func (*DryRunner) DryRun added in v0.11.281 

func (r *DryRunner) DryRun(ctx context.Context, p pipeline.Pipeline, a pipeline.Asset, c *config.Config) (map[string]any, error)

type DryRunnerQuerier added in v0.11.281 

type DryRunnerQuerier interface {
	QueryDryRun(ctx context.Context, queryObj *query.Query) (*bigquery.QueryStatistics, error)
}

type MetadataPushOperator added in v0.9.0 

type MetadataPushOperator struct {
	// contains filtered or unexported fields
}

func NewMetadataPushOperator added in v0.9.0 

func NewMetadataPushOperator(conn config.ConnectionGetter) *MetadataPushOperator

func (*MetadataPushOperator) Run added in v0.9.0 

func (o *MetadataPushOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error

type MetadataUpdater added in v0.9.0 

type MetadataUpdater interface {
	UpdateTableMetadataIfNotExist(ctx context.Context, asset *pipeline.Asset) error
}

type NoMetadataUpdatedError added in v0.9.0 

type NoMetadataUpdatedError struct{}

func (NoMetadataUpdatedError) Error added in v0.9.0 

func (m NoMetadataUpdatedError) Error() string

type PatternCheck added in v0.10.1 

type PatternCheck struct {
	// contains filtered or unexported fields
}

func (*PatternCheck) Check added in v0.10.1 

func (c *PatternCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error

type Querier 

type Querier interface {
	RunQueryWithoutResult(ctx context.Context, query *query.Query) error
	Ping(ctx context.Context) error
}

type QuerySensor added in v0.11.171 

type QuerySensor struct {
	// contains filtered or unexported fields
}

func NewQuerySensor added in v0.11.171 

func NewQuerySensor(conn config.ConnectionGetter, extractor query.QueryExtractor, sensorMode string) *QuerySensor

func (*QuerySensor) Run added in v0.11.171 

func (o *QuerySensor) Run(ctx context.Context, ti scheduler.TaskInstance) error

func (*QuerySensor) RunTask added in v0.11.171 

func (o *QuerySensor) RunTask(ctx context.Context, p *pipeline.Pipeline, t *pipeline.Asset) error

type Selector 

type Selector interface {
	Select(ctx context.Context, query *query.Query) ([][]interface{}, error)
	SelectWithSchema(ctx context.Context, queryObj *query.Query) (*query.QueryResult, error)
}

type TableManager added in v0.11.129 

type TableManager interface {
	IsPartitioningOrClusteringMismatch(ctx context.Context, meta *bigquery.TableMetadata, asset *pipeline.Asset) bool
	CreateDataSetIfNotExist(asset *pipeline.Asset, ctx context.Context) error
	IsMaterializationTypeMismatch(ctx context.Context, meta *bigquery.TableMetadata, asset *pipeline.Asset) bool
	DropTableOnMismatch(ctx context.Context, tableName string, asset *pipeline.Asset) error
	BuildTableExistsQuery(tableName string) (string, error)
}

type TableSensor added in v0.11.172 

type TableSensor struct {
	// contains filtered or unexported fields
}

func NewTableSensor added in v0.11.172 

func NewTableSensor(conn config.ConnectionGetter, sensorMode string, extractor query.QueryExtractor) *TableSensor

func (*TableSensor) Run added in v0.11.172 

func (ts *TableSensor) Run(ctx context.Context, ti scheduler.TaskInstance) error

func (*TableSensor) RunTask added in v0.11.172 

func (ts *TableSensor) RunTask(ctx context.Context, p *pipeline.Pipeline, t *pipeline.Asset) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.