bigquery

package
v0.11.196 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 29, 2025 License: Apache-2.0 Imports: 23 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func IsSameClustering added in v0.11.129 

func IsSameClustering(meta *bigquery.TableMetadata, asset *pipeline.Asset) bool

func IsSamePartitioning added in v0.11.129 

func IsSamePartitioning(meta *bigquery.TableMetadata, asset *pipeline.Asset) bool

func NewMaterializer added in v0.5.0 

func NewMaterializer(fullRefresh bool) *pipeline.Materializer

Types

type AcceptedValuesCheck 

type AcceptedValuesCheck struct {
	// contains filtered or unexported fields
}

func (*AcceptedValuesCheck) Check 

func (c *AcceptedValuesCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error

type BasicOperator 

type BasicOperator struct {
	// contains filtered or unexported fields
}

func NewBasicOperator 

func NewBasicOperator(conn connectionFetcher, extractor queryExtractor, materializer materializer) *BasicOperator

func (BasicOperator) Run 

func (o BasicOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error

func (BasicOperator) RunTask 

func (o BasicOperator) RunTask(ctx context.Context, p *pipeline.Pipeline, t *pipeline.Asset) error

type Client 

type Client struct {
	// contains filtered or unexported fields
}

func NewDB 

func NewDB(c *Config) (*Client, error)

func (*Client) BuildTableExistsQuery added in v0.11.172 

func (d *Client) BuildTableExistsQuery(tableName string) (string, error)

func (*Client) CreateDataSetIfNotExist added in v0.11.129 

func (d *Client) CreateDataSetIfNotExist(asset *pipeline.Asset, ctx context.Context) error

func (*Client) DropTableOnMismatch added in v0.11.132 

func (d *Client) DropTableOnMismatch(ctx context.Context, tableName string, asset *pipeline.Asset) error

func (*Client) GetIngestrURI added in v0.7.0 

func (d *Client) GetIngestrURI() (string, error)

func (*Client) IsMaterializationTypeMismatch added in v0.11.132 

func (d *Client) IsMaterializationTypeMismatch(ctx context.Context, meta *bigquery.TableMetadata, asset *pipeline.Asset) bool

func (*Client) IsPartitioningOrClusteringMismatch added in v0.11.132 

func (d *Client) IsPartitioningOrClusteringMismatch(ctx context.Context, meta *bigquery.TableMetadata, asset *pipeline.Asset) bool

func (*Client) IsValid 

func (d *Client) IsValid(ctx context.Context, query *query.Query) (bool, error)

func (*Client) Ping added in v0.11.106 

func (d *Client) Ping(ctx context.Context) error

Test runs a simple query (SELECT 1) to validate the connection.

func (*Client) RunQueryWithoutResult 

func (d *Client) RunQueryWithoutResult(ctx context.Context, query *query.Query) error

func (*Client) Select 

func (d *Client) Select(ctx context.Context, query *query.Query) ([][]interface{}, error)

func (*Client) SelectWithSchema added in v0.11.53 

func (d *Client) SelectWithSchema(ctx context.Context, queryObj *query.Query) (*query.QueryResult, error)

func (*Client) UpdateTableMetadataIfNotExist added in v0.9.0 

func (d *Client) UpdateTableMetadataIfNotExist(ctx context.Context, asset *pipeline.Asset) error

type ColumnCheckOperator 

type ColumnCheckOperator struct {
	// contains filtered or unexported fields
}

func NewColumnCheckOperator 

func NewColumnCheckOperator(manager connectionFetcher) (*ColumnCheckOperator, error)

func (ColumnCheckOperator) Run 

func (o ColumnCheckOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error

type Config 

type Config struct {
	ProjectID           string `envconfig:"BIGQUERY_PROJECT"`
	CredentialsFilePath string `envconfig:"BIGQUERY_CREDENTIALS_FILE"`
	CredentialsJSON     string
	Credentials         *google.Credentials
	Location            string `envconfig:"BIGQUERY_LOCATION"`
}

func (Config) GetConnectionURI added in v0.6.13 

func (c Config) GetConnectionURI() (string, error)

func (Config) GetIngestrURI added in v0.7.0 

func (c Config) GetIngestrURI() (string, error)

func (Config) IsValid 

func (c Config) IsValid() bool

type DB 

type DB interface {
	Querier
	Selector
	MetadataUpdater
	TableManager
}

type MetadataPushOperator added in v0.9.0 

type MetadataPushOperator struct {
	// contains filtered or unexported fields
}

func NewMetadataPushOperator added in v0.9.0 

func NewMetadataPushOperator(conn connectionFetcher) *MetadataPushOperator

func (*MetadataPushOperator) Run added in v0.9.0 

func (o *MetadataPushOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error

type MetadataUpdater added in v0.9.0 

type MetadataUpdater interface {
	UpdateTableMetadataIfNotExist(ctx context.Context, asset *pipeline.Asset) error
}

type NoMetadataUpdatedError added in v0.9.0 

type NoMetadataUpdatedError struct{}

func (NoMetadataUpdatedError) Error added in v0.9.0 

func (m NoMetadataUpdatedError) Error() string

type PatternCheck added in v0.10.1 

type PatternCheck struct {
	// contains filtered or unexported fields
}

func (*PatternCheck) Check added in v0.10.1 

func (c *PatternCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error

type Querier 

type Querier interface {
	RunQueryWithoutResult(ctx context.Context, query *query.Query) error
	Ping(ctx context.Context) error
}

type QuerySensor added in v0.11.171 

type QuerySensor struct {
	// contains filtered or unexported fields
}

func NewQuerySensor added in v0.11.171 

func NewQuerySensor(conn connectionFetcher, extractor query.QueryExtractor, sensorMode string) *QuerySensor

func (*QuerySensor) Run added in v0.11.171 

func (o *QuerySensor) Run(ctx context.Context, ti scheduler.TaskInstance) error

func (*QuerySensor) RunTask added in v0.11.171 

func (o *QuerySensor) RunTask(ctx context.Context, p *pipeline.Pipeline, t *pipeline.Asset) error

type Selector 

type Selector interface {
	Select(ctx context.Context, query *query.Query) ([][]interface{}, error)
	SelectWithSchema(ctx context.Context, queryObj *query.Query) (*query.QueryResult, error)
}

type TableManager added in v0.11.129 

type TableManager interface {
	IsPartitioningOrClusteringMismatch(ctx context.Context, meta *bigquery.TableMetadata, asset *pipeline.Asset) bool
	CreateDataSetIfNotExist(asset *pipeline.Asset, ctx context.Context) error
	IsMaterializationTypeMismatch(ctx context.Context, meta *bigquery.TableMetadata, asset *pipeline.Asset) bool
	DropTableOnMismatch(ctx context.Context, tableName string, asset *pipeline.Asset) error
	BuildTableExistsQuery(tableName string) (string, error)
}

type TableSensor added in v0.11.172 

type TableSensor struct {
	// contains filtered or unexported fields
}

func NewTableSensor added in v0.11.172 

func NewTableSensor(conn connectionFetcher, sensorMode string) *TableSensor

func (*TableSensor) Run added in v0.11.172 

func (ts *TableSensor) Run(ctx context.Context, ti scheduler.TaskInstance) error

func (*TableSensor) RunTask added in v0.11.172 

func (ts *TableSensor) RunTask(ctx context.Context, p *pipeline.Pipeline, t *pipeline.Asset) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.