bigquery

package

v0.2.0 Latest Latest Go to latest Published: Mar 31, 2025 License: Apache-2.0 Imports: 26 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/getsynq/dwhsupport

Links

Open Source Insights

Documentation ¶

Index ¶

Variables
func QueryAndProcessMany[T any](ctx context.Context, conn Executor, sql string, ...) error
func QueryMany[T any](ctx context.Context, conn Executor, sql string, ...) ([]*T, error)
type BigQueryScrapper
- func NewBigQueryScrapper(ctx context.Context, conf *BigQueryScrapperConf) (*BigQueryScrapper, error)
type BigQueryScrapperConf
type Executor
type ShardedTable

Constants ¶

This section is empty.

Variables ¶

View Source

var BaseExpectedPermissions = []string{
	"bigquery.datasets.get",
	"bigquery.datasets.getIamPolicy",
	"bigquery.jobs.create",
	"bigquery.jobs.get",
	"bigquery.jobs.list",
	"bigquery.jobs.listAll",
	"bigquery.routines.get",
	"bigquery.routines.list",
	"bigquery.tables.get",
	"bigquery.tables.getData",
	"bigquery.tables.list",
	"resourcemanager.projects.get",
}

Functions ¶

func QueryAndProcessMany ¶

func QueryAndProcessMany[T any](ctx context.Context, conn Executor, sql string, handler func(ctx context.Context, rows []*T) error, opts ...dwhexec.QueryManyOpt[T]) error

func QueryMany ¶

func QueryMany[T any](ctx context.Context, conn Executor, sql string, opts ...dwhexec.QueryManyOpt[T]) ([]*T, error)

Types ¶

type BigQueryScrapper ¶

type BigQueryScrapper struct {
	// contains filtered or unexported fields
}

func NewBigQueryScrapper ¶

func NewBigQueryScrapper(ctx context.Context, conf *BigQueryScrapperConf) (*BigQueryScrapper, error)

func (*BigQueryScrapper) Close ¶

func (e *BigQueryScrapper) Close() error

func (*BigQueryScrapper) DialectType ¶ added in v0.1.8

func (e *BigQueryScrapper) DialectType() string

func (*BigQueryScrapper) Executor ¶

func (e *BigQueryScrapper) Executor() *dwhexecbigquery.BigQueryExecutor

func (*BigQueryScrapper) IsPermissionError ¶ added in v0.1.1

func (e *BigQueryScrapper) IsPermissionError(err error) bool

func (*BigQueryScrapper) QueryCatalog ¶

func (e *BigQueryScrapper) QueryCatalog(ctx context.Context) ([]*scrapper.CatalogColumnRow, error)

func (*BigQueryScrapper) QueryCustomMetrics ¶

func (e *BigQueryScrapper) QueryCustomMetrics(ctx context.Context, sql string, args ...any) ([]*scrapper.CustomMetricsRow, error)

func (*BigQueryScrapper) QueryDatabases ¶

func (e *BigQueryScrapper) QueryDatabases(ctx context.Context) ([]*scrapper.DatabaseRow, error)

func (*BigQueryScrapper) QuerySegments ¶

func (e *BigQueryScrapper) QuerySegments(ctx context.Context, sql string, args ...any) ([]*scrapper.SegmentRow, error)

func (*BigQueryScrapper) QuerySqlDefinitions ¶

func (e *BigQueryScrapper) QuerySqlDefinitions(ctx context.Context) ([]*scrapper.SqlDefinitionRow, error)

func (*BigQueryScrapper) QueryTableMetrics ¶

func (e *BigQueryScrapper) QueryTableMetrics(ctx context.Context, lastMetricsFetchTime time.Time) ([]*scrapper.TableMetricsRow, error)

func (*BigQueryScrapper) QueryTables ¶

func (e *BigQueryScrapper) QueryTables(ctx context.Context) ([]*scrapper.TableRow, error)

func (*BigQueryScrapper) SqlDialect ¶ added in v0.1.8

func (e *BigQueryScrapper) SqlDialect() sqldialect.Dialect

func (*BigQueryScrapper) ValidateConfiguration ¶

func (e *BigQueryScrapper) ValidateConfiguration(ctx context.Context) ([]string, error)

type BigQueryScrapperConf ¶

type BigQueryScrapperConf struct {
	dwhexecbigquery.BigQueryConf
	Blocklist string
}

type Executor ¶

type Executor interface {
	// contains filtered or unexported methods
}

type ShardedTable ¶

type ShardedTable struct {
	TableName     string
	Shards        []time.Time
	LatestShardId string
}

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL