Documentation
¶
Overview ¶
Package scrapper is a generated GoMock package.
Index ¶
- Constants
- Variables
- func As[T any](s Scrapper) (T, bool)
- func AsBigRat(v Value) (*big.Rat, bool)
- func AsFloat64(v Value) (float64, bool)
- func AsGo(v Value) any
- func AsInt64(v Value) (int64, bool)
- func AsString(v Value) (string, bool)
- func AsTime(v Value) (time.Time, bool)
- func GetTableRowOption[T any](tableRow *TableRow, optionName string) T
- func IsSinglePrecisionFloat(dialect string, nativeType string) bool
- func IsValidString(s string) bool
- func NormalizeTime(t time.Time) time.Time
- func NormalizeTimeOfKind(t time.Time, kind ValueKind) time.Time
- func ParseTimestamp(s string) (time.Time, bool)
- func SanitizeString(s string) string
- func SanitizeStringPtr(s *string)
- func ScanAll[T any](ctx context.Context, rows *sqlx.Rows, source string) ([]*T, error)
- func SqlLiteral(d sqldialect.Dialect, kind ValueKind, v Value) (string, error)
- type Annotation
- type BigIntValue
- type Capabilities
- type CatalogColumnRow
- func (r CatalogColumnRow) GetComment() string
- func (r CatalogColumnRow) GetTableComment() string
- func (r *CatalogColumnRow) HasValidIdentity() bool
- func (r *CatalogColumnRow) ResolveNumeric(dialect string) *CatalogColumnRow
- func (r *CatalogColumnRow) Sanitize()
- func (r CatalogColumnRow) TableFqn() DwhFqn
- type ColumnValue
- type CustomMetricsRow
- type DatabaseRow
- type DoubleValue
- type DwhFqn
- type EstimateQueryCapability
- type HasTableFqn
- type IgnoredValue
- type InstanceHost
- type InstanceIdentity
- type InstanceIdentityQuerier
- type IntValue
- type JsonValue
- type MockScrapper
- func (m *MockScrapper) Capabilities() Capabilities
- func (m *MockScrapper) Close() error
- func (m *MockScrapper) DialectType() string
- func (m *MockScrapper) EXPECT() *MockScrapperMockRecorder
- func (m *MockScrapper) EstimateQuery(ctx context.Context, sql string) (*QueryEstimate, error)
- func (m *MockScrapper) IsPermissionError(err error) bool
- func (m *MockScrapper) QueryCatalog(ctx context.Context) ([]*CatalogColumnRow, error)
- func (m *MockScrapper) QueryCustomMetrics(ctx context.Context, sql string, args ...any) ([]*CustomMetricsRow, error)
- func (m *MockScrapper) QueryDatabases(ctx context.Context) ([]*DatabaseRow, error)
- func (m *MockScrapper) QueryPlatformUsers(ctx context.Context) (*PlatformUsers, error)
- func (m *MockScrapper) QuerySchemas(ctx context.Context) ([]*SchemaRow, error)
- func (m *MockScrapper) QuerySegments(ctx context.Context, sql string, args ...any) ([]*SegmentRow, error)
- func (m *MockScrapper) QueryShape(ctx context.Context, sql string) ([]*QueryShapeColumn, error)
- func (m *MockScrapper) QuerySqlDefinitions(ctx context.Context) ([]*SqlDefinitionRow, error)
- func (m *MockScrapper) QueryTableConstraints(ctx context.Context) ([]*TableConstraintRow, error)
- func (m *MockScrapper) QueryTableMetrics(ctx context.Context, lastMetricsFetchTime time.Time) ([]*TableMetricsRow, error)
- func (m *MockScrapper) QueryTables(ctx context.Context, opts ...QueryTablesOption) ([]*TableRow, error)
- func (m *MockScrapper) RunRawQuery(ctx context.Context, sql string) (RawQueryRowIterator, error)
- func (m *MockScrapper) SqlDialect() sqldialect.Dialect
- func (m *MockScrapper) ValidateConfiguration(ctx context.Context) ([]string, error)
- type MockScrapperCapabilitiesCall
- type MockScrapperCloseCall
- type MockScrapperDialectTypeCall
- type MockScrapperEstimateQueryCall
- func (c *MockScrapperEstimateQueryCall) Do(f func(context.Context, string) (*QueryEstimate, error)) *MockScrapperEstimateQueryCall
- func (c *MockScrapperEstimateQueryCall) DoAndReturn(f func(context.Context, string) (*QueryEstimate, error)) *MockScrapperEstimateQueryCall
- func (c *MockScrapperEstimateQueryCall) Return(arg0 *QueryEstimate, arg1 error) *MockScrapperEstimateQueryCall
- type MockScrapperIsPermissionErrorCall
- func (c *MockScrapperIsPermissionErrorCall) Do(f func(error) bool) *MockScrapperIsPermissionErrorCall
- func (c *MockScrapperIsPermissionErrorCall) DoAndReturn(f func(error) bool) *MockScrapperIsPermissionErrorCall
- func (c *MockScrapperIsPermissionErrorCall) Return(arg0 bool) *MockScrapperIsPermissionErrorCall
- type MockScrapperMockRecorder
- func (mr *MockScrapperMockRecorder) Capabilities() *MockScrapperCapabilitiesCall
- func (mr *MockScrapperMockRecorder) Close() *MockScrapperCloseCall
- func (mr *MockScrapperMockRecorder) DialectType() *MockScrapperDialectTypeCall
- func (mr *MockScrapperMockRecorder) EstimateQuery(ctx, sql any) *MockScrapperEstimateQueryCall
- func (mr *MockScrapperMockRecorder) IsPermissionError(err any) *MockScrapperIsPermissionErrorCall
- func (mr *MockScrapperMockRecorder) QueryCatalog(ctx any) *MockScrapperQueryCatalogCall
- func (mr *MockScrapperMockRecorder) QueryCustomMetrics(ctx, sql any, args ...any) *MockScrapperQueryCustomMetricsCall
- func (mr *MockScrapperMockRecorder) QueryDatabases(ctx any) *MockScrapperQueryDatabasesCall
- func (mr *MockScrapperMockRecorder) QueryPlatformUsers(ctx any) *MockScrapperQueryPlatformUsersCall
- func (mr *MockScrapperMockRecorder) QuerySchemas(ctx any) *MockScrapperQuerySchemasCall
- func (mr *MockScrapperMockRecorder) QuerySegments(ctx, sql any, args ...any) *MockScrapperQuerySegmentsCall
- func (mr *MockScrapperMockRecorder) QueryShape(ctx, sql any) *MockScrapperQueryShapeCall
- func (mr *MockScrapperMockRecorder) QuerySqlDefinitions(ctx any) *MockScrapperQuerySqlDefinitionsCall
- func (mr *MockScrapperMockRecorder) QueryTableConstraints(ctx any) *MockScrapperQueryTableConstraintsCall
- func (mr *MockScrapperMockRecorder) QueryTableMetrics(ctx, lastMetricsFetchTime any) *MockScrapperQueryTableMetricsCall
- func (mr *MockScrapperMockRecorder) QueryTables(ctx any, opts ...any) *MockScrapperQueryTablesCall
- func (mr *MockScrapperMockRecorder) RunRawQuery(ctx, sql any) *MockScrapperRunRawQueryCall
- func (mr *MockScrapperMockRecorder) SqlDialect() *MockScrapperSqlDialectCall
- func (mr *MockScrapperMockRecorder) ValidateConfiguration(ctx any) *MockScrapperValidateConfigurationCall
- type MockScrapperQueryCatalogCall
- func (c *MockScrapperQueryCatalogCall) Do(f func(context.Context) ([]*CatalogColumnRow, error)) *MockScrapperQueryCatalogCall
- func (c *MockScrapperQueryCatalogCall) DoAndReturn(f func(context.Context) ([]*CatalogColumnRow, error)) *MockScrapperQueryCatalogCall
- func (c *MockScrapperQueryCatalogCall) Return(arg0 []*CatalogColumnRow, arg1 error) *MockScrapperQueryCatalogCall
- type MockScrapperQueryCustomMetricsCall
- func (c *MockScrapperQueryCustomMetricsCall) Do(f func(context.Context, string, ...any) ([]*CustomMetricsRow, error)) *MockScrapperQueryCustomMetricsCall
- func (c *MockScrapperQueryCustomMetricsCall) DoAndReturn(f func(context.Context, string, ...any) ([]*CustomMetricsRow, error)) *MockScrapperQueryCustomMetricsCall
- func (c *MockScrapperQueryCustomMetricsCall) Return(arg0 []*CustomMetricsRow, arg1 error) *MockScrapperQueryCustomMetricsCall
- type MockScrapperQueryDatabasesCall
- func (c *MockScrapperQueryDatabasesCall) Do(f func(context.Context) ([]*DatabaseRow, error)) *MockScrapperQueryDatabasesCall
- func (c *MockScrapperQueryDatabasesCall) DoAndReturn(f func(context.Context) ([]*DatabaseRow, error)) *MockScrapperQueryDatabasesCall
- func (c *MockScrapperQueryDatabasesCall) Return(arg0 []*DatabaseRow, arg1 error) *MockScrapperQueryDatabasesCall
- type MockScrapperQueryPlatformUsersCall
- func (c *MockScrapperQueryPlatformUsersCall) Do(f func(context.Context) (*PlatformUsers, error)) *MockScrapperQueryPlatformUsersCall
- func (c *MockScrapperQueryPlatformUsersCall) DoAndReturn(f func(context.Context) (*PlatformUsers, error)) *MockScrapperQueryPlatformUsersCall
- func (c *MockScrapperQueryPlatformUsersCall) Return(arg0 *PlatformUsers, arg1 error) *MockScrapperQueryPlatformUsersCall
- type MockScrapperQuerySchemasCall
- func (c *MockScrapperQuerySchemasCall) Do(f func(context.Context) ([]*SchemaRow, error)) *MockScrapperQuerySchemasCall
- func (c *MockScrapperQuerySchemasCall) DoAndReturn(f func(context.Context) ([]*SchemaRow, error)) *MockScrapperQuerySchemasCall
- func (c *MockScrapperQuerySchemasCall) Return(arg0 []*SchemaRow, arg1 error) *MockScrapperQuerySchemasCall
- type MockScrapperQuerySegmentsCall
- func (c *MockScrapperQuerySegmentsCall) Do(f func(context.Context, string, ...any) ([]*SegmentRow, error)) *MockScrapperQuerySegmentsCall
- func (c *MockScrapperQuerySegmentsCall) DoAndReturn(f func(context.Context, string, ...any) ([]*SegmentRow, error)) *MockScrapperQuerySegmentsCall
- func (c *MockScrapperQuerySegmentsCall) Return(arg0 []*SegmentRow, arg1 error) *MockScrapperQuerySegmentsCall
- type MockScrapperQueryShapeCall
- func (c *MockScrapperQueryShapeCall) Do(f func(context.Context, string) ([]*QueryShapeColumn, error)) *MockScrapperQueryShapeCall
- func (c *MockScrapperQueryShapeCall) DoAndReturn(f func(context.Context, string) ([]*QueryShapeColumn, error)) *MockScrapperQueryShapeCall
- func (c *MockScrapperQueryShapeCall) Return(arg0 []*QueryShapeColumn, arg1 error) *MockScrapperQueryShapeCall
- type MockScrapperQuerySqlDefinitionsCall
- func (c *MockScrapperQuerySqlDefinitionsCall) Do(f func(context.Context) ([]*SqlDefinitionRow, error)) *MockScrapperQuerySqlDefinitionsCall
- func (c *MockScrapperQuerySqlDefinitionsCall) DoAndReturn(f func(context.Context) ([]*SqlDefinitionRow, error)) *MockScrapperQuerySqlDefinitionsCall
- func (c *MockScrapperQuerySqlDefinitionsCall) Return(arg0 []*SqlDefinitionRow, arg1 error) *MockScrapperQuerySqlDefinitionsCall
- type MockScrapperQueryTableConstraintsCall
- func (c *MockScrapperQueryTableConstraintsCall) Do(f func(context.Context) ([]*TableConstraintRow, error)) *MockScrapperQueryTableConstraintsCall
- func (c *MockScrapperQueryTableConstraintsCall) DoAndReturn(f func(context.Context) ([]*TableConstraintRow, error)) *MockScrapperQueryTableConstraintsCall
- func (c *MockScrapperQueryTableConstraintsCall) Return(arg0 []*TableConstraintRow, arg1 error) *MockScrapperQueryTableConstraintsCall
- type MockScrapperQueryTableMetricsCall
- func (c *MockScrapperQueryTableMetricsCall) Do(f func(context.Context, time.Time) ([]*TableMetricsRow, error)) *MockScrapperQueryTableMetricsCall
- func (c *MockScrapperQueryTableMetricsCall) DoAndReturn(f func(context.Context, time.Time) ([]*TableMetricsRow, error)) *MockScrapperQueryTableMetricsCall
- func (c *MockScrapperQueryTableMetricsCall) Return(arg0 []*TableMetricsRow, arg1 error) *MockScrapperQueryTableMetricsCall
- type MockScrapperQueryTablesCall
- func (c *MockScrapperQueryTablesCall) Do(f func(context.Context, ...QueryTablesOption) ([]*TableRow, error)) *MockScrapperQueryTablesCall
- func (c *MockScrapperQueryTablesCall) DoAndReturn(f func(context.Context, ...QueryTablesOption) ([]*TableRow, error)) *MockScrapperQueryTablesCall
- func (c *MockScrapperQueryTablesCall) Return(arg0 []*TableRow, arg1 error) *MockScrapperQueryTablesCall
- type MockScrapperRunRawQueryCall
- func (c *MockScrapperRunRawQueryCall) Do(f func(context.Context, string) (RawQueryRowIterator, error)) *MockScrapperRunRawQueryCall
- func (c *MockScrapperRunRawQueryCall) DoAndReturn(f func(context.Context, string) (RawQueryRowIterator, error)) *MockScrapperRunRawQueryCall
- func (c *MockScrapperRunRawQueryCall) Return(arg0 RawQueryRowIterator, arg1 error) *MockScrapperRunRawQueryCall
- type MockScrapperSqlDialectCall
- type MockScrapperValidateConfigurationCall
- func (c *MockScrapperValidateConfigurationCall) Do(f func(context.Context) ([]string, error)) *MockScrapperValidateConfigurationCall
- func (c *MockScrapperValidateConfigurationCall) DoAndReturn(f func(context.Context) ([]string, error)) *MockScrapperValidateConfigurationCall
- func (c *MockScrapperValidateConfigurationCall) Return(warnings []string, err error) *MockScrapperValidateConfigurationCall
- type NoEnoughPermissionsError
- type NumericCategory
- type PlatformUser
- type PlatformUserFact
- type PlatformUserListing
- func PlatformUserSourceError(source string, kind PlatformUserSourceKind, err error, ...) *PlatformUserListing
- func RefusedPlatformUserSource(source string, kind PlatformUserSourceKind, err error) *PlatformUserListing
- func UnavailablePlatformUserSource(source string, kind PlatformUserSourceKind, err error) *PlatformUserListing
- func (p *PlatformUserListing) Answered() bool
- func (p *PlatformUserListing) AssignRoles(rolesByLogin map[string][]string)
- func (p *PlatformUserListing) DependentSkipKind() PlatformUserSkipKind
- func (p *PlatformUserListing) Finish() *PlatformUserListing
- func (p *PlatformUserListing) IsSkipped(fact PlatformUserFact) bool
- func (p *PlatformUserListing) Sanitize()
- func (p *PlatformUserListing) Skip(fact PlatformUserFact, kind PlatformUserSkipKind, reason string)
- func (p *PlatformUserListing) SkippedFact(fact PlatformUserFact) (SkippedPlatformUserFact, bool)
- type PlatformUserSkipKind
- type PlatformUserSourceKind
- type PlatformUsers
- type PlatformUsersCapability
- type PlatformUsersCompleteness
- type QueryEstimate
- type QueryShapeColumn
- func (c *QueryShapeColumn) HasValidIdentity() bool
- func (c *QueryShapeColumn) NumericCategory() NumericCategory
- func (c *QueryShapeColumn) Sanitize()
- func (c *QueryShapeColumn) SetDecimalSize(precision, scale int64, ok bool) *QueryShapeColumn
- func (c *QueryShapeColumn) SetFloatLength(length int64, ok bool) *QueryShapeColumn
- type QueryTablesConfig
- type QueryTablesOption
- type RawQueryRowIterator
- type SchemaColumnField
- type SchemaRow
- type Scrapper
- type SegmentRow
- type SegmentValue
- type SkippedPlatformUserFact
- type SqlDefinitionRow
- type StringValue
- type TableChangeEvent
- type TableChangeHistoryProvider
- type TableConstraintRow
- type TableMetricsRow
- type TableRow
- type Tag
- type TimeValue
- type Unwrapper
- type Value
- type ValueKind
- type WithSetInstance
Constants ¶
const ( // InstanceIdentityAccountLocator is the identifier the warehouse assigned // the account itself (Snowflake CURRENT_ACCOUNT()). InstanceIdentityAccountLocator = "account_locator" // InstanceIdentityOrganizationName is the organization the account belongs // to (Snowflake CURRENT_ORGANIZATION_NAME()). InstanceIdentityOrganizationName = "organization_name" // InstanceIdentityAccountName is the account's name within its organization // (Snowflake CURRENT_ACCOUNT_NAME()). InstanceIdentityAccountName = "account_name" // InstanceIdentityRegion is the region identifier, which is NOT the same // string as the region fragment appearing in the account's hostname // (Snowflake CURRENT_REGION() returns AWS_US_EAST_2 for us-east-2.aws). InstanceIdentityRegion = "region" )
Identity function/field keys used in InstanceIdentity.Raw. Each names the warehouse construct that produced the value, so a caller can tell an account locator from an organization-qualified name without re-deriving either.
const ( // InstanceHostRegional is the account's region-qualified hostname, built // from the account locator (Snowflake SNOWFLAKE_DEPLOYMENT). InstanceHostRegional = "regional" // InstanceHostRegionless is the account's region-independent hostname, // built from the organization-qualified account name (Snowflake // SNOWFLAKE_DEPLOYMENT_REGIONLESS). InstanceHostRegionless = "regionless" )
Host kinds used in InstanceHost.Kind.
const ( ConstraintTypePrimaryKey = "PRIMARY KEY" ConstraintTypeUniqueIndex = "UNIQUE INDEX" ConstraintTypeForeignKey = "FOREIGN KEY" ConstraintTypeIndex = "INDEX" ConstraintTypeSortingKey = "SORTING KEY" ConstraintTypePartitionBy = "PARTITION BY" ConstraintTypeClusterBy = "CLUSTER BY" ConstraintTypeDistributionKey = "DISTRIBUTION KEY" ConstraintTypeProjection = "PROJECTION" ConstraintTypeCheck = "CHECK" )
Constraint type constants
const ( // Snowflake USERS.TYPE. A user created before TYPE existed has none. PlatformUserTypeSnowflakePerson = "PERSON" PlatformUserTypeSnowflakeService = "SERVICE" PlatformUserTypeSnowflakeLegacyService = "LEGACY_SERVICE" // Databricks SCIM: a workspace user, or a service principal whose Login is // its application id (what query history reports as the user). PlatformUserTypeDatabricksUser = "user" PlatformUserTypeDatabricksServicePrincipal = "service_principal" // BigQuery IAM member prefixes: an address under gserviceaccount.com is a // serviceAccount, a person a user. PlatformUserTypeBigQueryUser = "user" PlatformUserTypeBigQueryServiceAccount = "serviceAccount" )
Values of PlatformUser.Type on the platforms that state one. The type is stored as the platform states it, never derived: "looks like a service account" is the caller's suggestion, not a fact of the platform.
const PlatformUsersReconciledSource = "reconciled"
PlatformUsersReconciledSource is the Source of a listing Reconcile made.
Variables ¶
var ErrUnsupported = errors.New("unsupported")
Functions ¶
func As ¶ added in v0.13.3
As walks the decorator chain rooted at s (following Unwrap) and returns the first value that type-asserts to T. Returns the zero T and false if no value in the chain satisfies T. Safe to call with a nil Scrapper.
func AsBigRat ¶ added in v0.19.0
AsBigRat reads a value as an exact rational: an integer, a double, or text holding a decimal number. This is the accessor for an exact numeric column, whose fractional values arrive as StringValue.
func AsFloat64 ¶ added in v0.19.0
AsFloat64 reads a value as a float64: any number, or text holding one (an exact decimal arrives as StringValue). Precision beyond float64 is lost, so use AsBigRat when the digits matter.
func AsGo ¶ added in v0.19.0
AsGo unwraps a value to the plain Go value it holds: int64, float64, *big.Int, string, time.Time (UTC), or the JSON text of a JsonValue as a string. nil and IgnoredValue give nil.
func AsInt64 ¶ added in v0.19.0
AsInt64 reads a value as an int64: an integer, a double with no fractional part, or text holding an integer. It reports false when the value is not a whole number or does not fit.
func AsString ¶ added in v0.19.0
AsString renders a value as text: a StringValue or JsonValue verbatim, a number in its shortest exact form, a time as RFC 3339 in UTC with every fractional digit it carries. It reports false for nil and IgnoredValue, which mean the value was not read, so a caller can say so instead of writing "{}" or "<nil>".
func AsTime ¶ added in v0.19.0
AsTime reads a value as an instant in UTC: a TimeValue, or text in any form ParseTimestamp knows.
func GetTableRowOption ¶ added in v0.4.13
func IsSinglePrecisionFloat ¶ added in v0.21.0
IsSinglePrecisionFloat reports whether a column's native type, as a driver reports it, holds a single-precision (32-bit) float on dialect: REAL, FLOAT4, Float32, BINARY_FLOAT. It is false for every type NativeValueKind does not map to KindFloat, so Oracle's FLOAT, which is a NUMBER, is not one.
The same name is a different size on different engines. A bare FLOAT is single precision on MySQL, Databricks, DuckDB, ClickHouse and Athena (whose driver reports REAL as float), and double everywhere else. REAL is double on MySQL and on Snowflake, where every float is a double whatever it is called.
The name alone cannot tell on Db2, whose driver reports REAL and DOUBLE both as FLOAT: QueryShapeColumn.SetFloatLength decides there.
func IsValidString ¶ added in v0.13.2
IsValidString reports whether s contains no NUL bytes and is valid UTF-8. Used both as the fast-path check in SanitizeString and as the identity-field validity predicate by the rejecting scrapper decorator.
func NormalizeTime ¶ added in v0.19.0
NormalizeTime returns t as an instant in UTC, the form every TimeValue carries.
A driver that has no zone for a value (a zone-less TIMESTAMP or a DATE) either labels it UTC (most drivers) or labels it with the process's local zone (trino-go-client, the Athena driver, go-sql-driver/mysql with loc=Local). Converting the second kind to UTC would move its wall clock by the host's offset, so a value labelled with time.Local is taken to be that wall clock in UTC instead. No driver we use labels a value it does know the zone of with time.Local: they build the zone from the value (FixedZone or LoadLocation), which is never the time.Local pointer.
func NormalizeTimeOfKind ¶ added in v0.19.0
NormalizeTimeOfKind is NormalizeTime for a value whose column kind is known, which settles the question NormalizeTime has to guess at: a zone-less timestamp or date keeps its wall clock whatever zone the driver labelled it with, and an instant is converted to UTC. Any other kind falls back to NormalizeTime.
func ParseTimestamp ¶ added in v0.19.0
ParseTimestamp reads a timestamp or date a warehouse rendered as text and returns it as an instant in UTC. Text that carries a zone or offset is converted to UTC; text without one is a wall clock and is read as UTC, which is how a zone-less timestamp is decoded from a driver too (NormalizeTime).
It reports false for text that is not a timestamp in any form listed in timestampLayouts. It does not guess.
func SanitizeString ¶ added in v0.13.2
SanitizeString drops NUL bytes and invalid UTF-8 sequences. Returns the input unchanged (no allocation) when it contains neither — the common case. Dropping rather than substituting (e.g. with U+FFFD) avoids visible replacement-character artifacts in downstream UIs that would otherwise surface data corruption to end users.
func SanitizeStringPtr ¶ added in v0.13.2
func SanitizeStringPtr(s *string)
SanitizeStringPtr applies SanitizeString through a *string, leaving nil pointers untouched.
func ScanAll ¶ added in v0.16.8
ScanAll reads an open result set into one entry per row.
Every catalog query in this repository ends in the same loop, and having one implementation of it is what lets the way a warehouse's columns are matched to a struct be decided once rather than per query. It goes through RowScanner, so the result set decides the shape: a column no field claims is discarded rather than fatal, a field no column feeds stays at its zero value, and the match is case-insensitive. A caller that cannot work without a given column plans the scan itself and says so with RequireColumns.
source names the object read ("MY_DB.information_schema.tables"). It is what a scan failure is reported against and what column drift is logged against — the caller's own wrapping says which step it belonged to.
func SqlLiteral ¶ added in v0.19.0
SqlLiteral writes a value read from a column of the given kind back into SQL as a literal for dialect d, such that `column = literal` holds on the warehouse the value came from. The kind is QueryShapeColumn.Kind of the column the value was read from, as RunRawQuery and QueryShape report it.
With KindUnknown the literal follows the Value alone: a TimeValue becomes a zone-less timestamp and a StringValue a string. That is right for text and numbers and for timestamps on the engines that coerce, and wrong for a DATE, UUID or exact NUMERIC column on those that do not (BigQuery, Trino).
It fails for nil, IgnoredValue and JsonValue, and for a value that does not fit the kind (text in an integer column), rather than writing a literal that compares with something else.
Types ¶
type Annotation ¶ added in v0.4.12
type Annotation struct {
AnnotationName string `json:"annotation_name"`
AnnotationValue string `json:"annotation_value"`
}
func (*Annotation) HasValidIdentity ¶ added in v0.13.2
func (a *Annotation) HasValidIdentity() bool
func (*Annotation) Sanitize ¶ added in v0.13.2
func (a *Annotation) Sanitize()
type BigIntValue ¶ added in v0.9.0
BigIntValue represents arbitrary precision integers (e.g., DuckDB hugeint, uint128)
func NewBigIntValue ¶ added in v0.9.0
func NewBigIntValue(v *big.Int) *BigIntValue
NewBigIntValue creates a BigIntValue from a *big.Int
func (*BigIntValue) BigInt ¶ added in v0.9.0
func (v *BigIntValue) BigInt() *big.Int
BigInt returns the underlying *big.Int
func (*BigIntValue) String ¶ added in v0.9.0
func (v *BigIntValue) String() string
String returns the string representation of the big integer
type Capabilities ¶ added in v0.10.6
type Capabilities struct {
// ConstraintsViaQueryTables indicates that QueryTables with WithConstraints()
// populates TableRow.Constraints, making a separate QueryTableConstraints call unnecessary.
ConstraintsViaQueryTables bool
// EstimateQuery describes EstimateQuery support for this dialect. Its zero
// value means EstimateQuery returns ErrUnsupported. Callers should consult
// this before calling EstimateQuery so they can pick an appropriate UI
// ("this will scan ~X bytes" vs "~Y rows") or skip the call entirely.
EstimateQuery EstimateQueryCapability
// PlatformUsers describes QueryPlatformUsers support. Its zero value means
// the platform has no user listing and QueryPlatformUsers returns
// ErrUnsupported, so a caller never calls it.
PlatformUsers PlatformUsersCapability
}
Capabilities describes what a scrapper supports beyond the base interface. Callers can use this to skip redundant calls (e.g., skip QueryTableConstraints when constraints are already provided via QueryTables).
type CatalogColumnRow ¶
type CatalogColumnRow struct {
Instance string `db:"instance" json:"instance" ch:"instance" bigquery:"instance"`
Database string `db:"database" json:"database" ch:"_database" bigquery:"database"`
Schema string `db:"schema" json:"schema" ch:"schema" bigquery:"schema"`
Table string `db:"table" json:"table" ch:"table" bigquery:"table"`
IsView bool `db:"is_view" json:"is_view" ch:"is_view" bigquery:"is_view"`
IsTable bool `db:"is_table" json:"is_table" ch:"is_table" bigquery:"is_table"`
IsMaterializedView bool `db:"is_materialized_view" json:"is_materialized_view" ch:"is_materialized_view" bigquery:"is_materialized_view"`
TableType string `db:"table_type" json:"table_type" ch:"table_type" bigquery:"table_type"`
Column string `db:"column" json:"column" ch:"column" bigquery:"column"`
Type string `db:"type" json:"type" ch:"type" bigquery:"type"`
Position int32 `db:"position" json:"position" ch:"position" bigquery:"position"`
Comment *string `db:"comment" json:"comment" ch:"comment" bigquery:"comment"`
TableComment *string `db:"table_comment" json:"table_comment" ch:"table_comment" bigquery:"table_comment"`
ColumnTags []*Tag ` json:"column_tags"`
TableTags []*Tag ` json:"table_tags"`
IsStructColumn bool `db:"is_struct_column" json:"is_struct_column" ch:"is_struct_column" bigquery:"is_struct_column"`
IsArrayColumn bool `db:"is_array_column" json:"is_array_column" ch:"is_array_column" bigquery:"is_array_column"`
FieldSchemas []*SchemaColumnField
// Precision and Scale are an exact numeric column's declared size, and
// NumericCategory what kind of number the column holds, all filled by
// ResolveNumeric. Both sizes are nil for every other column, and for a
// numeric whose size the warehouse does not report: a nil scale is not a
// scale of 0.
Precision *int64 `db:"numeric_precision" json:"precision,omitempty" ch:"numeric_precision"`
Scale *int64 `db:"numeric_scale" json:"scale,omitempty" ch:"numeric_scale"`
NumericCategory NumericCategory ` json:"numeric_category,omitempty"`
}
func (CatalogColumnRow) GetComment ¶
func (r CatalogColumnRow) GetComment() string
func (CatalogColumnRow) GetTableComment ¶
func (r CatalogColumnRow) GetTableComment() string
func (*CatalogColumnRow) HasValidIdentity ¶ added in v0.13.2
func (r *CatalogColumnRow) HasValidIdentity() bool
func (*CatalogColumnRow) ResolveNumeric ¶ added in v0.21.0
func (r *CatalogColumnRow) ResolveNumeric(dialect string) *CatalogColumnRow
ResolveNumeric fills a catalog row's Precision, Scale and NumericCategory from its Type on dialect (Scrapper.DialectType()), and returns r. Every QueryCatalog calls it on each row it returns.
A catalog query that reads the size from the warehouse's own columns (Snowflake's numeric_precision and numeric_scale) scans it into Precision and Scale first; otherwise the size is read from Type (DECIMAL(20,6)). Either way it is kept only for an exact numeric, since information_schema reports a precision for integers and floats too, in bits on some engines, and only when it is a size a column can have (SetDecimalSize). A numeric declared without a size keeps both nil: a nil scale is not a scale of 0.
func (*CatalogColumnRow) Sanitize ¶ added in v0.13.2
func (r *CatalogColumnRow) Sanitize()
func (CatalogColumnRow) TableFqn ¶
func (r CatalogColumnRow) TableFqn() DwhFqn
type ColumnValue ¶
func (*ColumnValue) HasValidIdentity ¶ added in v0.13.2
func (v *ColumnValue) HasValidIdentity() bool
func (*ColumnValue) Sanitize ¶ added in v0.13.2
func (v *ColumnValue) Sanitize()
type CustomMetricsRow ¶
type CustomMetricsRow struct {
Segments []*SegmentValue
ColumnValues []*ColumnValue
}
func (*CustomMetricsRow) HasValidIdentity ¶ added in v0.13.2
func (r *CustomMetricsRow) HasValidIdentity() bool
HasValidIdentity on CustomMetricsRow requires every segment and column name to be valid; a single bad identifier changes the meaning of the whole row, so we drop rather than partially filter.
func (*CustomMetricsRow) Sanitize ¶ added in v0.13.2
func (r *CustomMetricsRow) Sanitize()
type DatabaseRow ¶
type DatabaseRow struct {
Instance string `db:"instance" json:"instance" ch:"instance" bigquery:"instance"`
Database string `db:"database" json:"database" ch:"_database" bigquery:"database"`
Description *string `db:"description" json:"description" ch:"description" bigquery:"description"`
DatabaseType *string `db:"database_type" json:"database_type" ch:"database_type" bigquery:"database_type"`
DatabaseOwner *string `db:"database_owner" json:"database_owner" ch:"database_owner" bigquery:"database_owner"`
}
func (*DatabaseRow) HasValidIdentity ¶ added in v0.13.2
func (r *DatabaseRow) HasValidIdentity() bool
func (*DatabaseRow) Sanitize ¶ added in v0.13.2
func (r *DatabaseRow) Sanitize()
func (*DatabaseRow) SetInstance ¶
func (r *DatabaseRow) SetInstance(instance string)
type DoubleValue ¶
type DoubleValue float64
type EstimateQueryCapability ¶ added in v0.16.0
type EstimateQueryCapability struct {
// Supported is true when EstimateQuery may return a *QueryEstimate rather
// than ErrUnsupported.
Supported bool
// ReportsBytes is true when the estimate can populate QueryEstimate.BytesScanned.
ReportsBytes bool
// ReportsRows is true when the estimate can populate QueryEstimate.Rows.
ReportsRows bool
// CanBeExact is true when the dialect is capable of authoritative estimates
// rather than planner guesses (BigQuery dry-run). Individual estimates may
// still come back with QueryEstimate.Exact=false (e.g. BigQuery non-PRECISE
// accuracy); planner-based dialects are never exact. Planner estimates
// depend on fresh table statistics and can be off by orders of magnitude.
CanBeExact bool
}
EstimateQueryCapability advertises what a dialect's EstimateQuery implementation can do. It is a static description of the dialect, distinct from the per-query QueryEstimate result: e.g. CanBeExact says a dialect is capable of authoritative estimates, while QueryEstimate.Exact reports whether a specific estimate actually was. Granularity differs by engine: some return bytes scanned (BigQuery, Snowflake), others only a planner row estimate (ClickHouse, Postgres), and many cannot estimate reliably at all.
type HasTableFqn ¶
type HasTableFqn interface {
TableFqn() DwhFqn
}
type IgnoredValue ¶
type IgnoredValue struct{}
type InstanceHost ¶ added in v0.16.2
type InstanceHost struct {
// Host is the hostname exactly as the warehouse reported it, including any
// vendor domain suffix.
Host string
// Kind classifies the host, since one account is typically reachable under
// several names that encode different identifier forms. One of the
// InstanceHost* constants; empty when the warehouse gave no classification.
Kind string
}
InstanceHost is one hostname the warehouse reports for itself.
type InstanceIdentity ¶ added in v0.16.2
type InstanceIdentity struct {
// Raw holds each value keyed by the warehouse construct that produced it
// (the InstanceIdentity* constants). A key is absent when the warehouse did
// not report it; no key is guaranteed present.
Raw map[string]string
// Hosts are the hostnames the warehouse reports for the instance itself.
// Hostnames of unrelated infrastructure the instance happens to talk to
// (object storage, OCSP responders) are not included.
Hosts []InstanceHost
}
InstanceIdentity is what a warehouse answers when asked what it calls itself.
It exists because a single warehouse instance is often addressable under several unrelated identifiers, and which one a given tool reports is decided by that tool, not by us. Snowflake is the motivating case: an account has an account locator and an organization-qualified name, and neither is derivable from the other by string manipulation — the mapping lives inside Snowflake. Two producers can therefore normalize an account correctly and still disagree about its identity. Asking the warehouse is the only way to learn every form.
The values are deliberately RAW: exactly what the warehouse returned, never parsed, cased or assembled into an identifier here. Callers own identifier construction and apply their own normalization, which this package must not try to reproduce.
func (*InstanceIdentity) Get ¶ added in v0.16.2
func (i *InstanceIdentity) Get(key string) string
Get returns the raw value for key, or "" when the warehouse did not report it.
type InstanceIdentityQuerier ¶ added in v0.16.2
type InstanceIdentityQuerier interface {
// QueryInstanceIdentity returns every form of its own identity the
// warehouse will report. It returns ErrUnsupported when the platform has
// no such concept.
QueryInstanceIdentity(ctx context.Context) (*InstanceIdentity, error)
}
InstanceIdentityQuerier is implemented by scrappers that can ask their warehouse what it calls itself. It is optional: most platforms have a single identifier form and need nothing here. Reach it through As, so the capability survives being wrapped by decorators:
if q, ok := scrapper.As[scrapper.InstanceIdentityQuerier](s); ok { ... }
type JsonValue ¶ added in v0.16.0
type JsonValue string
JsonValue carries a complex / nested cell — an array/list/vector, a struct/record/named-tuple, a map, or a semi-structured object/variant — as canonical JSON text. Warehouses expose these either as native nested values (ClickHouse Array/Map/Tuple/Nested, BigQuery repeated fields & STRUCT/RECORD, DuckDB LIST/STRUCT/MAP, Postgres arrays, Trino array/map/row) or as JSON/semi-structured strings (Snowflake ARRAY/OBJECT/VARIANT, Redshift SUPER, Fabric/MSSQL nvarchar JSON). Rather than flatten them to fmt.Sprint garbage or lossy scalars, RunRawQuery normalises them to JSON so a single frontend renderer can show a structured tree uniformly across every warehouse.
The text is always valid JSON. Precision-sensitive scalars are encoded so the JSON round-trip is lossless at the text level: arbitrary-precision integers as JSON number literals, decimals as JSON numbers, and timestamps as RFC3339 strings. JsonValue is only emitted on the RunRawQuery path — the metrics / profile path (QueryCustomMetrics) still collapses non-scalar cells to IgnoredValue, since a nested cell cannot be a metric or a segment.
func NewJsonValueFromGo ¶ added in v0.16.0
NewJsonValueFromGo converts an arbitrary driver-returned value — an arbitrarily nested tree of Go slices, arrays, maps, pointers and scalars — into a JsonValue holding canonical JSON text. It is the single normaliser used by every RunRawQuery implementation for complex/nested cells so the wire format is identical regardless of warehouse.
bytesAsInts controls how a []byte / []uint8 is rendered, because Go cannot tell a "small-int array" from a "byte blob" — both are []uint8:
- true → the value is a native integer array (e.g. ClickHouse Array(UInt8), which the driver hands back as []uint8); render as a JSON array of numbers.
- false → the value is a byte blob (BYTES/BLOB column); render as a JSON string (invalid UTF-8 scrubbed), matching the scalar []byte→string path.
It returns ok=false only when the value cannot be represented as JSON at all; unknown scalar types fall back to their fmt.Stringer / fmt.Sprint form so nothing is silently dropped.
func NewJsonValueFromJSONText ¶ added in v0.16.0
NewJsonValueFromJSONText wraps text that is already JSON (Snowflake ARRAY/OBJECT/VARIANT, Redshift SUPER, Fabric/MSSQL nvarchar JSON) into a JsonValue, re-serialised to canonical (compact) form. Returns ok=false when the text is not valid JSON so the caller can fall back to StringValue.
type MockScrapper ¶ added in v0.4.7
type MockScrapper struct {
// contains filtered or unexported fields
}
MockScrapper is a mock of Scrapper interface.
func NewMockScrapper ¶ added in v0.4.7
func NewMockScrapper(ctrl *gomock.Controller) *MockScrapper
NewMockScrapper creates a new mock instance.
func (*MockScrapper) Capabilities ¶ added in v0.10.6
func (m *MockScrapper) Capabilities() Capabilities
Capabilities mocks base method.
func (*MockScrapper) Close ¶ added in v0.4.7
func (m *MockScrapper) Close() error
Close mocks base method.
func (*MockScrapper) DialectType ¶ added in v0.4.7
func (m *MockScrapper) DialectType() string
DialectType mocks base method.
func (*MockScrapper) EXPECT ¶ added in v0.4.7
func (m *MockScrapper) EXPECT() *MockScrapperMockRecorder
EXPECT returns an object that allows the caller to indicate expected use.
func (*MockScrapper) EstimateQuery ¶ added in v0.16.0
func (m *MockScrapper) EstimateQuery(ctx context.Context, sql string) (*QueryEstimate, error)
EstimateQuery mocks base method.
func (*MockScrapper) IsPermissionError ¶ added in v0.4.7
func (m *MockScrapper) IsPermissionError(err error) bool
IsPermissionError mocks base method.
func (*MockScrapper) QueryCatalog ¶ added in v0.4.7
func (m *MockScrapper) QueryCatalog(ctx context.Context) ([]*CatalogColumnRow, error)
QueryCatalog mocks base method.
func (*MockScrapper) QueryCustomMetrics ¶ added in v0.4.7
func (m *MockScrapper) QueryCustomMetrics(ctx context.Context, sql string, args ...any) ([]*CustomMetricsRow, error)
QueryCustomMetrics mocks base method.
func (*MockScrapper) QueryDatabases ¶ added in v0.4.7
func (m *MockScrapper) QueryDatabases(ctx context.Context) ([]*DatabaseRow, error)
QueryDatabases mocks base method.
func (*MockScrapper) QueryPlatformUsers ¶ added in v0.21.0
func (m *MockScrapper) QueryPlatformUsers(ctx context.Context) (*PlatformUsers, error)
QueryPlatformUsers mocks base method.
func (*MockScrapper) QuerySchemas ¶ added in v0.14.5
func (m *MockScrapper) QuerySchemas(ctx context.Context) ([]*SchemaRow, error)
QuerySchemas mocks base method.
func (*MockScrapper) QuerySegments ¶ added in v0.4.7
func (m *MockScrapper) QuerySegments(ctx context.Context, sql string, args ...any) ([]*SegmentRow, error)
QuerySegments mocks base method.
func (*MockScrapper) QueryShape ¶ added in v0.8.3
func (m *MockScrapper) QueryShape(ctx context.Context, sql string) ([]*QueryShapeColumn, error)
QueryShape mocks base method.
func (*MockScrapper) QuerySqlDefinitions ¶ added in v0.4.7
func (m *MockScrapper) QuerySqlDefinitions(ctx context.Context) ([]*SqlDefinitionRow, error)
QuerySqlDefinitions mocks base method.
func (*MockScrapper) QueryTableConstraints ¶ added in v0.9.0
func (m *MockScrapper) QueryTableConstraints(ctx context.Context) ([]*TableConstraintRow, error)
QueryTableConstraints mocks base method.
func (*MockScrapper) QueryTableMetrics ¶ added in v0.4.7
func (m *MockScrapper) QueryTableMetrics(ctx context.Context, lastMetricsFetchTime time.Time) ([]*TableMetricsRow, error)
QueryTableMetrics mocks base method.
func (*MockScrapper) QueryTables ¶ added in v0.4.7
func (m *MockScrapper) QueryTables(ctx context.Context, opts ...QueryTablesOption) ([]*TableRow, error)
QueryTables mocks base method.
func (*MockScrapper) RunRawQuery ¶ added in v0.13.7
func (m *MockScrapper) RunRawQuery(ctx context.Context, sql string) (RawQueryRowIterator, error)
RunRawQuery mocks base method.
func (*MockScrapper) SqlDialect ¶ added in v0.4.7
func (m *MockScrapper) SqlDialect() sqldialect.Dialect
SqlDialect mocks base method.
func (*MockScrapper) ValidateConfiguration ¶ added in v0.4.7
func (m *MockScrapper) ValidateConfiguration(ctx context.Context) ([]string, error)
ValidateConfiguration mocks base method.
type MockScrapperCapabilitiesCall ¶ added in v0.10.6
MockScrapperCapabilitiesCall wrap *gomock.Call
func (*MockScrapperCapabilitiesCall) Do ¶ added in v0.10.6
func (c *MockScrapperCapabilitiesCall) Do(f func() Capabilities) *MockScrapperCapabilitiesCall
Do rewrite *gomock.Call.Do
func (*MockScrapperCapabilitiesCall) DoAndReturn ¶ added in v0.10.6
func (c *MockScrapperCapabilitiesCall) DoAndReturn(f func() Capabilities) *MockScrapperCapabilitiesCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperCapabilitiesCall) Return ¶ added in v0.10.6
func (c *MockScrapperCapabilitiesCall) Return(arg0 Capabilities) *MockScrapperCapabilitiesCall
Return rewrite *gomock.Call.Return
type MockScrapperCloseCall ¶ added in v0.4.7
MockScrapperCloseCall wrap *gomock.Call
func (*MockScrapperCloseCall) Do ¶ added in v0.4.7
func (c *MockScrapperCloseCall) Do(f func() error) *MockScrapperCloseCall
Do rewrite *gomock.Call.Do
func (*MockScrapperCloseCall) DoAndReturn ¶ added in v0.4.7
func (c *MockScrapperCloseCall) DoAndReturn(f func() error) *MockScrapperCloseCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperCloseCall) Return ¶ added in v0.4.7
func (c *MockScrapperCloseCall) Return(arg0 error) *MockScrapperCloseCall
Return rewrite *gomock.Call.Return
type MockScrapperDialectTypeCall ¶ added in v0.4.7
MockScrapperDialectTypeCall wrap *gomock.Call
func (*MockScrapperDialectTypeCall) Do ¶ added in v0.4.7
func (c *MockScrapperDialectTypeCall) Do(f func() string) *MockScrapperDialectTypeCall
Do rewrite *gomock.Call.Do
func (*MockScrapperDialectTypeCall) DoAndReturn ¶ added in v0.4.7
func (c *MockScrapperDialectTypeCall) DoAndReturn(f func() string) *MockScrapperDialectTypeCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperDialectTypeCall) Return ¶ added in v0.4.7
func (c *MockScrapperDialectTypeCall) Return(arg0 string) *MockScrapperDialectTypeCall
Return rewrite *gomock.Call.Return
type MockScrapperEstimateQueryCall ¶ added in v0.16.0
MockScrapperEstimateQueryCall wrap *gomock.Call
func (*MockScrapperEstimateQueryCall) Do ¶ added in v0.16.0
func (c *MockScrapperEstimateQueryCall) Do(f func(context.Context, string) (*QueryEstimate, error)) *MockScrapperEstimateQueryCall
Do rewrite *gomock.Call.Do
func (*MockScrapperEstimateQueryCall) DoAndReturn ¶ added in v0.16.0
func (c *MockScrapperEstimateQueryCall) DoAndReturn(f func(context.Context, string) (*QueryEstimate, error)) *MockScrapperEstimateQueryCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperEstimateQueryCall) Return ¶ added in v0.16.0
func (c *MockScrapperEstimateQueryCall) Return(arg0 *QueryEstimate, arg1 error) *MockScrapperEstimateQueryCall
Return rewrite *gomock.Call.Return
type MockScrapperIsPermissionErrorCall ¶ added in v0.4.7
MockScrapperIsPermissionErrorCall wrap *gomock.Call
func (*MockScrapperIsPermissionErrorCall) Do ¶ added in v0.4.7
func (c *MockScrapperIsPermissionErrorCall) Do(f func(error) bool) *MockScrapperIsPermissionErrorCall
Do rewrite *gomock.Call.Do
func (*MockScrapperIsPermissionErrorCall) DoAndReturn ¶ added in v0.4.7
func (c *MockScrapperIsPermissionErrorCall) DoAndReturn(f func(error) bool) *MockScrapperIsPermissionErrorCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperIsPermissionErrorCall) Return ¶ added in v0.4.7
func (c *MockScrapperIsPermissionErrorCall) Return(arg0 bool) *MockScrapperIsPermissionErrorCall
Return rewrite *gomock.Call.Return
type MockScrapperMockRecorder ¶ added in v0.4.7
type MockScrapperMockRecorder struct {
// contains filtered or unexported fields
}
MockScrapperMockRecorder is the mock recorder for MockScrapper.
func (*MockScrapperMockRecorder) Capabilities ¶ added in v0.10.6
func (mr *MockScrapperMockRecorder) Capabilities() *MockScrapperCapabilitiesCall
Capabilities indicates an expected call of Capabilities.
func (*MockScrapperMockRecorder) Close ¶ added in v0.4.7
func (mr *MockScrapperMockRecorder) Close() *MockScrapperCloseCall
Close indicates an expected call of Close.
func (*MockScrapperMockRecorder) DialectType ¶ added in v0.4.7
func (mr *MockScrapperMockRecorder) DialectType() *MockScrapperDialectTypeCall
DialectType indicates an expected call of DialectType.
func (*MockScrapperMockRecorder) EstimateQuery ¶ added in v0.16.0
func (mr *MockScrapperMockRecorder) EstimateQuery(ctx, sql any) *MockScrapperEstimateQueryCall
EstimateQuery indicates an expected call of EstimateQuery.
func (*MockScrapperMockRecorder) IsPermissionError ¶ added in v0.4.7
func (mr *MockScrapperMockRecorder) IsPermissionError(err any) *MockScrapperIsPermissionErrorCall
IsPermissionError indicates an expected call of IsPermissionError.
func (*MockScrapperMockRecorder) QueryCatalog ¶ added in v0.4.7
func (mr *MockScrapperMockRecorder) QueryCatalog(ctx any) *MockScrapperQueryCatalogCall
QueryCatalog indicates an expected call of QueryCatalog.
func (*MockScrapperMockRecorder) QueryCustomMetrics ¶ added in v0.4.7
func (mr *MockScrapperMockRecorder) QueryCustomMetrics(ctx, sql any, args ...any) *MockScrapperQueryCustomMetricsCall
QueryCustomMetrics indicates an expected call of QueryCustomMetrics.
func (*MockScrapperMockRecorder) QueryDatabases ¶ added in v0.4.7
func (mr *MockScrapperMockRecorder) QueryDatabases(ctx any) *MockScrapperQueryDatabasesCall
QueryDatabases indicates an expected call of QueryDatabases.
func (*MockScrapperMockRecorder) QueryPlatformUsers ¶ added in v0.21.0
func (mr *MockScrapperMockRecorder) QueryPlatformUsers(ctx any) *MockScrapperQueryPlatformUsersCall
QueryPlatformUsers indicates an expected call of QueryPlatformUsers.
func (*MockScrapperMockRecorder) QuerySchemas ¶ added in v0.14.5
func (mr *MockScrapperMockRecorder) QuerySchemas(ctx any) *MockScrapperQuerySchemasCall
QuerySchemas indicates an expected call of QuerySchemas.
func (*MockScrapperMockRecorder) QuerySegments ¶ added in v0.4.7
func (mr *MockScrapperMockRecorder) QuerySegments(ctx, sql any, args ...any) *MockScrapperQuerySegmentsCall
QuerySegments indicates an expected call of QuerySegments.
func (*MockScrapperMockRecorder) QueryShape ¶ added in v0.8.3
func (mr *MockScrapperMockRecorder) QueryShape(ctx, sql any) *MockScrapperQueryShapeCall
QueryShape indicates an expected call of QueryShape.
func (*MockScrapperMockRecorder) QuerySqlDefinitions ¶ added in v0.4.7
func (mr *MockScrapperMockRecorder) QuerySqlDefinitions(ctx any) *MockScrapperQuerySqlDefinitionsCall
QuerySqlDefinitions indicates an expected call of QuerySqlDefinitions.
func (*MockScrapperMockRecorder) QueryTableConstraints ¶ added in v0.9.0
func (mr *MockScrapperMockRecorder) QueryTableConstraints(ctx any) *MockScrapperQueryTableConstraintsCall
QueryTableConstraints indicates an expected call of QueryTableConstraints.
func (*MockScrapperMockRecorder) QueryTableMetrics ¶ added in v0.4.7
func (mr *MockScrapperMockRecorder) QueryTableMetrics(ctx, lastMetricsFetchTime any) *MockScrapperQueryTableMetricsCall
QueryTableMetrics indicates an expected call of QueryTableMetrics.
func (*MockScrapperMockRecorder) QueryTables ¶ added in v0.4.7
func (mr *MockScrapperMockRecorder) QueryTables(ctx any, opts ...any) *MockScrapperQueryTablesCall
QueryTables indicates an expected call of QueryTables.
func (*MockScrapperMockRecorder) RunRawQuery ¶ added in v0.13.7
func (mr *MockScrapperMockRecorder) RunRawQuery(ctx, sql any) *MockScrapperRunRawQueryCall
RunRawQuery indicates an expected call of RunRawQuery.
func (*MockScrapperMockRecorder) SqlDialect ¶ added in v0.4.7
func (mr *MockScrapperMockRecorder) SqlDialect() *MockScrapperSqlDialectCall
SqlDialect indicates an expected call of SqlDialect.
func (*MockScrapperMockRecorder) ValidateConfiguration ¶ added in v0.4.7
func (mr *MockScrapperMockRecorder) ValidateConfiguration(ctx any) *MockScrapperValidateConfigurationCall
ValidateConfiguration indicates an expected call of ValidateConfiguration.
type MockScrapperQueryCatalogCall ¶ added in v0.4.7
MockScrapperQueryCatalogCall wrap *gomock.Call
func (*MockScrapperQueryCatalogCall) Do ¶ added in v0.4.7
func (c *MockScrapperQueryCatalogCall) Do(f func(context.Context) ([]*CatalogColumnRow, error)) *MockScrapperQueryCatalogCall
Do rewrite *gomock.Call.Do
func (*MockScrapperQueryCatalogCall) DoAndReturn ¶ added in v0.4.7
func (c *MockScrapperQueryCatalogCall) DoAndReturn(f func(context.Context) ([]*CatalogColumnRow, error)) *MockScrapperQueryCatalogCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperQueryCatalogCall) Return ¶ added in v0.4.7
func (c *MockScrapperQueryCatalogCall) Return(arg0 []*CatalogColumnRow, arg1 error) *MockScrapperQueryCatalogCall
Return rewrite *gomock.Call.Return
type MockScrapperQueryCustomMetricsCall ¶ added in v0.4.7
MockScrapperQueryCustomMetricsCall wrap *gomock.Call
func (*MockScrapperQueryCustomMetricsCall) Do ¶ added in v0.4.7
func (c *MockScrapperQueryCustomMetricsCall) Do(f func(context.Context, string, ...any) ([]*CustomMetricsRow, error)) *MockScrapperQueryCustomMetricsCall
Do rewrite *gomock.Call.Do
func (*MockScrapperQueryCustomMetricsCall) DoAndReturn ¶ added in v0.4.7
func (c *MockScrapperQueryCustomMetricsCall) DoAndReturn(f func(context.Context, string, ...any) ([]*CustomMetricsRow, error)) *MockScrapperQueryCustomMetricsCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperQueryCustomMetricsCall) Return ¶ added in v0.4.7
func (c *MockScrapperQueryCustomMetricsCall) Return(arg0 []*CustomMetricsRow, arg1 error) *MockScrapperQueryCustomMetricsCall
Return rewrite *gomock.Call.Return
type MockScrapperQueryDatabasesCall ¶ added in v0.4.7
MockScrapperQueryDatabasesCall wrap *gomock.Call
func (*MockScrapperQueryDatabasesCall) Do ¶ added in v0.4.7
func (c *MockScrapperQueryDatabasesCall) Do(f func(context.Context) ([]*DatabaseRow, error)) *MockScrapperQueryDatabasesCall
Do rewrite *gomock.Call.Do
func (*MockScrapperQueryDatabasesCall) DoAndReturn ¶ added in v0.4.7
func (c *MockScrapperQueryDatabasesCall) DoAndReturn(f func(context.Context) ([]*DatabaseRow, error)) *MockScrapperQueryDatabasesCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperQueryDatabasesCall) Return ¶ added in v0.4.7
func (c *MockScrapperQueryDatabasesCall) Return(arg0 []*DatabaseRow, arg1 error) *MockScrapperQueryDatabasesCall
Return rewrite *gomock.Call.Return
type MockScrapperQueryPlatformUsersCall ¶ added in v0.21.0
MockScrapperQueryPlatformUsersCall wrap *gomock.Call
func (*MockScrapperQueryPlatformUsersCall) Do ¶ added in v0.21.0
func (c *MockScrapperQueryPlatformUsersCall) Do(f func(context.Context) (*PlatformUsers, error)) *MockScrapperQueryPlatformUsersCall
Do rewrite *gomock.Call.Do
func (*MockScrapperQueryPlatformUsersCall) DoAndReturn ¶ added in v0.21.0
func (c *MockScrapperQueryPlatformUsersCall) DoAndReturn(f func(context.Context) (*PlatformUsers, error)) *MockScrapperQueryPlatformUsersCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperQueryPlatformUsersCall) Return ¶ added in v0.21.0
func (c *MockScrapperQueryPlatformUsersCall) Return(arg0 *PlatformUsers, arg1 error) *MockScrapperQueryPlatformUsersCall
Return rewrite *gomock.Call.Return
type MockScrapperQuerySchemasCall ¶ added in v0.14.5
MockScrapperQuerySchemasCall wrap *gomock.Call
func (*MockScrapperQuerySchemasCall) Do ¶ added in v0.14.5
func (c *MockScrapperQuerySchemasCall) Do(f func(context.Context) ([]*SchemaRow, error)) *MockScrapperQuerySchemasCall
Do rewrite *gomock.Call.Do
func (*MockScrapperQuerySchemasCall) DoAndReturn ¶ added in v0.14.5
func (c *MockScrapperQuerySchemasCall) DoAndReturn(f func(context.Context) ([]*SchemaRow, error)) *MockScrapperQuerySchemasCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperQuerySchemasCall) Return ¶ added in v0.14.5
func (c *MockScrapperQuerySchemasCall) Return(arg0 []*SchemaRow, arg1 error) *MockScrapperQuerySchemasCall
Return rewrite *gomock.Call.Return
type MockScrapperQuerySegmentsCall ¶ added in v0.4.7
MockScrapperQuerySegmentsCall wrap *gomock.Call
func (*MockScrapperQuerySegmentsCall) Do ¶ added in v0.4.7
func (c *MockScrapperQuerySegmentsCall) Do(f func(context.Context, string, ...any) ([]*SegmentRow, error)) *MockScrapperQuerySegmentsCall
Do rewrite *gomock.Call.Do
func (*MockScrapperQuerySegmentsCall) DoAndReturn ¶ added in v0.4.7
func (c *MockScrapperQuerySegmentsCall) DoAndReturn(f func(context.Context, string, ...any) ([]*SegmentRow, error)) *MockScrapperQuerySegmentsCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperQuerySegmentsCall) Return ¶ added in v0.4.7
func (c *MockScrapperQuerySegmentsCall) Return(arg0 []*SegmentRow, arg1 error) *MockScrapperQuerySegmentsCall
Return rewrite *gomock.Call.Return
type MockScrapperQueryShapeCall ¶ added in v0.8.3
MockScrapperQueryShapeCall wrap *gomock.Call
func (*MockScrapperQueryShapeCall) Do ¶ added in v0.8.3
func (c *MockScrapperQueryShapeCall) Do(f func(context.Context, string) ([]*QueryShapeColumn, error)) *MockScrapperQueryShapeCall
Do rewrite *gomock.Call.Do
func (*MockScrapperQueryShapeCall) DoAndReturn ¶ added in v0.8.3
func (c *MockScrapperQueryShapeCall) DoAndReturn(f func(context.Context, string) ([]*QueryShapeColumn, error)) *MockScrapperQueryShapeCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperQueryShapeCall) Return ¶ added in v0.8.3
func (c *MockScrapperQueryShapeCall) Return(arg0 []*QueryShapeColumn, arg1 error) *MockScrapperQueryShapeCall
Return rewrite *gomock.Call.Return
type MockScrapperQuerySqlDefinitionsCall ¶ added in v0.4.7
MockScrapperQuerySqlDefinitionsCall wrap *gomock.Call
func (*MockScrapperQuerySqlDefinitionsCall) Do ¶ added in v0.4.7
func (c *MockScrapperQuerySqlDefinitionsCall) Do(f func(context.Context) ([]*SqlDefinitionRow, error)) *MockScrapperQuerySqlDefinitionsCall
Do rewrite *gomock.Call.Do
func (*MockScrapperQuerySqlDefinitionsCall) DoAndReturn ¶ added in v0.4.7
func (c *MockScrapperQuerySqlDefinitionsCall) DoAndReturn(f func(context.Context) ([]*SqlDefinitionRow, error)) *MockScrapperQuerySqlDefinitionsCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperQuerySqlDefinitionsCall) Return ¶ added in v0.4.7
func (c *MockScrapperQuerySqlDefinitionsCall) Return(arg0 []*SqlDefinitionRow, arg1 error) *MockScrapperQuerySqlDefinitionsCall
Return rewrite *gomock.Call.Return
type MockScrapperQueryTableConstraintsCall ¶ added in v0.9.0
MockScrapperQueryTableConstraintsCall wrap *gomock.Call
func (*MockScrapperQueryTableConstraintsCall) Do ¶ added in v0.9.0
func (c *MockScrapperQueryTableConstraintsCall) Do(f func(context.Context) ([]*TableConstraintRow, error)) *MockScrapperQueryTableConstraintsCall
Do rewrite *gomock.Call.Do
func (*MockScrapperQueryTableConstraintsCall) DoAndReturn ¶ added in v0.9.0
func (c *MockScrapperQueryTableConstraintsCall) DoAndReturn(f func(context.Context) ([]*TableConstraintRow, error)) *MockScrapperQueryTableConstraintsCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperQueryTableConstraintsCall) Return ¶ added in v0.9.0
func (c *MockScrapperQueryTableConstraintsCall) Return(arg0 []*TableConstraintRow, arg1 error) *MockScrapperQueryTableConstraintsCall
Return rewrite *gomock.Call.Return
type MockScrapperQueryTableMetricsCall ¶ added in v0.4.7
MockScrapperQueryTableMetricsCall wrap *gomock.Call
func (*MockScrapperQueryTableMetricsCall) Do ¶ added in v0.4.7
func (c *MockScrapperQueryTableMetricsCall) Do(f func(context.Context, time.Time) ([]*TableMetricsRow, error)) *MockScrapperQueryTableMetricsCall
Do rewrite *gomock.Call.Do
func (*MockScrapperQueryTableMetricsCall) DoAndReturn ¶ added in v0.4.7
func (c *MockScrapperQueryTableMetricsCall) DoAndReturn(f func(context.Context, time.Time) ([]*TableMetricsRow, error)) *MockScrapperQueryTableMetricsCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperQueryTableMetricsCall) Return ¶ added in v0.4.7
func (c *MockScrapperQueryTableMetricsCall) Return(arg0 []*TableMetricsRow, arg1 error) *MockScrapperQueryTableMetricsCall
Return rewrite *gomock.Call.Return
type MockScrapperQueryTablesCall ¶ added in v0.4.7
MockScrapperQueryTablesCall wrap *gomock.Call
func (*MockScrapperQueryTablesCall) Do ¶ added in v0.4.7
func (c *MockScrapperQueryTablesCall) Do(f func(context.Context, ...QueryTablesOption) ([]*TableRow, error)) *MockScrapperQueryTablesCall
Do rewrite *gomock.Call.Do
func (*MockScrapperQueryTablesCall) DoAndReturn ¶ added in v0.4.7
func (c *MockScrapperQueryTablesCall) DoAndReturn(f func(context.Context, ...QueryTablesOption) ([]*TableRow, error)) *MockScrapperQueryTablesCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperQueryTablesCall) Return ¶ added in v0.4.7
func (c *MockScrapperQueryTablesCall) Return(arg0 []*TableRow, arg1 error) *MockScrapperQueryTablesCall
Return rewrite *gomock.Call.Return
type MockScrapperRunRawQueryCall ¶ added in v0.13.7
MockScrapperRunRawQueryCall wrap *gomock.Call
func (*MockScrapperRunRawQueryCall) Do ¶ added in v0.13.7
func (c *MockScrapperRunRawQueryCall) Do(f func(context.Context, string) (RawQueryRowIterator, error)) *MockScrapperRunRawQueryCall
Do rewrite *gomock.Call.Do
func (*MockScrapperRunRawQueryCall) DoAndReturn ¶ added in v0.13.7
func (c *MockScrapperRunRawQueryCall) DoAndReturn(f func(context.Context, string) (RawQueryRowIterator, error)) *MockScrapperRunRawQueryCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperRunRawQueryCall) Return ¶ added in v0.13.7
func (c *MockScrapperRunRawQueryCall) Return(arg0 RawQueryRowIterator, arg1 error) *MockScrapperRunRawQueryCall
Return rewrite *gomock.Call.Return
type MockScrapperSqlDialectCall ¶ added in v0.4.7
MockScrapperSqlDialectCall wrap *gomock.Call
func (*MockScrapperSqlDialectCall) Do ¶ added in v0.4.7
func (c *MockScrapperSqlDialectCall) Do(f func() sqldialect.Dialect) *MockScrapperSqlDialectCall
Do rewrite *gomock.Call.Do
func (*MockScrapperSqlDialectCall) DoAndReturn ¶ added in v0.4.7
func (c *MockScrapperSqlDialectCall) DoAndReturn(f func() sqldialect.Dialect) *MockScrapperSqlDialectCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperSqlDialectCall) Return ¶ added in v0.4.7
func (c *MockScrapperSqlDialectCall) Return(arg0 sqldialect.Dialect) *MockScrapperSqlDialectCall
Return rewrite *gomock.Call.Return
type MockScrapperValidateConfigurationCall ¶ added in v0.4.7
MockScrapperValidateConfigurationCall wrap *gomock.Call
func (*MockScrapperValidateConfigurationCall) Do ¶ added in v0.4.7
func (c *MockScrapperValidateConfigurationCall) Do(f func(context.Context) ([]string, error)) *MockScrapperValidateConfigurationCall
Do rewrite *gomock.Call.Do
func (*MockScrapperValidateConfigurationCall) DoAndReturn ¶ added in v0.4.7
func (c *MockScrapperValidateConfigurationCall) DoAndReturn(f func(context.Context) ([]string, error)) *MockScrapperValidateConfigurationCall
DoAndReturn rewrite *gomock.Call.DoAndReturn
func (*MockScrapperValidateConfigurationCall) Return ¶ added in v0.4.7
func (c *MockScrapperValidateConfigurationCall) Return(warnings []string, err error) *MockScrapperValidateConfigurationCall
Return rewrite *gomock.Call.Return
type NoEnoughPermissionsError ¶
type NoEnoughPermissionsError struct {
Err error
}
func NewNoEnoughPermissionsError ¶
func NewNoEnoughPermissionsError(err error) *NoEnoughPermissionsError
func (*NoEnoughPermissionsError) Error ¶
func (r *NoEnoughPermissionsError) Error() string
type NumericCategory ¶ added in v0.21.0
type NumericCategory string
NumericCategory is the kind of number a column holds, which decides how many significant digits its values carry. It is the one answer to that question for every surface that describes a column: a query's shape (QueryShapeColumn.NumericCategory) and a table's catalog (CatalogColumnRow.NumericCategory) both come from NumericCategoryOf.
const ( // NumericCategoryNone is a column that does not hold a number, or whose // type is not recognised. NumericCategoryNone NumericCategory = "" // NumericCategoryInteger is a whole number: an integer type, or an exact // numeric declared with scale 0. NumericCategoryInteger NumericCategory = "integer" // NumericCategoryDecimal is an exact decimal that may hold fractions, // sized by its precision and scale when they are known. NumericCategoryDecimal NumericCategory = "decimal" // NumericCategoryFloat is a double-precision binary float, about 15 // significant digits. NumericCategoryFloat NumericCategory = "float" // NumericCategorySinglePrecisionFloat is a single-precision binary float, // about 7 significant digits. NumericCategorySinglePrecisionFloat NumericCategory = "single_precision_float" )
func NumericCategoryOf ¶ added in v0.21.0
func NumericCategoryOf(kind ValueKind, scale *int64, singlePrecision bool) NumericCategory
NumericCategoryOf is what kind of number a column of kind holds. scale is its declared scale when known, and singlePrecision is IsSinglePrecisionFloat (or what the driver reported) for a float.
An exact numeric declared with scale 0 holds only whole numbers and is an integer: Snowflake declares every integer type as NUMBER(38, 0), and Oracle's NUMBER(19) is one too. One whose scale is not known may hold fractions, and stays a decimal.
type PlatformUser ¶ added in v0.21.0
type PlatformUser struct {
// Login is the name the platform authenticates, as the platform shows it.
// It is the same string query history reports as the user who ran a
// statement, so the two can be joined.
Login string `json:"login"`
// PlatformId is the platform's own stable id for the user, where it has
// one that survives a rename (Snowflake USER_ID, Databricks SCIM id,
// Postgres oid).
PlatformId string `json:"platform_id,omitempty"`
// Type is the kind of login as the platform states it; see the
// PlatformUserType* constants for the platforms that have one.
Type string `json:"type,omitempty"`
// Email the platform holds for the user.
Email string `json:"email,omitempty"`
// DisplayName is the user's name for humans, where it differs from Login.
DisplayName string `json:"display_name,omitempty"`
// Comment is the free-text description an administrator left on the user.
// It often says what a service login is for.
Comment string `json:"comment,omitempty"`
// Disabled is true when the login exists but cannot sign in (disabled,
// locked or expired, as the platform reports it). Nil when unknown.
Disabled *bool `json:"disabled,omitempty"`
// CreatedAt is when the user was created. Nil when unknown.
CreatedAt *time.Time `json:"created_at,omitempty"`
// LastLoginAt is the user's last successful sign-in. Nil when unknown or
// never.
LastLoginAt *time.Time `json:"last_login_at,omitempty"`
// DefaultRole is the role a session starts with.
DefaultRole string `json:"default_role,omitempty"`
// Roles are the roles or groups granted to the user directly, sorted and
// without duplicates. Roles say a lot about what a login is for
// (LOADER, TRANSFORMER, REPORTER).
Roles []string `json:"roles,omitempty"`
}
PlatformUser is one login on a platform instance, with every fact the platform states about it. Only Login is always set; every other field is best effort and stays empty when the platform does not state it or the connecting role may not read it (the listing's SkippedFacts says which).
func (*PlatformUser) HasValidIdentity ¶ added in v0.21.0
func (u *PlatformUser) HasValidIdentity() bool
HasValidIdentity reports whether the login is usable as an identity: set, free of NUL bytes and valid UTF-8. A repaired login could name another user, so the rejecting decorator drops such a user instead of sanitizing it.
func (*PlatformUser) Sanitize ¶ added in v0.21.0
func (u *PlatformUser) Sanitize()
type PlatformUserFact ¶ added in v0.21.0
type PlatformUserFact string
PlatformUserFact names one fact of a PlatformUser, so a listing can say which facts it could not read.
const ( PlatformUserFactPlatformId PlatformUserFact = "platform_id" PlatformUserFactType PlatformUserFact = "type" PlatformUserFactEmail PlatformUserFact = "email" PlatformUserFactDisplayName PlatformUserFact = "display_name" PlatformUserFactComment PlatformUserFact = "comment" PlatformUserFactDisabled PlatformUserFact = "disabled" PlatformUserFactCreatedAt PlatformUserFact = "created_at" PlatformUserFactLastLoginAt PlatformUserFact = "last_login_at" PlatformUserFactDefaultRole PlatformUserFact = "default_role" PlatformUserFactRoles PlatformUserFact = "roles" )
type PlatformUserListing ¶ added in v0.21.0
type PlatformUserListing struct {
// Source names where the listing was read, as <platform>.<source>
// ("snowflake.account_usage.users", "bigquery.iam_policy"). A reconciled
// listing has PlatformUsersReconciledSource.
Source string `json:"source"`
// Kind says whether the source is SQL or an API. Empty on a reconciled
// listing.
Kind PlatformUserSourceKind `json:"kind,omitempty"`
// Refused is set, with the platform's error, when the source refused the
// connecting role: a grant would let it answer.
Refused string `json:"refused,omitempty"`
// source (a view, a column or an API it lacks): no grant fixes it.
Unavailable string `json:"unavailable,omitempty"`
// Failed is set when reading the source failed for any other reason (a
// timeout, a 5xx, a dropped connection): a later attempt may answer.
//
// A listing with any of Refused, Unavailable or Failed set holds no users
// and claims no completeness; see Answered.
Failed string `json:"failed,omitempty"`
// Users sorted by Login, one per login.
Users []*PlatformUser `json:"users"`
// Completeness says how much of the platform's users the listing holds.
Completeness PlatformUsersCompleteness `json:"completeness,omitempty"`
// CompletenessReason explains a limited, empty or unknown listing, and
// names the grant that would complete it where one would.
CompletenessReason string `json:"completeness_reason,omitempty"`
// SkippedFacts lists the facts the listing could not read in full, with
// the kind of skip and the reason: absent on the platform, refused for
// every user, hidden for the users the role may not see the details of
// (Snowflake SHOW USERS), or read only in part (Redshift group roles
// without its RBAC roles). A fact a user simply does not have (a user
// without an email) is not listed.
SkippedFacts []SkippedPlatformUserFact `json:"skipped_facts,omitempty"`
// contains filtered or unexported fields
}
PlatformUserListing is what one source said about the platform's users, or, from PlatformUsers.Reconcile, what all of them said together.
func PlatformUserSourceError ¶ added in v0.21.0
func PlatformUserSourceError( source string, kind PlatformUserSourceKind, err error, isPermission, isUnavailable func(error) bool, ) *PlatformUserListing
PlatformUserSourceError records a source that could not be read, in the state its error calls for: refused when isPermission says the role was refused, unavailable when isUnavailable says the platform has no such source, failed otherwise. Either classifier may be nil.
func RefusedPlatformUserSource ¶ added in v0.21.0
func RefusedPlatformUserSource(source string, kind PlatformUserSourceKind, err error) *PlatformUserListing
RefusedPlatformUserSource records a source that refused the connecting role.
func UnavailablePlatformUserSource ¶ added in v0.21.0
func UnavailablePlatformUserSource(source string, kind PlatformUserSourceKind, err error) *PlatformUserListing
UnavailablePlatformUserSource records a source this platform's version or edition does not have.
func (*PlatformUserListing) Answered ¶ added in v0.21.0
func (p *PlatformUserListing) Answered() bool
Answered reports whether the source was read: it is not refused, unavailable or failed.
func (*PlatformUserListing) AssignRoles ¶ added in v0.21.0
func (p *PlatformUserListing) AssignRoles(rolesByLogin map[string][]string)
AssignRoles adds roles to the users they belong to, keyed by login. Roles of a login that is not in the listing are dropped: a grant to a user we did not list does not add a user.
func (*PlatformUserListing) DependentSkipKind ¶ added in v0.21.0
func (p *PlatformUserListing) DependentSkipKind() PlatformUserSkipKind
DependentSkipKind is how another source skips a fact only this source states: the way this source did not answer, so a grant is asked for when it was refused, and unavailable when it answered or this platform lacks it, since the other source never states the fact itself. A nil source counts as unavailable.
func (*PlatformUserListing) Finish ¶ added in v0.21.0
func (p *PlatformUserListing) Finish() *PlatformUserListing
Finish puts a listing into its final shape: users without a login are dropped, users sharing a login are merged (the first non-empty value of each fact wins, roles are united), roles are sorted and deduplicated, users are sorted by login, and an empty listing is marked PlatformUsersEmpty whatever completeness was claimed. Every implementation returns through it.
func (*PlatformUserListing) IsSkipped ¶ added in v0.21.0
func (p *PlatformUserListing) IsSkipped(fact PlatformUserFact) bool
IsSkipped reports whether fact was recorded as skipped.
func (*PlatformUserListing) Sanitize ¶ added in v0.21.0
func (p *PlatformUserListing) Sanitize()
Sanitize cleans every string of the listing (see SanitizeString).
func (*PlatformUserListing) Skip ¶ added in v0.21.0
func (p *PlatformUserListing) Skip(fact PlatformUserFact, kind PlatformUserSkipKind, reason string)
Skip records that fact could not be read in full, with the kind of skip and the reason. A fact skipped twice keeps its first kind and reason.
func (*PlatformUserListing) SkippedFact ¶ added in v0.21.0
func (p *PlatformUserListing) SkippedFact(fact PlatformUserFact) (SkippedPlatformUserFact, bool)
SkippedFact returns how fact was skipped, and whether it was.
type PlatformUserSkipKind ¶ added in v0.21.0
type PlatformUserSkipKind string
PlatformUserSkipKind says what would let a skipped fact be read, in the same three states a source that did not answer has, so a caller can decide what to recommend without reading the reason.
const ( // PlatformUserSkipRefused: the connecting role may not read the fact, for // every user or for some of them. A grant would let it, and the reason // names the grant where the platform has one name for it. PlatformUserSkipRefused PlatformUserSkipKind = "refused" // have the fact, or this source does not state it. No grant fixes it. PlatformUserSkipUnavailable PlatformUserSkipKind = "unavailable" // PlatformUserSkipFailed: reading the fact failed for any other reason (a // timeout, a 5xx, a dropped connection), so a later run may read it. PlatformUserSkipFailed PlatformUserSkipKind = "failed" )
func PlatformUserSkipKindOf ¶ added in v0.21.0
func PlatformUserSkipKindOf(err error, isPermission, isUnavailable func(error) bool) PlatformUserSkipKind
PlatformUserSkipKindOf classifies the error a fact query failed with, the way PlatformUserSourceError classifies a source's: refused when isPermission says so, unavailable when isUnavailable does, failed otherwise. Either classifier may be nil.
type PlatformUserSourceKind ¶ added in v0.21.0
type PlatformUserSourceKind string
PlatformUserSourceKind says how a source of platform users was read.
const ( // PlatformUserSourceSQL is a catalog view or command read over the // warehouse connection. PlatformUserSourceSQL PlatformUserSourceKind = "sql" // PlatformUserSourceAPI is the platform's REST API (Databricks SCIM, Google // IAM, Fabric). PlatformUserSourceAPI PlatformUserSourceKind = "api" )
type PlatformUsers ¶ added in v0.21.0
type PlatformUsers struct {
// Sources in the order the platform trusts them, most trusted first:
// Reconcile takes a fact from the first source that has it.
Sources []*PlatformUserListing `json:"sources"`
}
PlatformUsers is the result of QueryPlatformUsers: what each source the platform has said, kept apart so a caller can weigh one against another (the warehouse's own view against the platform's API, a complete source against a fresher one). Reconcile merges them for a caller that wants one list.
A platform reads every source that can add a user or a fact. A fallback that only repeats a subset of another source (Oracle ALL_USERS next to DBA_USERS) is read when that source did not answer, and the source that did not answer is kept with the reason, so a caller sees what was missed and why.
func CollectPlatformUsers ¶ added in v0.21.0
func CollectPlatformUsers(ctx context.Context, sources ...*PlatformUserListing) (*PlatformUsers, error)
CollectPlatformUsers is how QueryPlatformUsers returns. A source that did not answer is a state of the result, not an error of the call: the call fails only when the context is done, or when no source answered and at least one of them failed outright, which is a real failure of the connection (a wrong password, an unreachable host) rather than something the platform or a grant decides. Every source refused or unavailable is a result, whose Reconcile says so.
func NewPlatformUsers ¶ added in v0.21.0
func NewPlatformUsers(sources ...*PlatformUserListing) *PlatformUsers
NewPlatformUsers finishes each source (see PlatformUserListing.Finish) and returns them as one result.
func (*PlatformUsers) Answered ¶ added in v0.21.0
func (p *PlatformUsers) Answered() bool
Answered reports whether any source was read.
func (*PlatformUsers) Reconcile ¶ added in v0.21.0
func (p *PlatformUsers) Reconcile() *PlatformUserListing
Reconcile merges the sources into one listing. It is the default reading, for a caller that has no reason to weigh the sources itself:
- a user is listed when any source lists it;
- each fact is taken from the first source, in trust order, that has it, and roles are united across sources;
- the listing is as complete as its most complete source, since a complete source saw every user: complete, else limited, else unknown;
- a fact is skipped only when every source that answered skipped it;
- a source that did not answer adds why to the reason of a listing that is not complete;
- when no source answered, the reconciled listing did not either: it is refused if any source refused (a grant would help), else unavailable if any was, else failed, with every source's reason.
func (*PlatformUsers) Sanitize ¶ added in v0.21.0
func (p *PlatformUsers) Sanitize()
Sanitize cleans every string of every source (see SanitizeString).
func (*PlatformUsers) Source ¶ added in v0.21.0
func (p *PlatformUsers) Source(name string) *PlatformUserListing
Source returns the listing of the named source, or nil.
type PlatformUsersCapability ¶ added in v0.21.0
type PlatformUsersCapability struct {
// Supported is true when QueryPlatformUsers may return a listing rather
// than ErrUnsupported.
Supported bool `json:"supported"`
// Grant names, in the platform's own terms, what lets the connecting role
// read every user and every fact, so a caller can quote it in a
// recommendation ("grant X to see all your users").
Grant string `json:"grant,omitempty"`
}
PlatformUsersCapability says whether a platform can list its users, and what a role needs to be granted to see all of them. Its zero value means the platform has no user listing at all, and QueryPlatformUsers returns ErrUnsupported: a caller checks Supported first and never calls it then.
"No listing on this platform" and "a listing our role may not read" are different things to a customer. The first is this capability; the second is a permission error from QueryPlatformUsers that IsPermissionError recognises.
type PlatformUsersCompleteness ¶ added in v0.21.0
type PlatformUsersCompleteness string
PlatformUsersCompleteness says how much of the platform's user list a listing holds, as far as the platform lets us tell.
const ( // PlatformUsersComplete: the listing holds every user of the platform. PlatformUsersComplete PlatformUsersCompleteness = "complete" // PlatformUsersLimited: the connecting role sees only part of the users. // CompletenessReason says why, and names the grant that lifts it. PlatformUsersLimited PlatformUsersCompleteness = "limited" // PlatformUsersEmpty: the listing worked and returned nobody. Every // platform has at least the login we connect with, so this is suspicious // and must not be read as "the platform has no users". PlatformUsersEmpty PlatformUsersCompleteness = "empty" // PlatformUsersUnknown: the platform gives no way to tell whether the // listing is complete. PlatformUsersUnknown PlatformUsersCompleteness = "unknown" )
type QueryEstimate ¶ added in v0.16.0
type QueryEstimate struct {
// BytesScanned is the estimated number of bytes the query would read/process.
BytesScanned *int64
// Rows is a planner estimate of rows processed, intended as a scan-size
// proxy. Where the engine exposes a plan tree (Postgres) this is the
// largest per-node estimate — i.e. the heaviest scan — rather than the
// final result cardinality, which collapses to 1 for aggregates.
Rows *int64
// Exact is true only when authoritative rather than a planner estimate
// (BigQuery dry-run: TotalBytesProcessed, with TotalBytesProcessedAccuracy).
Exact bool
}
QueryEstimate is a pre-execution estimate of what a SELECT will scan. It is produced from engine metadata (dry-run / EXPLAIN) — the query itself is never executed. Nil fields mean the engine could not estimate that dimension.
type QueryShapeColumn ¶ added in v0.8.3
type QueryShapeColumn struct {
Name string `json:"name"`
NativeType string `json:"native_type"`
Position int32 `json:"position"`
// Kind is what NativeType means on this warehouse, filled in by
// RunRawQuery and QueryShape. Pass it to SqlLiteral to write a value of
// this column back into SQL. KindUnknown when the type is not one
// NativeValueKind recognises.
Kind ValueKind `json:"kind,omitempty"`
// Precision and Scale are an exact numeric column's declared size
// (KindNumeric), from the driver (sql.ColumnType.DecimalSize) or, when it
// reports none, from NativeType (DECIMAL(20,6)). Both are nil for every
// other kind, and for a numeric whose size is not known: one declared
// without it (a Postgres NUMERIC, an Oracle NUMBER), or one whose driver
// reports neither and names it bare (Databricks, Athena). A nil scale is
// not a scale of 0.
Precision *int64 `json:"precision,omitempty"`
Scale *int64 `json:"scale,omitempty"`
// SinglePrecision marks a KindFloat column that holds a single-precision
// float (IsSinglePrecisionFloat), about 7 significant digits rather than a
// double's 15. Always false for every other kind.
SinglePrecision bool `json:"single_precision,omitempty"`
}
func (*QueryShapeColumn) HasValidIdentity ¶ added in v0.13.2
func (c *QueryShapeColumn) HasValidIdentity() bool
func (*QueryShapeColumn) NumericCategory ¶ added in v0.21.0
func (c *QueryShapeColumn) NumericCategory() NumericCategory
NumericCategory is what kind of number the column holds (NumericCategoryOf).
func (*QueryShapeColumn) Sanitize ¶ added in v0.13.2
func (c *QueryShapeColumn) Sanitize()
func (*QueryShapeColumn) SetDecimalSize ¶ added in v0.21.0
func (c *QueryShapeColumn) SetDecimalSize(precision, scale int64, ok bool) *QueryShapeColumn
SetDecimalSize records an exact numeric column's precision and scale, as sql.ColumnType.DecimalSize reports them, and returns c. Kind must be set first: any other kind is left without a size, since drivers report one for timestamps and floats too, where it means something else.
When the driver reports nothing, the size is read from NativeType. A size no column can have is not recorded, which is how the drivers describe a numeric declared without one: lib/pq reports an unconstrained NUMERIC as 65535 digits, go-mssqldb an unknown size as math.MaxInt64, and go-ora a NUMBER without precision as (38, 255). A scale above the precision is one of those, though Postgres 15 can declare it: such a column is reported without a size rather than with a wrong one.
func (*QueryShapeColumn) SetFloatLength ¶ added in v0.21.0
func (c *QueryShapeColumn) SetFloatLength(length int64, ok bool) *QueryShapeColumn
SetFloatLength records whether a KindFloat column is single precision, from its storage length in bytes as sql.ColumnType.Length reports it, and returns c. A length of 4 is a single-precision float and 8 a double, whatever the type is called: go-db2 names REAL and DOUBLE both FLOAT, and tells them apart only by length. Any other length, or none, leaves what the native type name said (IsSinglePrecisionFloat).
type QueryTablesConfig ¶ added in v0.10.6
type QueryTablesConfig struct {
IncludeConstraints bool
}
QueryTablesConfig holds options for QueryTables.
func ApplyQueryTablesOptions ¶ added in v0.10.6
func ApplyQueryTablesOptions(opts ...QueryTablesOption) QueryTablesConfig
ApplyQueryTablesOptions applies the given options to a QueryTablesConfig.
type QueryTablesOption ¶ added in v0.10.6
type QueryTablesOption func(*QueryTablesConfig)
QueryTablesOption configures QueryTables behavior.
func WithConstraints ¶ added in v0.10.6
func WithConstraints() QueryTablesOption
WithConstraints instructs QueryTables to include table constraints (partitioning, clustering, etc.) in the returned TableRow.Constraints field. This avoids the need for a separate QueryTableConstraints call when the scrapper already fetches the necessary metadata as part of QueryTables.
type RawQueryRowIterator ¶ added in v0.13.7
type RawQueryRowIterator interface {
// Columns describes the shape of the result. Stable for the lifetime of
// the iterator — valid before the first Next call and unchanged after.
Columns() []*QueryShapeColumn
// Next returns the next row or io.EOF when exhausted.
//
// The returned slice is positional with Columns(); each cell is a fresh
// *ColumnValue the caller may retain. NULL cells have IsNull=true and
// Value=nil.
//
// Implementations MUST auto-close resources before returning io.EOF.
// Safe to call after EOF (keeps returning io.EOF).
Next(ctx context.Context) ([]*ColumnValue, error)
// Close releases the underlying cursor / driver rows / warehouse job.
// Safe to call multiple times.
Close() error
}
RawQueryRowIterator streams typed rows from RunRawQuery. Semantics mirror querylogs.QueryLogIterator: Next returns io.EOF when exhausted, auto-closes on EOF, and Close is idempotent.
type SchemaColumnField ¶
type SchemaColumnField struct {
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
// Human readable name of the column as present in dbt or data warehouse.
HumanName string `protobuf:"bytes,2,opt,name=human_name,json=humanName,proto3" json:"human_name,omitempty"`
// Native data type of the column as present in data warehouse.
NativeType string `protobuf:"bytes,4,opt,name=native_type,json=nativeType,proto3" json:"native_type,omitempty"`
// Description of the column
Description *string `protobuf:"bytes,5,opt,name=description,proto3" json:"description,omitempty"`
// Ordinal position of the column in the table, starting from 1
OrdinalPosition int32 `protobuf:"varint,6,opt,name=ordinal_position,json=ordinalPosition,proto3" json:"ordinal_position,omitempty"`
// Indicates that the column type could be used as a struct/json in a data warehouse
IsStruct bool `protobuf:"varint,7,opt,name=is_struct,json=isStruct,proto3" json:"is_struct,omitempty"`
// Indicates that the column is a repeated field in a data warehouse (e.g. array)
IsRepeated bool `protobuf:"varint,8,opt,name=is_repeated,json=isRepeated,proto3" json:"is_repeated,omitempty"`
// Fields inside of the struct/record like column
Fields []*SchemaColumnField `protobuf:"bytes,9,rep,name=fields,proto3" json:"fields,omitempty"`
}
func (*SchemaColumnField) HasValidIdentity ¶ added in v0.13.2
func (f *SchemaColumnField) HasValidIdentity() bool
func (*SchemaColumnField) Sanitize ¶ added in v0.13.2
func (f *SchemaColumnField) Sanitize()
type SchemaRow ¶ added in v0.14.5
type SchemaRow struct {
Instance string `db:"instance" json:"instance" ch:"instance" bigquery:"instance"`
Database string `db:"database" json:"database" ch:"_database" bigquery:"database"`
Schema string `db:"schema" json:"schema" ch:"schema" bigquery:"schema"`
Description *string `db:"description" json:"description" ch:"description" bigquery:"description"`
SchemaType *string `db:"schema_type" json:"schema_type" ch:"schema_type" bigquery:"schema_type"`
SchemaOwner *string `db:"schema_owner" json:"schema_owner" ch:"schema_owner" bigquery:"schema_owner"`
}
SchemaRow describes a single schema (or the warehouse concept we map to a schema — e.g. a ClickHouse database, a BigQuery dataset, an Oracle user). It mirrors DatabaseRow one level deeper in the (database, schema) hierarchy.
func (*SchemaRow) HasValidIdentity ¶ added in v0.14.5
func (*SchemaRow) SetInstance ¶ added in v0.14.5
type Scrapper ¶
type Scrapper interface {
DialectType() string
SqlDialect() sqldialect.Dialect
IsPermissionError(err error) bool
// Capabilities returns what this scrapper supports beyond the base interface.
Capabilities() Capabilities
ValidateConfiguration(ctx context.Context) (warnings []string, err error)
QueryCatalog(ctx context.Context) ([]*CatalogColumnRow, error)
QueryTableMetrics(ctx context.Context, lastMetricsFetchTime time.Time) ([]*TableMetricsRow, error)
QuerySqlDefinitions(ctx context.Context) ([]*SqlDefinitionRow, error)
QueryTables(ctx context.Context, opts ...QueryTablesOption) ([]*TableRow, error)
QueryDatabases(ctx context.Context) ([]*DatabaseRow, error)
// QuerySchemas lists the schemas (or the warehouse concept mapped to a
// schema, e.g. ClickHouse databases, BigQuery datasets, Oracle users)
// visible to the connection. Scope filtering is driven by the context
// (scope.WithScope) exactly like QueryTables, not by a parameter.
QuerySchemas(ctx context.Context) ([]*SchemaRow, error)
QuerySegments(ctx context.Context, sql string, args ...any) ([]*SegmentRow, error)
QueryCustomMetrics(ctx context.Context, sql string, args ...any) ([]*CustomMetricsRow, error)
QueryShape(ctx context.Context, sql string) ([]*QueryShapeColumn, error)
// RunRawQuery executes an arbitrary user-supplied SELECT and returns a
// streaming iterator over typed rows. Unlike QueryCustomMetrics it does
// not filter segment* columns and does not collapse text columns to
// IgnoredValue — it is designed for generic "run this query" surfaces
// rather than the agent's metrics/profile/segments path.
//
// The caller must Close() the iterator. Callers that want a row cap
// should stop iteration after N calls to Next().
RunRawQuery(ctx context.Context, sql string) (RawQueryRowIterator, error)
// EstimateQuery returns a pre-execution scan estimate for an arbitrary
// SELECT, or ErrUnsupported when the dialect cannot estimate reliably.
// It must obtain the estimate from engine metadata (dry-run / EXPLAIN) and
// MUST NOT execute the query. Consult Capabilities().EstimateQuery to learn
// which dimensions (bytes vs rows) a given dialect can populate.
EstimateQuery(ctx context.Context, sql string) (*QueryEstimate, error)
QueryTableConstraints(ctx context.Context) ([]*TableConstraintRow, error)
// QueryPlatformUsers lists the platform's users (logins) with every fact
// the platform states about them, one listing per source. It returns
// ErrUnsupported when Capabilities().PlatformUsers is not supported.
// Otherwise a source the role may not read, that this version lacks, or
// that failed is a state of the result (PlatformUserListing.Refused,
// Unavailable, Failed), and a fact that cannot be read is skipped with the
// reason: the call fails only on a real failure of the connection (see
// CollectPlatformUsers). Users are not scoped: scope.WithScope selects
// tables, not logins.
QueryPlatformUsers(ctx context.Context) (*PlatformUsers, error)
// This will close underlying execer, such scrapper can't be used anymore
Close() error
}
type SegmentRow ¶
func (*SegmentRow) HasValidIdentity ¶ added in v0.13.2
func (r *SegmentRow) HasValidIdentity() bool
func (*SegmentRow) Sanitize ¶ added in v0.13.2
func (r *SegmentRow) Sanitize()
type SegmentValue ¶ added in v0.1.7
func (*SegmentValue) HasValidIdentity ¶ added in v0.13.2
func (v *SegmentValue) HasValidIdentity() bool
func (*SegmentValue) Sanitize ¶ added in v0.13.2
func (v *SegmentValue) Sanitize()
type SkippedPlatformUserFact ¶ added in v0.21.0
type SkippedPlatformUserFact struct {
Fact PlatformUserFact `json:"fact"`
// Kind says whether a grant would let the fact be read. Empty only on a
// listing stored before skips had a kind, which says nothing either way.
Kind PlatformUserSkipKind `json:"kind,omitempty"`
Reason string `json:"reason"`
}
SkippedPlatformUserFact says that a fact could not be read, what kind of skip it is, and why.
type SqlDefinitionRow ¶
type SqlDefinitionRow struct {
Instance string `db:"instance" json:"instance" ch:"instance" bigquery:"instance"`
Database string `db:"database" json:"database" ch:"_database" bigquery:"database"`
Schema string `db:"schema" json:"schema" ch:"schema" bigquery:"schema"`
Table string `db:"table" json:"table" ch:"table" bigquery:"table"`
IsView bool `db:"is_view" json:"is_view" ch:"is_view" bigquery:"is_view"`
IsTable bool `db:"is_table" json:"is_table" ch:"is_table" bigquery:"is_table"`
IsMaterializedView bool `db:"is_materialized_view" json:"is_materialized_view" ch:"is_materialized_view" bigquery:"is_materialized_view"`
TableType string `db:"table_type" json:"table_type" ch:"table_type" bigquery:"table_type"`
Sql string `db:"sql" json:"sql" ch:"sql" bigquery:"sql"`
Description *string ` json:"description,omitempty"`
Tags []*Tag ` json:"tags,omitempty"`
}
func (*SqlDefinitionRow) HasValidIdentity ¶ added in v0.13.2
func (r *SqlDefinitionRow) HasValidIdentity() bool
func (*SqlDefinitionRow) Sanitize ¶ added in v0.13.2
func (r *SqlDefinitionRow) Sanitize()
func (SqlDefinitionRow) TableFqn ¶
func (r SqlDefinitionRow) TableFqn() DwhFqn
type StringValue ¶ added in v0.13.7
type StringValue string
StringValue preserves text values verbatim. Used by RunRawQuery so generic "run this query" surfaces see string columns as strings rather than having them collapsed to IgnoredValue (as QueryCustomMetrics does, which is correct for the metrics/profile agent path but wrong for raw data preview).
type TableChangeEvent ¶ added in v0.9.2
type TableChangeEvent struct {
// When the change was committed. Use for time-travel AS OF.
Timestamp time.Time
// Database-specific version identifier.
// Databricks: Delta version number. BigQuery: job_id. Snowflake: empty (hourly bucket).
Version string
// Type of operation: "INSERT", "UPDATE", "DELETE", "MERGE", "COPY", "TRUNCATE", "OTHER"
Operation string
// Row counts (nil if not available)
RowsInserted *int64
RowsUpdated *int64
RowsDeleted *int64
}
TableChangeEvent represents a single modification event on a table.
type TableChangeHistoryProvider ¶ added in v0.9.2
type TableChangeHistoryProvider interface {
FetchTableChangeHistory(
ctx context.Context,
fqn DwhFqn,
from, to time.Time,
limit int,
) ([]*TableChangeEvent, error)
}
TableChangeHistoryProvider returns recent modification events for a table, ordered by timestamp descending (most recent first).
Enables constructing time-travel queries:
AT(TIMESTAMP => event.Timestamp - 1s) -- Snowflake FOR SYSTEM_TIME AS OF event.Timestamp -- BigQuery TIMESTAMP AS OF event.Timestamp -- Databricks
type TableConstraintRow ¶ added in v0.9.0
type TableConstraintRow struct {
Instance string `db:"instance" json:"instance" ch:"instance" bigquery:"instance"`
Database string `db:"database" json:"database" ch:"_database" bigquery:"database"`
Schema string `db:"schema" json:"schema" ch:"schema" bigquery:"schema"`
Table string `db:"table" json:"table" ch:"table" bigquery:"table"`
ConstraintName string `db:"constraint_name" json:"constraint_name" ch:"constraint_name" bigquery:"constraint_name"`
ColumnName string `db:"column_name" json:"column_name" ch:"column_name" bigquery:"column_name"`
ConstraintType string `db:"constraint_type" json:"constraint_type" ch:"constraint_type" bigquery:"constraint_type"`
ColumnPosition int32 `db:"column_position" json:"column_position" ch:"column_position" bigquery:"column_position"`
ConstraintExpression string `db:"constraint_expression" json:"constraint_expression" ch:"constraint_expression" bigquery:"constraint_expression"`
IsEnforced *bool `db:"is_enforced" json:"is_enforced" ch:"is_enforced" bigquery:"is_enforced"`
}
TableConstraintRow represents a constraint or key on a table column. This covers traditional indexes, primary keys, unique constraints, as well as warehouse-specific concepts like BigQuery partitioning/clustering, ClickHouse sorting keys, and Snowflake clustering keys.
func (*TableConstraintRow) HasValidIdentity ¶ added in v0.13.2
func (r *TableConstraintRow) HasValidIdentity() bool
func (*TableConstraintRow) Sanitize ¶ added in v0.13.2
func (r *TableConstraintRow) Sanitize()
func (TableConstraintRow) TableFqn ¶ added in v0.9.0
func (r TableConstraintRow) TableFqn() DwhFqn
type TableMetricsRow ¶
type TableMetricsRow struct {
Instance string `db:"instance" json:"instance" ch:"instance" bigquery:"instance"`
Database string `db:"database" json:"database" ch:"_database" bigquery:"database"`
Schema string `db:"schema" json:"schema" ch:"schema" bigquery:"schema"`
Table string `db:"table" json:"table" ch:"table" bigquery:"table"`
RowCount *int64 `db:"row_count" json:"row_count" ch:"row_count" bigquery:"row_count"`
UpdatedAt *time.Time `db:"updated_at" json:"updated_at" ch:"updated_at" bigquery:"updated_at"`
SizeBytes *int64 `db:"size_bytes" json:"size_bytes" ch:"size_bytes" bigquery:"size_bytes"`
}
func (*TableMetricsRow) HasValidIdentity ¶ added in v0.13.2
func (r *TableMetricsRow) HasValidIdentity() bool
func (*TableMetricsRow) Sanitize ¶ added in v0.13.2
func (r *TableMetricsRow) Sanitize()
func (TableMetricsRow) TableFqn ¶
func (r TableMetricsRow) TableFqn() DwhFqn
type TableRow ¶
type TableRow struct {
Instance string `db:"instance" json:"instance" ch:"instance" bigquery:"instance"`
Database string `db:"database" json:"database" ch:"_database" bigquery:"database"`
Schema string `db:"schema" json:"schema" ch:"schema" bigquery:"schema"`
Table string `db:"table" json:"table" ch:"table" bigquery:"table"`
TableType string `db:"table_type" json:"table_type" ch:"table_type" bigquery:"table_type"`
Description *string `db:"description" json:"description" ch:"description" bigquery:"description"`
Tags []*Tag `db:"tags" json:"tags"`
IsView bool `db:"is_view" json:"is_view" ch:"is_view"`
IsTable bool `db:"is_table" json:"is_table" ch:"is_table"`
IsMaterializedView bool `db:"is_materialized_view" json:"is_materialized_view" ch:"is_materialized_view"`
Options map[string]interface{} `db:"options" json:"options"`
Annotations []*Annotation `db:"annotations" json:"annotations"`
// Constraints is optionally populated by QueryTables when WithConstraints() is passed.
// When non-nil, callers may skip a separate QueryTableConstraints call.
Constraints []*TableConstraintRow `json:"constraints,omitempty"`
}
func (*TableRow) HasValidIdentity ¶ added in v0.13.2
type Tag ¶
type Tag struct {
// Optionally prefix (`tag.` / `policytag.` / etc)
TagName string `json:"tag_name"`
TagValue string `json:"tag_value"`
}
func (*Tag) HasValidIdentity ¶ added in v0.13.2
type Unwrapper ¶ added in v0.13.3
type Unwrapper interface {
Unwrap() Scrapper
}
Unwrapper is implemented by Scrapper decorators (sanitize, reject, scope, ...) so callers can walk to an inner scrapper of a specific type. It mirrors the errors.Unwrap idiom: together with As it lets code type-assert past arbitrary decorator chains without each decorator having to know about every interface a consumer might care about.
type Value ¶
type Value interface {
// contains filtered or unexported methods
}
func DecimalValueFromRat ¶ added in v0.19.0
DecimalValueFromRat decodes an exact rational (BigQuery NUMERIC and BIGNUMERIC arrive as *big.Rat) the way DecimalValueFromText decodes text. The fraction is written with the fewest digits that represent it exactly, which for a base-10 warehouse decimal is at most its scale.
func DecimalValueFromText ¶ added in v0.19.0
DecimalValueFromText decodes the text of an exact numeric (NUMERIC, DECIMAL, NUMBER, Snowflake's FIXED) without going through float64: an integer becomes IntValue, or BigIntValue when it does not fit, and anything with a fractional part stays StringValue with its digits exactly as the warehouse wrote them. Text that is not a plain decimal number (Postgres' 'NaN', an exponent) also stays StringValue.
func MetricValueFromText ¶ added in v0.19.0
MetricValueFromText is how the metrics path (QueryCustomMetrics) reads a cell a driver handed back as text: an integer, a float, a boolean (as 1/0), or a timestamp in RFC 3339 or "2006-01-02 15:04:05.999999999". Anything else is IgnoredValue.
It is deliberately narrower than ParseTimestamp. The metrics path returns metrics, not the data they were computed from, and callers rely on it never handing back text; each form it learns to parse is more of a table's contents it returns. A caller that needs a value read back as it is uses RunRawQuery.
type ValueKind ¶ added in v0.19.0
type ValueKind string
ValueKind is the family of warehouse types a column belongs to, as far as reading its values and writing them back as literals is concerned. It is what separates cases a Value alone cannot: a TimeValue from a DATE column needs a date literal, and a StringValue from a UUID or NUMERIC column needs a literal of that type on the engines that will not compare it with text.
const ( KindUnknown ValueKind = "" // KindTimestamp is a timestamp without a zone (TIMESTAMP_NTZ, DATETIME, // DATETIME2, Postgres TIMESTAMP). Its TimeValue is the wall clock, in UTC. KindTimestamp ValueKind = "timestamp" // KindTimestampTz is an instant (TIMESTAMPTZ, TIMESTAMP_TZ/_LTZ, // DATETIMEOFFSET, BigQuery TIMESTAMP, ClickHouse DateTime). KindTimestampTz ValueKind = "timestamptz" KindDate ValueKind = "date" KindInteger ValueKind = "integer" // KindNumeric is an exact decimal (NUMERIC, DECIMAL, NUMBER). A fractional // value arrives as StringValue holding its exact digits. KindNumeric ValueKind = "numeric" KindFloat ValueKind = "float" KindBool ValueKind = "bool" KindText ValueKind = "text" KindUUID ValueKind = "uuid" )
func NativeValueKind ¶ added in v0.19.0
NativeValueKind maps a column's native type, as a driver reports it (QueryShapeColumn.NativeType from RunRawQuery or QueryShape), to its ValueKind. dialect is Scrapper.DialectType(), because the same name means different things on different engines: TIMESTAMP is an instant on BigQuery and Databricks and a wall clock everywhere else, and DATE carries a time of day on Oracle.
Type parameters and ClickHouse's Nullable/LowCardinality wrappers are ignored, so "DECIMAL(20,6)", "timestamp(6) with time zone" and "Nullable(DateTime64(6, 'UTC'))" all resolve. An unrecognised type is KindUnknown.
type WithSetInstance ¶
type WithSetInstance interface {
SetInstance(instance string)
}
Source Files
¶
Directories
¶
| Path | Synopsis |
|---|---|
|
Package athena implements the scrapper.Scrapper interface for Amazon Athena.
|
Package athena implements the scrapper.Scrapper interface for Amazon Athena. |
|
Package fabric implements a scrapper for Microsoft Fabric Warehouses and Lakehouse SQL analytics endpoints.
|
Package fabric implements a scrapper for Microsoft Fabric Warehouses and Lakehouse SQL analytics endpoints. |
|
Package reject provides a Scrapper decorator that drops rows whose identity fields (the parts forming an object's fully-qualified name) contain NUL bytes or invalid UTF-8.
|
Package reject provides a Scrapper decorator that drops rows whose identity fields (the parts forming an object's fully-qualified name) contain NUL bytes or invalid UTF-8. |
|
Package sanitize provides a Scrapper decorator that strips NUL bytes and repairs invalid UTF-8 in every row returned by the underlying scrapper.
|
Package sanitize provides a Scrapper decorator that strips NUL bytes and repairs invalid UTF-8 in every row returned by the underlying scrapper. |
|
Package scrappertest provides a compliance test suite that can be embedded in warehouse-specific integration test suites to validate that a scrapper implementation conforms to the expected contract.
|
Package scrappertest provides a compliance test suite that can be embedded in warehouse-specific integration test suites to validate that a scrapper implementation conforms to the expected contract. |