Documentation
¶
Index ¶
- func GetObjectNameFromFqn(fqn string) string
- func ParseCommentClause(ddl string) *string
- func ParseCreateStatementsPerObject(ctx context.Context, ddls string) (map[string]string, error)
- func ParseWithTagClause(ddl string) []*scrapper.Tag
- func QuoteLiteral(literal string) string
- func ScopeFromConf(conf *SnowflakeScrapperConf) *scope.ScopeFilter
- func UnQuote(key string) string
- type DatabaseAndSchema
- type DbDesc
- type ShareDesc
- type ShareObject
- type ShowStreamsRow
- type SnowflakeQueryLogSchema
- type SnowflakeScrapper
- func (e *SnowflakeScrapper) Capabilities() scrapper.Capabilities
- func (e *SnowflakeScrapper) Close() error
- func (e *SnowflakeScrapper) DialectType() string
- func (e *SnowflakeScrapper) Executor() *dwhexecsnowflake.SnowflakeExecutor
- func (s *SnowflakeScrapper) FetchQueryLogs(ctx context.Context, from, to time.Time, obfuscator querylogs.QueryObfuscator) (querylogs.QueryLogIterator, error)
- func (e *SnowflakeScrapper) FetchTableChangeHistory(ctx context.Context, fqn scrapper.DwhFqn, from, to time.Time, limit int) ([]*scrapper.TableChangeEvent, error)
- func (e *SnowflakeScrapper) GetDatabasesToQuery(ctx context.Context) ([]string, error)
- func (e *SnowflakeScrapper) GetExistingDbs(ctx context.Context) ([]*DbDesc, error)
- func (e *SnowflakeScrapper) IsPermissionError(err error) bool
- func (e *SnowflakeScrapper) QueryCatalog(origCtx context.Context) ([]*scrapper.CatalogColumnRow, error)
- func (e *SnowflakeScrapper) QueryCustomMetrics(ctx context.Context, sql string, args ...any) ([]*scrapper.CustomMetricsRow, error)
- func (e *SnowflakeScrapper) QueryDatabases(ctx context.Context) ([]*scrapper.DatabaseRow, error)
- func (e *SnowflakeScrapper) QuerySegments(ctx context.Context, sql string, args ...any) ([]*scrapper.SegmentRow, error)
- func (e *SnowflakeScrapper) QueryShape(ctx context.Context, sql string) ([]*scrapper.QueryShapeColumn, error)
- func (e *SnowflakeScrapper) QuerySqlDefinitions(origCtx context.Context) ([]*scrapper.SqlDefinitionRow, error)
- func (e *SnowflakeScrapper) QueryTableConstraints(ctx context.Context) ([]*scrapper.TableConstraintRow, error)
- func (e *SnowflakeScrapper) QueryTableMetrics(origCtx context.Context, lastMetricsFetchTime time.Time) ([]*scrapper.TableMetricsRow, error)
- func (e *SnowflakeScrapper) QueryTables(origCtx context.Context, opts ...scrapper.QueryTablesOption) ([]*scrapper.TableRow, error)
- func (e *SnowflakeScrapper) SqlDialect() sqldialect.Dialect
- func (e *SnowflakeScrapper) ValidateConfiguration(ctx context.Context) ([]string, error)
- type SnowflakeScrapperConf
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func GetObjectNameFromFqn ¶ added in v0.4.19
func ParseCommentClause ¶ added in v0.10.5
ParseCommentClause extracts the table-level COMMENT='...' value from a Snowflake DDL statement. It only considers COMMENT tokens that appear after the outermost closing parenthesis of the column definitions, to avoid matching column-level comments.
func ParseCreateStatementsPerObject ¶ added in v0.4.19
func ParseWithTagClause ¶ added in v0.10.5
ParseWithTagClause extracts tags from a Snowflake DDL's WITH TAG (...) clause. It skips UNKNOWN_TAG sentinel entries that indicate insufficient permissions.
func QuoteLiteral ¶
func ScopeFromConf ¶ added in v0.10.0
func ScopeFromConf(conf *SnowflakeScrapperConf) *scope.ScopeFilter
ScopeFromConf translates the Snowflake config's Databases field into a ScopeFilter. Returns nil if no database filtering is configured.
Types ¶
type DatabaseAndSchema ¶ added in v0.4.19
type DbDesc ¶
type ShareObject ¶
type ShareObject struct {
}
type ShowStreamsRow ¶ added in v0.4.12
type ShowStreamsRow struct {
// CreatedOn is the timestamp when the stream was created
CreatedOn string `db:"created_on"`
// Name is the name of the stream
Name string `db:"name"`
// DatabaseName is the database containing the stream
DatabaseName string `db:"database_name"`
// SchemaName is the schema containing the stream
SchemaName string `db:"schema_name"`
// Owner is the owner of the stream
Owner string `db:"owner"`
// Comment is the comment/description for the stream
Comment string `db:"comment"`
// TableName is the name of the source table the stream tracks
TableName string `db:"table_name"`
// SourceType is the type of source object (Table, View, etc.)
SourceType string `db:"source_type"`
// BaseTables lists the base tables the stream tracks
BaseTables string `db:"base_tables"`
// Type is the stream type (DELTA, etc.)
Type string `db:"type"`
// Stale indicates whether the stream is stale (true/false)
Stale string `db:"stale"`
// Mode is the stream mode: DEFAULT (standard/delta), APPEND_ONLY, or INSERT_ONLY
Mode string `db:"mode"`
// StaleAfter is the timestamp when the stream becomes stale
StaleAfter string `db:"stale_after"`
// InvalidReason provides the reason if the stream is invalid
InvalidReason string `db:"invalid_reason"`
// OwnerRoleType is the type of owner role
OwnerRoleType string `db:"owner_role_type"`
}
ShowStreamsRow represents the structure of a row returned by SHOW STREAMS command
type SnowflakeQueryLogSchema ¶ added in v0.5.0
type SnowflakeQueryLogSchema struct {
QueryID string `db:"QUERY_ID"`
QueryText string `db:"QUERY_TEXT"`
DatabaseID *int64 `db:"DATABASE_ID"`
DatabaseName *string `db:"DATABASE_NAME"`
SchemaID *int64 `db:"SCHEMA_ID"`
SchemaName *string `db:"SCHEMA_NAME"`
QueryType string `db:"QUERY_TYPE"`
SessionID int64 `db:"SESSION_ID"`
UserName string `db:"USER_NAME"`
RoleName *string `db:"ROLE_NAME"`
WarehouseID *int64 `db:"WAREHOUSE_ID"`
WarehouseName *string `db:"WAREHOUSE_NAME"`
WarehouseSize *string `db:"WAREHOUSE_SIZE"`
WarehouseType *string `db:"WAREHOUSE_TYPE"`
ClusterNumber *int32 `db:"CLUSTER_NUMBER"`
QueryTag string `db:"QUERY_TAG"`
ExecutionStatus string `db:"EXECUTION_STATUS"`
ErrorCode *int32 `db:"ERROR_CODE"`
ErrorMessage *string `db:"ERROR_MESSAGE"`
StartTime time.Time `db:"START_TIME"`
EndTime time.Time `db:"END_TIME"`
TotalElapsedTime int64 `db:"TOTAL_ELAPSED_TIME"`
BytesScanned int64 `db:"BYTES_SCANNED"`
PercentageScannedFromCache float64 `db:"PERCENTAGE_SCANNED_FROM_CACHE"`
BytesWritten int64 `db:"BYTES_WRITTEN"`
BytesWrittenToResult int64 `db:"BYTES_WRITTEN_TO_RESULT"`
BytesReadFromResult int64 `db:"BYTES_READ_FROM_RESULT"`
RowsProduced *int64 `db:"ROWS_PRODUCED"`
RowsInserted int64 `db:"ROWS_INSERTED"`
RowsUpdated int64 `db:"ROWS_UPDATED"`
RowsDeleted int64 `db:"ROWS_DELETED"`
RowsUnloaded int64 `db:"ROWS_UNLOADED"`
BytesDeleted int64 `db:"BYTES_DELETED"`
PartitionsScanned int64 `db:"PARTITIONS_SCANNED"`
PartitionsTotal int64 `db:"PARTITIONS_TOTAL"`
BytesSpilledToLocalStorage int64 `db:"BYTES_SPILLED_TO_LOCAL_STORAGE"`
BytesSpilledToRemoteStorage int64 `db:"BYTES_SPILLED_TO_REMOTE_STORAGE"`
BytesSentOverTheNetwork int64 `db:"BYTES_SENT_OVER_THE_NETWORK"`
CompilationTime int64 `db:"COMPILATION_TIME"`
ExecutionTime int64 `db:"EXECUTION_TIME"`
QueuedProvisioningTime int64 `db:"QUEUED_PROVISIONING_TIME"`
QueuedRepairTime int64 `db:"QUEUED_REPAIR_TIME"`
QueuedOverloadTime int64 `db:"QUEUED_OVERLOAD_TIME"`
TransactionBlockedTime int64 `db:"TRANSACTION_BLOCKED_TIME"`
OutboundDataTransferCloud *string `db:"OUTBOUND_DATA_TRANSFER_CLOUD"`
OutboundDataTransferRegion *string `db:"OUTBOUND_DATA_TRANSFER_REGION"`
OutboundDataTransferBytes *int64 `db:"OUTBOUND_DATA_TRANSFER_BYTES"`
InboundDataTransferCloud *string `db:"INBOUND_DATA_TRANSFER_CLOUD"`
InboundDataTransferRegion *string `db:"INBOUND_DATA_TRANSFER_REGION"`
InboundDataTransferBytes *int64 `db:"INBOUND_DATA_TRANSFER_BYTES"`
ListExternalFilesTime int64 `db:"LIST_EXTERNAL_FILES_TIME"`
CreditsUsedCloudServices float64 `db:"CREDITS_USED_CLOUD_SERVICES"`
ReleaseVersion string `db:"RELEASE_VERSION"`
ExternalFunctionTotalInvocations int64 `db:"EXTERNAL_FUNCTION_TOTAL_INVOCATIONS"`
ExternalFunctionTotalSentRows int64 `db:"EXTERNAL_FUNCTION_TOTAL_SENT_ROWS"`
ExternalFunctionTotalReceivedRows int64 `db:"EXTERNAL_FUNCTION_TOTAL_RECEIVED_ROWS"`
ExternalFunctionTotalSentBytes int64 `db:"EXTERNAL_FUNCTION_TOTAL_SENT_BYTES"`
ExternalFunctionTotalReceivedBytes int64 `db:"EXTERNAL_FUNCTION_TOTAL_RECEIVED_BYTES"`
QueryLoadPercent *int64 `db:"QUERY_LOAD_PERCENT"`
IsClientGeneratedStatement bool `db:"IS_CLIENT_GENERATED_STATEMENT"`
QueryAccelerationBytesScanned int64 `db:"QUERY_ACCELERATION_BYTES_SCANNED"`
QueryAccelerationPartitionsScanned int64 `db:"QUERY_ACCELERATION_PARTITIONS_SCANNED"`
QueryAccelerationUpperLimitScaleFactor int64 `db:"QUERY_ACCELERATION_UPPER_LIMIT_SCALE_FACTOR"`
ChildQueriesWaitTime int64 `db:"CHILD_QUERIES_WAIT_TIME"`
TransactionID int64 `db:"TRANSACTION_ID"`
RoleType *string `db:"ROLE_TYPE"`
QueryHash *string `db:"QUERY_HASH"`
QueryHashVersion *int64 `db:"QUERY_HASH_VERSION"`
QueryParameterizedHash *string `db:"QUERY_PARAMETERIZED_HASH"`
QueryParameterizedHashVersion *int64 `db:"QUERY_PARAMETERIZED_HASH_VERSION"`
}
SnowflakeQueryLogSchema represents Snowflake's ACCOUNT_USAGE.QUERY_HISTORY schema Source: https://docs.snowflake.com/en/sql-reference/account-usage/query_history
type SnowflakeScrapper ¶
type SnowflakeScrapper struct {
// contains filtered or unexported fields
}
func NewSnowflakeScrapper ¶
func NewSnowflakeScrapper(ctx context.Context, conf *SnowflakeScrapperConf) (*SnowflakeScrapper, error)
func (*SnowflakeScrapper) Capabilities ¶ added in v0.10.6
func (e *SnowflakeScrapper) Capabilities() scrapper.Capabilities
func (*SnowflakeScrapper) Close ¶
func (e *SnowflakeScrapper) Close() error
func (*SnowflakeScrapper) DialectType ¶ added in v0.1.8
func (e *SnowflakeScrapper) DialectType() string
func (*SnowflakeScrapper) Executor ¶
func (e *SnowflakeScrapper) Executor() *dwhexecsnowflake.SnowflakeExecutor
func (*SnowflakeScrapper) FetchQueryLogs ¶ added in v0.5.0
func (s *SnowflakeScrapper) FetchQueryLogs( ctx context.Context, from, to time.Time, obfuscator querylogs.QueryObfuscator, ) (querylogs.QueryLogIterator, error)
func (*SnowflakeScrapper) FetchTableChangeHistory ¶ added in v0.9.2
func (*SnowflakeScrapper) GetDatabasesToQuery ¶ added in v0.11.0
func (e *SnowflakeScrapper) GetDatabasesToQuery(ctx context.Context) ([]string, error)
GetDatabasesToQuery returns the configured databases filtered to only those that exist and are accepted by the scope filter. This is the standard way to get the list of databases to iterate over in scrapper query methods.
func (*SnowflakeScrapper) GetExistingDbs ¶
func (e *SnowflakeScrapper) GetExistingDbs(ctx context.Context) ([]*DbDesc, error)
func (*SnowflakeScrapper) IsPermissionError ¶ added in v0.1.1
func (e *SnowflakeScrapper) IsPermissionError(err error) bool
func (*SnowflakeScrapper) QueryCatalog ¶
func (e *SnowflakeScrapper) QueryCatalog(origCtx context.Context) ([]*scrapper.CatalogColumnRow, error)
func (*SnowflakeScrapper) QueryCustomMetrics ¶
func (e *SnowflakeScrapper) QueryCustomMetrics(ctx context.Context, sql string, args ...any) ([]*scrapper.CustomMetricsRow, error)
func (*SnowflakeScrapper) QueryDatabases ¶
func (e *SnowflakeScrapper) QueryDatabases(ctx context.Context) ([]*scrapper.DatabaseRow, error)
func (*SnowflakeScrapper) QuerySegments ¶
func (e *SnowflakeScrapper) QuerySegments(ctx context.Context, sql string, args ...any) ([]*scrapper.SegmentRow, error)
func (*SnowflakeScrapper) QueryShape ¶ added in v0.8.3
func (e *SnowflakeScrapper) QueryShape(ctx context.Context, sql string) ([]*scrapper.QueryShapeColumn, error)
func (*SnowflakeScrapper) QuerySqlDefinitions ¶
func (e *SnowflakeScrapper) QuerySqlDefinitions(origCtx context.Context) ([]*scrapper.SqlDefinitionRow, error)
func (*SnowflakeScrapper) QueryTableConstraints ¶ added in v0.9.0
func (e *SnowflakeScrapper) QueryTableConstraints(ctx context.Context) ([]*scrapper.TableConstraintRow, error)
func (*SnowflakeScrapper) QueryTableMetrics ¶
func (e *SnowflakeScrapper) QueryTableMetrics(origCtx context.Context, lastMetricsFetchTime time.Time) ([]*scrapper.TableMetricsRow, error)
func (*SnowflakeScrapper) QueryTables ¶
func (e *SnowflakeScrapper) QueryTables(origCtx context.Context, opts ...scrapper.QueryTablesOption) ([]*scrapper.TableRow, error)
func (*SnowflakeScrapper) SqlDialect ¶ added in v0.1.8
func (e *SnowflakeScrapper) SqlDialect() sqldialect.Dialect
func (*SnowflakeScrapper) ValidateConfiguration ¶
func (e *SnowflakeScrapper) ValidateConfiguration(ctx context.Context) ([]string, error)
type SnowflakeScrapperConf ¶
type SnowflakeScrapperConf struct {
dwhexecsnowflake.SnowflakeConf
NoGetDll bool
AccountUsageDb *string
QueryLogsDatabases []string
// UseAccessHistoryForTableChanges enables ACCESS_HISTORY for FetchTableChangeHistory instead of
// TABLE_DML_HISTORY. ACCESS_HISTORY provides ~3h lag vs ~6h but requires Snowflake Enterprise edition.
UseAccessHistoryForTableChanges bool
}
func (*SnowflakeScrapperConf) UpperDatabasesLiteral ¶
func (c *SnowflakeScrapperConf) UpperDatabasesLiteral() string
FIXME: I couldn't make it work with `foo IN (?)` binding, so I'm using this