Documentation
¶
Index ¶
- func IngestrTypeHints() map[string]string
- func IngestrTypeWrappers() map[string]bool
- func NewColumnCheckOperator(manager config.ConnectionGetter) *ansisql.ColumnCheckOperator
- type AcceptedValuesCheck
- type AssetMaterializationMap
- type BasicOperator
- type ClickHouseClient
- type ClickHouseConfig
- type Client
- func (c *Client) BuildTableExistsQuery(tableName string) (string, error)
- func (c *Client) GetColumns(ctx context.Context, databaseName, tableName string) ([]*ansisql.DBColumn, error)
- func (c *Client) GetDatabaseSummary(ctx context.Context) (*ansisql.DBDatabase, error)
- func (c *Client) GetDatabases(ctx context.Context) ([]string, error)
- func (c *Client) GetIngestrURI() (string, error)
- func (c *Client) GetTables(ctx context.Context, databaseName string) ([]string, error)
- func (c *Client) IngestrTypeHints() map[string]string
- func (c *Client) IngestrTypeWrappers() map[string]bool
- func (c *Client) Ping(ctx context.Context) error
- func (c *Client) RunQueryWithoutResult(ctx context.Context, query *query.Query) error
- func (c *Client) Select(ctx context.Context, q *query.Query) ([][]interface{}, error)
- func (c *Client) SelectWithSchema(ctx context.Context, queryObj *query.Query) (*query.QueryResult, error)
- type Config
- type Materializer
- type MaterializerFunc
- type PatternCheck
- type Renderer
- type RowScanner
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func IngestrTypeHints ¶ added in v0.11.681
IngestrTypeHints maps ClickHouse-native column type names (and ClickHouse-specific aliases) to ingestr/dlt types used for --columns hints on seed and ingest assets.
Parameterized forms such as DateTime64(3), Decimal64(2), FixedString(16), Enum8('a' = 1), and Array(UInt64) resolve via their base name. Nullable(T) and LowCardinality(T) are peeled by pkg/python so the inner type is used.
Entries that also exist in the shared defaults intentionally override them when this destination is ClickHouse (e.g. int8 is Int8 here, not PostgreSQL bigint; time is the Int64 alias, not a time-of-day type).
func IngestrTypeWrappers ¶ added in v0.11.681
IngestrTypeWrappers are ClickHouse parameterized types whose inner type should be resolved for ingestr --columns hints (e.g. Nullable(DateTime64(3))).
func NewColumnCheckOperator ¶
func NewColumnCheckOperator(manager config.ConnectionGetter) *ansisql.ColumnCheckOperator
Types ¶
type AcceptedValuesCheck ¶
type AcceptedValuesCheck struct {
// contains filtered or unexported fields
}
func (*AcceptedValuesCheck) Check ¶
func (c *AcceptedValuesCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error
type AssetMaterializationMap ¶
type AssetMaterializationMap map[pipeline.MaterializationType]map[pipeline.MaterializationStrategy]MaterializerFunc
type BasicOperator ¶
type BasicOperator struct {
// contains filtered or unexported fields
}
func NewBasicOperator ¶
func NewBasicOperator(conn config.ConnectionGetter, extractor query.QueryExtractor, materializer materializer, parser *sqlparser.SQLParser) *BasicOperator
func (BasicOperator) Run ¶
func (o BasicOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error
type ClickHouseClient ¶
type ClickHouseConfig ¶
type ClickHouseConfig interface {
ToClickHouseOptions() *click_house.Options
GetIngestrURI() string
GetDatabase() string
}
type Client ¶
type Client struct {
// contains filtered or unexported fields
}
func NewClient ¶
func NewClient(c ClickHouseConfig) (*Client, error)
func (*Client) BuildTableExistsQuery ¶ added in v0.11.296
func (*Client) GetColumns ¶ added in v0.11.247
func (*Client) GetDatabaseSummary ¶ added in v0.11.242
func (*Client) GetDatabases ¶ added in v0.11.247
func (*Client) GetIngestrURI ¶
func (*Client) IngestrTypeHints ¶ added in v0.11.681
IngestrTypeHints implements the optional connection capability used by pkg/python when building ingestr --columns hints for this destination.
func (*Client) IngestrTypeWrappers ¶ added in v0.11.681
IngestrTypeWrappers implements the optional connection capability used by pkg/python when peeling destination-specific type wrappers.
func (*Client) RunQueryWithoutResult ¶
func (*Client) SelectWithSchema ¶
type Config ¶
type Config struct {
Username string
Password string
Host string
Port int
Database string
HTTPPort int
Secure *int
}
func (*Config) GetDatabase ¶ added in v0.11.296
func (*Config) GetIngestrURI ¶
func (*Config) ToClickHouseOptions ¶
func (c *Config) ToClickHouseOptions() *click_house.Options
type Materializer ¶
type Materializer struct {
MaterializationMap AssetMaterializationMap
// contains filtered or unexported fields
}
The other packages all use a materializer that renders the query to a single string. Due to the quirks of athena we need to create a different materializer that returns a slice of strings, since athena server requires us to send separate batches for certain things.
func NewMaterializer ¶
func NewMaterializer(fullRefresh bool) *Materializer
func (*Materializer) LogIfFullRefreshAndDDL ¶ added in v0.11.222
func (m *Materializer) LogIfFullRefreshAndDDL(writer interface{}, asset *pipeline.Asset) error
type MaterializerFunc ¶
type PatternCheck ¶
type PatternCheck struct {
// contains filtered or unexported fields
}
func (*PatternCheck) Check ¶
func (c *PatternCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error
type Renderer ¶
type Renderer struct {
// contains filtered or unexported fields
}
func NewRenderer ¶
type RowScanner ¶
type RowScanner struct {
// contains filtered or unexported fields
}
func (*RowScanner) Scan ¶
func (s *RowScanner) Scan(src any) error
Source Files