Documentation
¶
Index ¶
- Constants
- Variables
- type Client
- func (c *Client) AddQueryHook(hook QueryHook)
- func (c *Client) Close() error
- func (c *Client) ColumnStats(ctx context.Context, database, table string) ([]TableColumnStat, error)
- func (c *Client) GetHistogramData(ctx context.Context, tableName, timestampField string, params HistogramParams) (*HistogramResult, error)
- func (c *Client) GetTableInfo(ctx context.Context, database, table string) (*TableInfo, error)
- func (c *Client) Ping(ctx context.Context, database string, table string) error
- func (c *Client) Query(ctx context.Context, query string) (*models.QueryResult, error)
- func (c *Client) QueryWithTimeout(ctx context.Context, query string, timeoutSeconds *int) (*models.QueryResult, error)
- func (c *Client) Reconnect(ctx context.Context) error
- func (c *Client) TableStats(ctx context.Context, database, table string) (*TableStat, error)
- type ClientOptions
- type ExtendedColumnInfo
- type HistogramData
- type HistogramParams
- type HistogramResult
- type LogContextParams
- type LogContextResult
- type LogQueryHook
- type LogQueryParams
- type LogQueryResult
- type Manager
- func (m *Manager) AddQueryHook(hook QueryHook)
- func (m *Manager) AddSource(source *models.Source) error
- func (m *Manager) Close() error
- func (m *Manager) CreateTemporaryClient(source *models.Source) (*Client, error)
- func (m *Manager) GetCachedHealth(sourceID models.SourceID) models.SourceHealth
- func (m *Manager) GetClient(sourceID models.SourceID) (*Client, error)
- func (m *Manager) GetConnection(sourceID models.SourceID) (*Client, error)
- func (m *Manager) GetHealth(sourceID models.SourceID) models.SourceHealth
- func (m *Manager) RemoveSource(sourceID models.SourceID) error
- func (m *Manager) StartBackgroundHealthChecks(interval time.Duration)
- func (m *Manager) StopBackgroundHealthChecks()
- type QueryBuilder
- type QueryHook
- type StructuredQueryLoggerHook
- type TableColumnStat
- type TableInfo
- type TableStat
- type TimeWindow
Constants ¶
const ( // DefaultQueryTimeout is the default max_execution_time in seconds if not specified DefaultQueryTimeout = 60 // MaxQueryTimeout is the maximum allowed timeout to prevent resource abuse MaxQueryTimeout = 300 // 5 minutes )
Default values for query execution
const ( DefaultQueryLimit = 100 HealthCheckTimeout = 1 * time.Second // Reduce to 1 second for faster health checks DefaultHealthCheckInterval = 30 * time.Second )
Default values
Variables ¶
var ( // ErrSourceNotConnected is returned when a source is not connected ErrSourceNotConnected = errors.New("source not connected") // ErrQueryTimeout is returned when a query times out ErrQueryTimeout = errors.New("query timeout exceeded") // ErrInvalidQuery is returned when a query is invalid ErrInvalidQuery = errors.New("invalid query") // ErrConnectionFailed is returned when a connection cannot be established ErrConnectionFailed = errors.New("connection failed") // ErrSourceExists is returned when trying to create a source that already exists ErrSourceExists = errors.New("source already exists") // ErrInvalidSourceType is returned when the source type is not supported ErrInvalidSourceType = errors.New("invalid source type") )
Common errors for the clickhouse package
Functions ¶
This section is empty.
Types ¶
type Client ¶
type Client struct {
// contains filtered or unexported fields
}
Client represents a connection to a ClickHouse database using the native protocol. It provides methods for executing queries and retrieving metadata.
func NewClient ¶
func NewClient(opts ClientOptions, logger *slog.Logger) (*Client, error)
NewClient establishes a new connection to a ClickHouse server using the native protocol. It takes connection options and a logger, creates the connection, and returns a Client instance. Note: This does not automatically verify the connection with a ping - callers should do that if needed.
func (*Client) AddQueryHook ¶
AddQueryHook registers a hook to be executed before and after queries run by this client.
func (*Client) ColumnStats ¶
func (c *Client) ColumnStats(ctx context.Context, database, table string) ([]TableColumnStat, error)
ColumnStats retrieves detailed statistics for each column of a specific table.
func (*Client) GetHistogramData ¶
func (c *Client) GetHistogramData(ctx context.Context, tableName, timestampField string, params HistogramParams) (*HistogramResult, error)
GetHistogramData generates histogram data by grouping log counts into time buckets. It uses the provided raw SQL as the base query and applies time window aggregation. Timeout is always applied.
func (*Client) GetTableInfo ¶
GetTableInfo retrieves detailed metadata about a table, including handling for Distributed tables by inspecting the underlying local table.
func (*Client) Ping ¶
Ping checks the connectivity to the ClickHouse server and optionally verifies a table exists. It uses short timeouts internally. Returns nil on success, or an error indicating the failure reason.
func (*Client) Query ¶
Query executes a SELECT query, processes the results, and applies query hooks. It automatically handles DDL statements by calling execDDL. The params argument is now unused but kept for potential future structured query building.
func (*Client) QueryWithTimeout ¶ added in v0.2.2
func (c *Client) QueryWithTimeout(ctx context.Context, query string, timeoutSeconds *int) (*models.QueryResult, error)
QueryWithTimeout executes a SELECT query with a timeout setting. The timeoutSeconds parameter is required and will always be applied.
type ClientOptions ¶
type ClientOptions struct {
Host string // Hostname or IP address.
Database string // Target database name.
Username string // Username for authentication.
Password string // Password for authentication.
Settings map[string]interface{} // Additional ClickHouse settings (e.g., max_execution_time).
}
ClientOptions holds configuration for establishing a new ClickHouse client connection.
type ExtendedColumnInfo ¶
type ExtendedColumnInfo struct {
Name string `json:"name"`
Type string `json:"type"`
IsNullable bool `json:"is_nullable"`
IsPrimaryKey bool `json:"is_primary_key"`
DefaultExpression string `json:"default_expression,omitempty"`
Comment string `json:"comment,omitempty"`
}
ExtendedColumnInfo provides detailed column metadata, including nullability, primary key status, default expressions, and comments, supplementing models.ColumnInfo.
type HistogramData ¶
type HistogramData struct {
Bucket time.Time `json:"bucket"` // Start time of the bucket.
LogCount int `json:"log_count"` // Number of logs in the bucket.
GroupValue string `json:"group_value"` // Value of the group for grouped histograms.
}
HistogramData represents a single time bucket and its log count in a histogram.
type HistogramParams ¶
type HistogramParams struct {
Window TimeWindow
Query string // Raw SQL query to use as base for histogram
GroupBy string // Optional: Field to group by for segmented histograms.
Timezone string // Optional: Timezone identifier for time-based operations.
// Query execution timeout in seconds. If not specified, uses default timeout.
QueryTimeout *int
}
HistogramParams defines parameters for generating histogram data.
type HistogramResult ¶
type HistogramResult struct {
Granularity string `json:"granularity"` // The time window used (e.g., "5m").
Data []HistogramData `json:"data"`
}
HistogramResult holds the complete histogram data and its granularity.
type LogContextParams ¶
LogContextParams defines parameters for fetching logs around a specific target time.
type LogContextResult ¶
type LogContextResult struct {
BeforeLogs []map[string]interface{}
TargetLogs []map[string]interface{} // Logs exactly at the target time.
AfterLogs []map[string]interface{}
Stats models.QueryStats
}
LogContextResult holds the logs retrieved before, at, and after the target time.
type LogQueryHook ¶
type LogQueryHook struct {
// Verbose logs all queries if true; otherwise, only logs failed queries.
Verbose bool
// contains filtered or unexported fields
}
LogQueryHook is a basic QueryHook implementation that logs query execution start and completion/failure, controlled by the Verbose flag.
func NewLogQueryHook ¶
func NewLogQueryHook(logger *slog.Logger, verbose bool) *LogQueryHook
NewLogQueryHook creates a new LogQueryHook.
func (*LogQueryHook) AfterQuery ¶
func (h *LogQueryHook) AfterQuery(ctx context.Context, query string, err error, duration time.Duration)
AfterQuery logs query completion status (success/failure) and duration. Success is only logged if Verbose is true.
func (*LogQueryHook) BeforeQuery ¶
BeforeQuery optionally logs the query before execution if Verbose is true.
type LogQueryParams ¶
type LogQueryParams struct {
Limit int
RawSQL string
// Query execution timeout in seconds. If not specified, uses default timeout.
QueryTimeout *int
}
LogQueryParams defines parameters for querying logs.
type LogQueryResult ¶
type LogQueryResult struct {
Data []map[string]interface{} `json:"data"`
Stats models.QueryStats `json:"stats"`
Columns []models.ColumnInfo `json:"columns"`
}
LogQueryResult represents the structured result of a log query.
type Manager ¶
type Manager struct {
// contains filtered or unexported fields
}
Manager handles pooling and management of multiple ClickHouse client connections, one per data source. It also manages query hooks and background health checks.
func NewManager ¶
NewManager creates a new ClickHouse connection manager.
func (*Manager) AddQueryHook ¶
AddQueryHook adds a query hook to the manager's list. The hook will be applied to all currently managed clients and any subsequently added clients via AddSource.
func (*Manager) AddSource ¶
AddSource creates a new ClickHouse client connection based on the source details, applies existing hooks, stores it in the manager pool, and initializes health. Modified to always create a client entry even if initial connection fails.
func (*Manager) Close ¶
Close iterates through all managed client connections and closes them, with a timeout for each client to prevent hanging on unhealthy connections. It also stops the background health checker and waits for it to complete.
func (*Manager) CreateTemporaryClient ¶
CreateTemporaryClient creates a new, unmanaged ClickHouse client instance, typically used for validating connection details before adding a source. The caller is responsible for closing the returned client.
func (*Manager) GetCachedHealth ¶
func (m *Manager) GetCachedHealth(sourceID models.SourceID) models.SourceHealth
GetCachedHealth retrieves the latest known health status for a source ID from the cache. Returns a default unhealthy status if the source hasn't been checked yet.
func (*Manager) GetClient ¶
GetClient is an alias for GetConnection for potential backward compatibility.
func (*Manager) GetConnection ¶
GetConnection returns the managed client connection for a given source ID. Returns ErrSourceNotConnected if the source is not currently managed.
func (*Manager) GetHealth ¶
func (m *Manager) GetHealth(sourceID models.SourceID) models.SourceHealth
GetHealth performs a LIVE health check on a specific source and updates the cache. Deprecated: Use GetCachedHealth for regular status checks. Use this only if an immediate, live check is explicitly required.
func (*Manager) RemoveSource ¶
RemoveSource closes the connection for the given source ID and removes it from the manager.
func (*Manager) StartBackgroundHealthChecks ¶
StartBackgroundHealthChecks launches a goroutine to periodically check the health of all managed connections.
func (*Manager) StopBackgroundHealthChecks ¶
func (m *Manager) StopBackgroundHealthChecks()
StopBackgroundHealthChecks signals the health check goroutine to stop.
type QueryBuilder ¶
type QueryBuilder struct {
// contains filtered or unexported fields
}
QueryBuilder assists in building and validating ClickHouse SQL queries.
func NewQueryBuilder ¶
func NewQueryBuilder(tableName string) *QueryBuilder
NewQueryBuilder creates a new QueryBuilder for a specific table.
func (*QueryBuilder) BuildRawQuery ¶
func (qb *QueryBuilder) BuildRawQuery(rawSQL string, limit int) (string, error)
BuildRawQuery parses, validates, potentially modifies (adds LIMIT), and reconstructs a raw SQL query string.
func (*QueryBuilder) RemoveLimitClause ¶
func (qb *QueryBuilder) RemoveLimitClause(rawSQL string) (string, error)
RemoveLimitClause parses the SQL and removes any LIMIT clause, then returns the modified query.
type QueryHook ¶
type QueryHook interface {
// BeforeQuery is called before a query is executed.
// It can return a modified context for the query execution.
BeforeQuery(ctx context.Context, query string) (context.Context, error)
// AfterQuery is called after a query has finished executing,
// regardless of whether it succeeded or failed.
AfterQuery(ctx context.Context, query string, err error, duration time.Duration)
}
QueryHook defines an interface for intercepting ClickHouse queries. Hooks can modify the context or log query details before and after execution.
type StructuredQueryLoggerHook ¶
type StructuredQueryLoggerHook struct {
// contains filtered or unexported fields
}
StructuredQueryLoggerHook implements QueryHook to log query details *before* execution using structured logging attributes.
func NewStructuredQueryLoggerHook ¶
func NewStructuredQueryLoggerHook(logger *slog.Logger) *StructuredQueryLoggerHook
NewStructuredQueryLoggerHook creates a new StructuredQueryLoggerHook.
func (*StructuredQueryLoggerHook) AfterQuery ¶
func (h *StructuredQueryLoggerHook) AfterQuery(ctx context.Context, query string, err error, duration time.Duration)
AfterQuery is a no-op for StructuredQueryLoggerHook.
func (*StructuredQueryLoggerHook) BeforeQuery ¶
func (h *StructuredQueryLoggerHook) BeforeQuery(ctx context.Context, query string) (context.Context, error)
BeforeQuery logs query details using structured logging.
type TableColumnStat ¶
type TableColumnStat struct {
Database string `json:"database"`
Table string `json:"table"`
Column string `json:"column"`
Compressed string `json:"compressed"` // Size on disk (human-readable).
Uncompressed string `json:"uncompressed"` // Original size (human-readable).
ComprRatio float64 `json:"compr_ratio"` // Compression ratio.
RowsCount uint64 `json:"rows_count"` // Number of rows in the column chunk.
AvgRowSize float64 `json:"avg_row_size"` // Average row size in bytes.
}
TableColumnStat represents statistics for a single column in a ClickHouse table, typically retrieved from system.parts_columns.
type TableInfo ¶
type TableInfo struct {
Database string `json:"database"`
Name string `json:"name"`
Engine string `json:"engine"` // e.g., "MergeTree", "Distributed"
EngineParams []string `json:"engine_params,omitempty"` // Parameters extracted from engine_full.
Columns []models.ColumnInfo `json:"columns"` // Basic column info (Name, Type).
ExtColumns []ExtendedColumnInfo `json:"ext_columns,omitempty"` // Detailed column info.
SortKeys []string `json:"sort_keys"` // Parsed sorting key columns.
CreateQuery string `json:"create_query,omitempty"` // Full CREATE TABLE statement.
}
TableInfo represents comprehensive metadata about a ClickHouse table, including engine details, column definitions (basic and extended), sorting keys, and the CREATE statement.
type TableStat ¶
type TableStat struct {
Database string `json:"database"`
Table string `json:"table"`
Compressed string `json:"compressed"` // Total size on disk (human-readable).
Uncompressed string `json:"uncompressed"` // Total original size (human-readable).
ComprRate float64 `json:"compr_rate"` // Overall compression rate.
Rows uint64 `json:"rows"` // Total rows in the table partition/part.
PartCount uint64 `json:"part_count"` // Number of data parts.
}
TableStat represents overall statistics for a ClickHouse table, typically retrieved from system.parts.
type TimeWindow ¶
type TimeWindow string
TimeWindow represents the desired granularity for time-based aggregations.
const ( // Second-based windows TimeWindow1s TimeWindow = "1s" // 1 second TimeWindow5s TimeWindow = "5s" // 5 seconds TimeWindow10s TimeWindow = "10s" // 10 seconds TimeWindow15s TimeWindow = "15s" // 15 seconds TimeWindow30s TimeWindow = "30s" // 30 seconds // Minute-based windows TimeWindow1m TimeWindow = "1m" // 1 minute TimeWindow5m TimeWindow = "5m" // 5 minutes TimeWindow10m TimeWindow = "10m" // 10 minutes TimeWindow15m TimeWindow = "15m" // 15 minutes TimeWindow30m TimeWindow = "30m" // 30 minutes // Hour-based windows TimeWindow1h TimeWindow = "1h" // 1 hour TimeWindow2h TimeWindow = "2h" // 2 hours TimeWindow3h TimeWindow = "3h" // 3 hours TimeWindow6h TimeWindow = "6h" // 6 hours TimeWindow12h TimeWindow = "12h" // 12 hours TimeWindow24h TimeWindow = "24h" // 24 hours )