clickhouse

package
v2.8.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 16, 2026 License: MIT Imports: 27 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	ShardBackupFull   = "full"
	ShardBackupNone   = "none"
	ShardBackupSchema = "schema-only"
)
View Source 
const CustomDiskTableLike = "%disk = disk(%"

CustomDiskTableLike - `system.tables.create_table_query` fragment which marks a table with an inline disk

View Source 
const TmpInternalDiskPrefix = "__tmp_internal_"

TmpInternalDiskPrefix - DiskSelector::TMP_INTERNAL_DISK_PREFIX, name prefix of a `disk(...)` disk declared without `name = '...'`, the suffix is sipHash128 of the canonical disk declaration

View Source 
const TmpStoragePolicyPrefix = "__"

TmpStoragePolicyPrefix - StoragePolicySelector::TMP_STORAGE_POLICY_PREFIX, a table with `disk = ...` gets a synthesized single-disk storage policy named `__<disk name>`

Variables

View Source 
var (
	ErrShardOperationVers = errors.New("sharded operations are only supported for " +
		"clickhouse-server >= v21.x")
)

Functions

func GetDisksByPaths 

func GetDisksByPaths(disks []Disk, dataPaths []string) (map[string]string, error)

GetDisksByPaths - map each disk name to the table data path which lives on it. A data path matching no disk path is still attributed to `default` (disk_mapping cross-cluster setups rely on it), but such a fallback never overwrites a data path which really matched, and two different data paths really matching the same disk is a misconfiguration we refuse instead of silently restoring parts into the wrong disk, fix https://github.com/Altinity/clickhouse-backup/issues/1121

func HasCustomDisk added in v2.8.1 

func HasCustomDisk(query string) bool

HasCustomDisk - true when the CREATE/ATTACH TABLE query declares an inline `disk = disk(...)`

Types

type BackupDataSize added in v2.5.0 

type BackupDataSize struct {
	Size uint64 `ch:"backup_data_size"`
}

BackupDataSize - info from system.parts or system.tables when embedded BACKUP statement return zero size

type ClickHouse 

type ClickHouse struct {
	Config *config.ClickHouseConfig

	ConnMutex *sync.Mutex

	IsOpen              bool
	BreakConnectOnError bool
	// ShutdownCtx stops the infinite reconnect loop when the process is asked to terminate,
	// otherwise a SIGTERM can't stop a command waiting for an unreachable ClickHouse
	ShutdownCtx context.Context
	// contains filtered or unexported fields
}

ClickHouse - provide

func NewClickHouse added in v2.6.42 

func NewClickHouse(cfg *config.ClickHouseConfig) *ClickHouse

func (*ClickHouse) AnalyzerOffContextIfNecessary added in v2.7.0 

func (ch *ClickHouse) AnalyzerOffContextIfNecessary(version int, query string) context.Context

AnalyzerOffContextIfNecessary returns a context with allow_experimental_analyzer=0 setting attached via clickhouse.Context+WithSettings, so the setting is sent in the same packet as the query. This avoids the race condition where SET on a pooled connection may not affect the connection used by the subsequent query.

func (*ClickHouse) ApplyMacros 

func (ch *ClickHouse) ApplyMacros(ctx context.Context, s string) (string, error)

func (*ClickHouse) ApplyMacrosToObjectLabels added in v2.6.0 

func (ch *ClickHouse) ApplyMacrosToObjectLabels(ctx context.Context, objectLabels map[string]string, backupName string) (map[string]string, error)

ApplyMacrosToObjectLabels https://github.com/Altinity/clickhouse-backup/issues/588

func (*ClickHouse) ApplyMutation 

func (ch *ClickHouse) ApplyMutation(ctx context.Context, tableMetadata metadata.TableMetadata, mutation metadata.MutationMetadata) error

func (*ClickHouse) AttachDataParts 

func (ch *ClickHouse) AttachDataParts(table metadata.TableMetadata, dstTable Table) error

AttachDataParts - execute ALTER TABLE ... ATTACH PART command for specific table

func (*ClickHouse) AttachTable 

func (ch *ClickHouse) AttachTable(ctx context.Context, table metadata.TableMetadata, dstTable Table) error

AttachTable - execute ATTACH TABLE command for specific table

func (*ClickHouse) CalculateMaxFileSize 

func (ch *ClickHouse) CalculateMaxFileSize(ctx context.Context, cfg *config.Config) (int64, error)

func (*ClickHouse) CanShardOperation 

func (ch *ClickHouse) CanShardOperation(ctx context.Context) error

func (*ClickHouse) CheckReplicationInProgress 

func (ch *ClickHouse) CheckReplicationInProgress(table metadata.TableMetadata) (bool, error)

CheckReplicationInProgress allow to avoid concurrent ATTACH PART https://github.com/Altinity/clickhouse-backup/issues/474

func (*ClickHouse) CheckSettingsExists 

func (ch *ClickHouse) CheckSettingsExists(ctx context.Context, settings map[string]bool) (map[string]bool, error)

func (*ClickHouse) CheckSystemPartsColumns 

func (ch *ClickHouse) CheckSystemPartsColumns(ctx context.Context, table *Table) error

func (*ClickHouse) CheckSystemPartsColumnsForTables added in v2.6.42 

func (ch *ClickHouse) CheckSystemPartsColumnsForTables(ctx context.Context, tables []Table) error

func (*ClickHouse) CheckTypesConsistency added in v2.4.24 

func (ch *ClickHouse) CheckTypesConsistency(table *Table, partColumnsDataTypes []ColumnDataTypes) error

func (*ClickHouse) Close 

func (ch *ClickHouse) Close()

Close - closing connection to ClickHouse

func (*ClickHouse) Connect 

func (ch *ClickHouse) Connect() error

Connect - establish connection to ClickHouse

func (*ClickHouse) CreateDatabase 

func (ch *ClickHouse) CreateDatabase(database string, cluster string) error

CreateDatabase - create ClickHouse database

func (*ClickHouse) CreateDatabaseFromQuery 

func (ch *ClickHouse) CreateDatabaseFromQuery(ctx context.Context, query, cluster string, args ...interface{}) error

func (*ClickHouse) CreateDatabaseWithEngine 

func (ch *ClickHouse) CreateDatabaseWithEngine(database, engine, cluster string, version int) error

func (*ClickHouse) CreateTable 

func (ch *ClickHouse) CreateTable(table Table, query string, dropTable, ignoreDependencies bool, onCluster string, version int, defaultDataPath string, asAttach bool, databaseEngine string) error

CreateTable - create ClickHouse table

func (*ClickHouse) CreateTableAsAttach added in v2.6.6 

func (ch *ClickHouse) CreateTableAsAttach(query string) error

func (*ClickHouse) CreateUserDefinedFunction 

func (ch *ClickHouse) CreateUserDefinedFunction(name string, query string, cluster string) error

func (*ClickHouse) DropOrDetachTable added in v2.6.6 

func (ch *ClickHouse) DropOrDetachTable(table Table, query, onCluster string, ignoreDependencies bool, version int, defaultDataPath string, useDetach bool, databaseEngine string) error

DropOrDetachTable - drop ClickHouse table

func (*ClickHouse) ExtractStoragePolicy added in v2.4.26 

func (ch *ClickHouse) ExtractStoragePolicy(query string) string

func (*ClickHouse) FreezeTable 

func (ch *ClickHouse) FreezeTable(ctx context.Context, table *Table, name string) error

FreezeTable - freeze all partitions for table This way available for ClickHouse since v19.1

func (*ClickHouse) FreezeTableByParts added in v2.5.0 

func (ch *ClickHouse) FreezeTableByParts(ctx context.Context, table *Table, name string) error

FreezeTableByParts - freeze all partitions in table one by one also ally `freeze_by_part_where`

func (*ClickHouse) GetAccessManagementPath 

func (ch *ClickHouse) GetAccessManagementPath(ctx context.Context, disks []Disk) (string, error)

GetAccessManagementPath extract path from following sources system.user_directories, access_control_path from /var/lib/clickhouse/preprocessed_configs/config.xml, system.disks

func (*ClickHouse) GetConn 

func (ch *ClickHouse) GetConn() driver.Conn

GetConn - return current connection

func (*ClickHouse) GetCustomDiskTables added in v2.8.1 

func (ch *ClickHouse) GetCustomDiskTables(ctx context.Context) ([]Table, error)

GetCustomDiskTables - tables which declare an inline `SETTINGS disk = disk(...)`, their DDL is the only source of credentials for such disks, https://github.com/Altinity/clickhouse-backup/issues/943 Tables whose DDL is still masked after the `metadata/<db>/<table>.sql` fallback are returned together with a non nil error, the caller decides whether missing credentials for those tables are fatal.

func (*ClickHouse) GetDatabaseEngine added in v2.6.16 

func (ch *ClickHouse) GetDatabaseEngine(database string) (string, error)

func (*ClickHouse) GetDatabases 

func (ch *ClickHouse) GetDatabases(ctx context.Context, cfg *config.Config, tablePattern string) ([]Database, error)

GetDatabases - return slice of all non system databases for backup

func (*ClickHouse) GetDefaultPath 

func (ch *ClickHouse) GetDefaultPath(disks []Disk) (string, error)

func (*ClickHouse) GetDisks 

func (ch *ClickHouse) GetDisks(ctx context.Context, enrich bool) ([]Disk, error)

GetDisks - return data from system.disks table

func (*ClickHouse) GetEmbeddedBackupPath 

func (ch *ClickHouse) GetEmbeddedBackupPath(disks []Disk) (string, error)

func (*ClickHouse) GetInProgressMutations 

func (ch *ClickHouse) GetInProgressMutations(ctx context.Context, database string, table string) ([]metadata.MutationMetadata, error)

func (*ClickHouse) GetInProgressMutationsBatch added in v2.7.3 

func (ch *ClickHouse) GetInProgressMutationsBatch(ctx context.Context) (map[metadata.TableTitle][]metadata.MutationMetadata, error)

func (*ClickHouse) GetPreprocessedConfigPath 

func (ch *ClickHouse) GetPreprocessedConfigPath(ctx context.Context) (string, error)

func (*ClickHouse) GetPreprocessedXMLSettings 

func (ch *ClickHouse) GetPreprocessedXMLSettings(ctx context.Context, settingsXPath map[string]string, fileName string) (map[string]string, error)

GetPreprocessedXMLSettings - @todo think about from_env and from_zookeeper corner cases

func (*ClickHouse) GetSettingsValues added in v2.6.3 

func (ch *ClickHouse) GetSettingsValues(ctx context.Context, settings []interface{}) (map[string]string, error)

func (*ClickHouse) GetTableDiskNameFromStoragePolicy added in v2.8.1 

func (ch *ClickHouse) GetTableDiskNameFromStoragePolicy(ctx context.Context, database, table string) (string, error)

GetTableDiskNameFromStoragePolicy - resolve the disk a `SETTINGS disk = ...` table lives on via system.tables.storage_policy = "__" + disk name; returns "" for tables which use a regular storage policy

func (*ClickHouse) GetTables 

func (ch *ClickHouse) GetTables(ctx context.Context, tablePattern string) ([]Table, error)

GetTables - return slice of all tables suitable for backup, MySQL and PostgresSQL database engine shall be skipped

func (*ClickHouse) GetUserDefinedFunctions 

func (ch *ClickHouse) GetUserDefinedFunctions(ctx context.Context) ([]Function, error)

func (*ClickHouse) GetUserDirectories added in v2.8.1 

func (ch *ClickHouse) GetUserDirectories(ctx context.Context) ([]UserDirectory, error)

GetUserDirectories returns all rows from system.user_directories with their type, look https://github.com/Altinity/clickhouse-backup/issues/881

func (*ClickHouse) GetVersion 

func (ch *ClickHouse) GetVersion(ctx context.Context) (int, error)

GetVersion - returned ClickHouse version in number format Example value: 19001005

func (*ClickHouse) GetVersionDescribe 

func (ch *ClickHouse) GetVersionDescribe(ctx context.Context) string

func (*ClickHouse) IsDbAtomicOrReplicated added in v2.6.16 

func (ch *ClickHouse) IsDbAtomicOrReplicated(database string) (bool, error)

func (*ClickHouse) LogQuery 

func (ch *ClickHouse) LogQuery(query string, args ...interface{}) string

func (*ClickHouse) ParseXML 

func (ch *ClickHouse) ParseXML(ctx context.Context, configName string) (string, *xmlquery.Node, error)

func (*ClickHouse) Query 

func (ch *ClickHouse) Query(query string, args ...interface{}) error

func (*ClickHouse) QueryContext 

func (ch *ClickHouse) QueryContext(ctx context.Context, query string, args ...interface{}) error

func (*ClickHouse) Select 

func (ch *ClickHouse) Select(dest interface{}, query string, args ...interface{}) error

func (*ClickHouse) SelectContext 

func (ch *ClickHouse) SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error

func (*ClickHouse) SelectSingleRow 

func (ch *ClickHouse) SelectSingleRow(ctx context.Context, dest interface{}, query string, args ...interface{}) error

func (*ClickHouse) SelectSingleRowNoCtx 

func (ch *ClickHouse) SelectSingleRowNoCtx(dest interface{}, query string, args ...interface{}) error

func (*ClickHouse) ShowCreateTable 

func (ch *ClickHouse) ShowCreateTable(ctx context.Context, database, name string) string

func (*ClickHouse) StructSelect 

func (ch *ClickHouse) StructSelect(dest interface{}, query string, args ...interface{}) error

type ColumnDataTypes added in v2.4.24 

type ColumnDataTypes struct {
	Column string   `ch:"column"`
	Types  []string `ch:"uniq_types"`
}

ColumnDataTypes - info from system.parts_columns

type ColumnDataTypesWithTable added in v2.6.42 

type ColumnDataTypesWithTable struct {
	Database string `ch:"database"`
	Table    string `ch:"table"`
	Column   string `ch:"column"`
	MinType  string `ch:"min_type"`
	MaxType  string `ch:"max_type"`
}

type CustomDisk added in v2.8.1 

type CustomDisk struct {
	// Args - `key = value` pairs in declaration order, values unquoted; the nested `disk = disk(...)` is not
	// stored here, a `disk = 'name'` / `disk = name` reference to another disk is stored as Args["disk"]
	Args map[string]string
	// Order - keys of Args in declaration order
	Order []string
	// Nested - inner `disk = disk(...)` of `cache` / `encrypted` wrappers, nil when `disk` references a disk by name
	Nested *CustomDisk
}

CustomDisk - a disk declared inline in table SETTINGS as `disk = disk(type = ..., ...)`, https://github.com/Altinity/clickhouse-backup/issues/943 ClickHouse registers such a disk in system.disks under a generated `__tmp_internal_<sipHash128>` name (or under `name = '...'` when given); the credentials exist only in the table DDL.

func ParseCustomDisk added in v2.8.1 

func ParseCustomDisk(query string) (*CustomDisk, error)

ParseCustomDisk - extract and parse the `disk = disk(...)` setting from a CREATE/ATTACH TABLE query, returns nil, nil when the query has no inline disk

func (*CustomDisk) Leaf added in v2.8.1 

func (d *CustomDisk) Leaf() *CustomDisk

Leaf - innermost disk of a `cache` / `encrypted` chain, the one which holds the object storage credentials

func (*CustomDisk) Type added in v2.8.1 

func (d *CustomDisk) Type() string

Type - lower-cased `type` argument, `local` when absent (ClickHouse DiskFactory default)

type Database 

type Database struct {
	Name   string `ch:"name"`
	Engine string `ch:"engine"`
	Query  string `ch:"query"`
}

Database - Clickhouse system.databases struct

type Disk 

type Disk struct {
	Name            string   `ch:"name"`
	Path            string   `ch:"path"`
	Type            string   `ch:"type"`
	MetadataType    string   `ch:"metadata_type"`
	FreeSpace       uint64   `ch:"free_space"`
	TotalSpace      uint64   `ch:"total_space"`
	StoragePolicies []string `ch:"storage_policies"`
	IsBackup        bool
	// RawPath - `system.disks.path` as reported by clickhouse-server, before GetDisks rewrites it: the
	// bucket key prefix of plain/plain_rewritable disks becomes a pseudo local path, a relative local path
	// (clickhouse-server started with a relative `<path>`) becomes absolute via disk_mapping. Empty for a
	// plain disk which lives in the bucket root (azure disk without `common_key_prefix` on ClickHouse 24.8/25.3)
	RawPath string
}

func (Disk) IsPlain added in v2.8.1 

func (d Disk) IsPlain() bool

IsPlain returns true for disks without local VFS metadata files (metadata_type=plain or plain_rewritable)

type Function 

type Function struct {
	Name        string `ch:"name"`
	CreateQuery string `ch:"create_query"`
}

Function - Clickhouse system.functions struct

type IsSystemTablesFieldPresent 

type IsSystemTablesFieldPresent struct {
	IsDataPathPresent         uint64 `ch:"is_data_path_present"`
	IsDataPathsPresent        uint64 `ch:"is_data_paths_present"`
	IsUUIDPresent             uint64 `ch:"is_uuid_present"`
	IsCreateTableQueryPresent uint64 `ch:"is_create_table_query_present"`
	IsTotalBytesPresent       uint64 `ch:"is_total_bytes_present"`
}

IsSystemTablesFieldPresent - ClickHouse `system.tables` varius field flags

type Macro 

type Macro struct {
	Macro        string `ch:"macro"`
	Substitution string `ch:"substitution"`
}

Macro - info from system.macros

type RBACObject added in v2.5.0 

type RBACObject struct {
	Id   string `ch:"id"`
	Name string `ch:"name"`
}

type ShardBackupType 

type ShardBackupType string

type SystemBackups 

type SystemBackups struct {
	Id                string            `ch:"id"`
	UUID              string            `ch:"uuid"`
	QueryId           string            `ch:"query_id"`
	BackupName        string            `ch:"backup_name"`
	BaseBackupName    string            `ch:"base_backup_name"`
	Name              string            `ch:"name"`
	Status            string            `ch:"status"`
	StatusChangedTime time.Time         `ch:"status_changed_time"`
	Error             string            `ch:"error"`
	Internal          bool              `ch:"internal"`
	StartTime         time.Time         `ch:"start_time"`
	EndTime           time.Time         `ch:"end_time"`
	CompressedSize    uint64            `ch:"compressed_size"`
	UncompressedSize  uint64            `ch:"uncompressed_size"`
	NumFiles          uint64            `ch:"num_files"`
	TotalSize         uint64            `ch:"total_size"`
	NumEntries        uint64            `ch:"num_entries"`
	FilesRead         uint64            `ch:"files_read"`
	BytesRead         uint64            `ch:"bytes_read"`
	ProfileEvents     map[string]uint64 `ch:"ProfileEvents"`
}

SystemBackups - info from system.backups, universal for all versions

type Table 

type Table struct {
	// common fields for all `clickhouse-server` versions
	Database string `ch:"database"`
	Name     string `ch:"name"`
	Engine   string `ch:"engine"`
	// fields depends on `clickhouse-server` version
	DataPath         string   `ch:"data_path"`
	DataPaths        []string `ch:"data_paths"`
	UUID             string   `ch:"uuid"`
	CreateTableQuery string   `ch:"create_table_query"`
	TotalBytes       uint64   `ch:"total_bytes"`
	// StoragePolicy - only filled by GetCustomDiskTables, `__<disk name>` for `SETTINGS disk = ...` tables
	StoragePolicy string `ch:"storage_policy"`
	Skip          bool
	BackupType    ShardBackupType
}

Table - ClickHouse table struct

type UserDirectory added in v2.5.0 

type UserDirectory struct {
	Name string `ch:"name"`
	Type string `ch:"type"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.