backup

package
v2.8.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 16, 2026 License: MIT Imports: 68 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// TimeFormatForBackup - default backup name format
	TimeFormatForBackup = "2006-01-02T15-04-05"
	MetaFileName        = "metadata.json"
)
View Source 
const DirectoryFormat = "directory"

Variables

View Source 
var CreateDatabaseRE = regexp.MustCompile(`(?m)^CREATE DATABASE (\s*)(\S+)(\s*)(.*)`)
View Source 
var (
	ErrBackupIsAlreadyExists = errors.New("backup is already exists")
)
View Source 
var (
	// ErrUnknownClickhouseDataPath -
	ErrUnknownClickhouseDataPath = errors.New("clickhouse data path is unknown, you can set data_path in config file")
)
View Source 
var UUIDWithMergeTreeRE = regexp.MustCompile(`^(.+)(UUID)(\s+)'([^']+)'(.+)({uuid})(.*)`)

Functions

func IsInformationSchema 

func IsInformationSchema(database string) bool

func NewBackupName 

func NewBackupName() string

NewBackupName - return default backup name

func ShallSkipDatabase 

func ShallSkipDatabase(cfg *config.Config, targetDB, tablePattern string) bool

Types

type BackupInfo added in v2.6.32 

type BackupInfo struct {
	BackupName     string
	CreationDate   time.Time
	Size           string
	Description    string
	RequiredBackup string // for incremental backup, this is the base full backup
	Type           string // local or remote
}

type Backuper 

type Backuper struct {
	DiskToPathMap          map[string]string
	DefaultDataPath        string
	EmbeddedBackupDataPath string

	// DryRun report what the command would do instead of doing it, see issues/1012
	DryRun bool
	// DryRunResult holds the report produced when DryRun is set, so REST API handlers can read it after the command returns
	DryRunResult *DryRunReport
	// DiskLimit - max allowed local disk usage in percent after download, 0 disables the check, see issues/1458
	DiskLimit int
	// contains filtered or unexported fields
}

func NewBackuper 

func NewBackuper(cfg *config.Config, opts ...BackuperOpt) *Backuper

func (*Backuper) AddTableToLocalBackup added in v2.5.0 

func (b *Backuper) AddTableToLocalBackup(ctx context.Context, backupName string, tablesDiffFromRemote map[metadata.TableTitle]metadata.TableMetadata, shadowBackupUUID string, diskList []clickhouse.Disk, table *clickhouse.Table, partitionsIdsMap common.EmptyMap, skipProjections []string, version int) (map[string][]metadata.Part, map[string]int64, map[string]int64, map[string]uint64, map[string]string, map[string][]metadata.Part, error)

AddTableToLocalBackup freezes a table and moves its parts into the local backup. It returns the per-disk parts/sizes/checksums plus the per-disk data parts that were skipped as broken. Broken parts are only tolerated (skipped instead of aborting) when general.max_broken_part_ratio > 0; the caller aggregates the broken/total counts across all tables, enforces the configured ratio and stores the broken parts in the table metadata, see https://github.com/Altinity/clickhouse-backup/issues/1418

func (*Backuper) CalculateMaxSize added in v2.6.0 

func (b *Backuper) CalculateMaxSize(ctx context.Context) error

CalculateMaxSize https://github.com/Altinity/clickhouse-backup/issues/404

func (*Backuper) CalculateNonSkipTables added in v2.6.3 

func (b *Backuper) CalculateNonSkipTables(tables []clickhouse.Table) int

func (*Backuper) CheckDisksUsage added in v2.6.6 

func (b *Backuper) CheckDisksUsage(backup storage.Backup, disks []clickhouse.Disk, isResumeExists bool, tablePattern string) error

CheckDisksUsage - https://github.com/Altinity/clickhouse-backup/issues/878

func (*Backuper) Classify added in v2.6.24 

func (b *Backuper) Classify(err error) retrier.Action

Classify need to log retries

func (*Backuper) Clean 

func (b *Backuper) Clean(commandId int, all bool, olderThan time.Duration) error

Clean - removed all data in shadow folder Clean unfreezes the shadow uuids recorded in <backup_name>/freezes.tmp of every local backup which is not processed by a live clickhouse-backup process, `olderThan` > 0 additionally removes unrecorded shadow/* directories (orphans of versions before freezes.tmp or manual FREEZE) not modified for that long, `all` restores the historical behavior and wipes the whole shadow/ on every disk, see https://github.com/Altinity/clickhouse-backup/issues/1563

func (*Backuper) CleanBrokenRetention added in v2.7.0 

func (b *Backuper) CleanBrokenRetention(commandId int, includeGlobs, excludeGlobs []string, commit bool) error

CleanBrokenRetention walks remote `path` and `object_disks_path` top-level entries and removes everything that is NOT present in the live BackupList and NOT matched by excludeGlobs. Uses BatchDeleter with retry and parallel batch deletion for object_disks_path orphans. When commit=false, only logs orphans without deleting (dry-run mode). When includeGlobs is non-empty, only orphans matching at least one includeGlob are considered. excludeGlobs and includeGlobs follow path.Match syntax (e.g. "prod-*", "snapshot-2026-??-*").

func (*Backuper) CleanLocalBroken added in v2.6.27 

func (b *Backuper) CleanLocalBroken(commandId int) error

func (*Backuper) CleanRemoteBroken 

func (b *Backuper) CleanRemoteBroken(commandId int, includeGlobs []string) error

func (*Backuper) CollectAllBackups added in v2.6.32 

func (b *Backuper) CollectAllBackups(ctx context.Context, ptype string) ([]BackupInfo, error)

func (*Backuper) CollectLocalBackups added in v2.6.32 

func (b *Backuper) CollectLocalBackups(ctx context.Context, ptype string) ([]BackupInfo, error)

func (*Backuper) CollectRemoteBackups added in v2.6.32 

func (b *Backuper) CollectRemoteBackups(ctx context.Context, ptype string) ([]BackupInfo, error)

func (*Backuper) CreateBackup 

func (b *Backuper) CreateBackup(backupName, diffFromRemote, tablePattern string, partitions []string, schemaOnly, createRBAC, rbacOnly, createConfigs, configsOnly, createNamedCollections, namedCollectionsOnly, skipCheckPartsColumns bool, skipProjections []string, resume bool, backupVersion string, commandId int) error

CreateBackup - create new backup of all tables matched by tablePattern If backupName is empty string will use default backup name

func (*Backuper) CreateToRemote 

func (b *Backuper) CreateToRemote(backupName string, deleteSource bool, diffFrom, diffFromRemote, tablePattern string, partitions, skipProjections []string, schemaOnly, backupRBAC, rbacOnly, backupConfigs, configsOnly, namedCollections, namedCollectionsOnly, skipCheckPartsColumns, resume, streaming bool, version string, commandId int) error

func (*Backuper) Delete 

func (b *Backuper) Delete(backupType, backupName string, force bool, commandId int) error

Delete - remove local or remote backup

func (*Backuper) Download 

func (b *Backuper) Download(backupName string, tablePattern string, partitions []string, schemaOnly, rbacOnly, configsOnly, namedCollectionsOnly, resume bool, hardlinkExistsFiles bool, backupVersion string, commandId int) error

func (*Backuper) GetLocalBackups 

func (b *Backuper) GetLocalBackups(ctx context.Context, disks []clickhouse.Disk) ([]LocalBackup, []clickhouse.Disk, error)

GetLocalBackups - return slice of all backups stored locally

func (*Backuper) GetLocalDataSize added in v2.6.3 

func (b *Backuper) GetLocalDataSize(ctx context.Context) (float64, error)

func (*Backuper) GetRemoteBackups 

func (b *Backuper) GetRemoteBackups(ctx context.Context, parseMetadata bool) ([]storage.Backup, error)

GetRemoteBackups - get all backups stored on remote storage

func (*Backuper) GetStateDir added in v2.6.3 

func (b *Backuper) GetStateDir() string

func (*Backuper) GetTableRowsForLive added in v2.8.0 

func (b *Backuper) GetTableRowsForLive(ctx context.Context, tablePattern string, printAll, listParts, listPartitions bool) ([]TableRow, error)

GetTableRowsForLive returns per-table rows for the live ClickHouse server, optionally including the parts/partitions breakdown from `system.parts` (see collectTablesFromLive); intended for callers like the REST API. When printAll is false, tables matching skip_tables are filtered out.

func (*Backuper) GetTableRowsForLocalBackup added in v2.7.0 

func (b *Backuper) GetTableRowsForLocalBackup(ctx context.Context, backupName, tablePattern string, printAll, listParts, listPartitions bool) ([]TableRow, error)

GetTableRowsForLocalBackup returns per-table rows (db, table, size, parts, disks, skip) for a local backup, reading metadata from disk; intended for callers like the REST API. When printAll is false, tables matching skip_tables are filtered out.

func (*Backuper) GetTableRowsForRemoteBackup added in v2.7.0 

func (b *Backuper) GetTableRowsForRemoteBackup(ctx context.Context, backupName, tablePattern string, printAll, listParts, listPartitions bool) ([]TableRow, error)

GetTableRowsForRemoteBackup returns per-table rows (db, table, size, parts, disks, skip) for a remote backup, downloading per-table metadata; intended for callers like the REST API. When printAll is false, tables matching skip_tables are filtered out.

func (*Backuper) GetTables 

func (b *Backuper) GetTables(ctx context.Context, tablePattern string) ([]clickhouse.Table, error)

GetTables - get all tables for use by CreateBackup, PrintTables, and API

func (*Backuper) List 

func (b *Backuper) List(what, ptype, format string) error

List - list backups to stdout from command line

func (*Backuper) NewBackupWatchName 

func (b *Backuper) NewBackupWatchName(ctx context.Context, backupType string) (string, error)

func (*Backuper) PrintBackup added in v2.6.32 

func (b *Backuper) PrintBackup(backupInfos []BackupInfo, format string) error

func (*Backuper) PrintTables 

func (b *Backuper) PrintTables(printAll bool, tablePattern, remoteBackup, localBackup, format string, listParts, listPartitions bool) error

PrintTables - print all tables suitable for backup. When localBackup or remoteBackup is set, list tables from the corresponding backup (per-table size and parts count are read from `metadata.TableMetadata`); both flags may be set simultaneously to render `local` and `remote` sections in one go. Otherwise tables are read from the live ClickHouse server. `format` controls output: text (default), json, yaml, csv, tsv. `listParts` and `listPartitions` are independent toggles, usable alone or together, against the live server or `--local-backup`/`--remote-backup` alike:

  • listParts attaches every physical part (name, partition_id, and against the live server also size), read from `system.parts` live or from backup metadata otherwise.
  • listPartitions attaches the distinct partitions (parts grouped by partition_id), with the human-readable partition value and size only available against the live server.

Against `--local-backup`/`--remote-backup`, partition_id is derived from each part's name (the `_`-delimited prefix, same convention as filesystemhelper.IsPartInPartition).

func (*Backuper) ReadBackupMetadataLocal 

func (b *Backuper) ReadBackupMetadataLocal(ctx context.Context, backupName string) (*metadata.BackupMetadata, error)

func (*Backuper) ReadBackupMetadataRemote 

func (b *Backuper) ReadBackupMetadataRemote(ctx context.Context, backupName string) (*metadata.BackupMetadata, error)

func (*Backuper) Rebalance added in v2.8.0 

func (b *Backuper) Rebalance(backupName string, tablePattern string, dryRun bool, commandId int) error

Rebalance - move data parts of a local backup between disks, https://github.com/Altinity/clickhouse-backup/issues/1024 Rule 1: the live part from system.parts is on the same disk as in the backup - do nothing; Rule 2: the live part is on another usable disk - hardlink it into the backup shadow on that disk; Rule 3: the part is not in system.parts - copy it to the policy disk of the same type with the most effective free space; a part on a still valid disk (present in system.disks and in the table storage policy) moves only to a strictly roomier disk, a part on an invalid disk must move or the command fails; parts on object disks are skipped, metadata in table.json / metadata.json is rewritten accordingly

func (*Backuper) Rebase added in v2.8.0 

func (b *Backuper) Rebase(backupName string, commandId int) error

Rebase - copy required parts from the required backups chain into backupName on remote storage and remove the required_backup dependency, so backupName becomes a full backup

func (*Backuper) RemoveBackupLocal 

func (b *Backuper) RemoveBackupLocal(ctx context.Context, backupName string, disks []clickhouse.Disk, force bool) error

RemoveBackupLocal - delete backupName from local disks, when force is false and some other local backup requires backupName via `required_backup`, the backup is kept and an error is returned, see https://github.com/Altinity/clickhouse-backup/issues/1493

func (*Backuper) RemoveBackupRemote 

func (b *Backuper) RemoveBackupRemote(ctx context.Context, backupName string, force bool) error

RemoveBackupRemote - delete backupName from remote storage, when some other remote backup requires backupName via `required_backup`, the behavior depends on `general.rebase_during_delete`: when enabled every dependent backup is rebased first (so the chain stays restorable), otherwise the backup is kept and an error is returned unless force is set, see https://github.com/Altinity/clickhouse-backup/issues/1493

func (*Backuper) RemoveOldBackupsLocal 

func (b *Backuper) RemoveOldBackupsLocal(ctx context.Context, keepLastBackup bool, disks []clickhouse.Disk) error

func (*Backuper) RemoveOldBackupsRemote added in v2.4.28 

func (b *Backuper) RemoveOldBackupsRemote(ctx context.Context) error

func (*Backuper) Restore 

func (b *Backuper) Restore(backupName, tablePattern string, databaseMapping, tableMapping, partitions, skipProjections []string, schemaOnly, dataOnly, dropExists, ignoreDependencies, restoreRBAC, rbacOnly, restoreConfigs, configsOnly, restoreNamedCollections, namedCollectionsOnly, resume, schemaAsAttach, replicatedCopyToDetached, skipEmptyTables bool, backupVersion string, commandId int) error

Restore - restore tables matched by tablePattern from backupName

func (*Backuper) RestoreCloud added in v2.8.1 

func (b *Backuper) RestoreCloud(opts RestoreCloudOptions, commandId int) error

func (*Backuper) RestoreData 

func (b *Backuper) RestoreData(ctx context.Context, backupName string, backupMetadata metadata.BackupMetadata, dataOnly bool, metadataPath, tablePattern string, partitions, skipProjections []string, disks []clickhouse.Disk, version int, replicatedCopyToDetached bool, filteredTables ListOfTables, existingTablesSnapshot []clickhouse.Table) error

RestoreData - restore data for tables matched by tablePattern from backupName

func (*Backuper) RestoreFromRemote 

func (b *Backuper) RestoreFromRemote(backupName, tablePattern string, databaseMapping, tableMapping, partitions, skipProjections []string, schemaOnly, dataOnly, dropExists, ignoreDependencies, restoreRBAC, rbacOnly, restoreConfigs, configsOnly, restoreNamedCollections, namedCollectionsOnly, resume, schemaAsAttach, replicatedCopyToDetached, skipEmptyTables, hardlinkExistsFiles, streaming bool, version string, commandId int) error

func (*Backuper) RestoreSchema 

func (b *Backuper) RestoreSchema(ctx context.Context, backupName string, backupMetadata metadata.BackupMetadata, disks []clickhouse.Disk, tablesForRestore ListOfTables, ignoreDependencies bool, version int, schemaAsAttach bool) error

RestoreSchema - restore schemas matched by tablePattern from backupName

func (*Backuper) Upload 

func (b *Backuper) Upload(backupName string, deleteSource bool, diffFrom, diffFromRemote, tablePattern string, partitions, skipProjections []string, schemaOnly, rbacOnly, configsOnly, namedCollectionsOnly, resume bool, backupVersion string, commandId int) error

func (*Backuper) ValidateWatchParams 

func (b *Backuper) ValidateWatchParams(watchInterval, fullInterval, watchBackupNameTemplate string, schedules []string) error

func (*Backuper) Watch 

func (b *Backuper) Watch(watchInterval, fullInterval, watchBackupNameTemplate string, schedules []string, tablePattern string, partitions, skipProjections []string, schemaOnly, backupRBAC, backupConfigs, backupNamedCollections, skipCheckPartsColumns, deleteSource, streaming bool, version string, commandId int, metrics *metrics.APIMetrics, cliCtx *cli.Command) error

Watch - run create_remote full + delete local full, even when upload failed

  • if success save backup type full, next will increment, until reach full interval
  • if fail save previous backup type empty, next try will also full

- each watch-interval, run create_remote increment --diff-from=prev-name + delete local increment, even when upload failed

  • save previous backup type incremental, next try will also incremental, until reach full interval

type BackuperOpt 

type BackuperOpt func(*Backuper)

func WithBackupSharder 

func WithBackupSharder(s backupSharder) BackuperOpt

func WithVersioner 

func WithVersioner(v versioner) BackuperOpt

type DryRunReport added in v2.8.1 

type DryRunReport struct {
	Command              string `json:"command"`
	BackupName           string `json:"backup_name,omitempty"`
	DryRun               bool   `json:"dry_run"` // always true
	TableCount           int    `json:"table_count"`
	PartsCount           int    `json:"parts_count,omitempty"`
	DataSize             uint64 `json:"data_size"`
	CompressedSize       uint64 `json:"compressed_size,omitempty"`
	ObjectDiskSize       uint64 `json:"object_disk_size,omitempty"`
	MetadataSize         uint64 `json:"metadata_size,omitempty"`
	RBACSize             uint64 `json:"rbac_size,omitempty"`
	ConfigSize           uint64 `json:"config_size,omitempty"`
	NamedCollectionsSize uint64 `json:"named_collections_size,omitempty"`
	TotalSize            uint64 `json:"total_size"`
	// UnknownSizeParts counts parts whose size is not recorded in metadata (older backups)
	UnknownSizeParts int `json:"unknown_size_parts,omitempty"`
	// create only: hardlinks cost nothing now, but become owned as merges remove originals
	HardlinkMaxSize     uint64 `json:"hardlink_max_size,omitempty"`
	HardlinkEstimate1d  uint64 `json:"hardlink_estimate_1d,omitempty"`
	HardlinkEstimate7d  uint64 `json:"hardlink_estimate_7d,omitempty"`
	HardlinkEstimate30d uint64 `json:"hardlink_estimate_30d,omitempty"`
	// DependentBackups is delete only, backups which reference the deleted one via required_backup
	DependentBackups []string `json:"dependent_backups,omitempty"`
}

DryRunReport describes what a command would do, without doing it. issues/1012

func (*DryRunReport) JSONString added in v2.8.1 

func (r *DryRunReport) JSONString() string

JSONString marshals the report for the `result` field of a status row, it returns an empty string for a nil report so callers don't need to check

type InfoResult added in v2.7.0 

type InfoResult struct {
	BackupName   string     `json:"backup_name" yaml:"backup_name"`
	BackupType   string     `json:"backup_type" yaml:"backup_type"`
	TablePattern string     `json:"table_pattern,omitempty" yaml:"table_pattern,omitempty"`
	TableCount   int        `json:"table_count" yaml:"table_count"`
	TotalBytes   uint64     `json:"total_bytes" yaml:"total_bytes"`
	TotalSize    string     `json:"total_size" yaml:"total_size"`
	TotalParts   int        `json:"total_parts" yaml:"total_parts"`
	Tables       []TableRow `json:"tables" yaml:"tables"`
}

InfoResult wraps a per-backup result with aggregate fields for JSON/YAML output of `tables --local-backup` / `tables --remote-backup`.

type ListOfTables 

type ListOfTables []*metadata.TableMetadata

func (ListOfTables) Sort 

func (lt ListOfTables) Sort(dropTable bool)

Sort - sorting ListOfTables slice orderly by engine priority

type LocalBackup 

type LocalBackup struct {
	metadata.BackupMetadata
	Broken string
}

func GetBackupsToDeleteLocal added in v2.4.28 

func GetBackupsToDeleteLocal(backups []LocalBackup, keep int) []LocalBackup

type PartRow added in v2.8.0 

type PartRow struct {
	Name        string `json:"name" yaml:"name" csv:"name"`
	PartitionID string `json:"partition_id" yaml:"partition_id" csv:"partition_id"`
	TotalBytes  uint64 `json:"total_bytes,omitempty" yaml:"total_bytes,omitempty" csv:"total_bytes"`
	Size        string `json:"size,omitempty" yaml:"size,omitempty" csv:"size"`
}

PartRow describes one physical data part of a table, as returned by `tables --list-parts` (alias `--parts`). Against the live server it comes straight from `system.parts`; against `--local-backup`/`--remote-backup` only Name and PartitionID are available (from backup metadata), PartitionID being the `_`-delimited prefix of Name, same convention as filesystemhelper.IsPartInPartition.

type PartitionRow added in v2.8.0 

type PartitionRow struct {
	PartitionID string `json:"partition_id" yaml:"partition_id" csv:"partition_id"`
	Partition   string `json:"partition,omitempty" yaml:"partition,omitempty" csv:"partition"`
	Parts       int    `json:"parts" yaml:"parts" csv:"parts"`
	TotalBytes  uint64 `json:"total_bytes,omitempty" yaml:"total_bytes,omitempty" csv:"total_bytes"`
	Size        string `json:"size,omitempty" yaml:"size,omitempty" csv:"size"`
}

PartitionRow describes one distinct partition of a table (parts grouped by partition_id), as returned by `tables --partitions` (alias `--list-partitions`). Against the live server it comes from `system.parts`, including the human-readable Partition value and total size. Against `--local-backup`/`--remote-backup` it's aggregated from part names (same convention as PartRow.PartitionID), so Partition/TotalBytes/Size stay empty/zero there.

type RestoreCloudOptions added in v2.8.1 

type RestoreCloudOptions struct {
	Prefix            string // key prefix of the backup inside the bucket/container (directory that contains `.backup`)
	Bucket            string
	Region            string
	Endpoint          string
	Container         string // azblob container, switches the source to AzureBlobStorage
	BasePrefix        string // prefix of the base backup for incremental backups with use_base
	S3RestoreURL      string // URL passed to RESTORE ... FROM S3('...'), default https://s3.<region>.amazonaws.com/<bucket>/<prefix>
	AzblobRestoreURL  string // blob endpoint passed to RESTORE ... FROM AzureBlobStorage(...), e.g. http://azurite:10000/devstoreaccount1, switches the source to AzureBlobStorage
	TablePattern      string
	Partitions        []string
	RestoreOnCluster  string // cluster name for CREATE/RESTORE ... ON CLUSTER, macros like {cluster} are resolved
	ReplicatedZkPath  string
	ReplicatedReplica string
	SkipEmptyTables   bool
	ContinueOnError   bool
	Drop              bool // DROP TABLE / DICTIONARY IF EXISTS ... SYNC before CREATE, to re-run a failed or interrupted restore
	Parallel          int  // tables restored concurrently inside one database, default runtime.NumCPU()
}

type TableRow added in v2.7.0 

type TableRow struct {
	Database   string         `json:"database" yaml:"database" csv:"database"`
	Table      string         `json:"table" yaml:"table" csv:"table"`
	TotalBytes uint64         `json:"total_bytes" yaml:"total_bytes" csv:"total_bytes"`
	Size       string         `json:"size" yaml:"size" csv:"size"`
	Parts      int            `json:"parts" yaml:"parts" csv:"parts"`
	Disks      []string       `json:"disks" yaml:"disks" csv:"-"`
	DisksStr   string         `json:"-" yaml:"-" csv:"disks"`
	Skip       bool           `json:"skip" yaml:"skip" csv:"skip"`
	BackupType string         `json:"backup_type,omitempty" yaml:"backup_type,omitempty" csv:"backup_type"`
	PartsList  []PartRow      `json:"parts_list,omitempty" yaml:"parts_list,omitempty" csv:"-"`
	Partitions []PartitionRow `json:"partitions,omitempty" yaml:"partitions,omitempty" csv:"-"`
}

TableRow is the output projection used by the `tables` command for non-text formats. Disks is exposed as a structured list in JSON/YAML and as a comma-joined string in CSV/TSV.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.