restore

package
v0.0.0-...-e056406 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 18, 2025 License: Apache-2.0, Apache-2.0 Imports: 48 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// DefaultMergeRegionSizeBytes is the default region split size, 96MB.
	// See https://github.com/tikv/tikv/blob/v4.0.8/components/raftstore/src/coprocessor/config.rs#L35-L38
	DefaultMergeRegionSizeBytes uint64 = 96 * units.MiB

	// DefaultMergeRegionKeyCount is the default region key count, 960000.
	DefaultMergeRegionKeyCount uint64 = 960000
)
View Source 
const (
	SplitRetryTimes       = 32
	SplitRetryInterval    = 50 * time.Millisecond
	SplitMaxRetryInterval = time.Second

	SplitCheckMaxRetryTimes = 64
	SplitCheckInterval      = 8 * time.Millisecond
	SplitMaxCheckInterval   = time.Second

	ScatterWaitMaxRetryTimes = 64
	ScatterWaitInterval      = 50 * time.Millisecond
	ScatterMaxWaitInterval   = time.Second
	ScatterWaitUpperInterval = 180 * time.Second

	ScanRegionPaginationLimit = 128

	RejectStoreCheckRetryTimes  = 64
	RejectStoreCheckInterval    = 100 * time.Millisecond
	RejectStoreMaxCheckInterval = 2 * time.Second
)

Constants for split retry machinery.

Variables

View Source 
var (
	ScanRegionAttemptTimes = 30
)

Functions

func CheckRegionConsistency 

func CheckRegionConsistency(startKey, endKey []byte, regions []*RegionInfo) error

func GetSSTMetaFromFile 

func GetSSTMetaFromFile(
	id []byte,
	file *backuppb.File,
	region *metapb.Region,
	regionRule *import_sstpb.RewriteRule,
) import_sstpb.SSTMeta

GetSSTMetaFromFile compares the keys in file, region and rewrite rules, then returns a sst conn. The range of the returned sst meta is [regionRule.NewKeyPrefix, append(regionRule.NewKeyPrefix, 0xff)].

func SortRanges 

func SortRanges(ranges []rtree.Range, rewriteRules *RewriteRules) ([]rtree.Range, error)

SortRanges checks if the range overlapped and sort them.

func SplitRanges 

func SplitRanges(
	ctx context.Context,
	client *Client,
	spliterConf SpliterConfig,
	ranges []rtree.Range,
	rewriteRules *RewriteRules,
	updateCh glue.Progress,
	isRawKv bool,
	needEncodeKey bool,
) error

SplitRanges splits region by 1. data range after rewrite. 2. rewrite rules.

Types

type Client 

type Client struct {
	// contains filtered or unexported fields
}

Client sends requests to restore files.

func NewRestoreClient 

func NewRestoreClient(
	pdClient pd.Client,
	tlsConf *tls.Config,
	keepaliveConf keepalive.ClientParameters,
	spliterConf SpliterConfig,
	isRawKv bool,
) (*Client, error)

NewRestoreClient returns a new RestoreClient.

func (*Client) Close 

func (rc *Client) Close()

Close a client.

func (*Client) EnableOnline 

func (rc *Client) EnableOnline()

EnableOnline sets the mode of restore to online.

func (*Client) GetAPIVersion 

func (rc *Client) GetAPIVersion() kvrpcpb.APIVersion

func (*Client) GetFilesInRawRange 

func (rc *Client) GetFilesInRawRange(startKey []byte, endKey []byte, cf string) ([]*backuppb.File, error)

GetFilesInRawRange gets all files that are in the given range or intersects with the given range.

func (*Client) GetPDClient 

func (rc *Client) GetPDClient() pd.Client

GetPDClient returns a pd client.

func (*Client) GetTLSConfig 

func (rc *Client) GetTLSConfig() *tls.Config

GetTLSConfig returns the tls config.

func (*Client) GetTS 

func (rc *Client) GetTS(ctx context.Context) (uint64, error)

GetTS gets a new timestamp from PD.

func (*Client) InitBackupMeta 

func (rc *Client) InitBackupMeta(
	c context.Context,
	backupMeta *backuppb.BackupMeta,
	backend *backuppb.StorageBackend,
	externalStorage storage.ExternalStorage,
	reader *metautil.MetaReader) error

InitBackupMeta loads schemas from BackupMeta to initialize RestoreClient.

func (*Client) IsOnline 

func (rc *Client) IsOnline() bool

IsOnline tells if it's a online restore.

func (*Client) IsRawKvMode 

func (rc *Client) IsRawKvMode() bool

IsRawKvMode checks whether the backup data is in raw kv format, in which case transactional recover is forbidden.

func (*Client) RestoreRaw 

func (rc *Client) RestoreRaw(
	ctx context.Context, startKey []byte, endKey []byte, files []*backuppb.File, updateCh glue.Progress,
) error

RestoreRaw tries to restore raw keys in the specified range.

func (*Client) SetConcurrency 

func (rc *Client) SetConcurrency(c uint)

SetConcurrency sets the concurrency of dbs tables files.

func (*Client) SetCrypter 

func (rc *Client) SetCrypter(crypter *backuppb.CipherInfo)

func (*Client) SetRateLimit 

func (rc *Client) SetRateLimit(rateLimit uint64)

SetRateLimit to set rateLimit.

func (*Client) SetStorage 

func (rc *Client) SetStorage(ctx context.Context, backend *backuppb.StorageBackend, opts *storage.ExternalStorageOptions) error

SetStorage set ExternalStorage for client.

func (*Client) SetSwitchModeInterval 

func (rc *Client) SetSwitchModeInterval(interval time.Duration)

SetSwitchModeInterval set switch mode interval for client.

func (*Client) SwitchToImportMode 

func (rc *Client) SwitchToImportMode(ctx context.Context)

SwitchToImportMode switch tikv cluster to import mode.

func (*Client) SwitchToNormalMode 

func (rc *Client) SwitchToNormalMode(ctx context.Context) error

SwitchToNormalMode switch tikv cluster to normal mode.

type FileImporter 

type FileImporter struct {
	// contains filtered or unexported fields
}

FileImporter used to import a file to TiKV.

func NewFileImporter 

func NewFileImporter(
	metaClient SplitClient,
	importClient ImporterClient,
	backend *backuppb.StorageBackend,
	isRawKvMode bool,
	apiVersion kvrpcpb.APIVersion,
) FileImporter

NewFileImporter returns a new file importClient.

func (*FileImporter) CheckMultiIngestSupport 

func (importer *FileImporter) CheckMultiIngestSupport(ctx context.Context, pdClient pd.Client) error

CheckMultiIngestSupport checks whether all stores support multi-ingest

func (*FileImporter) Import 

func (importer *FileImporter) Import(
	ctx context.Context,
	files []*backuppb.File,
	rewriteRules *RewriteRules,
	cipher *backuppb.CipherInfo,
) error

Import tries to import a file. All rules must contain encoded keys.

func (*FileImporter) SetRawRange 

func (importer *FileImporter) SetRawRange(startKey, endKey []byte) error

SetRawRange sets the range to be restored in raw kv mode.

type ImporterClient 

type ImporterClient interface {
	DownloadSST(
		ctx context.Context,
		storeID uint64,
		req *import_sstpb.DownloadRequest,
	) (*import_sstpb.DownloadResponse, error)

	IngestSST(
		ctx context.Context,
		storeID uint64,
		req *import_sstpb.IngestRequest,
	) (*import_sstpb.IngestResponse, error)
	MultiIngest(
		ctx context.Context,
		storeID uint64,
		req *import_sstpb.MultiIngestRequest,
	) (*import_sstpb.IngestResponse, error)

	SetDownloadSpeedLimit(
		ctx context.Context,
		storeID uint64,
		req *import_sstpb.SetDownloadSpeedLimitRequest,
	) (*import_sstpb.SetDownloadSpeedLimitResponse, error)

	GetImportClient(
		ctx context.Context,
		storeID uint64,
	) (import_sstpb.ImportSSTClient, error)

	SupportMultiIngest(ctx context.Context, stores []uint64) (bool, error)
}

ImporterClient is used to import a file to TiKV.

func NewImportClient 

func NewImportClient(metaClient SplitClient, tlsConf *tls.Config, keepaliveConf keepalive.ClientParameters) ImporterClient

NewImportClient returns a new ImporterClient.

type MergeRangesStat 

type MergeRangesStat struct {
	TotalFiles           int
	TotalWriteCFFile     int
	TotalDefaultCFFile   int
	TotalRegions         int
	RegionKeysAvg        int
	RegionBytesAvg       int
	MergedRegions        int
	MergedRegionKeysAvg  int
	MergedRegionBytesAvg int
}

MergeRangesStat holds statistics for the MergeRanges.

func MergeFileRanges 

func MergeFileRanges(
	files []*backuppb.File, splitSizeBytes, splitKeyCount uint64,
) ([]rtree.Range, *MergeRangesStat, error)

MergeFileRanges returns ranges of the files are merged based on splitSizeBytes and splitKeyCount.

By merging small ranges, it speeds up restoring a backup that contains many small ranges (regions) as it reduces split region and scatter region.

type OnSplitFunc 

type OnSplitFunc func(key [][]byte)

OnSplitFunc is called before split a range.

type Range 

type Range struct {
	Start []byte
	End   []byte
}

Range record start and end key for localStoreDir.DB so we can write it to tikv in streaming

type RegionInfo 

type RegionInfo struct {
	Region *metapb.Region
	Leader *metapb.Peer
}

RegionInfo includes a region and the leader of the region.

func NeedSplit 

func NeedSplit(splitKey []byte, regions []*RegionInfo, needEncodeKey bool) *RegionInfo

NeedSplit checks whether a key is necessary to split, if true returns the split region.

func PaginateScanRegion 

func PaginateScanRegion(
	ctx context.Context, client SplitClient, startKey, endKey []byte, limit int,
) ([]*RegionInfo, error)

PaginateScanRegion scan regions with a limit pagination and return all regions at once. It reduces max gRPC message size.

func (*RegionInfo) ContainsInterior 

func (region *RegionInfo) ContainsInterior(key []byte) bool

ContainsInterior returns whether the region contains the given key, and also that the key does not fall on the boundary (start key) of the region.

type RegionSplitter 

type RegionSplitter struct {
	// contains filtered or unexported fields
}

RegionSplitter is a executor of region split by rules.

func NewRegionSplitter 

func NewRegionSplitter(client SplitClient, conf SpliterConfig) *RegionSplitter

NewRegionSplitter returns a new RegionSplitter.

func (*RegionSplitter) ScatterRegions 

func (rs *RegionSplitter) ScatterRegions(ctx context.Context, newRegions []*RegionInfo)

ScatterRegions scatter the regions.

func (*RegionSplitter) ScatterRegionsWithBackoffer 

func (rs *RegionSplitter) ScatterRegionsWithBackoffer(ctx context.Context, newRegions []*RegionInfo, backoffer utils.Backoffer)

ScatterRegionsWithBackoffer scatter the region with some backoffer. This function is for testing the retry mechanism. For a real cluster, directly use ScatterRegions would be fine.

func (*RegionSplitter) Split 

func (rs *RegionSplitter) Split(
	ctx context.Context,
	ranges []rtree.Range,
	rewriteRules *RewriteRules,
	needEncodeKey bool,
	onSplit OnSplitFunc,
) error

Split executes a region split. It will split regions by the rewrite rules, then it will split regions by the end key of each range. tableRules includes the prefix of a table, since some ranges may have a prefix with record sequence or index sequence. note: all ranges and rewrite rules must have raw key.

type RewriteRules 

type RewriteRules struct {
	Data []*import_sstpb.RewriteRule
}

RewriteRules contains rules for rewriting keys of tables.

func EmptyRewriteRule 

func EmptyRewriteRule() *RewriteRules

EmptyRewriteRule make a new, empty rewrite rule.

func (*RewriteRules) Append 

func (r *RewriteRules) Append(other RewriteRules)

Append append its argument to this rewrite rules.

type SplitClient 

type SplitClient interface {
	// GetStore gets a store by a store id.
	GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error)
	// GetRegion gets a region which includes a specified key.
	GetRegion(ctx context.Context, key []byte) (*RegionInfo, error)
	// GetRegionByID gets a region by a region id.
	GetRegionByID(ctx context.Context, regionID uint64) (*RegionInfo, error)
	// SplitRegion splits a region from a key, if key is not included in the region, it will return nil.
	// note: the key should not be encoded
	SplitRegion(ctx context.Context, regionInfo *RegionInfo, key []byte) (*RegionInfo, error)
	// BatchSplitRegions splits a region from a batch of keys.
	// note: the keys should not be encoded
	BatchSplitRegions(ctx context.Context, regionInfo *RegionInfo, keys [][]byte) ([]*RegionInfo, error)
	// BatchSplitRegionsWithOrigin splits a region from a batch of keys and return the original region and split new regions
	BatchSplitRegionsWithOrigin(ctx context.Context, regionInfo *RegionInfo, keys [][]byte) (*RegionInfo, []*RegionInfo, error)
	// ScatterRegion scatters a specified region.
	ScatterRegion(ctx context.Context, regionInfo *RegionInfo) error
	// ScatterRegions scatters regions in a batch.
	ScatterRegions(ctx context.Context, regionInfo []*RegionInfo) error
	// GetOperator gets the status of operator of the specified region.
	GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error)
	// ScanRegion gets a list of regions, starts from the region that contains key.
	// Limit limits the maximum number of regions returned.
	ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*RegionInfo, error)
}

SplitClient is an external client used by RegionSplitter.

func NewSplitClient 

func NewSplitClient(client pd.Client, tlsConf *tls.Config, spliterConf SpliterConfig, isRawKv bool) SplitClient

NewSplitClient returns a client used by RegionSplitter.

type SpliterConfig 

type SpliterConfig struct {
	GRPCMaxRecvMsgSize int
	SplitRegionMaxKeys int
}

SpliterConfig is the config for region split.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.