backup

package
v0.3.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 10, 2019 License: GPL-3.0 Imports: 23 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	MsgRsyncInfo  = "RsyncInfo"
	MsgGolangInfo = "GolangInfo"

	MsgFolderBackupTypeSkipDescription      = "FolderBackupTypeSkipDescription"
	MsgFolderBackupTypeRecursiveDescription = "FolderBackupTypeRecursiveDescription"
	MsgFolderBackupTypeContentDescription   = "FolderBackupTypeContentDescription"

	MsgLogPlanStageStarting                  = "LogPlanStageStarting"
	MsgLogPlanStageStartTime                 = "LogPlanStageStartTime"
	MsgLogPlanStageEndTime                   = "LogPlanStageEndTime"
	MsgLogPlanStartIterateViaNSources        = "LogPlanStartIterateViaNSources"
	MsgLogPlanStageInquirySource             = "LogPlanStageInquirySource"
	MsgLogPlanStageSourceFolderCountInfo     = "LogPlanStageSourceFolderCountInfo"
	MsgLogPlanStageSourceSkipFolderCountInfo = "LogPlanStageSourceSkipFolderCountInfo"
	MsgLogPlanStageSourceTotalSizeInfo       = "LogPlanStageSourceTotalSizeInfo"
	MsgLogPlanStageUseTemporaryFolder        = "LogPlanStageUseTemporaryFolder"
	MsgLogPlanStageBuildFolderError          = "LogPlanStageBuildFolderError"

	MsgLogBackupStageStarting                               = "LogBackupStageStarting"
	MsgLogBackupStageStartTime                              = "LogBackupStageStartTime"
	MsgLogBackupStageEndTime                                = "LogBackupStageEndTime"
	MsgLogBackupStageBackupToDestination                    = "LogBackupStageBackupToDestination"
	MsgLogBackupStagePreviousBackupDiscoveryPermissionError = "LogBackupStagePreviousBackupDiscoveryPermissionError"
	MsgLogBackupStagePreviousBackupDiscoveryOtherError      = "LogBackupStagePreviousBackupDiscoveryOtherError"
	MsgLogBackupStagePreviousBackupFoundAndWillBeUsed       = "LogBackupStagePreviousBackupFoundAndWillBeUsed"
	MsgLogBackupStagePreviousBackupFoundButDisabled         = "LogBackupStagePreviousBackupFoundButDisabled"
	MsgLogBackupStagePreviousBackupNotFound                 = "LogBackupStagePreviousBackupNotFound"
	MsgLogBackupStageStartToBackupFromSource                = "LogBackupStageStartToBackupFromSource"
	MsgLogBackupStageRenameDestination                      = "LogBackupStageRenameDestination"
	MsgLogBackupStageFailedToCreateFolder                   = "LogBackupStageFailedToCreateFolder"
	MsgLogBackupDetectedTotalBackupSizeGetChanged           = "LogBackupDetectedTotalBackupSizeGetChanged"
	MsgLogBackupStageProgressBackupSuccess                  = "LogBackupStageProgressBackupSuccess"
	MsgLogBackupStageProgressBackupError                    = "LogBackupStageProgressBackupError"
	MsgLogBackupStageProgressSkipBackupError                = "LogBackupStageProgressSkipBackupError"
	MsgLogBackupStageCriticalError                          = "LogBackupStageCriticalError"
	MsgLogBackupStageDiscoveringPreviousBackups             = "LogBackupStageDiscoveringPreviousBackups"
	MsgLogBackupStageRecoveredFromError                     = "LogBackupStageRecoveredFromError"
	MsgLogBackupStageSaveRsyncExtraLogTo                    = "LogBackupStageSaveRsyncExtraLogTo"
	MsgLogBackupStageSaveLogTo                              = "LogBackupStageSaveLogTo"
	MsgLogBackupStageExitMessage                            = "LogBackupStageExitMessage"

	MsgLogStatisticsSummaryCaption                            = "LogStatisticsSummaryCaption"
	MsgLogStatisticsEnvironmentCaption                        = "LogStatisticsEnvironmentCaption"
	MsgLogStatisticsResultsCaption                            = "LogStatisticsResultsCaption"
	MsgLogStatisticsStatusCaption                             = "LogStatisticsStatusCaption"
	MsgLogStatisticsStatusSuccessfullyCompleted               = "LogStatisticsStatusSuccessfullyCompleted"
	MsgLogStatisticsStatusCompletedWithErrors                 = "LogStatisticsStatusCompletedWithErrors"
	MsgLogStatisticsPlanStageCaption                          = "LogStatisticsPlanStageCaption"
	MsgLogStatisticsPlanStageSourceToBackup                   = "LogStatisticsPlanStageSourceToBackup"
	MsgLogStatisticsPlanStageTotalSize                        = "LogStatisticsPlanStageTotalSize"
	MsgLogStatisticsPlanStageFolderCount                      = "LogStatisticsPlanStageFolderCount"
	MsgLogStatisticsPlanStageFolderSkipCount                  = "LogStatisticsPlanStageFolderSkipCount"
	MsgLogStatisticsPlanStageTimeTaken                        = "LogStatisticsPlanStageTimeTaken"
	MsgLogStatisticsBackupStageCaption                        = "LogStatisticsBackupStageCaption"
	MsgLogStatisticsBackupStageDestinationPath                = "LogStatisticsBackupStageDestinationPath"
	MsgLogStatisticsBackupStagePreviousBackupFound            = "LogStatisticsBackupStagePreviousBackupFound"
	MsgLogStatisticsBackupStagePreviousBackupFoundButDisabled = "LogStatisticsBackupStagePreviousBackupFoundButDisabled"
	MsgLogStatisticsBackupStageNoValidPreviousBackupFound     = "LogStatisticsBackupStageNoValidPreviousBackupFound"
	MsgLogStatisticsBackupStageTotalSize                      = "LogStatisticsBackupStageTotalSize"
	MsgLogStatisticsBackupStageSkippedSize                    = "LogStatisticsBackupStageSkippedSize"
	MsgLogStatisticsBackupStageFailedToBackupSize             = "LogStatisticsBackupStageFailedToBackupSize"
	MsgLogStatisticsBackupStageTimeTaken                      = "LogStatisticsBackupStageTimeTaken"
)
View Source 
const MaxInt = int(MaxUint >> 1)
View Source 
const MaxUint = ^uint(0)
View Source 
const MinInt = -MaxInt - 1
View Source 
const (
	TAB_RUNE = '\t'
)

Variables

View Source 
var (
	DoubleSplitLogLine string = strings.Repeat("=", 100)
	SingleSplitLogLine string = strings.Repeat("-", 100)
)
View Source 
var LocalLog = logger.NewPackageLogger("backup",

	logger.InfoLevel,
)

Functions

func BuildBackupPlan 

func BuildBackupPlan(ctx context.Context, lg logger.PackageLog, config *Config,
	modules []Module, notifier Notifier) (*Plan, *Progress, error)

Perform 1st stage (plan stage) to measure RSYNC source volume to backup and find optimal traverse path of source directory tree. Use plan built in 1st stage later in 2nd stage.

func CreateMetadataSignatureFile 

func CreateMetadataSignatureFile(modules []Module, destPath string) error

CreateMetadataSignatureFile serialize RSYNC sources plus destination subpaths to the special "backup session signature" file.

func EncodeSignatures 

func EncodeSignatures(signs NodeSignatures) (string, error)

EncodeSignatures encode NodeSignatures object to self-describing binary format.

func GenerateSourceID 

func GenerateSourceID(rsyncSource string) string

GenerateSourceID convert RSYNC source URL to unique identifier.

func GetBackupFolderName 

func GetBackupFolderName(incomplete bool, date *time.Time) string

GetBackupFolderName return new folder name for ongoing backup process.

func GetBackupTypeDescription 

func GetBackupTypeDescription(backupType core.FolderBackupType) string

GetBackupTypeDescription return localized description of how application will backup specific directory described by core.Dir object. It could be 3 options: 1) full backup: backup full content include all nested folders; 2) flat backup: backup only direct files in folder, ignore nested folders; 3) skip backup: skip folder backup (this happens when specific signature file found).

func GetLogFileName 

func GetLogFileName() string

GetLogFileName return the name of general backup process log.

func GetMetadataSignatureFileName 

func GetMetadataSignatureFileName() string

GetMetadataSignatureFileName return the name of specific file which describe all sources used in backup process.

func GetRsyncLogFileName 

func GetRsyncLogFileName() string

GetRsyncLogFileName return the name of specific low-level RSYNC utility log.

func MeasureDir 

func MeasureDir(ctx context.Context, password *string, dir *core.Dir, retryCount *int, log *rsync.Logging,
	blockSize *backupBlockSizeSettings) (int, error)

Types

type Config 

type Config struct {
	SigFileIgnoreBackup                string `toml:"sig_file_ignore_backup"`
	RsyncRetryCount                    *int   `toml:"retry_count"`
	AutoManageBackupBlockSize          *bool  `toml:"auto_manage_backup_block_size"`
	MaxBackupBlockSizeMb               *int   `toml:"max_backup_block_size_mb"`
	UsePreviousBackup                  *bool  `toml:"use_previous_backup"`
	NumberOfPreviousBackupToUse        *int   `toml:"number_of_previous_backup_to_use"`
	EnableLowLevelLogForRsync          *bool  `toml:"enable_low_level_log_rsync"`
	EnableIntensiveLowLevelLogForRsync *bool  `toml:"enable_intensive_low_level_log_rsync"`
	// rsync --compress
	RsyncCompressFileTransfer *bool `toml:"rsync_compress_file_transfer"`
	// rsync --links
	RsyncRecreateSymlinks *bool `toml:"rsync_recreate_symlinks"`
	// rsync --perms
	RsyncTransferSourcePermissions *bool `toml:"rsync_transfer_source_permissions"`
	// rsync --group
	RsyncTransferSourceGroup *bool `toml:"rsync_transfer_source_group"`
	// rsync --owner
	RsyncTransferSourceOwner *bool `toml:"rsync_transfer_source_owner"`
	// rsync --devices
	RsyncTransferDeviceFiles *bool `toml:"rsync_transfer_device_files"`
	// rsync --specials
	RsyncTransferSpecialFiles *bool `toml:"rsync_transfer_special_files"`
}

Config keeps backup session configuration. Config instance is initialized mainly from GLIB GSettings in ui/gtkui package.

func NewConfig 

func NewConfig(filePath string) (*Config, error)

func (*Config) GetRsyncParams 

func (conf *Config) GetRsyncParams(addExtraParams []string) []string

Prepare RSYNC CLI parameters to run console RSYNC process.

type IRsyncConfigurable 

type IRsyncConfigurable interface {
	GetRsyncParams(addExtraParams []string) []string
}

type LogFiles 

type LogFiles struct {
	// contains filtered or unexported fields
}

LogFiles track log files during backup session. Has functionality to relocate log files from one storage to another: used when log files moved from /tmp partition to permanent destination location.

func NewLogFiles 

func NewLogFiles() *LogFiles

func (*LogFiles) ChangeRootPath 

func (v *LogFiles) ChangeRootPath(newRootPath string) error

ChangeRootPath relocate log files from one storage to another. Used to move from 1st backup stage (plan stage) to 2nd (backup stage). In 1st stage we keep log files in /tmp partition, in 2nd stage we relocate and save them to destination location.

func (*LogFiles) Close 

func (v *LogFiles) Close() error

func (*LogFiles) GetAppendFile 

func (v *LogFiles) GetAppendFile(suffixPath string) (*os.File, error)

type Module 

type Module struct {
	SourceRsync          string  `toml:"src_rsync"`
	DestSubPath          string  `toml:"dst_subpath"`
	ChangeFilePermission string  `toml:"rsync_change_file_permission"`
	AuthPassword         *string `toml:"module_auth_password"`
}

func (*Module) GetRsyncParams 

func (module *Module) GetRsyncParams(addExtraParams []string) []string

Prepare RSYNC CLI parameters to run console RSYNC process.

type Node 

type Node struct {
	Module  Module
	RootDir *core.Dir
}

Node contain information about single rsync source backup.

type NodeSignature 

type NodeSignature struct {
	SourceRsyncCipher string
	DestSubPath       string
}

NodeSignature keep RSYNC source path crypted with hash function and destination subpath. RSYNC source path crypted with hash function is used as "source identifier" to search repeated backup sessions to use for deduplication. Content of this object is serialized to the file stored in backup session root folder.

func GetSignature 

func GetSignature(module Module) NodeSignature

GetSignature builds NodeSignature object on the basis of BackupNodePath data.

type NodeSignatures 

type NodeSignatures struct {
	Signatures []NodeSignature
}

NodeSignatures keeps list of RSYNC source to backup in one session.

func DecodeSignatures 

func DecodeSignatures(str string) (*NodeSignatures, error)

DecodeSignatures decode NodeSignatures object from self-describing binary format.

func GetNodeSignatures 

func GetNodeSignatures(modules []Module) NodeSignatures

func (NodeSignatures) FindFirstSignature 

func (v NodeSignatures) FindFirstSignature(signature string) *NodeSignature

type Notifier 

type Notifier interface {
	NotifyPlanStage_NodeStructureStartInquiry(sourceID int,
		sourceRsync string) error
	NotifyPlanStage_NodeStructureDoneInquiry(sourceID int,
		sourceRsync string, dir *core.Dir) error
	NotifyBackupStage_FolderStartBackup(rootDest string,
		paths core.SrcDstPath, backupType core.FolderBackupType,
		leftToBackup core.FolderSize,
		timePassed time.Duration, eta *time.Duration,
	) error
	NotifyBackupStage_FolderDoneBackup(rootDest string,
		paths core.SrcDstPath, backupType core.FolderBackupType,
		leftToBackup core.FolderSize, sizeDone core.SizeProgress,
		timePassed time.Duration, eta *time.Duration,
		sessionErr error) error
}

type Plan 

type Plan struct {
	Config     *Config
	Nodes      []Node
	BackupSize core.FolderSize
}

Plan keep all necessary information obtained from preferences and 1st backup pass to start backup process.

func (*Plan) GetModules 

func (v *Plan) GetModules() []Module

func (*Plan) RunBackup 

func (this *Plan) RunBackup(progress *Progress, destPath string, errorHookCall rsync.ErrorHookCall) error

Perform whole 2nd stage (backup stage) here, than save and report completion to session logs.

type PrevBackup 

type PrevBackup struct {
	// Full path to signature file name
	SignatureFileName string
	Signature         NodeSignature
}

PrevBackup describe previous backup found, which contain same RSYNC source. Such previous backups used for RSYNC utility deduplication, which significantly decrease size and time for new backup session.

func (PrevBackup) GetDirPath 

func (v PrevBackup) GetDirPath() string

GetDirPath returns full path to data copied in previous successful backup session.

type PrevBackups 

type PrevBackups struct {
	Backups []PrevBackup
}

PrevBackups keeps list of previous backup found. See description of PrevBackup.

func FindPrevBackupPathsByNodeSignatures 

func FindPrevBackupPathsByNodeSignatures(lg logger.PackageLog, destPath string,
	signs NodeSignatures, lastN int) (*PrevBackups, error)

FindPrevBackupPathsByNodeSignatures search for previous backup sessions which might significantly decrease backup size and speed up process. In the end it should return list of previous backup sessions sorted by date/time in descending order (recent go first).

func (*PrevBackups) FilterBySourceID 

func (v *PrevBackups) FilterBySourceID(sourceID string) *PrevBackups

FilterBySourceID choose backup sessions which contains same source as specified by sourceID.

func (*PrevBackups) GetDirPaths 

func (v *PrevBackups) GetDirPaths() []string

GetDirPaths provide file system paths to previous backup sessions found.

type Progress 

type Progress struct {
	Context       context.Context
	LogFiles      *LogFiles
	Log           logger.PackageLog
	RsyncLog      *rsync.Logging
	Progress      *core.SizeProgress
	TotalProgress *core.SizeProgress

	// Notifier interface used to plug external objects to notify
	Notifier Notifier

	// Time stamps in 1st stage
	StartPlanTime time.Time
	EndPlanTime   time.Time

	// Time stamps in 2nd stage
	StartBackupTime time.Time
	EndBackupTime   time.Time

	// Previous backup sessions found to use for deduplicaton
	PrevBackups *PrevBackups

	RootDest     string
	BackupFolder string

	// Notify only once (theoretically it never happens)
	SizeChangedNotified bool
}

Progress tracks all aspects of backup session progress including time points, volume processed and so on. Progress employ Notifier interface to report progress to plug-in interface.

func (*Progress) CalcTimePassedAndETA 

func (this *Progress) CalcTimePassedAndETA(plan *Plan) (time.Duration, *time.Duration)

CalcTimePassedAndETA count total time passed in backup stage (2nd stage) and compute ETA (estimated time of arrival) - time left.

func (*Progress) Close 

func (this *Progress) Close() error

Close release any resources occupied.

func (*Progress) EventBackupStage_FolderDoneBackup 

func (this *Progress) EventBackupStage_FolderDoneBackup(paths core.SrcDstPath,
	backupType core.FolderBackupType, plan *Plan,
	sizeDone core.SizeProgress, sessionErr error) error

EventBackupStage_FolderStartBackup report about backup folder end (2nd stage).

func (*Progress) EventBackupStage_FolderStartBackup 

func (this *Progress) EventBackupStage_FolderStartBackup(paths core.SrcDstPath,
	backupType core.FolderBackupType, plan *Plan) error

EventBackupStage_FolderStartBackup report about backup folder start (2nd stage).

func (*Progress) EventPlanStage_NodeStructureDoneInquiry 

func (this *Progress) EventPlanStage_NodeStructureDoneInquiry(sourceID int,
	sourceRsync string, dir *core.Dir) error

EventPlanStage_NodeStructureDoneInquiry report about end inquiry of RSYNC source (1st stage).

func (*Progress) EventPlanStage_NodeStructureStartInquiry 

func (this *Progress) EventPlanStage_NodeStructureStartInquiry(sourceID int,
	sourceRsync string) error

EventPlanStage_NodeStructureStartInquiry report about start inquiry of RSYNC source (1st stage).

func (*Progress) FinishBackupStage 

func (this *Progress) FinishBackupStage()

Save the end time of 2nd stage.

func (*Progress) FinishPlanStage 

func (this *Progress) FinishPlanStage()

FinishPlanStage save the end time of 1st stage.

func (*Progress) GetBackupFullPath 

func (this *Progress) GetBackupFullPath(backupFolder string) string

func (*Progress) GetTotalTimeTaken 

func (this *Progress) GetTotalTimeTaken() time.Duration

GetTotalTimeTaken count up total time of backup session execution.

func (*Progress) LeftToBackup 

func (this *Progress) LeftToBackup(plan *Plan) core.FolderSize

LeftToBackup return size left to process in 2nd stage.

func (*Progress) PrevBackupsUsed 

func (this *Progress) PrevBackupsUsed(prevBackups *PrevBackups)

func (*Progress) PrintTotalStatistics 

func (this *Progress) PrintTotalStatistics(lg logger.PackageLog, plan *Plan) error

PrintTotalStatistics print results on backup session completion. Print all statistics including time taken, volume processed, errors happens and so on.

func (*Progress) SayGoodbye 

func (this *Progress) SayGoodbye(lg logger.PackageLog)

SayGoodbye report completion to the backup session log.

func (*Progress) SetBackupFolder 

func (this *Progress) SetBackupFolder(backupFolder string) error

func (*Progress) SetRootDestination 

func (this *Progress) SetRootDestination(rootDestPath string)

func (*Progress) SizeBackedUp 

func (this *Progress) SizeBackedUp() core.FolderSize

SizeBackedUp return total size processed during 2nd stage.

func (*Progress) StartBackupStage 

func (this *Progress) StartBackupStage()

Save the start time of 2nd stage.

func (*Progress) StartPlanStage 

func (this *Progress) StartPlanStage()

StartPlanStage save the start time of 1st stage.

Source Files

  • abstract.go
  • common.go
  • config.go
  • deduplication.go
  • heuristic.go
  • logfiles.go
  • messagekeys.go
  • process.go
  • progress.go
  • utils.go

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.