 Documentation
      ¶
      Documentation
      ¶
    
    
  
    
  
    Overview ¶
Package model gathers the main structs and interfaces used in sync library.
Index ¶
- Constants
- func Ignores(endpoint Endpoint, name string) bool
- func IsConfigError(e error) bool
- func IsFolderHiddenFile(name string) bool
- func IsIgnoredFile(path string, ignores ...glob.Glob) (ignored bool)
- func NewConfigError(err error) error
- func NodeRequiresChecksum(node *tree.Node) bool
- func Retry(f func() error, seconds ...time.Duration) error
- func RetryWithCtx(ctx context.Context, f func(retry int) error, seconds ...time.Duration) error
- func StringContentToETag(uuid string) string
- func ZapEndpoint(key string, e Endpoint) zapcore.Field
- type AuthRequester
- type BulkLoader
- type CachedBranchProvider
- type ChecksumProvider
- type Command
- type ConfigError
- type Conflicter
- type DataSyncSource
- type DataSyncTarget
- type DirectionType
- type Endpoint
- type EndpointInfo
- type EndpointOptions
- type EndpointRootStat
- type EndpointStatus
- type ErrorType
- type EventInfo
- type EventType
- type HashStoreReader
- type LockBranchProvider
- type LockEvent
- type LockEventType
- type MetadataProvider
- type MetadataReceiver
- type MultiStater
- type Networking
- type PathSyncSource
- type PathSyncTarget
- type ProcessingStatus
- func (p *ProcessingStatus) AtomicProgress() bool
- func (p *ProcessingStatus) EndpointURI() string
- func (p *ProcessingStatus) Error() error
- func (p *ProcessingStatus) IsError() bool
- func (p *ProcessingStatus) MarshalJSON() ([]byte, error)
- func (p *ProcessingStatus) Node() *tree.Node
- func (p *ProcessingStatus) Progress() float32
- func (p *ProcessingStatus) SetEndpoint(uri string) *ProcessingStatus
- func (p *ProcessingStatus) SetError(e error) *ProcessingStatus
- func (p *ProcessingStatus) SetNode(node *tree.Node) *ProcessingStatus
- func (p *ProcessingStatus) SetProgress(pg float32, atomic ...bool) *ProcessingStatus
- func (p *ProcessingStatus) Status() int
- func (p *ProcessingStatus) String() string
- func (p *ProcessingStatus) Type() StatusScope
- func (p *ProcessingStatus) UnmarshalJSON(data []byte) error
 
- type ProcessorEvent
- type SessionProvider
- type SnapshotFactory
- type SnapshotUpdater
- type Snapshoter
- type Stater
- type Status
- type StatusProvider
- type StatusScope
- type SyncCmd
- type TaskStatus
- type UuidFoldersRefresher
- type UuidProvider
- type UuidReceiver
- type Versioner
- type WalkNodesFunc
- type WatchConnectionInfo
- type WatchObject
- func (w *WatchObject) Close()
- func (w *WatchObject) ConnectionInfos() chan WatchConnectionInfo
- func (w WatchObject) Done() <-chan bool
- func (w *WatchObject) Errors() chan error
- func (w *WatchObject) Events() chan EventInfo
- func (w WatchObject) NextError() error
- func (w WatchObject) NextEvent() EventInfo
 
Constants ¶
const ( // Use unique path separator everywhere InternalPathSeparator = "/" GlobSeparator = '/' // Minio default Etag when a new file is detected DefaultEtag = "00000000000000000000000000000000-1" DirectionLeft DirectionType = 1 DirectionRight DirectionType = 2 DirectionBi DirectionType = 3 Interrupt SyncCmd = iota Pause Resume )
const ( StatusScopeTask = iota StatusScopeProcessing )
Variables ¶
This section is empty.
Functions ¶
func Ignores ¶
Ignores checks if a specific name should be ignored by the given Endpoint
func IsFolderHiddenFile ¶
IsFolderHiddenFile checks if file is .pydio
func Retry ¶
Retry tries to apply an operation as many time as required
func RetryWithCtx ¶
RetryWithCtx does like Retry with an additionnal cancellable context
Types ¶
type BulkLoader ¶
type BulkLoader interface {
	BulkLoadNodes(ctx context.Context, nodes map[string]string) (map[string]interface{}, error)
}
    A BulkLoader can stream calls to ReadNode - Better use CachedBranchProvider
type CachedBranchProvider ¶
type CachedBranchProvider interface {
	Endpoint
	GetCachedBranches(ctx context.Context, roots ...string) PathSyncSource
}
    A CachedBranchProvider can quickly load a full branch recursively in memory and expose it as a PathSyncSource
type ChecksumProvider ¶
ChecksumProvider is able to compute a checksum for a given node (typically an Etag)
type Command ¶
Command is a pub/sub for dispatching SyncCmd
type ConfigError ¶
type ConfigError struct {
	// contains filtered or unexported fields
}
    func (*ConfigError) HasConfigError ¶
func (c *ConfigError) HasConfigError() bool
type DataSyncSource ¶
type DataSyncSource interface {
	PathSyncSource
	// GetReaderOn provides a ReadCloser for reading content of a node located at a given path
	GetReaderOn(path string) (out io.ReadCloser, err error)
}
    DataSyncSource provides a way to read the actual content of the nodes
func AsDataSyncSource ¶
func AsDataSyncSource(endpoint Endpoint) (DataSyncSource, bool)
AsDataSyncSource tries to cast an Endpoint to a DataSyncSource
type DataSyncTarget ¶
type DataSyncTarget interface {
	PathSyncTarget
	// GetWriterOn provides a writeCloser for writing content to a given path.
	GetWriterOn(cancel context.Context, path string, targetSize int64) (out io.WriteCloser, writeDone chan bool, writeErr chan error, err error)
}
    DataSyncTarget provides a way to write some actual content to the nodes
func AsDataSyncTarget ¶
func AsDataSyncTarget(endpoint Endpoint) (DataSyncTarget, bool)
AsDataSyncTarget tries to cast an Endpoint to a DataSyncTarget
type Endpoint ¶
type Endpoint interface {
	// LoadNode loads a given node by its path from this endpoint
	LoadNode(ctx context.Context, path string, extendedStats ...bool) (node *tree.Node, err error)
	// GetEndpointInfo returns static informations about this endpoint
	GetEndpointInfo() EndpointInfo
}
    Endpoint is the most basic interface for representing an endpoint for synchronization. It is just able to return some info and to load a node
type EndpointInfo ¶
type EndpointInfo struct {
	URI                   string
	RequiresNormalization bool
	RequiresFoldersRescan bool
	IsAsynchronous        bool
	EchoTime              time.Duration
	Ignores               []string
}
    EndpointInfo provides static info about a given Endpoint (returned by GetEndpointInfo method)
type EndpointOptions ¶
EndpointOptions is used to configure an Endpoint at creation time
type EndpointRootStat ¶
type EndpointRootStat struct {
	HasChildrenInfo bool
	HasSizeInfo     bool
	Size    int64
	Folders int64
	Files   int64
	PgSize     int64 `json:"-"`
	PgChildren int64 `json:"-"`
	PgFolders  int64 `json:"-"`
	PgFiles    int64 `json:"-"`
	LastPg float64 `json:"-"`
}
    EndpointRootStat gives information about the size/files/folders of an endpoint
func (*EndpointRootStat) Children ¶
func (e *EndpointRootStat) Children() int64
Children returns the sum of Folders and Files
func (*EndpointRootStat) IsEmpty ¶
func (e *EndpointRootStat) IsEmpty() bool
IsEmpty checks if the given stat seems empty. Call to IsKnown should be done before otherwise it can be a false negative
func (*EndpointRootStat) IsKnown ¶
func (e *EndpointRootStat) IsKnown() bool
IsKnown returns true if either HasSizeInfo or HasChildrenInfo is set
type EndpointStatus ¶
type EndpointStatus struct {
	EndpointInfo
	WatchConnection WatchConnectionInfo
	Stats           *EndpointRootStat
}
    type EventInfo ¶
type EventInfo struct {
	Time           string
	Size           int64
	Etag           string
	Folder         bool
	Path           string
	Source         PathSyncSource `json:"-"`
	Type           EventType
	Host           string
	Port           string
	UserAgent      string
	OperationId    string
	ScanEvent      bool
	ScanSourceNode *tree.Node
	Metadata       map[string]string
	MoveSource     *tree.Node
	MoveTarget     *tree.Node
}
    EventInfo contains the information of the event that occurred and the source IP:PORT of the client which triggerred the event.
func NodeToEventInfo ¶
type EventType ¶
type EventType string
EventType represents the type of the event occurred.
const ( // EventCreate notifies when a new object is created EventCreate EventType = "ObjectCreated" // EventRename notifies when a new object is renamed EventRename EventType = "ObjectRenamed" // EventRemove notifies when a new object is deleted EventRemove EventType = "ObjectRemoved" // EventAccessed notifies when an object is accessed. EventAccessed EventType = "ObjectAccessed" // EventAccessedRead notifies when an object is accessed (specifically read). EventAccessedRead EventType = "ObjectAccessed:Read" // EventAccessedStat notifies when an object is accessed (specifically stat). EventAccessedStat EventType = "ObjectAccessed:Stat" // EventOther notifies any other events EventOther EventType = "ObjectOther" // EventOther notifies any other events EventSureMove EventType = "NodeMoved" )
type HashStoreReader ¶
type HashStoreReader interface {
	// SetRefHashStore passes a reference to a loaded snapshot
	SetRefHashStore(source PathSyncSource)
}
    HashStoreReader can maintain a reference to a snapshot to quickly find hashes for nodes directly from the snapshot if they have not been modified
type LockBranchProvider ¶
type LockBranchProvider interface {
	// Lock set a lock on a branch, with a preset sessionUUID and an expiration time
	LockBranch(ctx context.Context, node *tree.Node, sessionUUID string, expireAfter time.Duration) error
	// Unlock removes lock manually from this branch, if it was not expired already
	UnlockBranch(ctx context.Context, sessionUUID string) error
}
    LockBranchProvider can set/remove a lock on a branch, with automatic expiration
type LockEvent ¶
type LockEvent struct {
	Type        LockEventType
	Source      PathSyncSource
	Path        string
	OperationId string
}
    LockEvent is a simple struct for signaling lock/unlock operation
type LockEventType ¶
type LockEventType int
const ( LockEventLock LockEventType = iota LockEventUnlock )
type MetadataProvider ¶
type MetadataProvider interface {
	// ProvidesMetadataNamespaces returns a list of patterns to check on provider nodes MetaStore.
	ProvidesMetadataNamespaces() ([]glob.Glob, bool)
}
    MetadataProvider declares metadata namespaces that may be mapped to target metadata
type MetadataReceiver ¶
type MetadataReceiver interface {
	// CreateMetadata add a metadata to the node
	CreateMetadata(ctx context.Context, node *tree.Node, namespace string, jsonValue string) error
	// UpdateMetadata updates an existing metadata value
	UpdateMetadata(ctx context.Context, node *tree.Node, namespace string, jsonValue string) error
	// DeleteMetadata deletes a metadata by namespace
	DeleteMetadata(ctx context.Context, node *tree.Node, namespace string) error
}
    MetadataReceiver implements methods for updating nodes metadata
type MultiStater ¶
func NewMultiStater ¶
func NewMultiStater() MultiStater
func (MultiStater) Stats ¶
func (m MultiStater) Stats() map[string]interface{}
func (MultiStater) String ¶
func (m MultiStater) String() string
type PathSyncSource ¶
type PathSyncSource interface {
	Endpoint
	// Walk walks the nodes with a callback
	Walk(walknFc WalkNodesFunc, root string, recursive bool) (err error)
	// Watch setup an event watcher on the nodes
	Watch(recursivePath string) (*WatchObject, error)
}
    PathSyncSource is a type of endpoint that can be used as a source of tree.Nodes for synchronization. It can browse and watch the nodes, but not get the nodes actual content (see DataSyncSource).
func AsPathSyncSource ¶
func AsPathSyncSource(endpoint Endpoint) (PathSyncSource, bool)
AsPathSyncSource tries to cast an Endpoint to a PathSyncSource
type PathSyncTarget ¶
type PathSyncTarget interface {
	Endpoint
	// CreateNode is used to create a node in the tree
	CreateNode(ctx context.Context, node *tree.Node, updateIfExists bool) (err error)
	// DeleteNode is used to remove a node (and all its children) from the tree
	DeleteNode(ctx context.Context, path string) (err error)
	// MoveNode is used to move a node (and all its children) from one place to another in the tree.
	MoveNode(ctx context.Context, oldPath string, newPath string) (err error)
}
    PathSyncTarget is a type of endpoint that can be used as a target for synchronization, typically an Index. It can be updated with a tree of nodes, but cannot store actual data (see DataSyncTarget)
func AsPathSyncTarget ¶
func AsPathSyncTarget(endpoint Endpoint) (PathSyncTarget, bool)
AsPathSyncTarget tries to cast an Endpoint to a PathSyncTarget
type ProcessingStatus ¶
type ProcessingStatus struct {
	// contains filtered or unexported fields
}
    func NewProcessingStatus ¶
func NewProcessingStatus(info string) *ProcessingStatus
func (*ProcessingStatus) AtomicProgress ¶
func (p *ProcessingStatus) AtomicProgress() bool
func (*ProcessingStatus) EndpointURI ¶
func (p *ProcessingStatus) EndpointURI() string
func (*ProcessingStatus) Error ¶
func (p *ProcessingStatus) Error() error
func (*ProcessingStatus) IsError ¶
func (p *ProcessingStatus) IsError() bool
func (*ProcessingStatus) MarshalJSON ¶
func (p *ProcessingStatus) MarshalJSON() ([]byte, error)
MarshalJSON implements custom JSON marshalling
func (*ProcessingStatus) Node ¶
func (p *ProcessingStatus) Node() *tree.Node
func (*ProcessingStatus) Progress ¶
func (p *ProcessingStatus) Progress() float32
func (*ProcessingStatus) SetEndpoint ¶
func (p *ProcessingStatus) SetEndpoint(uri string) *ProcessingStatus
func (*ProcessingStatus) SetError ¶
func (p *ProcessingStatus) SetError(e error) *ProcessingStatus
func (*ProcessingStatus) SetNode ¶
func (p *ProcessingStatus) SetNode(node *tree.Node) *ProcessingStatus
func (*ProcessingStatus) SetProgress ¶
func (p *ProcessingStatus) SetProgress(pg float32, atomic ...bool) *ProcessingStatus
func (*ProcessingStatus) Status ¶
func (p *ProcessingStatus) Status() int
func (*ProcessingStatus) String ¶
func (p *ProcessingStatus) String() string
func (*ProcessingStatus) Type ¶
func (p *ProcessingStatus) Type() StatusScope
func (*ProcessingStatus) UnmarshalJSON ¶
func (p *ProcessingStatus) UnmarshalJSON(data []byte) error
UnmarshalJSON implements custom JSON unmarshalling
type SessionProvider ¶
type SessionProvider interface {
	// StartSession opens a new indexation session and returns it
	StartSession(ctx context.Context, rootNode *tree.Node, silent bool) (*tree.IndexationSession, error)
	// FlushSession calls the Flush method on the underlying service without closing the session yet
	FlushSession(ctx context.Context, sessionUuid string) error
	// FinishSession closes the indexation session
	FinishSession(ctx context.Context, sessionUuid string) error
}
    SessionProvider has internal mechanism to start/flush/finish an IndexationSession
func AsSessionProvider ¶
func AsSessionProvider(endpoint Endpoint) (SessionProvider, bool)
AsSessionProvider tries to cast an Endpoint to a SessionProvider
type SnapshotFactory ¶
type SnapshotFactory interface {
	// Load creates the snapshot and return it
	Load(source PathSyncSource) (Snapshoter, error)
	Close(ctx context.Context) error
	Reset(ctx context.Context) error
}
    SnapshotFactory provides dependency injection for creating snapshots using a specific persistence layer.
type SnapshotUpdater ¶
type SnapshotUpdater interface {
	// SetUpdateSnapshot stores internal reference to a Snapshot
	SetUpdateSnapshot(PathSyncTarget)
	// PatchUpdateSnaptshot applies a patch of operations to the internal snapshot
	PatchUpdateSnapshot(ctx context.Context, patch interface{})
}
    SnapshotUpdater is an endpoint that can embed a reference to a snapshot and update it afterward
type Snapshoter ¶
type Snapshoter interface {
	PathSyncSource
	// IsEmpty indicates whether this snapshot is properly initialized or not (e.g. underlying DB not found)
	IsEmpty() bool
	// Capture walks the source and stores all paths inside snapshot. The paths parameter allows to filter the branches to capture.
	Capture(ctx context.Context, source PathSyncSource, paths ...string) error
}
    Snapshoter is an extended version of PathSyncSource that can capture another source at once.
type Stater ¶
Stater can provide a json-serializable description of its content
type Status ¶
type StatusProvider ¶
type StatusProvider interface {
	// SetupChannels register channels for listening to status and done infos
	SetupChannels(status chan Status, done chan interface{}, cmd *Command)
	// Status notify of a new ProcessStatus
	Status(s Status)
	// Done notify the patch is processed, can send any useful info to the associated channel
	Done(info interface{})
}
    StatusProvider can register channels to send status/done events during processing
type TaskStatus ¶
type TaskStatus int
const ( TaskStatusIdle TaskStatus = iota TaskStatusPaused TaskStatusDisabled TaskStatusProcessing TaskStatusError TaskStatusRestarting TaskStatusStopping TaskStatusRemoved )
type UuidFoldersRefresher ¶
type UuidFoldersRefresher interface {
	// ExistingFolders lists all folders with their UUID
	ExistingFolders(ctx context.Context) (map[string][]*tree.Node, error)
	// UpdateFolderUuid refreshes a given folder UUID and return it.
	UpdateFolderUuid(ctx context.Context, node *tree.Node) (*tree.Node, error)
}
    UuidFoldersRefresher provides tools to detect UUID duplicates and update them if necessary
type UuidProvider ¶
type UuidProvider interface {
	// LoadNodeByUuid loads a node by UUID.
	LoadNodeByUuid(ctx context.Context, uuid string) (node *tree.Node, err error)
}
    UuidProvider declares an endpoint to be able to load a node by its unique UUID
type UuidReceiver ¶
type UuidReceiver interface {
	// UpdateNodeUuid refresh node UUID and returns the new node
	UpdateNodeUuid(ctx context.Context, node *tree.Node) (*tree.Node, error)
}
    UuidReceiver is able to update an existing node UUID
type Versioner ¶
type WatchConnectionInfo ¶
type WatchConnectionInfo int
const ( WatchConnected WatchConnectionInfo = iota WatchDisconnected WatchActive WatchIdle WatchStats )
type WatchObject ¶
type WatchObject struct {
	// eventInfo will be put on this chan
	EventInfoChan chan EventInfo
	// errors will be put on this chan
	ErrorChan chan error
	// will stop the watcher goroutines
	DoneChan chan bool
	// Provides info about the internal watcher connection
	// Can be nill if the internal watcher does not support such notifications
	ConnectionInfo chan WatchConnectionInfo
}
    func (*WatchObject) Close ¶
func (w *WatchObject) Close()
Close the watcher, will stop all goroutines
func (*WatchObject) ConnectionInfos ¶
func (w *WatchObject) ConnectionInfos() chan WatchConnectionInfo
func (WatchObject) Done ¶
func (w WatchObject) Done() <-chan bool
Done returns a channel that unblocks when Close has been called
func (*WatchObject) Errors ¶
func (w *WatchObject) Errors() chan error
Errors returns the chan receiving errors
func (*WatchObject) Events ¶
func (w *WatchObject) Events() chan EventInfo
Events returns the chan receiving events
func (WatchObject) NextError ¶
func (w WatchObject) NextError() error
NextError pops the next error from the ErrorChan
func (WatchObject) NextEvent ¶
func (w WatchObject) NextEvent() EventInfo
NextEvent pops the next event from the EventInfoChan
       Source Files
      ¶
      Source Files
      ¶
    
- command.go
- consts.go
- endpoints.go
- errors.go
- events.go
- funcs.go
- retry.go
- stater.go
- status.go
- watchers.go