catalog

package
v0.31.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 8, 2021 License: Apache-2.0 Imports: 30 Imported by: 0

Documentation

Index

Constants

View Source
const (
	DefaultBranchName       = "master"
	DefaultImportBranchName = "import-from-inventory"
	DefaultPathDelimiter    = "/"
)
View Source
const (
	RangeFSName     = "range"
	MetaRangeFSName = "meta-range"
)
View Source
const (
	DBEntryFieldChecksum        = "checksum"
	DBEntryFieldPhysicalAddress = "physical_address"
)
View Source
const (
	ListRepositoriesLimitMax = 1000
	ListBranchesLimitMax     = 1000
	ListTagsLimitMax         = 1000
	DiffLimitMax             = 1000
	ListEntriesLimitMax      = 10000
)
View Source
const (
	MaxPathLength = 1024
)

Variables

View Source
var (
	ErrInvalidReference            = errors.New("invalid reference")
	ErrInvalidMetadataSrcFormat    = errors.New("invalid metadata src format")
	ErrExpired                     = errors.New("expired from storage")
	ErrByteSliceTypeAssertion      = errors.New("type assertion to []byte failed")
	ErrFeatureNotSupported         = errors.New("feature not supported")
	ErrBranchNotFound              = fmt.Errorf("branch %w", db.ErrNotFound)
	ErrCommitNotFound              = fmt.Errorf("commit %w", db.ErrNotFound)
	ErrRepositoryNotFound          = fmt.Errorf("repository %w", db.ErrNotFound)
	ErrEntryNotFound               = fmt.Errorf("entry %w", db.ErrNotFound)
	ErrUnexpected                  = errors.New("unexpected error")
	ErrReadEntryTimeout            = errors.New("read entry timeout")
	ErrInvalidValue                = errors.New("invalid value")
	ErrNonDirectNotSupported       = errors.New("non direct diff not supported")
	ErrSameBranchMergeNotSupported = errors.New("same branch merge not supported")
	ErrLineageCorrupted            = errors.New("lineage corrupted")
	ErrOperationNotPermitted       = errors.New("operation not permitted")
	ErrNothingToCommit             = errors.New("nothing to commit")
	ErrInvalidLockValue            = errors.New("invalid lock value")
	ErrNoDifferenceWasFound        = errors.New("no difference was found")
	ErrConflictFound               = errors.New("conflict found")
	ErrUnsupportedRelation         = errors.New("unsupported relation")
	ErrUnsupportedDelimiter        = errors.New("unsupported delimiter")
	ErrBadTypeConversion           = errors.New("bad type")
	ErrExportFailed                = errors.New("export failed")
	ErrRollbackWithActiveBranch    = fmt.Errorf("%w: rollback with active branch", ErrFeatureNotSupported)
)
View Source
var (
	ErrInvalidType   = errors.New("invalid type")
	ErrRequiredValue = errors.New("required value")
)
View Source
var ErrUnknownDiffType = errors.New("unknown graveler difference type")
View Source
var File_catalog_proto protoreflect.FileDescriptor
View Source
var ValidatePathOptional = MakeValidateOptional(ValidatePath)
View Source
var ValidateTagIDOptional = MakeValidateOptional(ValidateTagID)

Functions

func EntryToValue added in v0.31.0

func EntryToValue(entry *Entry) (*graveler.Value, error)

func MustEntryToValue added in v0.31.0

func MustEntryToValue(entry *Entry) *graveler.Value

func NewEntryToValueIterator added in v0.31.0

func NewEntryToValueIterator(it EntryIterator) *entryValueIterator

func NewValueToEntryIterator added in v0.31.0

func NewValueToEntryIterator(it graveler.ValueIterator) *valueEntryIterator

func Validate

func Validate(args []ValidateArg) error

func ValidateBranchID added in v0.31.0

func ValidateBranchID(v interface{}) error

func ValidateCommitID added in v0.31.0

func ValidateCommitID(v interface{}) error

func ValidateNonNegativeInt added in v0.31.0

func ValidateNonNegativeInt(v interface{}) error

func ValidatePath

func ValidatePath(v interface{}) error

func ValidateRef added in v0.31.0

func ValidateRef(v interface{}) error

func ValidateRepositoryID added in v0.31.0

func ValidateRepositoryID(v interface{}) error

func ValidateRequiredString added in v0.31.0

func ValidateRequiredString(v interface{}) error

func ValidateStorageNamespace

func ValidateStorageNamespace(v interface{}) error

func ValidateTagID added in v0.31.0

func ValidateTagID(v interface{}) error

Types

type Branch

type Branch struct {
	Name      string `db:"name"`
	Reference string
}

type Cataloger

type Cataloger interface {
	// CreateRepository create a new repository pointing to 'storageNamespace' (ex: s3://bucket1/repo) with default branch name 'branch'
	CreateRepository(ctx context.Context, repository string, storageNamespace string, branch string) (*Repository, error)

	// GetRepository get repository information
	GetRepository(ctx context.Context, repository string) (*Repository, error)

	// DeleteRepository delete a repository
	DeleteRepository(ctx context.Context, repository string) error

	// ListRepositories list repositories information, the bool returned is true when more repositories can be listed.
	// In this case pass the last repository name as 'after' on the next call to ListRepositories
	ListRepositories(ctx context.Context, limit int, after string) ([]*Repository, bool, error)

	CreateBranch(ctx context.Context, repository, branch string, sourceRef string) (*CommitLog, error)
	DeleteBranch(ctx context.Context, repository, branch string) error
	ListBranches(ctx context.Context, repository string, prefix string, limit int, after string) ([]*Branch, bool, error)
	BranchExists(ctx context.Context, repository string, branch string) (bool, error)
	GetBranchReference(ctx context.Context, repository, branch string) (string, error)
	ResetBranch(ctx context.Context, repository, branch string) error

	CreateTag(ctx context.Context, repository, tagID string, ref string) (string, error)
	DeleteTag(ctx context.Context, repository, tagID string) error
	ListTags(ctx context.Context, repository string, limit int, after string) ([]*Tag, bool, error)
	GetTag(ctx context.Context, repository, tagID string) (string, error)

	// GetEntry returns the current entry for path in repository branch reference.  Returns
	// the entry with ExpiredError if it has expired from underlying storage.
	GetEntry(ctx context.Context, repository, reference string, path string, params GetEntryParams) (*DBEntry, error)
	CreateEntry(ctx context.Context, repository, branch string, entry DBEntry) error
	CreateEntries(ctx context.Context, repository, branch string, entries []DBEntry) error
	DeleteEntry(ctx context.Context, repository, branch string, path string) error
	ListEntries(ctx context.Context, repository, reference string, prefix, after string, delimiter string, limit int) ([]*DBEntry, bool, error)
	ResetEntry(ctx context.Context, repository, branch string, path string) error
	ResetEntries(ctx context.Context, repository, branch string, prefix string) error

	Commit(ctx context.Context, repository, branch string, message string, committer string, metadata Metadata) (*CommitLog, error)
	GetCommit(ctx context.Context, repository, reference string) (*CommitLog, error)
	ListCommits(ctx context.Context, repository, branch string, fromReference string, limit int) ([]*CommitLog, bool, error)

	// RollbackCommit sets the branch to point at the given commit, losing all later commits.
	RollbackCommit(ctx context.Context, repository, branch string, reference string) error
	// Revert creates a reverse patch to the given commit, and applies it as a new commit on the given branch.
	Revert(ctx context.Context, repository, branch string, params RevertParams) error

	Diff(ctx context.Context, repository, leftReference string, rightReference string, params DiffParams) (Differences, bool, error)
	Compare(ctx context.Context, repository, leftReference string, rightReference string, params DiffParams) (Differences, bool, error)
	DiffUncommitted(ctx context.Context, repository, branch string, limit int, after string) (Differences, bool, error)

	Merge(ctx context.Context, repository, destinationBranch, sourceRef, committer, message string, metadata Metadata) (*MergeResult, error)

	Hooks() *CatalogerHooks

	io.Closer
}

func NewCataloger

func NewCataloger(db db.Database, cfg *config.Config) (Cataloger, error)

type CatalogerHooks added in v0.15.0

type CatalogerHooks struct {
	// PostCommit hooks are called at the end of a commit.
	PostCommit []PostCommitFunc

	// PostMerge hooks are called at the end of a merge.
	PostMerge []PostMergeFunc
}

CatalogerHooks describes the hooks available for some operations on the catalog. Hooks are called after the transaction ends; if they return an error they do not affect commit/merge.

func (*CatalogerHooks) AddPostCommit added in v0.15.0

func (h *CatalogerHooks) AddPostCommit(f PostCommitFunc) *CatalogerHooks

func (*CatalogerHooks) AddPostMerge added in v0.15.0

func (h *CatalogerHooks) AddPostMerge(f PostMergeFunc) *CatalogerHooks

type CommitLog

type CommitLog struct {
	Reference    string
	Committer    string    `db:"committer"`
	Message      string    `db:"message"`
	CreationDate time.Time `db:"creation_date"`
	Metadata     Metadata  `db:"metadata"`
	MetaRangeID  string    `db:"meta_range_id"`
	Parents      []string
}

type DBEntry added in v0.31.0

type DBEntry struct {
	CommonLevel     bool
	Path            string    `db:"path"`
	PhysicalAddress string    `db:"physical_address"`
	CreationDate    time.Time `db:"creation_date"`
	Size            int64     `db:"size"`
	Checksum        string    `db:"checksum"`
	Metadata        Metadata  `db:"metadata"`
	Expired         bool      `db:"is_expired"`
}

type DiffParams added in v0.15.0

type DiffParams struct {
	Limit            int
	After            string
	AdditionalFields []string // db fields names that will be load in additional to Path on Difference's Entry
}

type DiffResultRecord added in v0.16.0

type DiffResultRecord struct {
	TargetEntryNotInDirectBranch bool // the entry is reflected via lineage, NOT in the branch itself
	Difference
	EntryCtid *string // CTID of the modified/added entry. Do not use outside of catalog diff-by-iterators. https://github.com/treeverse/lakeFS/issues/831
}

type Difference

type Difference struct {
	DBEntry                // Partially filled. Path is always set.
	Type    DifferenceType `db:"diff_type"`
}

func (Difference) String

func (d Difference) String() string

type DifferenceType

type DifferenceType int
const (
	DifferenceTypeAdded DifferenceType = iota
	DifferenceTypeRemoved
	DifferenceTypeChanged
	DifferenceTypeConflict
	DifferenceTypeNone
)

type Differences

type Differences []Difference

func (Differences) Equal

func (d Differences) Equal(other Differences) bool

type Entry

type Entry struct {
	Address      string                 `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
	LastModified *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=last_modified,json=lastModified,proto3" json:"last_modified,omitempty"`
	Size         int64                  `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"`
	ETag         string                 `protobuf:"bytes,4,opt,name=e_tag,json=eTag,proto3" json:"e_tag,omitempty"`
	Metadata     map[string]string      `` /* 157-byte string literal not displayed */
	// contains filtered or unexported fields
}

func EntryFromCatalogEntry added in v0.31.0

func EntryFromCatalogEntry(entry DBEntry) *Entry

func ValueToEntry added in v0.31.0

func ValueToEntry(value *graveler.Value) (*Entry, error)

func (*Entry) Descriptor deprecated added in v0.31.0

func (*Entry) Descriptor() ([]byte, []int)

Deprecated: Use Entry.ProtoReflect.Descriptor instead.

func (*Entry) GetAddress added in v0.31.0

func (x *Entry) GetAddress() string

func (*Entry) GetETag added in v0.31.0

func (x *Entry) GetETag() string

func (*Entry) GetLastModified added in v0.31.0

func (x *Entry) GetLastModified() *timestamppb.Timestamp

func (*Entry) GetMetadata added in v0.31.0

func (x *Entry) GetMetadata() map[string]string

func (*Entry) GetSize added in v0.31.0

func (x *Entry) GetSize() int64

func (*Entry) ProtoMessage added in v0.31.0

func (*Entry) ProtoMessage()

func (*Entry) ProtoReflect added in v0.31.0

func (x *Entry) ProtoReflect() protoreflect.Message

func (*Entry) Reset added in v0.31.0

func (x *Entry) Reset()

func (*Entry) String added in v0.31.0

func (x *Entry) String() string

type EntryCatalog added in v0.31.0

type EntryCatalog struct {
	Store Store
}

func NewEntryCatalog added in v0.31.0

func NewEntryCatalog(cfg *config.Config, db db.Database) (*EntryCatalog, error)

func (*EntryCatalog) AddCommit added in v0.31.0

func (e *EntryCatalog) AddCommit(ctx context.Context, repositoryID graveler.RepositoryID, commit graveler.Commit) (graveler.CommitID, error)

func (*EntryCatalog) AddCommitToBranchHead added in v0.31.0

func (e *EntryCatalog) AddCommitToBranchHead(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID, commit graveler.Commit) (graveler.CommitID, error)

func (*EntryCatalog) Commit added in v0.31.0

func (e *EntryCatalog) Commit(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID, commitParams graveler.CommitParams) (graveler.CommitID, error)

func (*EntryCatalog) Compare added in v0.31.0

func (e *EntryCatalog) Compare(ctx context.Context, repositoryID graveler.RepositoryID, from graveler.Ref, to graveler.Ref) (EntryDiffIterator, error)

func (*EntryCatalog) CreateBranch added in v0.31.0

func (e *EntryCatalog) CreateBranch(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID, ref graveler.Ref) (*graveler.Branch, error)

func (*EntryCatalog) CreateRepository added in v0.31.0

func (e *EntryCatalog) CreateRepository(ctx context.Context, repositoryID graveler.RepositoryID, storageNamespace graveler.StorageNamespace, branchID graveler.BranchID) (*graveler.Repository, error)

func (*EntryCatalog) CreateTag added in v0.31.0

func (e *EntryCatalog) CreateTag(ctx context.Context, repositoryID graveler.RepositoryID, tagID graveler.TagID, commitID graveler.CommitID) error

func (*EntryCatalog) DeleteBranch added in v0.31.0

func (e *EntryCatalog) DeleteBranch(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID) error

func (*EntryCatalog) DeleteEntry added in v0.31.0

func (e *EntryCatalog) DeleteEntry(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID, path Path) error

func (*EntryCatalog) DeleteRepository added in v0.31.0

func (e *EntryCatalog) DeleteRepository(ctx context.Context, repositoryID graveler.RepositoryID) error

func (*EntryCatalog) DeleteTag added in v0.31.0

func (e *EntryCatalog) DeleteTag(ctx context.Context, repositoryID graveler.RepositoryID, tagID graveler.TagID) error

func (*EntryCatalog) Dereference added in v0.31.0

func (e *EntryCatalog) Dereference(ctx context.Context, repositoryID graveler.RepositoryID, ref graveler.Ref) (graveler.CommitID, error)

func (*EntryCatalog) Diff added in v0.31.0

func (e *EntryCatalog) Diff(ctx context.Context, repositoryID graveler.RepositoryID, left, right graveler.Ref) (EntryDiffIterator, error)

func (*EntryCatalog) DiffUncommitted added in v0.31.0

func (e *EntryCatalog) DiffUncommitted(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID) (EntryDiffIterator, error)

func (*EntryCatalog) GetBranch added in v0.31.0

func (e *EntryCatalog) GetBranch(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID) (*graveler.Branch, error)

func (*EntryCatalog) GetCommit added in v0.31.0

func (e *EntryCatalog) GetCommit(ctx context.Context, repositoryID graveler.RepositoryID, commitID graveler.CommitID) (*graveler.Commit, error)

func (*EntryCatalog) GetEntry added in v0.31.0

func (e *EntryCatalog) GetEntry(ctx context.Context, repositoryID graveler.RepositoryID, ref graveler.Ref, path Path) (*Entry, error)

func (*EntryCatalog) GetRepository added in v0.31.0

func (e *EntryCatalog) GetRepository(ctx context.Context, repositoryID graveler.RepositoryID) (*graveler.Repository, error)

func (*EntryCatalog) GetTag added in v0.31.0

func (e *EntryCatalog) GetTag(ctx context.Context, repositoryID graveler.RepositoryID, tagID graveler.TagID) (*graveler.CommitID, error)

func (*EntryCatalog) ListBranches added in v0.31.0

func (e *EntryCatalog) ListBranches(ctx context.Context, repositoryID graveler.RepositoryID) (graveler.BranchIterator, error)

func (*EntryCatalog) ListEntries added in v0.31.0

func (e *EntryCatalog) ListEntries(ctx context.Context, repositoryID graveler.RepositoryID, ref graveler.Ref, prefix, delimiter Path) (EntryListingIterator, error)

func (*EntryCatalog) ListRepositories added in v0.31.0

func (e *EntryCatalog) ListRepositories(ctx context.Context) (graveler.RepositoryIterator, error)

func (*EntryCatalog) ListTags added in v0.31.0

func (e *EntryCatalog) ListTags(ctx context.Context, repositoryID graveler.RepositoryID) (graveler.TagIterator, error)

func (*EntryCatalog) Log added in v0.31.0

func (*EntryCatalog) Merge added in v0.31.0

func (e *EntryCatalog) Merge(ctx context.Context, repositoryID graveler.RepositoryID, destination graveler.BranchID, source graveler.Ref, commitParams graveler.CommitParams) (graveler.CommitID, graveler.DiffSummary, error)

func (*EntryCatalog) Reset added in v0.31.0

func (e *EntryCatalog) Reset(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID) error

func (*EntryCatalog) ResetKey added in v0.31.0

func (e *EntryCatalog) ResetKey(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID, path Path) error

func (*EntryCatalog) ResetPrefix added in v0.31.0

func (e *EntryCatalog) ResetPrefix(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID, prefix Path) error

func (*EntryCatalog) Revert added in v0.31.0

func (e *EntryCatalog) Revert(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID, ref graveler.Ref, parentNumber int, commitParams graveler.CommitParams) (graveler.CommitID, graveler.DiffSummary, error)

func (*EntryCatalog) SetEntry added in v0.31.0

func (e *EntryCatalog) SetEntry(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID, path Path, entry *Entry) error

func (*EntryCatalog) UpdateBranch added in v0.31.0

func (e *EntryCatalog) UpdateBranch(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID, ref graveler.Ref) (*graveler.Branch, error)

func (*EntryCatalog) WriteMetaRange added in v0.31.0

func (e *EntryCatalog) WriteMetaRange(ctx context.Context, repositoryID graveler.RepositoryID, it EntryIterator) (*graveler.MetaRangeID, error)

type EntryDiff added in v0.31.0

type EntryDiff struct {
	Type  graveler.DiffType
	Path  Path
	Entry *Entry
}

type EntryDiffIterator added in v0.31.0

type EntryDiffIterator interface {
	Next() bool
	SeekGE(id Path)
	Value() *EntryDiff
	Err() error
	Close()
}

func NewEntryDiffIterator added in v0.31.0

func NewEntryDiffIterator(it graveler.DiffIterator) EntryDiffIterator

type EntryIterator added in v0.31.0

type EntryIterator interface {
	Next() bool
	SeekGE(id Path)
	Value() *EntryRecord
	Err() error
	Close()
}

func NewPrefixIterator added in v0.31.0

func NewPrefixIterator(it EntryIterator, prefix Path) EntryIterator

type EntryListing added in v0.31.0

type EntryListing struct {
	CommonPrefix bool
	Path
	*Entry
}

type EntryListingIterator added in v0.31.0

type EntryListingIterator interface {
	Next() bool
	SeekGE(id Path)
	Value() *EntryListing
	Err() error
	Close()
}

func NewEntryListingIterator added in v0.31.0

func NewEntryListingIterator(it EntryIterator, prefix Path, delimiter Path) EntryListingIterator

type EntryRecord added in v0.31.0

type EntryRecord struct {
	Path Path
	*Entry
}

type ExpireResult

type ExpireResult struct {
	Repository        string
	Branch            string
	PhysicalAddress   string
	InternalReference string
}

type ExpiryRows

type ExpiryRows interface {
	Close()
	Next() bool
	Err() error
	// Read returns the current from ExpiryRows, or an error on failure.  Call it only after
	// successfully calling Next.
	Read() (*ExpireResult, error)
}

ExpiryRows is a database iterator over ExpiryResults. Use Next to advance from row to row.

type GetEntryParams

type GetEntryParams struct {
	// For entries to expired objects the Expired bit is set.  If true, GetEntry returns
	// successfully for expired entries, otherwise it returns the entry with ErrExpired.
	ReturnExpired bool
}

GetEntryParams configures what entries GetEntry returns.

type MergeResult

type MergeResult struct {
	Summary   map[DifferenceType]int
	Reference string
}

type Metadata

type Metadata map[string]string

func (*Metadata) Scan

func (j *Metadata) Scan(src interface{}) error

func (Metadata) Value

func (j Metadata) Value() (driver.Value, error)

type Path added in v0.31.0

type Path string

func (Path) String added in v0.31.0

func (id Path) String() string

type PostCommitFunc added in v0.17.0

type PostCommitFunc func(ctx context.Context, repo, branch string, commitLog CommitLog) error

type PostMergeFunc added in v0.17.0

type PostMergeFunc func(ctx context.Context, repo, branch string, mergeResult MergeResult) error

type Repository

type Repository struct {
	Name             string    `db:"name"`
	StorageNamespace string    `db:"storage_namespace"`
	DefaultBranch    string    `db:"default_branch"`
	CreationDate     time.Time `db:"creation_date"`
}

type RevertParams added in v0.31.0

type RevertParams struct {
	Reference    string // the commit to revert
	ParentNumber int    // if reverting a merge commit, the change will be reversed relative to this parent number (1-based).
	Committer    string
}

type Store added in v0.31.0

type Tag added in v0.31.0

type Tag struct {
	ID       string
	CommitID string
}

type ValidateArg added in v0.31.0

type ValidateArg struct {
	Name  string
	Value interface{}
	Fn    ValidateFunc
}

type ValidateFunc

type ValidateFunc func(v interface{}) error

func MakeValidateOptional added in v0.31.0

func MakeValidateOptional(fn ValidateFunc) ValidateFunc

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL