Documentation
¶
Index ¶
- Variables
- func NewInverted(dts ds.Batching) *invertedIndexImpl
- type FSIndexRepo
- func (l *FSIndexRepo) AddFullIndex(key shard.Key, index carindex.Index) (err error)
- func (l *FSIndexRepo) DropFullIndex(key shard.Key) (dropped bool, err error)
- func (l *FSIndexRepo) ForEach(f func(shard.Key) (bool, error)) error
- func (l *FSIndexRepo) GetFullIndex(key shard.Key) (carindex.Index, error)
- func (l *FSIndexRepo) Len() (int, error)
- func (l *FSIndexRepo) Size() (uint64, error)
- func (l *FSIndexRepo) StatFullIndex(key shard.Key) (Stat, error)
- type FullIndexRepo
- type Inverted
- type Manifest
- type ManifestKey
- type ManifestRepo
- type MemIndexRepo
- func (m *MemIndexRepo) AddFullIndex(key shard.Key, index index.Index) (err error)
- func (m *MemIndexRepo) DropFullIndex(key shard.Key) (dropped bool, err error)
- func (m *MemIndexRepo) ForEach(f func(shard.Key) (bool, error)) error
- func (m *MemIndexRepo) GetFullIndex(key shard.Key) (idx index.Index, err error)
- func (m *MemIndexRepo) Len() (int, error)
- func (m *MemIndexRepo) Size() (uint64, error)
- func (m *MemIndexRepo) StatFullIndex(key shard.Key) (Stat, error)
- type MultihashIterator
- type Repo
- type Stat
Constants ¶
This section is empty.
Variables ¶
var ErrNotFound = errors.New("index not found")
Functions ¶
func NewInverted ¶ added in v0.4.5
NewInverted returns a new inverted index that uses `go-indexer-core` as it's storage backend. We use `go-indexer-core` as the backend here as it's been optimized to store (multihash -> Value) kind of data and supports bulk updates via context ID and metadata-deduplication which are useful properties for our use case here.
Types ¶
type FSIndexRepo ¶
type FSIndexRepo struct {
// contains filtered or unexported fields
}
FSIndexRepo implements FullIndexRepo using the local file system to store the indices
func NewFSRepo ¶
func NewFSRepo(baseDir string) (*FSIndexRepo, error)
NewFSRepo creates a new index repo that stores indices on the local filesystem with the given base directory as the root
func (*FSIndexRepo) AddFullIndex ¶
func (*FSIndexRepo) DropFullIndex ¶
func (l *FSIndexRepo) DropFullIndex(key shard.Key) (dropped bool, err error)
func (*FSIndexRepo) GetFullIndex ¶
func (*FSIndexRepo) Len ¶
func (l *FSIndexRepo) Len() (int, error)
Len counts all index files in the base path
func (*FSIndexRepo) Size ¶
func (l *FSIndexRepo) Size() (uint64, error)
Size sums the size of all index files in the base path
func (*FSIndexRepo) StatFullIndex ¶
func (l *FSIndexRepo) StatFullIndex(key shard.Key) (Stat, error)
type FullIndexRepo ¶
type FullIndexRepo interface {
// GetFullIndex returns the full index for the specified shard.
GetFullIndex(key shard.Key) (index.Index, error)
// AddFullIndex persists a full index for a shard.
AddFullIndex(key shard.Key, index index.Index) error
// DropFullIndex drops the full index for the specified shard. If the error
// is nil, it returns whether an index was effectively dropped.
DropFullIndex(key shard.Key) (dropped bool, err error)
// StatFullIndex stats a full index.
StatFullIndex(key shard.Key) (Stat, error)
// Len returns the number of indices in the repo.
Len() (int, error)
// ForEach calls the callback with the key for each index.
//
// Returning true from the callback will continue the traversal.
// Returning false will terminate the traversal.
//
// A non-nil error will abort the traversal, and the error will be
// propagated to the caller.
ForEach(func(shard.Key) (bool, error)) error
// Size returns the size of the repo in bytes.
Size() (uint64, error)
}
type Inverted ¶ added in v0.4.5
type Inverted interface {
// AddMultihashesForShard adds a (multihash -> shard key) mapping for all multihashes returned by the given MultihashIterator.
AddMultihashesForShard(ctx context.Context, mhIter MultihashIterator, s shard.Key) error
// GetShardsForMultihash returns keys for all the shards that has the given multihash.
GetShardsForMultihash(ctx context.Context, h multihash.Multihash) ([]shard.Key, error)
}
Inverted is the top-level inverted index that maps a multihash to all the shards it is present in.
type Manifest ¶
type Manifest interface {
// Contains checks whether a given CID is contained in the manifest.
Contains(c cid.Cid) (bool, error)
// Len returns the count of entries this manifest has.
Len() (l int64, err error)
// ForEach traverses the manifest using an visitor pattern. The supplied
// callback will be called for each manifest entry, in no particular order.
//
// Returning true from the callback will continue the traversal.
// Returning false will terminate the traversal.
//
// A non-nil error will abort the traversal, and the error will be
// propagated to the caller.
ForEach(func(c cid.Cid) (ok bool, err error)) error
}
Manifest are sets of CIDs with no offset indication.
type ManifestKey ¶
ManifestKey identifies a manifest. It is a triple that can act as a composite key, comprising the shard key and generation metadata.
type ManifestRepo ¶
type ManifestRepo interface {
// ListManifests returns the available manifests for a given shard,
// identified by their ManifestKey.
ListManifests(key shard.Key) ([]ManifestKey, error)
// GetManifest returns the Manifest identified by a given ManifestKey.
GetManifest(key ManifestKey) (Manifest, error)
// AddManifest adds a Manifest to the ManifestRepo.
AddManifest(key ManifestKey, manifest Manifest) error
// DropManifest drops a Manifest from the ManifestRepo.
DropManifest(key ManifestKey) (bool, error)
// StatManifest stats a Manifest.
StatManifest(key ManifestKey) (Stat, error)
}
TODO unimplemented.
type MemIndexRepo ¶
type MemIndexRepo struct {
// contains filtered or unexported fields
}
MemIndexRepo implements FullIndexRepo with an in-memory map.
func NewMemoryRepo ¶
func NewMemoryRepo() *MemIndexRepo
func (*MemIndexRepo) AddFullIndex ¶
func (*MemIndexRepo) DropFullIndex ¶
func (m *MemIndexRepo) DropFullIndex(key shard.Key) (dropped bool, err error)
func (*MemIndexRepo) GetFullIndex ¶
func (*MemIndexRepo) Len ¶
func (m *MemIndexRepo) Len() (int, error)
func (*MemIndexRepo) Size ¶
func (m *MemIndexRepo) Size() (uint64, error)
func (*MemIndexRepo) StatFullIndex ¶
func (m *MemIndexRepo) StatFullIndex(key shard.Key) (Stat, error)
type MultihashIterator ¶ added in v0.4.5
type Repo ¶
type Repo interface {
FullIndexRepo
ManifestRepo
}
Repo is the central index repository object that manages full indices and manifests.