retriever

package
v0.24.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 17, 2025 License: Apache-2.0, MIT Imports: 30 Imported by: 0

Documentation

Index

Constants

View Source 
const BufferWindow = 5 * time.Millisecond
View Source 
const HttpDefaultInitialWait time.Duration = 2 * time.Millisecond

Connect() is currently a noop, so this simply allows parallel goroutines to queue and the scoring logic to select one to start.

Variables

View Source 
var (
	ErrHttpSelectorRequest = errors.New("HTTP retrieval for an explicit selector request")
	ErrNoHttpForPeer       = errors.New("no HTTP url for peer")
	ErrBadPathForRequest   = errors.New("bad path for request")
)
View Source 
var (
	ErrRetrieverNotStarted         = errors.New("retriever not started")
	ErrDealProposalFailed          = errors.New("deal proposal failed")
	ErrNoCandidates                = errors.New("no candidates")
	ErrUnexpectedRetrieval         = errors.New("unexpected active retrieval")
	ErrHitRetrievalLimit           = errors.New("hit retrieval limit")
	ErrProposalCreationFailed      = errors.New("proposal creation failed")
	ErrRetrievalRegistrationFailed = errors.New("retrieval registration failed")
	ErrRetrievalFailed             = errors.New("retrieval failed")
	ErrAllRetrievalsFailed         = errors.New("all retrievals failed")
	ErrConnectFailed               = errors.New("unable to connect to provider")
	ErrAllQueriesFailed            = errors.New("all queries failed")
	ErrRetrievalTimedOut           = errors.New("retrieval timed out")
	ErrRetrievalAlreadyRunning     = errors.New("retrieval already running for CID")
)

Functions

func NewHttpRetriever 

func NewHttpRetriever(session Session, client *http.Client) types.CandidateRetriever

NewHttpRetriever makes a new CandidateRetriever for verified CAR HTTP retrievals (transport-ipfs-gateway-http).

func NewHttpRetrieverWithDeps 

func NewHttpRetrieverWithDeps(
	session Session,
	client *http.Client,
	clock clock.Clock,
	awaitReceivedCandidates chan<- struct{},
	initialPause time.Duration,
	noDirtyClose bool,
) types.CandidateRetriever

Types

type AssignableCandidateFinder 

type AssignableCandidateFinder struct {
	// contains filtered or unexported fields
}

AssignableCandidateFinder finds and filters candidates for a given retrieval

func NewAssignableCandidateFinder 

func NewAssignableCandidateFinder(candidateSource types.CandidateSource, filterIndexerCandidate FilterIndexerCandidate) AssignableCandidateFinder

func NewAssignableCandidateFinderWithClock 

func NewAssignableCandidateFinderWithClock(candidateSource types.CandidateSource, filterIndexerCandidate FilterIndexerCandidate, clock clock.Clock) AssignableCandidateFinder

func (AssignableCandidateFinder) FindCandidates 

func (acf AssignableCandidateFinder) FindCandidates(ctx context.Context, request types.RetrievalRequest, eventsCallback func(types.RetrievalEvent), onCandidates func([]types.RetrievalCandidate)) error

type DirectCandidateSource 

type DirectCandidateSource struct {
	// contains filtered or unexported fields
}

DirectCandidateSource finds candidate protocols from a fixed set of peers

func NewDirectCandidateSource 

func NewDirectCandidateSource(providers []types.Provider) *DirectCandidateSource

NewDirectCandidateSource returns a new DirectCandidateFinder for the given providers

func (*DirectCandidateSource) FindCandidates 

func (d *DirectCandidateSource) FindCandidates(ctx context.Context, c cid.Cid, cb func(types.RetrievalCandidate)) error

FindCandidates returns candidates for each configured provider

type ErrHttpRequestFailure 

type ErrHttpRequestFailure struct {
	Code int
}

func (ErrHttpRequestFailure) Error 

func (e ErrHttpRequestFailure) Error() string

type FilterIndexerCandidate 

type FilterIndexerCandidate func(types.RetrievalCandidate) (bool, types.RetrievalCandidate)

type GetStorageProviderTimeout 

type GetStorageProviderTimeout func(peer peer.ID) time.Duration

type HybridRetriever 

type HybridRetriever struct {
	// contains filtered or unexported fields
}

HybridRetriever wraps an existing retriever and adds fallback to per-block retrieval when the primary retriever fails with a missing block error.

func NewHybridRetriever 

func NewHybridRetriever(
	inner types.Retriever,
	candidateSource types.CandidateSource,
	httpClient *http.Client,
) *HybridRetriever

NewHybridRetriever creates a new hybrid retriever that wraps an existing retriever and falls back to per-block fetching on partial responses.

func (*HybridRetriever) Retrieve 

func (hr *HybridRetriever) Retrieve(
	ctx context.Context,
	request types.RetrievalRequest,
	eventsCallback func(types.RetrievalEvent),
) (*types.RetrievalStats, error)

type ProtocolHttp 

type ProtocolHttp struct {
	Client *http.Client
	Clock  clock.Clock
}

func (ProtocolHttp) Code 

func (ph ProtocolHttp) Code() multicodec.Code

func (*ProtocolHttp) Connect 

func (ph *ProtocolHttp) Connect(ctx context.Context, retrieval *retrieval, candidate types.RetrievalCandidate) (time.Duration, error)

func (ProtocolHttp) GetMergedMetadata 

func (ph ProtocolHttp) GetMergedMetadata(cid cid.Cid, currentMetadata, newMetadata metadata.Protocol) metadata.Protocol

func (*ProtocolHttp) Retrieve 

func (ph *ProtocolHttp) Retrieve(
	ctx context.Context,
	retrieval *retrieval,
	shared *retrievalShared,
	timeout time.Duration,
	candidate types.RetrievalCandidate,
) (*types.RetrievalStats, error)

type Retriever 

type Retriever struct {
	// contains filtered or unexported fields
}

func NewRetriever 

func NewRetriever(
	ctx context.Context,
	session Session,
	candidateSource types.CandidateSource,
	candidateRetriever types.CandidateRetriever,
	protocol multicodec.Code,
) (*Retriever, error)

func NewRetrieverWithClock 

func NewRetrieverWithClock(
	ctx context.Context,
	session Session,
	candidateSource types.CandidateSource,
	candidateRetriever types.CandidateRetriever,
	protocol multicodec.Code,
	clock clock.Clock,
) (*Retriever, error)

func (*Retriever) RegisterSubscriber 

func (retriever *Retriever) RegisterSubscriber(subscriber types.RetrievalEventSubscriber) func()

RegisterSubscriber registers a subscriber to receive all events fired during the process of making a retrieval, including the process of querying available storage providers to find compatible ones to attempt retrieval from.

func (*Retriever) Retrieve 

func (retriever *Retriever) Retrieve(
	ctx context.Context,
	request types.RetrievalRequest,
	eventsCB func(types.RetrievalEvent),
) (*types.RetrievalStats, error)

Retrieve attempts to retrieve the given CID using the configured CandidateSource to find storage providers that should have the CID.

func (*Retriever) Start 

func (retriever *Retriever) Start()

Start will start the retriever events system

func (*Retriever) Stop 

func (retriever *Retriever) Stop() chan struct{}

Stop will stop the retriever events system and return a channel that will be closed when shutdown has completed

func (*Retriever) WrapWithHybrid 

func (retriever *Retriever) WrapWithHybrid(candidateSource types.CandidateSource, httpClient *http.Client)

WrapWithHybrid wraps the executor with a HybridRetriever for per-block fallback.

type Session 

type Session interface {
	GetStorageProviderTimeout(storageProviderId peer.ID) time.Duration
	FilterIndexerCandidate(candidate types.RetrievalCandidate) (bool, types.RetrievalCandidate)

	RegisterRetrieval(retrievalId types.RetrievalID, cid cid.Cid, selector datamodel.Node) bool
	AddToRetrieval(retrievalId types.RetrievalID, storageProviderIds []peer.ID) error
	EndRetrieval(retrievalId types.RetrievalID) error

	RecordConnectTime(storageProviderId peer.ID, connectTime time.Duration)
	RecordFirstByteTime(storageProviderId peer.ID, firstByteTime time.Duration)
	RecordFailure(retrievalId types.RetrievalID, storageProviderId peer.ID) error
	RecordSuccess(storageProviderId peer.ID, bandwidthBytesPerSecond uint64)

	ChooseNextProvider(peers []peer.ID, metadata []metadata.Protocol) int
}

type TransportProtocol 

type TransportProtocol interface {
	Code() multicodec.Code
	GetMergedMetadata(cid cid.Cid, currentMetadata, newMetadata metadata.Protocol) metadata.Protocol
	Connect(ctx context.Context, retrieval *retrieval, candidate types.RetrievalCandidate) (time.Duration, error)
	Retrieve(
		ctx context.Context,
		retrieval *retrieval,
		shared *retrievalShared,
		timeout time.Duration,
		candidate types.RetrievalCandidate,
	) (*types.RetrievalStats, error)
}

TransportProtocol implements the protocol-specific portions of a parallel- peer retriever. It is responsible for communicating with individual peers and also bears responsibility for some of the peer-selection logic.

Source Files

View all Source files

Directories

Path Synopsis
Package prioritywaitqueue implements a blocking queue for prioritised coordination of goroutine execution.
 Package prioritywaitqueue implements a blocking queue for prioritised coordination of goroutine execution.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.