fetcher

package
v0.15.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 8, 2026 License: BSD-3-Clause Imports: 35 Imported by: 1

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func ToFS added in v0.9.0 

func ToFS(ctx context.Context, f Fetcher) fsFetcher

Turn a Fetcher into a fs.FS virtual filesystem

func ToFSFile added in v0.9.0 

func ToFSFile(ctx context.Context, r Resource) fs.File

Turn a Resource into a fs.File virtual file

Types

type ArchiveFetcher 

type ArchiveFetcher struct {
	// contains filtered or unexported fields
}

Provides access to entries of an archive.

func NewArchiveFetcher 

func NewArchiveFetcher(a archive.Archive) *ArchiveFetcher

func NewArchiveFetcherFromPath 

func NewArchiveFetcherFromPath(ctx context.Context, path string) (*ArchiveFetcher, error)

func NewArchiveFetcherFromPathWithFactory 

func NewArchiveFetcherFromPathWithFactory(ctx context.Context, path string, factory archive.ArchiveFactory) (*ArchiveFetcher, error)

func NewArchiveFetcherFromURLWithFactory added in v0.9.0 

func NewArchiveFetcherFromURLWithFactory(ctx context.Context, url url.URL, factory archive.ArchiveFactory) (*ArchiveFetcher, error)

func NewArchiveFetcherFromURLWithFactoryAndContext added in v0.9.0 

func NewArchiveFetcherFromURLWithFactoryAndContext(ctx context.Context, url url.URL, factory archive.SchemeSpecificArchiveFactory) (*ArchiveFetcher, error)

func (*ArchiveFetcher) Close 

func (f *ArchiveFetcher) Close()

Close implements Fetcher

func (*ArchiveFetcher) Get 

func (f *ArchiveFetcher) Get(ctx context.Context, link manifest.Link) Resource

Get implements Fetcher 

func (f *ArchiveFetcher) Links(ctx context.Context) (manifest.LinkList, error)

Links implements Fetcher

type BytesResource 

type BytesResource struct {
	// contains filtered or unexported fields
}

BytesResource is a Resource serving a lazy-loaded bytes buffer.

func NewBytesResource 

func NewBytesResource(link manifest.Link, loader func() []byte) *BytesResource

NewBytesResource creates a new BytesResources from a lazy loader callback.

func (*BytesResource) Close 

func (r *BytesResource) Close()

Close implements Resource

func (*BytesResource) File 

func (r *BytesResource) File() string

File implements Resource

func (*BytesResource) Length 

func (r *BytesResource) Length(ctx context.Context) (int64, *ResourceError)

Length implements Resource 

func (r *BytesResource) Link() manifest.Link

Link implements Resource

func (*BytesResource) Properties 

func (r *BytesResource) Properties() manifest.Properties

Properties implements Resource

func (*BytesResource) Read 

func (r *BytesResource) Read(ctx context.Context, start int64, end int64) ([]byte, *ResourceError)

Read implements Resource

func (*BytesResource) Stream 

func (r *BytesResource) Stream(ctx context.Context, w io.Writer, start int64, end int64) (int64, *ResourceError)

Stream implements Resource

type CompressedResource added in v0.5.0 

type CompressedResource interface {
	CompressedAs(compressionMethod archive.CompressionMethod) bool
	CompressedLength(ctx context.Context) int64
	StreamCompressed(ctx context.Context, w io.Writer) (int64, *ResourceError)
	StreamCompressedGzip(ctx context.Context, w io.Writer) (int64, *ResourceError)
	ReadCompressed(ctx context.Context) ([]byte, *ResourceError)
	ReadCompressedGzip(ctx context.Context) ([]byte, *ResourceError)
	CRC32Checksum(ctx context.Context) *uint32
}

type EfficientStreamer added in v0.15.0 

type EfficientStreamer interface {
	// HasEfficientStream reports whether Stream retrieves only the requested
	// range from the underlying source. See [EfficientStreamer].
	HasEfficientStream() bool
}

EfficientStreamer is implemented by resources that can report whether their Stream method retrieves just the requested byte range from the underlying source with bounded per-request overhead — no reading and discarding of the range's prefix, and no degradation into one remote request per tiny chunk.

When HasEfficientStream reports true, Stream is at least as efficient as Read even when the resource is backed by a remote source (HTTP, S3, GCS), while using bounded memory and delivering the first byte as soon as it is available. When it reports false — e.g. for a deflate-compressed entry in an archive, where a ranged Stream must decompress from the start of the entry — callers serving remote content should prefer Read for ranged access.

type EmptyFetcher 

type EmptyFetcher struct{}

A Fetcher providing no resources at all.

func (EmptyFetcher) Close 

func (f EmptyFetcher) Close()

func (EmptyFetcher) Get 

func (f EmptyFetcher) Get(ctx context.Context, link manifest.Link) Resource
func (f EmptyFetcher) Links(ctx context.Context) (manifest.LinkList, error)

type FailureResource 

type FailureResource struct {
	// contains filtered or unexported fields
}

Creates a Resource that will always return the given [error].

func NewFailureResource 

func NewFailureResource(link manifest.Link, ex *ResourceError) FailureResource

func (FailureResource) Close 

func (r FailureResource) Close()

Close implements Resource

func (FailureResource) File 

func (r FailureResource) File() string

File implements Resource

func (FailureResource) Length 

func (r FailureResource) Length(ctx context.Context) (int64, *ResourceError)

Length implements Resource 

func (r FailureResource) Link() manifest.Link

Link implements Resource

func (FailureResource) Properties 

func (r FailureResource) Properties() manifest.Properties

func (FailureResource) Read 

func (r FailureResource) Read(ctx context.Context, start int64, end int64) ([]byte, *ResourceError)

Read implements Resource

func (FailureResource) Stream 

func (r FailureResource) Stream(ctx context.Context, w io.Writer, start int64, end int64) (int64, *ResourceError)

Stream implements Resource

type Fetcher 

type Fetcher interface {

	/**
	 * Known resources available in the medium, such as file paths on the file system
	 * or entries in a ZIP archive. This list is not exhaustive, and additional
	 * unknown resources might be reachable.
	 *
	 * If the medium has an inherent resource order, it should be followed.
	 * Otherwise, HREFs are sorted alphabetically.
	 */
	Links(ctx context.Context) (manifest.LinkList, error)

	/**
	 * Returns the [Resource] at the given [link]'s HREF.
	 *
	 * A [Resource] is always returned, since for some cases we can't know if it exists before
	 * actually fetching it, such as HTTP. Therefore, errors are handled at the Resource level.
	 */
	Get(ctx context.Context, link manifest.Link) Resource

	// Closes this object and releases any resources associated with it.
	// If the object is already closed then invoking this method has no effect.
	Close()
}

Fetcher provides access to a Resource from a Link.

type FileFetcher 

type FileFetcher struct {
	// contains filtered or unexported fields
}

Provides access to resources on the local file system.

func NewFileFetcher 

func NewFileFetcher(href string, fpath string) *FileFetcher

func (*FileFetcher) Close 

func (f *FileFetcher) Close()

Close implements Fetcher

func (*FileFetcher) Get 

func (f *FileFetcher) Get(ctx context.Context, link manifest.Link) Resource

Get implements Fetcher 

func (f *FileFetcher) Links(ctx context.Context) (manifest.LinkList, error)

Links implements Fetcher

type FileResource 

type FileResource struct {
	// contains filtered or unexported fields
}

func NewFileResource 

func NewFileResource(link manifest.Link, abspath string) *FileResource

func (*FileResource) Close 

func (r *FileResource) Close()

Close implements Resource

func (*FileResource) File 

func (r *FileResource) File() string

File implements Resource

func (*FileResource) Length 

func (r *FileResource) Length(ctx context.Context) (int64, *ResourceError)

Length implements Resource 

func (r *FileResource) Link() manifest.Link

Link implements Resource

func (*FileResource) Properties 

func (r *FileResource) Properties() manifest.Properties

Properties implements Resource

func (*FileResource) Read 

func (r *FileResource) Read(ctx context.Context, start int64, end int64) ([]byte, *ResourceError)

Read implements Resource. Ranged reads (end > 0) are safe for concurrent use.

func (*FileResource) Stream 

func (r *FileResource) Stream(ctx context.Context, w io.Writer, start int64, end int64) (int64, *ResourceError)

Stream implements Resource

type GCSFetcher added in v0.9.0 

type GCSFetcher struct {
	// contains filtered or unexported fields
}

func NewGCSFetcher added in v0.9.0 

func NewGCSFetcher(href string, client *storage.Client, handle *storage.ObjectHandle) *GCSFetcher

func (*GCSFetcher) Close added in v0.9.0 

func (f *GCSFetcher) Close()

func (*GCSFetcher) Get added in v0.9.0 

func (f *GCSFetcher) Get(ctx context.Context, link manifest.Link) Resource

Get implements Fetcher 

func (f *GCSFetcher) Links(ctx context.Context) (manifest.LinkList, error)

Links implements Fetcher

type HTTPFetcher added in v0.9.0 

type HTTPFetcher struct {
	// contains filtered or unexported fields
}

func NewHTTPFetcher added in v0.9.0 

func NewHTTPFetcher(href string, client *http.Client, url url.AbsoluteURL) *HTTPFetcher

func (*HTTPFetcher) Close added in v0.9.0 

func (f *HTTPFetcher) Close()

func (*HTTPFetcher) Get added in v0.9.0 

func (f *HTTPFetcher) Get(ctx context.Context, link manifest.Link) Resource

Get implements Fetcher 

func (f *HTTPFetcher) Links(ctx context.Context) (manifest.LinkList, error)

Links implements Fetcher

type LazyResource added in v0.9.0 

type LazyResource struct {
	// contains filtered or unexported fields
}

Wraps a Resource which will be created only when first accessing one of its members.

func NewLazyResource added in v0.9.0 

func NewLazyResource(factory func() Resource) *LazyResource

func (*LazyResource) CRC32Checksum added in v0.13.0 

func (r *LazyResource) CRC32Checksum(ctx context.Context) *uint32

CRC32Checksum implements CompressedResource

func (*LazyResource) Close added in v0.9.0 

func (r *LazyResource) Close()

Close implements Resource

func (*LazyResource) CompressedAs added in v0.9.0 

func (r *LazyResource) CompressedAs(compressionMethod archive.CompressionMethod) bool

CompressedAs implements CompressedResource

func (*LazyResource) CompressedLength added in v0.9.0 

func (r *LazyResource) CompressedLength(ctx context.Context) int64

CompressedLength implements CompressedResource

func (*LazyResource) File added in v0.9.0 

func (r *LazyResource) File() string

File implements Resource

func (*LazyResource) Length added in v0.9.0 

func (r *LazyResource) Length(ctx context.Context) (int64, *ResourceError)

Length implements Resource 

func (r *LazyResource) Link() manifest.Link

Link implements Resource

func (*LazyResource) Properties added in v0.9.0 

func (r *LazyResource) Properties() manifest.Properties

func (*LazyResource) Read added in v0.9.0 

func (r *LazyResource) Read(ctx context.Context, start int64, end int64) ([]byte, *ResourceError)

Read implements Resource

func (*LazyResource) ReadCompressed added in v0.9.0 

func (r *LazyResource) ReadCompressed(ctx context.Context) ([]byte, *ResourceError)

ReadCompressed implements CompressedResource

func (*LazyResource) ReadCompressedGzip added in v0.9.0 

func (r *LazyResource) ReadCompressedGzip(ctx context.Context) ([]byte, *ResourceError)

ReadCompressedGzip implements CompressedResource

func (*LazyResource) Stream added in v0.9.0 

func (r *LazyResource) Stream(ctx context.Context, w io.Writer, start int64, end int64) (int64, *ResourceError)

Stream implements Resource

func (*LazyResource) StreamCompressed added in v0.9.0 

func (r *LazyResource) StreamCompressed(ctx context.Context, w io.Writer) (int64, *ResourceError)

StreamCompressed implements CompressedResource

func (*LazyResource) StreamCompressedGzip added in v0.9.0 

func (r *LazyResource) StreamCompressedGzip(ctx context.Context, w io.Writer) (int64, *ResourceError)

StreamCompressedGzip implements CompressedResource

type ProxyResource 

type ProxyResource struct {
	Res Resource
}

A base class for a Resource which acts as a proxy to another one. Every function is delegating to the proxied resource, and subclasses should override some of them.

func (ProxyResource) CRC32Checksum added in v0.13.0 

func (r ProxyResource) CRC32Checksum(ctx context.Context) *uint32

CRC32Checksum implements CompressedResource

func (ProxyResource) Close 

func (r ProxyResource) Close()

Close implements Resource

func (ProxyResource) CompressedAs added in v0.5.0 

func (r ProxyResource) CompressedAs(compressionMethod archive.CompressionMethod) bool

CompressedAs implements CompressedResource

func (ProxyResource) CompressedLength added in v0.5.0 

func (r ProxyResource) CompressedLength(ctx context.Context) int64

CompressedLength implements CompressedResource

func (ProxyResource) File 

func (r ProxyResource) File() string

File implements Resource

func (ProxyResource) Length 

func (r ProxyResource) Length(ctx context.Context) (int64, *ResourceError)

Length implements Resource 

func (r ProxyResource) Link() manifest.Link

Link implements Resource

func (ProxyResource) Properties 

func (r ProxyResource) Properties() manifest.Properties

func (ProxyResource) Read 

func (r ProxyResource) Read(ctx context.Context, start int64, end int64) ([]byte, *ResourceError)

Read implements Resource

func (ProxyResource) ReadCompressed added in v0.5.0 

func (r ProxyResource) ReadCompressed(ctx context.Context) ([]byte, *ResourceError)

ReadCompressed implements CompressedResource

func (ProxyResource) ReadCompressedGzip added in v0.5.0 

func (r ProxyResource) ReadCompressedGzip(ctx context.Context) ([]byte, *ResourceError)

ReadCompressedGzip implements CompressedResource

func (ProxyResource) Stream 

func (r ProxyResource) Stream(ctx context.Context, w io.Writer, start int64, end int64) (int64, *ResourceError)

Stream implements Resource

func (ProxyResource) StreamCompressed added in v0.5.0 

func (r ProxyResource) StreamCompressed(ctx context.Context, w io.Writer) (int64, *ResourceError)

StreamCompressed implements CompressedResource

func (ProxyResource) StreamCompressedGzip added in v0.5.0 

func (r ProxyResource) StreamCompressedGzip(ctx context.Context, w io.Writer) (int64, *ResourceError)

StreamCompressedGzip implements CompressedResource

type Resource 

type Resource interface {

	// Direct filepath for this resource, when available.
	// Not guaranteed to be set, for example if the resource underwent transformations or is being read from an archive.
	File() string

	// Closes this object and releases any resources associated with it.
	// If the object is already closed then invoking this method has no effect.
	Close()

	// Returns the link from which the resource was retrieved.
	// It might be modified by the [Resource] to include additional metadata, e.g. the `Content-Type` HTTP header in [Link.Type].
	Link() manifest.Link

	// Returns the properties associated with the resource.
	// This is opened for extensions.
	Properties() manifest.Properties

	// Returns data length from metadata if available, or calculated from reading the bytes otherwise.
	// This value must be treated as a hint, as it might not reflect the actual bytes length. To get the real length, you need to read the whole resource.
	Length(ctx context.Context) (int64, *ResourceError)

	// Reads the bytes at the given range.
	// When start and end are zero, the whole content is returned. Out-of-range indexes are clamped to the available length automatically.
	Read(ctx context.Context, start int64, end int64) ([]byte, *ResourceError)

	// Stream the bytes at the given range to a writer.
	// When start and end are zero, the whole content is returned. Out-of-range indexes are clamped to the available length automatically.
	Stream(ctx context.Context, w io.Writer, start int64, end int64) (int64, *ResourceError)
}

Acts as a proxy to an actual resource by handling read access.

func NewHTTPResource added in v0.15.0 

func NewHTTPResource(link manifest.Link, client *http.Client, url url.AbsoluteURL) Resource

Creates a Resource serving the contents of the given HTTP(S) URL.

type ResourceError 

type ResourceError struct {
	Cause error
	Code  ResourceErrorCode
}

Errors occurring while accessing a resource.

func BadRequest 

func BadRequest(cause error) *ResourceError

Equivalent to a 400 HTTP error.

func Forbidden 

func Forbidden(cause error) *ResourceError

Equivalent to a 403 HTTP error. This can be returned when trying to read a resource protected with a DRM that is not unlocked.

func NewResourceError 

func NewResourceError(code ResourceErrorCode) *ResourceError

func NewResourceErrorWithCause 

func NewResourceErrorWithCause(code ResourceErrorCode, cause error) *ResourceError

func NotFound 

func NotFound(cause error) *ResourceError

Equivalent to a 404 HTTP error.

func OsErrorToException 

func OsErrorToException(err error) *ResourceError

Convert a Go os error to an exception

func Other 

func Other(cause error) *ResourceError

For any other error, such as HTTP 500.

func OutOfMemory 

func OutOfMemory(cause error) *ResourceError

Equivalent to a 507 HTTP error. Used when the requested range is too large to be read in memory.

func RangeNotSatisfiable 

func RangeNotSatisfiable(cause error) *ResourceError

Equivalent to a 416 HTTP error. Used when the requested range is not satisfiable (invalid)

func ReadResourceAsJSON 

func ReadResourceAsJSON(ctx context.Context, r Resource) (map[string]interface{}, *ResourceError)

func ReadResourceAsString 

func ReadResourceAsString(ctx context.Context, r Resource) (string, *ResourceError)

func ReadResourceAsXML 

func ReadResourceAsXML(ctx context.Context, r Resource) (*xmlquery.Node, *ResourceError)

func Timeout 

func Timeout(cause error) *ResourceError

Equivalent to a 504 HTTP error. Used when a request for a file times out (e.g. when fetching from remote storage)

func Unavailable 

func Unavailable(cause error) *ResourceError

Equivalent to a 503 HTTP error. Used when the source can't be reached, e.g. no Internet connection, or an issue with the file system. Usually this is a temporary error.

func (*ResourceError) Error 

func (ex *ResourceError) Error() string

func (*ResourceError) HTTPStatus 

func (ex *ResourceError) HTTPStatus() int

type ResourceErrorCode 

type ResourceErrorCode uint16
const (
	CodeBadRequest                   ResourceErrorCode = http.StatusBadGateway
	CodeNotFound                     ResourceErrorCode = http.StatusNotFound
	CodeForbidden                    ResourceErrorCode = http.StatusForbidden
	CodeServiceUnavailable           ResourceErrorCode = http.StatusServiceUnavailable
	CodeInsufficientStorage          ResourceErrorCode = http.StatusInsufficientStorage
	CodeRequestedRangeNotSatisfiable ResourceErrorCode = http.StatusRequestedRangeNotSatisfiable
	CodeGatewayTimeout               ResourceErrorCode = http.StatusGatewayTimeout
	CodeInternalServerError          ResourceErrorCode = http.StatusInternalServerError
)

Error codes with HTTP equivalents 

const (
	Offline ResourceErrorCode
	Cancelled
)

The rest of the codes

type ResourceReadSeeker 

type ResourceReadSeeker struct {
	// contains filtered or unexported fields
}

For opening a fetcher.Resource as a io.ReadSeeker

func NewResourceReadSeeker 

func NewResourceReadSeeker(r Resource) *ResourceReadSeeker

func (*ResourceReadSeeker) Read 

func (rs *ResourceReadSeeker) Read(p []byte) (n int, err error)

Read implements io.Reader

func (*ResourceReadSeeker) Seek 

func (rs *ResourceReadSeeker) Seek(offset int64, whence int) (int64, error)

Seek implements io.ReadSeeker

type ResourceTransformer 

type ResourceTransformer func(Resource) Resource

*

  • Implements the transformation of a Resource. It can be used, for example, to decrypt,
  • deobfuscate, inject CSS or JavaScript, correct content – e.g. adding a missing dir="rtl" in an
  • HTML document, pre-process – e.g. before indexing a publication's content, etc. *
  • If the transformation doesn't apply, simply return resource unchanged.

type S3Fetcher added in v0.9.0 

type S3Fetcher struct {
	// contains filtered or unexported fields
}

func NewS3Fetcher added in v0.9.0 

func NewS3Fetcher(href string, client *s3.Client, bucket, key string) *S3Fetcher

func (*S3Fetcher) Close added in v0.9.0 

func (f *S3Fetcher) Close()

func (*S3Fetcher) Get added in v0.9.0 

func (f *S3Fetcher) Get(ctx context.Context, link manifest.Link) Resource

Get implements Fetcher 

func (f *S3Fetcher) Links(ctx context.Context) (manifest.LinkList, error)

Links implements Fetcher

type TransformingFetcher 

type TransformingFetcher struct {
	// contains filtered or unexported fields
}

Transforms the resources' content of a child fetcher using a list of ResourceTransformer functions.

func NewTransformingFetcher 

func NewTransformingFetcher(fetcher Fetcher, transformers ...ResourceTransformer) *TransformingFetcher

func (*TransformingFetcher) Close 

func (f *TransformingFetcher) Close()

Close implements Fetcher

func (*TransformingFetcher) Get 

func (f *TransformingFetcher) Get(ctx context.Context, link manifest.Link) Resource

Get implements Fetcher 

func (f *TransformingFetcher) Links(ctx context.Context) (manifest.LinkList, error)

Links implements Fetcher

type TransformingResource 

type TransformingResource struct {
	// contains filtered or unexported fields
}

*

  • Transforms the bytes of [resource] on-the-fly. *
  • Warning: The transformation runs on the full content of [resource], so it's not appropriate for
  • large resources which can't be held in memory. Pass [cacheBytes] = true to cache the result of
  • the transformation. This may be useful if multiple ranges will be read.

func NewTransformingResource added in v0.9.0 

func NewTransformingResource(resource Resource, cacheBytes bool, transform func([]byte) []byte) *TransformingResource

func (*TransformingResource) Close added in v0.9.0 

func (r *TransformingResource) Close()

Close implements Resource

func (*TransformingResource) File added in v0.9.0 

func (r *TransformingResource) File() string

File implements Resource

func (*TransformingResource) Length added in v0.9.0 

func (r *TransformingResource) Length(ctx context.Context) (int64, *ResourceError)

Length implements Resource 

func (r *TransformingResource) Link() manifest.Link

Link implements Resource

func (*TransformingResource) Properties added in v0.9.0 

func (r *TransformingResource) Properties() manifest.Properties

func (*TransformingResource) Read added in v0.9.0 

func (r *TransformingResource) Read(ctx context.Context, start int64, end int64) ([]byte, *ResourceError)

Read implements Resource

func (*TransformingResource) Stream added in v0.9.0 

func (r *TransformingResource) Stream(ctx context.Context, w io.Writer, start int64, end int64) (int64, *ResourceError)

Stream implements Resource

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.