Documentation
¶
Overview ¶
Package kindstream is Ether's unified byte-acquisition and streaming layer for plugin-provided content kinds (spec §4, §5, §15.1). Every plugin media item — comic page, ebook file, audiobook chapter, podcast episode, or video part — resolves through kindstream into a random-access ByteSource.
Index ¶
- Constants
- Variables
- func ExpandHeaders(ctx context.Context, headers map[string]string, ...) (map[string]string, error)
- func Handler(src ByteSource) http.Handler
- func MediaSource(src ByteSource, releaseID string) core.MediaSource
- func ReadAhead(ctx context.Context, r io.ReaderAt, size int64) io.ReaderAt
- func Seeker(src ByteSource) io.ReadSeekCloser
- func ServeLoopback(src ByteSource) (urlStr string, closeFn func() error, err error)
- func TranscodeSource(open OpenFunc, releaseID string) func(ctx context.Context) (core.MediaSource, error)
- func UnwrapMediaSource(bs ByteSource) core.MediaSource
- type ByteSource
- func ExtractMember(ctx context.Context, c core.ContainerRef, format archive.Format, ...) (ByteSource, error)
- func Open(ctx context.Context, src core.PlayableSource, deps Deps) (ByteSource, error)
- func OpenContainer(ctx context.Context, c core.ContainerRef, declaredMIME string, maxBytes int64, ...) (ByteSource, error)
- func OpenSources(ctx context.Context, sources []core.PlayableSource, deps Deps) (ByteSource, core.PlayableSource, error)
- type Deps
- type DigestReporter
- type DirectSource
- func (d *DirectSource) Close() error
- func (d *DirectSource) DirectHeaders() map[string]string
- func (d *DirectSource) DirectURL() string
- func (d *DirectSource) MIME() string
- func (d *DirectSource) Name() string
- func (d *DirectSource) ReadAt(p []byte, off int64) (int, error)
- func (d *DirectSource) Size() int64
- type HTTPReaderAt
- func (r *HTTPReaderAt) Close() error
- func (r *HTTPReaderAt) MIME() string
- func (r *HTTPReaderAt) Name() string
- func (r *HTTPReaderAt) ReadAt(p []byte, off int64) (int, error)
- func (r *HTTPReaderAt) ReadAtContext(ctx context.Context, p []byte, off int64) (int, error)
- func (r *HTTPReaderAt) Size() int64
- func (r *HTTPReaderAt) SniffBytes() []byte
- type HostBackoff
- type Leaf
- type OpenFunc
- type Sequence
- func (s *Sequence) Close() error
- func (s *Sequence) IsLeafOpen(i int) bool
- func (s *Sequence) Leaf(i int) (ByteSource, error)
- func (s *Sequence) LeafContext(ctx context.Context, i int) (ByteSource, error)
- func (s *Sequence) Len() int
- func (s *Sequence) MIME() string
- func (s *Sequence) Name() string
- func (s *Sequence) OpenLeafCount() int
- func (s *Sequence) ReadAt(p []byte, off int64) (int, error)
- func (s *Sequence) ReadAtContext(ctx context.Context, p []byte, off int64) (int, error)
- func (s *Sequence) Size() int64
- type SourceFailure
- type TemplateVars
- type URLTemplate
Constants ¶
const ( PatternFile = "file" PatternRange = "range" PatternDownload = "download" PatternDirect = "direct" PatternContainer = "container" PatternRelease = "release" )
Retrieval pattern constants (spec §4.1, §4.3).
const DefaultMaxNonVideoReleaseGrabs = 3
DefaultMaxNonVideoReleaseGrabs bounds how many non-video Usenet release candidates OpenSources will fetch in a single play attempt before stopping, so a failing play never burns an entire page of indexer grabs (#659).
Variables ¶
var ( // ErrDigestMismatch is returned when served bytes do not match a digest // or expected size the plugin declared on the source. ErrDigestMismatch = errors.New("kindstream: bytes do not match the declared digest") // ErrPluginAnswer marks an open failure that points at the plugin's own // answer — a missing archive member, an out-of-range member index, a // digest or size mismatch — rather than at a host being unavailable. A // fallback that opens after such a failure must not be saved as the // item's resolution: the plugin's first choice is broken, not absent. ErrPluginAnswer = errors.New("kindstream: source contradicts the plugin's answer") )
var ( // ErrMemberUndecodable is returned when an archive member is encrypted // without seekable/sequential support or uses an archive format/compression // method that internal/archive cannot decode sequentially. It wraps // core.ErrArchiveUnstreamable so frontends surface it plainly. ErrMemberUndecodable = fmt.Errorf("kindstream: archive member not sequentially decodable: %w", core.ErrArchiveUnstreamable) // ErrHostNotAllowed is returned when a source URL's host is rejected by the // plugin's declared network allowlist. ErrHostNotAllowed = errors.New("kindstream: host not in plugin allowlist") // ErrSizeLimitExceeded is returned when a download or range-demoted source // exceeds its configured maxBytes or library cache quota. ErrSizeLimitExceeded = errors.New("kindstream: source exceeds download size limit") // ErrMIMEMismatch is returned when sniffed magic bytes disagree with the // plugin-declared MIME at the family level (e.g. text/html error page for an // image/* source). ErrMIMEMismatch = errors.New("kindstream: sniffed MIME family disagrees with declared MIME") )
var ( // ErrInvalidTemplate is returned when a URL template or secret placeholder // is malformed or expands to a disallowed URL. ErrInvalidTemplate = errors.New("kindstream: invalid URL template") )
Functions ¶
func ExpandHeaders ¶
func ExpandHeaders(ctx context.Context, headers map[string]string, secretFn func(ctx context.Context, name string) (string, error)) (map[string]string, error)
ExpandHeaders resolves {secret:name} placeholders in header values via secretFn. Resolved values must never be logged by callers.
func Handler ¶
func Handler(src ByteSource) http.Handler
Handler returns an http.Handler that serves a ByteSource with full HTTP Range support (206 Partial Content, Accept-Ranges: bytes, Content-Type, and Content-Length).
func MediaSource ¶
func MediaSource(src ByteSource, releaseID string) core.MediaSource
MediaSource adapts a ByteSource into a core.MediaSource so any pipeline accepting core.MediaSource can consume a kindstream ByteSource directly.
func ReadAhead ¶
ReadAhead wraps r for one sequential consumer (one HTTP response): small sequential reads such as http.ServeContent's 32 KiB copies are coalesced into ranged reads that grow to 4 MiB, while a seek drops back to a small window. Create one per request; it is not meant to be shared.
func Seeker ¶
func Seeker(src ByteSource) io.ReadSeekCloser
Seeker wraps any ByteSource as an io.ReadSeekCloser so it can be passed to http.ServeContent, standard library readers, or transcode pipelines.
func ServeLoopback ¶
func ServeLoopback(src ByteSource) (urlStr string, closeFn func() error, err error)
ServeLoopback starts a loopback HTTP server (http://127.0.0.1:<port>/<name>) backed by src so local ffmpeg / ffprobe sessions in internal/transcode can seek and stream any ByteSource over HTTP Range (spec §15.1). Calling closeFn shuts down the loopback server without closing src.
func TranscodeSource ¶
func TranscodeSource(open OpenFunc, releaseID string) func(ctx context.Context) (core.MediaSource, error)
TranscodeSource adapts a kindstream ByteSource opener into the opener shape the playback session pipeline expects (spec §15.1). A plugin video/audio leaf therefore becomes a normal transcoder input: the existing decision, HLS/DASH and probe machinery runs unchanged over the returned MediaSource.
releaseID names the underlying source for session accounting and snapshots; when empty the ByteSource's Name is used. The ByteSource is opened on the first call to the returned function, so a caller that never plays pays nothing.
func UnwrapMediaSource ¶
func UnwrapMediaSource(bs ByteSource) core.MediaSource
UnwrapMediaSource returns the underlying core.MediaSource when bs wraps one directly without archive extraction, preserving optional playback interfaces.
Types ¶
type ByteSource ¶
type ByteSource interface {
io.ReaderAt
Size() int64 // total byte length; always > 0 once Open returns
Name() string // filename for Content-Disposition and container sniffing
MIME() string // MIME type (plugin-declared, verified/refined by sniffing first 512 B)
Close() error
}
ByteSource is the uniform random-access stream interface produced by kindstream for all plugin content kinds (spec §5.1).
func ExtractMember ¶
func ExtractMember( ctx context.Context, c core.ContainerRef, format archive.Format, selected archive.Member, nonDirIdx int, declaredMIME string, maxBytes int64, deps Deps, ) (ByteSource, error)
ExtractMember sequentially decodes the target member from a compressed or solid archive into deps.Cache and returns a random-access ByteSource over the extracted file.
Compressed ZIP, non-solid 7z, RAR4, and RAR5 volumes are decoded directly over ranged volume readers without caching whole volumes first. Solid 7z archives over remote URL volumes cache the volume first via ensureVolumeCached, honouring ctx and the library cache quota. Encrypted non-stored members and unsupported archive formats/methods fail with ErrMemberUndecodable.
func Open ¶
func Open(ctx context.Context, src core.PlayableSource, deps Deps) (ByteSource, error)
Open resolves a single core.PlayableSource into a random-access ByteSource according to its retrieval pattern (spec §4.3, §5.1).
func OpenContainer ¶
func OpenContainer(ctx context.Context, c core.ContainerRef, declaredMIME string, maxBytes int64, deps Deps) (ByteSource, error)
OpenContainer opens an archive container (RAR4/RAR5/7z/ZIP) over one or more VolumeRefs and returns a ByteSource for the requested inner member (spec §5.3).
Store-mode members (including encrypted store-mode RAR/7z when PasswordSecret is supplied) are served directly via archive.NewMemberReader without extracting the archive. Compressed ZIP, 7z (single-coder LZMA/LZMA2), RAR4, and RAR5 members are sequentially decoded into deps.Cache via ExtractMember. Encrypted non-stored members and unsupported archive formats/methods fail with ErrMemberUndecodable (wrapping core.ErrArchiveUnstreamable).
func OpenSources ¶
func OpenSources(ctx context.Context, sources []core.PlayableSource, deps Deps) (ByteSource, core.PlayableSource, error)
OpenSources tries each candidate core.PlayableSource in order, returning the first ByteSource that opens and validates cleanly. When every candidate fails with core.ErrArchiveUnstreamable, that error is preserved rather than masked as core.ErrSourceUnavailable; when every candidate fails with a permanent rejection (e.g. core.ErrReleaseRejected, archive.ErrMissingVolume, core.ErrNoUsableFile, core.ErrArticleMissing), the returned error wraps core.ErrReleaseRejected rather than core.ErrSourceUnavailable; otherwise all-failed candidates wrap core.ErrSourceUnavailable (spec §4.5).
For non-video release candidates (#659), OpenSources caps Usenet release grabs per play (DefaultMaxNonVideoReleaseGrabs) and stops trying further candidates of the same shape after the first structural core.ErrNoUsableFile verdict so one failed play never burns a full page of indexer grabs.
type Deps ¶
type Deps struct {
OpenFile func(context.Context, core.PlayableSource) (ByteSource, error)
HTTP *http.Client
Cache *blobcache.Namespace
OpenRelease func(ctx context.Context, r core.ReleaseHint) (core.MediaSource, error)
// OpenReleaseSource, when set, is preferred over OpenRelease so release
// opening receives the full PlayableSource (MIME, Container, Release).
OpenReleaseSource func(ctx context.Context, src core.PlayableSource) (core.MediaSource, error)
// DefaultReleaseMIME is the kind-level fallback MIME (e.g. "audio/*",
// "application/epub+zip") when a release source omits src.MIME.
DefaultReleaseMIME string
// MaxNonVideoReleaseGrabs caps how many non-video Usenet release grabs
// OpenSources attempts per play (0 uses DefaultMaxNonVideoReleaseGrabs;
// negative disables the cap).
MaxNonVideoReleaseGrabs int
// CandidateOpenTimeout, when positive, bounds how long OpenSources spends
// opening any single candidate before moving on to the next candidate.
CandidateOpenTimeout time.Duration
Secret func(ctx context.Context, name string) (string, error)
AllowHost func(host string) bool
MaxDownloadBytes int64
ItemIndex int
Backoff *HostBackoff
// OnSourceFailed, when set, is called each time a candidate fails to open,
// regardless of whether a later candidate succeeds.
OnSourceFailed func(attempt, total int, src core.PlayableSource, took time.Duration, err error)
// OnSourceSkipped, when set, is called for each candidate that failed
// before a later one opened, so the caller can log it and decide
// whether the winner may be saved as the item's resolution.
OnSourceSkipped func(failures []SourceFailure, winning core.PlayableSource)
}
Deps supplies the runtime dependencies needed by Open and OpenSources (spec §5.1).
type DigestReporter ¶
type DigestReporter interface {
DigestState() (declared, verified bool)
}
DigestReporter is implemented by a ByteSource whose declared digest is still being verified as it streams. DigestState reports whether a digest was declared and whether the full bytes have been verified yet.
type DirectSource ¶
type DirectSource struct {
// contains filtered or unexported fields
}
DirectSource represents a validated "direct" core.PlayableSource. Callers that support client redirects (spec §4.3) can extract the target URL via DirectURL() and issue a 307 Temporary Redirect without proxying bytes; callers that require a ByteSource stream (such as tests or local transcode bridges) can call ReadAt, which lazily opens the underlying HTTP source.
func AsDirectSource ¶
func AsDirectSource(bs ByteSource) (*DirectSource, bool)
AsDirectSource extracts the *DirectSource if bs was opened from a "direct" core.PlayableSource.
func (*DirectSource) Close ¶
func (d *DirectSource) Close() error
func (*DirectSource) DirectHeaders ¶
func (d *DirectSource) DirectHeaders() map[string]string
DirectHeaders returns the resolved request headers for the direct URL.
func (*DirectSource) DirectURL ¶
func (d *DirectSource) DirectURL() string
DirectURL returns the validated, template-expanded target URL for a 307 redirect.
func (*DirectSource) MIME ¶
func (d *DirectSource) MIME() string
func (*DirectSource) Name ¶
func (d *DirectSource) Name() string
func (*DirectSource) Size ¶
func (d *DirectSource) Size() int64
type HTTPReaderAt ¶
type HTTPReaderAt struct {
// contains filtered or unexported fields
}
HTTPReaderAt implements ByteSource (and io.ReaderAt) over an HTTP Range or range-templated URL source.
func (*HTTPReaderAt) Close ¶
func (r *HTTPReaderAt) Close() error
Close is a no-op for stateless HTTP range readers.
func (*HTTPReaderAt) MIME ¶
func (r *HTTPReaderAt) MIME() string
MIME returns the resource's MIME type.
func (*HTTPReaderAt) Name ¶
func (r *HTTPReaderAt) Name() string
Name returns the resource's filename.
func (*HTTPReaderAt) ReadAt ¶
func (r *HTTPReaderAt) ReadAt(p []byte, off int64) (int, error)
ReadAt reads len(p) bytes starting at byte offset off.
func (*HTTPReaderAt) ReadAtContext ¶
ReadAtContext reads len(p) bytes starting at byte offset off using ctx.
func (*HTTPReaderAt) Size ¶
func (r *HTTPReaderAt) Size() int64
Size returns the total byte size of the resource.
func (*HTTPReaderAt) SniffBytes ¶
func (r *HTTPReaderAt) SniffBytes() []byte
SniffBytes returns up to the first 512 bytes captured during the initial probe.
type HostBackoff ¶
type HostBackoff struct {
// contains filtered or unexported fields
}
HostBackoff tracks per-host exponential backoff and Retry-After windows on 429 and 503 responses.
func NewHostBackoff ¶
func NewHostBackoff() *HostBackoff
NewHostBackoff creates a per-host backoff tracker.
func (*HostBackoff) IsBackingOff ¶
func (b *HostBackoff) IsBackingOff(rawURL string) bool
IsBackingOff reports whether rawURL's host currently has an active rate-limit or service-unavailable backoff window.
type Leaf ¶
type Leaf struct {
Name string
MIME string
Size int64
Open func(ctx context.Context) (ByteSource, error)
}
Leaf describes one part/page/chapter in a multi-part Sequence (spec §4.4, §5.1).
type OpenFunc ¶
type OpenFunc func(ctx context.Context) (ByteSource, error)
OpenFunc opens one kindstream ByteSource. It is the leaf-level opener a resolver hands to the transcode bridge: called lazily, once per open, with the session's context so cancellation and stream-limit accounting apply.
type Sequence ¶
type Sequence struct {
// contains filtered or unexported fields
}
Sequence presents an ordered list of Leaf sources both as discrete addressable leaves (Leaf(i)) and as a single concatenated ByteSource (ReadAt) when every leaf has a known positive Size (spec §4.4, §5.1).
Sequence opens leaves lazily on demand, holds at most 2 leaves open at a time (current + next), and prefetches the next leaf when a read crosses 90% of the current leaf.
func NewSequence ¶
NewSequence constructs a Sequence over leaves.
func (*Sequence) IsLeafOpen ¶
IsLeafOpen reports whether leaf i is currently open in the sequence pool.
func (*Sequence) Leaf ¶
func (s *Sequence) Leaf(i int) (ByteSource, error)
Leaf opens (or returns the already-open) ByteSource for leaf i (0-based). The returned ByteSource shares the sequence's open-leaf pool; calling Close on the returned leaf handle is a no-op (Sequence.Close closes all open leaves).
func (*Sequence) LeafContext ¶
LeafContext is Leaf with an explicit context for opening the leaf on cache miss.
func (*Sequence) OpenLeafCount ¶
OpenLeafCount returns the number of currently open leaves (at most 2).
func (*Sequence) ReadAtContext ¶
ReadAtContext reads len(p) bytes at offset off across the concatenated leaves using ctx.
type SourceFailure ¶
type SourceFailure struct {
Source core.PlayableSource
Err error
}
SourceFailure is one candidate that failed to open before a later candidate succeeded (or before every candidate failed).
func (SourceFailure) PluginAnswer ¶
func (f SourceFailure) PluginAnswer() bool
PluginAnswer reports whether the failure points at the plugin's own answer (see ErrPluginAnswer) rather than at an unavailable host.
type TemplateVars ¶
TemplateVars holds the integer values allowed in a URL template (spec §4.2). Values are strictly non-negative integers; arbitrary string interpolation is never performed.
type URLTemplate ¶
type URLTemplate struct {
// contains filtered or unexported fields
}
URLTemplate is a parsed and validated URL template.
func ParseURLTemplate ¶
func ParseURLTemplate(raw string, allowHost func(host string) bool) (*URLTemplate, error)
ParseURLTemplate validates that raw contains only known placeholders ({itemIndex}, {pageNumber}, {byteStart}, {byteEnd}, {byteRange}) and that a trial expansion with zero values yields a valid http/https URL with a host.
func (*URLTemplate) Expand ¶
func (t *URLTemplate) Expand(v TemplateVars, allowHost func(host string) bool) (string, error)
Expand substitutes integer placeholders and validates the resulting URL's scheme (http/https), non-empty host, and optional host allowlist.
func (*URLTemplate) HasByteRange ¶
func (t *URLTemplate) HasByteRange() bool
HasByteRange reports whether the template embeds a bounded byte range ({byteRange} or both {byteStart} and {byteEnd}) in the URL itself.
func (*URLTemplate) HasByteStartOnly ¶
func (t *URLTemplate) HasByteStartOnly() bool
HasByteStartOnly reports whether the template embeds {byteStart} without an end byte, indicating an upstream that seeks from {byteStart} without a Range request header (spec §4.2).
func (*URLTemplate) UsesRangePlaceholders ¶
func (t *URLTemplate) UsesRangePlaceholders() bool
UsesRangePlaceholders reports whether any byte-offset placeholder is present.