Documentation
¶
Overview ¶
Package fetch contains the reusable web fetch core shared by browser- and tool-facing adapters.
Index ¶
- Constants
- Variables
- func DefaultHTTPClient(resolver webcore.HostResolver) *http.Client
- func Err(msg string) error
- func FetchWithRedirect(client HTTPClient, method, url string, headers map[string]string, ...) (resp *http.Response, err error)
- func HTMLToMarkdown(html string, baseURL string) string
- func IsPreapproved(hostname string) bool
- func IsPreapprovedPath(hostname, pathname string) bool
- func NewReadCloser(data []byte) io.ReadCloser
- func NormalizeRenderMode(raw string) (string, error)
- func NormalizeURL(raw string) (string, *url.URL, error)
- func ReadBody(resp *http.Response) ([]byte, error)
- type Cache
- type CacheEntry
- type Config
- type DecisionCache
- type FetchedContent
- type HTTPClient
- type ReadCloser
- type RedirectInfo
- type Request
- type Service
Constants ¶
const ( // RenderModeAuto keeps the routing policy in charge of choosing the backend. RenderModeAuto = webcore.RenderModeAuto // RenderModeHTTP forces the fast HTTP path. RenderModeHTTP = webcore.RenderModeHTTP // RenderModeBrowser forces JS-rendered extraction through the browser runtime. RenderModeBrowser = webcore.RenderModeBrowser )
const BrowserSnapshotTextLimit = 12000
BrowserSnapshotTextLimit bounds the browser-rendered text payload returned to fetch callers.
Variables ¶
var ( // ErrInvalidURL indicates an invalid URL ErrInvalidURL = errors.New("invalid URL") // ErrDomainBlocked indicates domain is blocked ErrDomainBlocked = errors.New("domain is blocked by security policy") // ErrDomainCheckFailed indicates domain check failure ErrDomainCheckFailed = errors.New("unable to verify domain safety") // ErrTooManyRedirects indicates too many redirects ErrTooManyRedirects = errors.New("too many redirects") // ErrContentTooLarge indicates content exceeds limit ErrContentTooLarge = errors.New("content exceeds maximum size") // ErrTimeout indicates request timeout ErrTimeout = errors.New("request timeout") // ErrAborted indicates request was aborted ErrAborted = errors.New("request aborted") )
var PathPrefixes = webcore.PreapprovedPathPrefixes
PathPrefixes keeps the old fetch-local name as an alias to the shared path-scoped allowlist.
var PreapprovedHosts = webcore.PreapprovedHosts
PreapprovedHosts re-exports the shared documentation host allowlist for backward compatibility.
Functions ¶
func DefaultHTTPClient ¶
func DefaultHTTPClient(resolver webcore.HostResolver) *http.Client
DefaultHTTPClient creates a default HTTP client
func FetchWithRedirect ¶
func FetchWithRedirect( client HTTPClient, method, url string, headers map[string]string, redirectChecker func(original, redirect string) bool, maxRedirects int, ) (resp *http.Response, err error)
FetchWithRedirect performs HTTP request with custom redirect handling
func HTMLToMarkdown ¶
HTMLToMarkdown converts HTML content into a compact markdown-like representation. The V1 pipeline prefers readability-style main-content extraction, then falls back to structural DOM selection, and only then uses the raw renderer.
func IsPreapproved ¶
IsPreapproved reports whether the hostname is covered by the shared preapproval policy.
func IsPreapprovedPath ¶
IsPreapprovedPath reports whether the host/path pair is covered by the shared preapproval policy.
func NewReadCloser ¶
func NewReadCloser(data []byte) io.ReadCloser
NewReadCloser creates a new ReadCloser from bytes
func NormalizeRenderMode ¶
NormalizeRenderMode converts empty or mixed-case render hints into the canonical routing values.
func NormalizeURL ¶
NormalizeURL upgrades plain HTTP inputs to HTTPS and ensures the fetch core always receives a canonical URL.
Types ¶
type Cache ¶
type Cache = webcore.TTLCache[CacheEntry]
Cache is a fetch-specific view over the shared web TTL cache.
func DefaultCache ¶
func DefaultCache() *Cache
DefaultCache creates the default fetch cache on top of the shared web cache implementation.
type CacheEntry ¶
type CacheEntry struct {
Content string
Bytes int
Code int
CodeText string
ContentType string
FinalURL string
PersistedPath string
PersistedSize int
BrowserRecommended bool
}
CacheEntry represents a cached URL response.
type Config ¶
type Config struct {
HTTPClient HTTPClient
BrowserManager webcore.BrowserManager
ArtifactStore webcore.ArtifactStore
Cache *Cache
DecisionCache *DecisionCache
Resolver webcore.HostResolver
Timeout time.Duration
MaxContentLength int
MaxRedirects int
RenderPoolEnabled bool
RenderPoolTTL time.Duration
RenderPoolMaxSessions int
}
Config configures the shared fetch service.
func DefaultConfig ¶
func DefaultConfig() *Config
DefaultConfig returns a pragmatic default configuration for the shared fetch service.
type DecisionCache ¶
DecisionCache stores stable auto-routing outcomes so repeated fetches avoid paying the HTTP-then-browser probe cost for hosts that are consistently browser-first or HTTP-friendly.
func DefaultDecisionCache ¶
func DefaultDecisionCache() *DecisionCache
type FetchedContent ¶
type FetchedContent = webcore.FetchResponse
FetchedContent represents normalized fetched content returned by the core service.
type HTTPClient ¶
type HTTPClient interface {
// Do executes an HTTP request
Do(req *http.Request) (*http.Response, error)
}
HTTPClient defines the HTTP client interface
type ReadCloser ¶
type ReadCloser struct {
// contains filtered or unexported fields
}
ReadCloser wraps a byte slice as an io.ReadCloser
func (*ReadCloser) Close ¶
func (rc *ReadCloser) Close() error
type RedirectInfo ¶
type RedirectInfo = webcore.RedirectInfo
RedirectInfo indicates that the caller should explicitly follow a host-changing redirect.
type Request ¶
type Request = webcore.FetchRequest
Request describes a backend fetch request independent from any specific tool wrapper.
type Service ¶
type Service struct {
// contains filtered or unexported fields
}
Service implements the reusable fetch core used by tool wrappers and future runtime integrations.
func NewService ¶
NewService creates a new shared fetch service with local defaults for HTTP and caching.
Source Files