webfetch

package
v0.14.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 29, 2026 License: MIT Imports: 27 Imported by: 0

Documentation

Index

Constants

View Source
const (
	ErrorCodeInvalidURL         = "invalid_url"
	ErrorCodePrivateNetwork     = "private_network_blocked"
	ErrorCodeUnsupportedReader  = "unsupported_reader"
	ErrorCodeNotHTML            = "not_html"
	ErrorCodeResponseTooLarge   = "response_too_large"
	ErrorCodeUpstreamHTTP       = "upstream_http"
	ErrorCodeAuthentication     = "authentication_failed"
	ErrorCodeRateLimited        = "rate_limited"
	ErrorCodeMissingAPIKey      = "missing_api_key"
	ErrorCodeRequestTimeout     = "request_timeout"
	ErrorCodeCanceled           = "canceled"
	ErrorCodeInvalidArgument    = "invalid_argument"
	ErrorCodeUnsupportedSearch  = "unsupported_search_option"
	ErrorCodeInvalidJSON        = "invalid_json"
	ErrorCodeNoInput            = "no_input"
	ErrorCodeProviderResponse   = "provider_response_invalid"
	ErrorCodeExtraction         = "extraction_failed"
	ErrorCodeReaderFallback     = "reader_fallback_failed"
	ErrorCodeBrowserUnavailable = "browser_unavailable"
	ErrorCodeRenderFailed       = "render_failed"
	ErrorCodeRenderBudget       = "render_budget_exceeded"
)
View Source
const (
	ReaderModeJina     = "jina"
	ReaderModeDefuddle = "defuddle"
	ReaderModeAuto     = "auto"
)
View Source
const (
	RenderModeNever  = "never"
	RenderModeAuto   = "auto"
	RenderModeAlways = "always"

	RenderWaitLoad        = "load"
	RenderWaitNetworkIdle = "networkidle"
)
View Source
const (
	DefaultTimeout               = 30 * time.Second
	DefaultMaxBodyBytes          = 8 << 20
	DefaultRenderTimeout         = 30 * time.Second
	DefaultRenderMaxConcurrency  = 2
	DefaultRenderMaxRequests     = 100
	DefaultRenderMaxNetworkBytes = 32 << 20
	DefaultSearchLimit           = 5
	MaxSearchLimit               = 50
	DefaultReaderEndpoint        = "https://r.jina.ai"
	DefaultSearchEndpoint        = "https://api.search.brave.com/res/v1/web/search"
	DefaultExaEndpoint           = "https://api.exa.ai/search"
	DefaultUserAgent             = "webfetch/0.1"
)
View Source
const ProjectionFormatMarkdown = "markdown"

ProjectionFormatMarkdown selects the default content projection.

Variables

View Source
var (
	// ErrNotHTML indicates that an HTML reader received a non-HTML response.
	ErrNotHTML = errors.New("response is not HTML")

	// ErrTooLarge indicates that a response exceeded the configured body limit.
	ErrTooLarge = errors.New("response body exceeds configured limit")

	// ErrUnsupportedReader indicates that a fetch request selected an unknown reader.
	ErrUnsupportedReader = errors.New("unsupported reader")
)

Functions

func ContentLineCount

func ContentLineCount(content string) int

ContentLineCount counts logical lines without treating a trailing newline as an additional empty line.

func NewCodedError

func NewCodedError(err error, code, suggestion string) error

NewCodedError wraps err with machine-readable recovery metadata.

func NormalizeProjectionFormat

func NormalizeProjectionFormat(format string) string

NormalizeProjectionFormat trims and lowercases a projection name.

func ProjectionFormatIsMarkdown

func ProjectionFormatIsMarkdown(format string) bool

ProjectionFormatIsMarkdown reports whether format selects the default view.

func TruncationMarker

func TruncationMarker(projection ContentProjection) string

TruncationMarker renders the stable human-readable output budget marker.

Types

type Client

type Client struct {
	// contains filtered or unexported fields
}

Client performs bounded HTTP requests with redirect and network safety checks.

func NewClient

func NewClient(cfg ClientConfig) *Client

NewClient constructs a bounded HTTP client from explicit configuration.

func (*Client) Do

func (c *Client) Do(ctx context.Context, method, rawURL string, headers http.Header, body []byte) (httpResponse, error)

Do performs one bounded HTTP operation and retries only safe methods.

func (*Client) Get

func (c *Client) Get(ctx context.Context, rawURL string, headers http.Header) (httpResponse, error)

Get performs a safe, retryable GET request.

type ClientConfig

type ClientConfig struct {
	Timeout              time.Duration
	MaxBodyBytes         int64
	AllowPrivateNetworks bool
	UserAgent            string
}

ClientConfig controls the shared HTTP client.

type CodedError

type CodedError struct {
	Err        error
	Code       string
	Suggestion string
}

CodedError preserves the original error while exposing stable, actionable metadata to machine-readable callers.

func (*CodedError) Error

func (e *CodedError) Error() string

func (*CodedError) Unwrap

func (e *CodedError) Unwrap() error

type Config

type Config struct {
	// UserAgent identifies the owning application for direct and rendered HTTP requests.
	UserAgent             string
	JinaAPIKey            string
	BraveAPIKey           string
	ExaAPIKey             string
	ReaderMode            string
	ReaderEndpoint        string
	SearchEndpoint        string
	ExaSearchEndpoint     string
	SearchProvider        string
	URLCacheTTL           time.Duration
	URLCacheDir           string
	Timeout               time.Duration
	MaxBodyBytes          int64
	AllowPrivateNetworks  bool
	RenderTimeout         time.Duration
	RenderMaxConcurrency  int
	RenderMaxRequests     int
	RenderMaxNetworkBytes int64
	RenderMaxHTMLBytes    int64
	ChromePath            string
}

Config controls provider endpoints and HTTP safety limits.

type ContentProjection

type ContentProjection struct {
	Content       string
	Format        string
	Truncated     bool
	TruncatedBy   string
	TotalBytes    int
	OutputBytes   int
	TotalLines    int
	OutputLines   int
	StartLine     int
	NextStartLine int
}

ContentProjection describes a bounded view of fetched content.

func ProjectContent

func ProjectContent(content string, limits OutputLimits) ContentProjection

ProjectContent applies byte and line limits without changing content format.

func ProjectFetchContent

func ProjectFetchContent(content, format string, startLine int, limits OutputLimits) (ContentProjection, error)

ProjectFetchContent derives a requested representation, applies a zero-based line offset, and then enforces output limits.

type Document

type Document struct {
	URL         string
	FinalURL    string
	StatusCode  int
	ContentType string
	Title       string
	Description string
	Domain      string
	Favicon     string
	Image       string
	Language    string
	Published   string
	Author      string
	Site        string
	WordCount   int
	Extractor   string
	Source      string
	Rendered    bool
	Warnings    []string
	Content     string
}

Document is the normalized result of a URL fetch.

type FetchRequest

type FetchRequest struct {
	URL        string
	Raw        bool
	Reader     string
	Render     string
	RenderWait string
}

FetchRequest describes one URL fetch.

type HTTPError

type HTTPError struct {
	URL        string
	StatusCode int
	Status     string
	Body       string
}

HTTPError represents a bounded non-success HTTP response.

func (*HTTPError) Error

func (e *HTTPError) Error() string

type OutputLimits

type OutputLimits struct {
	MaxBytes int
	MaxLines int
}

OutputLimits bounds projected content. Zero means unlimited.

func NewOutputLimits

func NewOutputLimits(maxBytes, maxLines int) (OutputLimits, error)

NewOutputLimits validates optional byte and line limits.

type SearchRequest

type SearchRequest struct {
	Query              string
	Limit              int
	Category           string
	IncludeDomains     []string
	StartPublishedDate string
	IncludeHighlights  bool
	HighlightSentences int
}

SearchRequest describes one web search.

func ParseSearchRequest

func ParseSearchRequest(args map[string]any) (SearchRequest, error)

ParseSearchRequest validates the provider-neutral arguments used by the machine protocol and applies its larger result-limit default.

type SearchResponse

type SearchResponse struct {
	Query    string         `json:"query"`
	Provider string         `json:"provider,omitempty"`
	Results  []SearchResult `json:"results"`
}

SearchResponse is the normalized search result.

type SearchResult

type SearchResult struct {
	Title       string   `json:"title"`
	URL         string   `json:"url"`
	Description string   `json:"description,omitempty"`
	PublishedAt string   `json:"published_at,omitempty"`
	Highlights  []string `json:"highlights,omitempty"`
}

SearchResult is one search hit.

type Service

type Service struct {
	// contains filtered or unexported fields
}

Service owns the configured fetch, search, cache, and lazy renderer lifecycle.

func NewService

func NewService(cfg Config) *Service

NewService constructs a web service without performing network or browser work.

func (*Service) Close

func (s *Service) Close(ctx context.Context) error

Close releases the lazily initialized browser renderer, if any.

func (*Service) Fetch

func (s *Service) Fetch(ctx context.Context, req FetchRequest) (Document, error)

Fetch retrieves and normalizes one URL according to the requested reader and render policy.

func (*Service) Search

func (s *Service) Search(ctx context.Context, req SearchRequest) (SearchResponse, error)

Search executes one normalized query against the configured provider.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL