Documentation
¶
Index ¶
- Constants
- Variables
- func ContentLineCount(content string) int
- func NewCodedError(err error, code, suggestion string) error
- func NormalizeProjectionFormat(format string) string
- func ProjectionFormatIsMarkdown(format string) bool
- func TruncationMarker(projection ContentProjection) string
- type Client
- type ClientConfig
- type CodedError
- type Config
- type ContentProjection
- type Document
- type FetchRequest
- type HTTPError
- type OutputLimits
- type SearchRequest
- type SearchResponse
- type SearchResult
- type Service
Constants ¶
const ( ErrorCodeInvalidURL = "invalid_url" ErrorCodePrivateNetwork = "private_network_blocked" ErrorCodeUnsupportedReader = "unsupported_reader" ErrorCodeNotHTML = "not_html" ErrorCodeResponseTooLarge = "response_too_large" ErrorCodeUpstreamHTTP = "upstream_http" ErrorCodeAuthentication = "authentication_failed" ErrorCodeRateLimited = "rate_limited" ErrorCodeMissingAPIKey = "missing_api_key" ErrorCodeRequestTimeout = "request_timeout" ErrorCodeCanceled = "canceled" ErrorCodeInvalidArgument = "invalid_argument" ErrorCodeUnsupportedSearch = "unsupported_search_option" ErrorCodeInvalidJSON = "invalid_json" ErrorCodeNoInput = "no_input" ErrorCodeProviderResponse = "provider_response_invalid" ErrorCodeExtraction = "extraction_failed" ErrorCodeReaderFallback = "reader_fallback_failed" ErrorCodeRenderFailed = "render_failed" ErrorCodeRenderBudget = "render_budget_exceeded" )
const ( ReaderModeJina = "jina" ReaderModeDefuddle = "defuddle" ReaderModeAuto = "auto" )
const ( RenderModeNever = "never" RenderModeAuto = "auto" RenderModeAlways = "always" RenderWaitLoad = "load" RenderWaitNetworkIdle = "networkidle" )
const ( DefaultTimeout = 30 * time.Second DefaultMaxBodyBytes = 8 << 20 DefaultRenderTimeout = 30 * time.Second DefaultRenderMaxConcurrency = 2 DefaultRenderMaxRequests = 100 DefaultRenderMaxNetworkBytes = 32 << 20 DefaultSearchLimit = 5 MaxSearchLimit = 50 DefaultReaderEndpoint = "https://r.jina.ai" DefaultSearchEndpoint = "https://api.search.brave.com/res/v1/web/search" DefaultExaEndpoint = "https://api.exa.ai/search" DefaultUserAgent = "webfetch/0.1" )
const ProjectionFormatMarkdown = "markdown"
ProjectionFormatMarkdown selects the default content projection.
Variables ¶
var ( // ErrNotHTML indicates that an HTML reader received a non-HTML response. ErrNotHTML = errors.New("response is not HTML") // ErrTooLarge indicates that a response exceeded the configured body limit. ErrTooLarge = errors.New("response body exceeds configured limit") // ErrUnsupportedReader indicates that a fetch request selected an unknown reader. ErrUnsupportedReader = errors.New("unsupported reader") )
Functions ¶
func ContentLineCount ¶
ContentLineCount counts logical lines without treating a trailing newline as an additional empty line.
func NewCodedError ¶
NewCodedError wraps err with machine-readable recovery metadata.
func NormalizeProjectionFormat ¶
NormalizeProjectionFormat trims and lowercases a projection name.
func ProjectionFormatIsMarkdown ¶
ProjectionFormatIsMarkdown reports whether format selects the default view.
func TruncationMarker ¶
func TruncationMarker(projection ContentProjection) string
TruncationMarker renders the stable human-readable output budget marker.
Types ¶
type Client ¶
type Client struct {
// contains filtered or unexported fields
}
Client performs bounded HTTP requests with redirect and network safety checks.
func NewClient ¶
func NewClient(cfg ClientConfig) *Client
NewClient constructs a bounded HTTP client from explicit configuration.
type ClientConfig ¶
type ClientConfig struct {
Timeout time.Duration
MaxBodyBytes int64
AllowPrivateNetworks bool
UserAgent string
}
ClientConfig controls the shared HTTP client.
type CodedError ¶
CodedError preserves the original error while exposing stable, actionable metadata to machine-readable callers.
func (*CodedError) Error ¶
func (e *CodedError) Error() string
func (*CodedError) Unwrap ¶
func (e *CodedError) Unwrap() error
type Config ¶
type Config struct {
// UserAgent identifies the owning application for direct and rendered HTTP requests.
UserAgent string
JinaAPIKey string
BraveAPIKey string
ExaAPIKey string
ReaderMode string
ReaderEndpoint string
SearchEndpoint string
ExaSearchEndpoint string
SearchProvider string
URLCacheTTL time.Duration
URLCacheDir string
Timeout time.Duration
MaxBodyBytes int64
AllowPrivateNetworks bool
RenderTimeout time.Duration
RenderMaxConcurrency int
RenderMaxRequests int
RenderMaxNetworkBytes int64
RenderMaxHTMLBytes int64
ChromePath string
}
Config controls provider endpoints and HTTP safety limits.
type ContentProjection ¶
type ContentProjection struct {
Content string
Format string
Truncated bool
TruncatedBy string
TotalBytes int
OutputBytes int
TotalLines int
OutputLines int
StartLine int
NextStartLine int
}
ContentProjection describes a bounded view of fetched content.
func ProjectContent ¶
func ProjectContent(content string, limits OutputLimits) ContentProjection
ProjectContent applies byte and line limits without changing content format.
func ProjectFetchContent ¶
func ProjectFetchContent(content, format string, startLine int, limits OutputLimits) (ContentProjection, error)
ProjectFetchContent derives a requested representation, applies a zero-based line offset, and then enforces output limits.
type Document ¶
type Document struct {
URL string
FinalURL string
StatusCode int
ContentType string
Title string
Description string
Domain string
Favicon string
Image string
Language string
Published string
Author string
Site string
WordCount int
Extractor string
Source string
Rendered bool
Warnings []string
Content string
}
Document is the normalized result of a URL fetch.
type FetchRequest ¶
FetchRequest describes one URL fetch.
type OutputLimits ¶
OutputLimits bounds projected content. Zero means unlimited.
func NewOutputLimits ¶
func NewOutputLimits(maxBytes, maxLines int) (OutputLimits, error)
NewOutputLimits validates optional byte and line limits.
type SearchRequest ¶
type SearchRequest struct {
Query string
Limit int
Category string
IncludeDomains []string
StartPublishedDate string
IncludeHighlights bool
HighlightSentences int
}
SearchRequest describes one web search.
func ParseSearchRequest ¶
func ParseSearchRequest(args map[string]any) (SearchRequest, error)
ParseSearchRequest validates the provider-neutral arguments used by the machine protocol and applies its larger result-limit default.
type SearchResponse ¶
type SearchResponse struct {
Query string `json:"query"`
Provider string `json:"provider,omitempty"`
Results []SearchResult `json:"results"`
}
SearchResponse is the normalized search result.
type SearchResult ¶
type SearchResult struct {
Title string `json:"title"`
URL string `json:"url"`
Description string `json:"description,omitempty"`
PublishedAt string `json:"published_at,omitempty"`
Highlights []string `json:"highlights,omitempty"`
}
SearchResult is one search hit.
type Service ¶
type Service struct {
// contains filtered or unexported fields
}
Service owns the configured fetch, search, cache, and lazy renderer lifecycle.
func NewService ¶
NewService constructs a web service without performing network or browser work.
func (*Service) Fetch ¶
Fetch retrieves and normalizes one URL according to the requested reader and render policy.
func (*Service) Search ¶
func (s *Service) Search(ctx context.Context, req SearchRequest) (SearchResponse, error)
Search executes one normalized query against the configured provider.