webfetch

package
v2026.909.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 9, 2026 License: MIT Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source
const (
	ExaAdapterName   = "exa"
	ExaFetchToolName = "web_fetch_exa"
)
View Source
const (
	MaxBodyBytes    = 5 * 1024 * 1024
	DefaultTimeoutS = 30
	MaxTimeoutSecs  = 120
	UserAgent       = defaultUserAgent
)
View Source
const (
	ParallelAdapterName   = "parallel"
	ParallelFetchToolName = "web_fetch"
)
View Source
const CloakAdapterName = "cloak"

Variables

View Source
var ErrNoContent = errors.New("fetch returned no content")

Functions

func ClassifyHTTPStatusForTest

func ClassifyHTTPStatusForTest(status int, body []byte) string

func DomainBlockedForTest

func DomainBlockedForTest(host string, rules []string) (bool, string)

func ResetSharedClientForTest

func ResetSharedClientForTest()

func RetryableFetchErrorForTest

func RetryableFetchErrorForTest(err error) bool

Types

type Attempt added in v2026.909.0

type Attempt struct {
	Backend    string `json:"backend"`
	Success    bool   `json:"success"`
	Error      string `json:"error,omitempty"`
	DurationMs int64  `json:"durationMs,omitempty"`
}

type ExaFetcher added in v2026.909.0

type ExaFetcher struct {
	// contains filtered or unexported fields
}

func NewExaFetcher added in v2026.909.0

func NewExaFetcher(caller MCPToolCaller, serverName string) (*ExaFetcher, error)

func (*ExaFetcher) Fetch added in v2026.909.0

func (f *ExaFetcher) Fetch(ctx context.Context, req Request) (Result, error)

type Fetcher added in v2026.909.0

type Fetcher interface {
	Fetch(context.Context, Request) (Result, error)
}

Fetcher is the runtime-owned web content fetch surface. FetchURL remains available as the legacy direct HTTP implementation during migration.

type MCPAdapterServer added in v2026.909.0

type MCPAdapterServer struct {
	ServerName string
	Adapter    string
}

type MCPToolCaller added in v2026.909.0

type MCPToolCaller interface {
	CallInternalTool(ctx context.Context, serverName, toolName, intent string, args map[string]any) (any, error)
}

MCPToolCaller is the narrow host-side surface needed by MCP-backed fetch adapters. The concrete MCP manager stays outside this package so adapters remain unit-testable without a live remote server.

type Metadata added in v2026.909.0

type Metadata struct {
	Provider     string         `json:"provider,omitempty"`
	Adapter      string         `json:"adapter,omitempty"`
	Fallback     bool           `json:"fallback,omitempty"`
	Partial      bool           `json:"partial,omitempty"`
	Attempts     []Attempt      `json:"attempts,omitempty"`
	ProviderData map[string]any `json:"providerData,omitempty"`
}

type NativeCloakFetcher added in v2026.909.0

type NativeCloakFetcher struct {
	// contains filtered or unexported fields
}

NativeCloakFetcher is the local browser fallback. CloakBrowser is used only to render the page; content extraction and Markdown conversion remain in Go alongside the legacy fetch implementation.

func NewNativeCloakFetcher added in v2026.909.0

func NewNativeCloakFetcher(browser cloak.Browser) (*NativeCloakFetcher, error)

func (*NativeCloakFetcher) Fetch added in v2026.909.0

func (f *NativeCloakFetcher) Fetch(ctx context.Context, req Request) (Result, error)

type ParallelFetcher added in v2026.909.0

type ParallelFetcher struct {
	// contains filtered or unexported fields
}

func NewParallelFetcher added in v2026.909.0

func NewParallelFetcher(caller MCPToolCaller, serverName string) (*ParallelFetcher, error)

func (*ParallelFetcher) Fetch added in v2026.909.0

func (f *ParallelFetcher) Fetch(ctx context.Context, req Request) (Result, error)

type Request added in v2026.909.0

type Request struct {
	URL            string
	TimeoutSeconds int
	Intent         string
	Extras         map[string]any
}

Request is the provider-neutral fetch request used by native web adapters. Intent is kept separate from Extras so every MCP call can use Solomon's standard intent-aware logging path.

type Result

type Result struct {
	URL         string
	Status      int
	ContentType string
	Markdown    string
	Title       string
	Metadata    *Metadata
}

func FetchURL

func FetchURL(ctx context.Context, rawURL string, timeoutSec int, cfg *config.Root) (Result, error)

type Router added in v2026.909.0

type Router struct {
	// contains filtered or unexported fields
}

func NewMCPRouter added in v2026.909.0

func NewMCPRouter(caller MCPToolCaller, servers []MCPAdapterServer, options RouterOptions) (*Router, error)

func NewRouter added in v2026.909.0

func NewRouter(adapters map[string]Fetcher, options RouterOptions) *Router

func (*Router) Fetch added in v2026.909.0

func (r *Router) Fetch(ctx context.Context, req Request) (Result, error)

type RouterError added in v2026.909.0

type RouterError struct {
	Attempts []Attempt
	Err      error
}

func (*RouterError) Error added in v2026.909.0

func (e *RouterError) Error() string

func (*RouterError) Unwrap added in v2026.909.0

func (e *RouterError) Unwrap() error

type RouterOptions added in v2026.909.0

type RouterOptions struct {
	Store    search.BalanceStore
	Now      func() time.Time
	Fallback Fetcher
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL