web

type AdapterCreatorFunc func(scraperConfig types.WebScraperConfig) (WebAdapter, error)

type AdapterFactory func() (WebAdapter, error)

type DefaultWebStrategy struct {
	// contains filtered or unexported fields
}

type Format string

type Image struct {
	URL   string `json:"url"`
	Title string `json:"title,omitempty"`
}

type Link struct {
	URL  string `json:"url"`
	Text string `json:"text"`
}

type ScrapeOptions struct {
	// Basic options
	Format        Format            `json:"format,omitempty"`         // Scraping format enumeration
	Timeout       int               `json:"timeout,omitempty"`        // Timeout in seconds
	Cookies       map[string]string `json:"cookies,omitempty"`        // Cookies to use for the request
	ImagesSummary bool              `json:"images_summary,omitempty"` // Whether to include a summary of unique images at the end
	LinksSummary  bool              `json:"links_summary,omitempty"`  // Whether to include a summary of unique links at the end
}

type WebAdapter interface {
	// Scrape is the core function for scraping a single page.
	Scrape(ctx context.Context, url string, options *ScrapeOptions) (*WebContent, error)

	// ScrapeMultiple scrapes multiple pages concurrently.
	ScrapeMultiple(ctx context.Context, urls []string, options *ScrapeOptions) ([]*WebContent, error)
}

type WebContent struct {
	// Basic information
	URL     string `json:"url"`
	Title   string `json:"title"`
	Content string `json:"content"` // Main content

	// Extracted structured data
	Links  []Link  `json:"links,omitempty"`
	Images []Image `json:"images,omitempty"`
}

type WebRegistry struct {
	// contains filtered or unexported fields
}

type WebScraper string

type WebStrategy interface {
	// Execute executes the web scraping strategy for a single URL.
	Execute(ctx context.Context, url string, options *ScrapeOptions) (*WebContent, error)

	// ExecuteMultiple executes the web scraping strategy for multiple URLs.
	ExecuteMultiple(ctx context.Context, urls []string, options *ScrapeOptions) ([]*WebContent, error)
}

type WebStrategyConfig struct {
	// DefaultScraper is the default scraper to use if no list is specified.
	DefaultScraper WebScraper `json:"default_scraper,omitempty"`

	// DefaultFallbackOrder is the default order of scrapers to try.
	DefaultFallbackOrder []WebScraper `json:"default_fallback_order,omitempty"`

	// EnableFallback determines whether to try the next scraper on failure.
	EnableFallback bool `json:"enable_fallback"`

	// FailFast determines whether to stop immediately after the first failure.
	FailFast bool `json:"fail_fast"`
}