scrape

type BrowserConn interface {
	// Fetch navigates to a URL and returns the rendered HTML. The context
	// bounds navigation and JS settling; cancellation unblocks the caller.
	Fetch(ctx context.Context, url string) (html string, err error)
	// Close releases browser resources.
	Close()
}

type FetchResult struct {
	Page        *Page
	RawHTML     string
	NotModified bool
	JSDetection string // "static", "likely_shell", "ambiguous"

	// Doc is the parsed document that ScrapeConditional used for JS-shell
	// detection. Downstream callers (e.g. link extraction during a crawl)
	// can reuse it to avoid re-parsing the same HTML. Nil when the page
	// was re-fetched via the browser — in that case RawHTML is the
	// rendered HTML and needs a fresh parse.
	Doc *goquery.Document

	// Source is the fetch path that produced Page (SourceHTTP or SourceBrowser).
	Source string
}

type Page struct {
	URL          string `json:"url"`
	FetchedURL   string `json:"fetched_url,omitempty"`
	Title        string `json:"title"`
	Markdown     string `json:"markdown"`
	ETag         string `json:"etag,omitempty"`
	LastModified string `json:"last_modified,omitempty"`
	ContentHash  string `json:"content_hash,omitempty"`
}

type PageCache interface {
	Get(url string) (*Page, string)
	Put(url string, page *Page, source string)
	GetRaw(url string) (rawHTML, source string, page *Page)
	PutRaw(url string, page *Page, source, rawHTML string)
}

type Scraper struct {
	// contains filtered or unexported fields
}