scrapers

type AnonymizedScraper struct {
	// contains filtered or unexported fields
}

type ArticleExtractor struct {
	// contains filtered or unexported fields
}

type ExtractResult struct {
	URL     string
	Chars   int
	Matched bool // the article selector matched an element
}

type FeedInspection struct {
	Diagnosis   models.FeedDiagnosis
	SampleLinks []string
	// SampleContentChars holds the plain-text length of each sampled entry's feed
	// content (content:encoded, falling back to description), aligned 1:1 with
	// SampleLinks. It lets a caller tell whether the feed already ships full bodies.
	SampleContentChars []int
	// SampleContent holds the plain text of each sampled entry's feed content, aligned
	// 1:1 with SampleLinks. A caller can match it against the real article page to
	// confirm the body is fully in the feed, not merely long (a teaser can be long too).
	SampleContent []string
	Title         string
}

type HTMLLinkScraper struct {
	// contains filtered or unexported fields
}

type LinkListCandidate struct {
	LinksSelector string   `json:"links_selector"`
	Count         int      `json:"count"`
	SampleHrefs   []string `json:"sample_hrefs"`
	DateSelector  string   `json:"date_selector,omitempty"`
	DateSample    string   `json:"date_sample,omitempty"`
	URLFilter     string   `json:"url_filter,omitempty"`
}

type RSSFeedScraper struct {
	// contains filtered or unexported fields
}

type RawResponse struct {
	URL         string
	FinalURL    string // after redirects
	Status      int
	ContentType string
	Body        []byte
	Duration    time.Duration
}

type Scraper interface {
	// Fetch returns the feed's items plus the raw HTTP response it parsed them
	// from, so the bytes stay inspectable (it is non-nil even when parsing fails,
	// and nil only when nothing was fetched, e.g. a network-level error).
	Fetch(url string, params map[string]any) ([]models.FeedItem, *RawResponse, error)
	ScrapeContent(url string, params map[string]any) (string, error)
}

type SelectorCandidate struct {
	Selector    string  `json:"selector"`
	Chars       int     `json:"chars"`        // trimmed text length
	LinkDensity float64 `json:"link_density"` // anchor text ÷ total text (0..1); high = nav/menu
	Snippet     string  `json:"snippet"`      // first printable chars of the element's text
}

type SelectorScore struct {
	Samples   int
	Usable    int     // count of results that passed Usable
	MeanChars int     // mean extracted length over usable results
	Score     float64 // 0..1: usable ratio penalized by length variance
	Reliable  bool    // Score high enough to recommend without hesitation
}

type TwitterScraper struct {
	// contains filtered or unexported fields
}