extract

type Config struct {
	Enabled              bool          `json:"enabled" mapstructure:"enabled"`
	DefaultMode          string        `json:"default_mode" mapstructure:"default_mode"`
	Timeout              time.Duration `json:"timeout" mapstructure:"timeout"`
	MaxBytes             int           `json:"max_bytes" mapstructure:"max_bytes"`
	MaxConcurrent        int           `json:"max_concurrent" mapstructure:"max_concurrent"`
	AllowPrivateNetworks bool          `json:"allow_private_networks" mapstructure:"allow_private_networks"`
}

type ExtractMeta struct {
	ModeUsed  string `json:"mode_used"`
	FetchedAt string `json:"fetched_at"`
	Bytes     int    `json:"bytes"`
	TookMs    int64  `json:"took_ms"`
}

type ExtractRequest struct {
	URL      string
	Mode     Mode
	ProxyURL string
	LangCode string
	Timeout  time.Duration
	MaxBytes int
	// FullPage selects whole-readable-body extraction instead of the default
	// article-only (trafilatura) extraction. LLM agents fetching arbitrary URLs
	// often want the full page; FullPage keeps nav/feature/landing content that
	// trafilatura strips. The zero value (false) preserves the cleaned default.
	FullPage bool
	// UseLLMSTxt, when set and the URL is a site root, probes /llms-full.txt then
	// /llms.txt and returns that LLM-optimized markdown instead of scraping HTML.
	UseLLMSTxt bool
	// MinRunes is the per-request auto-mode escalation floor: raw output below
	// this many extracted-text runes escalates to a render. 0 uses defaultMinRunes.
	MinRunes int
}

type ExtractResult struct {
	URL         string            `json:"url"`
	Title       string            `json:"title"`
	Description string            `json:"description"`
	Markdown    string            `json:"markdown"`
	Text        string            `json:"text"`
	Headings    []Heading         `json:"headings,omitempty"`
	Links       []Link            `json:"links,omitempty"`
	Canonical   string            `json:"canonical,omitempty"`
	Lang        string            `json:"lang,omitempty"`
	SchemaOrg   []json.RawMessage `json:"schema_org,omitempty"`
	OGTags      map[string]string `json:"og_tags,omitempty"`
	Meta        ExtractMeta       `json:"meta"`
}

type Extractor struct {
	RawFetch      RawFetcher
	RenderedFetch RenderedFetcher
	Cfg           Config
}

type FetchResponse struct {
	StatusCode int
	Body       []byte
}

type Heading struct {
	Level int    `json:"level"`
	Text  string `json:"text"`
}

type Link struct {
	Text string `json:"text"`
	URL  string `json:"url"`
}

type Mode string

type RawFetcher func(ctx context.Context, req ExtractRequest) (*FetchResponse, error)

type RenderedFetcher func(ctx context.Context, req ExtractRequest) (*FetchResponse, error)