scrape

type Config struct {
	ID       ConfigID
	Writer   output.WriterConfig `yaml:"writer,omitempty"`
	Scrapers []Scraper           `yaml:"scrapers,omitempty"`
	Global   GlobalConfig        `yaml:"global,omitempty"`
	Records  output.Records
}

type ConfigID struct {
	Slug  string
	ID    string
	Field string
	SubID string
}

type DateComponent struct {
	Covers          date.CoveredDateParts `yaml:"covers"`
	ElementLocation ElementLocation       `yaml:"location"`
	Layout          []string              `yaml:"layout"`
	Transform       []TransformConfig     `yaml:"transform,omitempty"`
}

type DerivedField struct {
	Source   string          `yaml:"source"`   // source field name
	Template string          `yaml:"template"` // "{name} | {date}" - delimiter-based
	Regex    string          `yaml:"regex"`    // fallback: named capture groups (?P<name>...)
	Outputs  []DerivedOutput `yaml:"outputs"`
	// contains filtered or unexported fields
}

type DerivedOutput struct {
	Symbol    string           `yaml:"symbol"`
	Target    string           `yaml:"target"`
	Condition *OutputCondition `yaml:"condition,omitempty"`
	Value     string           `yaml:"value,omitempty"` // override extracted value
}

type ElementLocation struct {
	Selector      string      `yaml:"selector,omitempty"`
	JsonSelector  string      `yaml:"json_selector,omitempty"`
	ChildIndex    int         `yaml:"child_index,omitempty"`
	RegexExtract  RegexConfig `yaml:"regex_extract,omitempty"`
	Attr          string      `yaml:"attr,omitempty"`
	MaxLength     int         `yaml:"max_length,omitempty"`
	EntireSubtree bool        `yaml:"entire_subtree,omitempty"`
	AllNodes      bool        `yaml:"all_nodes,omitempty"`
	Separator     string      `yaml:"separator,omitempty"`      // Intra-node sibling separator (default: \x1F)
	NodeSeparator string      `yaml:"node_separator,omitempty"` // Inter-node separator (default: \x1E)
}

type ElementLocations []ElementLocation

type FetchConfig struct {
	UseJavascript          bool   `yaml:"use_javascript,omitempty"`           // Enable headless browser
	WaitSelector           string `yaml:"wait_selector,omitempty"`            // CSS selector to wait for
	WaitTimeoutMs          int    `yaml:"wait_timeout_ms,omitempty"`          // Timeout for wait (default 30000)
	FerretQL               string `yaml:"ferret_ql,omitempty"`                // Custom FerretQL script (advanced)
	Script                 string `yaml:"script,omitempty"`                   // JavaScript to run after page load (Rod only, runs before scraping)
	InfiniteScrollSelector string `yaml:"infinite_scroll_selector,omitempty"` // CSS selector for "Load More" button (Rod clicks it repeatedly)
}

type Field struct {
	Name             string           `yaml:"name"`
	Value            string           `yaml:"value,omitempty"`
	Type             string           `yaml:"type,omitempty"`     // can be text (default), html, url, or date_time_tz_ranges
	ElementLocations ElementLocations `yaml:"location,omitempty"` // elements are extracted strings joined with newlines
	Default          string           `yaml:"default,omitempty"`  // the default for a dynamic field (text or url) if no value is found
	// If a field can be found on a detail page the following variable has to
	// contain a field name of a field of type 'url' that is located on the main
	// page.
	OnDetailPage string            `yaml:"on_detail_page,omitempty"` // applies to text, url, date
	Required     bool              `yaml:"required,omitempty"`       // applies to text, url - if true, skip record when field is empty
	Components   []DateComponent   `yaml:"components,omitempty"`     // applies to date
	DateLocation string            `yaml:"date_location,omitempty"`  // applies to date
	DateLanguage string            `yaml:"date_language,omitempty"`  // applies to date
	Hide         bool              `yaml:"hide,omitempty"`           // applies to text, url, date
	GuessYear    bool              `yaml:"guess_year,omitempty"`     // applies to date
	Transform    []TransformConfig `yaml:"transform,omitempty"`      // applies to text
}

type FieldNameComponents struct {
	Hash          string // 8-char hex hash
	Attribute     string // Attribute name, or empty for text content
	TextNodeIndex int    // 0-based text node index
}

type Filter struct {
	Field           string `yaml:"field"`
	Type            string
	Expression      string `yaml:"exp"` // changed from 'regex' to 'exp' in version 0.5.7
	RegexComp       *regexp.Regexp
	DateComp        time.Time
	DateOp          string
	Match           bool   `yaml:"match"`
	Condition       string `yaml:"condition,omitempty"`        // matches, not_matches, missing, missing_or_matches, exists
	CaseInsensitive bool   `yaml:"case_insensitive,omitempty"` // for regex matching
}

type GlobalConfig struct {
	UserAgent string `yaml:"user-agent"`
}

type OutputCondition struct {
	Equals          string `yaml:"equals,omitempty"`
	NotEquals       string `yaml:"not_equals,omitempty"`
	Matches         string `yaml:"matches,omitempty"`
	NotMatches      string `yaml:"not_matches,omitempty"`
	CaseInsensitive bool   `yaml:"case_insensitive,omitempty"`
	// contains filtered or unexported fields
}

type Pagination struct {
	Type           string `yaml:"type"`            // query_param, scroll, next_button
	ParamName      string `yaml:"param_name"`      // for query_param: e.g. "start", "page"
	StartValue     int    `yaml:"start_value"`     // starting value (usually 0)
	Increment      int    `yaml:"increment"`       // increment per page
	MaxPages       int    `yaml:"max_pages"`       // safety limit
	ButtonSelector string `yaml:"button_selector"` // for scroll/next_button types
	WaitMs         int    `yaml:"wait_ms"`         // delay between pages (milliseconds)
}

type Paginator struct {
	Location ElementLocation `yaml:"location,omitempty"`
	MaxPages int             `yaml:"max_pages,omitempty"`
}

type RegexConfig struct {
	RegexPattern string `yaml:"exp"`
	Index        int    `yaml:"index"`
}

type Scraper struct {
	Interaction  []*fetch.Interaction `yaml:"interaction,omitempty"`
	Name         string               `yaml:"name"`
	PageLoadWait int                  `yaml:"page_load_wait,omitempty"` // milliseconds. Only taken into account when render_js = true
	RenderJs     bool                 `yaml:"render_js,omitempty"`
	Selector     string               `yaml:"selector"`
	Strategy     string               `yaml:"strategy,omitempty"` // "nested" (default) or "sequential"
	URL          string               `yaml:"url"`
	Validation   *ValidationConfig    `yaml:"validation,omitempty"`
	Fields       []Field              `yaml:"fields,omitempty"`
	Filters      []*Filter            `yaml:"filters,omitempty"`
	Paginators   []Paginator          `yaml:"paginators,omitempty"`

	// New declarative config fields
	Fetch         *FetchConfig   `yaml:"fetch,omitempty"`          // Fetch configuration
	Pagination    *Pagination    `yaml:"pagination,omitempty"`     // Declarative pagination
	DerivedFields []DerivedField `yaml:"derived_fields,omitempty"` // Template-based field derivation

	// MergeKey makes this an independent scraper (not a detail-page follower).
	// When set, this scraper is run separately via Page() and its records are
	// merged into the primary scraper's records by matching on this field name.
	// The field must exist in both scrapers' output.
	MergeKey string `yaml:"merge_key,omitempty"`
}

type TransformConfig struct {
	TransformType string `yaml:"type,omitempty"`    // only regex-replace for now
	RegexPattern  string `yaml:"regex,omitempty"`   // a container for the pattern
	Replacement   string `yaml:"replace,omitempty"` // a plain string for replacement
}

type ValidationConfig struct {
	RequiresCTASelector string `yaml:"requires_cta_selector,omitempty"`
}