types

type FetchError struct {
	URL        string
	StatusCode int
	Err        error
	Retryable  bool
	RetryAfter time.Duration // populated from Retry-After header on HTTP 429
}

type Item struct {
	// Fields stores the extracted key-value data.
	Fields map[string]any

	// URL is the source page URL this item was extracted from.
	URL string

	// SpiderName identifies which spider produced this item.
	SpiderName string

	// Timestamp is when this item was created.
	Timestamp time.Time

	// Depth is the crawl depth at which this item was found.
	Depth int

	// Checksum is a hash of the item content for deduplication.
	Checksum string
}

type ParseError struct {
	URL      string
	Selector string
	Err      error
}

type PipelineError struct {
	Stage string
	Item  *Item
	Err   error
}

type Request struct {
	// URL is the target URL to fetch.
	URL *url.URL

	// Method is the HTTP method (GET, POST, etc.). Defaults to GET.
	Method string

	// Headers are custom HTTP headers to send with the request.
	Headers http.Header

	// Body is the request body for POST/PUT requests.
	Body []byte

	// Depth is the crawl depth from the seed URL.
	Depth int

	// Priority controls scheduling order (lower = higher priority).
	Priority int

	// MaxRetries is the maximum number of retries for this request.
	MaxRetries int

	// RetryCount tracks the current retry attempt.
	RetryCount int

	// Timeout overrides the global request timeout for this request.
	Timeout time.Duration

	// Meta stores arbitrary metadata attached to this request.
	Meta map[string]any

	// Tag categorizes this request (e.g., "listing", "detail", "pagination").
	Tag string

	// FetcherType specifies which fetcher to use: "http" or "browser".
	FetcherType string

	// Callbacks are the names of callback functions to invoke on response.
	Callbacks []string

	// ParentURL tracks which page this request was discovered on.
	ParentURL string

	// CreatedAt is when this request was created.
	CreatedAt time.Time

	// ID is a unique identifier for this request.
	ID string
}

type Response struct {
	// StatusCode is the HTTP status code.
	StatusCode int

	// Headers are the response HTTP headers.
	Headers http.Header

	// Body is the raw response body bytes.
	Body []byte

	// Request is a reference to the original request.
	Request *Request

	// ContentType is the MIME type of the response.
	ContentType string

	// ContentLength is the size of the response body in bytes.
	ContentLength int64

	// FinalURL is the URL after any redirects.
	FinalURL string

	// Doc is a parsed goquery document (lazily loaded).
	Doc *goquery.Document

	// FetchDuration is how long the fetch took.
	FetchDuration time.Duration

	// FetchedAt is when this response was received.
	FetchedAt time.Time

	// Meta stores arbitrary metadata.
	Meta map[string]any
}

type StorageError struct {
	Backend string
	Err     error
}