extractors

type BasicGenericExtractor struct {
	// contains filtered or unexported fields
}

type BloggerExtractor struct{}

type CacheEntry struct {
	Extractor   *custom.CustomExtractor
	LoadTime    time.Time
	AccessTime  time.Time
	AccessCount int64
	Element     *list.Element // For LRU tracking
}

type CollectAllPagesOptions struct {
	NextPageURL   string
	HTML          string
	Doc           *goquery.Document
	MetaCache     map[string]interface{}
	Result        map[string]interface{}
	Extractor     interface{}
	Title         interface{}
	URL           string
	Resource      ResourceInterface
	RootExtractor *RootExtractorInterface
}

type ContentExtractor struct {
	Selectors        parser.SelectorList      `json:"-"`                   // Type-safe selectors
	SelectorsLegacy  []interface{}            `json:"selectors,omitempty"` // Deprecated: JSON compatibility
	AllowMultiple    bool                     `json:"allowMultiple,omitempty"`
	DefaultCleaner   bool                     `json:"defaultCleaner"`
	Clean            []string                 `json:"clean,omitempty"`      // Selectors to remove
	Transforms       parser.TransformRegistry `json:"-"`                    // Type-safe transforms
	TransformsLegacy map[string]interface{}   `json:"transforms,omitempty"` // Deprecated: JSON compatibility
}

type CustomExtractorAdapter struct {
	// contains filtered or unexported fields
}

type DetectByHTMLFunc func(*goquery.Document) Extractor

type ExtractOptions struct {
	Doc            *goquery.Document
	URL            string
	Extractor      interface{}
	ContentOnly    bool
	ExtractedTitle interface{}
	Fallback       bool
}

type ExtractOpts struct {
	Doc            *goquery.Document
	URL            string
	Extractor      interface{}
	ContentOnly    bool
	ExtractedTitle interface{}
	Fallback       bool
}

type Extractor = parser.Extractor

type ExtractorCandidate struct {
	Extractor Extractor
	Priority  int    // Lower number = higher priority
	Source    string // "api_hostname", "api_domain", "static_hostname", etc.
	Key       string // The key used to find this extractor
}

type ExtractorChecker interface {
	CanHandle(doc *goquery.Document, url string) (bool, error)
	GetPriority() int
	GetSource() string
}

type ExtractorError struct {
	Error   bool   `json:"error"`
	Message string `json:"message"`
}

type ExtractorLoader struct {
	// contains filtered or unexported fields
}

type ExtractorOptions struct {
	StripUnlikelyCandidates bool
	WeightNodes             bool
	CleanConditionally      bool
	URL                     string
	Content                 string
	Title                   string
}

type ExtractorStats struct {
	TotalCandidates   int
	CheckedInParallel int
	FastestMatch      time.Duration
	SlowestMatch      time.Duration
	AverageCheckTime  time.Duration
	WinningExtractor  string
	WinningPriority   int
}

type FieldExtractor struct {
	Selectors       parser.SelectorList `json:"-"`                   // Type-safe selectors
	SelectorsLegacy []interface{}       `json:"selectors,omitempty"` // Deprecated: JSON compatibility
	AllowMultiple   bool                `json:"allowMultiple,omitempty"`
	DefaultCleaner  bool                `json:"defaultCleaner"` // defaults to true in JavaScript
}

type FullExtractor struct {
	Domain           string   `json:"domain"`
	SupportedDomains []string `json:"supportedDomains,omitempty"`

	// Field extractors for standard fields
	Title         *FieldExtractor   `json:"title,omitempty"`
	Author        *FieldExtractor   `json:"author,omitempty"`
	Content       *ContentExtractor `json:"content,omitempty"`
	DatePublished *FieldExtractor   `json:"date_published,omitempty"`
	LeadImageURL  *FieldExtractor   `json:"lead_image_url,omitempty"`
	Dek           *FieldExtractor   `json:"dek,omitempty"`
	NextPageURL   *FieldExtractor   `json:"next_page_url,omitempty"`
	Excerpt       *FieldExtractor   `json:"excerpt,omitempty"`
	WordCount     *FieldExtractor   `json:"word_count,omitempty"`
	Direction     *FieldExtractor   `json:"direction,omitempty"`
	URL           *FieldExtractor   `json:"url,omitempty"`

	// Extended types for custom fields
	Extend map[string]*FieldExtractor `json:"extend,omitempty"`
}

type LoaderConfig struct {
	MaxCacheSize       int           // Maximum number of cached extractors
	CacheExpiration    time.Duration // How long to keep extractors in cache
	PreloadCommonSites bool          // Whether to preload popular extractors
	EnableMetrics      bool          // Whether to track performance metrics
	MaxLoadAttempts    int           // Maximum attempts to load an extractor
	LoadTimeout        time.Duration // Timeout for loading operations
}

type LoaderMetrics struct {
	CacheHits       int64
	CacheMisses     int64
	LoadSuccesses   int64
	LoadFailures    int64
	TotalLoadTime   time.Duration
	AverageLoadTime time.Duration
	EvictionCount   int64
}

type MediumExtractor struct{}

type ParallelExtractorConfig struct {
	MaxConcurrentChecks int           // Maximum number of goroutines
	CheckTimeout        time.Duration // Timeout for individual extractor checks
	EnableStats         bool          // Whether to collect detailed statistics
	EnableCaching       bool          // Whether to cache extractor decisions
}

type ParallelExtractorResult struct {
	Candidate *ExtractorCandidate
	Error     error
	Duration  time.Duration
}

type ResourceInterface interface {
	Create(url string, preparedResponse string, parsedURL interface{}, headers map[string]string) (*goquery.Document, error)
}

type RootExtractorInterface struct{}

type SelectOptions struct {
	Doc            *goquery.Document
	Type           string
	ExtractionOpts interface{}
	ExtractHTML    bool
	URL            string
}

type SelectOpts struct {
	Doc            *goquery.Document
	Type           string
	ExtractionOpts interface{}
	ExtractHTML    bool
	URL            string
}

type SelectorEntry struct {
	Selector        string
	Attribute       string
	TransformFunc   func(string) string
	IsMultiSelector bool
	IsAttribute     bool
}

type SimpleExtractor struct {
	Domain string
}

type SimpleRootExtractor struct{}

type TransformFunc func(*goquery.Selection, *goquery.Document) interface{}