Documentation
¶
Index ¶
- func FetchWithBrowser(url string, timeout int, proxyProfile *config.ProxyProfile) (string, error)
- func NewHTTPClient(cfg *config.ScraperSettings, globalProxy *config.ProxyConfig, ...) (*resty.Client, *config.ProxyProfile, error)
- func NewHTTPClientWithDefaults(cfg *config.Config) (*resty.Client, *config.ProxyProfile, error)
- type DMMConfig
- func (c *DMMConfig) GetDownloadProxy() any
- func (c *DMMConfig) GetMaxRetries() int
- func (c *DMMConfig) GetProxy() any
- func (c *DMMConfig) GetRequestDelay() int
- func (c *DMMConfig) GetUserAgent() string
- func (c *DMMConfig) IsEnabled() bool
- func (c *DMMConfig) ValidateConfig(sc *config.ScraperSettings) error
- type JSONLDAggregateRating
- type JSONLDBrand
- type JSONLDOffer
- type JSONLDProduct
- type JSONLDVideoObject
- type Scraper
- func (s *Scraper) Close() error
- func (s *Scraper) Config() *config.ScraperSettings
- func (s *Scraper) GetURL(id string) (string, error)
- func (s *Scraper) IsEnabled() bool
- func (s *Scraper) Name() string
- func (s *Scraper) ResolveContentID(id string) (string, error)
- func (s *Scraper) ResolveDownloadProxyForHost(host string) (*config.ProxyConfig, *config.ProxyConfig, bool)
- func (s *Scraper) Search(id string) (*models.ScraperResult, error)
- func (s *Scraper) ValidateConfig(cfg *config.ScraperSettings) error
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func FetchWithBrowser ¶
FetchWithBrowser fetches a URL using Chrome browser automation with age verification cookies
func NewHTTPClient ¶
func NewHTTPClient(cfg *config.ScraperSettings, globalProxy *config.ProxyConfig, globalFlareSolverr config.FlareSolverrConfig) (*resty.Client, *config.ProxyProfile, error)
NewHTTPClient creates an HTTP client for the DMM scraper. HTTP-01: Per-scraper HTTP client ownership. Returns client, effective proxyProfile (for browser use), and error.
func NewHTTPClientWithDefaults ¶
NewHTTPClientWithDefaults creates an HTTP client using DMM-specific defaults. Deprecated: This function is kept for backward compatibility but is no longer used. Scraper.New() now constructs ScraperConfig from ScraperSettings directly.
Types ¶
type DMMConfig ¶
type DMMConfig struct {
Enabled bool `yaml:"enabled" json:"enabled"`
RequestDelay int `yaml:"request_delay" json:"request_delay"`
MaxRetries int `yaml:"max_retries" json:"max_retries"`
UserAgent string `yaml:"user_agent" json:"user_agent"`
Proxy *config.ProxyConfig `yaml:"proxy,omitempty" json:"proxy,omitempty"`
DownloadProxy *config.ProxyConfig `yaml:"download_proxy,omitempty" json:"download_proxy,omitempty"`
Priority int `yaml:"priority" json:"priority"` // Scraper's priority (higher = higher priority)
// NEW: Per-scraper browser and scrape_actress settings
UseBrowser bool `yaml:"use_browser" json:"use_browser"`
ScrapeActress bool `yaml:"scrape_actress" json:"scrape_actress"`
}
Config holds DMM/Fanza scraper configuration. YAML tags are defined here for unmarshaling via config.ScrapersConfig.
func (*DMMConfig) GetDownloadProxy ¶
GetDownloadProxy implements scraperutil.ScraperConfigInterface.
func (*DMMConfig) GetMaxRetries ¶
GetMaxRetries implements scraperutil.ScraperConfigInterface.
func (*DMMConfig) GetRequestDelay ¶
GetRequestDelay implements scraperutil.ScraperConfigInterface.
func (*DMMConfig) GetUserAgent ¶
GetUserAgent implements scraperutil.ScraperConfigInterface.
func (*DMMConfig) ValidateConfig ¶
func (c *DMMConfig) ValidateConfig(sc *config.ScraperSettings) error
ValidateConfig implements config.ConfigValidator for DMMConfig.
type JSONLDAggregateRating ¶
type JSONLDAggregateRating struct {
Type string `json:"@type"`
RatingValue float64 `json:"ratingValue"`
RatingCount int `json:"ratingCount"`
}
JSONLDAggregateRating represents the AggregateRating schema
type JSONLDBrand ¶
JSONLDBrand represents the Brand schema
type JSONLDOffer ¶
type JSONLDOffer struct {
Type string `json:"@type"`
Availability string `json:"availability"`
PriceCurrency string `json:"priceCurrency"`
Price float64 `json:"price"`
}
JSONLDOffer represents the Offer schema
type JSONLDProduct ¶
type JSONLDProduct struct {
Context string `json:"@context"`
Type string `json:"@type"`
Name string `json:"name"`
Description string `json:"description"`
Image interface{} `json:"image"` // Can be string or array
SKU string `json:"sku"`
Brand *JSONLDBrand `json:"brand"`
SubjectOf *JSONLDVideoObject `json:"subjectOf"`
Offers *JSONLDOffer `json:"offers"`
AggregateRating *JSONLDAggregateRating `json:"aggregateRating"`
}
JSONLDProduct represents the Product schema from JSON-LD
type JSONLDVideoObject ¶
type JSONLDVideoObject struct {
Type string `json:"@type"`
Name string `json:"name"`
Description string `json:"description"`
ContentURL string `json:"contentUrl"`
ThumbnailURL string `json:"thumbnailUrl"`
UploadDate string `json:"uploadDate"`
Genre []string `json:"genre"`
}
JSONLDVideoObject represents the VideoObject schema
type Scraper ¶
type Scraper struct {
// contains filtered or unexported fields
}
Scraper implements the DMM/Fanza scraper
func New ¶
func New(settings config.ScraperSettings, globalConfig *config.ScrapersConfig, contentIDRepo *database.ContentIDMappingRepository) *Scraper
New creates a new DMM scraper
func (*Scraper) Config ¶
func (s *Scraper) Config() *config.ScraperSettings
Config returns the scraper's configuration
func (*Scraper) ResolveContentID ¶
ResolveContentID attempts to resolve the display ID to an actual DMM content ID by first checking the cache, then scraping DMM search if needed
func (*Scraper) ResolveDownloadProxyForHost ¶
func (s *Scraper) ResolveDownloadProxyForHost(host string) (*config.ProxyConfig, *config.ProxyConfig, bool)
ResolveDownloadProxyForHost declares DMM-owned media hosts for downloader proxy routing.
func (*Scraper) Search ¶
func (s *Scraper) Search(id string) (*models.ScraperResult, error)
Search searches for and scrapes metadata for a given movie ID
func (*Scraper) ValidateConfig ¶
func (s *Scraper) ValidateConfig(cfg *config.ScraperSettings) error
ValidateConfig validates the scraper configuration. Returns error if config is invalid, nil if valid.
Source Files