Documentation
¶
Index ¶
- func NewHTTPClient(cfg *config.ScraperSettings, globalProxy *config.ProxyConfig, ...) (*resty.Client, error)
- type R18DevConfig
- func (c *R18DevConfig) GetDownloadProxy() any
- func (c *R18DevConfig) GetMaxRetries() int
- func (c *R18DevConfig) GetProxy() any
- func (c *R18DevConfig) GetRequestDelay() int
- func (c *R18DevConfig) GetUserAgent() string
- func (c *R18DevConfig) IsEnabled() bool
- func (c *R18DevConfig) ValidateConfig(sc *config.ScraperSettings) error
- type R18Response
- type Scraper
- func (s *Scraper) CanHandleURL(rawURL string) bool
- func (s *Scraper) Close() error
- func (s *Scraper) Config() *config.ScraperSettings
- func (s *Scraper) ExtractIDFromURL(urlStr string) (string, error)
- func (s *Scraper) GetURL(id string) (string, error)
- func (s *Scraper) IsEnabled() bool
- func (s *Scraper) Name() string
- func (s *Scraper) ResolveDownloadProxyForHost(host string) (*config.ProxyConfig, *config.ProxyConfig, bool)
- func (s *Scraper) ScrapeURL(ctx context.Context, urlStr string) (*models.ScraperResult, error)
- func (s *Scraper) Search(ctx context.Context, id string) (*models.ScraperResult, error)
- func (s *Scraper) ValidateConfig(cfg *config.ScraperSettings) error
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func NewHTTPClient ¶
func NewHTTPClient(cfg *config.ScraperSettings, globalProxy *config.ProxyConfig, globalFlareSolverr config.FlareSolverrConfig) (*resty.Client, error)
NewHTTPClient creates a new HTTP client for the R18.dev scraper. HTTP-01: Per-scraper HTTP client ownership. Uses builder pattern for consistent HTTP client construction.
Types ¶
type R18DevConfig ¶
type R18DevConfig struct {
Enabled bool `yaml:"enabled" json:"enabled"`
Language string `yaml:"language" json:"language"` // Language code: en, ja (default: en)
RequestDelay int `yaml:"request_delay" json:"request_delay"` // Delay between requests in milliseconds (0 = no delay)
MaxRetries int `yaml:"max_retries" json:"max_retries"` // Maximum number of retry attempts for rate-limited requests
RespectRetryAfter bool `yaml:"respect_retry_after" json:"respect_retry_after"` // Whether to respect Retry-After header from server
UserAgent string `yaml:"user_agent" json:"user_agent"` // Custom User-Agent for this scraper
Proxy *config.ProxyConfig `yaml:"proxy,omitempty" json:"proxy,omitempty"` // Optional scraper-specific proxy override
DownloadProxy *config.ProxyConfig `yaml:"download_proxy,omitempty" json:"download_proxy,omitempty"` // Optional scraper-specific download proxy override
Priority int `yaml:"priority" json:"priority"` // Scraper's priority (higher = higher priority)
FlareSolverr config.FlareSolverrConfig `yaml:"flaresolverr" json:"flaresolverr"` // FlareSolverr config for Cloudflare bypass
}
R18DevConfig holds R18.dev scraper configuration. YAML tags are defined here for unmarshaling via config.ScrapersConfig.
func (*R18DevConfig) GetDownloadProxy ¶
func (c *R18DevConfig) GetDownloadProxy() any
GetDownloadProxy implements scraperutil.ScraperConfigInterface.
func (*R18DevConfig) GetMaxRetries ¶
func (c *R18DevConfig) GetMaxRetries() int
GetMaxRetries implements scraperutil.ScraperConfigInterface.
func (*R18DevConfig) GetProxy ¶
func (c *R18DevConfig) GetProxy() any
GetProxy implements scraperutil.ScraperConfigInterface.
func (*R18DevConfig) GetRequestDelay ¶
func (c *R18DevConfig) GetRequestDelay() int
GetRequestDelay implements scraperutil.ScraperConfigInterface.
func (*R18DevConfig) GetUserAgent ¶
func (c *R18DevConfig) GetUserAgent() string
GetUserAgent implements scraperutil.ScraperConfigInterface.
func (*R18DevConfig) IsEnabled ¶
func (c *R18DevConfig) IsEnabled() bool
IsEnabled implements scraperutil.ScraperConfigInterface.
func (*R18DevConfig) ValidateConfig ¶
func (c *R18DevConfig) ValidateConfig(sc *config.ScraperSettings) error
ValidateConfig implements config.ConfigValidator for R18DevConfig.
type R18Response ¶
type R18Response struct {
DVDID string `json:"dvd_id"`
ContentID string `json:"content_id"`
TitleJA string `json:"title_ja"` // Japanese title
TitleEn string `json:"title_en"` // English title (may be null)
Description string `json:"description"` // Legacy field (not used by current API)
DescriptionEn string `json:"description_en"` // English description field
ReleaseDate string `json:"release_date"`
Runtime int `json:"runtime_mins"` // API uses runtime_mins, not runtime
// Top-level jacket URLs
JacketFullURL string `json:"jacket_full_url"`
JacketThumbURL string `json:"jacket_thumb_url"`
// Gallery/screenshots
Gallery []struct {
ImageFull string `json:"image_full"`
ImageThumb string `json:"image_thumb"`
} `json:"gallery"`
// Sample video URL
SampleURL string `json:"sample_url"`
// Director - support both flat string and directors array
Director string `json:"director"` // Legacy flat string
DirectorEn string `json:"director_en"` // Legacy English director field
Directors []struct {
ID int `json:"id"`
NameKana string `json:"name_kana"`
NameKanji string `json:"name_kanji"`
NameRomaji string `json:"name_romaji"`
} `json:"directors"` // New directors array format
// Maker - support both nested and flat structures
Maker struct {
Name string `json:"name"`
} `json:"maker"`
MakerNameEn string `json:"maker_name_en"` // Flat English field
MakerNameJa string `json:"maker_name_ja"` // Flat Japanese field
// Label - support both nested and flat structures
Label struct {
Name string `json:"name"`
} `json:"label"`
LabelNameEn string `json:"label_name_en"` // Flat English field
LabelNameJa string `json:"label_name_ja"` // Flat Japanese field
// Series can be nested object or string
Series struct {
Name string `json:"name"`
} `json:"series"`
SeriesName string `json:"series_name"` // Fallback
SeriesNameEn string `json:"series_name_en"` // English series field
SeriesNameJa string `json:"series_name_ja"` // Japanese series field
// Categories - support both old name field and new name_en/name_ja fields
Categories []struct {
ID int `json:"id"`
Name string `json:"name"` // Legacy field
NameEn string `json:"name_en"` // New English field
NameJa string `json:"name_ja"` // New Japanese field
NameEnIsMachineTranslation bool `json:"name_en_is_machine_translation"`
} `json:"categories"`
// Actresses with detailed fields
Actresses []struct {
ID int `json:"id"`
ImageURL string `json:"image_url"`
NameKana string `json:"name_kana"`
NameKanji string `json:"name_kanji"`
NameRomaji string `json:"name_romaji"`
} `json:"actresses"`
// Images are now nested differently
Images struct {
JacketImage struct {
Large string `json:"large"`
Large2 string `json:"large2"`
} `json:"jacket_image"`
SampleImages []string `json:"sample_images"`
} `json:"images"`
// Sample/trailer
Sample struct {
High string `json:"high"`
Low string `json:"low"`
} `json:"sample"`
}
R18Response represents the JSON response from R18.dev API (current format)
type Scraper ¶
type Scraper struct {
// contains filtered or unexported fields
}
Scraper implements the R18.dev scraper
func New ¶
func New(settings config.ScraperSettings, globalProxy *config.ProxyConfig, globalFlareSolverr config.FlareSolverrConfig) *Scraper
New creates a new R18.dev scraper
func (*Scraper) CanHandleURL ¶
CanHandleURL returns true if this scraper can handle the given URL
func (*Scraper) Config ¶
func (s *Scraper) Config() *config.ScraperSettings
Config returns the scraper's configuration
func (*Scraper) ExtractIDFromURL ¶
ExtractIDFromURL extracts the movie ID from an R18.dev URL
func (*Scraper) ResolveDownloadProxyForHost ¶
func (s *Scraper) ResolveDownloadProxyForHost(host string) (*config.ProxyConfig, *config.ProxyConfig, bool)
ResolveDownloadProxyForHost declares R18.dev-owned media hosts for downloader proxy routing.
func (*Scraper) Search ¶
Search searches for and scrapes metadata for a given movie ID Search searches for and scrapes metadata for a given movie ID with context support
func (*Scraper) ValidateConfig ¶
func (s *Scraper) ValidateConfig(cfg *config.ScraperSettings) error
ValidateConfig validates the scraper configuration. Returns error if config is invalid, nil if valid. Called by callers before creating the scraper to validate configuration.