dmm

package

v0.3.1-alpha Latest Latest Go to latest Published: Apr 30, 2026 License: MIT Imports: 23 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/javinizer/javinizer-go

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
Variables
func FetchWithBrowser(parentCtx context.Context, url string, timeout int, ...) (string, error)
func GetExtraPlaceholderHashes(settings *config.ScraperSettings) []string
func GetPlaceholderThreshold(settings *config.ScraperSettings) int
func IsPlaceholder(ctx context.Context, client *resty.Client, url string, thresholdBytes int64, ...) (bool, error)
func MergePlaceholderHashes(settings *config.ScraperSettings) []string
type DMMConfig
- func (c *DMMConfig) ToScraperSettings() *config.ScraperSettings
- func (c *DMMConfig) ValidateConfig(sc *config.ScraperSettings) error
type JSONLDAggregateRating
type JSONLDBrand
type JSONLDOffer
type JSONLDProduct
type JSONLDVideoObject
type Scraper
- func New(settings config.ScraperSettings, globalConfig *config.ScrapersConfig, ...) *Scraper

Constants ¶

View Source

const (
	ConfigKeyPlaceholderThreshold   = placeholder.ConfigKeyThreshold
	ConfigKeyExtraPlaceholderHashes = placeholder.ConfigKeyHashes
)

View Source

const DefaultPlaceholderThresholdKB = placeholder.DefaultThresholdKB

Variables ¶

View Source

var DefaultPlaceholderHashes = placeholder.DefaultDMMPlaceholderHashes

Functions ¶

func FetchWithBrowser ¶

func FetchWithBrowser(parentCtx context.Context, url string, timeout int, proxyProfile *config.ProxyProfile) (string, error)

FetchWithBrowser fetches a URL using Chrome browser automation with age verification cookies

func GetExtraPlaceholderHashes ¶

func GetExtraPlaceholderHashes(settings *config.ScraperSettings) []string

func GetPlaceholderThreshold ¶

func GetPlaceholderThreshold(settings *config.ScraperSettings) int

func IsPlaceholder ¶

func IsPlaceholder(ctx context.Context, client *resty.Client, url string, thresholdBytes int64, hashes []string) (bool, error)

func MergePlaceholderHashes ¶

func MergePlaceholderHashes(settings *config.ScraperSettings) []string

Types ¶

type DMMConfig ¶

type DMMConfig struct {
	config.BaseScraperConfig `yaml:",inline"`
	UseBrowser               bool     `yaml:"use_browser" json:"use_browser"`
	ScrapeActress            bool     `yaml:"scrape_actress" json:"scrape_actress"`
	PlaceholderThresholdKB   int      `yaml:"placeholder_threshold" json:"placeholder_threshold"`
	ExtraPlaceholderHashes   []string `yaml:"extra_placeholder_hashes" json:"extra_placeholder_hashes"`
}

func (*DMMConfig) ToScraperSettings ¶

func (c *DMMConfig) ToScraperSettings() *config.ScraperSettings

func (*DMMConfig) ValidateConfig ¶

func (c *DMMConfig) ValidateConfig(sc *config.ScraperSettings) error

type JSONLDAggregateRating ¶

type JSONLDAggregateRating struct {
	Type        string  `json:"@type"`
	RatingValue float64 `json:"ratingValue"`
	RatingCount int     `json:"ratingCount"`
}

JSONLDAggregateRating represents the AggregateRating schema

type JSONLDBrand ¶

type JSONLDBrand struct {
	Type string `json:"@type"`
	Name string `json:"name"`
}

JSONLDBrand represents the Brand schema

type JSONLDOffer ¶

type JSONLDOffer struct {
	Type          string  `json:"@type"`
	Availability  string  `json:"availability"`
	PriceCurrency string  `json:"priceCurrency"`
	Price         float64 `json:"price"`
}

JSONLDOffer represents the Offer schema

type JSONLDProduct ¶

type JSONLDProduct struct {
	Context         string                 `json:"@context"`
	Type            string                 `json:"@type"`
	Name            string                 `json:"name"`
	Description     string                 `json:"description"`
	Image           interface{}            `json:"image"` // Can be string or array
	SKU             string                 `json:"sku"`
	Brand           *JSONLDBrand           `json:"brand"`
	SubjectOf       *JSONLDVideoObject     `json:"subjectOf"`
	Offers          *JSONLDOffer           `json:"offers"`
	AggregateRating *JSONLDAggregateRating `json:"aggregateRating"`
}

JSONLDProduct represents the Product schema from JSON-LD

type JSONLDVideoObject ¶

type JSONLDVideoObject struct {
	Type         string   `json:"@type"`
	Name         string   `json:"name"`
	Description  string   `json:"description"`
	ContentURL   string   `json:"contentUrl"`
	ThumbnailURL string   `json:"thumbnailUrl"`
	UploadDate   string   `json:"uploadDate"`
	Genre        []string `json:"genre"`
}

JSONLDVideoObject represents the VideoObject schema

type Scraper ¶

type Scraper struct {
	// contains filtered or unexported fields
}

func New ¶

func New(settings config.ScraperSettings, globalConfig *config.ScrapersConfig, contentIDRepo *database.ContentIDMappingRepository) *Scraper

func (*Scraper) CanHandleURL ¶

func (s *Scraper) CanHandleURL(rawURL string) bool

func (*Scraper) Close ¶

func (s *Scraper) Close() error

func (*Scraper) Config ¶

func (s *Scraper) Config() *config.ScraperSettings

func (*Scraper) ExtractIDFromURL ¶

func (s *Scraper) ExtractIDFromURL(urlStr string) (string, error)

func (*Scraper) GetURL ¶

func (s *Scraper) GetURL(id string) (string, error)

func (*Scraper) IsEnabled ¶

func (s *Scraper) IsEnabled() bool

func (*Scraper) Name ¶

func (s *Scraper) Name() string

func (*Scraper) ResolveContentID ¶

func (s *Scraper) ResolveContentID(id string) (string, error)

func (*Scraper) ResolveContentIDCtx ¶

func (s *Scraper) ResolveContentIDCtx(ctx context.Context, id string) (string, error)

func (*Scraper) ResolveDownloadProxyForHost ¶

func (s *Scraper) ResolveDownloadProxyForHost(host string) (*config.ProxyConfig, *config.ProxyConfig, bool)

func (*Scraper) ScrapeURL ¶

func (s *Scraper) ScrapeURL(ctx context.Context, url string) (*models.ScraperResult, error)

func (*Scraper) Search ¶

func (s *Scraper) Search(ctx context.Context, id string) (*models.ScraperResult, error)

func (*Scraper) ValidateConfig ¶

func (s *Scraper) ValidateConfig(cfg *config.ScraperSettings) error

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL