scraperutil

package

v0.2.10-alpha Latest Latest Go to latest Published: Apr 25, 2026 License: MIT Imports: 4 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/javinizer/javinizer-go

Links

Open Source Insights

Documentation ¶

Index ¶

func CleanString(v string) string
func DefaultFlattenConfigFromConfig(c ScraperConfigInterface, overrides FlattenOverrides, builder SettingsBuilder) any
func GetDefaultScraperSettings() map[string]any
func GetDefaults() map[string]DefaultSettings
func GetPriorities() []string
func GetScraperConfigs() map[string]ScraperConfigAccessor
func GetScraperConstructor(name string) (any, bool)
func GetScraperConstructors() map[string]any
func IntPtr(i int) *int
func NormalizeLanguage(lang string) string
func ParseDate(s string) *time.Time
func RegisterModule(module ScraperModule)
func ResetAllRegistries()
func ResetConfigFactories()
func ResetConstructors()
func ResetDefaults()
func ResetDefaultsRegistries()
func ResetFlattenFuncs()
func ResetScraperConfigs()
func ResetScraperOptions()
func ResetValidators()
func ResolveURL(base, raw string) string
type ConfigFactory
- func GetConfigFactory(name string) ConfigFactory
type DefaultSettings
type FlattenFunc
- func DefaultFlattenConfig(overrides FlattenOverrides, builder SettingsBuilder) FlattenFunc
- func DefaultFlattenConfigWithRaw(overrides FlattenOverrides, builder SettingsBuilderWithRaw) FlattenFunc
- func GetFlattenFunc(name string) FlattenFunc
type FlattenOverrides
type FlattenedConfig
- func ExtractFlattenedConfig(cfg any) (*FlattenedConfig, bool)
type ScraperConfigAccessor
type ScraperConfigInterface
type ScraperModule
type ScraperOptionsProvider
- func GetScraperOptions(name string) (ScraperOptionsProvider, bool)
type SettingsBuilder
type SettingsBuilderWithRaw
type ValidatorFunc
- func GetValidator(name string) ValidatorFunc

Constants ¶

This section is empty.

Variables ¶

This section is empty.

Functions ¶

func CleanString ¶

func CleanString(v string) string

func DefaultFlattenConfigFromConfig ¶

func DefaultFlattenConfigFromConfig(c ScraperConfigInterface, overrides FlattenOverrides, builder SettingsBuilder) any

func GetDefaultScraperSettings ¶

func GetDefaultScraperSettings() map[string]any

func GetDefaults ¶

func GetDefaults() map[string]DefaultSettings

func GetPriorities ¶

func GetPriorities() []string

func GetScraperConfigs ¶

func GetScraperConfigs() map[string]ScraperConfigAccessor

func GetScraperConstructor ¶

func GetScraperConstructor(name string) (any, bool)

func GetScraperConstructors ¶

func GetScraperConstructors() map[string]any

func IntPtr ¶

func IntPtr(i int) *int

func NormalizeLanguage ¶

func NormalizeLanguage(lang string) string

func ParseDate ¶

func ParseDate(s string) *time.Time

func RegisterModule ¶

func RegisterModule(module ScraperModule)

func ResetAllRegistries ¶

func ResetAllRegistries()

func ResetConfigFactories ¶

func ResetConfigFactories()

func ResetConstructors ¶

func ResetConstructors()

func ResetDefaults ¶

func ResetDefaults()

func ResetDefaultsRegistries ¶

func ResetDefaultsRegistries()

func ResetFlattenFuncs ¶

func ResetFlattenFuncs()

func ResetScraperConfigs ¶

func ResetScraperConfigs()

func ResetScraperOptions ¶

func ResetScraperOptions()

func ResetValidators ¶

func ResetValidators()

func ResolveURL ¶

func ResolveURL(base, raw string) string

Types ¶

type ConfigFactory ¶

type ConfigFactory func() any

func GetConfigFactory ¶

func GetConfigFactory(name string) ConfigFactory

type DefaultSettings ¶

type DefaultSettings struct {
	Settings any
	Priority int
}

type FlattenFunc ¶

type FlattenFunc func(any) any

func DefaultFlattenConfig ¶

func DefaultFlattenConfig(overrides FlattenOverrides, builder SettingsBuilder) FlattenFunc

func DefaultFlattenConfigWithRaw ¶

func DefaultFlattenConfigWithRaw(overrides FlattenOverrides, builder SettingsBuilderWithRaw) FlattenFunc

func GetFlattenFunc ¶

func GetFlattenFunc(name string) FlattenFunc

type FlattenOverrides ¶

type FlattenOverrides struct {
	BaseURL       string
	Language      string
	UseBrowser    bool
	ScrapeActress *bool
	Cookies       map[string]string
	Extra         map[string]any
}

type FlattenedConfig ¶

type FlattenedConfig struct {
	Enabled       bool
	RateLimit     int
	Proxy         any
	DownloadProxy any
}

func ExtractFlattenedConfig ¶

func ExtractFlattenedConfig(cfg any) (*FlattenedConfig, bool)

type ScraperConfigAccessor ¶

type ScraperConfigAccessor func(any) any

type ScraperConfigInterface ¶

type ScraperConfigInterface interface {
	IsEnabled() bool
	GetUserAgent() string
	GetRequestDelay() int
	GetMaxRetries() int
	GetProxy() any
	GetDownloadProxy() any
}

type ScraperModule ¶

type ScraperModule interface {
	Name() string
	Description() string
	Constructor() any
	Validator() any
	ConfigFactory() any
	Options() any
	Defaults() any
	Priority() int
	FlattenFunc() any
}

type ScraperOptionsProvider ¶

type ScraperOptionsProvider struct {
	DisplayTitle string
	Options      []any
}

func GetScraperOptions ¶

func GetScraperOptions(name string) (ScraperOptionsProvider, bool)

type SettingsBuilder ¶

type SettingsBuilder func(*FlattenedConfig, FlattenOverrides) any

type SettingsBuilderWithRaw ¶

type SettingsBuilderWithRaw func(*FlattenedConfig, FlattenOverrides, any) any

type ValidatorFunc ¶

type ValidatorFunc func(any) error

func GetValidator ¶

func GetValidator(name string) ValidatorFunc

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL