extractors

type DefaultExtractor ¶

type DefaultExtractor struct {
	HTTPClient      httputils.HTTPClient
	Filters         htmlselector.OptimizedFilterGroup
	LinkTransformer models.LinkTransformer
}

DefaultExtractor ...

func (extractor DefaultExtractor) ExtractLinks(
	ctx context.Context,
	threadID int,
	link string,
) ([]string, error)

type DelayingExtractor ¶

type DelayingExtractor struct {
	// contains filtered or unexported fields
}

DelayingExtractor ...

func NewDelayingExtractor(
	minimalDelay time.Duration,
	sleeper SleepHandler,
	linkExtractor models.LinkExtractor,
) *DelayingExtractor

func (extractor *DelayingExtractor) ExtractLinks(
	ctx context.Context,
	threadID int,
	link string,
) ([]string, error)

type ExtractorGroup ¶ added in v1.7.1

type ExtractorGroup struct {
	Name           string
	LinkExtractors []models.LinkExtractor
	Logger         log.Logger
}

ExtractorGroup ...

func (extractors ExtractorGroup) ExtractLinks(
	ctx context.Context,
	threadID int,
	link string,
) ([]string, error)

type HTTPClient ¶

type HTTPClient interface {
	httputils.HTTPClient
}

HTTPClient ...

It's used only for mock generating.

type LinkExtractor ¶

type LinkExtractor interface {
	models.LinkExtractor
}

LinkExtractor ...

It's used only for mock generating.

type LinkLoader ¶ added in v1.7.1

type LinkLoader interface {
	LoadLink(link string, options interface{}) ([]byte, error)
}

LinkLoader ...

It's used only for mock generating.

type LinkTransformer ¶ added in v1.11.1

type LinkTransformer interface {
	models.LinkTransformer
}

LinkTransformer ...

It's used only for mock generating.

type Logger ¶

type Logger interface {
	log.Logger
}

Logger ...

It's used only for mock generating.

type RepeatingExtractor ¶

type RepeatingExtractor struct {
	LinkExtractor models.LinkExtractor
	RepeatCount   int
	RepeatDelay   time.Duration
	Logger        log.Logger
	SleepHandler  SleepHandler
}

RepeatingExtractor ...

func (extractor RepeatingExtractor) ExtractLinks(
	ctx context.Context,
	threadID int,
	link string,
) ([]string, error)

type SitemapExtractor ¶ added in v1.7.1

type SitemapExtractor struct {
	SitemapRegister registers.SitemapRegister
	Logger          log.Logger
}

SitemapExtractor ...

func (extractor SitemapExtractor) ExtractLinks(
	ctx context.Context,
	threadID int,
	link string,
) ([]string, error)

type SleepHandler ¶

type SleepHandler func(duration time.Duration)

SleepHandler ...

type Sleeper ¶

type Sleeper interface {
	Sleep(duration time.Duration)
}

Sleeper ...

It's used only for mock generating.

type TrimmingExtractor ¶ added in v1.10.1

type TrimmingExtractor struct {
	TrimLink      urlutils.LinkTrimming
	LinkExtractor models.LinkExtractor
}

TrimmingExtractor ...

func (extractor TrimmingExtractor) ExtractLinks(
	ctx context.Context,
	threadID int,
	link string,
) ([]string, error)