runner

package
v1.5.3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 2, 2024 License: MIT Imports: 22 Imported by: 0

Documentation

Index

Constants

View Source
const (
	RunModeFile = iota + 1
	RunModeDatabase
	RunModeDatabaseProduce
	RunModeInstallPlaywright
	RunModeWeb
)

Variables

View Source
var (
	ErrInvalidRunMode = errors.New("invalid run mode")
)

Functions

func Banner()

func CreateSeedJobs

func CreateSeedJobs(
	langCode string,
	r io.Reader,
	maxDepth int,
	email bool,
	geoCoordinates string,
	zoom int,
	dedup deduper.Deduper,
	exitMonitor exiter.Exiter,
) (jobs []scrapemate.IJob, err error)

func LoadCustomWriter

func LoadCustomWriter(pluginDir, pluginName string) (scrapemate.ResultWriter, error)

func Telemetry added in v1.3.4

func Telemetry() tlmt.Telemetry

Types

type Config

type Config struct {
	Concurrency              int
	CacheDir                 string
	MaxDepth                 int
	InputFile                string
	ResultsFile              string
	JSON                     bool
	LangCode                 string
	Debug                    bool
	Dsn                      string
	ProduceOnly              bool
	ExitOnInactivityDuration time.Duration
	Email                    bool
	CustomWriter             string
	GeoCoordinates           string
	Zoom                     int
	RunMode                  int
	DisableTelemetry         bool
	WebRunner                bool
	DataFolder               string
	Proxies                  []string
}

func ParseConfig

func ParseConfig() *Config

type Runner

type Runner interface {
	Run(context.Context) error
	Close(context.Context) error
}

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL