Documentation
¶
Index ¶
- Constants
- Variables
- func Banner()
- func CreateSeedJobs(fastmode bool, langCode string, r io.Reader, maxDepth int, email bool, ...) (jobs []scrapemate.IJob, err error)
- func LoadCustomWriter(pluginDir, pluginName string) (scrapemate.ResultWriter, error)
- func Telemetry() tlmt.Telemetry
- type Config
- type Runner
- type S3Uploader
Constants ¶
View Source
const ( RunModeFile = iota + 1 RunModeDatabase RunModeDatabaseProduce RunModeInstallPlaywright RunModeWeb RunModeAwsLambda RunModeAwsLambdaInvoker )
Variables ¶
View Source
var (
ErrInvalidRunMode = errors.New("invalid run mode")
)
Functions ¶
func CreateSeedJobs ¶
func LoadCustomWriter ¶
func LoadCustomWriter(pluginDir, pluginName string) (scrapemate.ResultWriter, error)
Types ¶
type Config ¶
type Config struct {
Concurrency int
CacheDir string
MaxDepth int
InputFile string
ResultsFile string
JSON bool
LangCode string
Debug bool
Dsn string
ProduceOnly bool
ExitOnInactivityDuration time.Duration
Email bool
CustomWriter string
GeoCoordinates string
Zoom int
RunMode int
DisableTelemetry bool
WebRunner bool
AwsLamdbaRunner bool
DataFolder string
Proxies []string
AwsAccessKey string
AwsSecretKey string
AwsRegion string
S3Uploader S3Uploader
S3Bucket string
AwsLambdaInvoker bool
FunctionName string
AwsLambdaChunkSize int
FastMode bool
Radius float64
Addr string
DisablePageReuse bool
ExtraReviews bool
LeadsDBAPIKey string
}
func ParseConfig ¶
func ParseConfig() *Config
Click to show internal directories.
Click to hide internal directories.