config

type Config struct {
	Job           string `mapstructure:"job"`
	JobPrometheus string
	JobPath       string

	// UseSeencheck exists just for convenience of not checking
	// !DisableSeencheck in the rest of the code, to make the code clearer
	DisableSeencheck bool `mapstructure:"disable-seencheck"`
	UseSeencheck     bool

	UserAgent                       string        `mapstructure:"user-agent"`
	Cookies                         string        `mapstructure:"cookies"`
	WARCPrefix                      string        `mapstructure:"warc-prefix"`
	WARCOperator                    string        `mapstructure:"warc-operator"`
	WARCTempDir                     string        `mapstructure:"warc-temp-dir"`
	WARCSize                        int           `mapstructure:"warc-size"`
	WARCOnDisk                      bool          `mapstructure:"warc-on-disk"`
	WARCPoolSize                    int           `mapstructure:"warc-pool-size"`
	WARCQueueSize                   int           `mapstructure:"warc-queue-size"`
	WARCDedupeSize                  int           `mapstructure:"warc-dedupe-size"`
	WARCDedupeCacheSize             int           `mapstructure:"warc-dedupe-cache-size"`
	WARCWriteAsync                  bool          `mapstructure:"async-warc-write"`
	WARCDiscardStatus               []int         `mapstructure:"warc-discard-status"`
	WARCDigestAlgorithm             string        `mapstructure:"warc-digest-algorithm"`
	CDXDedupeServer                 string        `mapstructure:"warc-cdx-dedupe-server"`
	CDXCookie                       string        `mapstructure:"warc-cdx-cookie"`
	DoppelgangerDedupeServer        string        `mapstructure:"warc-doppelganger-dedupe-server"`
	HQAddress                       string        `mapstructure:"hq-address"`
	HQKey                           string        `mapstructure:"hq-key"`
	HQSecret                        string        `mapstructure:"hq-secret"`
	HQProject                       string        `mapstructure:"hq-project"`
	HQTimeout                       int           `mapstructure:"hq-timeout"`
	HQBatchSize                     int           `mapstructure:"hq-batch-size"`
	HQBatchConcurrency              int           `mapstructure:"hq-batch-concurrency"`
	HQSeencheckCacheSize            int           `mapstructure:"hq-seencheck-cache-size"`
	HQSeencheckURL                  string        `mapstructure:"hq-seencheck-url"`
	HQGZIPRequests                  bool          `mapstructure:"hq-gzip-requests"`
	DisableHTMLTag                  []string      `mapstructure:"disable-html-tag"`
	ExcludeHosts                    []string      `mapstructure:"exclude-host"`
	IncludeHosts                    []string      `mapstructure:"include-host"`
	IncludeString                   []string      `mapstructure:"include-string"`
	ExcludeString                   []string      `mapstructure:"exclude-string"`
	ExclusionFile                   []string      `mapstructure:"exclusion-file"`
	ExclusionFileLiveReload         bool          `mapstructure:"exclusion-file-live-reload"`
	ExclusionFileLiveReloadInterval time.Duration `mapstructure:"exclusion-file-live-reload-interval"`
	WorkersCount                    int           `mapstructure:"workers"`
	MaxConcurrentAssets             int           `mapstructure:"max-concurrent-assets"`
	MaxHops                         int           `mapstructure:"max-hops"`
	MaxRedirect                     int           `mapstructure:"max-redirect"`
	MaxCSSJump                      int           `mapstructure:"max-css-jump"`
	MaxRetry                        int           `mapstructure:"max-retry"`
	MaxContentLengthMiB             int           `mapstructure:"max-content-length"`
	MaxOutlinks                     int           `mapstructure:"max-outlinks"`
	HTTPTimeout                     time.Duration `mapstructure:"http-timeout"`
	ConnReadDeadline                time.Duration `mapstructure:"conn-read-deadline"`
	CrawlTimeLimit                  int           `mapstructure:"crawl-time-limit"`
	CrawlMaxTimeLimit               int           `mapstructure:"crawl-max-time-limit"`
	MinSpaceRequired                float64       `mapstructure:"min-space-required"`
	DomainsCrawl                    []string      `mapstructure:"domains-crawl"`
	DomainsCrawlFile                []string      `mapstructure:"domains-crawl-file"`
	CaptureAlternatePages           bool          `mapstructure:"capture-alternate-pages"`
	StrictRegex                     bool          `mapstructure:"strict-regex"`
	DisableLocalDedupe              bool          `mapstructure:"disable-local-dedupe"`
	CertValidation                  bool          `mapstructure:"cert-validation"`
	MaxSegmentRepetition            int           `mapstructure:"max-segment-repetition"`
	MaxSegmentRepetitionThreshold   int           `mapstructure:"max-segment-repetition-threshold"`
	MaxURLLength                    int           `mapstructure:"max-url-length"`
	DisableAssetsCapture            bool          `mapstructure:"disable-assets-capture"`
	UseHQ                           bool          // Special field to check if HQ is enabled depending on the command called

	// Headless
	Headless                 bool     `mapstructure:"headless"`
	HeadlessHeadful          bool     `mapstructure:"headless-headful"`
	HeadlessTrace            bool     `mapstructure:"headless-trace"`
	HeadlessChromiumRevision int      `mapstructure:"headless-chromium-revision"`
	HeadlessChromiumBin      string   `mapstructure:"headless-chromium-bin"`
	HeadlessDevTools         bool     `mapstructure:"headless-dev-tools"`
	HeadlessStealth          bool     `mapstructure:"headless-stealth"`
	HeadlessUserMode         bool     `mapstructure:"headless-user-mode"`
	HeadlessUserDataDir      string   `mapstructure:"headless-user-data-dir"`
	HeadlessAllowedMethods   []string `mapstructure:"headless-allowed-methods"`

	HeadlessPageTimeout       time.Duration `mapstructure:"headless-page-timeout"`
	HeadlessPageLoadTimeout   time.Duration `mapstructure:"headless-page-load-timeout"`
	HeadlessPagePostLoadDelay time.Duration `mapstructure:"headless-page-post-load-delay"`

	HeadlessBehaviors       []string      `mapstructure:"headless-behaviors"`
	HeadlessBehaviorTimeout time.Duration `mapstructure:"headless-behavior-timeout"`

	// Network
	Proxy         string   `mapstructure:"proxy"`
	DNSServers    []string `mapstructure:"dns-server"`
	RandomLocalIP bool     `mapstructure:"random-local-ip"`
	DisableIPv4   bool     `mapstructure:"disable-ipv4"`
	DisableIPv6   bool     `mapstructure:"disable-ipv6"`
	IPv6AnyIP     bool     `mapstructure:"ipv6-anyip"`

	// Rate limiting
	DisableRateLimit          bool          `mapstructure:"disable-rate-limit"`
	RateLimitCapacity         float64       `mapstructure:"rate-limit-capacity"`
	RateLimitRefillRate       float64       `mapstructure:"rate-limit-refill-rate"`
	RateLimitCleanupFrequency time.Duration `mapstructure:"rate-limit-cleanup-frequency"`

	// Logging
	NoStdoutLogging  bool   `mapstructure:"no-stdout-log"`
	NoStderrLogging  bool   `mapstructure:"no-stderr-log"`
	NoColorLogging   bool   `mapstructure:"no-color-logs"`
	NoFileLogging    bool   `mapstructure:"no-log-file"`
	E2ELogging       bool   `mapstructure:"log-e2e"`
	E2ELevel         string `mapstructure:"log-e2e-level"`
	StdoutLogLevel   string `mapstructure:"log-level"`
	TUI              bool   `mapstructure:"tui"`
	TUILogLevel      string `mapstructure:"tui-log-level"`
	LogFileLevel     string `mapstructure:"log-file-level"`
	LogFileOutputDir string `mapstructure:"log-file-output-dir"`
	LogFilePrefix    string `mapstructure:"log-file-prefix"`
	LogFileRotation  string `mapstructure:"log-file-rotation"`

	// Profiling
	PyroscopeAddress    string        `mapstructure:"pyroscope-address"`
	PyroscopeUploadRate time.Duration `mapstructure:"pyroscope-upload-rate"`
	SentryDSN           string        `mapstructure:"sentry-dsn"`

	// API
	APIPort int  `mapstructure:"api-port"`
	API     bool `mapstructure:"api"`

	// Prometheus and metrics
	Prometheus       bool   `mapstructure:"prometheus"`
	PrometheusPrefix string `mapstructure:"prometheus-prefix"`

	// Consul
	ConsulAddress      string   `mapstructure:"consul-address"`
	ConsulPort         string   `mapstructure:"consul-port"`
	ConsulACLToken     string   `mapstructure:"consul-acl-token"`
	ConsulRegister     bool     `mapstructure:"consul-register"`
	ConsulRegisterTags []string `mapstructure:"consul-register-tags"`

	InputSeeds []string // Special field to store the input URLs
	// contains filtered or unexported fields
}