config

type Config struct {
	Env             string         `yaml:"env" env-default:"development"`
	Workers         int            `yaml:"workers" env-default:"5"`
	Delay           *time.Duration `yaml:"delay" env-default:"60s"`
	RandomDelay     *time.Duration `yaml:"random_delay" env-default:"150s"`
	ManticoreIndex  []Index        `yaml:"manticore"`
	EntryChanBuffer int            `yaml:"entry_chan_buffer" env-default:"20"`
	Splitter        Splitter       `yaml:"splitter"`
	Parsers         []Parser       `yaml:"parsers"`
}

type Index struct {
	Name string `yaml:"index"`
}

type Parser struct {
	Url         string         `yaml:"url"`
	Current     bool           `yaml:"current" env-default:"true"`
	Previous    bool           `yaml:"previous" env-default:"false"`
	Pages       int            `yaml:"pages,omitempty" env-default:"1"`
	FetchAll    bool           `yaml:"fetch_all" env-default:"false"`
	UserAgent   *string        `yaml:"user_agent,omitempty"`
	Delay       *time.Duration `yaml:"delay,omitempty"`
	RandomDelay *time.Duration `yaml:"random_delay,omitempty"`
}

type Splitter struct {
	OptChunkSize int `yaml:"opt_chunk_size" env-default:"1800"`
	MaxChunkSize int `yaml:"max_chunk_size" env-default:"3600"`
}