config

type Config struct {
	Global GlobalConfig `yaml:"global"`
	Tasks  []TaskConfig `yaml:"tasks"`
}

type FieldConfig struct {
	Name      string `yaml:"name"`
	Selector  string `yaml:"selector"`
	Attr      string `yaml:"attr,omitempty"`       // HTML attribute to extract (e.g., "src", "href")
	CleanHTML bool   `yaml:"clean_html,omitempty"` // Whether to strip HTML tags from the result
	BaseURL   string `yaml:"base_url,omitempty"`   // Base URL to prepend to the result
}

type GlobalConfig struct {
	MaxConcurrency int               `yaml:"max_concurrency"`
	Headers        map[string]string `yaml:"headers"`
}

type TaskConfig struct {
	TaskID       string            `yaml:"task_id"`
	URL          string            `yaml:"url"`
	ListSelector string            `yaml:"list_selector,omitempty"` // Selector for the container of each item in a list
	ParserType   string            `yaml:"parser_type"`
	Concurrency  int               `yaml:"concurrency"`
	Delay        int               `yaml:"delay,omitempty"` // Delay in milliseconds before executing the task
	Headers      map[string]string `yaml:"headers"`
	Fields       []FieldConfig     `yaml:"fields"`
	Model        string            `yaml:"model,omitempty"`  // Used for parser_type: "llm"
	Prompt       string            `yaml:"prompt,omitempty"` // Used for parser_type: "llm"
}