crawler

type CrawlOptions struct {
	URL               string
	Depth             int
	UserAgent         string
	Verbose           bool
	Offsite           bool
	WithNumbers       bool
	ExtractEmails     bool
	ExtractMeta       bool
	CapturePaths      bool
	CaptureSubdomains bool
	CaptureDomain     bool
	MaxPages          int
	Threads           int
	MaxContext        int
	NoCache           bool
	CacheTTL          time.Duration
	MaxFiles          int
	ExtractSecrets    bool
	ProxyURL          string
	AuthType          string
	AuthUser          string
	AuthPass          string
	Headers           []string
	ExcludePaths      []string
	DumpDir           string
}

type CrawlResult struct {
	Words    []string
	Emails   []string
	Metadata []string
	Secrets  []string
	Context  string
	URL      string
	Title    string
	Pages    int
}

type Source interface {
	Crawl(ctx context.Context, opts CrawlOptions) (*CrawlResult, error)
}