crawl

type Config struct {
	Concurrency int           // worker pool size (default 4)
	Limit       int           // max URLs to process this run (0 = drain the frontier)
	HTMLDelay   time.Duration // per-host spacing for HTML hosts (default 5s)
	Types       []string      // restrict to these entity types (empty = all)
	Source      string        // restrict to this source (empty = all)
	Logf        func(format string, args ...any)
}

type Engine struct {
	// contains filtered or unexported fields
}

type FrodoClient interface {
	Get(ctx context.Context, path string, params url.Values) ([]byte, error)
}

type HTMLClient interface {
	Get(ctx context.Context, url string) ([]byte, error)
}

type Stats struct {
	Processed  int
	Done       int
	Failed     int
	Blocked    int
	Skipped    int
	Discovered int
}