web

type Crawler interface {
	BasicAuth(username, password string) Crawler
	DigestAuth(username, password string) Crawler
	Download(options Request) (filePath string, err error)
	Probe(Request) (int, []byte, error)
}

type Request struct {
	Method  string
	Url     string
	Body    []byte
	Headers []map[string]string
	Timeout time.Duration
	Context context.Context
}