fetcher

type Fetcher interface {
	// FetchURLAsString fetches the content of a URL as a string.
	FetchURLAsString(ctx context.Context, url string) (string, error)
	// FetchURL fetches the content of a URL as a byte slice.
	FetchURL(ctx context.Context, url string) ([]byte, error)
	// StreamURL streams the content of a URL to the provided writer.
	StreamURL(ctx context.Context, url string, writer io.Writer) error
}

type FetcherConfig struct {
	// UserAgent is the user agent string to use for HTTP requests
	UserAgent string `toml:"user_agent"`
	// Timeout for HTTP requests
	Timeout time.Duration `toml:"timeout"`
	// RateLimit is the time in seconds to wait between requests
	RateLimit float64 `toml:"rate_limit"`
}

type HTTPClient interface {
	Do(req *http.Request) (*http.Response, error)
}

type HTTPFetcher struct {
	// contains filtered or unexported fields
}

type HTTPFetcherOption func(*HTTPFetcher)

type LimitedFetcher struct {
	Fetcher *HTTPFetcher
	// contains filtered or unexported fields
}

type RateLimiter struct {
	// contains filtered or unexported fields
}