Why Go
- Case Studies
  
  Common problems companies solve with Go
- Use Cases
  
  Stories about how and why companies use Go
- Security
  
  How Go can help keep you secure by default
Learn
Docs
- Effective Go
  
  Tips for writing clear, performant, and idiomatic Go code
- Go User Manual
  
  A complete introduction to building software with Go
- Standard library
  
  Reference documentation for Go's standard library
- Release Notes
  
  Learn what's new in each Go release
- API
  
  Reference documentation for the pkg.go.dev API
Packages
Community
- Recorded Talks
  
  Videos from prior events
- Meetups
  
  Meet other local Go developers
- Conferences
  
  Learn and network with Go developers from around the world
- Go blog
  
  The Go project's official blog.
- Go project
  
  Get help and stay informed from Go
- Get connected

crawler

package

Go to main page

Versions in this module

v0

v0.5.0

May 11, 2026

v0.4.0

May 5, 2026

v0.2.0

May 3, 2026

v0.1.3

Jul 4, 2026

v0.1.1

Jun 25, 2026

Changes in this version

+ func ParseHTMLDoc(pageURL string, body []byte) (*html.Node, []Link, []Form, error)

+ func ResolveURL(base, href string) string

+ type CircuitBreakerRegistry struct

+ func NewCircuitBreakerRegistry(threshold int, cooldown time.Duration) *CircuitBreakerRegistry

+ func (r *CircuitBreakerRegistry) AllowRequest(host string) bool

+ func (r *CircuitBreakerRegistry) RecordFailure(host string)

+ func (r *CircuitBreakerRegistry) RecordSuccess(host string)

+ type CircuitState int

+ const CircuitClosed

+ const CircuitHalfOpen

+ const CircuitOpen

type Config

+ AllowPrivateIPs bool

+ CircuitBreaker *CircuitBreakerRegistry

+ Fetcher Fetcher

+ Logger *slog.Logger

+ MaxPages int

+ type Fetcher interface

+ Fetch func(ctx context.Context, page *Page, targetURL string) error

+ type HostCircuitBreaker struct

+ func (b *HostCircuitBreaker) AllowRequest() bool

+ func (b *HostCircuitBreaker) RecordFailure()

+ func (b *HostCircuitBreaker) RecordSuccess()

+ func (b *HostCircuitBreaker) State() CircuitState

type Page

+ AuthRequired bool

+ Doc *html.Node

+ type ParseResult struct

+ Forms []Form

+ Links []Link

+ ParseErr error

+ func ParseHTML(pageURL string, body []byte) ParseResult

type RobotsCache

+ func (rc *RobotsCache) CrawlDelay(origin, userAgent string) time.Duration

v0.1.0

May 2, 2026

Changes in this version

+ func FetchSitemapURLs(ctx context.Context, client *http.Client, sitemapURLs []string) []string

+ func ServeDir(ctx context.Context, dir string) (*http.Server, string, error)

+ type Config struct

+ AuthHeader string

+ AuthValue string

+ Concurrency int

+ CookieJar http.CookieJar

+ Exclude []string

+ FollowRedirects int

+ MaxDepth int

+ PageTimeout time.Duration

+ RateLimit int

+ RespectRobots bool

+ RetryAttempts int

+ RetryDelay time.Duration

+ Timeout time.Duration

+ UserAgent string

+ type Crawler struct

+ func New(cfg Config) *Crawler

+ func (c *Crawler) Crawl(ctx context.Context, startURL string) ([]*Page, error)

+ type Form struct

+ Action string

+ HasCSRF bool

+ ID string

+ Inputs []FormInput

+ Method string

+ type FormInput struct

+ Name string

+ Required bool

+ Type string

+ Value string

+ type Link struct

+ Anchor bool

+ External bool

+ Href string

+ Rel string

+ Resource bool

+ Tag string

+ Text string

+ type Page struct

+ Body []byte

+ Depth int

+ Duration time.Duration

+ Error error

+ Forms []Form

+ Headers http.Header

+ Links []Link

+ ParentURL string

+ StatusCode int

+ URL string

+ type RobotsCache struct

+ func NewRobotsCache() *RobotsCache

+ func (rc *RobotsCache) Allowed(rawURL, userAgent string) bool

+ func (rc *RobotsCache) Fetch(ctx context.Context, client *http.Client, origin string)

+ func (rc *RobotsCache) Sitemaps(origin string) []string

+ type SitemapURL struct

+ Changefreq string

+ Lastmod string

+ Loc string

+ Priority string