crawler

type Config struct {
	// Maximum crawl depth
	MaxDepth int

	// Maximum pages to crawl
	MaxPages int

	// Include patterns (glob)
	IncludePatterns []string

	// Exclude patterns (glob)
	ExcludePatterns []string

	// Wait for SPA hydration
	WaitForSPA bool

	// SPA framework indicators
	SPAIndicators []string

	// Respect robots.txt
	RespectRobots bool

	// Use sitemap.xml
	UseSitemap bool

	// Delay between requests
	Delay time.Duration

	// Request timeout
	Timeout time.Duration

	// Concurrency limit
	Concurrency int
}

type Crawler struct {
	// contains filtered or unexported fields
}

type Page struct {
	URL            string        `json:"url"`
	Title          string        `json:"title"`
	Depth          int           `json:"depth"`
	DiscoveredFrom string        `json:"discoveredFrom"`
	IsSPA          bool          `json:"isSPA"`
	SPAFramework   string        `json:"spaFramework,omitempty"`
	LoadTime       time.Duration `json:"loadTime"`
	Links          []string      `json:"links"`
	Error          string        `json:"error,omitempty"`
}

type Result struct {
	StartURL    string        `json:"startUrl"`
	Pages       []Page        `json:"pages"`
	TotalPages  int           `json:"totalPages"`
	Duration    time.Duration `json:"duration"`
	RobotsTxt   string        `json:"robotsTxt,omitempty"`
	SitemapURLs []string      `json:"sitemapUrls,omitempty"`
}