robots

package

v0.7.2 Latest Latest Go to latest Published: Jan 27, 2026 License: MIT Imports: 11 Imported by: 0

Details

This section is empty.

View Source

var GlobalRobotsCache = &RobotsCache{
	cache: make(map[string]*robotsEntry),
}

func GetSitemaps(targetURL string) ([]string, error)

func IsURLAllowed(targetURL string) bool

Check if URL is allowed

type RobotsCache struct {
	RobotName string
	// contains filtered or unexported fields
}

func (rc *RobotsCache) GetRobotsForDomain(scheme string, domain string) (*robotsEntry, error)

Fetch and parse robots.txt with caching

func (rc *RobotsCache) SetClient(client *http.Client)

type Sitemap struct {
	Loc     string    `xml:"loc"`
	Lastmod time.Time `xml:"lastmod"`
	URLSet  URLSet
}

type SitemapIndex struct {
	URL      string
	XMLName  xml.Name  `xml:"sitemapindex"`
	Sitemaps []Sitemap `xml:"sitemap"`
}

func NewSitemapIndex(link string) *SitemapIndex

func (s *SitemapIndex) GetURLs() []URL

func (s *SitemapIndex) Process() bool

type SitemapInterface interface {
	Process() bool
	GetURLs() []string
}

type URL struct {
	Loc     string    `xml:"loc"`
	Lastmod time.Time `xml:"lastmod"`
}

func ExtractURLs(smap string) []URL

type URLSet struct {
	URL     string
	XMLName xml.Name `xml:"urlset"`
	URLs    []URL    `xml:"url"`
}

func NewURLSet(link string) *URLSet

func (s *URLSet) GetURLs() []URL

func (s *URLSet) Process() bool