utils

package

v1.1.0 Latest Latest Go to latest Published: Aug 18, 2025 License: MIT Imports: 14 Imported by: 0

Details

This section is empty.

This section is empty.

func CheckAbility(visited map[string]struct{}, rules Rules, normURL string) bool

func CheckDomain(domain *url.URL, rawURL string) (bool, error)

func GetHTML(rawURL string) ([]byte, error)

func GetRobots(rawURL string) ([]byte, error)

func Normalize(rawURL string) (string, error)

type Queue []string

queue implementation

func (q *Queue) CheckEmpty() bool

func (q *Queue) Dequeue() (string, error)

func (q *Queue) Enqueue(url string)

func (q *Queue) Peek() (string, error)

func (q *Queue) Size() int

type QueueOps interface {
	Enqueue(string)
	Dequeue() (string, error)
	Peek() (string, error)
	Empty() bool
	Size() int
}

type Response struct {
	Title   string
	Content []string
	Links   []string
}

func ParseHTML(domain *url.URL, page []byte) (Response, error)

type Rules struct {
	Allowed    []string
	Disallowed []string
	Delay      int
}

func ParseRobots(normURL string, textFile []byte) (Rules, error)