crawler

package
v2.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 16, 2026 License: BSD-2-Clause Imports: 22 Imported by: 0

Documentation

Overview

Package crawler contains the parsing and download rules used while importing external pages.

Keep framework and browser quirks here. The main application should provide callbacks for effects such as downloading resources, deciding whether to persist a resource, and mapping imported documents to local paths. This keeps new parsing rules testable without a database, HTTP server, or admin UI.

Index

Constants

This section is empty.

Variables

View Source 
var KnownResourceKindsByExtension = map[string]string{}/* 142 elements not displayed */

Functions

func CanonicalWholeSitePagePath 

func CanonicalWholeSitePagePath(rawPath string) string

func CleanPath 

func CleanPath(rawPath string) string

func CloneURL 

func CloneURL(sourceURL *url.URL) *url.URL

func CurrentWholeSiteImportURL 

func CurrentWholeSiteImportURL(pageQueue []WholeSitePageJob) string

func DownloadHTML 

func DownloadHTML(client *http.Client, pageURL *url.URL, applyHeaders func(*http.Request)) (string, bool, error)

func DownloadHTMLContext 

func DownloadHTMLContext(ctx context.Context, client *http.Client, pageURL *url.URL, applyHeaders func(*http.Request)) (string, bool, error)

func EffectiveResourceContentType 

func EffectiveResourceContentType(resourceURL, contentType string) string

func ErrorReason 

func ErrorReason(err error) string
func ExtractPageLinks(htmlSource string, baseURL, siteURL *url.URL) []*url.URL

func FirstPathSegment 

func FirstPathSegment(rawPath string) string

func HasAllowedResourceExtension 

func HasAllowedResourceExtension(resourceURL string) bool

func IsPageURL 

func IsPageURL(pageURL *url.URL) bool

func IsSuspiciousReference 

func IsSuspiciousReference(rawRef string) bool

func IsWholeSitePageURL 

func IsWholeSitePageURL(pageURL *url.URL) bool

func IsWholeSitePageURLString 

func IsWholeSitePageURLString(rawURL string) bool

func NewSessionClient 

func NewSessionClient(timeout time.Duration, transport http.RoundTripper) *http.Client

func NormalizeMirroredAssetReference 

func NormalizeMirroredAssetReference(assetPath string) string

func NormalizeURL 

func NormalizeURL(rawRef string, baseURL *url.URL, referenceContext ReferenceContext) (string, bool)

func NormalizedResourceContentType 

func NormalizedResourceContentType(contentTypeHeader string) string

func OriginRootURL 

func OriginRootURL(baseURL *url.URL) *url.URL

func PreviewResourceKind 

func PreviewResourceKind(tagName, attributeName, rawRef string) string

func ResourceExtension 

func ResourceExtension(rawRef string) string

func ResourceExtensionFromContentType 

func ResourceExtensionFromContentType(contentType string) string

func ResourceKindFromContentType 

func ResourceKindFromContentType(contentType string) string

func ResourceKindFromURL 

func ResourceKindFromURL(resourceURL string) string

func RewriteCSSImportReferences 

func RewriteCSSImportReferences(source string, rewriteSingle func(string) string) string

func RewriteCSSURLReferences 

func RewriteCSSURLReferences(source string, rewriteSingle func(string) string) string

func RewriteDocumentResourceReferences 

func RewriteDocumentResourceReferences(source string, rewriteSingle func(string) string) string

func RewriteSrcSetReferences 

func RewriteSrcSetReferences(source string, rewriteSingle func(string) string) string

func SameHost 

func SameHost(leftURL, rightURL *url.URL) bool

func ShouldRewriteJSResourceReference 

func ShouldRewriteJSResourceReference(rawReference string) bool

func SplitDataURL 

func SplitDataURL(rawRef string) (string, string, bool)

func SplitStaticResourceURLTrailingText 

func SplitStaticResourceURLTrailingText(rawURL string) (string, string)
func WholeSiteLocalLink(basePath string, startURL, pageURL *url.URL) string

func WholeSiteLocalPath 

func WholeSiteLocalPath(basePath string, startURL, pageURL *url.URL) string

func WholeSitePageKey 

func WholeSitePageKey(pageURL *url.URL) string

Types

type DecodeResult 

type DecodeResult struct {
	Text           string
	Encoding       string
	EncodingSource string
	Certain        bool
}

func DecodeHTML 

func DecodeHTML(htmlBytes []byte, contentType string) DecodeResult

func DecodeText 

func DecodeText(textBytes []byte, contentType string) DecodeResult

type HTMLDownloadResult 

type HTMLDownloadResult struct {
	HTML            string
	IsHTML          bool
	ResolvedURL     *url.URL
	Status          string
	StatusCode      int
	Encoding        string
	EncodingSource  string
	EncodingCertain bool
}

HTMLDownloadResult carries the document and transport metadata together so callers can keep HTTP details out of application-level download code.

func DownloadHTMLPageContext 

func DownloadHTMLPageContext(ctx context.Context, client *http.Client, pageURL *url.URL, applyHeaders func(*http.Request)) (HTMLDownloadResult, error)

func DownloadHTMLPageWithRetriesContext 

func DownloadHTMLPageWithRetriesContext(ctx context.Context, client *http.Client, pageURL *url.URL, applyHeaders func(*http.Request), options HTMLDownloadRetryOptions) (HTMLDownloadResult, error)

type HTMLDownloadRetryOptions 

type HTMLDownloadRetryOptions struct {
	Attempts  int
	Delay     time.Duration
	OnAttempt func(attempt, total int, pageURL *url.URL)
	OnRetry   func(attempt, total int, pageURL *url.URL, err error, delay time.Duration)
}

type ImportRequest 

type ImportRequest struct {
	Domain               string
	PagePath             string
	SourceURL            string
	RemoteSourceURL      *url.URL
	HTML                 string
	Context              context.Context
	ProgressToken        string
	DownloadTotal        int
	DownloadTotalBytes   int64
	SelectedResourceURLs map[string]struct{}
	SourceOptions        SourceOptions
}

ImportRequest is the application-facing input for single page and whole site imports.

type ImportResult 

type ImportResult struct {
	RedirectPath  string
	FailedTotal   int
	FailedURLs    []string
	FailedReasons map[string]string
}

ImportResult is returned after the importer has persisted everything it could fetch.

type ImportedPage 

type ImportedPage struct {
	SourceURL string
	LocalPath string
	HTML      string
}

ImportedPage is a crawled remote document mapped to a local SiteBrush page.

type Parser 

type Parser struct {
	NormalizeURL                         func(rawRef string, baseURL *url.URL, referenceContext ReferenceContext) (string, bool)
	RewriteResourceReference             func(rawRef string, baseURL *url.URL, depth int, referenceContext ReferenceContext) string
	RewriteDocumentResourceReference     func(rawRef string, baseURL *url.URL, depth int) string
	DocumentURLRewriter                  func(normalizedURL string) (string, bool)
	ShouldBlankEmbeddedDocumentReference func(tagName, normalizedURL string) bool
	ShouldRewriteImageAltResource        func(rawRef string, baseURL *url.URL) bool
}

Parser rewrites references found in external pages.

The package owns parsing rules. The application owns side effects: fetching, quota decisions, asset persistence, whole-site page routing, and progress. Add new site-specific or framework-specific parsing rules here by extending the relevant Rewrite* method or adding a narrow helper next to the rule it supports.

func (Parser) RewriteJavaScriptReferences 

func (parser Parser) RewriteJavaScriptReferences(source, baseRawURL string, depth int) string

func (Parser) RewriteStaticURLTextReferences 

func (parser Parser) RewriteStaticURLTextReferences(source string, baseURL *url.URL, depth int) string

func (Parser) RewriteTextReferences 

func (parser Parser) RewriteTextReferences(source, baseRawURL string, depth int) string

type ProgressEvent 

type ProgressEvent struct {
	Token                  string            `json:"token"`
	Stage                  string            `json:"stage"`
	FoundTotal             int               `json:"found_total"`
	DownloadedTotal        int               `json:"downloaded_total"`
	DownloadTotal          int               `json:"download_total,omitempty"`
	DownloadedBytes        int64             `json:"downloaded_bytes,omitempty"`
	DownloadTotalBytes     int64             `json:"download_total_bytes,omitempty"`
	FailedTotal            int               `json:"failed_total"`
	FailedURLs             []string          `json:"failed_urls,omitempty"`
	FailedReasons          map[string]string `json:"failed_reasons,omitempty"`
	RetryAttempt           int               `json:"retry_attempt,omitempty"`
	RetryTotal             int               `json:"retry_total,omitempty"`
	RetryDelaySeconds      int               `json:"retry_delay_seconds,omitempty"`
	CurrentURL             string            `json:"current_url"`
	CurrentError           string            `json:"current_error,omitempty"`
	CurrentPercent         int               `json:"current_percent"`
	CurrentDownloadedBytes int64             `json:"current_downloaded_bytes"`
	CurrentSizeBytes       int64             `json:"current_size_bytes"`
	CompletedPercent       int               `json:"completed_percent"`
	Message                string            `json:"message"`
}

ProgressEvent is emitted by page and site import crawlers.

type ProgressTracker 

type ProgressTracker struct {
	// contains filtered or unexported fields
}

ProgressTracker routes progress events by token using a channel actor.

func NewProgressTracker 

func NewProgressTracker() *ProgressTracker

func (*ProgressTracker) Publish 

func (tracker *ProgressTracker) Publish(event ProgressEvent)

func (*ProgressTracker) Subscribe 

func (tracker *ProgressTracker) Subscribe(token string) chan ProgressEvent

func (*ProgressTracker) Unsubscribe 

func (tracker *ProgressTracker) Unsubscribe(token string, stream chan ProgressEvent)

type ReferenceContext 

type ReferenceContext int

ReferenceContext tells parser rules where a reference was found. Browsers do not resolve every string literal the same way: HTML attributes, CSS url(...) values, and JavaScript module manifests each have their own conventions. Keeping the context explicit makes new crawler quirks local to this package instead of spreading conditional URL handling through the app. 

const (
	ReferenceDocument ReferenceContext = iota
	ReferenceJavaScript
)

type ResourcePreview 

type ResourcePreview struct {
	URL       string `json:"url"`
	Kind      string `json:"kind"`
	SizeBytes int64  `json:"size_bytes"`
}

ResourcePreview describes one downloadable remote asset before import confirmation.

type SourceOptions 

type SourceOptions struct {
	IP           string
	LanguageCode string
}

SourceOptions carry transport-level source overrides shared by previews and imports.

type WholeSitePageJob 

type WholeSitePageJob struct {
	URL  *url.URL
	HTML string
}

WholeSitePageJob is the queue entry used while crawling a whole remote site.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.