extractor

package
v2.0.26 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 2, 2026 License: AGPL-3.0 Imports: 32 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	LinkRegexStrict = MustStrictHTTPOnly()
	LinkRegex       = regexp.MustCompile(`(?i)https?://[^<>'",\s/]+\.[^<>'",\s/]+(?:/[^<>'",\s]*)?`) // Adapted from heritrix3's UriUtils (Apache License 2.0)

	AssetsRegex = `(?i)\b(?:src|href)=["']([^"']+\.(?:css|js|png|jpg|jpeg|gif|svg|webp|woff|woff2|ttf|eot))["']`
)
View Source 
var ObjectStorageServers = func() (s []string) {
	s = append(s, s3CompatibleServers...)
	s = append(s, azureServers...)
	return s
}()

All the supported object storage servers

Functions

func AddAtImportLinksToItemChild 

func AddAtImportLinksToItemChild(item *models.Item, atImportLinks []*models.URL)

Add the @import links to the item as children items if the @import jump is less than the config.MaxCSSJump.

func ExtractFromStringCSS 

func ExtractFromStringCSS(css string, inline bool) (links []string, atImportLinks []string)

func ExtractFromURLCSS 

func ExtractFromURLCSS(URL *models.URL) (links []*models.URL, atImportLinks []*models.URL, err error)

ExtractFromURLCSS extracts URLs from a CSS URL

func GetEmbeddedCSSJump 

func GetEmbeddedCSSJump(item *models.Item) int

Returns the number of @import jumps to reach the HTML item.

for example: 

CSS -> HTML: 0 @import jump
CSS -> CSS -> HTML: 1 @import jump
CSS -> CSS -> CSS -> HTML: 2 @import jump
CSS -> ItemGotRedirected -> CSS -> CSS -> HTML: still 2 @import jump (ignores the redirection items)

func GetURLsFromJSON 

func GetURLsFromJSON(decoder *json.Decoder) (assets, outlinks []string, err error)

func HTMLAssets 

func HTMLAssets(item *models.Item) (assets []*models.URL, err error)

func IsCSS 

func IsCSS(URL *models.URL) bool

https://html.spec.whatwg.org/multipage/links.html#link-type-stylesheet:process-the-linked-resource According to the spec, we should only check the Content-Type header if the resource is came from a HTTP(S) request.

func IsEmbeddedCSS 

func IsEmbeddedCSS(item *models.Item) bool

IsEmbeddedCSS checks if the item is an embedded CSS. An embedded CSS is a CSS item that is linked from an HTML item.

func IsHTML 

func IsHTML(URL *models.URL) bool

func IsJSON 

func IsJSON(URL *models.URL) bool

func IsM3U8 

func IsM3U8(URL *models.URL) bool

func IsSitemapXML 

func IsSitemapXML(URL *models.URL) bool

func IsXML 

func IsXML(URL *models.URL) bool

check if the Content-Type or MIME-type indicates XML exclude sitemap and SVG

func JSON 

func JSON(URL *models.URL) (assets, outlinks []*models.URL, err error)

func M3U8 

func M3U8(URL *models.URL) (assets []*models.URL, err error)

func MustStrictHTTPOnly 

func MustStrictHTTPOnly() *regexp.Regexp

func QuotedLinkRegexFindAll 

func QuotedLinkRegexFindAll(s string) []string

Helper function to call FindAllStringSubmatch on quotedLinkRegex and return only the capturing group (Quoted URL).

func TransformDocument 

func TransformDocument(u *models.URL) (doc *goquery.Document, err error)

TransformDocument transforms the document of a URL by detecting its encoding and creating a utf-8 goquery document.

func XML 

func XML(URL *models.URL) (assets, outlinks []*models.URL, err error)

Types

type AzureBlob 

type AzureBlob struct {
	Name         string `xml:"Name"` // path/to/file.txt, no leading slash
	LastModified string `xml:"Properties>Last-Modified"`
	Size         int64  `xml:"Properties>Content-Length"`
}

type AzureBlobEnumerationResults 

type AzureBlobEnumerationResults struct {
	XMLName    xml.Name    `xml:"EnumerationResults"`
	Prefix     string      `xml:"Prefix"`
	Marker     string      `xml:"Marker"`
	Blobs      []AzureBlob `xml:"Blobs>Blob"`
	NextMarker string      `xml:"NextMarker"`
}

AzureBlobEnumerationResults represents the XML structure of an Azure Blob Storage listing <https://learn.microsoft.com/en-us/rest/api/storageservices/enumerating-blob-resources>

type CommonPrefix 

type CommonPrefix struct {
	Prefix []string `xml:"Prefix"`
}

type HTMLOutlinkExtractor 

type HTMLOutlinkExtractor struct{}

func (HTMLOutlinkExtractor) Extract 

func (HTMLOutlinkExtractor) Extract(URL *models.URL) (outlinks []*models.URL, err error)

func (HTMLOutlinkExtractor) Match 

func (HTMLOutlinkExtractor) Match(URL *models.URL) bool

func (HTMLOutlinkExtractor) Support 

func (HTMLOutlinkExtractor) Support(m Mode) bool

type LinkHeaderExtractor 

type LinkHeaderExtractor struct{}
func (l LinkHeaderExtractor) ExtractLink(URL *models.URL) (URLs []*models.URL)

ExtractLink parses a raw Link header in the form: 

<url1>; rel="what", <url2>; rel="any"; another="yes", <url3>; rel="thing"

returning a slice of models.URL structs Each of these are separated by a `, ` and the in turn by a `; `, with the first always being the url, and the remaining the key-val pairs See: https://simon-frey.com/blog/link-header/, https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Link

func (LinkHeaderExtractor) Match 

func (l LinkHeaderExtractor) Match(URL *models.URL) bool

func (LinkHeaderExtractor) Support 

func (l LinkHeaderExtractor) Support(m Mode) bool

type Mode 

type Mode int
const (
	ModeGeneral  Mode = iota // Zeno is running in general archiving mode
	ModeHeadless             // Zeno is running in headless browser mode
)

type ObjectStorageOutlinkExtractor 

type ObjectStorageOutlinkExtractor struct{}

func (ObjectStorageOutlinkExtractor) Extract 

func (ObjectStorageOutlinkExtractor) Extract(URL *models.URL) ([]*models.URL, error)

ObjectStorage decides which helper to call based on the object storage server

func (ObjectStorageOutlinkExtractor) Match 

func (ObjectStorageOutlinkExtractor) Match(URL *models.URL) bool

Check if the response is from an object storage server

func (ObjectStorageOutlinkExtractor) Support 

func (ObjectStorageOutlinkExtractor) Support(m Mode) bool

type PDFOutlinkExtractor 

type PDFOutlinkExtractor struct{}

func (PDFOutlinkExtractor) Extract 

func (PDFOutlinkExtractor) Extract(URL *models.URL) (outlinks []*models.URL, err error)

func (PDFOutlinkExtractor) Match 

func (PDFOutlinkExtractor) Match(URL *models.URL) bool

func (PDFOutlinkExtractor) Support 

func (PDFOutlinkExtractor) Support(m Mode) bool

type S3ListBucketResult 

type S3ListBucketResult struct {
	XMLName               xml.Name       `xml:"ListBucketResult"`
	Name                  string         `xml:"Name"`
	Prefix                string         `xml:"Prefix"`
	Marker                string         `xml:"Marker"`
	Contents              []S3Object     `xml:"Contents"`
	CommonPrefixes        []CommonPrefix `xml:"CommonPrefixes"`
	IsTruncated           bool           `xml:"IsTruncated"`
	NextContinuationToken string         `xml:"NextContinuationToken"`
}

S3ListBucketResult represents the XML structure of an S3 bucket listing

type S3Object 

type S3Object struct {
	Key          string `xml:"Key"`
	LastModified string `xml:"LastModified"`
	Size         int64  `xml:"Size"`
}

type SitemapXMLOutlinkExtractor 

type SitemapXMLOutlinkExtractor struct{}

func (SitemapXMLOutlinkExtractor) Extract 

func (SitemapXMLOutlinkExtractor) Extract(URL *models.URL) ([]*models.URL, error)

func (SitemapXMLOutlinkExtractor) Match 

func (SitemapXMLOutlinkExtractor) Match(URL *models.URL) bool

func (SitemapXMLOutlinkExtractor) Support 

func (SitemapXMLOutlinkExtractor) Support(m Mode) bool

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.