engine

package
v0.2.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 13, 2026 License: MIT Imports: 55 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	CDNCloudflare  = "cloudflare"
	CDNCloudFront  = "cloudfront"
	CDNFastly      = "fastly"
	CDNAkamai      = "akamai"
	CDNVarnish     = "varnish"
	CDNNetlify     = "netlify"
	CDNVercel      = "vercel"
	CDNAzure       = "azure"
	CDNGCP         = "gcp"
	CDNFly         = "fly"
	CDNDeno        = "deno"
	CDNHeroku      = "heroku"
	CDNRender      = "render"
	CDNRailway     = "railway"
	CDNShopify     = "shopify"
	CDNSquarespace = "squarespace"
	CDNWix         = "wix"
	CDNWebflow     = "webflow"
)
View Source 
const (
	DefaultClientTimeout = 30 * time.Second

	DefaultMaxRetryWait = 60 * time.Second

	DefaultTotalRetryTimeout = 120 * time.Second

	DefaultRetryBackoffBase = time.Second

	DefaultSitemapMaxDepth = 5
	DefaultSitemapMaxURLs  = 50_000

	DefaultFeedMaxItems = 200
)
View Source 
const (
	DefaultScrapeMaxPages      = 50
	DefaultScrapeMaxPerPattern = 8
	DefaultScrapeTimeout       = 60 * time.Second
)
View Source 
const DefaultAccept = "text/markdown, text/html;q=0.9, application/xhtml+xml;q=0.8, application/xml;q=0.7, */*;q=0.1"
View Source 
const DefaultAcceptEncoding = "gzip, deflate, br, zstd"
View Source 
const DefaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"

Variables

View Source 
var (
	ErrSecurityScheme     = errors.New("scheme not allowed by security policy")
	ErrSecurityDomain     = errors.New("host blocked by security policy")
	ErrPrivateIPBlocked   = errors.New("target resolves to a private/internal IP")
	ErrSecurityResolve    = errors.New("could not resolve target host")
	ErrResponseTooLarge   = errors.New("response exceeds the configured size cap")
	ErrDecompressTooLarge = errors.New("decompressed body exceeds the configured size cap")
)
View Source 
var ErrBlockedByRobots = errors.New("blocked by robots.txt")
View Source 
var ErrInvalidBaseURL = errors.New("seaportal: ScrapeOptions.BaseURL is not a valid absolute URL")
View Source 
var ErrMissingBaseURL = errors.New("seaportal: ScrapeOptions.BaseURL is required")
View Source 
var ErrNeedsBrowser = errors.New("needs-browser")

Functions

func ApplySchema 

func ApplySchema(htmlStr string, schema Schema) (map[string]interface{}, error)

func CanonicalizeURL 

func CanonicalizeURL(rawURL string) (string, error)

func ChangeSignificance 

func ChangeSignificance(oldContent, newContent string) int

func CleanupMarkdown 

func CleanupMarkdown(md string) string

func ContentChanged 

func ContentChanged(oldContent, newContent string) bool

func ConvertLinksToCitations 

func ConvertLinksToCitations(md string) string

func CountMarkdownHeadings 

func CountMarkdownHeadings(content string) int
func CountMarkdownLinks(content string) int

func CountPattern 

func CountPattern(html string, pattern string) int

func DedupeLines 

func DedupeLines(content string) string

func DetectBlocked 

func DetectBlocked(html string) bool

func DetectJSChallenge 

func DetectJSChallenge(html string, contentType string, length int) bool

func DetectLanguage 

func DetectLanguage(markdown string) string

func DetectSPA 

func DetectSPA(html string) (signals []string, isSPA bool)

func ExtractFromHTML 

func ExtractFromHTML(html string, targetURL string) (string, error)

func ExtractMetaAuthors 

func ExtractMetaAuthors(rawHTML string) []string

func ExtractPDFText 

func ExtractPDFText(body []byte) (out string, err error)

func FetchBytes 

func FetchBytes(ctx context.Context, rawURL string, opts FetchBytesOptions) ([]byte, http.Header, int, error)

func LDJSONToMarkdown 

func LDJSONToMarkdown(blocks []LDJSONBlock) string

func NearDuplicateScore 

func NearDuplicateScore(a, b string) int

func PickCanonical 

func PickCanonical(rawURL, html string) string

func PreprocessHTML 

func PreprocessHTML(htmlStr string) string

func PreprocessHTMLWithURL 

func PreprocessHTMLWithURL(htmlStr string, _ *url.URL) string

func PruneToContent 

func PruneToContent(htmlStr string) string

func QuickNeedsBrowser 

func QuickNeedsBrowser(html string) (needsBrowser bool, reason string)

func RenderScrapeJSON added in v0.2.0 

func RenderScrapeJSON(res *ScrapeResult) ([]byte, error)

func RenderScrapeMarkdown added in v0.2.0 

func RenderScrapeMarkdown(res *ScrapeResult) string
func ResolveCanonicalLink(htmlStr string, baseURL string) string

func ResolveUserAgent 

func ResolveUserAgent(s string) string

func ResultToTEIXML 

func ResultToTEIXML(r Result) ([]byte, error)

func SanitizeHTML 

func SanitizeHTML(html string) string

func SemanticFingerprint 

func SemanticFingerprint(content string) string

func ShouldUseIndexFallback 

func ShouldUseIndexFallback(readabilityResult Result, indexResult IndexPageResult) bool

func StripInvisibleUnicode 

func StripInvisibleUnicode(text string) string

func TruncateMarkdownAtParagraph 

func TruncateMarkdownAtParagraph(md string, maxTokens int) (string, bool)

func WriteScrapeDirectory added in v0.2.0 

func WriteScrapeDirectory(res *ScrapeResult, dir string) ([]string, error)

Types

type BrowserDecision 

type BrowserDecision string
const (
	DecisionStaticHighConfidence BrowserDecision = "static-high-confidence"
	DecisionStaticOK             BrowserDecision = "static-ok"
	DecisionStaticCaution        BrowserDecision = "static-caution"
	DecisionBrowserNeeded        BrowserDecision = "browser-needed"
	DecisionBlocked              BrowserDecision = "blocked"
	DecisionAuthRequired         BrowserDecision = "auth-required"
	DecisionUnreachable          BrowserDecision = "unreachable"
	DecisionNotFound             BrowserDecision = "not-found"
	DecisionUnsupported          BrowserDecision = "unsupported"
)

type CDNInfo added in v0.2.0 

type CDNInfo struct {
	CDNProvider string   `json:"cdnProvider,omitempty"`
	CDNSignals  []string `json:"cdnSignals,omitempty"`

	ViaHops     []ViaHop `json:"viaHops,omitempty"`
	ProxyLayers int      `json:"proxyLayers,omitempty"`
}

type CacheAnalysis added in v0.2.0 

type CacheAnalysis struct {
	CacheAge       int  `json:"cacheAge,omitempty"`
	CacheMaxAge    int  `json:"cacheMaxAge,omitempty"`
	CacheSMaxAge   int  `json:"cacheSMaxAge,omitempty"`
	CacheFreshness int  `json:"cacheFreshness,omitempty"`
	CacheIsStale   bool `json:"cacheIsStale,omitempty"`

	CacheStaleWhileRevalidate int  `json:"cacheStaleWhileRevalidate,omitempty"`
	CacheStaleIfError         int  `json:"cacheStaleIfError,omitempty"`
	CacheHasSWR               bool `json:"cacheHasSwr,omitempty"`

	CDNEdgeLocation string `json:"cdnEdgeLocation,omitempty"`

	CacheMustRevalidate  bool     `json:"cacheMustRevalidate,omitempty"`
	CacheProxyRevalidate bool     `json:"cacheProxyRevalidate,omitempty"`
	CacheNoCache         bool     `json:"cacheNoCache,omitempty"`
	CacheNoCacheFields   []string `json:"cacheNoCacheFields,omitempty"`
	CacheNoStore         bool     `json:"cacheNoStore,omitempty"`
	CacheImmutable       bool     `json:"cacheImmutable,omitempty"`
	CachePrivate         bool     `json:"cachePrivate,omitempty"`
	CachePublic          bool     `json:"cachePublic,omitempty"`

	CachePolicySummary string `json:"cachePolicySummary,omitempty"`
	CachePolicyScore   int    `json:"cachePolicyScore,omitempty"`

	EffectiveCDNTTL       int      `json:"effectiveCdnTtl,omitempty"`
	CacheRecommendations  []string `json:"cacheRecommendations,omitempty"`
	CDNOptimizationIssues []string `json:"cdnOptimizationIssues,omitempty"`

	SurrogateMaxAge               int  `json:"surrogateMaxAge,omitempty"`
	SurrogateStaleWhileRevalidate int  `json:"surrogateStaleWhileRevalidate,omitempty"`
	SurrogateStaleIfError         int  `json:"surrogateStaleIfError,omitempty"`
	SurrogateNoStore              bool `json:"surrogateNoStore,omitempty"`
	SurrogateNoStoreRemote        bool `json:"surrogateNoStoreRemote,omitempty"`

	RequestID string `json:"requestId,omitempty"`

	CacheHitRateEstimate  string `json:"cacheHitRateEstimate,omitempty"`
	BandwidthSavingsLevel string `json:"bandwidthSavingsLevel,omitempty"`
	CacheCostAnalysis     string `json:"cacheCostAnalysis,omitempty"`
}

type CardItem 

type CardItem struct {
	Title   string
	URL     string
	Teaser  string
	Section string
}

type Chunk 

type Chunk struct {
	Index   int    `json:"index"`
	Heading string `json:"heading,omitempty"`
	Text    string `json:"text"`
	Tokens  int    `json:"tokens"`
}

func ChunkMarkdown 

func ChunkMarkdown(md string, cfg ChunkConfig) []Chunk

type ChunkConfig 

type ChunkConfig struct {
	Strategy ChunkStrategy
	Size     int
	Overlap  int
}

func ParseChunkConfig 

func ParseChunkConfig(s string) (ChunkConfig, error)

func (ChunkConfig) String 

func (c ChunkConfig) String() string

type ChunkStrategy 

type ChunkStrategy int
const (
	ChunkOff ChunkStrategy = iota
	ChunkHeading
	ChunkSentence
	ChunkWindow
)

type CommentRef 

type CommentRef struct {
	Author    string `json:"author,omitempty"`
	Text      string `json:"text"`
	Timestamp string `json:"timestamp,omitempty"`
}

func ExtractComments 

func ExtractComments(htmlStr string, _ string) []CommentRef

type CorpusEntry 

type CorpusEntry = corpus.Entry

func LoadCorpus 

func LoadCorpus(path string) ([]CorpusEntry, error)

type CrawlDelayCache 

type CrawlDelayCache struct {
	// contains filtered or unexported fields
}

func NewCrawlDelayCache 

func NewCrawlDelayCache() *CrawlDelayCache

func (*CrawlDelayCache) GetDelay 

func (c *CrawlDelayCache) GetDelay(ctx context.Context, domain string, userAgent string) time.Duration

func (*CrawlDelayCache) GetDelayWithScheme 

func (c *CrawlDelayCache) GetDelayWithScheme(ctx context.Context, domain string, userAgent string, scheme string) time.Duration

func (*CrawlDelayCache) IsAllowed 

func (c *CrawlDelayCache) IsAllowed(ctx context.Context, domain string, userAgent string, scheme string, path string) bool

type DedupeOptions 

type DedupeOptions struct {
	MinBlockLen         int
	NormalizeWhitespace bool
	CaseSensitive       bool
	NearDup             bool
}

func DefaultDedupeOptions 

func DefaultDedupeOptions() DedupeOptions

type DedupeResult 

type DedupeResult struct {
	Content              string   `json:"content,omitempty"`
	OriginalBlocks       int      `json:"originalBlocks,omitempty"`
	UniqueBlocks         int      `json:"uniqueBlocks,omitempty"`
	DuplicatesFound      int      `json:"duplicatesFound,omitempty"`
	DuplicateSignals     []string `json:"duplicateSignals,omitempty"`
	NearDuplicatesFound  int      `json:"nearDuplicatesFound,omitempty"`
	NearDuplicateSignals []string `json:"nearDuplicateSignals,omitempty"`
}

func Dedupe 

func Dedupe(content string) DedupeResult

func DedupeWithOptions 

func DedupeWithOptions(content string, opts DedupeOptions) DedupeResult

type DedupeStats added in v0.2.0 

type DedupeStats struct {
	DedupeApplied         bool     `json:"dedupeApplied,omitempty"`
	DuplicatesRemoved     int      `json:"duplicatesRemoved,omitempty"`
	DuplicateSignals      []string `json:"duplicateSignals,omitempty"`
	NearDuplicatesRemoved int      `json:"nearDuplicatesRemoved,omitempty"`
	NearDuplicateSignals  []string `json:"nearDuplicateSignals,omitempty"`
	OriginalBlockCount    int      `json:"originalBlockCount,omitempty"`
	UniqueBlockCount      int      `json:"uniqueBlockCount,omitempty"`
}

type DiskCache 

type DiskCache struct {
	// contains filtered or unexported fields
}

func NewDiskCache 

func NewDiskCache(dir string, ttl time.Duration) (*DiskCache, error)

func (*DiskCache) Get 

func (c *DiskCache) Get(url string, req *http.Request) (*cachedResponse, []byte, bool)

func (*DiskCache) GetStale 

func (c *DiskCache) GetStale(url string, req *http.Request) (*cachedResponse, []byte, bool, bool)

func (*DiskCache) GetStaleWithTolerance 

func (c *DiskCache) GetStaleWithTolerance(url string, req *http.Request, tolerance time.Duration) (*cachedResponse, []byte, bool, bool, bool)

func (*DiskCache) Put 

func (c *DiskCache) Put(url string, req *http.Request, status int, headers http.Header, body []byte) error

func (*DiskCache) TouchByKey 

func (c *DiskCache) TouchByKey(key string) error

type DomainRetry 

type DomainRetry struct {
	MaxRetries   int
	MaxRetryWait time.Duration
}

type ExtractionOutcome 

type ExtractionOutcome string
const (
	OutcomeExtract        ExtractionOutcome = "extract"
	OutcomeExtractWarning ExtractionOutcome = "extract-with-warning"
	OutcomeFailFast       ExtractionOutcome = "fail-fast"
	OutcomeNeedsBrowser   ExtractionOutcome = "needs-browser"
)

type FeedItem 

type FeedItem struct {
	Title     string `json:"title,omitempty"`
	Link      string `json:"link,omitempty"`
	Published string `json:"published,omitempty"`
	Summary   string `json:"summary,omitempty"`
	Author    string `json:"author,omitempty"`
	GUID      string `json:"guid,omitempty"`
}

func ParseFeed 

func ParseFeed(ctx context.Context, feedURL string, opts ParseFeedOptions) ([]FeedItem, error)

type FetchBytesOptions 

type FetchBytesOptions struct {
	Client    *http.Client
	Timeout   time.Duration
	Security  *SecurityPolicy
	Accept    string
	UserAgent string
}

type FieldSpec 

type FieldSpec struct {
	Selector string               `json:"selector,omitempty" yaml:"selector,omitempty"`
	Attr     string               `json:"attr,omitempty" yaml:"attr,omitempty"`
	Multiple bool                 `json:"multiple,omitempty" yaml:"multiple,omitempty"`
	Fields   map[string]FieldSpec `json:"fields,omitempty" yaml:"fields,omitempty"`
}

type FlattenSitemapOptions 

type FlattenSitemapOptions struct {
	MaxDepth int
	MaxURLs  int
	Timeout  time.Duration
	Client   *http.Client
	Security *SecurityPolicy
	Since    time.Time
}

type HostRateLimiter 

type HostRateLimiter struct {
	// contains filtered or unexported fields
}

func NewHostRateLimiter 

func NewHostRateLimiter() *HostRateLimiter

func (*HostRateLimiter) Wait 

func (l *HostRateLimiter) Wait(ctx context.Context, host string, minInterval time.Duration) error

type IPResolver added in v0.2.0 

type IPResolver interface {
	LookupIP(ctx context.Context, network, host string) ([]net.IP, error)
}

type ImageRef 

type ImageRef struct {
	Src    string `json:"src"`
	Alt    string `json:"alt,omitempty"`
	Srcset string `json:"srcset,omitempty"`
	Title  string `json:"title,omitempty"`
}

func ExtractImages 

func ExtractImages(htmlStr string, baseURL string) []ImageRef

type IndexPageResult 

type IndexPageResult struct {
	IsIndexPage   bool
	Items         []CardItem
	Markdown      string
	Confidence    int
	ArticleCount  int
	HeadlineCount int
}

func DetectIndexPage 

func DetectIndexPage(htmlStr string) IndexPageResult

type LDJSONBlock 

type LDJSONBlock struct {
	Type        string `json:"type,omitempty"`
	Headline    string `json:"headline,omitempty"`
	Description string `json:"description,omitempty"`
	Author      string `json:"author,omitempty"`
	DatePub     string `json:"datePublished,omitempty"`
	Publisher   string `json:"publisher,omitempty"`
	URL         string `json:"url,omitempty"`
	Keywords    string `json:"keywords,omitempty"`
	Language    string `json:"inLanguage,omitempty"`
	Section     string `json:"articleSection,omitempty"`
	Body        string `json:"articleBody,omitempty"`
}

func ExtractLDJSON 

func ExtractLDJSON(html string) []LDJSONBlock

type LinkRef 

type LinkRef struct {
	Href string `json:"href"`
	Text string `json:"text,omitempty"`
	Rel  string `json:"rel,omitempty"`
}
func ExtractLinks(htmlStr string, baseURL string) []LinkRef

type LinkRetention 

type LinkRetention int
const (
	LinkRetentionAll LinkRetention = iota
	LinkRetentionNone
	LinkRetentionText
	LinkRetentionFooter
)

func ParseLinkRetention 

func ParseLinkRetention(s string) (LinkRetention, error)

func (LinkRetention) String 

func (m LinkRetention) String() string

type Metadata 

type Metadata struct {
	Author        string
	PublishedDate string
	Language      string
	Section       string
	Description   string
	ImageURL      string
	OGType        string
	Keywords      string
}

func ExtractMetadata 

func ExtractMetadata(rawHTML string) Metadata

type Options 

type Options struct {
	FailFast          bool
	FastMode          bool
	ProbeSearch       bool
	NoPooling         bool
	MaxRetries        int
	MaxRetryWait      time.Duration
	TotalRetryTimeout time.Duration

	ClientTimeout time.Duration

	RetryBackoffBase time.Duration

	HeadPreflight        bool
	ContentTypePreflight bool
	RetryLogger          func(event RetryEvent)
	DomainRetryConfig    map[string]DomainRetry
	UserAgent            string
	DomainUserAgent      map[string]string
	DomainTimeout        map[string]time.Duration
	RespectCrawlDelay    bool
	RespectRobots        bool
	CrawlDelayCache      *CrawlDelayCache
	RateLimit            time.Duration
	RateLimiter          *HostRateLimiter
	RequestID            string
	SendRequestID        bool
	Dedupe               bool
	NoNearDedupe         bool
	WithLinks            bool
	WithImages           bool
	WithTables           bool
	WithComments         bool
	Citations            bool
	LinkRetention        LinkRetention
	Chunk                ChunkConfig
	SelectCSS            string
	StripCSS             string
	MaxTokens            int
	HeadOnly             bool
	NoPruneFallback      bool
	Proxy                string
	CacheDir             string
	CacheTTL             time.Duration
	CacheStaleTolerance  time.Duration
	NoCache              bool
	NoPDF                bool
	SchemaPath           string
	Schema               *Schema
	Query                string
	TopN                 int
	FilterByQuery        bool
	SplitOut             string
	SplitBytes           int

	Transport http.RoundTripper

	Security *SecurityPolicy

	Context context.Context
}

type OutputFormat added in v0.2.0 

type OutputFormat string
const (
	OutputJSON      OutputFormat = "json"
	OutputMarkdown  OutputFormat = "md"
	OutputDirectory OutputFormat = "directory"
)

type PageClass 

type PageClass string
const (
	PageStatic   PageClass = "static"
	PageSSR      PageClass = "ssr"
	PageHydrated PageClass = "hydrated"
	PageSPA      PageClass = "spa"
	PageDynamic  PageClass = "dynamic"
	PageBlocked  PageClass = "blocked"
)

type PageGroup added in v0.2.0 

type PageGroup struct {
	Pattern        string       `json:"pattern"`
	TotalInSitemap int          `json:"totalInSitemap"`
	Sampled        int          `json:"sampled"`
	Pages          []PageObject `json:"pages"`
}

type PageObject added in v0.2.0 

type PageObject struct {
	URL           string            `json:"url"`
	Title         string            `json:"title"`
	Status        int               `json:"status"`
	Meta          map[string]string `json:"meta,omitempty"`
	Markdown      string            `json:"markdown"`
	Schema        []map[string]any  `json:"schema,omitempty"`
	Performance   *PagePerformance  `json:"performance,omitempty"`
	ContentType   string            `json:"contentType"`
	InternalLinks int               `json:"internalLinks"`
	ExternalLinks int               `json:"externalLinks"`
	Error         string            `json:"error,omitempty"`
}

type PagePerformance added in v0.2.0 

type PagePerformance struct {
	TTFBMillis int64 `json:"ttfbMs"`
	TotalBytes int64 `json:"totalBytes"`
	Requests   int   `json:"requests"`
}

type PageProfile 

type PageProfile struct {
	Class              PageClass         `json:"class"`
	Outcome            ExtractionOutcome `json:"outcome"`
	Decision           BrowserDecision   `json:"decision"`
	BrowserRecommended bool              `json:"browserRecommended"`
	Reasons            []string          `json:"reasons"`
	Confidence         int               `json:"confidence"`
	Trustworthy        bool              `json:"trustworthy"`
}

func ClassifyPage 

func ClassifyPage(result Result) PageProfile

func (PageProfile) String 

func (p PageProfile) String() string

type ParseFeedOptions 

type ParseFeedOptions struct {
	MaxItems int
	Timeout  time.Duration
	Client   *http.Client
	Security *SecurityPolicy
}

type PatternGroup added in v0.2.0 

type PatternGroup struct {
	Pattern        string
	URLs           []string
	TotalInSitemap int
}

type QualityMetrics 

type QualityMetrics = quality.Metrics

func ComputeQuality 

func ComputeQuality(markdown string) QualityMetrics

type RankedSection 

type RankedSection struct {
	Score   float64 `json:"score"`
	Heading string  `json:"heading,omitempty"`
	Text    string  `json:"text"`
	Tokens  int     `json:"tokens"`
}

func RankSections 

func RankSections(content, query string, k1, b float64, topN int) []RankedSection

type ResponseHeaders added in v0.2.0 

type ResponseHeaders struct {
	ResponseETag         string `json:"responseEtag,omitempty"`
	ResponseLastModified string `json:"responseLastModified,omitempty"`

	ResponseContentEncoding string `json:"responseContentEncoding,omitempty"`
	ResponseServer          string `json:"responseServer,omitempty"`
	ResponseXForwardedFor   string `json:"responseXForwardedFor,omitempty"`

	RequestAcceptEncoding string `json:"requestAcceptEncoding,omitempty"`

	TTFBMs     int64 `json:"ttfbMs,omitempty"`
	DownloadMs int64 `json:"downloadMs,omitempty"`

	ResponseVia        string `json:"responseVia,omitempty"`
	ResponseConnection string `json:"responseConnection,omitempty"`
	ResponseAge        string `json:"responseAge,omitempty"`

	ResponseCacheControl string `json:"responseCacheControl,omitempty"`
	ResponseXCache       string `json:"responseXCache,omitempty"`
	ResponseVary         string `json:"responseVary,omitempty"`

	ResponseXCacheHits       string `json:"responseXCacheHits,omitempty"`
	ResponseSurrogateControl string `json:"responseSurrogateControl,omitempty"`
	ResponseCFCacheStatus    string `json:"responseCfCacheStatus,omitempty"`

	ResponseXServedBy        string `json:"responseXServedBy,omitempty"`
	ResponseXFastlyRequestID string `json:"responseXFastlyRequestId,omitempty"`

	ResponseXAkamaiTransformed string `json:"responseXAkamaiTransformed,omitempty"`
	ResponseXAkamaiSessionInfo string `json:"responseXAkamaiSessionInfo,omitempty"`
	ResponseXAkamaiRequestID   string `json:"responseXAkamaiRequestId,omitempty"`

	ResponseXRequestId     string `json:"responseXRequestId,omitempty"`
	ResponseXCorrelationId string `json:"responseXCorrelationId,omitempty"`

	ResponseXVarnish string `json:"responseXVarnish,omitempty"`

	ResponseXCDN string `json:"responseXCdn,omitempty"`

	ResponseXTraceId        string `json:"responseXTraceId,omitempty"`
	ResponseXB3TraceId      string `json:"responseXB3TraceId,omitempty"`
	ResponseXB3SpanId       string `json:"responseXB3SpanId,omitempty"`
	ResponseXB3ParentSpanId string `json:"responseXB3ParentSpanId,omitempty"`
	ResponseXB3Sampled      string `json:"responseXB3Sampled,omitempty"`
	ResponseB3              string `json:"responseB3,omitempty"`
	ResponseTraceparent     string `json:"responseTraceparent,omitempty"`
	ResponseTracestate      string `json:"responseTracestate,omitempty"`
	ResponseXAmznTraceId    string `json:"responseXAmznTraceId,omitempty"`

	TraceInfo

	ResponseNEL string `json:"responseNel,omitempty"`

	ResponseReportTo string `json:"responseReportTo,omitempty"`

	ResponsePermissionsPolicy  string `json:"responsePermissionsPolicy,omitempty"`
	ResponseExpectCT           string `json:"responseExpectCt,omitempty"`
	ResponseFeaturePolicy      string `json:"responseFeaturePolicy,omitempty"`
	ResponseReportingEndpoints string `json:"responseReportingEndpoints,omitempty"`
	ResponseCSP                string `json:"responseCsp,omitempty"`
	ResponseCSPReportOnly      string `json:"responseCspReportOnly,omitempty"`

	ResponseCORP string `json:"responseCorp,omitempty"`
	ResponseCOEP string `json:"responseCoep,omitempty"`
	ResponseCOOP string `json:"responseCoop,omitempty"`

	ResponseHSTS string `json:"responseHsts,omitempty"`

	ResponseXContentTypeOptions string `json:"responseXContentTypeOptions,omitempty"`
	ResponseXFrameOptions       string `json:"responseXFrameOptions,omitempty"`

	ResponseReferrerPolicy string `json:"responseReferrerPolicy,omitempty"`

	ResponseXXSSProtection                string `json:"responseXXssProtection,omitempty"`
	ResponseXPermittedCrossDomainPolicies string `json:"responseXPermittedCrossDomainPolicies,omitempty"`
	ResponseXDownloadOptions              string `json:"responseXDownloadOptions,omitempty"`

	ResponseClearSiteData string `json:"responseClearSiteData,omitempty"`

	ResponseTimingAllowOrigin string `json:"responseTimingAllowOrigin,omitempty"`

	ResponseOriginAgentCluster string `json:"responseOriginAgentCluster,omitempty"`

	ResponseDocumentPolicy string `json:"responseDocumentPolicy,omitempty"`

	ResponseAcceptCH string `json:"responseAcceptCH,omitempty"`

	ResponseSecCHUA                 string `json:"responseSecChUa,omitempty"`
	ResponseSecCHUAMobile           string `json:"responseSecChUaMobile,omitempty"`
	ResponseSecCHUAPlatform         string `json:"responseSecChUaPlatform,omitempty"`
	ResponseSecCHUAFullVersionList  string `json:"responseSecChUaFullVersionList,omitempty"`
	ResponseSecCHPrefersColorScheme string `json:"responseSecChPrefersColorScheme,omitempty"`

	ResponseCriticalCH string `json:"responseCriticalCh,omitempty"`

	ResponseCOEPReportOnly string `json:"responseCoepReportOnly,omitempty"`
	ResponseCOOPReportOnly string `json:"responseCoopReportOnly,omitempty"`

	ResponseDocumentPolicyReportOnly string `json:"responseDocumentPolicyReportOnly,omitempty"`

	ResponseSourceMap string `json:"responseSourceMap,omitempty"`

	ResponseAccessControlAllowOrigin      string `json:"responseAccessControlAllowOrigin,omitempty"`
	ResponseAccessControlAllowMethods     string `json:"responseAccessControlAllowMethods,omitempty"`
	ResponseAccessControlAllowHeaders     string `json:"responseAccessControlAllowHeaders,omitempty"`
	ResponseAccessControlAllowCredentials string `json:"responseAccessControlAllowCredentials,omitempty"`
	ResponseAccessControlExposeHeaders    string `json:"responseAccessControlExposeHeaders,omitempty"`
	ResponseAccessControlMaxAge           string `json:"responseAccessControlMaxAge,omitempty"`

	ResponseLink string `json:"responseLink,omitempty"`

	LLMsTxtURL string `json:"llmsTxtUrl,omitempty"`

	LDJSONBlocks []LDJSONBlock `json:"ldJsonBlocks,omitempty"`

	HasLLMContent bool `json:"hasLlmContent,omitempty"`

	ResponseXRobotsTag string `json:"responseXRobotsTag,omitempty"`

	ResponseContentDisposition string `json:"responseContentDisposition,omitempty"`

	ResponseXContentDuration string `json:"responseXContentDuration,omitempty"`

	ResponseRefresh string `json:"responseRefresh,omitempty"`

	ResponseContentLanguage string `json:"responseContentLanguage,omitempty"`

	ResponseXUACompatible string `json:"responseXUaCompatible,omitempty"`

	ResponseAcceptRanges string `json:"responseAcceptRanges,omitempty"`

	ResponseTransferEncoding string `json:"responseTransferEncoding,omitempty"`

	ResponseContentRange string `json:"responseContentRange,omitempty"`

	ResponsePragma string `json:"responsePragma,omitempty"`

	ResponseXPoweredBy string `json:"responseXPoweredBy,omitempty"`

	ResponseXAspNetVersion string `json:"responseXAspNetVersion,omitempty"`

	ResponseXAspNetMvcVersion string `json:"responseXAspNetMvcVersion,omitempty"`

	ResponseServerTiming string `json:"responseServerTiming,omitempty"`

	ResponseXGenerator string `json:"responseXGenerator,omitempty"`

	ResponseXRuntime string `json:"responseXRuntime,omitempty"`

	ResponseXDrupalCache string `json:"responseXDrupalCache,omitempty"`

	ResponseXMagentoCacheControl string `json:"responseXMagentoCacheControl,omitempty"`

	ResponseXDrupalDynamicCache string `json:"responseXDrupalDynamicCache,omitempty"`

	ResponseXMagentoTags string `json:"responseXMagentoTags,omitempty"`

	ResponseXShopifyStage string `json:"responseXShopifyStage,omitempty"`

	ResponseXShopifyRequestID string `json:"responseXShopifyRequestId,omitempty"`

	ResponseXWPTotal      string `json:"responseXWpTotal,omitempty"`
	ResponseXWPTotalPages string `json:"responseXWpTotalPages,omitempty"`

	ResponseXCraftCache string `json:"responseXCraftCache,omitempty"`

	ResponseXDiscourseRoute string `json:"responseXDiscourseRoute,omitempty"`

	ResponseXGhostCacheStatus string `json:"responseXGhostCacheStatus,omitempty"`

	ResponseXJoomlaCache string `json:"responseXJoomlaCache,omitempty"`

	ResponseXDiscourseMediaType string `json:"responseXDiscourseMediaType,omitempty"`

	ResponseXPrestaShopCache string `json:"responseXPrestaShopCache,omitempty"`

	ResponseXMagentoCacheDebug string `json:"responseXMagentoCacheDebug,omitempty"`

	ResponseXTypo3Cache string `json:"responseXTypo3Cache,omitempty"`

	ResponseXWixRequestId string `json:"responseXWixRequestId,omitempty"`

	ResponseXSquarespaceRequestId string `json:"responseXSquarespaceRequestId,omitempty"`

	ResponseXWebflowRequestId string `json:"responseXWebflowRequestId,omitempty"`

	ResponseXContentfulRequestId string `json:"responseXContentfulRequestId,omitempty"`

	ResponseXNetlifyRequestId string `json:"responseXNetlifyRequestId,omitempty"`

	ResponseXVercelId string `json:"responseXVercelId,omitempty"`

	ResponseXHerokuRequestId string `json:"responseXHerokuRequestId,omitempty"`

	ResponseXRenderRequestId string `json:"responseXRenderRequestId,omitempty"`

	ResponseXRailwayRequestId string `json:"responseXRailwayRequestId,omitempty"`

	ResponseXFlyRequestId string `json:"responseXFlyRequestId,omitempty"`

	ResponseXDenoRegion string `json:"responseXDenoRegion,omitempty"`

	ResponseXCloudflareWorkersRequestId string `json:"responseXCloudflareWorkersRequestId,omitempty"`

	ResponseXAzureRef string `json:"responseXAzureRef,omitempty"`

	ResponseXGCPRegion string `json:"responseXGcpRegion,omitempty"`

	ResponseXAmzCfId string `json:"responseXAmzCfId,omitempty"`
}

type Result 

type Result struct {
	URL              string   `json:"url"`
	CanonicalURL     string   `json:"canonicalUrl,omitempty"`
	Title            string   `json:"title"`
	Content          string   `json:"content"`
	Byline           string   `json:"byline"`
	PublishedDate    string   `json:"publishedDate,omitempty"`
	Language         string   `json:"language,omitempty"`
	Charset          string   `json:"charset,omitempty"`
	Section          string   `json:"section,omitempty"`
	Description      string   `json:"description,omitempty"`
	ImageURL         string   `json:"imageUrl,omitempty"`
	Excerpt          string   `json:"excerpt"`
	SiteName         string   `json:"sitename"`
	Length           int      `json:"length"`
	TimeMs           int64    `json:"timeMs"`
	Confidence       int      `json:"confidence"`
	IsSPA            bool     `json:"isSpa"`
	IsBlocked        bool     `json:"isBlocked"`
	SPASignals       []string `json:"spaSignals,omitempty"`
	Error            string   `json:"error,omitempty"`
	FromCache        bool     `json:"fromCache,omitempty"`
	CacheHit         bool     `json:"cacheHit,omitempty"`
	CacheRevalidated bool     `json:"cacheRevalidated,omitempty"`
	CacheStale       bool     `json:"cacheStale,omitempty"`
	StatusCode       int      `json:"statusCode,omitempty"`
	HeadOnly         bool     `json:"headOnly,omitempty"`
	Protocol         string   `json:"protocol,omitempty"`

	BlockedByRobots bool `json:"blockedByRobots,omitempty"`

	HeadingCount   int `json:"headingCount"`
	LinkCount      int `json:"linkCount"`
	ParagraphCount int `json:"paragraphCount"`

	Quality     float64        `json:"quality"`
	QualityInfo QualityMetrics `json:"qualityInfo"`
	Profile     PageProfile    `json:"profile"`
	PageClass   PageClass      `json:"pageClass"`
	Fingerprint string         `json:"fingerprint"`
	Validation  Validation     `json:"validation"`

	TransportInfo

	IsSoft404    bool     `json:"isSoft404,omitempty"`
	Soft404Hints []string `json:"soft404Hints,omitempty"`

	ResponseHeaders

	NormalizedCacheStatus string `json:"normalizedCacheStatus,omitempty"`
	CacheStatusSource     string `json:"cacheStatusSource,omitempty"`

	CDNInfo

	CacheAnalysis

	DedupeStats

	Links    []LinkRef    `json:"links,omitempty"`
	Images   []ImageRef   `json:"images,omitempty"`
	Tables   []TableRef   `json:"tables,omitempty"`
	Comments []CommentRef `json:"comments,omitempty"`
	Chunks   []Chunk      `json:"chunks,omitempty"`

	SplitFiles []SplitFile `json:"splitFiles,omitempty"`

	RankedSections []RankedSection `json:"rankedSections,omitempty"`

	Schema map[string]interface{} `json:"schema,omitempty"`

	Warnings []string `json:"warnings,omitempty"`

	Truncated bool `json:"truncated,omitempty"`

	PruneFallbackUsed bool `json:"pruneFallbackUsed,omitempty"`

	ExtractionMethod string `json:"extractionMethod,omitempty"`

	SecurityBlock string `json:"securityBlock,omitempty"`
	// contains filtered or unexported fields
}

func FromHTML 

func FromHTML(html string, targetURL string) Result

func FromHTMLWithOptions 

func FromHTMLWithOptions(html string, targetURL string, opts Options) Result

func FromResponse 

func FromResponse(resp *http.Response, targetURL string, start time.Time) (result Result)

func FromURL 

func FromURL(targetURL string) Result

func FromURLContext added in v0.2.0 

func FromURLContext(ctx context.Context, targetURL string, opts Options) Result

func FromURLWithDedupe 

func FromURLWithDedupe(targetURL string) Result

func FromURLWithOptions 

func FromURLWithOptions(targetURL string, opts Options) Result

func (*Result) Err added in v0.2.0 

func (r *Result) Err() error

type RetryEvent 

type RetryEvent struct {
	Attempt    int
	StatusCode int
	WaitTime   time.Duration
	Error      error
	Outcome    string
}

type SampleStrategy added in v0.2.0 

type SampleStrategy string
const (
	SampleBalanced SampleStrategy = "balanced"
	SampleRandom   SampleStrategy = "random"
	SamplePriority SampleStrategy = "priority"
)

type Schema 

type Schema struct {
	Fields map[string]FieldSpec `json:"fields" yaml:"fields"`
}

func LoadSchema 

func LoadSchema(path string) (Schema, error)

type ScrapeOptions added in v0.2.0 

type ScrapeOptions struct {
	BaseURL         string
	MaxPages        int
	MaxPerPattern   int
	Full            bool
	IncludePatterns []string
	ExcludePatterns []string
	SampleStrategy  SampleStrategy
	Output          OutputFormat
	WithPerformance bool
	RespectRobots   *bool
	Timeout         time.Duration
	UserAgent       string

	Security *SecurityPolicy

	Since time.Time
}

type ScrapeResult added in v0.2.0 

type ScrapeResult struct {
	Site       SiteInfo      `json:"site"`
	PageGroups []PageGroup   `json:"pageGroups"`
	Pages      []PageObject  `json:"pages"`
	Summary    ScrapeSummary `json:"summary"`
}

func ScrapeSite added in v0.2.0 

func ScrapeSite(ctx context.Context, opts *ScrapeOptions) (*ScrapeResult, error)

type ScrapeSummary added in v0.2.0 

type ScrapeSummary struct {
	ContentTypes      map[string]int `json:"contentTypes"`
	Recommendations   []string       `json:"recommendations"`
	UnsampledPatterns int            `json:"unsampledPatterns,omitempty"`
	Warnings          []string       `json:"warnings,omitempty"`
}

type SecurityPolicy 

type SecurityPolicy struct {
	BlockPrivateIPs bool

	TrustedResolveCIDRs []string

	AllowedDomains []string
	DeniedDomains  []string

	AllowedSchemes []string

	MaxRedirects int

	RevalidateRedirects bool

	MaxResponseBytes     int64
	MaxDecompressedBytes int64

	Resolver IPResolver

	URLFilter func(ctx context.Context, rawURL string) error `json:"-"`
}

func DefaultSecurityPolicy 

func DefaultSecurityPolicy() *SecurityPolicy

func (*SecurityPolicy) ValidateURL 

func (p *SecurityPolicy) ValidateURL(ctx context.Context, rawURL string) error

type SiteInfo added in v0.2.0 

type SiteInfo struct {
	BaseURL            string    `json:"baseURL"`
	Title              string    `json:"title"`
	DiscoveredAt       time.Time `json:"discoveredAt"`
	SitemapFound       bool      `json:"sitemapFound"`
	TotalURLsInSitemap int       `json:"totalURLsInSitemap"`
	SampledPages       int       `json:"sampledPages"`
}

type SitemapEntry 

type SitemapEntry struct {
	Loc        string `json:"loc"`
	LastMod    string `json:"lastmod,omitempty"`
	ChangeFreq string `json:"changefreq,omitempty"`
	Priority   string `json:"priority,omitempty"`
}

func FlattenSitemap 

func FlattenSitemap(ctx context.Context, sitemapURL string, opts FlattenSitemapOptions) ([]SitemapEntry, error)

type SnapshotNode 

type SnapshotNode struct {
	Role        string         `json:"role"`
	Name        string         `json:"name,omitempty"`
	Tag         string         `json:"tag,omitempty"`
	Ref         string         `json:"ref,omitempty"`
	Selector    string         `json:"selector,omitempty"`
	Depth       int            `json:"depth,omitempty"`
	Interactive bool           `json:"interactive,omitempty"`
	Level       int            `json:"level,omitempty"`
	Value       string         `json:"value,omitempty"`
	Href        string         `json:"href,omitempty"`
	Checked     *bool          `json:"checked,omitempty"`
	Disabled    bool           `json:"disabled,omitempty"`
	Children    []SnapshotNode `json:"children,omitempty"`
}

func BuildSnapshot 

func BuildSnapshot(htmlStr string) (*SnapshotNode, error)

func BuildSnapshotWithOptions 

func BuildSnapshotWithOptions(htmlStr string, opts SnapshotOptions) (*SnapshotNode, error)

func (*SnapshotNode) ToCompact 

func (n *SnapshotNode) ToCompact() string

type SnapshotOptions 

type SnapshotOptions struct {
	FilterInteractive bool
	MaxTokens         int
}

type SplitConfig 

type SplitConfig struct {
	Dir      string
	MaxBytes int
	BaseName string
	Format   string
}

type SplitFile 

type SplitFile struct {
	Path  string `json:"path"`
	Index int    `json:"index"`
	Of    int    `json:"of"`
	Bytes int    `json:"bytes"`
}

func SplitResultToFiles 

func SplitResultToFiles(r Result, cfg SplitConfig) ([]SplitFile, error)

type TableKind 

type TableKind int
const (
	TableData TableKind = iota
	TableLayout
)

type TableRef 

type TableRef struct {
	Caption string     `json:"caption,omitempty"`
	Headers []string   `json:"headers,omitempty"`
	Rows    [][]string `json:"rows"`
}

func ExtractTables 

func ExtractTables(htmlStr string, _ string) []TableRef

type TextFallbackResult 

type TextFallbackResult struct {
	Content  string
	Length   int
	Headings int
	Links    int
}

func TextFallback 

func TextFallback(htmlStr string) TextFallbackResult

type TraceInfo added in v0.2.0 

type TraceInfo struct {
	TraceFormats     []string `json:"traceFormats,omitempty"`
	TraceCorrelation string   `json:"traceCorrelation,omitempty"`
}

type TransportInfo added in v0.2.0 

type TransportInfo struct {
	RetryCount          int           `json:"retryCount,omitempty"`
	TotalRetryWait      time.Duration `json:"totalRetryWait,omitempty"`
	HeadPreflightStatus int           `json:"headPreflightStatus,omitempty"`

	FetchTimeMs   int64 `json:"fetchTimeMs,omitempty"`
	ParseTimeMs   int64 `json:"parseTimeMs,omitempty"`
	ConvertTimeMs int64 `json:"convertTimeMs,omitempty"`

	ContentLength       int64  `json:"contentLength,omitempty"`
	ResponseContentType string `json:"responseContentType,omitempty"`

	RedirectCount int      `json:"redirectCount,omitempty"`
	RedirectChain []string `json:"redirectChain,omitempty"`
	FinalURL      string   `json:"finalUrl,omitempty"`
}

type Validation 

type Validation struct {
	IsValid       bool     `json:"isValid"`
	NeedsBrowser  bool     `json:"needsBrowser"`
	Confidence    float64  `json:"confidence"`
	Issues        []string `json:"issues"`
	LinkDensity   float64  `json:"linkDensity"`
	SkeletonScore float64  `json:"skeletonScore"`
	ContentRatio  float64  `json:"contentRatio"`
}

func ValidateExtraction 

func ValidateExtraction(r *Result) Validation

type ViaHop 

type ViaHop struct {
	Protocol string `json:"protocol,omitempty"`
	Host     string `json:"host,omitempty"`
	Comment  string `json:"comment,omitempty"`
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.