storage

package

Go to main page

Versions in this module

v0

v0.12.7

May 27, 2026

v0.12.6

May 19, 2026

v0.12.5

May 18, 2026

v0.12.4

May 1, 2026

v0.12.3

Apr 3, 2026

Changes in this version

+ const AlterPagesV7StructuredData

+ const AlterPagesV8CWV

+ const AlterSessionsV3

+ const CreateHreflangIssues

+ const CreateStructuredDataItems

+ var CriticalTables = []string

+ var HreflangIssueFilters = map[string]FilterDef

+ var HreflangIssueSortColumns = map[string]string

type Buffer

+ func (b *Buffer) AddStructuredData(items []schema.StructuredDataItem)

type CrawlSession

+ Label string

+ type HreflangIssue struct

+ Detail string

+ IssueType string

+ SourceLang string

+ SourceURL string

+ TargetLang string

+ TargetURL string

+ type HreflangValidationResult struct

+ Issues []HreflangIssue

+ Summary map[string]uint64

+ Total uint64

type PageLinkInserter

+ InsertStructuredData func(ctx context.Context, items []schema.StructuredDataItem) error

type PageRow

+ CWVCLS float64

+ CWVLCP float64

+ CWVMeasured bool

+ CWVTTFB float64

+ SchemaErrorCount uint16

+ SchemaValidCount uint16

+ SchemaWarningCount uint16

type Store

+ func (s *Store) ComputeHreflangValidation(ctx context.Context, sessionID string) error

+ func (s *Store) Conn() driver.Conn

+ func (s *Store) ExportCriticalTables(ctx context.Context, dir string, retain int) error

+ func (s *Store) GetStructuredData(ctx context.Context, sessionID, url string) ([]schema.StructuredDataItem, error)

+ func (s *Store) HreflangValidation(ctx context.Context, sessionID string, issueType string, pageURL string, ...) (*HreflangValidationResult, error)

+ func (s *Store) ImportCriticalTable(ctx context.Context, table string, r io.Reader) error

+ func (s *Store) InsertStructuredData(ctx context.Context, items []schema.StructuredDataItem) error

+ func (s *Store) URLDirectories(ctx context.Context, sessionID string, depth, minPages int) ([]URLDirectory, error)

+ func (s *Store) URLHosts(ctx context.Context, sessionID string) ([]URLHost, error)

+ func (s *Store) URLParams(ctx context.Context, sessionID string, limit int) ([]URLParam, error)

+ func (s *Store) URLPatterns(ctx context.Context, sessionID string, depth int) ([]URLPattern, error)

+ func (s *Store) UpdateSessionLabel(ctx context.Context, sessionID, label string) error

+ type URLDirectory struct

+ AvgPR float64

+ Errors uint64

+ Indexable uint64

+ Path string

+ Total uint64

+ WithParams uint64

+ type URLHost struct

+ AvgPR float64

+ Errors uint64

+ Host string

+ Indexable uint64

+ Status200 uint64

+ Total uint64

+ type URLParam struct

+ Indexable uint64

+ NonIndexable uint64

+ Occurrences uint64

+ Param string

+ UniqueURLs uint64

+ type URLPattern struct

+ AvgPageRank float64

+ Indexable uint64

+ NonIndexable uint64

+ Pattern string

+ Status200 uint64

+ Status3xx uint64

+ Status4xx uint64

+ StatusOther uint64

+ Total uint64

+ WithParams uint64

v0.12.2

Mar 11, 2026

v0.12.1

Mar 11, 2026

v0.12.0

Mar 10, 2026

Changes in this version

+ const AlterInterlinkingOpportunitiesV2

+ const CreateInterlinkingOpportunities

+ const CreateInterlinkingSimulationResults

+ const CreateInterlinkingSimulations

+ const CreatePageEmbeddings

+ var InterlinkingFilters = map[string]FilterDef

+ var InterlinkingSortColumns = map[string]string

+ var SimulationResultFilters = map[string]FilterDef

+ var SimulationResultSortColumns = map[string]string

type FilterType

+ const FilterFloat

+ type InterlinkingOpportunity struct

+ Category string

+ CrawlSessionID string

+ Method string

+ OpportunityScore float64

+ Similarity float64

+ SourcePageRank float64

+ SourceTitle string

+ SourceURL string

+ SourceWordCount uint32

+ TargetPageRank float64

+ TargetTitle string

+ TargetURL string

+ TargetWordCount uint32

+ type PageMetadata struct

+ Canonical string

+ CanonicalSelf bool

+ Lang string

+ PageRank float64

+ Title string

+ WordCount uint32

+ type PageRankGraph struct

+ IDToURL []string

+ N uint32

+ OutLinks [][]uint32

+ TotalOutLinks []uint32

+ URLToID map[string]uint32

+ type SimulationMeta struct

+ AvgDiff float64

+ ComputedAt time.Time

+ CrawlSessionID string

+ ID string

+ MaxDiff float64

+ PagesDeclined uint32

+ PagesImproved uint32

+ VirtualLinksCount uint32

+ type SimulationResultRow struct

+ PageRankAfter float64

+ PageRankBefore float64

+ PageRankDiff float64

+ URL string

type Store

+ func (s *Store) DeleteInterlinkingOpportunities(ctx context.Context, sessionID string) error

+ func (s *Store) GetSimulation(ctx context.Context, sessionID, simID string) (*SimulationMeta, error)

+ func (s *Store) InsertInterlinkingOpportunities(ctx context.Context, sessionID string, opps []InterlinkingOpportunity) error

+ func (s *Store) InsertSimulation(ctx context.Context, sessionID string, simID string, ...) error

+ func (s *Store) ListInterlinkingOpportunities(ctx context.Context, sessionID string, limit, offset int, ...) ([]InterlinkingOpportunity, int, error)

+ func (s *Store) ListSimulationResults(ctx context.Context, sessionID, simID string, limit, offset int, ...) ([]SimulationResultRow, int, error)

+ func (s *Store) ListSimulations(ctx context.Context, sessionID string) ([]SimulationMeta, error)

+ func (s *Store) LoadInternalLinkSet(ctx context.Context, sessionID string) (map[[2]string]struct{}, error)

+ func (s *Store) LoadPageMetadata(ctx context.Context, sessionID string) (map[string]PageMetadata, error)

+ func (s *Store) LoadPageRankGraph(ctx context.Context, sessionID string) (*PageRankGraph, error)

+ type VirtualLink struct

+ SourceURL string

+ TargetURL string

v0.11.1

Mar 10, 2026

Changes in this version

+ func ComputePageRankIterations(n uint32, outLinks [][]uint32, totalOutLinks []uint32) []float64

v0.11.0

Mar 10, 2026

v0.10.3

Mar 10, 2026

Changes in this version

+ const AlterExternalLinkChecksNS

type ExternalDomainCheck

+ NSDead uint64

type ExternalLinkCheck

+ NSError string

+ NSExists bool

type SessionStats

+ UniqueExtDomains uint64

v0.10.2

Mar 9, 2026

Changes in this version

+ const CreateNearDuplicatePairs

type Store

+ func (s *Store) ComputeNearDuplicates(ctx context.Context, sessionID string) error

+ func (s *Store) UpdateContentHashes(ctx context.Context, sessionID string, hashes map[string]uint64) error

v0.10.1

Mar 9, 2026

Changes in this version

+ var ExternalCheckSortColumns = map[string]string

+ var ExternalDomainCheckHavingFilters = map[string]FilterDef

+ var ExternalDomainCheckSortColumns = map[string]string

+ type ExternalLinkCheckWithSource struct

+ SourceDepth uint16

+ SourcePageRank float64

+ SourceURL string

v0.10.0

Mar 7, 2026

v0.9.1

Mar 7, 2026

v0.9.0

Mar 7, 2026

Changes in this version

+ const CreateRetryAttempts

+ type CSVImportResult struct

+ RowsImported int

+ RowsSkipped int

+ Session *CrawlSession

+ type CSVSource string

+ const CSVSourceAddressBased

+ const CSVSourceURLBased

+ type StatusTimelineBucket struct

+ ClientErr uint64

+ FetchErr uint64

+ OK uint64

+ Redirect uint64

+ Retried403 uint64

+ Retried429 uint64

+ Retried5xx uint64

+ ServerErr uint64

+ Status403 uint64

+ Status429 uint64

+ Timestamp time.Time

+ Total uint64

type Store

+ func (s *Store) ImportCSVSession(ctx context.Context, r io.Reader, projectID string) (*CSVImportResult, error)

+ func (s *Store) InsertRetryAttempt(ctx context.Context, sessionID string, attemptedAt time.Time, statusCode int, ...) error

+ func (s *Store) StatusTimeline(ctx context.Context, sessionID string) ([]StatusTimelineBucket, error)

+ func (s *Store) StatusTimelineRecent(ctx context.Context, sessionID string) ([]StatusTimelineBucket, error)

+ func (s *Store) StreamCrawledURLs(ctx context.Context, sessionID string, fn func(string)) (int, error)

v0.8.3

Mar 6, 2026

v0.8.2

Mar 5, 2026

v0.8.1

Mar 5, 2026

v0.8.0

Mar 5, 2026

v0.7.1

Mar 4, 2026

v0.7.0

Mar 4, 2026

Changes in this version

+ var BacklinkFilters = map[string]FilterDef

+ var BacklinkSortColumns = map[string]string

type ProviderBacklinkRow

+ CitationFlow float64

+ SourceTTFTopic string

+ TrustFlow float64

type WeightedPageRankPage

+ TTFTopic *string

v0.6.1

Mar 4, 2026

Changes in this version

type Store

+ func (s *Store) WeightedPageRankTop(ctx context.Context, sessionID, projectID string, limit, offset int, ...) (*WeightedPageRankResult, error)

+ type WeightedPageRankPage struct

+ CitationFlow *uint8

+ Depth uint16

+ ExtBackLinks *int64

+ InternalLinksOut uint32

+ PageRank float64

+ RefDomains *int64

+ StatusCode uint16

+ Title string

+ TrustFlow *uint8

+ URL string

+ WeightedPR float64

+ type WeightedPageRankResult struct

+ Pages []WeightedPageRankPage

+ Total uint64

v0.6.0

Mar 4, 2026

Changes in this version

+ const CreateProviderData

+ var ProviderDataFilters = map[string]FilterDef

+ var ProviderDataSortColumns = map[string]string

+ type ProviderDataRow struct

+ CitationFlow uint8

+ DataType string

+ Domain string

+ DomainRank float64

+ ExtBacklinks int64

+ FetchedAt time.Time

+ ItemURL string

+ NumData map[string]float64

+ Provider string

+ RefDomains int64

+ StrData map[string]string

+ TrustFlow uint8

type Store

+ func (s *Store) InsertProviderData(ctx context.Context, projectID string, rows []ProviderDataRow) error

+ func (s *Store) ProviderData(ctx context.Context, projectID, provider, dataType string, limit, offset int, ...) ([]ProviderDataRow, int, error)

+ func (s *Store) ProviderDataAge(ctx context.Context, projectID, provider, dataType string) (time.Time, error)

v0.5.1

Mar 4, 2026

v0.5.0

Mar 3, 2026

v0.4.0

Mar 3, 2026

Changes in this version

type Store

+ func (s *Store) GetSitemapCoverageURLs(ctx context.Context, sessionID, filter string, limit, offset int) ([]SitemapURLRow, error)

v0.3.0

Mar 3, 2026

Changes in this version

+ const CreateExtractions

type Buffer

+ func (b *Buffer) AddExtractions(rows []extraction.ExtractionRow)

+ type ExtractionQueryRow struct

+ ExtractorName string

+ URL string

+ Value string

type PageLinkInserter

+ InsertExtractions func(ctx context.Context, rows []extraction.ExtractionRow) error

type Store

+ func (s *Store) DeleteExtractions(ctx context.Context, sessionID string) error

+ func (s *Store) GetExtractions(ctx context.Context, sessionID string, limit, offset int) (*extraction.ExtractionResult, error)

+ func (s *Store) HasStoredHTML(ctx context.Context, sessionID string) (bool, error)

+ func (s *Store) InsertExtractions(ctx context.Context, rows []extraction.ExtractionRow) error

+ func (s *Store) RunExtractionsPostCrawl(ctx context.Context, sessionID string, extractors []extraction.Extractor) (*extraction.ExtractionResult, error)

+ func (s *Store) StreamPagesHTMLForExtraction(ctx context.Context, sessionID string) (<-chan PageHTMLRow, error)

v0.2.5

Mar 3, 2026

v0.2.4

Mar 3, 2026

v0.2.3

Mar 3, 2026

v0.2.2

Mar 3, 2026

v0.2.1

Mar 2, 2026

Changes in this version

+ var RedirectFilters = map[string]FilterDef

+ var RedirectSortColumns = map[string]string

+ type RedirectPageRow struct

+ FinalURL string

+ InboundInternalLinks uint64

+ StatusCode uint16

+ URL string

type Store

+ func (s *Store) ListRedirectPages(ctx context.Context, sessionID string, limit, offset int, ...) ([]RedirectPageRow, error)

v0.2.0

Mar 2, 2026

Changes in this version

+ var LinkSortColumns = map[string]string

+ var PageSortColumns = map[string]string

+ func BuildOrderByClause(sort *SortParam, defaultOrderBy string) string

+ type SortParam struct

+ Column string

+ Order string

+ func ParseSort(sortKey, orderStr string, whitelist map[string]string) *SortParam

v0.1.1

Mar 2, 2026

Changes in this version

+ const AlterPagesV2

+ const AlterPagesV3

+ const AlterPagesV4

+ const AlterPagesV5

+ const AlterPagesV6

+ const AlterSessionsV2

+ const CreateApplicationLogs

+ const CreateCrawlSessions

+ const CreateDatabase

+ const CreateExternalLinkChecks

+ const CreateGSCAnalytics

+ const CreateGSCInspection

+ const CreateLinks

+ const CreateLinksV2

+ const CreatePageResourceChecks

+ const CreatePageResourceRefs

+ const CreatePages

+ const CreatePagesV2

+ const CreateProviderAPICalls

+ const CreateProviderBacklinks

+ const CreateProviderDomainMetrics

+ const CreateProviderRankings

+ const CreateProviderRefDomains

+ const CreateProviderTopPages

+ const CreateProviderVisibility

+ const CreateRobotsTxt

+ const CreateRobotsTxtV2

+ const CreateSitemapURLs

+ const CreateSitemapURLsV2

+ const CreateSitemaps

+ const CreateSitemapsV2

+ const ExportFormatVersion

+ const RecordLink

+ const RecordMeta

+ const RecordPage

+ const RecordRobots

+ const RecordSitemap

+ const RecordSitemapURL

+ var ExternalCheckFilters = map[string]FilterDef

+ var ExternalDomainCheckFilters = map[string]FilterDef

+ var LinkFilters = map[string]FilterDef

+ var Migrations = []Migration

+ var PageFilters = map[string]FilterDef

+ var PageResourceCheckFilters = map[string]FilterDef

+ func BuildWhereClause(filters []ParsedFilter) (string, []interface{}, error)

+ type AnchorCount struct

+ Anchor string

+ Count uint64

+ type AuditContent struct

+ H1Missing uint64

+ H1Multiple uint64

+ HTMLPages uint64

+ ImagesNoAltTotal uint64

+ ImagesTotal uint64

+ MetaDescMissing uint64

+ MetaDescTooLong uint64

+ MetaDescTooShort uint64

+ PagesWithImagesNoAlt uint64

+ Thin100300 uint64

+ ThinUnder100 uint64

+ TitleDuplicates uint64

+ TitleMissing uint64

+ TitleTooLong uint64

+ TitleTooShort uint64

+ Total uint64

+ type AuditInternational struct

+ LangDistribution []LangCount

+ PagesWithHreflang uint64

+ PagesWithLang uint64

+ PagesWithSchema uint64

+ SchemaDistribution []SchemaCount

+ type AuditLinks struct

+ BrokenInternal uint64

+ ExternalDofollow uint64

+ ExternalNofollow uint64

+ PagesHighInternalOut uint64

+ PagesNoExternal uint64

+ PagesNoInternalOut uint64

+ TopAnchors []AnchorCount

+ TopExternalDomains []ExternalDomain

+ TotalExternal uint64

+ TotalInternal uint64

+ type AuditResult struct

+ Content *AuditContent

+ International *AuditInternational

+ Links *AuditLinks

+ Sitemaps *AuditSitemaps

+ Structure *AuditStructure

+ Technical *AuditTechnical

+ type AuditSitemaps struct

+ CrawledOnly uint64

+ InBoth uint64

+ SitemapOnly uint64

+ TotalSitemapURLs uint64

+ type AuditStructure struct

+ Directories []DirectoryCount

+ OrphanPages uint64

+ type AuditTechnical struct

+ CanonicalMissing uint64

+ CanonicalOther uint64

+ CanonicalSelf uint64

+ ContentTypes []ContentTypeCount

+ ErrorPages uint64

+ HasRedirect uint64

+ Indexable uint64

+ NoindexReasons []NoindexReason

+ NonIndexable uint64

+ RedirectChainsOver2 uint64

+ ResponseFast uint64

+ ResponseOK uint64

+ ResponseSlow uint64

+ ResponseVerySlow uint64

+ type BFSResult struct

+ Depths map[string]uint16

+ FoundOn map[string]string

+ func ComputeBFSDepths(seedURLs []string, crawledSet map[string]bool, adj map[string][]string) BFSResult

+ type Buffer struct

+ func NewBuffer(store PageLinkInserter, batchSize int, flushInterval time.Duration, ...) *Buffer

+ func (b *Buffer) AddLinks(links []LinkRow)

+ func (b *Buffer) AddPage(page PageRow)

+ func (b *Buffer) Close()

+ func (b *Buffer) ErrorState() BufferErrorState

+ func (b *Buffer) Flush()

+ func (b *Buffer) PageCount() int

+ func (b *Buffer) SetOnDataLost(fn func(lostPages, lostLinks int64))

+ type BufferErrorState struct

+ LastError error

+ LostLinks int64

+ LostPages int64

+ PendingLinks int

+ PendingPages int

+ type CompareStatsResult struct

+ SessionA string

+ SessionB string

+ StatsA *SessionStats

+ StatsB *SessionStats

+ type ContentTypeCount struct

+ ContentType string

+ Count uint64

+ type CrawlSession struct

+ Config string

+ FinishedAt time.Time

+ ID string

+ PagesCrawled uint64

+ ProjectID *string

+ SeedURLs []string

+ StartedAt time.Time

+ Status string

+ UserAgent string

+ type DirectoryCount struct

+ Count uint64

+ Directory string

+ type ExpiredDomain struct

+ DeadURLsChecked uint64

+ RegistrableDomain string

+ Sources []ExpiredDomainSource

+ type ExpiredDomainSource struct

+ SourceURL string

+ TargetURL string

+ type ExpiredDomainsResult struct

+ Domains []ExpiredDomain

+ Total uint64

+ type ExternalDomain struct

+ Count uint64

+ Domain string

+ type ExternalDomainCheck struct

+ AvgResponseMs uint32

+ ClientErrors uint64

+ Domain string

+ OK uint64

+ Redirects uint64

+ ServerErrors uint64

+ TotalURLs uint64

+ Unreachable uint64

+ type ExternalLinkCheck struct

+ CheckedAt time.Time

+ ContentType string

+ CrawlSessionID string

+ Error string

+ RedirectURL string

+ ResponseTimeMs uint32

+ StatusCode uint16

+ URL string

+ type FilterDef struct

+ Column string

+ Type FilterType

+ type FilterType int

+ const FilterArray

+ const FilterBool

+ const FilterLike

+ const FilterUint

+ type GSCAnalyticsInsertRow struct

+ CTR float32

+ Clicks uint32

+ Country string

+ Date time.Time

+ Device string

+ Impressions uint32

+ Page string

+ Position float32

+ Query string

+ type GSCCountryRow struct

+ CTR float64

+ Clicks uint64

+ Country string

+ Impressions uint64

+ Position float64

+ type GSCDeviceRow struct

+ CTR float64

+ Clicks uint64

+ Device string

+ Impressions uint64

+ Position float64

+ type GSCInspectionInsertRow struct

+ CanonicalURL string

+ CoverageState string

+ CrawledAs string

+ IndexingState string

+ IsGoogleCanonical bool

+ LastCrawlTime time.Time

+ MobileUsability string

+ RichResultsItems uint16

+ RobotsTxtState string

+ URL string

+ Verdict string

+ type GSCInspectionRow struct

+ CanonicalURL string

+ CoverageState string

+ CrawledAs string

+ IndexingState string

+ IsGoogleCanonical bool

+ LastCrawlTime string

+ MobileUsability string

+ RichResultsItems uint16

+ RobotsTxtState string

+ URL string

+ Verdict string

+ type GSCOverviewStats struct

+ AvgCTR float64

+ AvgPosition float64

+ DateMax string

+ DateMin string

+ TotalClicks uint64

+ TotalImpressions uint64

+ TotalPages uint64

+ TotalQueries uint64

+ type GSCPageRow struct

+ CTR float64

+ Clicks uint64

+ Impressions uint64

+ Page string

+ Position float64

+ type GSCQueryRow struct

+ CTR float64

+ Clicks uint64

+ Impressions uint64

+ Position float64

+ Query string

+ type GSCTimelineRow struct

+ Clicks uint64

+ Date string

+ Impressions uint64

+ type GlobalSessionStats struct

+ AvgFetchMs float64

+ ErrorCount uint64

+ SessionID string

+ TotalLinks uint64

+ TotalPages uint64

+ type HreflangRow struct

+ Lang string

+ URL string

+ type LangCount struct

+ Count uint64

+ Lang string

+ type LinkDiffResult struct

+ Links []LinkDiffRow

+ TotalAdded uint64

+ TotalRemoved uint64

+ type LinkDiffRow struct

+ AnchorText string

+ DiffType string

+ SourceURL string

+ TargetURL string

+ type LinkRow struct

+ AnchorText string

+ CrawlSessionID string

+ CrawledAt time.Time

+ IsInternal bool

+ Rel string

+ SourceURL string

+ Tag string

+ TargetURL string

+ type Migration struct

+ DDL string

+ Fn func(ctx context.Context, conn driver.Conn) error

+ Name string

+ type NearDuplicatePair struct

+ CanonicalA string

+ CanonicalB string

+ Similarity float64

+ TitleA string

+ TitleB string

+ URLa string

+ URLb string

+ WordCountA uint32

+ WordCountB uint32

+ type NearDuplicatesResult struct

+ Pairs []NearDuplicatePair

+ Total uint64

+ type NoindexReason struct

+ Count uint64

+ Reason string

+ type PageBody struct

+ BodyHTML string

+ URL string

+ type PageDiffResult struct

+ Pages []PageDiffRow

+ TotalAdded uint64

+ TotalChanged uint64

+ TotalRemoved uint64

+ type PageDiffRow struct

+ CanonicalA string

+ CanonicalB string

+ DepthA uint16

+ DepthB uint16

+ DiffType string

+ H1A string

+ H1B string

+ IsIndexableA bool

+ IsIndexableB bool

+ MetaDescriptionA string

+ MetaDescriptionB string

+ PageRankA float64

+ PageRankB float64

+ StatusCodeA uint16

+ StatusCodeB uint16

+ TitleA string

+ TitleB string

+ URL string

+ WordCountA uint32

+ WordCountB uint32

+ type PageHTMLRow struct

+ HTML string

+ URL string

+ type PageLinkInserter interface

+ InsertLinks func(ctx context.Context, links []LinkRow) error

+ InsertPages func(ctx context.Context, pages []PageRow) error

+ type PageLinksResult struct

+ InLinks []LinkRow

+ InLinksCount uint64

+ OutLinks []LinkRow

+ OutLinksCount uint64

+ type PageRankBucket struct

+ AvgPR float64

+ Count uint64

+ Max float64

+ Min float64

+ type PageRankDistributionResult struct

+ Avg float64

+ Buckets []PageRankBucket

+ Median float64

+ P90 float64

+ P99 float64

+ TotalWithPR uint64

+ type PageRankEntry struct

+ PageRank float64

+ URL string

+ type PageRankTopPage struct

+ Depth uint16

+ ExternalLinksOut uint32

+ InternalLinksOut uint32

+ PageRank float64

+ StatusCode uint16

+ Title string

+ URL string

+ WordCount uint32

+ type PageRankTopResult struct

+ Pages []PageRankTopPage

+ Total uint64

+ type PageRankTreemapEntry struct

+ AvgPR float64

+ MaxPR float64

+ PageCount uint64

+ Path string

+ TotalPR float64

+ type PageResourceCheck struct

+ CheckedAt time.Time

+ ContentType string

+ CrawlSessionID string

+ Error string

+ IsInternal bool

+ PageCount uint64

+ RedirectURL string

+ ResourceType string

+ ResponseTimeMs uint32

+ StatusCode uint16

+ URL string

+ type PageResourceRef struct

+ CrawlSessionID string

+ IsInternal bool

+ PageURL string

+ ResourceType string

+ ResourceURL string

+ type PageRow struct

+ BodyHTML string

+ BodySize uint64

+ BodyTruncated bool

+ Canonical string

+ CanonicalIsSelf bool

+ ContentEncoding string

+ ContentHash uint64

+ ContentType string

+ CrawlSessionID string

+ CrawledAt time.Time

+ Depth uint16

+ Error string

+ ExternalLinksOut uint32

+ FetchDurationMs uint64

+ FinalURL string

+ FoundOn string

+ H1 []string

+ H2 []string

+ H3 []string

+ H4 []string

+ H5 []string

+ H6 []string

+ Headers map[string]string

+ Hreflang []HreflangRow

+ ImagesCount uint16

+ ImagesNoAlt uint16

+ IndexReason string

+ InternalLinksOut uint32

+ IsIndexable bool

+ JSAddedImages int32

+ JSAddedLinks int32

+ JSAddedSchema bool

+ JSChangedCanonical bool

+ JSChangedContent bool

+ JSChangedDescription bool

+ JSChangedH1 bool

+ JSChangedTitle bool

+ JSRenderDurationMs uint64

+ JSRenderError string

+ JSRendered bool

+ Lang string

+ MetaDescLength uint16

+ MetaDescription string

+ MetaKeywords string

+ MetaRobots string

+ OGDescription string

+ OGImage string

+ OGTitle string

+ PageRank float64

+ RedirectChain []RedirectHopRow

+ RenderedBodyHTML string

+ RenderedCanonical string

+ RenderedH1 []string

+ RenderedImagesCount uint16

+ RenderedLinksCount uint32

+ RenderedMetaDescription string

+ RenderedMetaRobots string

+ RenderedSchemaTypes []string

+ RenderedTitle string

+ RenderedWordCount uint32

+ SchemaTypes []string

+ StatusCode uint16

+ Title string

+ TitleLength uint16

+ URL string

+ WordCount uint32

+ XRobotsTag string

+ type PageWithAuthority struct

+ CitationFlow *uint8

+ Depth uint16

+ ExtBackLinks *int64

+ PageRank float64

+ RefDomains *int64

+ StatusCode uint16

+ Title string

+ TrustFlow *uint8

+ URL string

+ WordCount uint32

+ type ParsedFilter struct

+ Def FilterDef

+ Value string

+ type ProviderAPICallRow struct

+ CalledAt time.Time

+ DurationMs uint32

+ Endpoint string

+ Error string

+ Method string

+ ProjectID string

+ Provider string

+ ResponseBody string

+ RowsReturned uint32

+ StatusCode uint16

+ type ProviderBacklinkRow struct

+ AnchorText string

+ Domain string

+ DomainRank float64

+ FetchedAt time.Time

+ FirstSeen time.Time

+ LastSeen time.Time

+ LinkType string

+ Nofollow bool

+ PageRank float64

+ Provider string

+ SourceDomain string

+ SourceURL string

+ TargetURL string

+ type ProviderDomainMetricsRow struct

+ BacklinksTotal int64

+ Domain string

+ DomainRank float64

+ FetchedAt time.Time

+ OrganicCost float64

+ OrganicKeywords int64

+ OrganicTraffic int64

+ Provider string

+ RefDomainsTotal int64

+ type ProviderRankingRow struct

+ CPC float64

+ Domain string

+ FetchedAt time.Time

+ Keyword string

+ Position uint16

+ Provider string

+ SearchBase string

+ SearchVolume int64

+ Traffic float64

+ TrafficPct float64

+ URL string

+ type ProviderRefDomainRow struct

+ BacklinkCount int64

+ Domain string

+ DomainRank float64

+ FetchedAt time.Time

+ FirstSeen time.Time

+ LastSeen time.Time

+ Provider string

+ RefDomain string

+ type ProviderTopPageRow struct

+ CitationFlow uint8

+ Domain string

+ ExtBackLinks int64

+ FetchedAt time.Time

+ Language string

+ Provider string

+ RefDomains int64

+ Title string

+ TopicalTrustFlow []TopicalTF

+ TrustFlow uint8

+ URL string

+ type ProviderVisibilityRow struct

+ Date time.Time

+ Domain string

+ FetchedAt time.Time

+ KeywordsCount int64

+ Provider string

+ SearchBase string

+ Visibility float64

+ type RedirectHopRow struct

+ StatusCode uint16

+ URL string

+ type ResourceTypeSummary struct

+ Errors uint64

+ External uint64

+ Internal uint64

+ OK uint64

+ ResourceType string

+ Total uint64

+ type RobotsRow struct

+ Content string

+ CrawlSessionID string

+ FetchedAt time.Time

+ Host string

+ StatusCode uint16

+ type SchemaCount struct

+ Count uint64

+ SchemaType string

+ type SessionStats struct

+ AvgFetchMs float64

+ AvgJSRenderMs float64

+ CrawlDurationSec float64

+ DepthDistribution map[uint16]uint64

+ ErrorCount uint64

+ ExternalLinks uint64

+ InternalLinks uint64

+ JSChangedContentCount uint64

+ JSChangedH1Count uint64

+ JSChangedTitleCount uint64

+ JSRenderedPages uint64

+ PagesPerSecond float64

+ StatusCodes map[uint16]uint64

+ TopPageRank []PageRankEntry

+ TotalLinks uint64

+ TotalPages uint64

+ type SitemapRow struct

+ CrawlSessionID string

+ FetchedAt time.Time

+ ParentURL string

+ StatusCode uint16

+ Type string

+ URL string

+ URLCount uint32

+ type SitemapURLRow struct

+ ChangeFreq string

+ CrawlSessionID string

+ LastMod string

+ Loc string

+ Priority string

+ SitemapURL string

+ type StorageStatsResult struct

+ Tables []TableStorageStats

+ type Store struct

+ func NewStore(host string, port int, database, username, password string) (*Store, error)

+ func (s *Store) Close() error

+ func (s *Store) CompareLinks(ctx context.Context, sessionA, sessionB, diffType string, limit, offset int) (*LinkDiffResult, error)

+ func (s *Store) ComparePages(ctx context.Context, sessionA, sessionB, diffType string, limit, offset int) (*PageDiffResult, error)

+ func (s *Store) CompareStats(ctx context.Context, sessionA, sessionB string) (*CompareStatsResult, error)

+ func (s *Store) ComputePageRank(ctx context.Context, sessionID string) error

+ func (s *Store) CountPages(ctx context.Context, sessionID string) (uint64, error)

+ func (s *Store) CrawledURLs(ctx context.Context, sessionID string) ([]string, error)

+ func (s *Store) DeleteFailedPages(ctx context.Context, sessionID string) (int, error)

+ func (s *Store) DeleteGSCData(ctx context.Context, projectID string) error

+ func (s *Store) DeletePagesByStatus(ctx context.Context, sessionID string, statusCode int) (int, error)

+ func (s *Store) DeleteProviderData(ctx context.Context, projectID, provider string) error

+ func (s *Store) DeleteSession(ctx context.Context, sessionID string) error

+ func (s *Store) ExportLogs(ctx context.Context) ([]applog.LogRow, error)

+ func (s *Store) ExportSession(ctx context.Context, sessionID string, w io.Writer, includeHTML bool) error

+ func (s *Store) ExternalLinks(ctx context.Context, sessionID string) ([]LinkRow, error)

+ func (s *Store) ExternalLinksPaginated(ctx context.Context, sessionID string, limit, offset int, ...) ([]LinkRow, error)

+ func (s *Store) FailedURLs(ctx context.Context, sessionID string) ([]string, error)

+ func (s *Store) GSCByCountry(ctx context.Context, projectID string) ([]GSCCountryRow, error)

+ func (s *Store) GSCByDevice(ctx context.Context, projectID string) ([]GSCDeviceRow, error)

+ func (s *Store) GSCInspectionResults(ctx context.Context, projectID string, limit, offset int) ([]GSCInspectionRow, int, error)

+ func (s *Store) GSCOverview(ctx context.Context, projectID string) (*GSCOverviewStats, error)

+ func (s *Store) GSCTimeline(ctx context.Context, projectID string) ([]GSCTimelineRow, error)

+ func (s *Store) GSCTopPages(ctx context.Context, projectID string, limit, offset int) ([]GSCPageRow, int, error)

+ func (s *Store) GSCTopQueries(ctx context.Context, projectID string, limit, offset int) ([]GSCQueryRow, int, error)

+ func (s *Store) GetExpiredDomains(ctx context.Context, sessionID string, limit, offset int) (*ExpiredDomainsResult, error)

+ func (s *Store) GetExternalLinkCheckDomains(ctx context.Context, sessionID string, limit, offset int, ...) ([]ExternalDomainCheck, error)

+ func (s *Store) GetExternalLinkChecks(ctx context.Context, sessionID string, limit, offset int, ...) ([]ExternalLinkCheck, error)

+ func (s *Store) GetPage(ctx context.Context, sessionID, url string) (*PageRow, error)

+ func (s *Store) GetPageBodies(ctx context.Context, sessionID string, limit, offset int) ([]PageBody, error)

+ func (s *Store) GetPageHTML(ctx context.Context, sessionID, url string) (string, error)

+ func (s *Store) GetPageLinks(ctx context.Context, sessionID, url string, ...) (*PageLinksResult, error)

+ func (s *Store) GetPageResourceChecks(ctx context.Context, sessionID string, limit, offset int, ...) ([]PageResourceCheck, error)

+ func (s *Store) GetPageResourceTypeSummary(ctx context.Context, sessionID string) ([]ResourceTypeSummary, error)

+ func (s *Store) GetRobotsContent(ctx context.Context, sessionID, host string) (*RobotsRow, error)

+ func (s *Store) GetRobotsHosts(ctx context.Context, sessionID string) ([]RobotsRow, error)

+ func (s *Store) GetSession(ctx context.Context, sessionID string) (*CrawlSession, error)

+ func (s *Store) GetSitemapURLs(ctx context.Context, sessionID, sitemapURL string, limit, offset int) ([]SitemapURLRow, error)

+ func (s *Store) GetSitemaps(ctx context.Context, sessionID string) ([]SitemapRow, error)

+ func (s *Store) GetURLsByHost(ctx context.Context, sessionID, host string) ([]string, error)

+ func (s *Store) GlobalStats(ctx context.Context) ([]GlobalSessionStats, *StorageStatsResult, error)

+ func (s *Store) ImportSession(ctx context.Context, r io.Reader) (*CrawlSession, error)

+ func (s *Store) InsertExternalLinkChecks(ctx context.Context, checks []ExternalLinkCheck) error

+ func (s *Store) InsertGSCAnalytics(ctx context.Context, projectID string, rows []GSCAnalyticsInsertRow) error

+ func (s *Store) InsertGSCInspection(ctx context.Context, projectID string, rows []GSCInspectionInsertRow) error

+ func (s *Store) InsertLinks(ctx context.Context, links []LinkRow) error

+ func (s *Store) InsertLogs(ctx context.Context, logs []applog.LogRow) error

+ func (s *Store) InsertPageResourceChecks(ctx context.Context, checks []PageResourceCheck) error

+ func (s *Store) InsertPageResourceRefs(ctx context.Context, refs []PageResourceRef) error

+ func (s *Store) InsertPages(ctx context.Context, pages []PageRow) error

+ func (s *Store) InsertProviderAPICalls(ctx context.Context, rows []ProviderAPICallRow) error

+ func (s *Store) InsertProviderBacklinks(ctx context.Context, projectID string, rows []ProviderBacklinkRow) error

+ func (s *Store) InsertProviderDomainMetrics(ctx context.Context, projectID string, rows []ProviderDomainMetricsRow) error

+ func (s *Store) InsertProviderRankings(ctx context.Context, projectID string, rows []ProviderRankingRow) error

+ func (s *Store) InsertProviderRefDomains(ctx context.Context, projectID string, rows []ProviderRefDomainRow) error

+ func (s *Store) InsertProviderTopPages(ctx context.Context, projectID string, rows []ProviderTopPageRow) error

+ func (s *Store) InsertProviderVisibility(ctx context.Context, projectID string, rows []ProviderVisibilityRow) error

+ func (s *Store) InsertRobotsData(ctx context.Context, rows []RobotsRow) error

+ func (s *Store) InsertSession(ctx context.Context, session *CrawlSession) error

+ func (s *Store) InsertSitemapURLs(ctx context.Context, rows []SitemapURLRow) error

+ func (s *Store) InsertSitemaps(ctx context.Context, rows []SitemapRow) error

+ func (s *Store) InternalLinksPaginated(ctx context.Context, sessionID string, limit, offset int, ...) ([]LinkRow, error)

+ func (s *Store) ListLogs(ctx context.Context, limit, offset int, level, component, search string) ([]applog.LogRow, int, error)

+ func (s *Store) ListPages(ctx context.Context, sessionID string, limit, offset int, ...) ([]PageRow, error)

+ func (s *Store) ListSessions(ctx context.Context, projectID ...string) ([]CrawlSession, error)

+ func (s *Store) ListSessionsPaginated(ctx context.Context, limit, offset int, projectID, search string) ([]CrawlSession, int, error)

+ func (s *Store) Migrate(ctx context.Context) error

+ func (s *Store) NearDuplicates(ctx context.Context, sessionID string, threshold int, limit, offset int) (*NearDuplicatesResult, error)

+ func (s *Store) PageRankDistribution(ctx context.Context, sessionID string, buckets int) (*PageRankDistributionResult, error)

+ func (s *Store) PageRankTop(ctx context.Context, sessionID string, limit, offset int, directory string) (*PageRankTopResult, error)

+ func (s *Store) PageRankTreemap(ctx context.Context, sessionID string, depth, minPages int) ([]PageRankTreemapEntry, error)

+ func (s *Store) PagesWithAuthority(ctx context.Context, sessionID, projectID string, limit, offset int) ([]PageWithAuthority, int, error)

+ func (s *Store) ProviderAPICalls(ctx context.Context, projectID, provider string, limit, offset int) ([]ProviderAPICallRow, int, error)

+ func (s *Store) ProviderBacklinks(ctx context.Context, projectID, provider string, limit, offset int) ([]ProviderBacklinkRow, int, error)

+ func (s *Store) ProviderDomainMetrics(ctx context.Context, projectID, provider string) (*ProviderDomainMetricsRow, error)

+ func (s *Store) ProviderRankings(ctx context.Context, projectID, provider string, limit, offset int) ([]ProviderRankingRow, int, error)

+ func (s *Store) ProviderRefDomains(ctx context.Context, projectID, provider string, limit, offset int) ([]ProviderRefDomainRow, int, error)

+ func (s *Store) ProviderTopPages(ctx context.Context, projectID, provider string, limit, offset int) ([]ProviderTopPageRow, int, error)

+ func (s *Store) ProviderVisibilityHistory(ctx context.Context, projectID, provider string) ([]ProviderVisibilityRow, error)

+ func (s *Store) RecomputeDepths(ctx context.Context, sessionID string, seedURLs []string) error

+ func (s *Store) RunCustomTestsSQL(ctx context.Context, sessionID string, rules []customtests.TestRule) (map[string]map[string]string, error)

+ func (s *Store) SessionAudit(ctx context.Context, sessionID string) (*AuditResult, error)

+ func (s *Store) SessionStats(ctx context.Context, sessionID string) (*SessionStats, error)

+ func (s *Store) SessionStorageStats(ctx context.Context) (map[string]uint64, error)

+ func (s *Store) StorageStats(ctx context.Context) (*StorageStatsResult, error)

+ func (s *Store) StreamPagesHTML(ctx context.Context, sessionID string) (<-chan PageHTMLRow, error)

+ func (s *Store) URLsByStatus(ctx context.Context, sessionID string, statusCode int) ([]string, error)

+ func (s *Store) UncrawledURLs(ctx context.Context, sessionID string) ([]string, error)

+ func (s *Store) UpdateSessionProject(ctx context.Context, sessionID string, projectID *string) error

+ type TableStorageStats struct

+ BytesOnDisk uint64

+ Name string

+ Rows uint64

+ type TopicalTF struct

+ Topic string

+ Value uint8

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL