server

Mar 9, 2026

Changes in this version

type CrawlStore

+ ComputeNearDuplicates func(ctx context.Context, sessionID string) error

+ UpdateContentHashes func(ctx context.Context, sessionID string, hashes map[string]uint64) error

Mar 7, 2026

Changes in this version

type CrawlStore

+ ImportCSVSession func(ctx context.Context, r io.Reader, projectID string) (*storage.CSVImportResult, error)

+ StatusTimeline func(ctx context.Context, sessionID string) ([]storage.StatusTimelineBucket, error)

+ StatusTimelineRecent func(ctx context.Context, sessionID string) ([]storage.StatusTimelineBucket, error)

Mar 5, 2026

Changes in this version

type Server

+ SetupMode bool

+ func NewSetupServer(cfg *config.Config) *Server

+ func (s *Server) SetDownloadProgress(p SetupProgress)

+ func (s *Server) TransitionToReady(store *storage.Store, keyStore *apikeys.Store)

+ type SetupProgress struct

+ BytesDownloaded int64

+ Percent int

+ TotalBytes int64

Mar 4, 2026

Changes in this version

type CrawlService

+ Phase func(sessionID string) string

Mar 4, 2026

Changes in this version

type CrawlStore

+ WeightedPageRankTop func(ctx context.Context, sessionID, projectID string, limit, offset int, ...) (*storage.WeightedPageRankResult, error)

Mar 4, 2026

Changes in this version

type ProviderStore

+ InsertProviderData func(ctx context.Context, projectID string, rows []storage.ProviderDataRow) error

+ ProviderData func(ctx context.Context, projectID, provider, dataType string, limit, offset int, ...) ([]storage.ProviderDataRow, int, error)

+ ProviderDataAge func(ctx context.Context, projectID, provider, dataType string) (time.Time, error)

Mar 3, 2026

Changes in this version

type CrawlStore

+ GetSitemapCoverageURLs func(ctx context.Context, sessionID, filter string, limit, offset int) ([]storage.SitemapURLRow, error)

Mar 3, 2026

Changes in this version

type CrawlStore

+ DeleteExtractions func(ctx context.Context, sessionID string) error

+ GetExtractions func(ctx context.Context, sessionID string, limit, offset int) (*extraction.ExtractionResult, error)

+ HasStoredHTML func(ctx context.Context, sessionID string) (bool, error)

+ InsertExtractions func(ctx context.Context, rows []extraction.ExtractionRow) error

+ RunExtractionsPostCrawl func(ctx context.Context, sessionID string, extractors []extraction.Extractor) (*extraction.ExtractionResult, error)

Mar 2, 2026

Changes in this version

type CrawlStore

+ ListRedirectPages func(ctx context.Context, sessionID string, limit, offset int, ...) ([]storage.RedirectPageRow, error)

Mar 2, 2026

Changes in this version

+ type CrawlService interface

+ BufferState func(sessionID string) storage.BufferErrorState

+ IsQueued func(sessionID string) bool

+ IsRunning func(sessionID string) bool

+ LastError func(sessionID string) string

+ Progress func(sessionID string) (int64, int, bool)

+ QueuedSessions func() []string

+ RecoverOrphanedSessions func(ctx context.Context)

+ ResumeCrawl func(sessionID string, overrides *crawler.CrawlRequest) (string, error)

+ RetryFailed func(sessionID string, overrides *crawler.CrawlRequest) (int, error)

+ Shutdown func(timeout time.Duration)

+ StartCrawl func(req crawler.CrawlRequest) (string, error)

+ StopCrawl func(sessionID string) error

+ type CrawlStore interface

+ CompareLinks func(ctx context.Context, sessionA, sessionB, diffType string, limit, offset int) (*storage.LinkDiffResult, error)

+ ComparePages func(ctx context.Context, sessionA, sessionB, diffType string, limit, offset int) (*storage.PageDiffResult, error)

+ CompareStats func(ctx context.Context, sessionA, sessionB string) (*storage.CompareStatsResult, error)

+ ComputePageRank func(ctx context.Context, sessionID string) error

+ DeleteSession func(ctx context.Context, sessionID string) error

+ ExportSession func(ctx context.Context, sessionID string, w io.Writer, includeHTML bool) error

+ ExternalLinksPaginated func(ctx context.Context, sessionID string, limit, offset int, ...) ([]storage.LinkRow, error)

+ GetExpiredDomains func(ctx context.Context, sessionID string, limit, offset int) (*storage.ExpiredDomainsResult, error)

+ GetExternalLinkCheckDomains func(ctx context.Context, sessionID string, limit, offset int, ...) ([]storage.ExternalDomainCheck, error)

+ GetExternalLinkChecks func(ctx context.Context, sessionID string, limit, offset int, ...) ([]storage.ExternalLinkCheck, error)

+ GetPage func(ctx context.Context, sessionID, url string) (*storage.PageRow, error)

+ GetPageBodies func(ctx context.Context, sessionID string, limit, offset int) ([]storage.PageBody, error)

+ GetPageHTML func(ctx context.Context, sessionID, url string) (string, error)

+ GetPageLinks func(ctx context.Context, sessionID, url string, ...) (*storage.PageLinksResult, error)

+ GetPageResourceChecks func(ctx context.Context, sessionID string, limit, offset int, ...) ([]storage.PageResourceCheck, error)

+ GetPageResourceTypeSummary func(ctx context.Context, sessionID string) ([]storage.ResourceTypeSummary, error)

+ GetRobotsContent func(ctx context.Context, sessionID, host string) (*storage.RobotsRow, error)

+ GetRobotsHosts func(ctx context.Context, sessionID string) ([]storage.RobotsRow, error)

+ GetSession func(ctx context.Context, sessionID string) (*storage.CrawlSession, error)

+ GetSitemapURLs func(ctx context.Context, sessionID, sitemapURL string, limit, offset int) ([]storage.SitemapURLRow, error)

+ GetSitemaps func(ctx context.Context, sessionID string) ([]storage.SitemapRow, error)

+ GetURLsByHost func(ctx context.Context, sessionID, host string) ([]string, error)

+ GlobalStats func(ctx context.Context) ([]storage.GlobalSessionStats, *storage.StorageStatsResult, error)

+ ImportSession func(ctx context.Context, r io.Reader) (*storage.CrawlSession, error)

+ InsertPageResourceChecks func(ctx context.Context, checks []storage.PageResourceCheck) error

+ InsertPageResourceRefs func(ctx context.Context, refs []storage.PageResourceRef) error

+ InternalLinksPaginated func(ctx context.Context, sessionID string, limit, offset int, ...) ([]storage.LinkRow, error)

+ ListPages func(ctx context.Context, sessionID string, limit, offset int, ...) ([]storage.PageRow, error)

+ ListSessions func(ctx context.Context, projectID ...string) ([]storage.CrawlSession, error)

+ ListSessionsPaginated func(ctx context.Context, limit, offset int, projectID, search string) ([]storage.CrawlSession, int, error)

+ NearDuplicates func(ctx context.Context, sessionID string, threshold int, limit, offset int) (*storage.NearDuplicatesResult, error)

+ PageRankDistribution func(ctx context.Context, sessionID string, buckets int) (*storage.PageRankDistributionResult, error)

+ PageRankTop func(ctx context.Context, sessionID string, limit, offset int, directory string) (*storage.PageRankTopResult, error)

+ PageRankTreemap func(ctx context.Context, sessionID string, depth, minPages int) ([]storage.PageRankTreemapEntry, error)

+ PagesWithAuthority func(ctx context.Context, sessionID, projectID string, limit, offset int) ([]storage.PageWithAuthority, int, error)

+ RecomputeDepths func(ctx context.Context, sessionID string, seedURLs []string) error

+ RunCustomTestsSQL func(ctx context.Context, sessionID string, rules []customtests.TestRule) (map[string]map[string]string, error)

+ SessionAudit func(ctx context.Context, sessionID string) (*storage.AuditResult, error)

+ SessionStats func(ctx context.Context, sessionID string) (*storage.SessionStats, error)

+ SessionStorageStats func(ctx context.Context) (map[string]uint64, error)

+ StorageStats func(ctx context.Context) (*storage.StorageStatsResult, error)

+ StreamPagesHTML func(ctx context.Context, sessionID string) (<-chan storage.PageHTMLRow, error)

+ UpdateSessionProject func(ctx context.Context, sessionID string, projectID *string) error

+ type GSCStore interface

+ DeleteGSCData func(ctx context.Context, projectID string) error

+ GSCByCountry func(ctx context.Context, projectID string) ([]storage.GSCCountryRow, error)

+ GSCByDevice func(ctx context.Context, projectID string) ([]storage.GSCDeviceRow, error)

+ GSCInspectionResults func(ctx context.Context, projectID string, limit, offset int) ([]storage.GSCInspectionRow, int, error)

+ GSCOverview func(ctx context.Context, projectID string) (*storage.GSCOverviewStats, error)

+ GSCTimeline func(ctx context.Context, projectID string) ([]storage.GSCTimelineRow, error)

+ GSCTopPages func(ctx context.Context, projectID string, limit, offset int) ([]storage.GSCPageRow, int, error)

+ GSCTopQueries func(ctx context.Context, projectID string, limit, offset int) ([]storage.GSCQueryRow, int, error)

+ InsertGSCAnalytics func(ctx context.Context, projectID string, rows []storage.GSCAnalyticsInsertRow) error

+ InsertGSCInspection func(ctx context.Context, projectID string, rows []storage.GSCInspectionInsertRow) error

+ type LogStore interface

+ ExportLogs func(ctx context.Context) ([]applog.LogRow, error)

+ InsertLogs func(ctx context.Context, logs []applog.LogRow) error

+ ListLogs func(ctx context.Context, limit, offset int, level, component, search string) ([]applog.LogRow, int, error)

+ type ProviderStore interface

+ DeleteProviderData func(ctx context.Context, projectID, provider string) error

+ InsertProviderAPICalls func(ctx context.Context, rows []storage.ProviderAPICallRow) error

+ InsertProviderBacklinks func(ctx context.Context, projectID string, rows []storage.ProviderBacklinkRow) error

+ InsertProviderDomainMetrics func(ctx context.Context, projectID string, rows []storage.ProviderDomainMetricsRow) error

+ InsertProviderRankings func(ctx context.Context, projectID string, rows []storage.ProviderRankingRow) error

+ InsertProviderRefDomains func(ctx context.Context, projectID string, rows []storage.ProviderRefDomainRow) error

+ InsertProviderTopPages func(ctx context.Context, projectID string, rows []storage.ProviderTopPageRow) error

+ InsertProviderVisibility func(ctx context.Context, projectID string, rows []storage.ProviderVisibilityRow) error

+ ProviderAPICalls func(ctx context.Context, projectID, provider string, limit, offset int) ([]storage.ProviderAPICallRow, int, error)

+ ProviderBacklinks func(ctx context.Context, projectID, provider string, limit, offset int) ([]storage.ProviderBacklinkRow, int, error)

+ ProviderDomainMetrics func(ctx context.Context, projectID, provider string) (*storage.ProviderDomainMetricsRow, error)

+ ProviderRankings func(ctx context.Context, projectID, provider string, limit, offset int) ([]storage.ProviderRankingRow, int, error)

+ ProviderRefDomains func(ctx context.Context, projectID, provider string, limit, offset int) ([]storage.ProviderRefDomainRow, int, error)

+ ProviderTopPages func(ctx context.Context, projectID, provider string, limit, offset int) ([]storage.ProviderTopPageRow, int, error)

+ ProviderVisibilityHistory func(ctx context.Context, projectID, provider string) ([]storage.ProviderVisibilityRow, error)

+ type Server struct

+ BackupOpts *backup.BackupOptions

+ IsDesktop bool

+ NoBrowserOpen bool

+ StartClickHouse func() error

+ StopClickHouse func()

+ UpdateStatus *updater.UpdateStatus

+ func New(cfg *config.Config, store *storage.Store, keyStore *apikeys.Store) *Server

+ func NewWithDeps(cfg *config.Config, store StorageService, keyStore *apikeys.Store, ...) *Server

+ func (s *Server) Handler() (http.Handler, error)

+ func (s *Server) Start() error

+ func (s *Server) Stop(ctx context.Context) error

+ type StorageService interface