ccrawl

package

Go to main page

Versions in this module

v0

v0.5.0

Jul 2, 2026

Changes in this version

+ func GenerateSeedREADME(s SeedDatasetStats) string

+ func HFSeedShardPath(crawlID string, shardIdx int) string

+ type CCSeedOptions struct

+ BlockSize int

+ Codec seed.Codec

+ Crawl string

+ FileRetries int

+ Limit int64

+ MaxFiles int

+ OutDir string

+ Progress func(CCSeedProgress)

+ Shards int

+ SkipFiles int

+ Subset string

+ Whole bool

+ Workers int

+ type CCSeedProgress struct

+ BytesFetched int64

+ Elapsed time.Duration

+ Failed int

+ FilesDone int

+ FilesTotal int

+ URLs int64

+ type CCSeedStats struct

+ BytesFetched int64

+ Crawl string

+ Elapsed time.Duration

+ FailedFiles []string

+ Files int

+ Scanned int64

+ SeedBytes int64

+ Shards int

+ Written int64

+ func BuildCCSeed(ctx context.Context, h *HTTPClient, cache *Cache, src Source, ...) (CCSeedStats, error)

type HTTPClient

+ func (h *HTTPClient) ContentLength(ctx context.Context, url string) (int64, error)

type ParquetWriter

+ func (p *ParquetWriter[T]) WriteRows(rows []T) error

+ type SeedDatasetStats struct

+ CrawlID string

+ ParquetBytes int64

+ PublishedShards int

+ Repo string

+ Rows int64

+ Subset string

+ TotalShards int

+ URLBytes int64

+ type SeedPublishConfig struct

+ CommitBatch int

+ CrawlID string

+ KeepParquet bool

+ Ledger *Ledger

+ MinFreeBytes int64

+ OutDir string

+ Progress func(SeedPublishStats)

+ Push bool

+ Repo string

+ SeedDir string

+ ShardParallel int

+ Subset string

+ type SeedPublishStats struct

+ ETA time.Duration

+ Elapsed time.Duration

+ Failed int

+ FreeDiskBytes int64

+ ParquetBytes int64

+ PublishS int64

+ Published int

+ Rows int64

+ ShardsPerHour float64

+ Skipped int

+ Total int

+ TranscodeS int64

+ URLBytes int64

+ func RunSeedPublish(ctx context.Context, hf *HFClient, cfg SeedPublishConfig) (SeedPublishStats, error)

v0.4.2

Jun 30, 2026

Changes in this version

+ const DefaultBackoff

+ const DefaultBackoffMax

type Config

+ Backoff time.Duration

+ BackoffMax time.Duration

v0.4.1

Jun 29, 2026

v0.4.0

Jun 29, 2026

Changes in this version

+ const DefaultWARCSize

+ func FetchWARCRecordRaw(ctx context.Context, h *HTTPClient, filename string, offset, length int64) ([]byte, error)

+ func FileLimit() uint64

+ func RaiseFileLimit() (uint64, error)

+ func ResolveCrawls(ctx context.Context, h *HTTPClient, cache *Cache, ref string) ([]string, error)

type MarkdownPackConfig

+ MaxRecords int

type RefetchExportConfig

+ CacheDir string

+ FetchOnly bool

type RefetchPackConfig

+ CacheDir string

+ FetchOnly bool

type RefetchRow

+ HTML string

type RefetchRunStats

+ ErrDNS int64

+ ErrOther int64

+ ErrRefused int64

+ ErrSkip int64

+ ErrTimeout int64

+ ExportS int64

+ ExtractS int64

+ Failures int64

+ FetchS int64

type RefetchStats

+ ErrDNS int64

+ ErrOther int64

+ ErrRefused int64

+ ErrSkip int64

+ ErrTimeout int64

+ type StreamingParquetWriter struct

+ func NewStreamingParquetWriter(dest io.Writer) *StreamingParquetWriter

+ func (s *StreamingParquetWriter) Close() error

+ func (s *StreamingParquetWriter) EmitRow(cols, vals []string) error

+ type WARCExporter struct

+ Files []string

+ func NewWARCExporter(dir, prefix, subprefix string, maxSize int64, info WARCInfo) *WARCExporter

+ func (e *WARCExporter) Close() error

+ func (e *WARCExporter) Records() int

+ func (e *WARCExporter) Write(raw []byte) error

+ type WARCInfo struct

+ Creator string

+ Description string

+ Format string

+ IsPartOf string

+ Operator string

+ Software string

v0.3.0

Jun 23, 2026

Changes in this version

+ func BuildDatasetShardFromChunk(ctx context.Context, prefix, workDir, crawlID, graphID, outPath string, ...) (int64, error)

+ func CDXBatchPath(workDir, prefix string, chunk int) string

+ func ExtractCDXBatch(ctx context.Context, h *HTTPClient, batchURLs []string, workDir string, ...) error

+ func GenerateMarkdownREADME(s MarkdownDatasetStats) string

+ func HFMarkdownPath(crawlID string, shardIdx int) string

+ func HFRefetchPath(crawlID string, shardIdx int) string

+ func HFShardPathChunk(crawlID, subset, prefix string, chunk int) string

+ func LoadCDXRawChunk(workDir, prefix string, chunk int, fn func(CDXRawOutputRow) error) error

+ func MarkdownDocID(rawURL string) string

type DatasetStats

+ CommittedBatches int

+ TotalBatches int

+ type Ledger struct

+ func OpenLedger(path string) (*Ledger, error)

+ func (l *Ledger) Close() error

+ func (l *Ledger) Count() int

+ func (l *Ledger) Has(idx int) bool

+ func (l *Ledger) Mark(idx int) error

+ type MarkdownDatasetStats struct

+ CommittedShards int

+ ConvertS int64

+ CrawlID string

+ DownloadS int64

+ ExportS int64

+ HTMLBytes int64

+ MDBytes int64

+ ParquetBytes int64

+ PublishS int64

+ Rows int64

+ TotalShards int

+ WARCBytes int64

+ type MarkdownExportConfig struct

+ CommitBatch int

+ ConvertWorkers int

+ CrawlID string

+ Indices []int

+ KeepParquet bool

+ Ledger *Ledger

+ MinFreeBytes int64

+ OutDir string

+ Progress func(MarkdownRunStats)

+ Push bool

+ Repo string

+ ShardParallel int

+ WARCPaths []string

+ type MarkdownPackConfig struct

+ ConvertSem chan struct{}

+ CrawlID string

+ OutPath string

+ Progress func(MarkdownStats)

+ ShardIdx int

+ WARCPath string

+ Workers int

+ type MarkdownRow struct

+ CrawlDate string

+ DocID string

+ HTMLLength int64

+ Host string

+ Markdown string

+ MarkdownLength int64

+ URL string

+ WARCRecordID string

+ type MarkdownRunStats struct

+ Committed int

+ ConvertS int64

+ ETA time.Duration

+ Elapsed time.Duration

+ Failed int

+ FreeDiskBytes int64

+ HTMLBytes int64

+ MDBytes int64

+ ParquetBytes int64

+ PublishS int64

+ Rows int64

+ ShardsPerHour float64

+ Skipped int

+ Total int

+ WARCBytes int64

+ func RunMarkdownExport(ctx context.Context, h *HTTPClient, hf *HFClient, cfg MarkdownExportConfig) (MarkdownRunStats, error)

+ type MarkdownStats struct

+ DurConvert time.Duration

+ DurDownload time.Duration

+ DurExport time.Duration

+ DurPublish time.Duration

+ HTMLBytes int64

+ MDBytes int64

+ ParquetBytes int64

+ Rows int64

+ ShardIdx int

+ WARCBytes int64

+ func PackMarkdownShard(ctx context.Context, h *HTTPClient, cfg MarkdownPackConfig) (MarkdownStats, error)

+ type RefetchDatasetStats struct

+ CommittedShards int

+ ConvertS int64

+ CrawlID string

+ FetchBytes int64

+ HTMLBytes int64

+ MDBytes int64

+ ParquetBytes int64

+ PublishS int64

+ Rows int64

+ TotalShards int

+ URLsFound int64

+ WARCBytes int64

+ type RefetchExportConfig struct

+ CommitBatch int

+ ConvertWorkers int

+ CrawlID string

+ FetchCfg config.Config

+ Indices []int

+ KeepParquet bool

+ Ledger *Ledger

+ MinFreeBytes int64

+ OutDir string

+ Progress func(RefetchRunStats)

+ Push bool

+ Repo string

+ ShardParallel int

+ WARCPaths []string

+ type RefetchPackConfig struct

+ ConvertSem chan struct{}

+ CrawlID string

+ FetchCfg config.Config

+ OutPath string

+ Progress func(RefetchStats)

+ ShardIdx int

+ WARCPath string

+ type RefetchRow struct

+ BodyLength int64

+ ContentType string

+ CrawlDate string

+ CrawlID string

+ Digest string

+ DocID string

+ ETag string

+ Error string

+ FetchDurMS int64

+ FetchedAt int64

+ FinalURL string

+ HTMLLength int64

+ Host string

+ IPAddress string

+ LastModified string

+ Markdown string

+ MarkdownLength int64

+ RespHeaders string

+ Status int32

+ TTFBMS int64

+ URL string

+ WARCRecordID string

+ type RefetchRunStats struct

+ Committed int

+ ConvertS int64

+ ETA time.Duration

+ Elapsed time.Duration

+ Failed int

+ FetchBytes int64

+ FreeDiskBytes int64

+ HTMLBytes int64

+ MDBytes int64

+ ParquetBytes int64

+ PublishS int64

+ Rows int64

+ ShardsPerHour float64

+ Skipped int

+ Total int

+ URLsFound int64

+ WARCBytes int64

+ func RunRefetchExport(ctx context.Context, h *HTTPClient, hf *HFClient, cfg RefetchExportConfig) (RefetchRunStats, error)

+ type RefetchStats struct

+ DurConvert time.Duration

+ DurExport time.Duration

+ DurExtract time.Duration

+ DurFetch time.Duration

+ Failed int64

+ FetchBytes int64

+ Fetched int64

+ HTMLBytes int64

+ MDBytes int64

+ MemRSSBytes int64

+ ParquetBytes int64

+ Redirected int64

+ Rows int64

+ ShardIdx int

+ URLsFound int64

+ WARCBytes int64

+ func PackRefetchShard(ctx context.Context, h *HTTPClient, cfg RefetchPackConfig) (RefetchStats, error)

+ type SeedExportOptions struct

+ Lang string

+ Limit int64

+ MIME string

+ Status int32

+ func DefaultSeedExportOptions() SeedExportOptions

+ type SeedExportStats struct

+ Scanned int64

+ Written int64

+ func ExportSeedFromCDX(ctx context.Context, shardPath string, opt SeedExportOptions, ...) (SeedExportStats, error)

+ type SeedRow struct

+ Digest string

+ Host string

+ Lang string

+ MIME string

+ Status int32

+ Time string

+ URL string

+ type WARCURLRecord struct

+ Date string

+ RecordID string

+ URL string

v0.2.5

Jun 19, 2026

v0.2.4

Jun 18, 2026

Changes in this version

+ func GenerateDatasetREADME(s DatasetStats) string

+ func HFShardPath(crawlID, subset, prefix string) string

+ type DatasetStats struct

+ CommittedShards int

+ CrawlID string

+ TotalBytes int64

+ TotalShards int

+ TotalURLs int64

+ type HFClient struct

+ func NewHFClient(token string) *HFClient

+ func (c *HFClient) CommitWithRetry(ctx context.Context, repoID, message string, ops []HFOperation, ...) (string, error)

+ func (c *HFClient) CreateCommit(ctx context.Context, repoID, message string, ops []HFOperation) (string, error)

+ func (c *HFClient) CreateDatasetRepo(ctx context.Context, repoID string, private bool) error

+ func (c *HFClient) PathsExist(ctx context.Context, repoID string, paths []string) (map[string]bool, error)

+ func (c *HFClient) Valid() bool

+ type HFOperation struct

+ LocalPath string

+ PathInRepo string

+ type RateLimitError struct

+ Msg string

+ RetryAfter time.Duration

+ func (e *RateLimitError) Error() string

v0.2.3

Jun 18, 2026

Changes in this version

+ func LoadCDXRawPrefix(workDir, prefix string, fn func(CDXRawOutputRow) error) error

type CDXRawOutputRow

+ Charset string

+ Crawl string

+ Digest string

+ MIMEDecl string

+ Proto string

+ Redir string

+ RobotsOK bool

+ Surt string

+ TLD string

+ Trunc string

+ URL string

+ WARCFile string

+ WARCOff int64

type CDXRawRow

+ ContentCharset string

+ ContentMIMEType string

+ ContentTruncated string

+ Crawl string

+ Digest string

+ FetchRedirect string

+ RobotsTXTForceGet bool

+ URL string

+ URLHostTLD string

+ URLProtocol string

+ URLSurtKey string

+ WARCFilename string

+ WARCRecordOffset int64

+ type URLDatasetRow struct

+ Bytes int64

+ Charset string

+ Crawl string

+ Digest string

+ GraphID string

+ HarmonicPos int64

+ HarmonicVal float64

+ Host string

+ Lang string

+ MIME string

+ MIMEDecl string

+ PageRankPos int64

+ PageRankVal float64

+ Proto string

+ RD string

+ Redir string

+ RobotsOK bool

+ ST int32

+ Surt string

+ TLD string

+ TS string

+ Trunc string

+ URL string

+ WARCFile string

+ WARCOff int64

v0.2.2

Jun 18, 2026

Changes in this version

+ var DatasetPrefixes = func() []string { ... }()

+ func AggregateCDXRaw(ctx context.Context, workDir string, prefixes []string, parallel int, ...) error

+ func BuildDatasetShard(ctx context.Context, prefix, workDir, crawlID, graphID, outPath string, ...) (int64, error)

+ func CDXSeqLocalSQL(localPath, prefix string) string

+ func DatasetPrefixReady(workDir, prefix string) bool

+ func DatasetWorkFiles(workDir, prefix string) (cdxFile, rankFile string)

+ func DownloadRankTable(ctx context.Context, rankURL, localPath string) error

+ func DownloadToFile(ctx context.Context, h *HTTPClient, url, localPath string) error

+ func ExtractCDXRaw(ctx context.Context, h *HTTPClient, parquetURLs []string, workDir string, ...) error

+ func HostCDXBatchSQL(parquetURLs []string, crawlID string, prefixes []string, ...) string

+ func HostCDXPrefixSQL(parquetURLs []string, crawlID, prefix, tempDir string) string

+ func LoadCDXPrefix(workDir, prefix string, fn func(HostCDXStats) error) error

+ func SaveCDXBatchByPrefix(ctx context.Context, parquetURLs []string, crawlID, workDir string, ...) (map[string]int64, error)

+ func SaveCDXSeqByPrefix(ctx context.Context, h *HTTPClient, parquetURLs []string, ...) error

+ func SaveCDXSplitByPrefix(ctx context.Context, parquetURLs []string, crawlID, workDir string, ...) (map[string]int64, error)

+ func SplitRankByPrefix(ctx context.Context, h *HTTPClient, rankURL, workDir string, ...) (map[string]int64, error)

+ func SplitRankFromFile(ctx context.Context, localPath, workDir string, progress func(total int64)) (map[string]int64, error)

+ type CDXBatchOptions struct

+ BatchSize int

+ DuckDBThreads int

+ MemoryLimit string

+ Parallel int

+ type CDXRawOutputRow struct

+ Bytes int64

+ Host string

+ Lang string

+ MIME string

+ RD string

+ ST int32

+ TS string

+ type CDXRawRow struct

+ ContentLanguages string

+ ContentMIMEDetected string

+ FetchStatus int32

+ FetchTime string

+ URLHostName string

+ URLHostRegisteredDomain string

+ WARCRecordLength int64

+ type HostDatasetRow struct

+ CrawlID string

+ FirstSeen string

+ GraphID string

+ HarmonicPos int64

+ HarmonicVal float64

+ Host string

+ HostRev string

+ InDegree int32

+ Language string

+ LastSeen string

+ OutDegree int32

+ PageRankPos int64

+ PageRankVal float64

+ RegisteredDomain string

+ Status2xx int64

+ Status3xx int64

+ Status4xx int64

+ Status5xx int64

+ TLD string

+ TopMIME string

+ TotalBytes int64

+ URLCount int64

v0.2.1

Jun 17, 2026

v0.2.0

Jun 17, 2026

Changes in this version

+ var DefaultBM25Params = BM25Params

+ var DefaultCrawlConfig = CrawlConfig

+ func BM25IDF(N, df int) float64

+ func BM25TF(tf, dl int, avgDL float64, p BM25Params) float64

+ func ComputeEdgeDegrees(ctx context.Context, h *HTTPClient, manifestURL string, nodeCount int64) (inDeg, outDeg []int32, err error)

+ func ContentSHA1(content []byte) string

+ func CrawlScore(harmonicVal, changeRate float64) float64

+ func CrawlTier(harmonicPos int64, changeRate float64) int

+ func DecodeVarInt(b []byte) (uint64, int)

+ func DiffCDX(ctx context.Context, urlsA, urlsB []string, crawlA, crawlB string, ...) error

+ func DiffCDXSQL(urlsA, urlsB []string, crawlA, crawlB string) string

+ func DocumentID(canonicalURL string) uint64

+ func EncodeVarInt(v uint64) []byte

+ func ExtractOutLinks(htmlBytes []byte, baseURL string) []string

+ func HTMLCanonicalURL(htmlBytes []byte) string

+ func HarmonicTier(harmonicPos int64, totalHosts int64) int

+ func HostCDXAgg(ctx context.Context, parquetURLs []string, crawlID, host string, ...) error

+ func HostCDXAggSQL(parquetURLs []string, crawlID, host string) string

+ func HostCDXSingleSQL(parquetURLs []string, crawlID, host string) string

+ func LinkBoost(bm25Score, harmonicVal, alpha float64) float64

+ func NormalizeURL(raw string) string

+ func RankStream(ctx context.Context, h *HTTPClient, url, tld string, fn func(Rank) error) error

+ func StripTrackingParams(rawQuery string) string

+ func TierBoost(tier int) float64

+ func TierInterval(tier int) int

+ func Tokenize(text string) []string

+ func VertexStream(ctx context.Context, h *HTTPClient, manifestURL string, ...) error

+ func WriteWARCResponse(w io.Writer, rec NewWARCRecord) error

+ type APIServer struct

+ func NewAPIServer(cfg ServeConfig, hosts HostStore, search SearchStore) *APIServer

+ func (s *APIServer) Addr() string

+ func (s *APIServer) ListenAndServe(ctx context.Context) error

+ type BM25Params struct

+ B float64

+ K1 float64

+ type CrawlConfig struct

+ MaxRedirect int

+ Timeout time.Duration

+ UserAgent string

+ type CrawlResult struct

+ Body []byte

+ ContentType string

+ Digest string

+ FetchedAt time.Time

+ FinalURL string

+ Links []string

+ Status int

+ URL string

+ func CrawlURL(ctx context.Context, rawURL string, cfg CrawlConfig) (*CrawlResult, error)

+ type EdgeDegrees struct

+ InDegree int32

+ NodeID int64

+ OutDegree int32

+ type ForwardDoc struct

+ CanonURL string

+ Description string

+ DocID uint64

+ Host string

+ Language string

+ LinkScore float32

+ Snippet string

+ Title string

+ URL string

+ WordCount int

+ type ForwardIndexWriter struct

+ func NewForwardIndexWriter(path string) (*ForwardIndexWriter, error)

+ func (fw *ForwardIndexWriter) Close() error

+ func (fw *ForwardIndexWriter) Write(d ForwardDoc) error

+ type Frontier struct

+ func NewFrontier(delay time.Duration) *Frontier

+ func (f *Frontier) Add(e FrontierEntry) bool

+ func (f *Frontier) Len() int

+ func (f *Frontier) Pop(now int64) (FrontierEntry, bool)

+ type FrontierEntry struct

+ Depth uint8

+ Host string

+ NextAt int64

+ Priority float32

+ Retries uint8

+ URL string

+ type HostCDXStats struct

+ FirstSeen string

+ Host string

+ Language string

+ LastSeen string

+ RegisteredDomain string

+ Status2xx int64

+ Status3xx int64

+ Status4xx int64

+ Status5xx int64

+ TopMIME string

+ TotalBytes int64

+ URLCount int64

+ type HostDiffEntry struct

+ ChangeRate float64

+ ChangedURLs int64

+ Host string

+ Tier int

+ TotalURLs int64

+ type HostRecord struct

+ FirstSeen string

+ HarmonicPos int64

+ HarmonicVal float64

+ Host string

+ HostRev string

+ InDegree int64

+ Language string

+ LastSeen string

+ OutDegree int64

+ PageRankPos int64

+ PageRankVal float64

+ RegisteredDomain string

+ Status2xx int64

+ Status3xx int64

+ Status4xx int64

+ Status5xx int64

+ TLD string

+ TopMIME string

+ TotalBytes int64

+ URLCount int64

+ func HostFromRank(r Rank) HostRecord

+ func HostLookupRank(ctx context.Context, h *HTTPClient, rankURL, host string) (HostRecord, error)

+ type HostSchedule struct

+ ChangeRate float64

+ HarmonicPos int64

+ HarmonicVal float64

+ Host string

+ IntervalH int

+ Score float64

+ Tier int

+ func HostScheduleFrom(r Rank, changeRate float64) HostSchedule

+ type HostStore interface

+ GetHost func(ctx context.Context, host string) (*HostRecord, error)

+ TopHosts func(ctx context.Context, n int, tld string) ([]HostRecord, error)

+ type IndexReader struct

+ AvgDL float64

+ N int

+ func OpenIndex(dir string) (*IndexReader, error)

+ func (r *IndexReader) Close() error

+ func (r *IndexReader) Lookup(term string) ([]PostingEntry, float64, bool)

+ func (r *IndexReader) Search(tokens []string, k int) []ScoredDoc

+ type IndexSearchStore struct

+ func NewIndexSearchStore(idx *IndexReader, forward map[uint64]ForwardDoc) *IndexSearchStore

+ func (s *IndexSearchStore) Search(_ context.Context, query string, k int) ([]SearchResult, error)

+ type InvertedIndexBuilder struct

+ N int

+ TermCount int

+ func NewInvertedIndexBuilder(dir string) (*InvertedIndexBuilder, error)

+ func (b *InvertedIndexBuilder) Add(docID uint64, tokens []string)

+ func (b *InvertedIndexBuilder) Flush() error

+ type MemHostStore struct

+ func NewMemHostStore(recs []HostRecord) *MemHostStore

+ func (s *MemHostStore) GetHost(_ context.Context, host string) (*HostRecord, error)

+ func (s *MemHostStore) TopHosts(_ context.Context, n int, tld string) ([]HostRecord, error)

+ type NewWARCRecord struct

+ Block []byte

+ Date string

+ RecordID string

+ TargetURI string

+ type PostingEntry struct

+ DL uint32

+ DocID uint64

+ TF uint32

+ type QualityResult struct

+ HasMainContent bool

+ IsParked bool

+ IsShortContent bool

+ SpamScore float64

+ TitleLength int

+ WordCount int

+ func QualitySignals(tr TextResult) QualityResult

+ type RobotsCache struct

+ func NewRobotsCache(ttl time.Duration, userAgent string) *RobotsCache

+ func (rc *RobotsCache) Get(host string) *RobotsEntry

+ func (rc *RobotsCache) Put(host string, e *RobotsEntry)

+ type RobotsEntry struct

+ CrawlDelay time.Duration

+ ExpiresAt int64

+ Rules []RobotsRule

+ func FetchRobots(ctx context.Context, h *HTTPClient, host, scheme string) *RobotsEntry

+ func (e *RobotsEntry) IsAllowed(path string) bool

+ type RobotsRule struct

+ Allow bool

+ Pattern string

+ type ScoredDoc struct

+ DocID uint64

+ Score float64

+ type SearchResult struct

+ DocID uint64

+ Host string

+ Language string

+ Score float64

+ Snippet string

+ Title string

+ URL string

+ type SearchStore interface

+ Search func(ctx context.Context, query string, k int) ([]SearchResult, error)

+ type ServeConfig struct

+ Addr string

+ DBPath string

+ IndexDir string

+ type TextResult struct

+ Body string

+ CanonURL string

+ Description string

+ Language string

+ Title string

+ WordCount int

+ func ExtractContent(htmlBytes []byte) TextResult

+ type VertexRecord struct

+ Host string

+ HostRev string

+ ID int64

+ type WebGraph struct

+ BaseURL string

+ HostArcs int64

+ HostNodes int64

+ ID string

+ Published string

+ func LatestWebGraph(ctx context.Context, h *HTTPClient, cache *Cache) (WebGraph, error)

+ func (g WebGraph) HostEdgesManifestURL() string

+ func (g WebGraph) HostRankURL() string

+ func (g WebGraph) HostVerticesManifestURL() string

v0.1.0

Jun 13, 2026

Changes in this version

+ const CDXBaseURL

+ const CollInfoURL

+ const ColumnarPrefix

+ const DataBaseURL

+ const DefaultDelay

+ const DefaultRetries

+ const DefaultTimeout

+ const DuckDBPrelude

+ const S3BaseURL

+ const UserAgent

+ var DefaultColumnarColumns = []string

+ var LocationColumns = []string

+ var PathKinds = []string

+ func CDXNumPages(ctx context.Context, h *HTTPClient, crawlID string, q CDXQuery) (int, error)

+ func CDXStream(ctx context.Context, h *HTTPClient, crawlID string, q CDXQuery, ...) error

+ func CanonicalURL(raw string) string

+ func ColumnarParquetURLs(ctx context.Context, h *HTTPClient, cache *Cache, crawlID, subset string, ...) ([]string, error)

+ func ColumnarSource(crawlID, subset string, src Source) string

+ func ConfigDir() string

+ func DownloadFiles(ctx context.Context, h *HTTPClient, src Source, paths []string, ...) error

+ func DuckDBAvailable() bool

+ func ExtractMarkdown(body []byte) (string, error)

+ func ExtractText(body []byte) string

+ func ExtractTitle(body []byte) string

+ func FetchPaths(ctx context.Context, h *HTTPClient, cache *Cache, crawlID, kind string) ([]string, error)

+ func FileURL(path string, src Source) string

+ func HTTPBody(block []byte) []byte

+ func HTTPHeaders(block []byte) []byte

+ func HTTPSURL(path string) string

+ func HostOf(raw string) string

+ func InferMatchType(pattern string) (cleanURL, matchType string)

+ func IterateWARC(r io.Reader, fn func(WARCRecord) error) error

+ func IterateWAT(r io.Reader, crawlID string, fn func(WATRecord) error) error

+ func IterateWET(r io.Reader, crawlID string, fn func(WETRecord) error) error

+ func LibraryDir() string

+ func ParquetListLiteral(urls []string) string

+ func ResolveCrawl(ctx context.Context, h *HTTPClient, cache *Cache, ref string) (string, error)

+ func RunColumnarDuckDB(ctx context.Context, sql string, emit func(map[string]any) error) error

+ func RunDuckDBJSON(ctx context.Context, dbPath, sql string, emit func(map[string]any) error) error

+ func SURT(raw string) string

+ func StreamPaths(ctx context.Context, h *HTTPClient, crawlID, kind string, ...) error

+ type CDXQuery struct

+ Filter []string

+ From string

+ Lang string

+ Limit int

+ MIME string

+ Match string

+ Status string

+ To string

+ URL string

+ type CDXRecord struct

+ Charset string

+ CrawlID string

+ Digest string

+ Filename string

+ Languages string

+ Length string

+ MIME string

+ MIMEDetected string

+ Offset string

+ Redirect string

+ Status string

+ Timestamp string

+ Truncated string

+ URL string

+ URLKey string

+ func CDXSearch(ctx context.Context, h *HTTPClient, crawlID string, q CDXQuery) ([]CDXRecord, error)

+ func (r CDXRecord) Location() Location

+ func (r CDXRecord) Time() time.Time

+ type Cache struct

+ func NewCache(dir string, enabled bool) *Cache

+ func (c *Cache) Clear() (int, error)

+ func (c *Cache) Dir() string

+ func (c *Cache) Get(key string, ttl time.Duration) ([]byte, bool)

+ func (c *Cache) Put(key string, data []byte)

+ type ColumnarQuery struct

+ Crawl string

+ Domain string

+ Host string

+ Lang string

+ Limit int

+ MIME string

+ PathPrefix string

+ Select []string

+ Status int

+ Subset string

+ TLD string

+ func (q ColumnarQuery) SQL(src Source) string

+ type Config struct

+ CacheDir string

+ CrawlID string

+ DBPath string

+ DataDir string

+ Delay time.Duration

+ Retries int

+ Source Source

+ Timeout time.Duration

+ UserAgent string

+ Workers int

+ func DefaultConfig() Config

+ func (c Config) ParquetDir() string

+ func (c Config) RawDir() string

+ type Crawl struct

+ CDXAPI string

+ From string

+ ID string

+ Name string

+ To string

+ func ListCrawls(ctx context.Context, h *HTTPClient, cache *Cache) ([]Crawl, error)

+ type DownloadResult struct

+ Bytes int64

+ Err error

+ LocalPath string

+ Path string

+ Skipped bool

+ type HTTPClient struct

+ func NewHTTPClient(cfg Config) *HTTPClient

+ func (h *HTTPClient) FetchBytes(ctx context.Context, url string) ([]byte, error)

+ func (h *HTTPClient) Get(ctx context.Context, url string) (*http.Response, error)

+ func (h *HTTPClient) GetDownload(ctx context.Context, url string) (*http.Response, error)

+ func (h *HTTPClient) GetRange(ctx context.Context, url string, offset, length int64) (*http.Response, error)

+ type Library struct

+ Crawl string

+ Root string

+ func NewLibrary(root, crawl string) Library

+ func (l Library) CrawlDir() string

+ func (l Library) ProcessedDir(format, kind string) string

+ func (l Library) RawDir(kind string) string

+ func (l Library) RawPath(kind, ccPath string) string

+ type Location struct

+ Filename string

+ Length int64

+ Offset int64

+ URL string

+ type NewsFile struct

+ Mon int

+ Path string

+ Year int

+ func ListNewsFiles(ctx context.Context, h *HTTPClient, year, month int) ([]NewsFile, error)

+ type ParquetWriter struct

+ func NewParquetWriter[T any](path string) (*ParquetWriter[T], error)

+ func (p *ParquetWriter[T]) Close() error

+ func (p *ParquetWriter[T]) Rows() int64

+ func (p *ParquetWriter[T]) Write(row T) error

+ type Rank struct

+ HarmonicPos int64

+ HarmonicVal float64

+ Key string

+ PageRankPos int64

+ PageRankVal float64

+ func RankLookup(ctx context.Context, h *HTTPClient, url, hostOrDomain string) (Rank, error)

+ func RankTop(ctx context.Context, h *HTTPClient, url, tld string, n int) ([]Rank, error)

+ type Source string

+ const SourceHTTPS

+ const SourceS3

+ type WARCHeader = warc.Header

+ type WARCParquetRow struct

+ ContentLength int64

+ ContentType string

+ CrawlID string

+ Date time.Time

+ HTTPMIME string

+ HTTPStatus int32

+ IPAddress string

+ Language string

+ Markdown string

+ PayloadDigest string

+ RecordID string

+ TargetURI string

+ Text string

+ Title string

+ Truncated string

+ WARCFilename string

+ WARCLength int64

+ WARCOffset int64

+ WARCType string

+ type WARCRecord = warc.Record

+ func FetchWARCRecord(ctx context.Context, h *HTTPClient, filename string, offset, length int64) (WARCRecord, error)

+ type WATLink = wat.Link

+ func ExtractLinks(base string, body []byte) []WATLink

+ type WATMeta = wat.Meta

+ type WATParquetRow struct

+ ContentType string

+ CrawlID string

+ Date time.Time

+ HTTPStatus int32

+ Links string

+ LinksCount int32

+ Metas string

+ RecordID string

+ Title string

+ URL string

+ type WATRecord = wat.Record

+ type WETParquetRow struct

+ ContentLanguage string

+ CrawlID string

+ Date time.Time

+ RecordID string

+ Text string

+ TextLength int32

+ URL string

+ type WETRecord = wet.Record

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL