Versions in this module Expand all Collapse all v1 v1.0.0 Feb 1, 2026 Changes in this version + func ComputeContentHash(content string) string + type BoltStore struct + func NewBoltStore(path string) (*BoltStore, error) + func (s *BoltStore) Close() error + func (s *BoltStore) Load() (*CrawlerState, error) + func (s *BoltStore) Save(state *CrawlerState) error + type CrawlError struct + Error string + Timestamp time.Time + URL string + type CrawlStats struct + APIEndpoints int + BytesTransferred int64 + Duration time.Duration + ErrorCount int + FormsFound int + PagesCrawled int + URLsDiscovered int + WebSocketEndpoints int + type CrawlerState struct + Config json.RawMessage + Endpoints []Endpoint + Errors []CrawlError + Forms []Form + QueueURLs []string + StartedAt time.Time + Stats CrawlStats + Target string + UpdatedAt time.Time + VisitedURLs []string + WebSockets []WebSocketEndpoint + type Deduplicator struct + func NewDeduplicator(estimatedItems int) *Deduplicator + func (d *Deduplicator) Add(url string) + func (d *Deduplicator) AddBatch(urls []string) + func (d *Deduplicator) Count() int + func (d *Deduplicator) FalsePositiveRate() float64 + func (d *Deduplicator) GetAll() []string + func (d *Deduplicator) HasSeen(url string) bool + func (d *Deduplicator) Merge(other *Deduplicator) + func (d *Deduplicator) Reset() + type Endpoint struct + ContentType string + Depth int + DiscoveredFrom string + Headers map[string]string + Method string + Parameters []Parameter + ResponseSize int64 + Source string + StatusCode int + Timestamp time.Time + URL string + type FileStore struct + func NewFileStore(path string, compressed bool) *FileStore + func (s *FileStore) Close() error + func (s *FileStore) Load() (*CrawlerState, error) + func (s *FileStore) Save(state *CrawlerState) error + type Form struct + Action string + Depth int + Enctype string + HasCSRF bool + Inputs []FormInput + Method string + Timestamp time.Time + URL string + type FormInput struct + MaxLength int + MinLength int + Name string + Pattern string + Placeholder string + Required bool + Type string + Value string + type HashAwareDeduplicator struct + func NewHashAwareDeduplicator(maxSize int) *HashAwareDeduplicator + func (d *HashAwareDeduplicator) AddBatch(urls []string) + func (d *HashAwareDeduplicator) ExtractRoutingFragment(rawURL string) string + func (d *HashAwareDeduplicator) GetAll() []string + func (d *HashAwareDeduplicator) GetContentHash(rawURL string) (string, bool) + func (d *HashAwareDeduplicator) HasDuplicateContent(rawURL string, contentHash string) (bool, string) + func (d *HashAwareDeduplicator) HasVisited(rawURL string) bool + func (d *HashAwareDeduplicator) MarkVisited(rawURL string) + func (d *HashAwareDeduplicator) NormalizeURL(rawURL string) string + func (d *HashAwareDeduplicator) SetContentHash(rawURL string, contentHash string) + func (d *HashAwareDeduplicator) ShouldSkipFragment(fragment string) bool + func (d *HashAwareDeduplicator) Stats() map[string]int + type Manager struct + func NewManager(store Store, estimatedURLs int) *Manager + func (m *Manager) AddAPIEndpoint() + func (m *Manager) AddBytes(n int64) + func (m *Manager) AddDiscoveredURL() + func (m *Manager) AddError() + func (m *Manager) AddForm() + func (m *Manager) AddWebSocket() + func (m *Manager) GetDeduplicator() *Deduplicator + func (m *Manager) GetHashDeduplicator() *HashAwareDeduplicator + func (m *Manager) GetSoftErrors() map[string]string + func (m *Manager) GetStats() CrawlStats + func (m *Manager) HasDuplicateContent(url, contentHash string) (bool, string) + func (m *Manager) HasVisited(url string) bool + func (m *Manager) IsSoftError(url string) (bool, string) + func (m *Manager) Load() (*CrawlerState, error) + func (m *Manager) MarkSoftError(url, errorMsg string) + func (m *Manager) MarkVisited(url string) bool + func (m *Manager) NormalizeURL(url string) string + func (m *Manager) Reset() + func (m *Manager) Save(state *CrawlerState) error + func (m *Manager) SetContentHash(url, contentHash string) + func (m *Manager) SetTarget(target string) + func (m *Manager) ShouldSkipFragment(fragment string) bool + func (m *Manager) Start(target string) + type MemoryStore struct + func NewMemoryStore() *MemoryStore + func (s *MemoryStore) Close() error + func (s *MemoryStore) Load() (*CrawlerState, error) + func (s *MemoryStore) Save(state *CrawlerState) error + type Parameter struct + Example string + Name string + Required bool + Type string + type Store interface + Close func() error + Load func() (*CrawlerState, error) + Save func(state *CrawlerState) error + type WebSocketEndpoint struct + DiscoveredFrom string + Protocols []string + SampleMessages []WebSocketMsg + Timestamp time.Time + URL string + type WebSocketMsg struct + Data string + Direction string + Timestamp time.Time + Type string
Go to main page