Documentation
¶
Overview ¶
Package queue provides URL queue implementations for the crawler.
Index ¶
- Variables
- type FastQueue
- func (fq *FastQueue) Clear() error
- func (fq *FastQueue) Close() error
- func (fq *FastQueue) Contains(url string) bool
- func (fq *FastQueue) IsEmpty() bool
- func (fq *FastQueue) Len() int
- func (fq *FastQueue) Pop() (*QueueItem, error)
- func (fq *FastQueue) PopBatch(n int) ([]*QueueItem, error)
- func (fq *FastQueue) PopWait() (*QueueItem, error)
- func (fq *FastQueue) Push(item *QueueItem) error
- func (fq *FastQueue) PushBatch(items []*QueueItem) (int, error)
- func (fq *FastQueue) Stats() QueueStats
- func (fq *FastQueue) URLs() []string
- type MemoryQueue
- func (mq *MemoryQueue) Clear() error
- func (mq *MemoryQueue) Close() error
- func (mq *MemoryQueue) Contains(url string) bool
- func (mq *MemoryQueue) IsEmpty() bool
- func (mq *MemoryQueue) Len() int
- func (mq *MemoryQueue) Peek() (*QueueItem, error)
- func (mq *MemoryQueue) Pop() (*QueueItem, error)
- func (mq *MemoryQueue) PopWait() (*QueueItem, error)
- func (mq *MemoryQueue) Push(item *QueueItem) error
- func (mq *MemoryQueue) URLs() []string
- type PersistentQueue
- func (pq *PersistentQueue) Clear() error
- func (pq *PersistentQueue) Close() error
- func (pq *PersistentQueue) Contains(url string) bool
- func (pq *PersistentQueue) IsEmpty() bool
- func (pq *PersistentQueue) Len() int
- func (pq *PersistentQueue) Peek() (*QueueItem, error)
- func (pq *PersistentQueue) Pop() (*QueueItem, error)
- func (pq *PersistentQueue) Push(item *QueueItem) error
- func (pq *PersistentQueue) Stats() (queueSize, visitedCount int)
- type Queue
- type QueueItem
- type QueueStats
Constants ¶
This section is empty.
Variables ¶
var ( ErrQueueEmpty = errors.New("queue is empty") ErrQueueClosed = errors.New("queue is closed") )
Functions ¶
This section is empty.
Types ¶
type FastQueue ¶
type FastQueue struct {
// contains filtered or unexported fields
}
FastQueue is a high-performance queue optimized for concurrent access. It uses sharding to reduce lock contention and supports batch operations.
func NewFastQueue ¶
NewFastQueue creates a new high-performance queue.
func (*FastQueue) Stats ¶
func (fq *FastQueue) Stats() QueueStats
type MemoryQueue ¶
type MemoryQueue struct {
// contains filtered or unexported fields
}
MemoryQueue is a thread-safe in-memory priority queue.
func NewMemoryQueue ¶
func NewMemoryQueue(capacity int) *MemoryQueue
NewMemoryQueue creates a new in-memory queue.
func (*MemoryQueue) Clear ¶
func (mq *MemoryQueue) Clear() error
Clear removes all items from the queue.
func (*MemoryQueue) Contains ¶
func (mq *MemoryQueue) Contains(url string) bool
Contains checks if a URL is in the queue.
func (*MemoryQueue) IsEmpty ¶
func (mq *MemoryQueue) IsEmpty() bool
IsEmpty returns true if the queue is empty.
func (*MemoryQueue) Len ¶
func (mq *MemoryQueue) Len() int
Len returns the number of items in the queue.
func (*MemoryQueue) Peek ¶
func (mq *MemoryQueue) Peek() (*QueueItem, error)
Peek returns the next item without removing it.
func (*MemoryQueue) Pop ¶
func (mq *MemoryQueue) Pop() (*QueueItem, error)
Pop removes and returns the next item from the queue.
func (*MemoryQueue) PopWait ¶
func (mq *MemoryQueue) PopWait() (*QueueItem, error)
PopWait removes and returns the next item, blocking if empty.
func (*MemoryQueue) Push ¶
func (mq *MemoryQueue) Push(item *QueueItem) error
Push adds an item to the queue.
func (*MemoryQueue) URLs ¶
func (mq *MemoryQueue) URLs() []string
URLs returns all URLs currently in the queue.
type PersistentQueue ¶
type PersistentQueue struct {
// contains filtered or unexported fields
}
PersistentQueue is a disk-backed queue using BoltDB.
func NewPersistentQueue ¶
func NewPersistentQueue(dbPath string, maxMemory int) (*PersistentQueue, error)
NewPersistentQueue creates a new persistent queue.
func (*PersistentQueue) Clear ¶
func (pq *PersistentQueue) Clear() error
Clear removes all items from the queue.
func (*PersistentQueue) Close ¶
func (pq *PersistentQueue) Close() error
Close closes the queue and database.
func (*PersistentQueue) Contains ¶
func (pq *PersistentQueue) Contains(url string) bool
Contains checks if a URL is in the queue or has been visited.
func (*PersistentQueue) IsEmpty ¶
func (pq *PersistentQueue) IsEmpty() bool
IsEmpty returns true if the queue is empty.
func (*PersistentQueue) Len ¶
func (pq *PersistentQueue) Len() int
Len returns the approximate number of items in the queue.
func (*PersistentQueue) Peek ¶
func (pq *PersistentQueue) Peek() (*QueueItem, error)
Peek returns the next item without removing it.
func (*PersistentQueue) Pop ¶
func (pq *PersistentQueue) Pop() (*QueueItem, error)
Pop removes and returns the next item from the queue.
func (*PersistentQueue) Push ¶
func (pq *PersistentQueue) Push(item *QueueItem) error
Push adds an item to the queue.
func (*PersistentQueue) Stats ¶
func (pq *PersistentQueue) Stats() (queueSize, visitedCount int)
Stats returns queue statistics.
type Queue ¶
type Queue interface {
// Push adds an item to the queue
Push(item *QueueItem) error
// Pop removes and returns the next item from the queue
Pop() (*QueueItem, error)
// Peek returns the next item without removing it
Peek() (*QueueItem, error)
// Len returns the number of items in the queue
Len() int
// IsEmpty returns true if the queue is empty
IsEmpty() bool
// Clear removes all items from the queue
Clear() error
// Close closes the queue and releases resources
Close() error
// Contains checks if a URL is already in the queue
Contains(url string) bool
}
Queue defines the interface for URL queues.
type QueueItem ¶
type QueueItem struct {
URL string
Method string
Depth int
ParentURL string
Headers map[string]string
Body []byte
Priority int
Timestamp time.Time
}
QueueItem represents an item in the crawl queue.
type QueueStats ¶
Stats returns queue statistics.
Source Files