scraper

package
v0.0.0-...-e7c4f5f Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 8, 2025 License: MIT Imports: 12 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Config

type Config struct {
	CacheDir string
	Workers  int
}

type Index

type Index struct {
	Path      string    `yaml:"path"`
	UpdatedAt time.Time `yaml:"updated_at"`
	Links     []Link    `yaml:"links"`
}

type IndexJob

type IndexJob struct {
	Path     string
	LatestAt time.Time
}

type IndexMsg

type IndexMsg struct {
	Found   int
	Indexed int
}
type Link struct {
	IsDir     bool      `yaml:"isDir"`
	Link      string    `yaml:"link"`
	Label     string    `yaml:"label"`
	Size      string    `yaml:"size,omitempty"`
	UpdatedAt time.Time `yaml:"updatedAt"`
	IndexedAt time.Time `yaml:"indexedAt"`
}

type ResultsMsg

type ResultsMsg struct {
	Index Index
	Err   error
}

type Scraper

type Scraper struct {
	Config Config
}

func New

func New(cacheDir string) Scraper

func (Scraper) CachedIndexRead

func (s Scraper) CachedIndexRead(path string) (index Index, err error)

func (Scraper) CachedIndexWrite

func (s Scraper) CachedIndexWrite(index Index)

func (Scraper) DoScrape

func (s Scraper) DoScrape(path string) (index Index, err error)

func (Scraper) IndexAll

func (s Scraper) IndexAll()

func (Scraper) IndexPreload

func (s Scraper) IndexPreload(index Index)

func (Scraper) IndexRecursive

func (s Scraper) IndexRecursive(wg *sync.WaitGroup, queue chan<- IndexJob, job IndexJob)

func (Scraper) IndexWorker

func (s Scraper) IndexWorker(wg *sync.WaitGroup, qr <-chan IndexJob, qw chan<- IndexJob)

func (Scraper) MsgScrape

func (s Scraper) MsgScrape(path string) tea.Cmd

func (Scraper) ScrapePath

func (s Scraper) ScrapePath(path string) (index Index, err error)

*

  • Scrape a page and extract the results

type StartedMsg

type StartedMsg struct{}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL