Documentation
¶
Index ¶
- func DefaultStatePath(resultsPath string) string
- func EntryIdentity(entry *gmaps.Entry) string
- func NewCSVAppendWriter(w *csv.Writer, writeHeader bool, ids *IdentitySet, tracker ResultTracker, ...) scrapemate.ResultWriter
- func NewJSONLAppendWriter(w io.Writer, ids *IdentitySet, tracker ResultTracker, syncOutput func() error) scrapemate.ResultWriter
- type IdentitySet
- func (s *IdentitySet) Add(key string)
- func (s *IdentitySet) AddEntry(entry *gmaps.Entry)
- func (s *IdentitySet) AddIfNotExists(_ context.Context, key string) bool
- func (s *IdentitySet) Clone() *IdentitySet
- func (s *IdentitySet) Has(key string) bool
- func (s *IdentitySet) HasEntry(entry *gmaps.Entry) bool
- type InputMarker
- type ProgressTracker
- type ResultTracker
- type State
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func DefaultStatePath ¶
DefaultStatePath returns the sidecar path for a results file.
func EntryIdentity ¶
EntryIdentity returns the preferred stable identity for entry.
func NewCSVAppendWriter ¶
func NewCSVAppendWriter( w *csv.Writer, writeHeader bool, ids *IdentitySet, tracker ResultTracker, syncOutput func() error, headers ...[]string, ) scrapemate.ResultWriter
NewCSVAppendWriter creates a CSV writer suitable for resume append mode. Optional headers preserve the column layout of an existing CSV file.
func NewJSONLAppendWriter ¶
func NewJSONLAppendWriter( w io.Writer, ids *IdentitySet, tracker ResultTracker, syncOutput func() error, ) scrapemate.ResultWriter
NewJSONLAppendWriter creates a JSONL writer suitable for resume append mode.
Types ¶
type IdentitySet ¶
type IdentitySet struct {
// contains filtered or unexported fields
}
IdentitySet tracks stable place identities already emitted to a result file.
func LoadResultIdentities ¶
func LoadResultIdentities(path string, jsonl bool) (*IdentitySet, error)
LoadResultIdentities reads existing CSV or JSONL results into an IdentitySet.
func NewIdentitySet ¶
func NewIdentitySet() *IdentitySet
NewIdentitySet creates an empty result identity set.
func (*IdentitySet) Add ¶
func (s *IdentitySet) Add(key string)
Add records a stable identity key. Empty keys are ignored.
func (*IdentitySet) AddEntry ¶
func (s *IdentitySet) AddEntry(entry *gmaps.Entry)
AddEntry records all stable identities for entry.
func (*IdentitySet) AddIfNotExists ¶
func (s *IdentitySet) AddIfNotExists(_ context.Context, key string) bool
AddIfNotExists implements deduper.Deduper for discovered place URLs.
func (*IdentitySet) Clone ¶
func (s *IdentitySet) Clone() *IdentitySet
Clone returns an independent copy of the identity set.
func (*IdentitySet) Has ¶
func (s *IdentitySet) Has(key string) bool
Has reports whether key has already been recorded.
type InputMarker ¶
InputMarker persists fully completed input IDs.
type ProgressTracker ¶
type ProgressTracker struct {
// contains filtered or unexported fields
}
ProgressTracker tracks per-input discovery and child job completion.
func NewProgressTracker ¶
func NewProgressTracker(marker InputMarker) *ProgressTracker
NewProgressTracker creates a per-input completion tracker.
func (*ProgressTracker) ResultPersisted ¶
func (t *ProgressTracker) ResultPersisted(inputID string, syncOutput func() error) error
ResultPersisted records that one output-producing child was durably handled.
func (*ProgressTracker) SeedDiscovered ¶
func (t *ProgressTracker) SeedDiscovered(inputID string, placesFound int) error
SeedDiscovered records that an input's search job finished discovery.
type ResultTracker ¶
ResultTracker acknowledges results after they have been persisted.
type State ¶
type State struct {
// contains filtered or unexported fields
}
State tracks input queries that fully completed in a resume-capable run.
func (*State) IsInputCompleted ¶
IsInputCompleted reports whether inputID has been marked complete.
func (*State) MarkInputCompleted ¶
MarkInputCompleted records and persists a completed input ID.