scrape

type GroupStats struct {
	Low        int64
	High       int64
	ProviderID string
}

type NNTPAdapter struct {
	// contains filtered or unexported fields
}

type Options struct {
	Newsgroups               []string
	BatchSize                int64
	Concurrency              int
	MaxBatches               int
	MaxNewSourceDaysPerPass  int
	BackfillUntilDateByGroup map[string]time.Time
	RangeCoordinator         RangeCoordinator
	RunObserver              func(context.Context, map[string]any, error)
	Timeframes               []Timeframe
	DeferredClaimOwner       string
	DeferredClaimLease       time.Duration
	DeferredMaxAttempts      int
}

type OverviewHeader struct {
	ArticleNumber int64
	MessageID     string
	Subject       string
	Poster        string
	DateUTC       *time.Time
	Bytes         int64
	Lines         int
	Xref          string
	RawOverview   map[string]any
}

type RangeCoordinator interface {
	BeginScrapeRange(ctx context.Context, request RangeRequest) (RangeDecision, error)
	CompleteScrapeRange(ctx context.Context, decision RangeDecision, result RangeResult) error
	FailScrapeRange(ctx context.Context, decision RangeDecision, cause error) error
}

type RangeDecision struct {
	PoolID            string
	ClaimID           string
	AssignmentID      string
	Group             string
	RangeStart        int64
	RangeEnd          int64
	WindowStart       *time.Time
	WindowEnd         *time.Time
	Skipped           bool
	AdvanceCheckpoint bool
	Reason            string
}

type RangeRequest struct {
	PoolID       string
	Mode         string
	AssignmentID string
	Group        string
	RangeStart   int64
	RangeEnd     int64
	WindowStart  *time.Time
	WindowEnd    *time.Time
}

type RangeResult struct {
	Mode             string
	Group            string
	RangeStart       int64
	RangeEnd         int64
	ArticleHeaders   int
	ArticlesInserted int64
}

type Service struct {
	// contains filtered or unexported fields
}

type Timeframe struct {
	ID    string
	Group string
	Start time.Time
	End   time.Time
}