crawl

type Decision struct {
	Terminal    bool
	FailureKind string
	RetryAt     time.Time
}

type Frontier interface {
	LeaseFrontierItems(ctx context.Context, worker string, now time.Time, leaseDuration time.Duration, limit, budget int) ([]corpus.FrontierItem, error)
	ReleaseFrontierItem(ctx context.Context, id int64, worker string, now time.Time) error
	CompleteFrontierItem(ctx context.Context, id int64, worker string, now time.Time) error
	RetryFrontierItem(ctx context.Context, id int64, worker, message string, earliestRunAt, now time.Time) error
	FailFrontierItem(ctx context.Context, id int64, worker, failureKind, message string, now time.Time) error
}

type Handler interface {
	Handle(context.Context, corpus.FrontierItem) error
}

type HandlerFunc func(context.Context, corpus.FrontierItem) error

type Stats struct {
	Leased    int
	Completed int
	Retried   int
	Failed    int
	Budget    int
}

type TerminalError struct {
	Kind string
	Err  error
}

type Worker struct {
	Frontier       Frontier
	Handler        Handler
	ID             string
	BatchSize      int
	Budget         int
	LeaseDuration  time.Duration
	HandlerTimeout time.Duration
	Backoff        time.Duration
	Now            func() time.Time
	Classify       func(error, corpus.FrontierItem, time.Time) Decision
}