Documentation
¶
Overview ¶
Package foreman is a Postgres-backed job queue for Go. Jobs are enqueued in the same transaction as your data.
Example ¶
package main
import (
"context"
"fmt"
"log"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/thnhdatle/foreman"
"github.com/thnhdatle/foreman/migrate"
)
type EmailArgs struct {
UserID int64 `json:"user_id"`
Tmpl string `json:"tmpl"`
}
func (EmailArgs) Kind() string { return "email" }
type EmailWorker struct{}
func (w *EmailWorker) Work(ctx context.Context, job *foreman.Job[EmailArgs]) error {
fmt.Printf("sending %s to user %d\n", job.Args.Tmpl, job.Args.UserID)
return nil
}
func main() {
if err := run(context.Background()); err != nil {
log.Fatal(err)
}
}
func run(ctx context.Context) error {
pool, err := pgxpool.New(ctx, "postgres://foreman:foreman@localhost:55432/foreman")
if err != nil {
return err
}
defer pool.Close()
if err := migrate.Up(ctx, pool); err != nil {
return err
}
workers := foreman.NewWorkers()
foreman.AddWorker(workers, &EmailWorker{})
client, err := foreman.NewClient(pool, foreman.Config{
Queues: map[string]foreman.QueueConfig{"default": {MaxWorkers: 50, PerKeyLimit: 5, Fairness: true}},
Workers: workers,
})
if err != nil {
return err
}
if err := client.Start(ctx); err != nil {
return err
}
defer func() { _ = client.Stop(ctx) }()
tx, err := pool.Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
if _, err := client.InsertTx(ctx, tx, EmailArgs{UserID: 42, Tmpl: "welcome"},
&foreman.InsertOpts{ConcurrencyKey: "tenant-7"}); err != nil {
return err
}
return tx.Commit(ctx)
}
Output:
Index ¶
- Constants
- Variables
- func AddWorker[T JobArgs](w *Workers, worker Worker[T], opts ...WorkerOption)
- func Cancel(err error) error
- func CompleteTx[T JobArgs](ctx context.Context, tx pgx.Tx, job *Job[T]) error
- func DecodeArgs[T JobArgs](row *JobRow) (T, error)
- func Discard(err error) error
- func Heartbeat(ctx context.Context)
- func Snooze(d time.Duration) error
- type AttemptError
- type Client
- func (c *Client) ID() string
- func (c *Client) Insert(ctx context.Context, args JobArgs, opts *InsertOpts) (*InsertResult, error)
- func (c *Client) InsertMany(ctx context.Context, params []InsertParams) ([]*InsertResult, error)
- func (c *Client) InsertManyTx(ctx context.Context, tx pgx.Tx, params []InsertParams) ([]*InsertResult, error)
- func (c *Client) InsertTx(ctx context.Context, tx pgx.Tx, args JobArgs, opts *InsertOpts) (*InsertResult, error)
- func (c *Client) IsLeader() bool
- func (c *Client) JobCancel(ctx context.Context, id int64) (*JobRow, error)
- func (c *Client) JobDeleteMany(ctx context.Context, f *JobFilter) (int, error)
- func (c *Client) JobDiscard(ctx context.Context, id int64) (*JobRow, error)
- func (c *Client) JobDiscardMany(ctx context.Context, f *JobFilter) (int, error)
- func (c *Client) JobGet(ctx context.Context, id int64) (*JobRow, error)
- func (c *Client) JobList(ctx context.Context, p *JobListParams) (*JobListResult, error)
- func (c *Client) JobRetry(ctx context.Context, id int64) (*JobRow, error)
- func (c *Client) JobRetryMany(ctx context.Context, f *JobFilter) (int, error)
- func (c *Client) JobWork(ctx context.Context, id int64) (*JobRow, error)
- func (c *Client) LeaderGet(ctx context.Context) (*LeaderInfo, error)
- func (c *Client) PeriodicList(ctx context.Context) ([]*PeriodicInfo, error)
- func (c *Client) Pool() *pgxpool.Pool
- func (c *Client) QueueList(ctx context.Context) ([]*QueueInfo, error)
- func (c *Client) QueuePause(ctx context.Context, name string) error
- func (c *Client) QueueResume(ctx context.Context, name string) error
- func (c *Client) Schema() string
- func (c *Client) Start(ctx context.Context) error
- func (c *Client) Stop(ctx context.Context) error
- func (c *Client) StopAndCancel(ctx context.Context) error
- type Clock
- type Config
- type ConstantBackoff
- type ErrorHandler
- type ErrorHandlerResult
- type ExponentialBackoff
- type InsertOpts
- type InsertParams
- type InsertResult
- type Job
- type JobArgs
- type JobArgsWithInsertOpts
- type JobFilter
- type JobListParams
- type JobListResult
- type JobRow
- type LeaderInfo
- type Middleware
- type MiddlewareDefaults
- type PeriodicInfo
- type PeriodicJob
- type QueueConfig
- type QueueInfo
- type RetryPolicy
- type RetryPolicyFunc
- type Schedule
- type State
- type Ticker
- type UniqueOpts
- type WorkFunc
- type Worker
- type WorkerOption
- type Workers
Examples ¶
- Package
- AddWorker
- Cancel
- Client.Insert
- Client.InsertMany
- Client.InsertManyTx
- Client.InsertTx
- Client.JobCancel
- Client.JobDeleteMany
- Client.JobDiscard
- Client.JobDiscardMany
- Client.JobGet
- Client.JobList
- Client.JobRetry
- Client.JobRetryMany
- Client.JobWork
- Client.LeaderGet
- Client.PeriodicList
- Client.QueueList
- Client.QueuePause
- Client.QueueResume
- Client.Start
- Client.StopAndCancel
- ClientFromContext
- CompleteTx
- Cron
- DecodeArgs
- Discard
- Every
- ExponentialBackoff
- Heartbeat
- MustCron
- NewClient
- NewWorkers
- Snooze
Constants ¶
const ( DefaultListLimit = 50 MaxListLimit = 500 )
Pagination limits of JobList.
const ( DefaultMaxWorkers = 10 DefaultFetchInterval = time.Second DefaultHeartbeatInterval = 30 * time.Second DefaultRescueStuckJobsAfter = time.Hour DefaultCompletedJobRetention = 24 * time.Hour DefaultCancelledJobRetention = 7 * 24 * time.Hour DefaultDiscardedJobRetention = 7 * 24 * time.Hour DefaultReindexInterval = 24 * time.Hour DefaultCleanerInterval = 30 * time.Second DefaultMaxArgsBytes = 1 << 20 DefaultSchema = "public" )
Defaults of Config.
const ( DefaultQueue = "default" DefaultPriority = 2 DefaultMaxAttempts = 25 )
Default insert values.
Variables ¶
var ( // ErrArgsTooLarge reports args whose JSON exceeds Config.MaxArgsBytes. ErrArgsTooLarge = errors.New("foreman: args too large") // ErrUniqueArgsNotCanonical reports a non-integral number in unique args. ErrUniqueArgsNotCanonical = unique.ErrArgsNotCanonical // ErrInvalidConfig wraps every problem found by NewClient. ErrInvalidConfig = errors.New("foreman: invalid config") // ErrInvalidInsertOpts wraps every problem with InsertOpts or UniqueOpts. ErrInvalidInsertOpts = errors.New("foreman: invalid insert options") // ErrInvalidArgument reports a nil or malformed argument. ErrInvalidArgument = errors.New("foreman: invalid argument") // ErrNotFound reports a job id that does not exist. ErrNotFound = job.ErrNotFound // ErrInvalidTransition reports an admin operation not allowed in the job's state. ErrInvalidTransition = job.ErrInvalidTransition // ErrUniqueConflict reports a retry that would duplicate a unique job. ErrUniqueConflict = job.ErrUniqueConflict // ErrStaleJob reports that the attempt is no longer owned by this client. ErrStaleJob = job.ErrStale // ErrNotInWorker reports a call that is only valid inside Worker.Work. ErrNotInWorker = errors.New("foreman: not called from inside Work") // ErrAlreadyStarted reports a second Start of a running client. ErrAlreadyStarted = errors.New("foreman: client already started") // ErrNotStarted reports Stop of a client that is not running. ErrNotStarted = errors.New("foreman: client not started") // ErrSchemaNotMigrated reports a database without the required Foreman schema. ErrSchemaNotMigrated = errors.New("foreman: schema not migrated") // ErrNoWorker reports a kind without a registered worker. ErrNoWorker = job.ErrNoWorker // ErrClientStopping is the cause of a job's context when the client cancels it on shutdown. ErrClientStopping = job.ErrClientStopping )
Errors returned by Foreman.
Functions ¶
func AddWorker ¶
func AddWorker[T JobArgs](w *Workers, worker Worker[T], opts ...WorkerOption)
AddWorker registers worker for the kind of T.
Example ¶
package main
import (
"context"
"fmt"
"time"
"github.com/thnhdatle/foreman"
)
type EmailArgs struct {
UserID int64 `json:"user_id"`
Tmpl string `json:"tmpl"`
}
func (EmailArgs) Kind() string { return "email" }
func main() {
workers := foreman.NewWorkers()
foreman.AddWorker(workers, foreman.WorkFunc[EmailArgs](func(ctx context.Context, job *foreman.Job[EmailArgs]) error {
return nil
}), foreman.WithTimeout(time.Minute), foreman.WithMaxAttempts(5),
foreman.WithRetryPolicy(foreman.ConstantBackoff{Delay: 10 * time.Second}))
fmt.Println(workers.Kinds())
}
Output: [email]
func Cancel ¶
Cancel returns an error that cancels the job.
Example ¶
package main
import (
"errors"
"fmt"
"github.com/thnhdatle/foreman"
)
func main() {
err := foreman.Cancel(errors.New("user deleted"))
fmt.Println(err)
}
Output: job cancelled: user deleted
func CompleteTx ¶
CompleteTx marks the job completed inside tx.
Example ¶
package main
import (
"context"
"github.com/thnhdatle/foreman"
)
type EmailArgs struct {
UserID int64 `json:"user_id"`
Tmpl string `json:"tmpl"`
}
func (EmailArgs) Kind() string { return "email" }
func main() {
_ = foreman.WorkFunc[EmailArgs](func(ctx context.Context, job *foreman.Job[EmailArgs]) error {
tx, err := foreman.ClientFromContext(ctx).Pool().Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
if err := foreman.CompleteTx(ctx, tx, job); err != nil {
return err
}
return tx.Commit(ctx)
})
}
Output:
func DecodeArgs ¶
DecodeArgs decodes the args of row into T. It fails if row is of another kind.
Example ¶
package main
import (
"fmt"
"github.com/thnhdatle/foreman"
)
type EmailArgs struct {
UserID int64 `json:"user_id"`
Tmpl string `json:"tmpl"`
}
func (EmailArgs) Kind() string { return "email" }
func main() {
row := &foreman.JobRow{ID: 1, Kind: "email", Args: []byte(`{"user_id":7,"tmpl":"welcome"}`)}
args, err := foreman.DecodeArgs[EmailArgs](row)
fmt.Println(args.UserID, args.Tmpl, err)
}
Output: 7 welcome <nil>
func Discard ¶
Discard returns an error that discards the job.
Example ¶
package main
import (
"errors"
"fmt"
"github.com/thnhdatle/foreman"
)
func main() {
err := foreman.Discard(errors.New("template does not exist"))
fmt.Println(err)
}
Output: job discarded: template does not exist
func Heartbeat ¶
Heartbeat refreshes the heartbeat of the running job.
Example ¶
package main
import (
"context"
"github.com/thnhdatle/foreman"
)
type EmailArgs struct {
UserID int64 `json:"user_id"`
Tmpl string `json:"tmpl"`
}
func (EmailArgs) Kind() string { return "email" }
func main() {
_ = foreman.WorkFunc[EmailArgs](func(ctx context.Context, job *foreman.Job[EmailArgs]) error {
for i := 0; i < 10; i++ {
foreman.Heartbeat(ctx)
}
return nil
})
}
Output:
Types ¶
type AttemptError ¶
type AttemptError struct {
At time.Time `json:"at"`
Attempt int `json:"attempt"`
Error string `json:"error"`
Trace string `json:"trace,omitempty"`
}
AttemptError records one failed attempt of a job.
type Client ¶
type Client struct {
// contains filtered or unexported fields
}
Client inserts jobs and runs worker pools and maintenance services.
func ClientFromContext ¶
ClientFromContext returns the client running the job, or nil outside Work.
Example ¶
package main
import (
"context"
"github.com/thnhdatle/foreman"
)
type EmailArgs struct {
UserID int64 `json:"user_id"`
Tmpl string `json:"tmpl"`
}
func (EmailArgs) Kind() string { return "email" }
func main() {
_ = foreman.WorkFunc[EmailArgs](func(ctx context.Context, job *foreman.Job[EmailArgs]) error {
client := foreman.ClientFromContext(ctx)
_, err := client.Insert(ctx, EmailArgs{UserID: job.Args.UserID, Tmpl: "follow-up"}, nil)
return err
})
}
Output:
func NewClient ¶
NewClient validates cfg and returns a client using pool.
Example ¶
package main
import (
"errors"
"fmt"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/thnhdatle/foreman"
)
func main() {
var pool *pgxpool.Pool
_, err := foreman.NewClient(pool, foreman.Config{})
fmt.Println(errors.Is(err, foreman.ErrInvalidArgument))
}
Output: true
func (*Client) Insert ¶
func (c *Client) Insert(ctx context.Context, args JobArgs, opts *InsertOpts) (*InsertResult, error)
Insert inserts one job. It is committed immediately.
Example ¶
package main
import (
"context"
"fmt"
"time"
"github.com/thnhdatle/foreman"
)
type EmailArgs struct {
UserID int64 `json:"user_id"`
Tmpl string `json:"tmpl"`
}
func (EmailArgs) Kind() string { return "email" }
func main() {
var client *foreman.Client
res, err := client.Insert(context.Background(), EmailArgs{UserID: 1, Tmpl: "reset"}, &foreman.InsertOpts{
Queue: "mail",
Priority: 1,
UniqueOpts: &foreman.UniqueOpts{ByArgs: true, ByPeriod: time.Hour},
})
if err == nil && res.UniqueSkippedAsDuplicate {
fmt.Println("already queued as job", res.Job.ID)
}
}
Output:
func (*Client) InsertMany ¶
func (c *Client) InsertMany(ctx context.Context, params []InsertParams) ([]*InsertResult, error)
InsertMany inserts a batch in one transaction.
Example ¶
package main
import (
"context"
"github.com/thnhdatle/foreman"
)
type EmailArgs struct {
UserID int64 `json:"user_id"`
Tmpl string `json:"tmpl"`
}
func (EmailArgs) Kind() string { return "email" }
func main() {
var client *foreman.Client
params := make([]foreman.InsertParams, 0, 100)
for i := int64(0); i < 100; i++ {
params = append(params, foreman.InsertParams{Args: EmailArgs{UserID: i, Tmpl: "digest"}})
}
_, _ = client.InsertMany(context.Background(), params)
}
Output:
func (*Client) InsertManyTx ¶
func (c *Client) InsertManyTx(ctx context.Context, tx pgx.Tx, params []InsertParams) ([]*InsertResult, error)
InsertManyTx inserts a batch in tx.
Example ¶
package main
import (
"context"
"github.com/jackc/pgx/v5"
"github.com/thnhdatle/foreman"
)
type EmailArgs struct {
UserID int64 `json:"user_id"`
Tmpl string `json:"tmpl"`
}
func (EmailArgs) Kind() string { return "email" }
func main() {
var (
client *foreman.Client
tx pgx.Tx
)
_, _ = client.InsertManyTx(context.Background(), tx, []foreman.InsertParams{
{Args: EmailArgs{UserID: 1}}, {Args: EmailArgs{UserID: 2}},
})
}
Output:
func (*Client) InsertTx ¶
func (c *Client) InsertTx(ctx context.Context, tx pgx.Tx, args JobArgs, opts *InsertOpts) (*InsertResult, error)
InsertTx inserts one job in tx.
Example ¶
package main
import (
"context"
"github.com/jackc/pgx/v5"
"github.com/thnhdatle/foreman"
)
type EmailArgs struct {
UserID int64 `json:"user_id"`
Tmpl string `json:"tmpl"`
}
func (EmailArgs) Kind() string { return "email" }
func main() {
var (
client *foreman.Client
tx pgx.Tx
)
_, _ = client.InsertTx(context.Background(), tx, EmailArgs{UserID: 2, Tmpl: "welcome"}, nil)
}
Output:
func (*Client) JobCancel ¶
JobCancel cancels a job.
Example ¶
package main
import (
"context"
"fmt"
"github.com/thnhdatle/foreman"
)
func main() {
var client *foreman.Client
job, err := client.JobCancel(context.Background(), 123)
if err == nil && job.State == foreman.StateRunning {
fmt.Println("cancel requested; the job ends as cancelled when Work returns")
}
}
Output:
func (*Client) JobDeleteMany ¶
JobDeleteMany deletes matching jobs except running ones and returns how many.
Example ¶
package main
import (
"context"
"time"
"github.com/thnhdatle/foreman"
)
func main() {
var client *foreman.Client
_, _ = client.JobDeleteMany(context.Background(), &foreman.JobFilter{
States: []foreman.State{foreman.StateCompleted}, CreatedBefore: time.Now().Add(-30 * 24 * time.Hour),
})
}
Output:
func (*Client) JobDiscard ¶
JobDiscard moves a job to discarded.
Example ¶
package main
import (
"context"
"github.com/thnhdatle/foreman"
)
func main() {
var client *foreman.Client
_, _ = client.JobDiscard(context.Background(), 123)
}
Output:
func (*Client) JobDiscardMany ¶
JobDiscardMany discards every non-final job matching f and returns how many.
Example ¶
package main
import (
"context"
"github.com/thnhdatle/foreman"
)
func main() {
var client *foreman.Client
_, _ = client.JobDiscardMany(context.Background(), &foreman.JobFilter{Queues: []string{"legacy"}})
}
Output:
func (*Client) JobGet ¶
JobGet returns one job, or ErrNotFound.
Example ¶
package main
import (
"context"
"errors"
"fmt"
"github.com/thnhdatle/foreman"
)
func main() {
var client *foreman.Client
job, err := client.JobGet(context.Background(), 123)
if errors.Is(err, foreman.ErrNotFound) {
return
}
fmt.Println(job.State, len(job.Errors))
}
Output:
func (*Client) JobList ¶
func (c *Client) JobList(ctx context.Context, p *JobListParams) (*JobListResult, error)
JobList lists jobs newest first with cursor pagination.
Example ¶
package main
import (
"context"
"fmt"
"github.com/thnhdatle/foreman"
)
func main() {
var client *foreman.Client
params := &foreman.JobListParams{States: []foreman.State{foreman.StateDiscarded}, Limit: 100}
for {
page, err := client.JobList(context.Background(), params)
if err != nil {
return
}
for _, j := range page.Jobs {
fmt.Println(j.ID, j.Kind)
}
if page.NextCursor == "" {
return
}
params.Cursor = page.NextCursor
}
}
Output:
func (*Client) JobRetry ¶
JobRetry moves a finished job back to available.
Example ¶
package main
import (
"context"
"github.com/thnhdatle/foreman"
)
func main() {
var client *foreman.Client
_, _ = client.JobRetry(context.Background(), 123)
}
Output:
func (*Client) JobRetryMany ¶
JobRetryMany retries every final job matching f and returns how many.
Example ¶
package main
import (
"context"
"fmt"
"time"
"github.com/thnhdatle/foreman"
)
func main() {
var client *foreman.Client
n, _ := client.JobRetryMany(context.Background(), &foreman.JobFilter{
Kinds: []string{"email"}, States: []foreman.State{foreman.StateDiscarded},
CreatedAfter: time.Now().Add(-24 * time.Hour),
})
fmt.Println("retried", n)
}
Output:
func (*Client) JobWork ¶
JobWork runs one available job synchronously on this client.
Example ¶
package main
import (
"context"
"fmt"
"github.com/thnhdatle/foreman"
)
type EmailArgs struct {
UserID int64 `json:"user_id"`
Tmpl string `json:"tmpl"`
}
func (EmailArgs) Kind() string { return "email" }
func main() {
var client *foreman.Client
res, _ := client.Insert(context.Background(), EmailArgs{UserID: 3}, nil)
job, _ := client.JobWork(context.Background(), res.Job.ID)
fmt.Println(job.State)
}
Output:
func (*Client) LeaderGet ¶
func (c *Client) LeaderGet(ctx context.Context) (*LeaderInfo, error)
LeaderGet returns the current lease, or ErrNotFound when no client holds one.
Example ¶
package main
import (
"context"
"fmt"
"github.com/thnhdatle/foreman"
)
func main() {
var client *foreman.Client
if l, err := client.LeaderGet(context.Background()); err == nil {
fmt.Println(l.LeaderID, l.ExpiresAt)
}
}
Output:
func (*Client) PeriodicList ¶
func (c *Client) PeriodicList(ctx context.Context) ([]*PeriodicInfo, error)
PeriodicList lists periodic jobs that have enqueued at least one tick.
Example ¶
package main
import (
"context"
"fmt"
"github.com/thnhdatle/foreman"
)
func main() {
var client *foreman.Client
ps, _ := client.PeriodicList(context.Background())
for _, p := range ps {
fmt.Println(p.Name, p.LastTick)
}
}
Output:
func (*Client) QueueList ¶
QueueList lists the queues known to the database.
Example ¶
package main
import (
"context"
"fmt"
"github.com/thnhdatle/foreman"
)
func main() {
var client *foreman.Client
queues, _ := client.QueueList(context.Background())
for _, q := range queues {
fmt.Println(q.Name, q.Paused, q.Available, q.Running)
}
}
Output:
func (*Client) QueuePause ¶
QueuePause stops clients from fetching jobs of the queue.
Example ¶
package main
import (
"context"
"github.com/thnhdatle/foreman"
)
func main() {
var client *foreman.Client
ctx := context.Background()
_ = client.QueuePause(ctx, "mail")
_ = client.QueueResume(ctx, "mail")
}
Output:
func (*Client) QueueResume ¶
QueueResume lets clients fetch jobs of the queue again.
Example ¶
package main
import (
"context"
"github.com/thnhdatle/foreman"
)
func main() {
var client *foreman.Client
_ = client.QueueResume(context.Background(), "mail")
}
Output:
func (*Client) Start ¶
Start runs the worker pools, heartbeats, LISTEN connection and leader election.
Example ¶
package main
import (
"context"
"log"
"time"
"github.com/thnhdatle/foreman"
)
func main() {
var client *foreman.Client
ctx := context.Background()
if err := client.Start(ctx); err != nil {
log.Fatal(err)
}
stopCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
_ = client.Stop(stopCtx)
}
Output:
func (*Client) StopAndCancel ¶
StopAndCancel cancels running jobs at once and waits for them.
Example ¶
package main
import (
"context"
"time"
"github.com/thnhdatle/foreman"
)
func main() {
var client *foreman.Client
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = client.StopAndCancel(ctx)
}
Output:
type Config ¶
type Config struct {
// Queues served by this client. With none it is producer-only.
Queues map[string]QueueConfig
Workers *Workers
PeriodicJobs []*PeriodicJob
Logger *slog.Logger
Clock Clock
// ID identifies the client. Default hostname-pid-random.
ID string
// Schema holding the Foreman tables. Default public.
Schema string
// JobTimeout is the default timeout of Work. 0 means none.
JobTimeout time.Duration
// RescueStuckJobsAfter is how long a running job may go without a heartbeat. Default 1h.
RescueStuckJobsAfter time.Duration
// HeartbeatInterval is the time between heartbeats. Default 30s.
HeartbeatInterval time.Duration
// RetryPolicy defaults to ExponentialBackoff{}.
RetryPolicy RetryPolicy
// CompletedJobRetention defaults to 24h. Negative keeps jobs forever.
CompletedJobRetention time.Duration
// CancelledJobRetention defaults to 7 days. Negative keeps jobs forever.
CancelledJobRetention time.Duration
// DiscardedJobRetention defaults to 7 days. Negative keeps jobs forever.
DiscardedJobRetention time.Duration
// ReindexInterval defaults to 24h. Negative disables it.
ReindexInterval time.Duration
// CleanerInterval defaults to 30s.
CleanerInterval time.Duration
ErrorHandler ErrorHandler
Middleware []Middleware
// PollOnly disables LISTEN/NOTIFY and relies on polling.
PollOnly bool
// MaxArgsBytes bounds the JSON size of args. Default 1 MiB.
MaxArgsBytes int
TracerProvider trace.TracerProvider
Registerer prometheus.Registerer
}
Config configures a client. Zero fields select their defaults.
type ConstantBackoff ¶
ConstantBackoff waits a fixed Delay.
type ErrorHandler ¶
type ErrorHandler interface {
HandleError(ctx context.Context, job *JobRow, err error) *ErrorHandlerResult
HandlePanic(ctx context.Context, job *JobRow, panicVal any, trace string) *ErrorHandlerResult
}
ErrorHandler is notified of failed attempts and may override the outcome.
type ErrorHandlerResult ¶
ErrorHandlerResult overrides the outcome of a failed attempt.
type ExponentialBackoff ¶
ExponentialBackoff waits an exponentially growing, jittered delay.
Example ¶
package main
import (
"fmt"
"time"
"github.com/thnhdatle/foreman"
)
func main() {
p := foreman.ExponentialBackoff{Base: time.Second, Cap: time.Hour}
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
next := p.NextRetry(foreman.JobRow{Attempt: 3}, now)
fmt.Println(next.Sub(now) < 4*time.Second)
}
Output: true
type InsertOpts ¶
type InsertOpts struct {
// Queue defaults to "default".
Queue string
// Priority is 1 to 4, where 1 runs first. Default 2.
Priority int
// ScheduledAt in the future schedules the job.
ScheduledAt time.Time
// MaxAttempts defaults to 25.
MaxAttempts int
Tags []string
// Metadata must be a JSON object.
Metadata json.RawMessage
// ConcurrencyKey groups jobs for per-key limits and fairness (for example a tenant id).
ConcurrencyKey string
UniqueOpts *UniqueOpts
Timeout time.Duration
}
InsertOpts are per-job options.
type InsertParams ¶
type InsertParams struct {
Args JobArgs
InsertOpts *InsertOpts
}
InsertParams is one job of an InsertMany call.
type InsertResult ¶
InsertResult is the outcome of inserting one job.
type Job ¶
type Job[T JobArgs] struct { ID int64 Args T Attempt int MaxAttempts int Queue string Priority int Tags []string Metadata json.RawMessage CreatedAt time.Time ScheduledAt time.Time Errors []AttemptError }
Job is a job with decoded, typed args, as passed to Worker.Work.
type JobArgsWithInsertOpts ¶
type JobArgsWithInsertOpts interface {
JobArgs
InsertOpts() InsertOpts
}
JobArgsWithInsertOpts is implemented by args types that provide insert defaults.
type JobFilter ¶
type JobFilter struct {
Kinds []string
Queues []string
States []State
CreatedAfter time.Time
CreatedBefore time.Time
}
JobFilter selects jobs for bulk operations.
type JobListParams ¶
type JobListParams struct {
States []State
Kinds []string
Queues []string
// Limit defaults to 50 and is at most 500.
Limit int
Cursor string
}
JobListParams filters JobList.
type JobListResult ¶
JobListResult is one page of jobs, newest first.
type JobRow ¶
type JobRow struct {
ID int64
Kind string
Queue string
Args json.RawMessage
State State
Priority int
Attempt int
MaxAttempts int
ScheduledAt time.Time
CreatedAt time.Time
AttemptedAt *time.Time
FinalizedAt *time.Time
HeartbeatAt *time.Time
CancelRequestedAt *time.Time
AttemptedBy []string
Errors []AttemptError
Tags []string
Metadata json.RawMessage
UniqueKey []byte
UniqueStates []State
ConcurrencyKey string
Timeout time.Duration
}
JobRow is a job as stored in the database, with undecoded args.
type LeaderInfo ¶
LeaderInfo describes the current leader lease.
type Middleware ¶
type Middleware interface {
InsertMany(ctx context.Context, params []InsertParams, next func(ctx context.Context) ([]*InsertResult, error)) (
[]*InsertResult, error)
Work(ctx context.Context, job *JobRow, next func(ctx context.Context) error) error
}
Middleware wraps inserts and work.
type MiddlewareDefaults ¶
type MiddlewareDefaults struct{}
MiddlewareDefaults is a pass-through Middleware to embed.
func (MiddlewareDefaults) InsertMany ¶
func (MiddlewareDefaults) InsertMany( ctx context.Context, _ []InsertParams, next func(ctx context.Context) ([]*InsertResult, error), ) ([]*InsertResult, error)
InsertMany calls next.
type PeriodicInfo ¶
PeriodicInfo summarises the ticks enqueued for one periodic job.
type PeriodicJob ¶
type PeriodicJob struct {
Name string
Schedule Schedule
Args func() (JobArgs, *InsertOpts)
RunOnStart bool
// CatchUp is how far back missed ticks are enqueued.
CatchUp time.Duration
// TimeZone evaluates cron expressions. Default UTC.
TimeZone *time.Location
}
PeriodicJob is a job the leader enqueues on a schedule.
type QueueConfig ¶
type QueueConfig struct {
// MaxWorkers is how many jobs run at once. Default 10.
MaxWorkers int
// FetchInterval is the polling interval. Default 1s.
FetchInterval time.Duration
// PerKeyLimit caps running jobs per concurrency key. 0 means no limit.
PerKeyLimit int
// Fairness alternates between concurrency keys when jobs are backlogged.
Fairness bool
}
QueueConfig configures one queue served by a client.
type QueueInfo ¶
type QueueInfo struct {
Name string
Paused bool
PausedAt *time.Time
LastSeenAt *time.Time
Available int64
Scheduled int64
Running int64
Retryable int64
}
QueueInfo describes a queue and its pending jobs.
type RetryPolicy ¶
RetryPolicy decides when a failed job runs again.
type RetryPolicyFunc ¶
RetryPolicyFunc adapts a function to RetryPolicy.
type Schedule ¶
Schedule computes the ticks of a periodic job.
func Cron ¶
Cron parses a cron expression.
Example ¶
package main
import (
"fmt"
"log"
"time"
"github.com/thnhdatle/foreman"
)
func main() {
s, err := foreman.Cron("0 9 * * MON")
if err != nil {
log.Fatal(err)
}
fmt.Println(s.Next(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)))
}
Output: 2026-01-05 09:00:00 +0000 UTC
func Every ¶
Every returns a schedule ticking every d.
Example ¶
package main
import (
"fmt"
"time"
"github.com/thnhdatle/foreman"
)
func main() {
s := foreman.Every(15 * time.Minute)
fmt.Println(s.Next(time.Date(2026, 1, 1, 10, 7, 0, 0, time.UTC)))
}
Output: 2026-01-01 10:15:00 +0000 UTC
func MustCron ¶
MustCron is Cron that panics on an invalid expression.
Example ¶
package main
import (
"fmt"
"time"
"github.com/thnhdatle/foreman"
)
type EmailArgs struct {
UserID int64 `json:"user_id"`
Tmpl string `json:"tmpl"`
}
func (EmailArgs) Kind() string { return "email" }
func main() {
job := &foreman.PeriodicJob{
Name: "daily-report",
Schedule: foreman.MustCron("@daily"),
Args: func() (foreman.JobArgs, *foreman.InsertOpts) { return EmailArgs{Tmpl: "report"}, nil },
CatchUp: time.Hour,
}
fmt.Println(job.Name, job.Schedule != nil, job.Args != nil, job.CatchUp)
}
Output: daily-report true true 1h0m0s
type UniqueOpts ¶
type UniqueOpts struct {
ByArgs bool
ByArgFields []string
ByPeriod time.Duration
ByQueue bool
// ByState defaults to available, scheduled, running, retryable and completed.
ByState []State
}
UniqueOpts selects the identity of a unique job.
type WorkerOption ¶
type WorkerOption func(*workerEntry)
WorkerOption configures the jobs of one kind.
func WithMaxAttempts ¶
func WithMaxAttempts(n int) WorkerOption
WithMaxAttempts sets the default max attempts of the kind.
func WithRetryPolicy ¶
func WithRetryPolicy(p RetryPolicy) WorkerOption
WithRetryPolicy sets the retry policy of the kind (default: client policy).
func WithTimeout ¶
func WithTimeout(d time.Duration) WorkerOption
WithTimeout sets the default timeout of the kind.
type Workers ¶
type Workers struct {
// contains filtered or unexported fields
}
Workers is the registry of workers by kind.
func NewWorkers ¶
func NewWorkers() *Workers
NewWorkers returns an empty registry.
Example ¶
package main
import (
"context"
"fmt"
"github.com/thnhdatle/foreman"
)
type EmailArgs struct {
UserID int64 `json:"user_id"`
Tmpl string `json:"tmpl"`
}
func (EmailArgs) Kind() string { return "email" }
type EmailWorker struct{}
func (w *EmailWorker) Work(ctx context.Context, job *foreman.Job[EmailArgs]) error {
fmt.Printf("sending %s to user %d\n", job.Args.Tmpl, job.Args.UserID)
return nil
}
func main() {
workers := foreman.NewWorkers()
foreman.AddWorker(workers, &EmailWorker{})
fmt.Println(len(workers.Kinds()))
}
Output: 1
Source Files
¶
Directories
¶
| Path | Synopsis |
|---|---|
|
cmd
|
|
|
foreman
command
Command foreman is the Foreman admin CLI.
|
Command foreman is the Foreman admin CLI. |
|
examples
|
|
|
basic
command
Command basic enqueues a job in a transaction and waits for it to complete.
|
Command basic enqueues a job in a transaction and waits for it to complete. |
|
cancel-long-job
command
Command cancel-long-job cancels a running job.
|
Command cancel-long-job cancels a running job. |
|
insert-from-python
command
Command insert-from-python runs a Go worker for jobs enqueued by a Python script.
|
Command insert-from-python runs a Go worker for jobs enqueued by a Python script. |
|
internal/exampledb
Package exampledb connects the examples to Postgres with a freshly migrated schema.
|
Package exampledb connects the examples to Postgres with a freshly migrated schema. |
|
multi-tenant-fairness
command
Command multi-tenant-fairness shows a small tenant served despite a large backlog.
|
Command multi-tenant-fairness shows a small tenant served despite a large backlog. |
|
otel
command
Command otel traces job work with OpenTelemetry.
|
Command otel traces job work with OpenTelemetry. |
|
periodic
command
Command periodic enqueues a job every second through the leader.
|
Command periodic enqueues a job every second through the leader. |
|
transactional-enqueue
command
Command transactional-enqueue shows a job inserted in a transaction.
|
Command transactional-enqueue shows a job inserted in a transaction. |
|
unique
command
Command unique inserts the same unique job twice.
|
Command unique inserts the same unique job twice. |
|
Package foremanhttp provides an embeddable admin JSON API for a Foreman client.
|
Package foremanhttp provides an embeddable admin JSON API for a Foreman client. |
|
Package foremantest provides helpers for testing code that uses Foreman.
|
Package foremantest provides helpers for testing code that uses Foreman. |
|
internal
|
|
|
adapter/pgstore
Package pgstore implements the job store ports on Postgres with pgx.
|
Package pgstore implements the job store ports on Postgres with pgx. |
|
app/completer
Package completer writes job completions and heartbeats to the database in batches.
|
Package completer writes job completions and heartbeats to the database in batches. |
|
app/elector
Package elector elects one leader per cluster with a lease row.
|
Package elector elects one leader per cluster with a lease row. |
|
app/executor
Package executor runs one attempt of a job and maps the result to a completion.
|
Package executor runs one attempt of a job and maps the result to a completion. |
|
app/maintenance
Package maintenance runs the leader-only background services and the cancel poller.
|
Package maintenance runs the leader-only background services and the cancel poller. |
|
app/notifier
Package notifier shares one Postgres LISTEN connection between subscribers and reconnects when it drops.
|
Package notifier shares one Postgres LISTEN connection between subscribers and reconnects when it drops. |
|
app/producer
Package producer runs the fetch loop of one queue and runs each fetched job on its own goroutine.
|
Package producer runs the fetch loop of one queue and runs each fetched job on its own goroutine. |
|
cli
Package cli implements the foreman admin CLI.
|
Package cli implements the foreman admin CLI. |
|
config
Package config loads and validates the configuration of the foreman CLI.
|
Package config loads and validates the configuration of the foreman CLI. |
|
domain/backoff
Package backoff provides retry delay policies.
|
Package backoff provides retry delay policies. |
|
domain/fairness
Package fairness decides how a fetch is split between concurrency keys.
|
Package fairness decides how a fetch is split between concurrency keys. |
|
domain/job
Package job defines the job row, its state machine and the mapping from a work result to a completion.
|
Package job defines the job row, its state machine and the mapping from a work result to a completion. |
|
domain/schedule
Package schedule provides cron and fixed-interval schedules and finds the ticks that are due.
|
Package schedule provides cron and fixed-interval schedules and finds the ticks that are due. |
|
domain/unique
Package unique builds the identity of unique jobs from their options.
|
Package unique builds the identity of unique jobs from their options. |
|
platform/clock
Package clock provides the injectable time source.
|
Package clock provides the injectable time source. |
|
platform/metrics
Package metrics defines the Prometheus collectors of a Foreman client.
|
Package metrics defines the Prometheus collectors of a Foreman client. |
|
platform/tracing
Package tracing creates OpenTelemetry spans for job work and inserts.
|
Package tracing creates OpenTelemetry spans for job work and inserts. |
|
testutil
Package testutil holds helpers shared by tests.
|
Package testutil holds helpers shared by tests. |
|
testutil/pgtest
Package pgtest provides Postgres for integration tests.
|
Package pgtest provides Postgres for integration tests. |
|
Package migrate installs and removes the Foreman schema with embedded SQL migrations.
|
Package migrate installs and removes the Foreman schema with embedded SQL migrations. |