Documentation
¶
Index ¶
Constants ¶
const ContextJobKey = "job-key"
const ContextQueueKey = "queue-key"
const MaxTime int64 = 9223372036854775807
Variables ¶
var JobError = joberror{ INVALID_JD: errors.New("invalid job description"), INVALID_JOB: errors.New("invalid job"), INVALID_WORKER: errors.New("invalid worker"), INVALID_PL: errors.New("invalid production line"), NOT_FOUND: errors.New("not found"), TERMINATING: errors.New("terminating"), DUPLICATE_JOB: errors.New("duplicate job"), }
var JobStatus = jobstatus{
INIT: "init",
PROCESSING: "processing",
RETRY: "retry",
ERROR: "error",
MAX_RETRY: "max_retry",
DONE: "done",
}
Functions ¶
func IsRetryable ¶ added in v1.1.0
IsRetryable reports whether err is a deadlock or lock wait timeout: MySQL/ MariaDB error 1213 or 1205, Postgres SQLSTATE 40P01 or 55P03, or the gorm store's StoreError.LOCK_TIMEOUT. These are an expected outcome of concurrent transactions touching the same rows and indexes (e.g. enqueueing while workers claim jobs), not a sign of a broken connection or a bad query, and running the transaction again normally succeeds.
It recognizes go-sql-driver/mysql's *MySQLError and any error with a SQLState() string method (pgx's *pgconn.PgError), wrapped or not, without importing either driver.
Retry the whole transaction, never just the failed statement: a MySQL deadlock has already rolled back everything the transaction did, and on Postgres any error aborts the transaction.
Types ¶
type Foreman ¶
type Foreman interface {
//AddWorker add a worker function for handler
AddWorker(title string, worker HandlerFunc, midl ...MiddlewareFunc) error
//AddJobTransaction add a new job to the queue. ctx is handed to JobStorage.CreateJob; with
//store/gorm it carries the caller's *gorm.DB, and when that is an open transaction the job
//is created inside it, so it only exists if that transaction commits.
//
//If it fails inside the caller's transaction with anything but a JobError (such as
//DUPLICATE_JOB, which leaves the transaction intact), treat the whole transaction as
//failed: return the error, roll back, and retry the transaction from the start if
//IsRetryable(err). Do not log it and carry on. On MySQL/MariaDB a deadlock has already
//rolled back everything the transaction wrote before this call, yet the transaction stays
//usable: later statements and the final Commit still succeed, persisting only the writes
//made after the failure. On Postgres any error aborts the transaction.
AddJobTransaction(ctx context.Context, job *Job) error
//AddJob add a new job to the queue without context, dont recommend but handy for multiple context
AddJob(job *Job) error
//AddJobWithSchedule add a new job to the queue to run at a schedule (unix seconds).
//It returns JobError.DUPLICATE_JOB if a schedule for the same title + job_id is still
//pending. An active job with that job_id doesn't block it, so a recurring job can
//schedule its own next run from its handler. If the schedule comes due while that
//job_id is still active, it stays pending and fires on a later scheduler pass.
AddJobWithSchedule(ctx context.Context, job *Job, runat int64) error
//Serve start the worker service. It blocks until Strike is called and then returns
//JobError.TERMINATING; store errors while fetching jobs are logged and retried with
//a backoff, never returned.
Serve() error
//Strike graceful shutdown service. Reject new jobs and wait for running one to finish. Force kill at TTL.
Strike(ttl int) error // ttl in seconds
//AddMiddleware add global middleware, order of adding matters
AddMiddleware(midl ...MiddlewareFunc)
//GetCounter return worker counter for debug purpose
GetCounter() int
}
func NewForeman ¶
func NewForeman(config *QueueConfig, store JobStorage, logger Logger) Foreman
type Governor ¶
type Governor interface {
AddJob(string)
DelJob(string)
NoJob()
Spawn() (bool, []string)
GetCounter() int
}
func NewLocalOndemandGovernor ¶
func NewLocalOndemandGovernor(max, min, maxworker int, jd map[string]JobDescription) Governor
OndemandGovernor ondemand governor
type HandlerFunc ¶
type Job ¶
type Job struct {
ID string `mapstructure:"id" yaml:"id" json:"id"`
JobID string `mapstructure:"job_id" yaml:"job_id" json:"job_id"`
Title string `mapstructure:"title" yaml:"title" json:"title"`
Payload string `mapstructure:"payload" yaml:"payload" json:"payload"`
Try int `mapstructure:"try" yaml:"try" json:"try"`
Priority int `mapstructure:"priority" yaml:"priority" json:"priority"`
Status string `mapstructure:"status" yaml:"status" json:"status"`
Result string `yaml:"result" mapstructure:"result" json:"result"`
Message string `yaml:"message" mapstructure:"message" json:"message"`
UpdatedAt int64 `yaml:"updated_at" mapstructure:"updated_at" json:"updated_at"`
}
type JobDescription ¶
type JobDescription struct {
Title string `mapstructure:"title" yaml:"title" json:"title"`
TTL int `mapstructure:"ttl" yaml:"ttl" json:"ttl"` // seconds
Concurrent int `mapstructure:"concurrent" yaml:"concurrent" json:"concurrent"` // 0 is invalid
Priority int `mapstructure:"priority" yaml:"priority" json:"priority"` // 0 is invalid here, 1 is highest
MaxRetry int `mapstructure:"max_retry" yaml:"max_retry" json:"max_retry"` // number of retry
Secure bool `mapstructure:"secure" yaml:"secure" json:"secure"`
}
Config models
type JobStorage ¶
type JobStorage interface {
CheckDuplicateJob(job Job) error
CreateJob(ctx context.Context, job Job) error
GetAndLockAvailableJob(jd map[string]JobDescription, ignorelist ...string) (*Job, error)
UpdateJobResult(job Job) error
InjectJob(Job) error
// CreateScheduleJob stores a schedule, returning JobError.DUPLICATE_JOB if
// one with the same title + job_id is already pending (status init).
// Active jobs with that job_id must not block it.
CreateScheduleJob(ctx context.Context, job ScheduleJob) error
GetScheduledJob(from, to int64) ([]*ScheduleJob, error)
UpdateScheduledJob(ScheduleJob) error
}
type LocalOndemandGovernor ¶
type LocalOndemandGovernor struct {
MaxSleep int
MinSleep int
CurSleep *int
WorkerCounter *int
Locker *sync.Mutex
JobCounter map[string]*int
MaxWorker int
JobDescription map[string]JobDescription
}
func (LocalOndemandGovernor) AddJob ¶
func (g LocalOndemandGovernor) AddJob(title string)
func (LocalOndemandGovernor) DelJob ¶
func (g LocalOndemandGovernor) DelJob(title string)
func (LocalOndemandGovernor) GetCounter ¶
func (g LocalOndemandGovernor) GetCounter() int
func (LocalOndemandGovernor) NoJob ¶
func (g LocalOndemandGovernor) NoJob()
func (LocalOndemandGovernor) Spawn ¶
func (g LocalOndemandGovernor) Spawn() (bool, []string)
type MiddlewareFunc ¶
type MiddlewareFunc func(HandlerFunc) HandlerFunc
type QueueConfig ¶
type QueueConfig struct {
JobDescription map[string]JobDescription `mapstructure:"job_description" yaml:"job_description" json:"job_description"`
CleanError bool `yaml:"clean_error" mapstructure:"clean_error" json:"clean_error"` // will the GC clear maxed try job
Concurrent int `yaml:"concurrent" mapstructure:"concurrent" json:"concurrent"` // number of workers can be run, will ignore JD.Concurrent if it >
BreakTime int `mapstructure:"break_time" yaml:"break_time" json:"break_time"` // miliseconds, break between loop
RampTime int `mapstructure:"ramp_time" yaml:"ramp_time" json:"ramp_time"`
// ScheduleInterval is the time between scheduler passes that fire due
// scheduled jobs, in milliseconds. 0 keeps the default of 59 seconds.
ScheduleInterval int `mapstructure:"schedule_interval" yaml:"schedule_interval" json:"schedule_interval"`
}
type ScheduleJob ¶
type ScheduleJob struct {
ID string `mapstructure:"id" yaml:"id" json:"id"`
JobID string `mapstructure:"job_id" yaml:"job_id" json:"job_id"`
Title string `mapstructure:"title" yaml:"title" json:"title"`
Payload string `mapstructure:"payload" yaml:"payload" json:"payload"`
Priority int `mapstructure:"priority" yaml:"priority" json:"priority"`
Status string `mapstructure:"status" yaml:"status" json:"status"`
UpdatedAt int64 `yaml:"updated_at" mapstructure:"updated_at" json:"updated_at"`
Schedule int64 `json:"schedule"`
ExecuteID string `json:"execute_id"`
Log string `json:"log"`
}
Directories
¶
| Path | Synopsis |
|---|---|
|
internal
|
|
|
sqlerr
Package sqlerr reads server error codes out of database driver errors without importing the drivers, so depending on job-queue doesn't pull in (or register) a driver the application doesn't use.
|
Package sqlerr reads server error codes out of database driver errors without importing the drivers, so depending on job-queue doesn't pull in (or register) a driver the application doesn't use. |
|
store
|
|
|
test
|
|
|
gorm
command
|
|
|
gorm-postgres
command
|
|
|
gorm-sqlite
command
|
|
|
gorvernortest
command
|
|
|
sleepstrike
command
|
|