|
|
@ -12,18 +12,20 @@ import sys
|
|
|
|
import time
|
|
|
|
import time
|
|
|
|
import traceback
|
|
|
|
import traceback
|
|
|
|
import warnings
|
|
|
|
import warnings
|
|
|
|
|
|
|
|
from datetime import datetime
|
|
|
|
|
|
|
|
|
|
|
|
from rq.compat import as_text, string_types, text_type
|
|
|
|
from rq.compat import as_text, string_types, text_type
|
|
|
|
|
|
|
|
|
|
|
|
from .connections import get_current_connection
|
|
|
|
from .connections import get_current_connection
|
|
|
|
from .exceptions import DequeueTimeout, NoQueueError
|
|
|
|
from .exceptions import DequeueTimeout, NoQueueError
|
|
|
|
from .job import Job, Status
|
|
|
|
from .job import Job, JobStatus
|
|
|
|
from .logutils import setup_loghandlers
|
|
|
|
from .logutils import setup_loghandlers
|
|
|
|
from .queue import get_failed_queue, Queue
|
|
|
|
from .queue import get_failed_queue, Queue
|
|
|
|
from .timeouts import UnixSignalDeathPenalty
|
|
|
|
from .timeouts import UnixSignalDeathPenalty
|
|
|
|
from .utils import import_attribute, make_colorizer, utcformat, utcnow
|
|
|
|
from .utils import import_attribute, make_colorizer, utcformat, utcnow, enum
|
|
|
|
from .version import VERSION
|
|
|
|
from .version import VERSION
|
|
|
|
from .registry import FinishedJobRegistry, StartedJobRegistry
|
|
|
|
from .registry import FinishedJobRegistry, StartedJobRegistry
|
|
|
|
|
|
|
|
from .suspension import is_suspended
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
from procname import setprocname
|
|
|
|
from procname import setprocname
|
|
|
@ -65,6 +67,12 @@ def signal_name(signum):
|
|
|
|
return 'SIG_UNKNOWN'
|
|
|
|
return 'SIG_UNKNOWN'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
WorkerStatus = enum('WorkerStatus',
|
|
|
|
|
|
|
|
STARTED='started', SUSPENDED='suspended', BUSY='busy',
|
|
|
|
|
|
|
|
IDLE='idle'
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Worker(object):
|
|
|
|
class Worker(object):
|
|
|
|
redis_worker_namespace_prefix = 'rq:worker:'
|
|
|
|
redis_worker_namespace_prefix = 'rq:worker:'
|
|
|
|
redis_workers_keys = 'rq:workers'
|
|
|
|
redis_workers_keys = 'rq:workers'
|
|
|
@ -333,6 +341,30 @@ class Worker(object):
|
|
|
|
signal.signal(signal.SIGINT, request_stop)
|
|
|
|
signal.signal(signal.SIGINT, request_stop)
|
|
|
|
signal.signal(signal.SIGTERM, request_stop)
|
|
|
|
signal.signal(signal.SIGTERM, request_stop)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def check_for_suspension(self, burst):
|
|
|
|
|
|
|
|
"""Check to see if the workers have been suspended by something like `rq suspend`"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
before_state = None
|
|
|
|
|
|
|
|
notified = False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
while not self.stopped and is_suspended(self.connection):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if burst:
|
|
|
|
|
|
|
|
self.log.info('Suspended in burst mode -- exiting.')
|
|
|
|
|
|
|
|
self.log.info('Note: There could still be unperformed jobs on the queue')
|
|
|
|
|
|
|
|
raise StopRequested
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if not notified:
|
|
|
|
|
|
|
|
self.log.info('Worker suspended, use "rq resume" command to resume')
|
|
|
|
|
|
|
|
before_state = self.get_state()
|
|
|
|
|
|
|
|
self.set_state(WorkerStatus.SUSPENDED)
|
|
|
|
|
|
|
|
notified = True
|
|
|
|
|
|
|
|
time.sleep(1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if before_state:
|
|
|
|
|
|
|
|
self.set_state(before_state)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def work(self, burst=False):
|
|
|
|
def work(self, burst=False):
|
|
|
|
"""Starts the work loop.
|
|
|
|
"""Starts the work loop.
|
|
|
|
|
|
|
|
|
|
|
@ -348,15 +380,19 @@ class Worker(object):
|
|
|
|
did_perform_work = False
|
|
|
|
did_perform_work = False
|
|
|
|
self.register_birth()
|
|
|
|
self.register_birth()
|
|
|
|
self.log.info('RQ worker started, version %s' % VERSION)
|
|
|
|
self.log.info('RQ worker started, version %s' % VERSION)
|
|
|
|
self.set_state('starting')
|
|
|
|
self.set_state(WorkerStatus.STARTED)
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
while True:
|
|
|
|
while True:
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
self.check_for_suspension(burst)
|
|
|
|
|
|
|
|
|
|
|
|
if self.stopped:
|
|
|
|
if self.stopped:
|
|
|
|
self.log.info('Stopping on request.')
|
|
|
|
self.log.info('Stopping on request.')
|
|
|
|
break
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
|
|
timeout = None if burst else max(1, self.default_worker_ttl - 60)
|
|
|
|
timeout = None if burst else max(1, self.default_worker_ttl - 60)
|
|
|
|
try:
|
|
|
|
|
|
|
|
result = self.dequeue_job_and_maintain_ttl(timeout)
|
|
|
|
result = self.dequeue_job_and_maintain_ttl(timeout)
|
|
|
|
if result is None:
|
|
|
|
if result is None:
|
|
|
|
break
|
|
|
|
break
|
|
|
@ -367,20 +403,22 @@ class Worker(object):
|
|
|
|
self.execute_job(job)
|
|
|
|
self.execute_job(job)
|
|
|
|
self.heartbeat()
|
|
|
|
self.heartbeat()
|
|
|
|
|
|
|
|
|
|
|
|
if job.get_status() == Status.FINISHED:
|
|
|
|
if job.get_status() == JobStatus.FINISHED:
|
|
|
|
queue.enqueue_dependents(job)
|
|
|
|
queue.enqueue_dependents(job)
|
|
|
|
|
|
|
|
|
|
|
|
did_perform_work = True
|
|
|
|
did_perform_work = True
|
|
|
|
|
|
|
|
|
|
|
|
finally:
|
|
|
|
finally:
|
|
|
|
if not self.is_horse:
|
|
|
|
if not self.is_horse:
|
|
|
|
self.register_death()
|
|
|
|
self.register_death()
|
|
|
|
return did_perform_work
|
|
|
|
return did_perform_work
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def dequeue_job_and_maintain_ttl(self, timeout):
|
|
|
|
def dequeue_job_and_maintain_ttl(self, timeout):
|
|
|
|
result = None
|
|
|
|
result = None
|
|
|
|
qnames = self.queue_names()
|
|
|
|
qnames = self.queue_names()
|
|
|
|
|
|
|
|
|
|
|
|
self.set_state('idle')
|
|
|
|
self.set_state(WorkerStatus.IDLE)
|
|
|
|
self.procline('Listening on %s' % ','.join(qnames))
|
|
|
|
self.procline('Listening on %s' % ','.join(qnames))
|
|
|
|
self.log.info('')
|
|
|
|
self.log.info('')
|
|
|
|
self.log.info('*** Listening on %s...' %
|
|
|
|
self.log.info('*** Listening on %s...' %
|
|
|
@ -477,12 +515,12 @@ class Worker(object):
|
|
|
|
timeout = (job.timeout or 180) + 60
|
|
|
|
timeout = (job.timeout or 180) + 60
|
|
|
|
|
|
|
|
|
|
|
|
with self.connection._pipeline() as pipeline:
|
|
|
|
with self.connection._pipeline() as pipeline:
|
|
|
|
self.set_state('busy', pipeline=pipeline)
|
|
|
|
self.set_state(WorkerStatus.BUSY, pipeline=pipeline)
|
|
|
|
self.set_current_job_id(job.id, pipeline=pipeline)
|
|
|
|
self.set_current_job_id(job.id, pipeline=pipeline)
|
|
|
|
self.heartbeat(timeout, pipeline=pipeline)
|
|
|
|
self.heartbeat(timeout, pipeline=pipeline)
|
|
|
|
registry = StartedJobRegistry(job.origin, self.connection)
|
|
|
|
registry = StartedJobRegistry(job.origin, self.connection)
|
|
|
|
registry.add(job, timeout, pipeline=pipeline)
|
|
|
|
registry.add(job, timeout, pipeline=pipeline)
|
|
|
|
job.set_status(Status.STARTED, pipeline=pipeline)
|
|
|
|
job.set_status(JobStatus.STARTED, pipeline=pipeline)
|
|
|
|
pipeline.execute()
|
|
|
|
pipeline.execute()
|
|
|
|
|
|
|
|
|
|
|
|
self.procline('Processing %s from %s since %s' % (
|
|
|
|
self.procline('Processing %s from %s since %s' % (
|
|
|
@ -511,7 +549,7 @@ class Worker(object):
|
|
|
|
result_ttl = job.get_ttl(self.default_result_ttl)
|
|
|
|
result_ttl = job.get_ttl(self.default_result_ttl)
|
|
|
|
if result_ttl != 0:
|
|
|
|
if result_ttl != 0:
|
|
|
|
job.ended_at = utcnow()
|
|
|
|
job.ended_at = utcnow()
|
|
|
|
job._status = Status.FINISHED
|
|
|
|
job._status = JobStatus.FINISHED
|
|
|
|
job.save(pipeline=pipeline)
|
|
|
|
job.save(pipeline=pipeline)
|
|
|
|
|
|
|
|
|
|
|
|
finished_job_registry = FinishedJobRegistry(job.origin, self.connection)
|
|
|
|
finished_job_registry = FinishedJobRegistry(job.origin, self.connection)
|
|
|
@ -523,7 +561,7 @@ class Worker(object):
|
|
|
|
pipeline.execute()
|
|
|
|
pipeline.execute()
|
|
|
|
|
|
|
|
|
|
|
|
except Exception:
|
|
|
|
except Exception:
|
|
|
|
job.set_status(Status.FAILED, pipeline=pipeline)
|
|
|
|
job.set_status(JobStatus.FAILED, pipeline=pipeline)
|
|
|
|
started_job_registry.remove(job, pipeline=pipeline)
|
|
|
|
started_job_registry.remove(job, pipeline=pipeline)
|
|
|
|
pipeline.execute()
|
|
|
|
pipeline.execute()
|
|
|
|
self.handle_exception(job, *sys.exc_info())
|
|
|
|
self.handle_exception(job, *sys.exc_info())
|
|
|
|