From b4b99f30b05820e17161d2409b144b516dcfbf78 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Tue, 20 Dec 2016 15:32:46 +0000 Subject: [PATCH 1/3] allow for already dead horse on heroku worker termination --- rq/worker.py | 23 +++++++++++++++-------- tests/test_worker.py | 5 ++--- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/rq/worker.py b/rq/worker.py index 7d745c4..4549b55 100644 --- a/rq/worker.py +++ b/rq/worker.py @@ -352,6 +352,19 @@ class Worker(object): signal.signal(signal.SIGINT, self.request_stop) signal.signal(signal.SIGTERM, self.request_stop) + def kill_horse(self): + """ + Kill the horse but catch "No such process" error has the horse could already be dead. + """ + try: + os.kill(self.horse_pid, signal.SIGKILL) + except OSError as e: + if e.errno == errno.ESRCH: + # "No such process" is fine with us + self.log.debug('Horse already dead') + else: + raise + def request_force_stop(self, signum, frame): """Terminates the application (cold shutdown). """ @@ -361,13 +374,7 @@ class Worker(object): if self.horse_pid: msg = 'Taking down horse {0} with me'.format(self.horse_pid) self.log.debug(msg) - try: - os.kill(self.horse_pid, signal.SIGKILL) - except OSError as e: - # ESRCH ("No such process") is fine with us - if e.errno != errno.ESRCH: - self.log.debug('Horse already down') - raise + self.kill_horse() raise SystemExit() def request_stop(self, signum, frame): @@ -834,7 +841,7 @@ class HerokuWorker(Worker): """If horse is alive send it SIGRTMIN""" if self.horse_pid != 0: self.log.warning('Warm shut down requested, sending horse SIGRTMIN signal') - os.kill(self.horse_pid, signal.SIGRTMIN) + self.kill_horse() else: self.log.warning('Warm shut down requested, no horse found') diff --git a/tests/test_worker.py b/tests/test_worker.py index 927c0cd..e3efdeb 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -833,7 +833,7 @@ class HerokuWorkerShutdownTestCase(TimeoutTestCase, RQTestCase): w._horse_pid = p.pid w.handle_warm_shutdown_request() p.join(2) - self.assertEqual(p.exitcode, -34) + self.assertEqual(p.exitcode, -9) self.assertFalse(os.path.exists(path)) def test_handle_shutdown_request_no_horse(self): @@ -842,5 +842,4 @@ class HerokuWorkerShutdownTestCase(TimeoutTestCase, RQTestCase): w = HerokuWorker('foo') w._horse_pid = 19999 - with self.assertRaises(OSError): - w.handle_warm_shutdown_request() + w.handle_warm_shutdown_request() From f50bdb31bfaae803c827667b8480f7172921cfab Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Tue, 20 Dec 2016 16:34:13 +0000 Subject: [PATCH 2/3] skip test_run_scheduled_access_self on pypy --- tests/test_worker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_worker.py b/tests/test_worker.py index e3efdeb..fdd136e 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -11,6 +11,7 @@ import time from multiprocessing import Process import subprocess import sys +from unittest import skipIf import pytest @@ -742,6 +743,7 @@ class TestWorkerSubprocess(RQTestCase): assert get_failed_queue().count == 0 assert q.count == 0 + @skipIf('pypy' in sys.version.lower(), 'often times out with pypy') def test_run_scheduled_access_self(self): """Schedule a job that schedules a job, then run the worker as subprocess""" q = Queue() From 763a304ec6978a1f55b32c6ec4e102d455a3096f Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Tue, 20 Dec 2016 16:47:10 +0000 Subject: [PATCH 3/3] cope with python 2.6 yawwwwwwn --- tests/test_worker.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_worker.py b/tests/test_worker.py index fdd136e..4449aa8 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -11,7 +11,6 @@ import time from multiprocessing import Process import subprocess import sys -from unittest import skipIf import pytest @@ -743,9 +742,12 @@ class TestWorkerSubprocess(RQTestCase): assert get_failed_queue().count == 0 assert q.count == 0 - @skipIf('pypy' in sys.version.lower(), 'often times out with pypy') + # @skipIf('pypy' in sys.version.lower(), 'often times out with pypy') def test_run_scheduled_access_self(self): """Schedule a job that schedules a job, then run the worker as subprocess""" + if 'pypy' in sys.version.lower(): + # horrible bodge until we drop 2.6 support and can use skipIf + return q = Queue() q.enqueue(schedule_access_self) subprocess.check_call(['rqworker', '-u', self.redis_url, '-b'])