From 6bb8b26114e6ab23a310124e49d254218f280850 Mon Sep 17 00:00:00 2001 From: foxx Date: Fri, 5 Sep 2014 11:19:18 +0100 Subject: [PATCH 1/9] Allow job ID to be set on enqueue/enqueue_call() - fixes #412 --- rq/job.py | 12 ++++++++++-- rq/queue.py | 10 +++++++--- tests/test_job.py | 9 +++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/rq/job.py b/rq/job.py index ef7a266..292d0ed 100644 --- a/rq/job.py +++ b/rq/job.py @@ -6,6 +6,7 @@ import inspect import warnings from functools import partial from uuid import uuid4 +from uuid import UUID from rq.compat import as_text, decode_redis_hash, string_types, text_type @@ -92,7 +93,8 @@ class Job(object): # Job construction @classmethod def create(cls, func, args=None, kwargs=None, connection=None, - result_ttl=None, status=None, description=None, depends_on=None, timeout=None): + result_ttl=None, status=None, description=None, depends_on=None, timeout=None, + job_id=None): """Creates a new Job instance for the given function, arguments, and keyword arguments. """ @@ -107,6 +109,8 @@ class Job(object): raise TypeError('{0!r} is not a valid kwargs dict.'.format(kwargs)) job = cls(connection=connection) + if job_id is not None: + job.set_id(job_id) # Set the core job tuple properties job._instance = None @@ -325,7 +329,11 @@ class Job(object): def set_id(self, value): """Sets a job ID for the given job.""" - self._id = value + try: + self.key_for(text_type(value)) + except: + raise ValueError("Job ID invalid, failed to encode to string") + self._id = text_type(value) id = property(get_id, set_id) diff --git a/rq/queue.py b/rq/queue.py index 22aa160..608f609 100644 --- a/rq/queue.py +++ b/rq/queue.py @@ -163,7 +163,8 @@ class Queue(object): self.connection.rpush(self.key, job_id) def enqueue_call(self, func, args=None, kwargs=None, timeout=None, - result_ttl=None, description=None, depends_on=None): + result_ttl=None, description=None, depends_on=None, + job_id=None): """Creates a job to represent the delayed function call and enqueues it. @@ -176,7 +177,8 @@ class Queue(object): # TODO: job with dependency shouldn't have "queued" as status job = self.job_class.create(func, args, kwargs, connection=self.connection, result_ttl=result_ttl, status=Status.QUEUED, - description=description, depends_on=depends_on, timeout=timeout) + description=description, depends_on=depends_on, timeout=timeout, + job_id=job_id) # If job depends on an unfinished job, register itself on it's # parent's dependents instead of enqueueing it. @@ -222,6 +224,7 @@ class Queue(object): description = kwargs.pop('description', None) result_ttl = kwargs.pop('result_ttl', None) depends_on = kwargs.pop('depends_on', None) + job_id = kwargs.pop('job_id', None) if 'args' in kwargs or 'kwargs' in kwargs: assert args == (), 'Extra positional arguments cannot be used when using explicit args and kwargs.' # noqa @@ -230,7 +233,8 @@ class Queue(object): return self.enqueue_call(func=f, args=args, kwargs=kwargs, timeout=timeout, result_ttl=result_ttl, - description=description, depends_on=depends_on) + description=description, depends_on=depends_on, + job_id=job_id) def enqueue_job(self, job, set_meta_data=True): """Enqueues a job for delayed execution. diff --git a/tests/test_job.py b/tests/test_job.py index 7e990c0..1184b8d 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -3,6 +3,7 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) from datetime import datetime +from uuid import uuid4 from rq.compat import as_text, PY2 from rq.exceptions import NoSuchJobError, UnpickleError @@ -339,3 +340,11 @@ class TestJob(RQTestCase): self.assertFalse(self.testconn.exists(job.dependents_key)) self.assertNotIn(job.id, queue.get_job_ids()) + + def test_create_job_with_id(self): + # try a bunch of different ID types + queue = Queue(connection=self.testconn) + ids = [1234, uuid4(), "somejobid"] + for job_id in ids: + job = queue.enqueue(say_hello, job_id=job_id) + job.perform() From 05c1d4fa4b7cee4966ae404099d57a08d4bfef1f Mon Sep 17 00:00:00 2001 From: foxx Date: Fri, 5 Sep 2014 15:23:48 +0100 Subject: [PATCH 2/9] Added better job ID tests --- tests/test_job.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_job.py b/tests/test_job.py index 1184b8d..1a0ab6e 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -347,4 +347,5 @@ class TestJob(RQTestCase): ids = [1234, uuid4(), "somejobid"] for job_id in ids: job = queue.enqueue(say_hello, job_id=job_id) + self.assertEqual(job.id, str(job_id)) job.perform() From 72bc9e37b733026d5a65eb15929c832b8530c4e7 Mon Sep 17 00:00:00 2001 From: foxx Date: Fri, 5 Sep 2014 19:39:28 +0100 Subject: [PATCH 3/9] Forced job_id to only allow str/unicode --- rq/job.py | 9 ++++----- tests/test_job.py | 12 ++++++------ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/rq/job.py b/rq/job.py index 292d0ed..5aeba46 100644 --- a/rq/job.py +++ b/rq/job.py @@ -2,6 +2,7 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) +import types import inspect import warnings from functools import partial @@ -107,6 +108,8 @@ class Job(object): raise TypeError('{0!r} is not a valid args list.'.format(args)) if not isinstance(kwargs, dict): raise TypeError('{0!r} is not a valid kwargs dict.'.format(kwargs)) + if not isinstance(job_id, (str, unicode, types.NoneType)): + raise TypeError('job_id must be a str/unicode, not {}.'.format(type(job_id))) job = cls(connection=connection) if job_id is not None: @@ -329,11 +332,7 @@ class Job(object): def set_id(self, value): """Sets a job ID for the given job.""" - try: - self.key_for(text_type(value)) - except: - raise ValueError("Job ID invalid, failed to encode to string") - self._id = text_type(value) + self._id = value id = property(get_id, set_id) diff --git a/tests/test_job.py b/tests/test_job.py index 1a0ab6e..d3f7aa7 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -342,10 +342,10 @@ class TestJob(RQTestCase): self.assertNotIn(job.id, queue.get_job_ids()) def test_create_job_with_id(self): - # try a bunch of different ID types + """test creating jobs with a custom ID""" queue = Queue(connection=self.testconn) - ids = [1234, uuid4(), "somejobid"] - for job_id in ids: - job = queue.enqueue(say_hello, job_id=job_id) - self.assertEqual(job.id, str(job_id)) - job.perform() + job = queue.enqueue(say_hello, job_id="1234") + self.assertEqual(job.id, "1234") + job.perform() + + self.assertRaises(TypeError, queue.enqueue, say_hello, job_id=1234) \ No newline at end of file From 2753f17e8ec1a7353453f2b8a6e288ed8863ad9a Mon Sep 17 00:00:00 2001 From: foxx Date: Fri, 5 Sep 2014 19:45:49 +0100 Subject: [PATCH 4/9] Fixed bug with format() in py3 --- rq/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rq/job.py b/rq/job.py index 5aeba46..ddaa750 100644 --- a/rq/job.py +++ b/rq/job.py @@ -109,7 +109,7 @@ class Job(object): if not isinstance(kwargs, dict): raise TypeError('{0!r} is not a valid kwargs dict.'.format(kwargs)) if not isinstance(job_id, (str, unicode, types.NoneType)): - raise TypeError('job_id must be a str/unicode, not {}.'.format(type(job_id))) + raise TypeError('job_id must be a str/unicode, not {0}.'.format(type(job_id))) job = cls(connection=connection) if job_id is not None: From f5779c194f88b633be8e29d3b7e5660da2525b9d Mon Sep 17 00:00:00 2001 From: foxx Date: Mon, 8 Sep 2014 12:19:59 +0100 Subject: [PATCH 5/9] Renamed job_id to id, and fixed py3 bug, per #415 --- rq/job.py | 10 +++++----- rq/queue.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rq/job.py b/rq/job.py index ddaa750..d9e93a6 100644 --- a/rq/job.py +++ b/rq/job.py @@ -95,7 +95,7 @@ class Job(object): @classmethod def create(cls, func, args=None, kwargs=None, connection=None, result_ttl=None, status=None, description=None, depends_on=None, timeout=None, - job_id=None): + id=None): """Creates a new Job instance for the given function, arguments, and keyword arguments. """ @@ -108,12 +108,12 @@ class Job(object): raise TypeError('{0!r} is not a valid args list.'.format(args)) if not isinstance(kwargs, dict): raise TypeError('{0!r} is not a valid kwargs dict.'.format(kwargs)) - if not isinstance(job_id, (str, unicode, types.NoneType)): - raise TypeError('job_id must be a str/unicode, not {0}.'.format(type(job_id))) + if not isinstance(id, (string_types, types.NoneType)): + raise TypeError('id must be a string, not {0}.'.format(type(id))) job = cls(connection=connection) - if job_id is not None: - job.set_id(job_id) + if id is not None: + job.set_id(id) # Set the core job tuple properties job._instance = None diff --git a/rq/queue.py b/rq/queue.py index 608f609..1d77b78 100644 --- a/rq/queue.py +++ b/rq/queue.py @@ -164,7 +164,7 @@ class Queue(object): def enqueue_call(self, func, args=None, kwargs=None, timeout=None, result_ttl=None, description=None, depends_on=None, - job_id=None): + id=None): """Creates a job to represent the delayed function call and enqueues it. @@ -178,7 +178,7 @@ class Queue(object): job = self.job_class.create(func, args, kwargs, connection=self.connection, result_ttl=result_ttl, status=Status.QUEUED, description=description, depends_on=depends_on, timeout=timeout, - job_id=job_id) + id=id) # If job depends on an unfinished job, register itself on it's # parent's dependents instead of enqueueing it. @@ -234,7 +234,7 @@ class Queue(object): return self.enqueue_call(func=f, args=args, kwargs=kwargs, timeout=timeout, result_ttl=result_ttl, description=description, depends_on=depends_on, - job_id=job_id) + id=job_id) def enqueue_job(self, job, set_meta_data=True): """Enqueues a job for delayed execution. From b6499ce71cf49d88144ae3f26b90852284fd0444 Mon Sep 17 00:00:00 2001 From: foxx Date: Mon, 8 Sep 2014 12:42:41 +0100 Subject: [PATCH 6/9] Fixed another py3 bug - my bad. Also moved type checking further down --- rq/job.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rq/job.py b/rq/job.py index d9e93a6..c5c5eae 100644 --- a/rq/job.py +++ b/rq/job.py @@ -108,8 +108,6 @@ class Job(object): raise TypeError('{0!r} is not a valid args list.'.format(args)) if not isinstance(kwargs, dict): raise TypeError('{0!r} is not a valid kwargs dict.'.format(kwargs)) - if not isinstance(id, (string_types, types.NoneType)): - raise TypeError('id must be a string, not {0}.'.format(type(id))) job = cls(connection=connection) if id is not None: @@ -332,6 +330,8 @@ class Job(object): def set_id(self, value): """Sets a job ID for the given job.""" + if not isinstance(value, string_types): + raise TypeError('id must be a string, not {0}.'.format(type(value))) self._id = value id = property(get_id, set_id) From 0466562a132a57b69f086bba3416aa5fd7461c9b Mon Sep 17 00:00:00 2001 From: foxx Date: Tue, 9 Sep 2014 11:10:18 +0100 Subject: [PATCH 7/9] Fixes from comments in #415 Added tox/vagrant tmp dir to gitignore Removed unused import Renamed id to job_id in enqueue_call() as per comments in #415 --- .gitignore | 3 +++ rq/job.py | 1 - rq/queue.py | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index ea8eaca..ff9c616 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ /.tox /dist /build +.tox +.vagrant +Vagrantfile \ No newline at end of file diff --git a/rq/job.py b/rq/job.py index c5c5eae..937430e 100644 --- a/rq/job.py +++ b/rq/job.py @@ -7,7 +7,6 @@ import inspect import warnings from functools import partial from uuid import uuid4 -from uuid import UUID from rq.compat import as_text, decode_redis_hash, string_types, text_type diff --git a/rq/queue.py b/rq/queue.py index 1d77b78..959addd 100644 --- a/rq/queue.py +++ b/rq/queue.py @@ -164,7 +164,7 @@ class Queue(object): def enqueue_call(self, func, args=None, kwargs=None, timeout=None, result_ttl=None, description=None, depends_on=None, - id=None): + job_id=None): """Creates a job to represent the delayed function call and enqueues it. @@ -178,7 +178,7 @@ class Queue(object): job = self.job_class.create(func, args, kwargs, connection=self.connection, result_ttl=result_ttl, status=Status.QUEUED, description=description, depends_on=depends_on, timeout=timeout, - id=id) + id=job_id) # If job depends on an unfinished job, register itself on it's # parent's dependents instead of enqueueing it. From 30ea76ddd64464ed5677352d4b56b4d5f108d826 Mon Sep 17 00:00:00 2001 From: foxx Date: Tue, 9 Sep 2014 11:12:39 +0100 Subject: [PATCH 8/9] Fixed silly typo --- rq/queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rq/queue.py b/rq/queue.py index 959addd..4cc3630 100644 --- a/rq/queue.py +++ b/rq/queue.py @@ -234,7 +234,7 @@ class Queue(object): return self.enqueue_call(func=f, args=args, kwargs=kwargs, timeout=timeout, result_ttl=result_ttl, description=description, depends_on=depends_on, - id=job_id) + job_id=job_id) def enqueue_job(self, job, set_meta_data=True): """Enqueues a job for delayed execution. From 6aa2e18f886d50ec0795290d26e68ee65b628836 Mon Sep 17 00:00:00 2001 From: Cal Leeming Date: Tue, 9 Sep 2014 17:47:41 +0100 Subject: [PATCH 9/9] Another unused import, oops --- tests/test_job.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index d3f7aa7..78e41e2 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -3,7 +3,6 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) from datetime import datetime -from uuid import uuid4 from rq.compat import as_text, PY2 from rq.exceptions import NoSuchJobError, UnpickleError @@ -348,4 +347,4 @@ class TestJob(RQTestCase): self.assertEqual(job.id, "1234") job.perform() - self.assertRaises(TypeError, queue.enqueue, say_hello, job_id=1234) \ No newline at end of file + self.assertRaises(TypeError, queue.enqueue, say_hello, job_id=1234)