From 9318825429df096f47363c7bf9b7f8f3970a4b40 Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Mon, 13 Feb 2012 13:20:09 +0100 Subject: [PATCH 1/2] Abstract away from the concrete pickle implementation. Choose cPickle, if available, for best performance. --- rq/job.py | 4 ++-- rq/pickle.py | 9 +++++++++ rq/worker.py | 2 +- tests/test_job.py | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 rq/pickle.py diff --git a/rq/job.py b/rq/job.py index 386e594..70f502f 100644 --- a/rq/job.py +++ b/rq/job.py @@ -1,6 +1,6 @@ import times from uuid import uuid4 -from pickle import loads, dumps +from .pickle import loads, dumps, UnpicklingError from .proxy import conn from .exceptions import UnpickleError, NoSuchJobError @@ -15,7 +15,7 @@ def unpickle(pickled_string): """ try: obj = loads(pickled_string) - except StandardError: + except (StandardError, UnpicklingError): raise UnpickleError('Could not unpickle.', pickled_string) return obj diff --git a/rq/pickle.py b/rq/pickle.py new file mode 100644 index 0000000..81e77b3 --- /dev/null +++ b/rq/pickle.py @@ -0,0 +1,9 @@ +from __future__ import absolute_import +try: + import cPickle as pickle +except ImportError: + import pickle # noqa + +loads = pickle.loads +dumps = pickle.dumps +UnpicklingError = pickle.UnpicklingError diff --git a/rq/worker.py b/rq/worker.py index a6bbd51..820346d 100644 --- a/rq/worker.py +++ b/rq/worker.py @@ -8,7 +8,7 @@ import procname import socket import signal import traceback -from pickle import dumps +from .pickle import dumps try: from logbook import Logger Logger = Logger # Does nothing except it shuts up pyflakes annoying error diff --git a/tests/test_job.py b/tests/test_job.py index c62c1d9..d913986 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -2,7 +2,7 @@ import times from datetime import datetime from tests import RQTestCase from tests.helpers import strip_milliseconds -from pickle import loads +from rq.pickle import loads from rq.job import Job from rq.exceptions import NoSuchJobError, UnpickleError From 8da204f74aa8e0670b0dd890f474c44d59f5b250 Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Mon, 13 Feb 2012 13:25:46 +0100 Subject: [PATCH 2/2] Always use cPickle, never 'regular' pickle. This fixes #18. --- rq/job.py | 2 +- rq/pickle.py | 9 --------- rq/worker.py | 2 +- tests/test_job.py | 2 +- 4 files changed, 3 insertions(+), 12 deletions(-) delete mode 100644 rq/pickle.py diff --git a/rq/job.py b/rq/job.py index 70f502f..c82a197 100644 --- a/rq/job.py +++ b/rq/job.py @@ -1,6 +1,6 @@ import times from uuid import uuid4 -from .pickle import loads, dumps, UnpicklingError +from cPickle import loads, dumps, UnpicklingError from .proxy import conn from .exceptions import UnpickleError, NoSuchJobError diff --git a/rq/pickle.py b/rq/pickle.py deleted file mode 100644 index 81e77b3..0000000 --- a/rq/pickle.py +++ /dev/null @@ -1,9 +0,0 @@ -from __future__ import absolute_import -try: - import cPickle as pickle -except ImportError: - import pickle # noqa - -loads = pickle.loads -dumps = pickle.dumps -UnpicklingError = pickle.UnpicklingError diff --git a/rq/worker.py b/rq/worker.py index 820346d..cb8c282 100644 --- a/rq/worker.py +++ b/rq/worker.py @@ -8,7 +8,7 @@ import procname import socket import signal import traceback -from .pickle import dumps +from cPickle import dumps try: from logbook import Logger Logger = Logger # Does nothing except it shuts up pyflakes annoying error diff --git a/tests/test_job.py b/tests/test_job.py index d913986..55e589e 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -2,7 +2,7 @@ import times from datetime import datetime from tests import RQTestCase from tests.helpers import strip_milliseconds -from rq.pickle import loads +from cPickle import loads from rq.job import Job from rq.exceptions import NoSuchJobError, UnpickleError