From 7f9f0f72ba1adac907791d604d5f56af8125fdfd Mon Sep 17 00:00:00 2001 From: Joachim Burket Date: Tue, 7 Mar 2023 04:03:31 +0100 Subject: [PATCH] Update arguments passed to the Sentinel Object when created from the settings (#1850) * Updated arguments passed to the Sentinel Object when created from the settings - added `USERNAME` key - added `CONNECTION_KWARGS` key to allow passing additionals arguments to the Redis connections - updated the documentation * added missing comma * tests(helpers): Added tests for Sentinel --------- Co-authored-by: Joachim Burket --- docs/docs/connections.md | 20 ++++++++++++----- rq/cli/helpers.py | 20 +++++++++++------ tests/test_helpers.py | 48 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 74 insertions(+), 14 deletions(-) diff --git a/docs/docs/connections.md b/docs/docs/connections.md index 87aba21..f58da10 100644 --- a/docs/docs/connections.md +++ b/docs/docs/connections.md @@ -136,11 +136,21 @@ Using this setting in conjunction with the systemd or docker containers with the automatic restart option allows workers and RQ to have a fault-tolerant connection to the redis. ```python -SENTINEL: {'INSTANCES':[('remote.host1.org', 26379), ('remote.host2.org', 26379), ('remote.host3.org', 26379)], - 'SOCKET_TIMEOUT': None, - 'PASSWORD': 'secret', - 'DB': 2, - 'MASTER_NAME': 'master'} +SENTINEL: { + 'INSTANCES':[('remote.host1.org', 26379), ('remote.host2.org', 26379), ('remote.host3.org', 26379)], + 'MASTER_NAME': 'master', + 'DB': 2, + 'USERNAME': 'redis-user', + 'PASSWORD': 'redis-secret', + 'SOCKET_TIMEOUT': None, + 'CONNECTION_KWARGS': { # Eventual addition Redis connection arguments + 'ssl_ca_path': None, + }, + 'SENTINEL_KWARGS': { # Eventual Sentinels connections arguments + 'username': 'sentinel-user', + 'password': 'sentinel-secret', + }, +} ``` diff --git a/rq/cli/helpers.py b/rq/cli/helpers.py index 53bc019..0f87d22 100644 --- a/rq/cli/helpers.py +++ b/rq/cli/helpers.py @@ -33,21 +33,27 @@ def get_redis_from_config(settings, connection_class=Redis): """Returns a StrictRedis instance from a dictionary of settings. To use redis sentinel, you must specify a dictionary in the configuration file. Example of a dictionary with keys without values: - SENTINEL = {'INSTANCES':, 'SOCKET_TIMEOUT':, 'PASSWORD':,'DB':, 'MASTER_NAME':} + SENTINEL = {'INSTANCES':, 'SOCKET_TIMEOUT':, 'USERNAME':, 'PASSWORD':, 'DB':, 'MASTER_NAME':, 'SENTINEL_KWARGS':} """ if settings.get('REDIS_URL') is not None: return connection_class.from_url(settings['REDIS_URL']) elif settings.get('SENTINEL') is not None: instances = settings['SENTINEL'].get('INSTANCES', [('localhost', 26379)]) - socket_timeout = settings['SENTINEL'].get('SOCKET_TIMEOUT', None) - password = settings['SENTINEL'].get('PASSWORD', None) - db = settings['SENTINEL'].get('DB', 0) master_name = settings['SENTINEL'].get('MASTER_NAME', 'mymaster') - ssl = settings['SENTINEL'].get('SSL', False) - arguments = {'password': password, 'ssl': ssl} + + connection_kwargs = { + 'db': settings['SENTINEL'].get('DB', 0), + 'username': settings['SENTINEL'].get('USERNAME', None), + 'password': settings['SENTINEL'].get('PASSWORD', None), + 'socket_timeout': settings['SENTINEL'].get('SOCKET_TIMEOUT', None), + 'ssl': settings['SENTINEL'].get('SSL', False), + } + connection_kwargs.update(settings['SENTINEL'].get('CONNECTION_KWARGS', {})) + sentinel_kwargs = settings['SENTINEL'].get('SENTINEL_KWARGS', {}) + sn = Sentinel( - instances, socket_timeout=socket_timeout, password=password, db=db, ssl=ssl, sentinel_kwargs=arguments + instances, sentinel_kwargs=sentinel_kwargs, **connection_kwargs ) return sn.master_for(master_name) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index b43f13b..5a84f71 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,11 +1,12 @@ from rq.cli.helpers import get_redis_from_config from tests import RQTestCase - +from unittest import mock class TestHelpers(RQTestCase): - def test_get_redis_from_config(self): + @mock.patch('rq.cli.helpers.Sentinel') + def test_get_redis_from_config(self, sentinel_class_mock): """Ensure Redis connection params are properly parsed""" settings = { 'REDIS_URL': 'redis://localhost:1/1' @@ -39,3 +40,46 @@ class TestHelpers(RQTestCase): self.assertEqual(connection_kwargs['db'], 2) self.assertEqual(connection_kwargs['port'], 2) self.assertEqual(connection_kwargs['password'], 'bar') + + # Add Sentinel to the settings + settings.update({ + 'SENTINEL': { + 'INSTANCES':[('remote.host1.org', 26379), ('remote.host2.org', 26379), ('remote.host3.org', 26379)], + 'MASTER_NAME': 'master', + 'DB': 2, + 'USERNAME': 'redis-user', + 'PASSWORD': 'redis-secret', + 'SOCKET_TIMEOUT': None, + 'CONNECTION_KWARGS': { + 'ssl_ca_path': None, + }, + 'SENTINEL_KWARGS': { + 'username': 'sentinel-user', + 'password': 'sentinel-secret', + }, + }, + }) + + # Ensure SENTINEL is preferred against REDIS_* parameters + redis = get_redis_from_config(settings) + sentinel_init_sentinels_args = sentinel_class_mock.call_args[0] + sentinel_init_sentinel_kwargs = sentinel_class_mock.call_args[1] + self.assertEqual( + sentinel_init_sentinels_args, + ([('remote.host1.org', 26379), ('remote.host2.org', 26379), ('remote.host3.org', 26379)],) + ) + self.assertDictEqual( + sentinel_init_sentinel_kwargs, + { + 'db': 2, + 'ssl': False, + 'username': 'redis-user', + 'password': 'redis-secret', + 'socket_timeout': None, + 'ssl_ca_path': None, + 'sentinel_kwargs': { + 'username': 'sentinel-user', + 'password': 'sentinel-secret', + } + } + )