Adds master-slave tests.

Makefile

@@ -17,6 +17,7 @@ $(VENV_ACTIVATE): requirements*.txt
 install_requirements: requirements*.txt
 	$(WITH_VENV) pip install --no-deps -r requirements.txt
 	$(WITH_VENV) pip install --no-deps -r requirements-dev.txt
+	$(WITH_VENV) pip install -r requirements-local.txt
 	$(WITH_VENV) pip install Django==$(DJANGO_VERSION)
 
 .PHONY: venv
@@ -46,6 +47,22 @@ redis_servers:
     		--port 0 \
     		--unixsocket /tmp/redis`echo $$i`.sock \
     		--unixsocketperm 755 ; \
+	    done
+
+	    ./redis/src/redis-server \
+			--pidfile /tmp/redis7.pid \
+			--requirepass yadayada \
+			--daemonize yes \
+			--port 6387 ;
+
+	for i in 8 9; do \
+		./redis/src/redis-server \
+            --pidfile /tmp/redis`echo $$i`.pid \
+            --requirepass yadayada \
+            --daemonize yes \
+            --masterauth yadayada \
+            --slaveof 127.0.0.1 6387 \
+            --port `echo 638$$i` ; \
     	done
 
 .PHONY: clean
@@ -66,7 +83,7 @@ teardown:
 .PHONY: test
 test: venv install_requirements redis_servers
 	$(WITH_VENV) PYTHONPATH=$(PYTHONPATH): django-admin.py test --settings=tests.settings -s
-	for i in 1 2 3 4 5 6; do kill `cat /tmp/redis$$i.pid`; done;
+	for i in 1 2 3 4 5 6 7 8 9; do kill `cat /tmp/redis$$i.pid`; done;
 
 .PHONY: shell
 shell: venv

redis_cache/backends/base.py

@@ -33,66 +33,30 @@ class BaseRedisCache(BaseCache):
         self.params = params or {}
         self.options = params.get('OPTIONS', {})
 
+        self.db = self.get_db()
+        self.password = self.get_password()
+        self.parser_class = self.get_parser_class()
+        self.pickle_version = self.get_pickle_version()
+        self.connection_pool_class = self.get_connection_pool_class()
+        self.connection_pool_class_kwargs = self.get_connection_pool_class_kwargs()
+
     def __getstate__(self):
         return {'params': self.params, 'server': self.server}
 
     def __setstate__(self, state):
         self.__init__(**state)
 
-    def create_client(self, server):
-        kwargs = {
-            'db': self.db,
-            'password': self.password,
-        }
-        if '://' in server:
-            client = redis.Redis.from_url(
-                server,
-                parser_class=self.parser_class,
-                **kwargs
-            )
-            kwargs.update(
-                client.connection_pool.connection_kwargs,
-                unix_socket_path=client.connection_pool.connection_kwargs.get('path'),
-            )
-        else:
-            unix_socket_path = None
-            if ':' in server:
-                host, port = server.rsplit(':', 1)
-                try:
-                    port = int(port)
-                except (ValueError, TypeError):
-                    raise ImproperlyConfigured("Port value must be an integer")
-            else:
-                host, port = None, None
-                unix_socket_path = server
-
-            kwargs.update(host=host, port=port, unix_socket_path=unix_socket_path)
-            client = redis.Redis(**kwargs)
-
-        kwargs.update(
-            parser_class=self.parser_class,
-            connection_pool_class=self.connection_pool_class,
-            connection_pool_class_kwargs=self.connection_pool_class_kwargs,
-        )
-
-        connection_pool = pool.get_connection_pool(**kwargs)
-        client.connection_pool = connection_pool
-        return client
-
-    @cached_property
-    def db(self):
+    def get_db(self):
         _db = self.params.get('db', self.options.get('DB', 1))
         try:
             return int(_db)
         except (ValueError, TypeError):
             raise ImproperlyConfigured("db value must be an integer")
 
-    @cached_property
-    def password(self):
+    def get_password(self):
         return self.params.get('password', self.options.get('PASSWORD', None))
 
-    @cached_property
-    def parser_class(self):
+    def get_parser_class(self):
         cls = self.options.get('PARSER_CLASS', None)
         if cls is None:
             return DefaultParser
@@ -106,8 +70,7 @@ class BaseRedisCache(BaseCache):
             raise ImproperlyConfigured("Could not find module '%s'" % e)
         return parser_class
 
-    @cached_property
-    def pickle_version(self):
+    def get_pickle_version(self):
         """
         Get the pickle version from the settings and save it for future use
         """
@@ -117,8 +80,7 @@ class BaseRedisCache(BaseCache):
         except (ValueError, TypeError):
             raise ImproperlyConfigured("pickle version value must be an integer")
 
-    @cached_property
-    def connection_pool_class(self):
+    def get_connection_pool_class(self):
         pool_class = self.options.get('CONNECTION_POOL_CLASS', 'redis.ConnectionPool')
         module_name, class_name = pool_class.rsplit('.', 1)
         module = import_module(module_name)
@@ -127,30 +89,53 @@ class BaseRedisCache(BaseCache):
         except AttributeError:
             raise ImportError('cannot import name %s' % class_name)
 
-    @cached_property
-    def connection_pool_class_kwargs(self):
+    def get_connection_pool_class_kwargs(self):
         return self.options.get('CONNECTION_POOL_CLASS_KWARGS', {})
 
-    @cached_property
-    def master_client(self):
-        """
-        Get the write server:port of the master cache
-        """
-        cache = self.options.get('MASTER_CACHE', None)
-        if cache is None:
-            self._master_client = None
+    def create_client(self, server):
+        kwargs = {
+            'db': self.db,
+            'password': self.password,
+        }
+        if '://' in server:
+            client = redis.Redis.from_url(
+                server,
+                parser_class=self.parser_class,
+                **kwargs
+            )
+            unix_socket_path = client.connection_pool.connection_kwargs.get('path')
+            kwargs.update(
+                client.connection_pool.connection_kwargs,
+                unix_socket_path=unix_socket_path,
+            )
         else:
-            self._master_client = None
-            try:
-                host, port = cache.split(":")
-            except ValueError:
-                raise ImproperlyConfigured("MASTER_CACHE must be in the form <host>:<port>")
-            for client in self.clients.itervalues():
-                connection_kwargs = client.connection_pool.connection_kwargs
-                if connection_kwargs['host'] == host and connection_kwargs['port'] == int(port):
-                    return client
-            if self._master_client is None:
-                raise ImproperlyConfigured("%s is not in the list of available redis-server instances." % cache)
+            unix_socket_path = None
+            if ':' in server:
+                host, port = server.rsplit(':', 1)
+                try:
+                    port = int(port)
+                except (ValueError, TypeError):
+                    raise ImproperlyConfigured("Port value must be an integer")
+            else:
+                host, port = None, None
+                unix_socket_path = server
+
+            kwargs.update(
+                host=host,
+                port=port,
+                unix_socket_path=unix_socket_path,
+            )
+            client = redis.Redis(**kwargs)
+
+        kwargs.update(
+            parser_class=self.parser_class,
+            connection_pool_class=self.connection_pool_class,
+            connection_pool_class_kwargs=self.connection_pool_class_kwargs,
+        )
+
+        connection_pool = pool.get_connection_pool(client, **kwargs)
+        client.connection_pool = connection_pool
+        return client
 
     def serialize(self, value):
         return pickle.dumps(value, self.pickle_version)

redis_cache/backends/multiple.py

@@ -38,18 +38,7 @@ class ShardedRedisCache(BaseRedisCache):
             if cache is None:
                 self._master_client = None
             else:
-                self._master_client = None
-                try:
-                    host, port = cache.split(":")
-                except ValueError:
-                    raise ImproperlyConfigured("MASTER_CACHE must be in the form <host>:<port>")
-                for client in self.clients:
-                    connection_kwargs = client.connection_pool.connection_kwargs
-                    if connection_kwargs['host'] == host and connection_kwargs['port'] == int(port):
-                        self._master_client = client
-                        break
-                if self._master_client is None:
-                    raise ImproperlyConfigured("%s is not in the list of available redis-server instances." % cache)
+                self._master_client = self.create_client(cache)
         return self._master_client
 
     def get_client(self, key, for_write=False):

redis_cache/connection.py

@@ -4,10 +4,24 @@ from redis.connection import UnixDomainSocketConnection, Connection
 class CacheConnectionPool(object):
 
     def __init__(self):
+        self._clients = {}
+        self._connection_pools = {}
+
+    def __contains__(self, server):
+        return server in self._clients
+
+    def __getitem__(self, server):
+        return self._clients.get(server, None)
+
+    def reset(self):
+        for pool in self._connection_pools.itervalues():
+            pool.disconnect()
+        self._clients = {}
         self._connection_pools = {}
 
     def get_connection_pool(
         self,
+        client,
         host='127.0.0.1',
         port=6379,
         db=1,
@@ -18,10 +32,9 @@ class CacheConnectionPool(object):
         connection_pool_class_kwargs=None,
         **kwargs
     ):
+        connection_identifier = (host, port, db, unix_socket_path)
 
-        connection_identifier = (
-            host, port, db, unix_socket_path
-        )
+        self._clients[connection_identifier] = client
 
         pool = self._connection_pools.get(connection_identifier)
 
@@ -46,10 +59,11 @@ class CacheConnectionPool(object):
             else:
                 kwargs['path'] = unix_socket_path
 
-            self._connection_pools[connection_identifier] = connection_pool_class(**kwargs)
-            self._connection_pools[connection_identifier].connection_identifier = (
-                connection_identifier
-            )
-        return self._connection_pools[connection_identifier]
+            pool = connection_pool_class(**kwargs)
+
+            self._connection_pools[connection_identifier] = pool
+            pool.connection_identifier = connection_identifier
+
+        return pool
 
 pool = CacheConnectionPool()

redis_cache/sharder.py

@@ -2,7 +2,6 @@ from bisect import insort, bisect
 from hashlib import md5
 from math import log
 import sys
-#from django.utils.encoding import smart_str
 
 
 DIGITS = int(log(sys.maxint) / log(16))
@@ -15,6 +14,7 @@ def make_hash(s):
 class Node(object):
     def __init__(self, node, i):
         self._node = node
+        self._i = i
         self._position = make_hash("%d:%s" % (i, str(self._node)))
 
     def __cmp__(self, other):

tests/testapp/tests/base_tests.py

@@ -26,8 +26,7 @@ class C:
         return 24
 
 
-class BaseRedisTestCase(object):
-
+class SetupMixin(object):
     def setUp(self):
         # use DB 16 for testing and hope there isn't any important data :->
         self.reset_pool()
@@ -42,13 +41,14 @@ class BaseRedisTestCase(object):
         self.cache.clear()
 
     def reset_pool(self):
-        if hasattr(self, 'cache'):
-            for client in self.cache.clients.itervalues():
-                client.connection_pool.disconnect()
+        pool.reset()
 
     def get_cache(self, backend=None):
         return get_cache(backend or 'default')
 
+
+class BaseRedisTestCase(SetupMixin):
+
     def test_simple(self):
         # Simple cache set/get works
         self.cache.set("key", "value")

tests/testapp/tests/master_slave_tests.py

+from django.test import TestCase
+try:
+    from django.test import override_settings
+except ImportError:
+    from django.test.utils import override_settings
+
+from redis_cache.connection import pool
+
+from tests.testapp.tests.base_tests import SetupMixin
+
+
+MASTER_LOCATION = "127.0.0.1:6387"
+LOCATIONS = [
+    '127.0.0.1:6387',
+    '127.0.0.1:6388',
+    '127.0.0.1:6389',
+]
+
+
+@override_settings(CACHES={
+    'default': {
+        'BACKEND': 'redis_cache.ShardedRedisCache',
+        'LOCATION': LOCATIONS,
+        'OPTIONS': {
+            'DB': 1,
+            'PASSWORD': 'yadayada',
+            'PARSER_CLASS': 'redis.connection.HiredisParser',
+            'PICKLE_VERSION': -1,
+            'MASTER_CACHE': MASTER_LOCATION,
+        },
+    },
+})
+class MasterSlaveTestCase(SetupMixin, TestCase):
+
+    def setUp(self):
+        super(MasterSlaveTestCase, self).setUp()
+        pool.reset()
+
+    def test_master_client(self):
+        cache = self.get_cache()
+        client = cache.master_client
+        self.assertEqual(
+            client.connection_pool.connection_identifier,
+            ('127.0.0.1', 6387, 1, None)
+        )
+        self.assertEqual(len(pool._connection_pools), 3)
+
+    def test_set(self):
+        cache = self.get_cache()
+        cache.set('a', 'a')
+        for client in self.cache.clients.itervalues():
+            key = cache.make_key('a')
+            self.assertIsNotNone(client.get(key))
+
+    def test_set_many(self):
+        cache = self.get_cache()
+        cache.set_many({'a': 'a', 'b': 'b'})
+        for client in self.cache.clients.itervalues():
+            self.assertNotIn(None, client.mget([
+                cache.make_key('a'),
+                cache.make_key('b'),
+            ]))
+
+    def test_incr(self):
+        cache = self.get_cache()
+        cache.set('a', 0)
+        cache.incr('a')
+        key = cache.make_key('a')
+        for client in self.cache.clients.itervalues():
+            self.assertEqual(client.get(key), '1')
+
+    def test_delete(self):
+        cache = self.get_cache()
+        cache.set('a', 'a')
+        self.assertEqual(cache.get('a'), 'a')
+        cache.delete('a')
+        key = cache.make_key('a')
+        for client in self.cache.clients.itervalues():
+            self.assertIsNone(client.get(key))
+
+    def test_clear(self):
+        cache = self.get_cache()
+        cache.set('a', 'a')
+        self.assertEqual(cache.get('a'), 'a')
+        cache.clear()
+        for client in self.cache.clients.itervalues():
+            self.assertEqual(len(client.keys('*')), 0)

tests/testapp/tests/multi_server_tests.py

@@ -16,10 +16,14 @@ class MultiServerTests(object):
 
     def test_key_distribution(self):
         n = 10000
+        self.cache.set('a', 'a')
         for i in xrange(n):
             self.cache.set(i, i)
-        keys = [len(client.keys('*')) for client in self.cache.clients.itervalues()]
-        self.assertTrue(((stddev(keys) / n) * 100.0) < 10)
+        keys = [
+            len(client.keys('*'))
+            for client in self.cache.clients.itervalues()
+        ]
+        self.assertLess(((stddev(keys) / n) * 100.0), 10)
 
     def test_removing_nodes(self):
         c1, c2, c3 = self.cache.clients.keys()

tests/testapp/tests/socket_tests.py

@@ -110,4 +110,3 @@ class MultipleHiredisTestCase(MultiServerTests, SocketTestCase):
 )
 class MultiplePythonParserTestCase(MultiServerTests, SocketTestCase):
     pass
-

tests/testapp/tests/tcp_tests.py

@@ -120,3 +120,4 @@ class MultipleHiredisTestCase(MultiServerTests, TCPTestCase):
 class MultiplePythonParserTestCase(MultiServerTests, TCPTestCase):
     pass
 
+