Use subprocess to start redis instances.

.gitignore

@@ -20,3 +20,4 @@ MANIFEST
 redis/
 */_build/
 requirements_local.txt
+requirements-local.txt

.travis.yml

@@ -11,6 +11,7 @@ env:
   - DJANGO_VERSION=1.7
   - DJANGO_VERSION=1.8
 # command to run tests
+install: install_redis.sh
 script: make test DJANGO_VERSION=$DJANGO_VERSION
 branches:
   only:

Makefile

 SHELL := /bin/bash
-PACKAGE_NAME=redis_cache
 
-VENV_DIR?=.venv
-VENV_ACTIVATE=$(VENV_DIR)/bin/activate
-WITH_VENV=. $(VENV_ACTIVATE);
+PACKAGE_NAME=redis_cache
 DJANGO_VERSION?=1.7
 
-default:
-	python setup.py check build
-
-$(VENV_ACTIVATE): requirements*.txt
-	test -f $@ || virtualenv --system-site-packages $(VENV_DIR)
-	touch $@
-
 .PHONY: install_requirements
 install_requirements: requirements*.txt
-	$(WITH_VENV) pip install --no-deps -r requirements.txt
-	$(WITH_VENV) pip install --no-deps -r requirements-dev.txt
-	$(WITH_VENV) $(test -f requirements-local.txt && pip install -r requirements-local.txt)
-	$(WITH_VENV) pip install Django==$(DJANGO_VERSION)
-
-.PHONY: venv
-venv: $(VENV_ACTIVATE)
-
-.PHONY: setup
-setup: venv
-
-.PHONY: redis_servers
-redis_servers:
-	test -d redis || git clone https://github.com/antirez/redis
-	git -C redis checkout 2.6
-	make -C redis
-	for i in 1 2 3; do \
-    	./redis/src/redis-server \
-    		--pidfile /tmp/redis`echo $$i`.pid \
-    		--requirepass yadayada \
-    		--daemonize yes \
-    		--port `echo 638$$i` ; \
-    	done
-
-	for i in 4 5 6; do \
-    	./redis/src/redis-server \
-    		--pidfile /tmp/redis`echo $$i`.pid \
-    		--requirepass yadayada \
-    		--daemonize yes \
-    		--port 0 \
-    		--unixsocket /tmp/redis`echo $$i`.sock \
-    		--unixsocketperm 755 ; \
-	    done
-
-	    ./redis/src/redis-server \
-			--pidfile /tmp/redis7.pid \
-			--requirepass yadayada \
-			--daemonize yes \
-			--port 6387 ;
-
-	for i in 8 9; do \
-		./redis/src/redis-server \
-            --pidfile /tmp/redis`echo $$i`.pid \
-            --requirepass yadayada \
-            --daemonize yes \
-            --masterauth yadayada \
-            --slaveof 127.0.0.1 6387 \
-            --port `echo 638$$i` ; \
-    	done
+	pip install --no-deps -r requirements.txt
+	pip install --no-deps -r requirements-dev.txt
+	pip install Django==$(DJANGO_VERSION)
 
 .PHONY: clean
 clean:
@@ -74,18 +18,11 @@ clean:
 	rm -rf __pycache__/
 	rm -f MANIFEST
 	rm -f test.db
-	find $(PACKAGE_NAME) -type f -name '*.pyc' -delete
-
-.PHONY: teardown
-teardown:
-	rm -rf $(VENV_DIR)/
 
 .PHONY: test
-test: venv install_requirements redis_servers
-	$(WITH_VENV) PYTHONPATH=$(PYTHONPATH): django-admin.py test --settings=tests.settings -s
-	for i in 1 2 3 4 5 6 7 8 9; do kill `cat /tmp/redis$$i.pid`; done;
+test: install_requirements
+	PYTHONPATH=$(PYTHONPATH): django-admin.py test --settings=tests.settings -s
 
 .PHONY: shell
-shell: venv
-	$(WITH_VENV) PYTHONPATH=$(PYTHONPATH): django-admin.py shell --settings=tests.settings
-
+shell:
+	PYTHONPATH=$(PYTHONPATH): django-admin.py shell --settings=tests.settings

install_redis.sh

+#!/bin/bash
+
+: ${REDIS_VERSION:="2.6"}
+
+test -d redis || git clone https://github.com/antirez/redis
+git -C redis checkout $REDIS_VERSION
+make -C redis

redis_cache/backends/base.py

 from django.core.cache.backends.base import BaseCache, InvalidCacheBackendError
 from django.core.exceptions import ImproperlyConfigured
 from django.utils import importlib
-from django.utils.functional import cached_property
 from django.utils.importlib import import_module
 
-from redis_cache.compat import bytes_type, smart_bytes, DEFAULT_TIMEOUT
+from redis_cache.compat import smart_bytes, DEFAULT_TIMEOUT
 
 try:
     import cPickle as pickle
@@ -14,12 +13,14 @@ except ImportError:
 try:
     import redis
 except ImportError:
-    raise InvalidCacheBackendError("Redis cache backend requires the 'redis-py' library")
+    raise InvalidCacheBackendError(
+        "Redis cache backend requires the 'redis-py' library"
+    )
 
 from redis.connection import DefaultParser
 
 from redis_cache.connection import pool
-from redis_cache.utils import CacheKey
+from redis_cache.utils import CacheKey, get_servers, parse_connection_kwargs
 
 
 from functools import wraps
@@ -34,7 +35,6 @@ def get_client(write=False):
             version = kwargs.pop('version', None)
             client = self.get_client(key, write=write)
             key = self.make_key(key, version=version)
-
             return method(self, client, key, *args, **kwargs)
 
         return wrapped
@@ -50,7 +50,7 @@ class BaseRedisCache(BaseCache):
         """
         super(BaseRedisCache, self).__init__(params)
         self.server = server
-        self.servers = self.get_servers(server)
+        self.servers = get_servers(server)
         self.params = params or {}
         self.options = params.get('OPTIONS', {})
         self.clients = {}
@@ -71,20 +71,6 @@ class BaseRedisCache(BaseCache):
     def __setstate__(self, state):
         self.__init__(**state)
 
-    def get_servers(self, server):
-        """returns a list of servers given the server argument passed in
-        from Django.
-        """
-        if isinstance(server, bytes_type):
-            servers = server.split(',')
-        elif hasattr(server, '__iter__'):
-            servers = server
-        else:
-            raise ImproperlyConfigured(
-                '"server" must be an iterable or string'
-            )
-        return servers
-
     def get_db(self):
         _db = self.params.get('db', self.options.get('DB', 1))
         try:
@@ -136,49 +122,29 @@ class BaseRedisCache(BaseCache):
         Get the write server:port of the master cache
         """
         cache = self.options.get('MASTER_CACHE', None)
-        return self.client_list[0] if cache is None else self.create_client(cache)
+        if cache is None:
+            return self.client_list[0]
 
-    def create_client(self, server):
-        kwargs = {
-            'db': self.db,
-            'password': self.password,
-        }
-        if '://' in server:
-            client = redis.Redis.from_url(
-                server,
-                parser_class=self.parser_class,
-                **kwargs
-            )
-            unix_socket_path = client.connection_pool.connection_kwargs.get('path')
-            kwargs.update(
-                client.connection_pool.connection_kwargs,
-                unix_socket_path=unix_socket_path,
-            )
-        else:
-            unix_socket_path = None
-            if ':' in server:
-                host, port = server.rsplit(':', 1)
-                try:
-                    port = int(port)
-                except (ValueError, TypeError):
-                    raise ImproperlyConfigured("Port value must be an integer")
-            else:
-                host, port = None, None
-                unix_socket_path = server
-
-            kwargs.update(
-                host=host,
-                port=port,
-                unix_socket_path=unix_socket_path,
-            )
-            client = redis.Redis(**kwargs)
+        kwargs = parse_connection_kwargs(cache, db=self.db)
+        return self.clients[(
+            kwargs['host'],
+            kwargs['port'],
+            kwargs['db'],
+            kwargs['unix_socket_path'],
+        )]
 
+    def create_client(self, server):
+        kwargs = parse_connection_kwargs(
+            server,
+            db=self.db,
+            password=self.password,
+        )
+        client = redis.Redis(**kwargs)
         kwargs.update(
             parser_class=self.parser_class,
             connection_pool_class=self.connection_pool_class,
             connection_pool_class_kwargs=self.connection_pool_class_kwargs,
         )
-
         connection_pool = pool.get_connection_pool(client, **kwargs)
         client.connection_pool = connection_pool
         return client

redis_cache/utils.py

-from redis_cache.compat import smart_text, python_2_unicode_compatible
+import warnings
+
+from django.core.exceptions import ImproperlyConfigured
+from redis.connection import SSLConnection, UnixDomainSocketConnection
+from redis._compat import iteritems, urlparse, parse_qs
+
+from redis_cache.compat import (
+    smart_text, python_2_unicode_compatible, bytes_type
+)
 
 
 @python_2_unicode_compatible
@@ -17,3 +25,132 @@ class CacheKey(object):
         return smart_text(self._versioned_key)
 
     __repr__ = __str__ = __unicode__
+
+
+def get_servers(location):
+    """Returns a list of servers given the server argument passed in from
+    Django.
+    """
+    if isinstance(location, bytes_type):
+        servers = location.split(',')
+    elif hasattr(location, '__iter__'):
+        servers = location
+    else:
+        raise ImproperlyConfigured(
+            '"server" must be an iterable or string'
+        )
+    return servers
+
+
+def parse_connection_kwargs(server, db=None, **kwargs):
+    """
+    Return a connection pool configured from the given URL.
+
+    For example::
+
+        redis://[:password]@localhost:6379/0
+        rediss://[:password]@localhost:6379/0
+        unix://[:password]@/path/to/socket.sock?db=0
+
+    Three URL schemes are supported:
+        redis:// creates a normal TCP socket connection
+        rediss:// creates a SSL wrapped TCP socket connection
+        unix:// creates a Unix Domain Socket connection
+
+    There are several ways to specify a database number. The parse function
+    will return the first specified option:
+        1. A ``db`` querystring option, e.g. redis://localhost?db=0
+        2. If using the redis:// scheme, the path argument of the url, e.g.
+           redis://localhost/0
+        3. The ``db`` argument to this function.
+
+    If none of these options are specified, db=0 is used.
+
+    Any additional querystring arguments and keyword arguments will be
+    passed along to the ConnectionPool class's initializer. In the case
+    of conflicting arguments, querystring arguments always win.
+
+    NOTE: taken from `redis.ConnectionPool.from_url` in redis-py
+    """
+    if '://' in server:
+        url = server
+        url_string = url
+        url = urlparse(url)
+        qs = ''
+
+        # in python2.6, custom URL schemes don't recognize querystring values
+        # they're left as part of the url.path.
+        if '?' in url.path and not url.query:
+            # chop the querystring including the ? off the end of the url
+            # and reparse it.
+            qs = url.path.split('?', 1)[1]
+            url = urlparse(url_string[:-(len(qs) + 1)])
+        else:
+            qs = url.query
+
+        url_options = {}
+
+        for name, value in iteritems(parse_qs(qs)):
+            if value and len(value) > 0:
+                url_options[name] = value[0]
+
+        # We only support redis:// and unix:// schemes.
+        if url.scheme == 'unix':
+            url_options.update({
+                'password': url.password,
+                'unix_socket_path': url.path,
+            })
+
+        else:
+            url_options.update({
+                'host': url.hostname,
+                'port': int(url.port or 6379),
+                'password': url.password,
+            })
+
+            # If there's a path argument, use it as the db argument if a
+            # querystring value wasn't specified
+            if 'db' not in url_options and url.path:
+                try:
+                    url_options['db'] = int(url.path.replace('/', ''))
+                except (AttributeError, ValueError):
+                    pass
+
+            if url.scheme == 'rediss':
+                url_options['connection_class'] = SSLConnection
+
+        # last shot at the db value
+        url_options['db'] = int(url_options.get('db', db or 0))
+
+        # update the arguments from the URL values
+        kwargs.update(url_options)
+
+        # backwards compatability
+        if 'charset' in kwargs:
+            warnings.warn(DeprecationWarning(
+                '"charset" is deprecated. Use "encoding" instead'))
+            kwargs['encoding'] = kwargs.pop('charset')
+        if 'errors' in kwargs:
+            warnings.warn(DeprecationWarning(
+                '"errors" is deprecated. Use "encoding_errors" instead'))
+            kwargs['encoding_errors'] = kwargs.pop('errors')
+    else:
+        unix_socket_path = None
+        if ':' in server:
+            host, port = server.rsplit(':', 1)
+            try:
+                port = int(port)
+            except (ValueError, TypeError):
+                raise ImproperlyConfigured("Port value must be an integer")
+        else:
+            host, port = None, None
+            unix_socket_path = server
+
+        kwargs.update(
+            host=host,
+            port=port,
+            unix_socket_path=unix_socket_path,
+            db=db,
+        )
+
+    return kwargs

start_redis_servers.sh

-#!/bin/bash
-
-: ${REDIS_VERSION:="2.6"}
-
-test -d redis || git clone https://github.com/antirez/redis
-git -C redis checkout $REDIS_VERSION
-make -C redis
-
-for i in 1 2 3; do \
-    ./redis/src/redis-server \
-        --pidfile /tmp/redis`echo $i`.pid \
-        --requirepass yadayada \
-        --daemonize yes \
-        --port `echo 638$i` ; \
-    done
-
-for i in 4 5 6; do \
-    ./redis/src/redis-server \
-        --pidfile /tmp/redis`echo $i`.pid \
-        --requirepass yadayada \
-        --daemonize yes \
-        --port 0 \
-        --unixsocket /tmp/redis`echo $i`.sock \
-        --unixsocketperm 755 ; \
-    done
-
-    ./redis/src/redis-server \
-        --pidfile /tmp/redis7.pid \
-        --requirepass yadayada \
-        --daemonize yes \
-        --port 6387 ;
-
-for i in 8 9; do \
-    ./redis/src/redis-server \
-        --pidfile /tmp/redis`echo $i`.pid \
-        --requirepass yadayada \
-        --daemonize yes \
-        --masterauth yadayada \
-        --slaveof 127.0.0.1 6387 \
-        --port `echo 638$i` ; \
-    done

stop_redis_servers.sh

-#!/bin/bash
-
-for i in 1 2 3 4 5 6 7 8 9; do kill `cat /tmp/redis$i.pid`; done;

tests/settings.py

@@ -32,3 +32,4 @@ CACHES = {
     },
 }
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
+MIDDLEWARE_CLASSES = tuple()

tests/testapp/tests/base_tests.py

 # -*- coding: utf-8 -*-
 from hashlib import sha1
+import os
+import subprocess
 import time
 import unittest
 
@@ -7,6 +9,7 @@ try:
     import cPickle as pickle
 except ImportError:
     import pickle
+
 from django.core.cache import get_cache
 from django.core.exceptions import ImproperlyConfigured
 from django.test import TestCase
@@ -20,6 +23,10 @@ import redis
 from tests.testapp.models import Poll, expensive_calculation
 from redis_cache.cache import RedisCache, pool
 from redis_cache.compat import DEFAULT_TIMEOUT
+from redis_cache.utils import get_servers, parse_connection_kwargs
+
+
+REDIS_PASSWORD = 'yadayada'
 
 
 LOCATION = "127.0.0.1:6381"
@@ -35,15 +42,92 @@ class C:
         return 24
 
 
+def start_redis_servers(servers, db=None, master=None):
+    """Creates redis instances using specified locations from the settings.
+
+    Returns list of Popen objects
+    """
+    processes = []
+    devnull = open(os.devnull, 'w')
+    master_connection_kwargs = master and parse_connection_kwargs(
+        master,
+        db=db,
+        password=REDIS_PASSWORD
+    )
+
+    for i, server in enumerate(servers):
+        connection_kwargs = parse_connection_kwargs(
+            server,
+            db=db,
+            password=REDIS_PASSWORD,  # will be overridden if specified in `server`
+        )
+        parameters = dict(
+            port=connection_kwargs.get('port', 0),
+            requirepass=connection_kwargs['password'],
+        )
+        is_socket = server.startswith('unix://') or server.startswith('/')
+        if is_socket:
+            parameters.update(
+                port=0,
+                unixsocket='/tmp/redis{}.sock'.format(i),
+                unixsocketperm=755,
+            )
+        if master and not connection_kwargs == master_connection_kwargs:
+            parameters.update(
+                masterauth=master_connection_kwargs['password'],
+                slaveof="{host} {port}".format(
+                    host=master_connection_kwargs['host'],
+                    port=master_connection_kwargs['port'],
+                )
+            )
+
+        args = ['./redis/src/redis-server'] + [
+            "--{parameter} {value}".format(parameter=parameter, value=value)
+            for parameter, value in parameters.items()
+        ]
+        p = subprocess.Popen(args, stdout=devnull)
+        processes.append(p)
+
+    return processes
+
+
 class SetupMixin(object):
+    processes = None
+
+    @classmethod
+    def tearDownClass(cls):
+        for p in cls.processes:
+            p.kill()
+        cls.processes = None
+
+        # Give redis processes some time to shutdown
+        # time.sleep(.1)
+
     def setUp(self):
-        # use DB 16 for testing and hope there isn't any important data :->
+        if self.__class__.processes is None:
+            from django.conf import settings
+
+            cache_settings = settings.CACHES['default']
+            servers = get_servers(cache_settings['LOCATION'])
+            options = cache_settings.get('OPTIONS', {})
+            db = options.get('db', 0)
+            master = options.get('MASTER_CACHE')
+            self.__class__.processes = start_redis_servers(
+                servers,
+                db=db,
+                master=master
+            )
+
+            # Give redis processes some time to startup
+            time.sleep(.1)
+
         self.reset_pool()
         self.cache = self.get_cache()
 
     def tearDown(self):
-        # Sometimes it will be necessary to skip this method because we need to test default
-        # initialization and that may be using a different port than the test redis server.
+        # Sometimes it will be necessary to skip this method because we need to
+        # test default initialization and that may be using a different port
+        # than the test redis server.
         if hasattr(self, '_skip_tearDown') and self._skip_tearDown:
             self._skip_tearDown = False
             return

tests/testapp/tests/master_slave_tests.py

@@ -50,15 +50,15 @@ class MasterSlaveTestCase(SetupMixin, TestCase):
     def test_set(self):
         cache = self.get_cache()
         cache.set('a', 'a')
-        time.sleep(.5)
-        for client in self.cache.clients.itervalues():
+        time.sleep(.1)
+        for client in self.cache.clients.values():
             key = cache.make_key('a')
             self.assertIsNotNone(client.get(key))
 
     def test_set_many(self):
         cache = self.get_cache()
         cache.set_many({'a': 'a', 'b': 'b'})
-        for client in self.cache.clients.itervalues():
+        for client in self.cache.clients.values():
             self.assertNotIn(None, client.mget([
                 cache.make_key('a'),
                 cache.make_key('b'),
@@ -68,24 +68,28 @@ class MasterSlaveTestCase(SetupMixin, TestCase):
         cache = self.get_cache()
         cache.set('a', 0)
         cache.incr('a')
-        time.sleep(.5)
+        time.sleep(.1)
         key = cache.make_key('a')
-        for client in self.cache.clients.itervalues():
+        for client in self.cache.clients.values():
             self.assertEqual(client.get(key), '1')
 
     def test_delete(self):
         cache = self.get_cache()
         cache.set('a', 'a')
+        time.sleep(.1)
         self.assertEqual(cache.get('a'), 'a')
         cache.delete('a')
+        time.sleep(.1)
         key = cache.make_key('a')
-        for client in self.cache.clients.itervalues():
+        for client in self.cache.clients.values():
             self.assertIsNone(client.get(key))
 
     def test_clear(self):
         cache = self.get_cache()
         cache.set('a', 'a')
+        time.sleep(.1)
         self.assertEqual(cache.get('a'), 'a')
         cache.clear()
-        for client in self.cache.clients.itervalues():
+        time.sleep(.1)
+        for client in self.cache.clients.values():
             self.assertEqual(len(client.keys('*')), 0)

tests/testapp/tests/socket_tests.py

@@ -12,11 +12,11 @@ from redis_cache.cache import ImproperlyConfigured
 from redis.connection import UnixDomainSocketConnection
 
 
-LOCATION = "unix://:yadayada@/tmp/redis4.sock?db=15"
+LOCATION = "unix://:yadayada@/tmp/redis0.sock?db=15"
 LOCATIONS = [
-    "unix://:yadayada@/tmp/redis4.sock?db=15",
-    "unix://:yadayada@/tmp/redis5.sock?db=15",
-    "unix://:yadayada@/tmp/redis6.sock?db=15",
+    "unix://:yadayada@/tmp/redis0.sock?db=15",
+    "unix://:yadayada@/tmp/redis1.sock?db=15",
+    "unix://:yadayada@/tmp/redis2.sock?db=15",
 ]
 
 

tests/testapp/tests/tcp_tests.py

@@ -24,6 +24,10 @@ class TCPTestCase(BaseRedisTestCase, TestCase):
     def test_default_initialization(self):
         self.reset_pool()
         self.cache = self.get_cache()
+        self.assertIn(
+            ('127.0.0.1', 6381, 15, None),
+            self.cache.clients,
+        )
         client = self.cache.clients[('127.0.0.1', 6381, 15, None)]
         connection_class = client.connection_pool.connection_class
         if connection_class is not UnixDomainSocketConnection: