Skip to content

D
Django-Redis-Cache
Commit a0879695 authored by Sean Bleier's avatar Sean Bleier
Browse files
Options

Use subprocess to start redis instances.

parent 4476c411
Hide whitespace changes
Inline Side-by-side
Showing with 284 additions and 186 deletions
.gitignore
View file @ a0879695
......@@ -20,3 +20,4 @@ MANIFEST
redis/
*/_build/
requirements_local.txt
requirements-local.txt
.travis.yml
View file @ a0879695
......@@ -11,6 +11,7 @@ env:
- DJANGO_VERSION=1.7
- DJANGO_VERSION=1.8
# command to run tests
install: install_redis.sh
script: make test DJANGO_VERSION=$DJANGO_VERSION
branches:
only:
......
Makefile
View file @ a0879695
SHELL := /bin/bash
PACKAGE_NAME=redis_cache
VENV_DIR?=.venv
VENV_ACTIVATE=$(VENV_DIR)/bin/activate
WITH_VENV=. $(VENV_ACTIVATE);
PACKAGE_NAME=redis_cache
DJANGO_VERSION?=1.7
default:
python setup.py check build
$(VENV_ACTIVATE): requirements*.txt
test -f $@ || virtualenv --system-site-packages $(VENV_DIR)
touch $@
.PHONY: install_requirements
install_requirements: requirements*.txt
$(WITH_VENV) pip install --no-deps -r requirements.txt
$(WITH_VENV) pip install --no-deps -r requirements-dev.txt
$(WITH_VENV) $(test -f requirements-local.txt && pip install -r requirements-local.txt)
$(WITH_VENV) pip install Django==$(DJANGO_VERSION)
.PHONY: venv
venv: $(VENV_ACTIVATE)
.PHONY: setup
setup: venv
.PHONY: redis_servers
redis_servers:
test -d redis || git clone https://github.com/antirez/redis
git -C redis checkout 2.6
make -C redis
for i in 1 2 3; do \
./redis/src/redis-server \
--pidfile /tmp/redis`echo $$i`.pid \
--requirepass yadayada \
--daemonize yes \
--port `echo 638$$i` ; \
done
for i in 4 5 6; do \
./redis/src/redis-server \
--pidfile /tmp/redis`echo $$i`.pid \
--requirepass yadayada \
--daemonize yes \
--port 0 \
--unixsocket /tmp/redis`echo $$i`.sock \
--unixsocketperm 755 ; \
done
./redis/src/redis-server \
--pidfile /tmp/redis7.pid \
--requirepass yadayada \
--daemonize yes \
--port 6387 ;
for i in 8 9; do \
./redis/src/redis-server \
--pidfile /tmp/redis`echo $$i`.pid \
--requirepass yadayada \
--daemonize yes \
--masterauth yadayada \
--slaveof 127.0.0.1 6387 \
--port `echo 638$$i` ; \
done
pip install --no-deps -r requirements.txt
pip install --no-deps -r requirements-dev.txt
pip install Django==$(DJANGO_VERSION)
.PHONY: clean
clean:
......@@ -74,18 +18,11 @@ clean:
rm -rf __pycache__/
rm -f MANIFEST
rm -f test.db
find $(PACKAGE_NAME) -type f -name '*.pyc' -delete
.PHONY: teardown
teardown:
rm -rf $(VENV_DIR)/
.PHONY: test
test: venv install_requirements redis_servers
$(WITH_VENV) PYTHONPATH=$(PYTHONPATH): django-admin.py test --settings=tests.settings -s
for i in 1 2 3 4 5 6 7 8 9; do kill `cat /tmp/redis$$i.pid`; done;
test: install_requirements
PYTHONPATH=$(PYTHONPATH): django-admin.py test --settings=tests.settings -s
.PHONY: shell
shell: venv
$(WITH_VENV) PYTHONPATH=$(PYTHONPATH): django-admin.py shell --settings=tests.settings
shell:
PYTHONPATH=$(PYTHONPATH): django-admin.py shell --settings=tests.settings
install_redis.sh 0 → 100755
View file @ a0879695
#!/bin/bash
: ${REDIS_VERSION:="2.6"}
test -d redis || git clone https://github.com/antirez/redis
git -C redis checkout $REDIS_VERSION
make -C redis
redis_cache/backends/base.py
View file @ a0879695
from django.core.cache.backends.base import BaseCache, InvalidCacheBackendError
from django.core.exceptions import ImproperlyConfigured
from django.utils import importlib
from django.utils.functional import cached_property
from django.utils.importlib import import_module
from redis_cache.compat import bytes_type, smart_bytes, DEFAULT_TIMEOUT
from redis_cache.compat import smart_bytes, DEFAULT_TIMEOUT
try:
import cPickle as pickle
......@@ -14,12 +13,14 @@ except ImportError:
try:
import redis
except ImportError:
raise InvalidCacheBackendError("Redis cache backend requires the 'redis-py' library")
raise InvalidCacheBackendError(
"Redis cache backend requires the 'redis-py' library"
)
from redis.connection import DefaultParser
from redis_cache.connection import pool
from redis_cache.utils import CacheKey
from redis_cache.utils import CacheKey, get_servers, parse_connection_kwargs
from functools import wraps
......@@ -34,7 +35,6 @@ def get_client(write=False):
version = kwargs.pop('version', None)
client = self.get_client(key, write=write)
key = self.make_key(key, version=version)
return method(self, client, key, *args, **kwargs)
return wrapped
......@@ -50,7 +50,7 @@ class BaseRedisCache(BaseCache):
"""
super(BaseRedisCache, self).__init__(params)
self.server = server
self.servers = self.get_servers(server)
self.servers = get_servers(server)
self.params = params or {}
self.options = params.get('OPTIONS', {})
self.clients = {}
......@@ -71,20 +71,6 @@ class BaseRedisCache(BaseCache):
def __setstate__(self, state):
self.__init__(**state)
def get_servers(self, server):
"""returns a list of servers given the server argument passed in
from Django.
"""
if isinstance(server, bytes_type):
servers = server.split(',')
elif hasattr(server, '__iter__'):
servers = server
else:
raise ImproperlyConfigured(
'"server" must be an iterable or string'
)
return servers
def get_db(self):
_db = self.params.get('db', self.options.get('DB', 1))
try:
......@@ -136,49 +122,29 @@ class BaseRedisCache(BaseCache):
Get the write server:port of the master cache
"""
cache = self.options.get('MASTER_CACHE', None)
return self.client_list[0] if cache is None else self.create_client(cache)
if cache is None:
return self.client_list[0]
def create_client(self, server):
kwargs = {
'db': self.db,
'password': self.password,
}
if '://' in server:
client = redis.Redis.from_url(
server,
parser_class=self.parser_class,
**kwargs
)
unix_socket_path = client.connection_pool.connection_kwargs.get('path')
kwargs.update(
client.connection_pool.connection_kwargs,
unix_socket_path=unix_socket_path,
)
else:
unix_socket_path = None
if ':' in server:
host, port = server.rsplit(':', 1)
try:
port = int(port)
except (ValueError, TypeError):
raise ImproperlyConfigured("Port value must be an integer")
else:
host, port = None, None
unix_socket_path = server
kwargs.update(
host=host,
port=port,
unix_socket_path=unix_socket_path,
)
client = redis.Redis(**kwargs)
kwargs = parse_connection_kwargs(cache, db=self.db)
return self.clients[(
kwargs['host'],
kwargs['port'],
kwargs['db'],
kwargs['unix_socket_path'],
)]
def create_client(self, server):
kwargs = parse_connection_kwargs(
server,
db=self.db,
password=self.password,
)
client = redis.Redis(**kwargs)
kwargs.update(
parser_class=self.parser_class,
connection_pool_class=self.connection_pool_class,
connection_pool_class_kwargs=self.connection_pool_class_kwargs,
)
connection_pool = pool.get_connection_pool(client, **kwargs)
client.connection_pool = connection_pool
return client
......
redis_cache/utils.py
View file @ a0879695
from redis_cache.compat import smart_text, python_2_unicode_compatible
import warnings
from django.core.exceptions import ImproperlyConfigured
from redis.connection import SSLConnection, UnixDomainSocketConnection
from redis._compat import iteritems, urlparse, parse_qs
from redis_cache.compat import (
smart_text, python_2_unicode_compatible, bytes_type
)
@python_2_unicode_compatible
......@@ -17,3 +25,132 @@ class CacheKey(object):
return smart_text(self._versioned_key)
__repr__ = __str__ = __unicode__
def get_servers(location):
"""Returns a list of servers given the server argument passed in from
Django.
"""
if isinstance(location, bytes_type):
servers = location.split(',')
elif hasattr(location, '__iter__'):
servers = location
else:
raise ImproperlyConfigured(
'"server" must be an iterable or string'
)
return servers
def parse_connection_kwargs(server, db=None, **kwargs):
"""
Return a connection pool configured from the given URL.
For example::
redis://[:password]@localhost:6379/0
rediss://[:password]@localhost:6379/0
unix://[:password]@/path/to/socket.sock?db=0
Three URL schemes are supported:
redis:// creates a normal TCP socket connection
rediss:// creates a SSL wrapped TCP socket connection
unix:// creates a Unix Domain Socket connection
There are several ways to specify a database number. The parse function
will return the first specified option:
1. A ``db`` querystring option, e.g. redis://localhost?db=0
2. If using the redis:// scheme, the path argument of the url, e.g.
redis://localhost/0
3. The ``db`` argument to this function.
If none of these options are specified, db=0 is used.
Any additional querystring arguments and keyword arguments will be
passed along to the ConnectionPool class's initializer. In the case
of conflicting arguments, querystring arguments always win.
NOTE: taken from `redis.ConnectionPool.from_url` in redis-py
"""
if '://' in server:
url = server
url_string = url
url = urlparse(url)
qs = ''
# in python2.6, custom URL schemes don't recognize querystring values
# they're left as part of the url.path.
if '?' in url.path and not url.query:
# chop the querystring including the ? off the end of the url
# and reparse it.
qs = url.path.split('?', 1)[1]
url = urlparse(url_string[:-(len(qs) + 1)])
else:
qs = url.query
url_options = {}
for name, value in iteritems(parse_qs(qs)):
if value and len(value) > 0:
url_options[name] = value[0]
# We only support redis:// and unix:// schemes.
if url.scheme == 'unix':
url_options.update({
'password': url.password,
'unix_socket_path': url.path,
})
else:
url_options.update({
'host': url.hostname,
'port': int(url.port or 6379),
'password': url.password,
})
# If there's a path argument, use it as the db argument if a
# querystring value wasn't specified
if 'db' not in url_options and url.path:
try:
url_options['db'] = int(url.path.replace('/', ''))
except (AttributeError, ValueError):
pass
if url.scheme == 'rediss':
url_options['connection_class'] = SSLConnection
# last shot at the db value
url_options['db'] = int(url_options.get('db', db or 0))
# update the arguments from the URL values
kwargs.update(url_options)
# backwards compatability
if 'charset' in kwargs:
warnings.warn(DeprecationWarning(
'"charset" is deprecated. Use "encoding" instead'))
kwargs['encoding'] = kwargs.pop('charset')
if 'errors' in kwargs:
warnings.warn(DeprecationWarning(
'"errors" is deprecated. Use "encoding_errors" instead'))
kwargs['encoding_errors'] = kwargs.pop('errors')
else:
unix_socket_path = None
if ':' in server:
host, port = server.rsplit(':', 1)
try:
port = int(port)
except (ValueError, TypeError):
raise ImproperlyConfigured("Port value must be an integer")
else:
host, port = None, None
unix_socket_path = server
kwargs.update(
host=host,
port=port,
unix_socket_path=unix_socket_path,
db=db,
)
return kwargs
start_redis_servers.sh deleted 100755 → 0
View file @ 4476c411
#!/bin/bash
: ${REDIS_VERSION:="2.6"}
test -d redis || git clone https://github.com/antirez/redis
git -C redis checkout $REDIS_VERSION
make -C redis
for i in 1 2 3; do \
./redis/src/redis-server \
--pidfile /tmp/redis`echo $i`.pid \
--requirepass yadayada \
--daemonize yes \
--port `echo 638$i` ; \
done
for i in 4 5 6; do \
./redis/src/redis-server \
--pidfile /tmp/redis`echo $i`.pid \
--requirepass yadayada \
--daemonize yes \
--port 0 \
--unixsocket /tmp/redis`echo $i`.sock \
--unixsocketperm 755 ; \
done
./redis/src/redis-server \
--pidfile /tmp/redis7.pid \
--requirepass yadayada \
--daemonize yes \
--port 6387 ;
for i in 8 9; do \
./redis/src/redis-server \
--pidfile /tmp/redis`echo $i`.pid \
--requirepass yadayada \
--daemonize yes \
--masterauth yadayada \
--slaveof 127.0.0.1 6387 \
--port `echo 638$i` ; \
done
stop_redis_servers.sh deleted 100755 → 0
View file @ 4476c411
#!/bin/bash
for i in 1 2 3 4 5 6 7 8 9; do kill `cat /tmp/redis$i.pid`; done;
tests/settings.py
View file @ a0879695
......@@ -32,3 +32,4 @@ CACHES = {
},
}
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
MIDDLEWARE_CLASSES = tuple()
tests/testapp/tests/base_tests.py
View file @ a0879695
# -*- coding: utf-8 -*-
from hashlib import sha1
import os
import subprocess
import time
import unittest
......@@ -7,6 +9,7 @@ try:
import cPickle as pickle
except ImportError:
import pickle
from django.core.cache import get_cache
from django.core.exceptions import ImproperlyConfigured
from django.test import TestCase
......@@ -20,6 +23,10 @@ import redis
from tests.testapp.models import Poll, expensive_calculation
from redis_cache.cache import RedisCache, pool
from redis_cache.compat import DEFAULT_TIMEOUT
from redis_cache.utils import get_servers, parse_connection_kwargs
REDIS_PASSWORD = 'yadayada'
LOCATION = "127.0.0.1:6381"
......@@ -35,15 +42,92 @@ class C:
return 24
def start_redis_servers(servers, db=None, master=None):
"""Creates redis instances using specified locations from the settings.
Returns list of Popen objects
"""
processes = []
devnull = open(os.devnull, 'w')
master_connection_kwargs = master and parse_connection_kwargs(
master,
db=db,
password=REDIS_PASSWORD
)
for i, server in enumerate(servers):
connection_kwargs = parse_connection_kwargs(
server,
db=db,
password=REDIS_PASSWORD, # will be overridden if specified in `server`
)
parameters = dict(
port=connection_kwargs.get('port', 0),
requirepass=connection_kwargs['password'],
)
is_socket = server.startswith('unix://') or server.startswith('/')
if is_socket:
parameters.update(
port=0,
unixsocket='/tmp/redis{}.sock'.format(i),
unixsocketperm=755,
)
if master and not connection_kwargs == master_connection_kwargs:
parameters.update(
masterauth=master_connection_kwargs['password'],
slaveof="{host} {port}".format(
host=master_connection_kwargs['host'],
port=master_connection_kwargs['port'],
)
)
args = ['./redis/src/redis-server'] + [
"--{parameter} {value}".format(parameter=parameter, value=value)
for parameter, value in parameters.items()
]
p = subprocess.Popen(args, stdout=devnull)
processes.append(p)
return processes
class SetupMixin(object):
processes = None
@classmethod
def tearDownClass(cls):
for p in cls.processes:
p.kill()
cls.processes = None
# Give redis processes some time to shutdown
# time.sleep(.1)
def setUp(self):
# use DB 16 for testing and hope there isn't any important data :->
if self.__class__.processes is None:
from django.conf import settings
cache_settings = settings.CACHES['default']
servers = get_servers(cache_settings['LOCATION'])
options = cache_settings.get('OPTIONS', {})
db = options.get('db', 0)
master = options.get('MASTER_CACHE')
self.__class__.processes = start_redis_servers(
servers,
db=db,
master=master
)
# Give redis processes some time to startup
time.sleep(.1)
self.reset_pool()
self.cache = self.get_cache()
def tearDown(self):
# Sometimes it will be necessary to skip this method because we need to test default
# initialization and that may be using a different port than the test redis server.
# Sometimes it will be necessary to skip this method because we need to
# test default initialization and that may be using a different port
# than the test redis server.
if hasattr(self, '_skip_tearDown') and self._skip_tearDown:
self._skip_tearDown = False
return
......
tests/testapp/tests/master_slave_tests.py
View file @ a0879695
......@@ -50,15 +50,15 @@ class MasterSlaveTestCase(SetupMixin, TestCase):
def test_set(self):
cache = self.get_cache()
cache.set('a', 'a')
time.sleep(.5)
for client in self.cache.clients.itervalues():
time.sleep(.1)
for client in self.cache.clients.values():
key = cache.make_key('a')
self.assertIsNotNone(client.get(key))
def test_set_many(self):
cache = self.get_cache()
cache.set_many({'a': 'a', 'b': 'b'})
for client in self.cache.clients.itervalues():
for client in self.cache.clients.values():
self.assertNotIn(None, client.mget([
cache.make_key('a'),
cache.make_key('b'),
......@@ -68,24 +68,28 @@ class MasterSlaveTestCase(SetupMixin, TestCase):
cache = self.get_cache()
cache.set('a', 0)
cache.incr('a')
time.sleep(.5)
time.sleep(.1)
key = cache.make_key('a')
for client in self.cache.clients.itervalues():
for client in self.cache.clients.values():
self.assertEqual(client.get(key), '1')
def test_delete(self):
cache = self.get_cache()
cache.set('a', 'a')
time.sleep(.1)
self.assertEqual(cache.get('a'), 'a')
cache.delete('a')
time.sleep(.1)
key = cache.make_key('a')
for client in self.cache.clients.itervalues():
for client in self.cache.clients.values():
self.assertIsNone(client.get(key))
def test_clear(self):
cache = self.get_cache()
cache.set('a', 'a')
time.sleep(.1)
self.assertEqual(cache.get('a'), 'a')
cache.clear()
for client in self.cache.clients.itervalues():
time.sleep(.1)
for client in self.cache.clients.values():
self.assertEqual(len(client.keys('*')), 0)
tests/testapp/tests/socket_tests.py
View file @ a0879695
......@@ -12,11 +12,11 @@ from redis_cache.cache import ImproperlyConfigured
from redis.connection import UnixDomainSocketConnection
LOCATION = "unix://:yadayada@/tmp/redis4.sock?db=15"
LOCATION = "unix://:yadayada@/tmp/redis0.sock?db=15"
LOCATIONS = [
"unix://:yadayada@/tmp/redis4.sock?db=15",
"unix://:yadayada@/tmp/redis5.sock?db=15",
"unix://:yadayada@/tmp/redis6.sock?db=15",
"unix://:yadayada@/tmp/redis0.sock?db=15",
"unix://:yadayada@/tmp/redis1.sock?db=15",
"unix://:yadayada@/tmp/redis2.sock?db=15",
]
......
tests/testapp/tests/tcp_tests.py
View file @ a0879695
......@@ -24,6 +24,10 @@ class TCPTestCase(BaseRedisTestCase, TestCase):
def test_default_initialization(self):
self.reset_pool()
self.cache = self.get_cache()
self.assertIn(
('127.0.0.1', 6381, 15, None),
self.cache.clients,
)
client = self.cache.clients[('127.0.0.1', 6381, 15, None)]
connection_class = client.connection_pool.connection_class
if connection_class is not UnixDomainSocketConnection:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment