Skip to content

D
Django-Redis-Cache
Commit 9ef12f52 authored by Sean Bleier's avatar Sean Bleier
Browse files
Options

Merge pull request #85 from sebleier/unstable 

Unstable
parents eb85ab99 176a0d2d
Hide whitespace changes
Inline Side-by-side
Showing with 409 additions and 203 deletions
.gitignore
View file @ 9ef12f52
......@@ -19,4 +19,3 @@ MANIFEST
.venv
redis/
*/_build/
requirements_local.txt
.travis.yml
View file @ 9ef12f52
language: python
python:
- "2.6"
- "2.7"
- "3.2"
- "3.3"
......@@ -11,6 +10,7 @@ env:
- DJANGO_VERSION=1.7
- DJANGO_VERSION=1.8
# command to run tests
install: ./install_redis.sh
script: make test DJANGO_VERSION=$DJANGO_VERSION
branches:
only:
......
Makefile
View file @ 9ef12f52
SHELL := /bin/bash
PACKAGE_NAME=redis_cache
VENV_DIR?=.venv
VENV_ACTIVATE=$(VENV_DIR)/bin/activate
WITH_VENV=. $(VENV_ACTIVATE);
PACKAGE_NAME=redis_cache
DJANGO_VERSION?=1.7
default:
python setup.py check build
$(VENV_ACTIVATE): requirements*.txt
test -f $@ || virtualenv --system-site-packages $(VENV_DIR)
touch $@
.PHONY: install_requirements
install_requirements: requirements*.txt
$(WITH_VENV) pip install --no-deps -r requirements.txt
$(WITH_VENV) pip install --no-deps -r requirements-dev.txt
$(WITH_VENV) $(test -f requirements-local.txt && pip install -r requirements-local.txt)
$(WITH_VENV) pip install Django==$(DJANGO_VERSION)
.PHONY: venv
venv: $(VENV_ACTIVATE)
.PHONY: setup
setup: venv
.PHONY: redis_servers
redis_servers:
test -d redis || git clone https://github.com/antirez/redis
git -C redis checkout 2.6
make -C redis
for i in 1 2 3; do \
./redis/src/redis-server \
--pidfile /tmp/redis`echo $$i`.pid \
--requirepass yadayada \
--daemonize yes \
--port `echo 638$$i` ; \
done
for i in 4 5 6; do \
./redis/src/redis-server \
--pidfile /tmp/redis`echo $$i`.pid \
--requirepass yadayada \
--daemonize yes \
--port 0 \
--unixsocket /tmp/redis`echo $$i`.sock \
--unixsocketperm 755 ; \
done
./redis/src/redis-server \
--pidfile /tmp/redis7.pid \
--requirepass yadayada \
--daemonize yes \
--port 6387 ;
for i in 8 9; do \
./redis/src/redis-server \
--pidfile /tmp/redis`echo $$i`.pid \
--requirepass yadayada \
--daemonize yes \
--masterauth yadayada \
--slaveof 127.0.0.1 6387 \
--port `echo 638$$i` ; \
done
pip install -r requirements.txt
pip install -r requirements-dev.txt
pip install Django==$(DJANGO_VERSION)
.PHONY: clean
clean:
......@@ -74,18 +18,11 @@ clean:
rm -rf __pycache__/
rm -f MANIFEST
rm -f test.db
find $(PACKAGE_NAME) -type f -name '*.pyc' -delete
.PHONY: teardown
teardown:
rm -rf $(VENV_DIR)/
.PHONY: test
test: venv install_requirements redis_servers
$(WITH_VENV) PYTHONPATH=$(PYTHONPATH): django-admin.py test --settings=tests.settings -s
for i in 1 2 3 4 5 6 7 8 9; do kill `cat /tmp/redis$$i.pid`; done;
test: install_requirements
PYTHONPATH=$(PYTHONPATH): django-admin.py test --settings=tests.settings -s
.PHONY: shell
shell: venv
$(WITH_VENV) PYTHONPATH=$(PYTHONPATH): django-admin.py shell --settings=tests.settings
shell:
PYTHONPATH=$(PYTHONPATH): django-admin.py shell --settings=tests.settings
install_redis.sh 0 → 100755
View file @ 9ef12f52
#!/bin/bash
: ${REDIS_VERSION:="2.6"}
test -d redis || git clone https://github.com/antirez/redis
git -C redis checkout $REDIS_VERSION
make -C redis
redis_cache/backends/base.py
View file @ 9ef12f52
from django.core.cache.backends.base import BaseCache, InvalidCacheBackendError
from django.core.exceptions import ImproperlyConfigured
from django.utils import importlib
from django.utils.functional import cached_property
from django.utils.importlib import import_module
from redis_cache.compat import bytes_type, smart_bytes, DEFAULT_TIMEOUT
from redis_cache.compat import smart_bytes, DEFAULT_TIMEOUT
try:
import cPickle as pickle
......@@ -14,12 +13,14 @@ except ImportError:
try:
import redis
except ImportError:
raise InvalidCacheBackendError("Redis cache backend requires the 'redis-py' library")
raise InvalidCacheBackendError(
"Redis cache backend requires the 'redis-py' library"
)
from redis.connection import DefaultParser
from redis_cache.connection import pool
from redis_cache.utils import CacheKey
from redis_cache.utils import CacheKey, get_servers, parse_connection_kwargs
from functools import wraps
......@@ -34,7 +35,6 @@ def get_client(write=False):
version = kwargs.pop('version', None)
client = self.get_client(key, write=write)
key = self.make_key(key, version=version)
return method(self, client, key, *args, **kwargs)
return wrapped
......@@ -50,7 +50,7 @@ class BaseRedisCache(BaseCache):
"""
super(BaseRedisCache, self).__init__(params)
self.server = server
self.servers = self.get_servers(server)
self.servers = get_servers(server)
self.params = params or {}
self.options = params.get('OPTIONS', {})
self.clients = {}
......@@ -71,20 +71,6 @@ class BaseRedisCache(BaseCache):
def __setstate__(self, state):
self.__init__(**state)
def get_servers(self, server):
"""returns a list of servers given the server argument passed in
from Django.
"""
if isinstance(server, bytes_type):
servers = server.split(',')
elif hasattr(server, '__iter__'):
servers = server
else:
raise ImproperlyConfigured(
'"server" must be an iterable or string'
)
return servers
def get_db(self):
_db = self.params.get('db', self.options.get('DB', 1))
try:
......@@ -136,49 +122,29 @@ class BaseRedisCache(BaseCache):
Get the write server:port of the master cache
"""
cache = self.options.get('MASTER_CACHE', None)
return self.client_list[0] if cache is None else self.create_client(cache)
if cache is None:
return next(iter(self.client_list))
def create_client(self, server):
kwargs = {
'db': self.db,
'password': self.password,
}
if '://' in server:
client = redis.Redis.from_url(
server,
parser_class=self.parser_class,
**kwargs
)
unix_socket_path = client.connection_pool.connection_kwargs.get('path')
kwargs.update(
client.connection_pool.connection_kwargs,
unix_socket_path=unix_socket_path,
)
else:
unix_socket_path = None
if ':' in server:
host, port = server.rsplit(':', 1)
try:
port = int(port)
except (ValueError, TypeError):
raise ImproperlyConfigured("Port value must be an integer")
else:
host, port = None, None
unix_socket_path = server
kwargs.update(
host=host,
port=port,
unix_socket_path=unix_socket_path,
)
client = redis.Redis(**kwargs)
kwargs = parse_connection_kwargs(cache, db=self.db)
return self.clients[(
kwargs['host'],
kwargs['port'],
kwargs['db'],
kwargs['unix_socket_path'],
)]
def create_client(self, server):
kwargs = parse_connection_kwargs(
server,
db=self.db,
password=self.password,
)
client = redis.Redis(**kwargs)
kwargs.update(
parser_class=self.parser_class,
connection_pool_class=self.connection_pool_class,
connection_pool_class_kwargs=self.connection_pool_class_kwargs,
)
connection_pool = pool.get_connection_pool(client, **kwargs)
client.connection_pool = connection_pool
return client
......
redis_cache/backends/multiple.py
View file @ 9ef12f52
from collections import defaultdict
from django.core.exceptions import ImproperlyConfigured
from redis_cache.backends.base import BaseRedisCache
from redis_cache.compat import DEFAULT_TIMEOUT
from redis_cache.sharder import HashRing
......@@ -20,9 +17,8 @@ class ShardedRedisCache(BaseRedisCache):
self.client_list = self.clients.values()
def get_client(self, key, write=False):
node = self.sharder.get_node(unicode(key))
node = self.sharder.get_node(key)
return self.clients[node]
def shard(self, keys, write=False, version=None):
......@@ -31,7 +27,9 @@ class ShardedRedisCache(BaseRedisCache):
"""
clients = defaultdict(list)
for key in keys:
clients[self.get_client(key, write)].append(self.make_key(key, version))
clients[self.get_client(key, write)].append(
self.make_key(key, version)
)
return clients
####################
......@@ -54,7 +52,7 @@ class ShardedRedisCache(BaseRedisCache):
namespace will be deleted. Otherwise, all keys will be deleted.
"""
if version is None:
for client in self.clients.itervalues():
for client in self.clients.values():
self._clear(client)
else:
self.delete_pattern('*', version=version)
......@@ -64,7 +62,13 @@ class ShardedRedisCache(BaseRedisCache):
clients = self.shard(keys, version=version)
for client, versioned_keys in clients.items():
original_keys = [key._original_key for key in versioned_keys]
data.update(self._get_many(client, original_keys, versioned_keys=versioned_keys))
data.update(
self._get_many(
client,
original_keys,
versioned_keys=versioned_keys
)
)
return data
def set_many(self, data, timeout=None, version=None):
......@@ -113,12 +117,12 @@ class ShardedRedisCache(BaseRedisCache):
def delete_pattern(self, pattern, version=None):
pattern = self.make_key(pattern, version=version)
for client in self.clients.itervalues():
for client in self.clients.values():
self._delete_pattern(client, pattern)
def reinsert_keys(self):
"""
Reinsert cache entries using the current pickle protocol version.
"""
for client in self.clients.itervalues():
for client in self.clients.values():
self._reinsert_keys(client)
redis_cache/backends/single.py
View file @ 9ef12f52
......@@ -5,7 +5,6 @@ except ImportError:
import random
from redis_cache.backends.base import BaseRedisCache
from redis_cache.compat import bytes_type, DEFAULT_TIMEOUT
class RedisCache(BaseRedisCache):
......@@ -26,7 +25,7 @@ class RedisCache(BaseRedisCache):
def get_client(self, key, write=False):
if write and self.master_client is not None:
return self.master_client
return random.choice(self.client_list)
return random.choice(list(self.client_list))
####################
# Django cache api #
......
redis_cache/compat.py
View file @ 9ef12f52
import sys
import django
PY3 = (sys.version_info >= (3,))
try:
......@@ -15,8 +16,11 @@ except ImportError:
if PY3:
bytes_type = bytes
from urllib.parse import parse_qs, urlparse
else:
bytes_type = str
from urlparse import parse_qs, urlparse
if django.VERSION[:2] >= (1, 6):
from django.core.cache.backends.base import DEFAULT_TIMEOUT as DJANGO_DEFAULT_TIMEOUT
......
redis_cache/connection.py
View file @ 9ef12f52
......@@ -14,7 +14,7 @@ class CacheConnectionPool(object):
return self._clients.get(server, None)
def reset(self):
for pool in self._connection_pools.itervalues():
for pool in self._connection_pools.values():
pool.disconnect()
self._clients = {}
self._connection_pools = {}
......
redis_cache/sharder.py
View file @ 9ef12f52
from bisect import insort, bisect
from hashlib import md5
from math import log
import sys
import hashlib
from redis_cache.compat import smart_text
try:
maxint = sys.maxint
except AttributeError:
maxint = sys.maxsize
DIGITS = int(log(maxint) / log(16))
DIGITS = 8
def make_hash(s):
return int(md5(s.encode('utf-8')).hexdigest()[:DIGITS], 16)
def get_slot(s):
_hash = hashlib.md5(s.encode('utf-8')).hexdigest()
return int(_hash[-DIGITS:], 16)
class Node(object):
def __init__(self, node, i):
self._node = node
self._i = i
self._position = make_hash("%d:%s" % (i, str(self._node)))
key = "{0}:{1}".format(
smart_text(i),
smart_text(self._node),
)
self._position = get_slot(key)
def __cmp__(self, other):
def __gt__(self, other):
if isinstance(other, int):
return cmp(self._position, other)
return self._position > other
elif isinstance(other, Node):
return cmp(self._position, other._position)
raise TypeError('Cannot compare this class with "%s" type' % type(other))
def __eq__(self, other):
return self._node == other._node
return self._position > other._position
raise TypeError(
'Cannot compare this class with "%s" type' % type(other)
)
class HashRing(object):
......@@ -42,7 +42,7 @@ class HashRing(object):
insort(self._nodes, Node(node, i))
def add(self, node, weight=1):
for i in xrange(weight * self.replicas):
for i in range(weight * self.replicas):
self._add(node, i)
def remove(self, node):
......@@ -52,5 +52,5 @@ class HashRing(object):
del self._nodes[n - i - 1]
def get_node(self, key):
i = bisect(self._nodes, make_hash(key)) - 1
i = bisect(self._nodes, get_slot(smart_text(key))) - 1
return self._nodes[i]._node
redis_cache/utils.py
View file @ 9ef12f52
from redis_cache.compat import smart_text, python_2_unicode_compatible
import warnings
from django.core.exceptions import ImproperlyConfigured
from redis.connection import SSLConnection
from redis_cache.compat import (
smart_text, python_2_unicode_compatible, parse_qs, urlparse
)
try:
basestring
except NameError:
basestring = str
@python_2_unicode_compatible
......@@ -16,4 +28,141 @@ class CacheKey(object):
def __unicode__(self):
return smart_text(self._versioned_key)
def __hash__(self):
return hash(self._versioned_key)
__repr__ = __str__ = __unicode__
def get_servers(location):
"""Returns a list of servers given the server argument passed in from
Django.
"""
if isinstance(location, basestring):
servers = location.split(',')
elif hasattr(location, '__iter__'):
servers = location
else:
raise ImproperlyConfigured(
'"server" must be an iterable or string'
)
return servers
def parse_connection_kwargs(server, db=None, **kwargs):
"""
Return a connection pool configured from the given URL.
For example::
redis://[:password]@localhost:6379/0
rediss://[:password]@localhost:6379/0
unix://[:password]@/path/to/socket.sock?db=0
Three URL schemes are supported:
redis:// creates a normal TCP socket connection
rediss:// creates a SSL wrapped TCP socket connection
unix:// creates a Unix Domain Socket connection
There are several ways to specify a database number. The parse function
will return the first specified option:
1. A ``db`` querystring option, e.g. redis://localhost?db=0
2. If using the redis:// scheme, the path argument of the url, e.g.
redis://localhost/0
3. The ``db`` argument to this function.
If none of these options are specified, db=0 is used.
Any additional querystring arguments and keyword arguments will be
passed along to the ConnectionPool class's initializer. In the case
of conflicting arguments, querystring arguments always win.
NOTE: taken from `redis.ConnectionPool.from_url` in redis-py
"""
if '://' in server:
url = server
url_string = url
url = urlparse(url)
qs = ''
# in python2.6, custom URL schemes don't recognize querystring values
# they're left as part of the url.path.
if '?' in url.path and not url.query:
# chop the querystring including the ? off the end of the url
# and reparse it.
qs = url.path.split('?', 1)[1]
url = urlparse(url_string[:-(len(qs) + 1)])
else:
qs = url.query
url_options = {}
for name, value in parse_qs(qs).items():
if value and len(value) > 0:
url_options[name] = value[0]
# We only support redis:// and unix:// schemes.
if url.scheme == 'unix':
url_options.update({
'password': url.password,
'unix_socket_path': url.path,
})
else:
url_options.update({
'host': url.hostname,
'port': int(url.port or 6379),
'password': url.password,
})
# If there's a path argument, use it as the db argument if a
# querystring value wasn't specified
if 'db' not in url_options and url.path:
try:
url_options['db'] = int(url.path.replace('/', ''))
except (AttributeError, ValueError):
pass
if url.scheme == 'rediss':
url_options['connection_class'] = SSLConnection
# last shot at the db value
url_options['db'] = int(url_options.get('db', db or 0))
# update the arguments from the URL values
kwargs.update(url_options)
# backwards compatability
if 'charset' in kwargs:
warnings.warn(DeprecationWarning(
'"charset" is deprecated. Use "encoding" instead'))
kwargs['encoding'] = kwargs.pop('charset')
if 'errors' in kwargs:
warnings.warn(DeprecationWarning(
'"errors" is deprecated. Use "encoding_errors" instead'))
kwargs['encoding_errors'] = kwargs.pop('errors')
else:
unix_socket_path = None
if ':' in server:
host, port = server.rsplit(':', 1)
try:
port = int(port)
except (ValueError, TypeError):
raise ImproperlyConfigured(
"{0} from {1} must be an integer".format(
repr(port),
server
)
)
else:
host, port = None, None
unix_socket_path = server
kwargs.update(
host=host,
port=port,
unix_socket_path=unix_socket_path,
db=db,
)
return kwargs
requirements-dev.txt
View file @ 9ef12f52
hiredis==0.2.0
django-nose==1.4
nose==1.3.6
unittest2==1.0.1
setup.py
View file @ 9ef12f52
......@@ -5,7 +5,7 @@ setup(
url="http://github.com/sebleier/django-redis-cache/",
author="Sean Bleier",
author_email="sebleier@gmail.com",
version="1.1.1",
version="1.2.0",
packages=["redis_cache", "redis_cache.backends"],
description="Redis Cache Backend for Django",
install_requires=['redis>=2.4.5'],
......
tests/settings.py
View file @ 9ef12f52
......@@ -32,3 +32,4 @@ CACHES = {
},
}
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
MIDDLEWARE_CLASSES = tuple()
tests/testapp/tests/base_tests.py
View file @ 9ef12f52
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from hashlib import sha1
import os
import subprocess
import sys
import time
import unittest
if sys.version_info < (2, 7):
import unittest2 as unittest
else:
import unittest
try:
import cPickle as pickle
except ImportError:
import pickle
from django.core.cache import get_cache
from django.core.exceptions import ImproperlyConfigured
from django.test import TestCase
......@@ -19,7 +30,11 @@ import redis
from tests.testapp.models import Poll, expensive_calculation
from redis_cache.cache import RedisCache, pool
from redis_cache.compat import DEFAULT_TIMEOUT
from redis_cache.compat import DEFAULT_TIMEOUT, smart_bytes
from redis_cache.utils import get_servers, parse_connection_kwargs
REDIS_PASSWORD = 'yadayada'
LOCATION = "127.0.0.1:6381"
......@@ -35,15 +50,91 @@ class C:
return 24
def start_redis_servers(servers, db=None, master=None):
"""Creates redis instances using specified locations from the settings.
Returns list of Popen objects
"""
processes = []
devnull = open(os.devnull, 'w')
master_connection_kwargs = master and parse_connection_kwargs(
master,
db=db,
password=REDIS_PASSWORD
)
for i, server in enumerate(servers):
connection_kwargs = parse_connection_kwargs(
server,
db=db,
password=REDIS_PASSWORD, # will be overridden if specified in `server`
)
parameters = dict(
port=connection_kwargs.get('port', 0),
requirepass=connection_kwargs['password'],
)
is_socket = server.startswith('unix://') or server.startswith('/')
if is_socket:
parameters.update(
port=0,
unixsocket='/tmp/redis{0}.sock'.format(i),
unixsocketperm=755,
)
if master and not connection_kwargs == master_connection_kwargs:
parameters.update(
masterauth=master_connection_kwargs['password'],
slaveof="{host} {port}".format(
host=master_connection_kwargs['host'],
port=master_connection_kwargs['port'],
)
)
args = ['./redis/src/redis-server'] + [
"--{parameter} {value}".format(parameter=parameter, value=value)
for parameter, value in parameters.items()
]
p = subprocess.Popen(args, stdout=devnull)
processes.append(p)
return processes
class SetupMixin(object):
processes = None
@classmethod
def tearDownClass(cls):
for p in cls.processes:
p.kill()
cls.processes = None
# Give redis processes some time to shutdown
# time.sleep(.1)
def setUp(self):
# use DB 16 for testing and hope there isn't any important data :->
if self.__class__.processes is None:
from django.conf import settings
cache_settings = settings.CACHES['default']
servers = get_servers(cache_settings['LOCATION'])
options = cache_settings.get('OPTIONS', {})
db = options.get('db', 0)
master = options.get('MASTER_CACHE')
self.__class__.processes = start_redis_servers(
servers,
db=db,
master=master
)
# Give redis processes some time to startup
time.sleep(.1)
self.reset_pool()
self.cache = self.get_cache()
def tearDown(self):
# Sometimes it will be necessary to skip this method because we need to test default
# initialization and that may be using a different port than the test redis server.
# Sometimes it will be necessary to skip this method because we need to
# test default initialization and that may be using a different port
# than the test redis server.
if hasattr(self, '_skip_tearDown') and self._skip_tearDown:
self._skip_tearDown = False
return
......@@ -226,10 +317,10 @@ class BaseRedisTestCase(SetupMixin):
def test_unicode(self):
# Unicode values can be cached
stuff = {
u'ascii': u'ascii_value',
u'unicode_ascii': u'Iñtërnâtiônàlizætiøn1',
u'Iñtërnâtiônàlizætiøn': u'Iñtërnâtiônàlizætiøn2',
u'ascii': {u'x': 1}
'ascii': 'ascii_value',
'unicode_ascii': 'Iñtërnâtiônàlizætiøn1',
'Iñtërnâtiônàlizætiøn': 'Iñtërnâtiônàlizætiøn2',
'ascii': {'x': 1}
}
for (key, value) in stuff.items():
self.cache.set(key, value)
......@@ -373,7 +464,7 @@ class BaseRedisTestCase(SetupMixin):
def test_reinsert_keys(self):
self.cache._pickle_version = 0
for i in range(2000):
s = sha1(str(i)).hexdigest()
s = sha1(smart_bytes(i)).hexdigest()
self.cache.set(s, self.cache)
self.cache._pickle_version = -1
self.cache.reinsert_keys()
......@@ -414,7 +505,7 @@ class BaseRedisTestCase(SetupMixin):
self.assertEqual(value, 42)
def assertMaxConnection(self, cache, max_num):
for client in cache.clients.itervalues():
for client in cache.clients.values():
self.assertLessEqual(client.connection_pool._created_connections, max_num)
def test_max_connections(self):
......@@ -425,7 +516,7 @@ class BaseRedisTestCase(SetupMixin):
pass
releases = {}
for client in cache.clients.itervalues():
for client in cache.clients.values():
releases[client.connection_pool] = client.connection_pool.release
client.connection_pool.release = noop
self.assertEqual(client.connection_pool.max_connections, 2)
......@@ -441,7 +532,7 @@ class BaseRedisTestCase(SetupMixin):
self.assertMaxConnection(cache, 2)
for client in cache.clients.itervalues():
for client in cache.clients.values():
client.connection_pool.release = releases[client.connection_pool]
client.connection_pool.max_connections = 2 ** 31
......
tests/testapp/tests/master_slave_tests.py
View file @ 9ef12f52
......@@ -11,11 +11,11 @@ from redis_cache.connection import pool
from tests.testapp.tests.base_tests import SetupMixin
MASTER_LOCATION = "127.0.0.1:6387"
MASTER_LOCATION = "127.0.0.2:6387"
LOCATIONS = [
'127.0.0.1:6387',
'127.0.0.1:6388',
'127.0.0.1:6389',
'127.0.0.2:6387',
'127.0.0.2:6388',
'127.0.0.2:6389',
]
......@@ -43,22 +43,22 @@ class MasterSlaveTestCase(SetupMixin, TestCase):
client = cache.master_client
self.assertEqual(
client.connection_pool.connection_identifier,
('127.0.0.1', 6387, 1, None)
('127.0.0.2', 6387, 1, None)
)
self.assertEqual(len(pool._connection_pools), 3)
def test_set(self):
cache = self.get_cache()
cache.set('a', 'a')
time.sleep(.5)
for client in self.cache.clients.itervalues():
time.sleep(.2)
for client in self.cache.clients.values():
key = cache.make_key('a')
self.assertIsNotNone(client.get(key))
def test_set_many(self):
cache = self.get_cache()
cache.set_many({'a': 'a', 'b': 'b'})
for client in self.cache.clients.itervalues():
for client in self.cache.clients.values():
self.assertNotIn(None, client.mget([
cache.make_key('a'),
cache.make_key('b'),
......@@ -68,24 +68,28 @@ class MasterSlaveTestCase(SetupMixin, TestCase):
cache = self.get_cache()
cache.set('a', 0)
cache.incr('a')
time.sleep(.5)
time.sleep(.2)
key = cache.make_key('a')
for client in self.cache.clients.itervalues():
self.assertEqual(client.get(key), '1')
for client in self.cache.clients.values():
self.assertEqual(int(client.get(key)), 1)
def test_delete(self):
cache = self.get_cache()
cache.set('a', 'a')
time.sleep(.2)
self.assertEqual(cache.get('a'), 'a')
cache.delete('a')
time.sleep(.2)
key = cache.make_key('a')
for client in self.cache.clients.itervalues():
for client in self.cache.clients.values():
self.assertIsNone(client.get(key))
def test_clear(self):
cache = self.get_cache()
cache.set('a', 'a')
time.sleep(.2)
self.assertEqual(cache.get('a'), 'a')
cache.clear()
for client in self.cache.clients.itervalues():
time.sleep(.2)
for client in self.cache.clients.values():
self.assertEqual(len(client.keys('*')), 0)
tests/testapp/tests/multi_server_tests.py
View file @ 9ef12f52
from collections import Counter
from math import sqrt
from redis_cache.sharder import HashRing
def mean(lst):
......@@ -14,15 +16,48 @@ def stddev(lst):
class MultiServerTests(object):
def test_distribution(self):
nodes = [node._position for node in self.cache.sharder._nodes]
nodes.sort()
diffs = [(b - a) for a, b in zip(nodes[:-1], nodes[1:])]
l = 16 ** 8
perfect_dist = l / len(nodes)
random_dist = sum(diffs) / len(diffs)
_max = max([perfect_dist, random_dist])
_min = min([perfect_dist, random_dist])
percentage = (1 - _max / _min) * 100
# Assert they are less than 2 percent of each other
self.assertLess(percentage, 2.0)
def test_make_key_distribution(self):
ring = HashRing()
nodes = set([str(node._node) for node in self.cache.sharder._nodes])
nodes = [
('127.0.0.1', 6379, 15, '/tmp/redis0.sock'),
('127.0.0.1', 6379, 15, '/tmp/redis1.sock'),
('127.0.0.1', 6379, 15, '/tmp/redis2.sock'),
]
for node in nodes:
ring.add(str(node))
n = 50000
counter = Counter(
[ring.get_node(str(i)) for i in range(n)]
)
self.assertLess(
((stddev(counter.values()) / n) * 100.0), 10, counter.values()
)
def test_key_distribution(self):
n = 10000
self.cache.set('a', 'a')
for i in xrange(n):
for i in range(n):
self.cache.set(i, i)
keys = [
len(client.keys('*'))
for client in self.cache.clients.itervalues()
for client in self.cache.clients.values()
]
self.assertEqual(sum(keys), n)
self.assertLess(((stddev(keys) / n) * 100.0), 10)
def test_removing_nodes(self):
......
tests/testapp/tests/socket_tests.py
View file @ 9ef12f52
# # -*- coding: utf-8 -*-
from collections import Counter
from tests.testapp.tests.base_tests import BaseRedisTestCase
from tests.testapp.tests.multi_server_tests import MultiServerTests
......@@ -8,15 +9,12 @@ except ImportError:
from django.test.utils import override_settings
from django.test import TestCase
from redis_cache.cache import ImproperlyConfigured
from redis.connection import UnixDomainSocketConnection
LOCATION = "unix://:yadayada@/tmp/redis4.sock?db=15"
LOCATION = "unix://:yadayada@/tmp/redis0.sock?db=15"
LOCATIONS = [
"unix://:yadayada@/tmp/redis4.sock?db=15",
"unix://:yadayada@/tmp/redis5.sock?db=15",
"unix://:yadayada@/tmp/redis6.sock?db=15",
"unix://:yadayada@/tmp/redis0.sock?db=15",
"unix://:yadayada@/tmp/redis1.sock?db=15",
"unix://:yadayada@/tmp/redis2.sock?db=15",
]
......@@ -87,7 +85,16 @@ class SinglePythonParserTestCase(SocketTestCase):
}
)
class MultipleHiredisTestCase(MultiServerTests, SocketTestCase):
pass
def test_equal_number_of_nodes(self):
counter = Counter(
[node._node[3] for node in self.cache.sharder._nodes]
)
self.assertEqual(counter, {
'/tmp/redis0.sock': 16,
'/tmp/redis1.sock': 16,
'/tmp/redis2.sock': 16,
})
@override_settings(
......
tests/testapp/tests/tcp_tests.py
View file @ 9ef12f52
......@@ -24,6 +24,10 @@ class TCPTestCase(BaseRedisTestCase, TestCase):
def test_default_initialization(self):
self.reset_pool()
self.cache = self.get_cache()
self.assertIn(
('127.0.0.1', 6381, 15, None),
self.cache.clients,
)
client = self.cache.clients[('127.0.0.1', 6381, 15, None)]
connection_class = client.connection_pool.connection_class
if connection_class is not UnixDomainSocketConnection:
......@@ -119,5 +123,3 @@ class MultipleHiredisTestCase(MultiServerTests, TCPTestCase):
)
class MultiplePythonParserTestCase(MultiServerTests, TCPTestCase):
pass
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment