Commit ef82a59a authored by Sean Bleier's avatar Sean Bleier

Fixing up some caching behavior.

parent 9b2dfcef
from collections import defaultdict from collections import defaultdict
from django.core.exceptions import ImproperlyConfigured
from redis_cache.backends.base import BaseRedisCache from redis_cache.backends.base import BaseRedisCache
from redis_cache.compat import DEFAULT_TIMEOUT, smart_text
from redis_cache.sharder import HashRing from redis_cache.sharder import HashRing
...@@ -21,7 +18,7 @@ class ShardedRedisCache(BaseRedisCache): ...@@ -21,7 +18,7 @@ class ShardedRedisCache(BaseRedisCache):
self.client_list = self.clients.values() self.client_list = self.clients.values()
def get_client(self, key, write=False): def get_client(self, key, write=False):
node = self.sharder.get_node(smart_text(key)) node = self.sharder.get_node(key)
return self.clients[node] return self.clients[node]
def shard(self, keys, write=False, version=None): def shard(self, keys, write=False, version=None):
......
from bisect import insort, bisect from bisect import insort, bisect
from hashlib import md5 import hashlib
from math import log from redis_cache.compat import smart_text
import sys
try:
maxint = sys.maxint
except AttributeError:
maxint = sys.maxsize
DIGITS = int(log(maxint) / log(16)) DIGITS = 8
def make_hash(s): def get_slot(s):
return int(md5(s.encode('utf-8')).hexdigest()[:DIGITS], 16) _hash = hashlib.md5(s.encode('utf-8')).hexdigest()
return int(_hash[-DIGITS:], 16)
class Node(object): class Node(object):
def __init__(self, node, i): def __init__(self, node, i):
self._node = node self._node = node
self._i = i self._i = i
self._position = make_hash("{0}:{1}".format(i, self._node)) key = "{0}:{1}".format(
smart_text(i),
smart_text(self._node),
)
self._position = get_slot(key)
def __gt__(self, other): def __gt__(self, other):
if isinstance(other, int): if isinstance(other, int):
...@@ -51,5 +52,5 @@ class HashRing(object): ...@@ -51,5 +52,5 @@ class HashRing(object):
del self._nodes[n - i - 1] del self._nodes[n - i - 1]
def get_node(self, key): def get_node(self, key):
i = bisect(self._nodes, make_hash(key)) - 1 i = bisect(self._nodes, get_slot(smart_text(key))) - 1
return self._nodes[i]._node return self._nodes[i]._node
from collections import Counter
from math import sqrt from math import sqrt
from redis_cache.sharder import HashRing
def mean(lst): def mean(lst):
...@@ -14,15 +16,48 @@ def stddev(lst): ...@@ -14,15 +16,48 @@ def stddev(lst):
class MultiServerTests(object): class MultiServerTests(object):
def test_distribution(self):
nodes = [node._position for node in self.cache.sharder._nodes]
nodes.sort()
diffs = [(b - a) for a, b in zip(nodes[:-1], nodes[1:])]
l = 16 ** 8
perfect_dist = l / len(nodes)
random_dist = sum(diffs) / len(diffs)
_max = max([perfect_dist, random_dist])
_min = min([perfect_dist, random_dist])
percentage = (1 - _max / _min) * 100
# Assert they are less than 2 percent of each other
self.assertLess(percentage, 2.0)
def test_make_key_distribution(self):
ring = HashRing()
nodes = set([str(node._node) for node in self.cache.sharder._nodes])
nodes = [
('127.0.0.1', 6379, 15, '/tmp/redis0.sock'),
('127.0.0.1', 6379, 15, '/tmp/redis1.sock'),
('127.0.0.1', 6379, 15, '/tmp/redis2.sock'),
]
for node in nodes:
ring.add(str(node))
n = 50000
counter = Counter(
[ring.get_node(str(i)) for i in range(n)]
)
self.assertLess(
((stddev(counter.values()) / n) * 100.0), 10, counter.values()
)
def test_key_distribution(self): def test_key_distribution(self):
n = 10000 n = 10000
self.cache.set('a', 'a')
for i in range(n): for i in range(n):
self.cache.set(i, i) self.cache.set(i, i)
keys = [ keys = [
len(client.keys('*')) len(client.keys('*'))
for client in self.cache.clients.values() for client in self.cache.clients.values()
] ]
self.assertEqual(sum(keys), n)
self.assertLess(((stddev(keys) / n) * 100.0), 10) self.assertLess(((stddev(keys) / n) * 100.0), 10)
def test_removing_nodes(self): def test_removing_nodes(self):
......
# # -*- coding: utf-8 -*- # # -*- coding: utf-8 -*-
from collections import Counter
from tests.testapp.tests.base_tests import BaseRedisTestCase from tests.testapp.tests.base_tests import BaseRedisTestCase
from tests.testapp.tests.multi_server_tests import MultiServerTests from tests.testapp.tests.multi_server_tests import MultiServerTests
...@@ -8,9 +9,6 @@ except ImportError: ...@@ -8,9 +9,6 @@ except ImportError:
from django.test.utils import override_settings from django.test.utils import override_settings
from django.test import TestCase from django.test import TestCase
from redis_cache.cache import ImproperlyConfigured
from redis.connection import UnixDomainSocketConnection
LOCATION = "unix://:yadayada@/tmp/redis0.sock?db=15" LOCATION = "unix://:yadayada@/tmp/redis0.sock?db=15"
LOCATIONS = [ LOCATIONS = [
...@@ -87,7 +85,16 @@ class SinglePythonParserTestCase(SocketTestCase): ...@@ -87,7 +85,16 @@ class SinglePythonParserTestCase(SocketTestCase):
} }
) )
class MultipleHiredisTestCase(MultiServerTests, SocketTestCase): class MultipleHiredisTestCase(MultiServerTests, SocketTestCase):
pass
def test_equal_number_of_nodes(self):
counter = Counter(
[node._node[3] for node in self.cache.sharder._nodes]
)
self.assertEqual(counter, {
'/tmp/redis0.sock': 16,
'/tmp/redis1.sock': 16,
'/tmp/redis2.sock': 16,
})
@override_settings( @override_settings(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment