Commit ef82a59a authored by Sean Bleier's avatar Sean Bleier

Fixing up some caching behavior.

parent 9b2dfcef
from collections import defaultdict
from django.core.exceptions import ImproperlyConfigured
from redis_cache.backends.base import BaseRedisCache
from redis_cache.compat import DEFAULT_TIMEOUT, smart_text
from redis_cache.sharder import HashRing
......@@ -21,7 +18,7 @@ class ShardedRedisCache(BaseRedisCache):
self.client_list = self.clients.values()
def get_client(self, key, write=False):
node = self.sharder.get_node(smart_text(key))
node = self.sharder.get_node(key)
return self.clients[node]
def shard(self, keys, write=False, version=None):
......
from bisect import insort, bisect
from hashlib import md5
from math import log
import sys
import hashlib
from redis_cache.compat import smart_text
try:
maxint = sys.maxint
except AttributeError:
maxint = sys.maxsize
DIGITS = int(log(maxint) / log(16))
DIGITS = 8
def make_hash(s):
return int(md5(s.encode('utf-8')).hexdigest()[:DIGITS], 16)
def get_slot(s):
_hash = hashlib.md5(s.encode('utf-8')).hexdigest()
return int(_hash[-DIGITS:], 16)
class Node(object):
def __init__(self, node, i):
self._node = node
self._i = i
self._position = make_hash("{0}:{1}".format(i, self._node))
key = "{0}:{1}".format(
smart_text(i),
smart_text(self._node),
)
self._position = get_slot(key)
def __gt__(self, other):
if isinstance(other, int):
......@@ -51,5 +52,5 @@ class HashRing(object):
del self._nodes[n - i - 1]
def get_node(self, key):
i = bisect(self._nodes, make_hash(key)) - 1
i = bisect(self._nodes, get_slot(smart_text(key))) - 1
return self._nodes[i]._node
from collections import Counter
from math import sqrt
from redis_cache.sharder import HashRing
def mean(lst):
......@@ -14,15 +16,48 @@ def stddev(lst):
class MultiServerTests(object):
def test_distribution(self):
nodes = [node._position for node in self.cache.sharder._nodes]
nodes.sort()
diffs = [(b - a) for a, b in zip(nodes[:-1], nodes[1:])]
l = 16 ** 8
perfect_dist = l / len(nodes)
random_dist = sum(diffs) / len(diffs)
_max = max([perfect_dist, random_dist])
_min = min([perfect_dist, random_dist])
percentage = (1 - _max / _min) * 100
# Assert they are less than 2 percent of each other
self.assertLess(percentage, 2.0)
def test_make_key_distribution(self):
ring = HashRing()
nodes = set([str(node._node) for node in self.cache.sharder._nodes])
nodes = [
('127.0.0.1', 6379, 15, '/tmp/redis0.sock'),
('127.0.0.1', 6379, 15, '/tmp/redis1.sock'),
('127.0.0.1', 6379, 15, '/tmp/redis2.sock'),
]
for node in nodes:
ring.add(str(node))
n = 50000
counter = Counter(
[ring.get_node(str(i)) for i in range(n)]
)
self.assertLess(
((stddev(counter.values()) / n) * 100.0), 10, counter.values()
)
def test_key_distribution(self):
n = 10000
self.cache.set('a', 'a')
for i in range(n):
self.cache.set(i, i)
keys = [
len(client.keys('*'))
for client in self.cache.clients.values()
]
self.assertEqual(sum(keys), n)
self.assertLess(((stddev(keys) / n) * 100.0), 10)
def test_removing_nodes(self):
......
# # -*- coding: utf-8 -*-
from collections import Counter
from tests.testapp.tests.base_tests import BaseRedisTestCase
from tests.testapp.tests.multi_server_tests import MultiServerTests
......@@ -8,9 +9,6 @@ except ImportError:
from django.test.utils import override_settings
from django.test import TestCase
from redis_cache.cache import ImproperlyConfigured
from redis.connection import UnixDomainSocketConnection
LOCATION = "unix://:yadayada@/tmp/redis0.sock?db=15"
LOCATIONS = [
......@@ -87,7 +85,16 @@ class SinglePythonParserTestCase(SocketTestCase):
}
)
class MultipleHiredisTestCase(MultiServerTests, SocketTestCase):
pass
def test_equal_number_of_nodes(self):
counter = Counter(
[node._node[3] for node in self.cache.sharder._nodes]
)
self.assertEqual(counter, {
'/tmp/redis0.sock': 16,
'/tmp/redis1.sock': 16,
'/tmp/redis2.sock': 16,
})
@override_settings(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment