Commit 2c2ece78 authored by Sean Bleier's avatar Sean Bleier

Some python 3 fixes.

parent 32daa42e
...@@ -123,7 +123,7 @@ class BaseRedisCache(BaseCache): ...@@ -123,7 +123,7 @@ class BaseRedisCache(BaseCache):
""" """
cache = self.options.get('MASTER_CACHE', None) cache = self.options.get('MASTER_CACHE', None)
if cache is None: if cache is None:
return self.client_list[0] return next(iter(self.client_list))
kwargs = parse_connection_kwargs(cache, db=self.db) kwargs = parse_connection_kwargs(cache, db=self.db)
return self.clients[( return self.clients[(
......
...@@ -3,7 +3,7 @@ from collections import defaultdict ...@@ -3,7 +3,7 @@ from collections import defaultdict
from django.core.exceptions import ImproperlyConfigured from django.core.exceptions import ImproperlyConfigured
from redis_cache.backends.base import BaseRedisCache from redis_cache.backends.base import BaseRedisCache
from redis_cache.compat import DEFAULT_TIMEOUT from redis_cache.compat import DEFAULT_TIMEOUT, smart_text
from redis_cache.sharder import HashRing from redis_cache.sharder import HashRing
...@@ -20,9 +20,8 @@ class ShardedRedisCache(BaseRedisCache): ...@@ -20,9 +20,8 @@ class ShardedRedisCache(BaseRedisCache):
self.client_list = self.clients.values() self.client_list = self.clients.values()
def get_client(self, key, write=False): def get_client(self, key, write=False):
node = self.sharder.get_node(unicode(key)) node = self.sharder.get_node(smart_text(key))
return self.clients[node] return self.clients[node]
def shard(self, keys, write=False, version=None): def shard(self, keys, write=False, version=None):
...@@ -31,7 +30,9 @@ class ShardedRedisCache(BaseRedisCache): ...@@ -31,7 +30,9 @@ class ShardedRedisCache(BaseRedisCache):
""" """
clients = defaultdict(list) clients = defaultdict(list)
for key in keys: for key in keys:
clients[self.get_client(key, write)].append(self.make_key(key, version)) clients[self.get_client(key, write)].append(
self.make_key(key, version)
)
return clients return clients
#################### ####################
...@@ -54,7 +55,7 @@ class ShardedRedisCache(BaseRedisCache): ...@@ -54,7 +55,7 @@ class ShardedRedisCache(BaseRedisCache):
namespace will be deleted. Otherwise, all keys will be deleted. namespace will be deleted. Otherwise, all keys will be deleted.
""" """
if version is None: if version is None:
for client in self.clients.itervalues(): for client in self.clients.values():
self._clear(client) self._clear(client)
else: else:
self.delete_pattern('*', version=version) self.delete_pattern('*', version=version)
...@@ -64,7 +65,13 @@ class ShardedRedisCache(BaseRedisCache): ...@@ -64,7 +65,13 @@ class ShardedRedisCache(BaseRedisCache):
clients = self.shard(keys, version=version) clients = self.shard(keys, version=version)
for client, versioned_keys in clients.items(): for client, versioned_keys in clients.items():
original_keys = [key._original_key for key in versioned_keys] original_keys = [key._original_key for key in versioned_keys]
data.update(self._get_many(client, original_keys, versioned_keys=versioned_keys)) data.update(
self._get_many(
client,
original_keys,
versioned_keys=versioned_keys
)
)
return data return data
def set_many(self, data, timeout=None, version=None): def set_many(self, data, timeout=None, version=None):
...@@ -113,12 +120,12 @@ class ShardedRedisCache(BaseRedisCache): ...@@ -113,12 +120,12 @@ class ShardedRedisCache(BaseRedisCache):
def delete_pattern(self, pattern, version=None): def delete_pattern(self, pattern, version=None):
pattern = self.make_key(pattern, version=version) pattern = self.make_key(pattern, version=version)
for client in self.clients.itervalues(): for client in self.clients.values():
self._delete_pattern(client, pattern) self._delete_pattern(client, pattern)
def reinsert_keys(self): def reinsert_keys(self):
""" """
Reinsert cache entries using the current pickle protocol version. Reinsert cache entries using the current pickle protocol version.
""" """
for client in self.clients.itervalues(): for client in self.clients.values():
self._reinsert_keys(client) self._reinsert_keys(client)
...@@ -26,7 +26,7 @@ class RedisCache(BaseRedisCache): ...@@ -26,7 +26,7 @@ class RedisCache(BaseRedisCache):
def get_client(self, key, write=False): def get_client(self, key, write=False):
if write and self.master_client is not None: if write and self.master_client is not None:
return self.master_client return self.master_client
return random.choice(self.client_list) return random.choice(list(self.client_list))
#################### ####################
# Django cache api # # Django cache api #
......
...@@ -15,8 +15,11 @@ except ImportError: ...@@ -15,8 +15,11 @@ except ImportError:
if PY3: if PY3:
bytes_type = bytes bytes_type = bytes
from urllib.parse import parse_qs, urlparse
else: else:
bytes_type = str bytes_type = str
from urlparse import parse_qs, urlparse
if django.VERSION[:2] >= (1, 6): if django.VERSION[:2] >= (1, 6):
from django.core.cache.backends.base import DEFAULT_TIMEOUT as DJANGO_DEFAULT_TIMEOUT from django.core.cache.backends.base import DEFAULT_TIMEOUT as DJANGO_DEFAULT_TIMEOUT
......
...@@ -14,7 +14,7 @@ class CacheConnectionPool(object): ...@@ -14,7 +14,7 @@ class CacheConnectionPool(object):
return self._clients.get(server, None) return self._clients.get(server, None)
def reset(self): def reset(self):
for pool in self._connection_pools.itervalues(): for pool in self._connection_pools.values():
pool.disconnect() pool.disconnect()
self._clients = {} self._clients = {}
self._connection_pools = {} self._connection_pools = {}
......
...@@ -2,6 +2,7 @@ from bisect import insort, bisect ...@@ -2,6 +2,7 @@ from bisect import insort, bisect
from hashlib import md5 from hashlib import md5
from math import log from math import log
import sys import sys
from functools import total_ordering
try: try:
maxint = sys.maxint maxint = sys.maxint
...@@ -15,21 +16,30 @@ def make_hash(s): ...@@ -15,21 +16,30 @@ def make_hash(s):
return int(md5(s.encode('utf-8')).hexdigest()[:DIGITS], 16) return int(md5(s.encode('utf-8')).hexdigest()[:DIGITS], 16)
@total_ordering
class Node(object): class Node(object):
def __init__(self, node, i): def __init__(self, node, i):
self._node = node self._node = node
self._i = i self._i = i
self._position = make_hash("%d:%s" % (i, str(self._node))) self._position = make_hash("{0}:{1}".format(i, self._node))
def __cmp__(self, other): def __lt__(self, other):
if isinstance(other, int): if isinstance(other, int):
return cmp(self._position, other) return self._position < other
elif isinstance(other, Node): elif isinstance(other, Node):
return cmp(self._position, other._position) return self._position < other._position
raise TypeError('Cannot compare this class with "%s" type' % type(other)) raise TypeError(
'Cannot compare this class with "%s" type' % type(other)
)
def __eq__(self, other): def __eq__(self, other):
if isinstance(other, int):
return self._node == other
elif isinstance(other, Node):
return self._node == other._node return self._node == other._node
raise TypeError(
'Cannot compare this class with "%s" type' % type(other)
)
class HashRing(object): class HashRing(object):
...@@ -42,7 +52,7 @@ class HashRing(object): ...@@ -42,7 +52,7 @@ class HashRing(object):
insort(self._nodes, Node(node, i)) insort(self._nodes, Node(node, i))
def add(self, node, weight=1): def add(self, node, weight=1):
for i in xrange(weight * self.replicas): for i in range(weight * self.replicas):
self._add(node, i) self._add(node, i)
def remove(self, node): def remove(self, node):
......
import warnings import warnings
from django.core.exceptions import ImproperlyConfigured from django.core.exceptions import ImproperlyConfigured
from redis.connection import SSLConnection, UnixDomainSocketConnection from redis.connection import SSLConnection
from redis._compat import iteritems, urlparse, parse_qs
from redis_cache.compat import ( from redis_cache.compat import (
smart_text, python_2_unicode_compatible, bytes_type smart_text, python_2_unicode_compatible, parse_qs, urlparse
) )
try: try:
...@@ -29,6 +28,9 @@ class CacheKey(object): ...@@ -29,6 +28,9 @@ class CacheKey(object):
def __unicode__(self): def __unicode__(self):
return smart_text(self._versioned_key) return smart_text(self._versioned_key)
def __hash__(self):
return hash(self._versioned_key)
__repr__ = __str__ = __unicode__ __repr__ = __str__ = __unicode__
...@@ -95,7 +97,7 @@ def parse_connection_kwargs(server, db=None, **kwargs): ...@@ -95,7 +97,7 @@ def parse_connection_kwargs(server, db=None, **kwargs):
url_options = {} url_options = {}
for name, value in iteritems(parse_qs(qs)): for name, value in parse_qs(qs).items():
if value and len(value) > 0: if value and len(value) > 0:
url_options[name] = value[0] url_options[name] = value[0]
......
...@@ -28,7 +28,7 @@ import redis ...@@ -28,7 +28,7 @@ import redis
from tests.testapp.models import Poll, expensive_calculation from tests.testapp.models import Poll, expensive_calculation
from redis_cache.cache import RedisCache, pool from redis_cache.cache import RedisCache, pool
from redis_cache.compat import DEFAULT_TIMEOUT from redis_cache.compat import DEFAULT_TIMEOUT, smart_bytes
from redis_cache.utils import get_servers, parse_connection_kwargs from redis_cache.utils import get_servers, parse_connection_kwargs
...@@ -60,7 +60,6 @@ def start_redis_servers(servers, db=None, master=None): ...@@ -60,7 +60,6 @@ def start_redis_servers(servers, db=None, master=None):
db=db, db=db,
password=REDIS_PASSWORD password=REDIS_PASSWORD
) )
for i, server in enumerate(servers): for i, server in enumerate(servers):
connection_kwargs = parse_connection_kwargs( connection_kwargs = parse_connection_kwargs(
server, server,
...@@ -463,7 +462,7 @@ class BaseRedisTestCase(SetupMixin): ...@@ -463,7 +462,7 @@ class BaseRedisTestCase(SetupMixin):
def test_reinsert_keys(self): def test_reinsert_keys(self):
self.cache._pickle_version = 0 self.cache._pickle_version = 0
for i in range(2000): for i in range(2000):
s = sha1(str(i)).hexdigest() s = sha1(smart_bytes(i)).hexdigest()
self.cache.set(s, self.cache) self.cache.set(s, self.cache)
self.cache._pickle_version = -1 self.cache._pickle_version = -1
self.cache.reinsert_keys() self.cache.reinsert_keys()
...@@ -504,7 +503,7 @@ class BaseRedisTestCase(SetupMixin): ...@@ -504,7 +503,7 @@ class BaseRedisTestCase(SetupMixin):
self.assertEqual(value, 42) self.assertEqual(value, 42)
def assertMaxConnection(self, cache, max_num): def assertMaxConnection(self, cache, max_num):
for client in cache.clients.itervalues(): for client in cache.clients.values():
self.assertLessEqual(client.connection_pool._created_connections, max_num) self.assertLessEqual(client.connection_pool._created_connections, max_num)
def test_max_connections(self): def test_max_connections(self):
...@@ -515,7 +514,7 @@ class BaseRedisTestCase(SetupMixin): ...@@ -515,7 +514,7 @@ class BaseRedisTestCase(SetupMixin):
pass pass
releases = {} releases = {}
for client in cache.clients.itervalues(): for client in cache.clients.values():
releases[client.connection_pool] = client.connection_pool.release releases[client.connection_pool] = client.connection_pool.release
client.connection_pool.release = noop client.connection_pool.release = noop
self.assertEqual(client.connection_pool.max_connections, 2) self.assertEqual(client.connection_pool.max_connections, 2)
...@@ -531,7 +530,7 @@ class BaseRedisTestCase(SetupMixin): ...@@ -531,7 +530,7 @@ class BaseRedisTestCase(SetupMixin):
self.assertMaxConnection(cache, 2) self.assertMaxConnection(cache, 2)
for client in cache.clients.itervalues(): for client in cache.clients.values():
client.connection_pool.release = releases[client.connection_pool] client.connection_pool.release = releases[client.connection_pool]
client.connection_pool.max_connections = 2 ** 31 client.connection_pool.max_connections = 2 ** 31
......
...@@ -71,7 +71,7 @@ class MasterSlaveTestCase(SetupMixin, TestCase): ...@@ -71,7 +71,7 @@ class MasterSlaveTestCase(SetupMixin, TestCase):
time.sleep(.1) time.sleep(.1)
key = cache.make_key('a') key = cache.make_key('a')
for client in self.cache.clients.values(): for client in self.cache.clients.values():
self.assertEqual(client.get(key), '1') self.assertEqual(int(client.get(key)), 1)
def test_delete(self): def test_delete(self):
cache = self.get_cache() cache = self.get_cache()
......
...@@ -17,11 +17,11 @@ class MultiServerTests(object): ...@@ -17,11 +17,11 @@ class MultiServerTests(object):
def test_key_distribution(self): def test_key_distribution(self):
n = 10000 n = 10000
self.cache.set('a', 'a') self.cache.set('a', 'a')
for i in xrange(n): for i in range(n):
self.cache.set(i, i) self.cache.set(i, i)
keys = [ keys = [
len(client.keys('*')) len(client.keys('*'))
for client in self.cache.clients.itervalues() for client in self.cache.clients.values()
] ]
self.assertLess(((stddev(keys) / n) * 100.0), 10) self.assertLess(((stddev(keys) / n) * 100.0), 10)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment