Commit 2c2ece78 authored by Sean Bleier's avatar Sean Bleier

Some python 3 fixes.

parent 32daa42e
......@@ -123,7 +123,7 @@ class BaseRedisCache(BaseCache):
"""
cache = self.options.get('MASTER_CACHE', None)
if cache is None:
return self.client_list[0]
return next(iter(self.client_list))
kwargs = parse_connection_kwargs(cache, db=self.db)
return self.clients[(
......
......@@ -3,7 +3,7 @@ from collections import defaultdict
from django.core.exceptions import ImproperlyConfigured
from redis_cache.backends.base import BaseRedisCache
from redis_cache.compat import DEFAULT_TIMEOUT
from redis_cache.compat import DEFAULT_TIMEOUT, smart_text
from redis_cache.sharder import HashRing
......@@ -20,9 +20,8 @@ class ShardedRedisCache(BaseRedisCache):
self.client_list = self.clients.values()
def get_client(self, key, write=False):
node = self.sharder.get_node(unicode(key))
node = self.sharder.get_node(smart_text(key))
return self.clients[node]
def shard(self, keys, write=False, version=None):
......@@ -31,7 +30,9 @@ class ShardedRedisCache(BaseRedisCache):
"""
clients = defaultdict(list)
for key in keys:
clients[self.get_client(key, write)].append(self.make_key(key, version))
clients[self.get_client(key, write)].append(
self.make_key(key, version)
)
return clients
####################
......@@ -54,7 +55,7 @@ class ShardedRedisCache(BaseRedisCache):
namespace will be deleted. Otherwise, all keys will be deleted.
"""
if version is None:
for client in self.clients.itervalues():
for client in self.clients.values():
self._clear(client)
else:
self.delete_pattern('*', version=version)
......@@ -64,7 +65,13 @@ class ShardedRedisCache(BaseRedisCache):
clients = self.shard(keys, version=version)
for client, versioned_keys in clients.items():
original_keys = [key._original_key for key in versioned_keys]
data.update(self._get_many(client, original_keys, versioned_keys=versioned_keys))
data.update(
self._get_many(
client,
original_keys,
versioned_keys=versioned_keys
)
)
return data
def set_many(self, data, timeout=None, version=None):
......@@ -113,12 +120,12 @@ class ShardedRedisCache(BaseRedisCache):
def delete_pattern(self, pattern, version=None):
pattern = self.make_key(pattern, version=version)
for client in self.clients.itervalues():
for client in self.clients.values():
self._delete_pattern(client, pattern)
def reinsert_keys(self):
"""
Reinsert cache entries using the current pickle protocol version.
"""
for client in self.clients.itervalues():
for client in self.clients.values():
self._reinsert_keys(client)
......@@ -26,7 +26,7 @@ class RedisCache(BaseRedisCache):
def get_client(self, key, write=False):
if write and self.master_client is not None:
return self.master_client
return random.choice(self.client_list)
return random.choice(list(self.client_list))
####################
# Django cache api #
......
......@@ -15,8 +15,11 @@ except ImportError:
if PY3:
bytes_type = bytes
from urllib.parse import parse_qs, urlparse
else:
bytes_type = str
from urlparse import parse_qs, urlparse
if django.VERSION[:2] >= (1, 6):
from django.core.cache.backends.base import DEFAULT_TIMEOUT as DJANGO_DEFAULT_TIMEOUT
......
......@@ -14,7 +14,7 @@ class CacheConnectionPool(object):
return self._clients.get(server, None)
def reset(self):
for pool in self._connection_pools.itervalues():
for pool in self._connection_pools.values():
pool.disconnect()
self._clients = {}
self._connection_pools = {}
......
......@@ -2,6 +2,7 @@ from bisect import insort, bisect
from hashlib import md5
from math import log
import sys
from functools import total_ordering
try:
maxint = sys.maxint
......@@ -15,21 +16,30 @@ def make_hash(s):
return int(md5(s.encode('utf-8')).hexdigest()[:DIGITS], 16)
@total_ordering
class Node(object):
def __init__(self, node, i):
self._node = node
self._i = i
self._position = make_hash("%d:%s" % (i, str(self._node)))
self._position = make_hash("{0}:{1}".format(i, self._node))
def __cmp__(self, other):
def __lt__(self, other):
if isinstance(other, int):
return cmp(self._position, other)
return self._position < other
elif isinstance(other, Node):
return cmp(self._position, other._position)
raise TypeError('Cannot compare this class with "%s" type' % type(other))
return self._position < other._position
raise TypeError(
'Cannot compare this class with "%s" type' % type(other)
)
def __eq__(self, other):
return self._node == other._node
if isinstance(other, int):
return self._node == other
elif isinstance(other, Node):
return self._node == other._node
raise TypeError(
'Cannot compare this class with "%s" type' % type(other)
)
class HashRing(object):
......@@ -42,7 +52,7 @@ class HashRing(object):
insort(self._nodes, Node(node, i))
def add(self, node, weight=1):
for i in xrange(weight * self.replicas):
for i in range(weight * self.replicas):
self._add(node, i)
def remove(self, node):
......
import warnings
from django.core.exceptions import ImproperlyConfigured
from redis.connection import SSLConnection, UnixDomainSocketConnection
from redis._compat import iteritems, urlparse, parse_qs
from redis.connection import SSLConnection
from redis_cache.compat import (
smart_text, python_2_unicode_compatible, bytes_type
smart_text, python_2_unicode_compatible, parse_qs, urlparse
)
try:
......@@ -29,6 +28,9 @@ class CacheKey(object):
def __unicode__(self):
return smart_text(self._versioned_key)
def __hash__(self):
return hash(self._versioned_key)
__repr__ = __str__ = __unicode__
......@@ -95,7 +97,7 @@ def parse_connection_kwargs(server, db=None, **kwargs):
url_options = {}
for name, value in iteritems(parse_qs(qs)):
for name, value in parse_qs(qs).items():
if value and len(value) > 0:
url_options[name] = value[0]
......
......@@ -28,7 +28,7 @@ import redis
from tests.testapp.models import Poll, expensive_calculation
from redis_cache.cache import RedisCache, pool
from redis_cache.compat import DEFAULT_TIMEOUT
from redis_cache.compat import DEFAULT_TIMEOUT, smart_bytes
from redis_cache.utils import get_servers, parse_connection_kwargs
......@@ -60,7 +60,6 @@ def start_redis_servers(servers, db=None, master=None):
db=db,
password=REDIS_PASSWORD
)
for i, server in enumerate(servers):
connection_kwargs = parse_connection_kwargs(
server,
......@@ -463,7 +462,7 @@ class BaseRedisTestCase(SetupMixin):
def test_reinsert_keys(self):
self.cache._pickle_version = 0
for i in range(2000):
s = sha1(str(i)).hexdigest()
s = sha1(smart_bytes(i)).hexdigest()
self.cache.set(s, self.cache)
self.cache._pickle_version = -1
self.cache.reinsert_keys()
......@@ -504,7 +503,7 @@ class BaseRedisTestCase(SetupMixin):
self.assertEqual(value, 42)
def assertMaxConnection(self, cache, max_num):
for client in cache.clients.itervalues():
for client in cache.clients.values():
self.assertLessEqual(client.connection_pool._created_connections, max_num)
def test_max_connections(self):
......@@ -515,7 +514,7 @@ class BaseRedisTestCase(SetupMixin):
pass
releases = {}
for client in cache.clients.itervalues():
for client in cache.clients.values():
releases[client.connection_pool] = client.connection_pool.release
client.connection_pool.release = noop
self.assertEqual(client.connection_pool.max_connections, 2)
......@@ -531,7 +530,7 @@ class BaseRedisTestCase(SetupMixin):
self.assertMaxConnection(cache, 2)
for client in cache.clients.itervalues():
for client in cache.clients.values():
client.connection_pool.release = releases[client.connection_pool]
client.connection_pool.max_connections = 2 ** 31
......
......@@ -71,7 +71,7 @@ class MasterSlaveTestCase(SetupMixin, TestCase):
time.sleep(.1)
key = cache.make_key('a')
for client in self.cache.clients.values():
self.assertEqual(client.get(key), '1')
self.assertEqual(int(client.get(key)), 1)
def test_delete(self):
cache = self.get_cache()
......
......@@ -17,11 +17,11 @@ class MultiServerTests(object):
def test_key_distribution(self):
n = 10000
self.cache.set('a', 'a')
for i in xrange(n):
for i in range(n):
self.cache.set(i, i)
keys = [
len(client.keys('*'))
for client in self.cache.clients.itervalues()
for client in self.cache.clients.values()
]
self.assertLess(((stddev(keys) / n) * 100.0), 10)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment