Merge pull request #88 from sebleier/unstable

Unstable

Makefile

@@ -21,7 +21,7 @@ clean:
 
 .PHONY: test
 test: install_requirements
-	PYTHONPATH=$(PYTHONPATH): django-admin.py test tests.testapp.tests.socket_timeout_tests:SocketTimeoutTestCase.test_socket_timeout --settings=tests.settings -s
+	PYTHONPATH=$(PYTHONPATH): django-admin.py test --settings=tests.settings -s
 
 .PHONY: shell
 shell:

README.rst

@@ -19,6 +19,12 @@ A Redis cache backend for Django
 Changelog
 =========
 
+1.5.0
+-----
+
+* Adds ability to compress/decompress cache values using pluggable compressors
+including zlib, bzip2, or a custom implementation.
+
 1.4.0
 -----
 

redis_cache/backends/base.py

@@ -3,8 +3,6 @@ from django.core.exceptions import ImproperlyConfigured
 from django.utils import importlib
 from django.utils.importlib import import_module
 
-from redis_cache.compat import smart_bytes, DEFAULT_TIMEOUT
-
 try:
     import redis
 except ImportError:
@@ -14,8 +12,11 @@ except ImportError:
 
 from redis.connection import DefaultParser
 
+from redis_cache.compat import smart_bytes, DEFAULT_TIMEOUT
 from redis_cache.connection import pool
-from redis_cache.utils import CacheKey, get_servers, parse_connection_kwargs
+from redis_cache.utils import (
+    CacheKey, get_servers, parse_connection_kwargs, import_class
+)
 
 
 from functools import wraps
@@ -60,12 +61,21 @@ class BaseRedisCache(BaseCache):
         self.connection_pool_class_kwargs = (
             self.get_connection_pool_class_kwargs()
         )
+
+        # Serializer
         self.serializer_class = self.get_serializer_class()
         self.serializer_class_kwargs = self.get_serializer_class_kwargs()
         self.serializer = self.serializer_class(
             **self.serializer_class_kwargs
         )
 
+        # Compressor
+        self.compressor_class = self.get_compressor_class()
+        self.compressor_class_kwargs = self.get_compressor_class_kwargs()
+        self.compressor = self.compressor_class(
+            **self.compressor_class_kwargs
+        )
+
     def __getstate__(self):
         return {'params': self.params, 'server': self.server}
 
@@ -83,18 +93,10 @@ class BaseRedisCache(BaseCache):
         return self.params.get('password', self.options.get('PASSWORD', None))
 
     def get_parser_class(self):
-        cls = self.options.get('PARSER_CLASS', None)
-        if cls is None:
+        parser_class = self.options.get('PARSER_CLASS', None)
+        if parser_class is None:
             return DefaultParser
-        mod_path, cls_name = cls.rsplit('.', 1)
-        try:
-            mod = importlib.import_module(mod_path)
-            parser_class = getattr(mod, cls_name)
-        except AttributeError:
-            raise ImproperlyConfigured("Could not find parser class '%s'" % parser_class)
-        except ImportError as ex:
-            raise ImproperlyConfigured("Could not find module '%s'" % ex)
-        return parser_class
+        return import_class(parser_class)
 
     def get_pickle_version(self):
         """
@@ -111,12 +113,7 @@ class BaseRedisCache(BaseCache):
 
     def get_connection_pool_class(self):
         pool_class = self.options.get('CONNECTION_POOL_CLASS', 'redis.ConnectionPool')
-        module_name, class_name = pool_class.rsplit('.', 1)
-        module = import_module(module_name)
-        try:
-            return getattr(module, class_name)
-        except AttributeError:
-            raise ImportError('cannot import name %s' % class_name)
+        return import_class(pool_class)
 
     def get_connection_pool_class_kwargs(self):
         return self.options.get('CONNECTION_POOL_CLASS_KWARGS', {})
@@ -126,16 +123,21 @@ class BaseRedisCache(BaseCache):
             'SERIALIZER_CLASS',
             'redis_cache.serializers.PickleSerializer'
         )
-        module_name, class_name = serializer_class.rsplit('.', 1)
-        module = import_module(module_name)
-        try:
-            return getattr(module, class_name)
-        except AttributeError:
-            raise ImportError('cannot import name %s' % class_name)
+        return import_class(serializer_class)
 
     def get_serializer_class_kwargs(self):
         return self.options.get('SERIALIZER_CLASS_KWARGS', {})
 
+    def get_compressor_class(self):
+        compressor_class = self.options.get(
+            'COMPRESSOR_CLASS',
+            'redis_cache.compressors.NoopCompressor'
+        )
+        return import_class(compressor_class)
+
+    def get_compressor_class_kwargs(self):
+        return self.options.get('COMPRESSOR_CLASS_KWARGS', {})
+
     def get_master_client(self):
         """
         Get the write server:port of the master cache
@@ -175,17 +177,25 @@ class BaseRedisCache(BaseCache):
     def deserialize(self, value):
         return self.serializer.deserialize(value)
 
+    def compress(self, value):
+        return self.compressor.compress(value)
+
+    def decompress(self, value):
+        return self.compressor.decompress(value)
+
     def get_value(self, original):
         try:
             value = int(original)
         except (ValueError, TypeError):
-            value = self.deserialize(original)
+            value = self.decompress(original)
+            value = self.deserialize(value)
         return value
 
     def prep_value(self, value):
         if isinstance(value, int) and not isinstance(value, bool):
             return value
-        return self.serialize(value)
+        value = self.serialize(value)
+        return self.compress(value)
 
     def make_key(self, key, version=None):
         if not isinstance(key, CacheKey):
@@ -382,8 +392,7 @@ class BaseRedisCache(BaseCache):
         keys = client.keys('*')
         for key in keys:
             timeout = client.ttl(key)
-            value = self.deserialize(client.get(key))
-
+            value = self.get_value(client.get(key))
             if timeout is None:
                 client.set(key, self.prep_value(value))
 

redis_cache/compressors.py

+import zlib
+
+try:
+    import bz2
+except ImportError:
+    pass
+
+
+class BaseCompressor(object):
+
+    def __init__(self, **kwargs):
+        super(BaseCompressor, self).__init__()
+
+    def compress(self, value):
+        raise NotImplementedError
+
+    def decompress(self, value):
+        raise NotImplementedError
+
+
+class NoopCompressor(BaseCompressor):
+
+    def compress(self, value):
+        return value
+
+    def decompress(self, value):
+        return value
+
+
+class ZLibCompressor(BaseCompressor):
+
+    def __init__(self, level=6):
+        self.level = level
+        super(ZLibCompressor, self).__init__()
+
+    def compress(self, value):
+        return zlib.compress(value, self.level)
+
+    def decompress(self, value):
+        return zlib.decompress(value)
+
+
+class BZip2Compressor(BaseCompressor):
+
+    def __init__(self, compresslevel=9):
+        self.compresslevel = compresslevel
+        super(BZip2Compressor, self).__init__()
+
+    def compress(self, value):
+        return bz2.compress(value, compresslevel=self.compresslevel)
+
+    def decompress(self, value):
+        return bz2.decompress(value)

redis_cache/utils.py

 import warnings
 
 from django.core.exceptions import ImproperlyConfigured
+from django.utils.importlib import import_module
 from redis.connection import SSLConnection
 
 from redis_cache.compat import (
@@ -49,6 +50,19 @@ def get_servers(location):
     return servers
 
 
+def import_class(path):
+    module_name, class_name = path.rsplit('.', 1)
+    try:
+        module = import_module(module_name)
+    except ImportError:
+        raise ImproperlyConfigured('Could not find module "%s"' % module_name)
+    else:
+        try:
+            return getattr(module, class_name)
+        except AttributeError:
+            raise ImproperlyConfigured('Cannot import "%s"' % class_name)
+
+
 def parse_connection_kwargs(server, db=None, **kwargs):
     """
     Return a connection pool configured from the given URL.

setup.py

@@ -5,7 +5,7 @@ setup(
     url="http://github.com/sebleier/django-redis-cache/",
     author="Sean Bleier",
     author_email="sebleier@gmail.com",
-    version="1.4.0",
+    version="1.5.1",
     packages=["redis_cache", "redis_cache.backends"],
     description="Redis Cache Backend for Django",
     install_requires=['redis>=2.10.3'],

tests/testapp/tests/compressor_tests.py

+# -*- coding: utf-8 -*-
+try:
+    from django.test import override_settings
+except ImportError:
+    from django.test.utils import override_settings
+from django.test import TestCase
+
+
+from tests.testapp.tests.base_tests import BaseRedisTestCase
+
+LOCATION = "127.0.0.1:6381"
+
+
+class CompressionTestCase(object):
+
+    def test_compression(self):
+        key = 'a'
+        noop_cache = self.get_cache('noop')
+        string = 10000 * 'a'
+
+        self.cache.set(key, string)
+        noop_cache.set(key, string)
+        self.assertEqual(self.cache.get(key), noop_cache.get(key))
+        self.assertNotEqual(self.cache, noop_cache)
+
+        noop_client, = list(noop_cache.clients.values())
+        default_client, = list(self.cache.clients.values())
+        versioned_key = self.cache.make_key(key)
+        self.assertLess(
+            len(default_client.get(versioned_key)),
+            len(noop_client.get(versioned_key)),
+        )
+
+
+@override_settings(
+    CACHES={
+        'default': {
+            'BACKEND': 'redis_cache.RedisCache',
+            'LOCATION': LOCATION,
+            'OPTIONS': {
+                'DB': 14,
+                'PASSWORD': 'yadayada',
+                'PARSER_CLASS': 'redis.connection.HiredisParser',
+                'PICKLE_VERSION': -1,
+                'COMPRESSOR_CLASS': 'redis_cache.compressors.ZLibCompressor',
+                'COMPRESSOR_CLASS_KWARGS': {
+                    'level': 5,
+                },
+                'CONNECTION_POOL_CLASS': 'redis.ConnectionPool',
+                'CONNECTION_POOL_CLASS_KWARGS': {
+                    'max_connections': 2,
+                },
+            },
+        },
+        'noop': {
+            'BACKEND': 'redis_cache.RedisCache',
+            'LOCATION': LOCATION,
+            'OPTIONS': {
+                'DB': 15,
+                'PASSWORD': 'yadayada',
+                'PARSER_CLASS': 'redis.connection.HiredisParser',
+                'PICKLE_VERSION': -1,
+                'COMPRESSOR_CLASS': 'redis_cache.compressors.NoopCompressor',
+            },
+        },
+    }
+)
+class ZLibTestCase(CompressionTestCase, BaseRedisTestCase, TestCase):
+    pass
+
+
+@override_settings(
+    CACHES={
+        'default': {
+            'BACKEND': 'redis_cache.RedisCache',
+            'LOCATION': LOCATION,
+            'OPTIONS': {
+                'DB': 14,
+                'PASSWORD': 'yadayada',
+                'PARSER_CLASS': 'redis.connection.HiredisParser',
+                'PICKLE_VERSION': -1,
+                'COMPRESSOR_CLASS': 'redis_cache.compressors.BZip2Compressor',
+                'COMPRESSOR_CLASS_KWARGS': {
+                    'compresslevel': 5,
+                },
+                'CONNECTION_POOL_CLASS': 'redis.ConnectionPool',
+                'CONNECTION_POOL_CLASS_KWARGS': {
+                    'max_connections': 2,
+                },
+            },
+        },
+        'noop': {
+            'BACKEND': 'redis_cache.RedisCache',
+            'LOCATION': LOCATION,
+            'OPTIONS': {
+                'DB': 15,
+                'PASSWORD': 'yadayada',
+                'PARSER_CLASS': 'redis.connection.HiredisParser',
+                'PICKLE_VERSION': -1,
+                'COMPRESSOR_CLASS': 'redis_cache.compressors.NoopCompressor',
+            },
+        },
+    }
+)
+class BZip2TestCase(CompressionTestCase, BaseRedisTestCase, TestCase):
+    pass

tests/testapp/tests/socket_timeout_tests.py

@@ -8,7 +8,7 @@ from django.test import TestCase
 from redis.exceptions import ConnectionError
 from tests.testapp.tests.base_tests import SetupMixin
 
-LOCATION = "127.0.0.1:6382"
+LOCATION = "127.0.0.1:6381"
 
 
 @override_settings(