Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Contribute to GitLab
Sign in
Toggle navigation
D
Django-Redis-Cache
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Registry
Registry
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Shared
Django-Redis-Cache
Commits
ef82a59a
Commit
ef82a59a
authored
Jul 16, 2015
by
Sean Bleier
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Fixing up some caching behavior.
parent
9b2dfcef
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
61 additions
and
21 deletions
+61
-21
multiple.py
redis_cache/backends/multiple.py
+1
-4
sharder.py
redis_cache/sharder.py
+13
-12
multi_server_tests.py
tests/testapp/tests/multi_server_tests.py
+36
-1
socket_tests.py
tests/testapp/tests/socket_tests.py
+11
-4
No files found.
redis_cache/backends/multiple.py
View file @
ef82a59a
from
collections
import
defaultdict
from
collections
import
defaultdict
from
django.core.exceptions
import
ImproperlyConfigured
from
redis_cache.backends.base
import
BaseRedisCache
from
redis_cache.backends.base
import
BaseRedisCache
from
redis_cache.compat
import
DEFAULT_TIMEOUT
,
smart_text
from
redis_cache.sharder
import
HashRing
from
redis_cache.sharder
import
HashRing
...
@@ -21,7 +18,7 @@ class ShardedRedisCache(BaseRedisCache):
...
@@ -21,7 +18,7 @@ class ShardedRedisCache(BaseRedisCache):
self
.
client_list
=
self
.
clients
.
values
()
self
.
client_list
=
self
.
clients
.
values
()
def
get_client
(
self
,
key
,
write
=
False
):
def
get_client
(
self
,
key
,
write
=
False
):
node
=
self
.
sharder
.
get_node
(
smart_text
(
key
)
)
node
=
self
.
sharder
.
get_node
(
key
)
return
self
.
clients
[
node
]
return
self
.
clients
[
node
]
def
shard
(
self
,
keys
,
write
=
False
,
version
=
None
):
def
shard
(
self
,
keys
,
write
=
False
,
version
=
None
):
...
...
redis_cache/sharder.py
View file @
ef82a59a
from
bisect
import
insort
,
bisect
from
bisect
import
insort
,
bisect
from
hashlib
import
md5
import
hashlib
from
math
import
log
from
redis_cache.compat
import
smart_text
import
sys
try
:
maxint
=
sys
.
maxint
except
AttributeError
:
maxint
=
sys
.
maxsize
DIGITS
=
int
(
log
(
maxint
)
/
log
(
16
))
DIGITS
=
8
def
make_hash
(
s
):
def
get_slot
(
s
):
return
int
(
md5
(
s
.
encode
(
'utf-8'
))
.
hexdigest
()[:
DIGITS
],
16
)
_hash
=
hashlib
.
md5
(
s
.
encode
(
'utf-8'
))
.
hexdigest
()
return
int
(
_hash
[
-
DIGITS
:],
16
)
class
Node
(
object
):
class
Node
(
object
):
def
__init__
(
self
,
node
,
i
):
def
__init__
(
self
,
node
,
i
):
self
.
_node
=
node
self
.
_node
=
node
self
.
_i
=
i
self
.
_i
=
i
self
.
_position
=
make_hash
(
"{0}:{1}"
.
format
(
i
,
self
.
_node
))
key
=
"{0}:{1}"
.
format
(
smart_text
(
i
),
smart_text
(
self
.
_node
),
)
self
.
_position
=
get_slot
(
key
)
def
__gt__
(
self
,
other
):
def
__gt__
(
self
,
other
):
if
isinstance
(
other
,
int
):
if
isinstance
(
other
,
int
):
...
@@ -51,5 +52,5 @@ class HashRing(object):
...
@@ -51,5 +52,5 @@ class HashRing(object):
del
self
.
_nodes
[
n
-
i
-
1
]
del
self
.
_nodes
[
n
-
i
-
1
]
def
get_node
(
self
,
key
):
def
get_node
(
self
,
key
):
i
=
bisect
(
self
.
_nodes
,
make_hash
(
key
))
-
1
i
=
bisect
(
self
.
_nodes
,
get_slot
(
smart_text
(
key
)
))
-
1
return
self
.
_nodes
[
i
]
.
_node
return
self
.
_nodes
[
i
]
.
_node
tests/testapp/tests/multi_server_tests.py
View file @
ef82a59a
from
collections
import
Counter
from
math
import
sqrt
from
math
import
sqrt
from
redis_cache.sharder
import
HashRing
def
mean
(
lst
):
def
mean
(
lst
):
...
@@ -14,15 +16,48 @@ def stddev(lst):
...
@@ -14,15 +16,48 @@ def stddev(lst):
class
MultiServerTests
(
object
):
class
MultiServerTests
(
object
):
def
test_distribution
(
self
):
nodes
=
[
node
.
_position
for
node
in
self
.
cache
.
sharder
.
_nodes
]
nodes
.
sort
()
diffs
=
[(
b
-
a
)
for
a
,
b
in
zip
(
nodes
[:
-
1
],
nodes
[
1
:])]
l
=
16
**
8
perfect_dist
=
l
/
len
(
nodes
)
random_dist
=
sum
(
diffs
)
/
len
(
diffs
)
_max
=
max
([
perfect_dist
,
random_dist
])
_min
=
min
([
perfect_dist
,
random_dist
])
percentage
=
(
1
-
_max
/
_min
)
*
100
# Assert they are less than 2 percent of each other
self
.
assertLess
(
percentage
,
2.0
)
def
test_make_key_distribution
(
self
):
ring
=
HashRing
()
nodes
=
set
([
str
(
node
.
_node
)
for
node
in
self
.
cache
.
sharder
.
_nodes
])
nodes
=
[
(
'127.0.0.1'
,
6379
,
15
,
'/tmp/redis0.sock'
),
(
'127.0.0.1'
,
6379
,
15
,
'/tmp/redis1.sock'
),
(
'127.0.0.1'
,
6379
,
15
,
'/tmp/redis2.sock'
),
]
for
node
in
nodes
:
ring
.
add
(
str
(
node
))
n
=
50000
counter
=
Counter
(
[
ring
.
get_node
(
str
(
i
))
for
i
in
range
(
n
)]
)
self
.
assertLess
(
((
stddev
(
counter
.
values
())
/
n
)
*
100.0
),
10
,
counter
.
values
()
)
def
test_key_distribution
(
self
):
def
test_key_distribution
(
self
):
n
=
10000
n
=
10000
self
.
cache
.
set
(
'a'
,
'a'
)
for
i
in
range
(
n
):
for
i
in
range
(
n
):
self
.
cache
.
set
(
i
,
i
)
self
.
cache
.
set
(
i
,
i
)
keys
=
[
keys
=
[
len
(
client
.
keys
(
'*'
))
len
(
client
.
keys
(
'*'
))
for
client
in
self
.
cache
.
clients
.
values
()
for
client
in
self
.
cache
.
clients
.
values
()
]
]
self
.
assertEqual
(
sum
(
keys
),
n
)
self
.
assertLess
(((
stddev
(
keys
)
/
n
)
*
100.0
),
10
)
self
.
assertLess
(((
stddev
(
keys
)
/
n
)
*
100.0
),
10
)
def
test_removing_nodes
(
self
):
def
test_removing_nodes
(
self
):
...
...
tests/testapp/tests/socket_tests.py
View file @
ef82a59a
# # -*- coding: utf-8 -*-
# # -*- coding: utf-8 -*-
from
collections
import
Counter
from
tests.testapp.tests.base_tests
import
BaseRedisTestCase
from
tests.testapp.tests.base_tests
import
BaseRedisTestCase
from
tests.testapp.tests.multi_server_tests
import
MultiServerTests
from
tests.testapp.tests.multi_server_tests
import
MultiServerTests
...
@@ -8,9 +9,6 @@ except ImportError:
...
@@ -8,9 +9,6 @@ except ImportError:
from
django.test.utils
import
override_settings
from
django.test.utils
import
override_settings
from
django.test
import
TestCase
from
django.test
import
TestCase
from
redis_cache.cache
import
ImproperlyConfigured
from
redis.connection
import
UnixDomainSocketConnection
LOCATION
=
"unix://:yadayada@/tmp/redis0.sock?db=15"
LOCATION
=
"unix://:yadayada@/tmp/redis0.sock?db=15"
LOCATIONS
=
[
LOCATIONS
=
[
...
@@ -87,7 +85,16 @@ class SinglePythonParserTestCase(SocketTestCase):
...
@@ -87,7 +85,16 @@ class SinglePythonParserTestCase(SocketTestCase):
}
}
)
)
class
MultipleHiredisTestCase
(
MultiServerTests
,
SocketTestCase
):
class
MultipleHiredisTestCase
(
MultiServerTests
,
SocketTestCase
):
pass
def
test_equal_number_of_nodes
(
self
):
counter
=
Counter
(
[
node
.
_node
[
3
]
for
node
in
self
.
cache
.
sharder
.
_nodes
]
)
self
.
assertEqual
(
counter
,
{
'/tmp/redis0.sock'
:
16
,
'/tmp/redis1.sock'
:
16
,
'/tmp/redis2.sock'
:
16
,
})
@
override_settings
(
@
override_settings
(
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment