Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Contribute to GitLab
Sign in
Toggle navigation
D
Django-Redis-Cache
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Registry
Registry
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Shared
Django-Redis-Cache
Commits
ef82a59a
Commit
ef82a59a
authored
Jul 16, 2015
by
Sean Bleier
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Fixing up some caching behavior.
parent
9b2dfcef
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
61 additions
and
21 deletions
+61
-21
multiple.py
redis_cache/backends/multiple.py
+1
-4
sharder.py
redis_cache/sharder.py
+13
-12
multi_server_tests.py
tests/testapp/tests/multi_server_tests.py
+36
-1
socket_tests.py
tests/testapp/tests/socket_tests.py
+11
-4
No files found.
redis_cache/backends/multiple.py
View file @
ef82a59a
from
collections
import
defaultdict
from
django.core.exceptions
import
ImproperlyConfigured
from
redis_cache.backends.base
import
BaseRedisCache
from
redis_cache.compat
import
DEFAULT_TIMEOUT
,
smart_text
from
redis_cache.sharder
import
HashRing
...
...
@@ -21,7 +18,7 @@ class ShardedRedisCache(BaseRedisCache):
self
.
client_list
=
self
.
clients
.
values
()
def
get_client
(
self
,
key
,
write
=
False
):
node
=
self
.
sharder
.
get_node
(
smart_text
(
key
)
)
node
=
self
.
sharder
.
get_node
(
key
)
return
self
.
clients
[
node
]
def
shard
(
self
,
keys
,
write
=
False
,
version
=
None
):
...
...
redis_cache/sharder.py
View file @
ef82a59a
from
bisect
import
insort
,
bisect
from
hashlib
import
md5
from
math
import
log
import
sys
import
hashlib
from
redis_cache.compat
import
smart_text
try
:
maxint
=
sys
.
maxint
except
AttributeError
:
maxint
=
sys
.
maxsize
DIGITS
=
int
(
log
(
maxint
)
/
log
(
16
))
DIGITS
=
8
def
make_hash
(
s
):
return
int
(
md5
(
s
.
encode
(
'utf-8'
))
.
hexdigest
()[:
DIGITS
],
16
)
def
get_slot
(
s
):
_hash
=
hashlib
.
md5
(
s
.
encode
(
'utf-8'
))
.
hexdigest
()
return
int
(
_hash
[
-
DIGITS
:],
16
)
class
Node
(
object
):
def
__init__
(
self
,
node
,
i
):
self
.
_node
=
node
self
.
_i
=
i
self
.
_position
=
make_hash
(
"{0}:{1}"
.
format
(
i
,
self
.
_node
))
key
=
"{0}:{1}"
.
format
(
smart_text
(
i
),
smart_text
(
self
.
_node
),
)
self
.
_position
=
get_slot
(
key
)
def
__gt__
(
self
,
other
):
if
isinstance
(
other
,
int
):
...
...
@@ -51,5 +52,5 @@ class HashRing(object):
del
self
.
_nodes
[
n
-
i
-
1
]
def
get_node
(
self
,
key
):
i
=
bisect
(
self
.
_nodes
,
make_hash
(
key
))
-
1
i
=
bisect
(
self
.
_nodes
,
get_slot
(
smart_text
(
key
)
))
-
1
return
self
.
_nodes
[
i
]
.
_node
tests/testapp/tests/multi_server_tests.py
View file @
ef82a59a
from
collections
import
Counter
from
math
import
sqrt
from
redis_cache.sharder
import
HashRing
def
mean
(
lst
):
...
...
@@ -14,15 +16,48 @@ def stddev(lst):
class
MultiServerTests
(
object
):
def
test_distribution
(
self
):
nodes
=
[
node
.
_position
for
node
in
self
.
cache
.
sharder
.
_nodes
]
nodes
.
sort
()
diffs
=
[(
b
-
a
)
for
a
,
b
in
zip
(
nodes
[:
-
1
],
nodes
[
1
:])]
l
=
16
**
8
perfect_dist
=
l
/
len
(
nodes
)
random_dist
=
sum
(
diffs
)
/
len
(
diffs
)
_max
=
max
([
perfect_dist
,
random_dist
])
_min
=
min
([
perfect_dist
,
random_dist
])
percentage
=
(
1
-
_max
/
_min
)
*
100
# Assert they are less than 2 percent of each other
self
.
assertLess
(
percentage
,
2.0
)
def
test_make_key_distribution
(
self
):
ring
=
HashRing
()
nodes
=
set
([
str
(
node
.
_node
)
for
node
in
self
.
cache
.
sharder
.
_nodes
])
nodes
=
[
(
'127.0.0.1'
,
6379
,
15
,
'/tmp/redis0.sock'
),
(
'127.0.0.1'
,
6379
,
15
,
'/tmp/redis1.sock'
),
(
'127.0.0.1'
,
6379
,
15
,
'/tmp/redis2.sock'
),
]
for
node
in
nodes
:
ring
.
add
(
str
(
node
))
n
=
50000
counter
=
Counter
(
[
ring
.
get_node
(
str
(
i
))
for
i
in
range
(
n
)]
)
self
.
assertLess
(
((
stddev
(
counter
.
values
())
/
n
)
*
100.0
),
10
,
counter
.
values
()
)
def
test_key_distribution
(
self
):
n
=
10000
self
.
cache
.
set
(
'a'
,
'a'
)
for
i
in
range
(
n
):
self
.
cache
.
set
(
i
,
i
)
keys
=
[
len
(
client
.
keys
(
'*'
))
for
client
in
self
.
cache
.
clients
.
values
()
]
self
.
assertEqual
(
sum
(
keys
),
n
)
self
.
assertLess
(((
stddev
(
keys
)
/
n
)
*
100.0
),
10
)
def
test_removing_nodes
(
self
):
...
...
tests/testapp/tests/socket_tests.py
View file @
ef82a59a
# # -*- coding: utf-8 -*-
from
collections
import
Counter
from
tests.testapp.tests.base_tests
import
BaseRedisTestCase
from
tests.testapp.tests.multi_server_tests
import
MultiServerTests
...
...
@@ -8,9 +9,6 @@ except ImportError:
from
django.test.utils
import
override_settings
from
django.test
import
TestCase
from
redis_cache.cache
import
ImproperlyConfigured
from
redis.connection
import
UnixDomainSocketConnection
LOCATION
=
"unix://:yadayada@/tmp/redis0.sock?db=15"
LOCATIONS
=
[
...
...
@@ -87,7 +85,16 @@ class SinglePythonParserTestCase(SocketTestCase):
}
)
class
MultipleHiredisTestCase
(
MultiServerTests
,
SocketTestCase
):
pass
def
test_equal_number_of_nodes
(
self
):
counter
=
Counter
(
[
node
.
_node
[
3
]
for
node
in
self
.
cache
.
sharder
.
_nodes
]
)
self
.
assertEqual
(
counter
,
{
'/tmp/redis0.sock'
:
16
,
'/tmp/redis1.sock'
:
16
,
'/tmp/redis2.sock'
:
16
,
})
@
override_settings
(
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment