jehiah · October 5, 2010 16:00 · Oct 5, 2010
diff --git a/MemcachedPool.py b/MemcachedPool.py
@@ -0,0 +1,85 @@
+import pylibmc
+import Queue
+import logging
+import functools
+
+"""
+This is a transparent pool library that wraps a pylibmc client
+
+from MemcachePool import mc
+mc.get(key)
+mc.set(key, value)
+
+"""
+
+import settings
+def servers():
+    return settings.get('memcached')
+
+server_pool = Queue.Queue(maxsize=50) # of max cached connections; not max # of open connections
+
+def newConnection():
+    """setup a new memcached connection"""
+    if callable(servers):
+        return pylibmc.Client(servers())
+    else:
+        return pylibmc.Client(servers)
+
+def pool(method):
+    """
+    this decorator implements a self growing pool and returns 
+    connections to it
+    If there is not a connection available in the pool create one
+    If we can't return the connection to the pool, drop it 
+    """
+    @functools.wraps(method)
+    def wrapper(*args, **kwargs):
+        try:
+            conn = server_pool.get_nowait()
+        except Queue.Empty:
+            conn = newConnection()
+        val = None
+        try:
+            ## method is wrapperFunction in PoolClient
+            key = args[0]
+            if not isinstance(key, (str, list, tuple)):
+                logging.warning('%r must be _utf8' % key)
+            try:
+                # convert to utf8 on the fly since memcached chokes on unicode objects
+                args = [_utf8(arg) for arg in args]
+                val = method(conn, *args, **kwargs)
+            except:
+                logging.exception('failed memcached call %s %s' % (str(args), str(kwargs)))
+                val = None
+        finally:
+            try:
+                server_pool.put_nowait(conn)
+            except Queue.Full:
+                pass
+        return val
+    return wrapper
+
+class PoolClient(object):
+    """
+    This is a wrapper class that passes all methods on to the actual client
+    getting a connection from the pool first, and returning it there after
+    """
+    def __getattribute__(self,name):
+        @pool
+        def wrapperFunction(*args, **kwargs):
+            ## called by acquireAndRelease;
+            ## args[0] == pylibmc.Client
+            ## name == method to call
+            ## args[1:] the original method arguments
+
+            ## debug:
+            # print "memcache", name, args[1]
+            return getattr(args[0], name)(*args[1:], **kwargs)
+        return wrapperFunction
+
+mc = PoolClient()
+
+def _utf8(s):
+    if isinstance(s, unicode):
+        return s.encode("utf-8")
+    return s
No results found