[Checkins] SVN: lovely.responsecache/trunk/ - added a disk purge utility to remove files from disk storage by using

Bernd Roessl bernd.roessl at lovelysystems.com
Fri Jun 20 12:11:29 EDT 2008


Log message for revision 87599:
   - added a disk purge utility to remove files from disk storage by using
     regular expressions. just get the util by getUtility(IPurge, 'disk')
     and call purge(expr) on it.
  
  

Changed:
  U   lovely.responsecache/trunk/CHANGES.txt
  U   lovely.responsecache/trunk/setup.py
  A   lovely.responsecache/trunk/src/lovely/responsecache/PURGEDISK.txt
  U   lovely.responsecache/trunk/src/lovely/responsecache/PURGEVIEW.txt
  U   lovely.responsecache/trunk/src/lovely/responsecache/configure.zcml
  U   lovely.responsecache/trunk/src/lovely/responsecache/event.py
  U   lovely.responsecache/trunk/src/lovely/responsecache/meta.zcml
  U   lovely.responsecache/trunk/src/lovely/responsecache/purge.py
  U   lovely.responsecache/trunk/src/lovely/responsecache/tests.py
  U   lovely.responsecache/trunk/src/lovely/responsecache/view.py
  U   lovely.responsecache/trunk/src/lovely/responsecache/zcml.py

-=-
Modified: lovely.responsecache/trunk/CHANGES.txt
===================================================================
--- lovely.responsecache/trunk/CHANGES.txt	2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/CHANGES.txt	2008-06-20 16:11:25 UTC (rev 87599)
@@ -2,6 +2,14 @@
 Changes for lovely.responsecache
 ================================
 
+
+2008/06/20 1.1.0
+================
+
+ - added a disk purge utility to remove files from disk storage by using
+   regular expressions. just get the util by getUtility(IPurge, 'disk')
+   and call purge(expr) on it.
+
 2008/06/16 1.0.0a1
 ==================
 

Modified: lovely.responsecache/trunk/setup.py
===================================================================
--- lovely.responsecache/trunk/setup.py	2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/setup.py	2008-06-20 16:11:25 UTC (rev 87599)
@@ -23,7 +23,7 @@
 
 setup(
     name = 'lovely.responsecache',
-    version = '1.0.0a1',
+    version = '1.1.0',
     author = "Lovely Systems",
     author_email = "office at lovelysystems.com",
     description = "Cache results of ContentProviders",

Added: lovely.responsecache/trunk/src/lovely/responsecache/PURGEDISK.txt
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/PURGEDISK.txt	                        (rev 0)
+++ lovely.responsecache/trunk/src/lovely/responsecache/PURGEDISK.txt	2008-06-20 16:11:25 UTC (rev 87599)
@@ -0,0 +1,105 @@
+==========================
+Purging a filesystem cache
+==========================
+
+ - multiple caches
+ - zmi UI to manually purge
+ - implemented as utility
+ - configured by zcml
+
+
+Let's create a purge utility. To ::
+
+    >>> from zope import component
+    >>> from lovely.responsecache.purge import PurgeDiskUtil
+    >>> import tempfile
+    >>> cachedir = tempfile.mkdtemp()
+    >>> util = PurgeDiskUtil([cachedir])
+    >>> component.provideUtility(util, name='disk')
+
+Let's purge an expression::
+
+    >>> util.purge('purge_expression1.*')
+    >>> print log_info
+    <BLANKLINE>
+
+Now lets commit the transaction::
+
+    >>> import transaction
+    >>> log_info.clear()
+    >>> transaction.commit()
+    >>> print log_info
+    lovely.responsecache.purge INFO
+      directory ... purged 'purge_expression1.*'
+
+If the transaction was commited get called twice there should not be more
+to purge::
+
+    >>> log_info.clear()
+    >>> transaction.commit()
+    >>> print log_info
+    <BLANKLINE>
+
+Now call purge with more than one expressions::
+
+    >>> log_info.clear()
+    >>> util.purge('http://domain/purge_expression1')
+    >>> util.purge('http://domain/purge_expression2')
+    >>> transaction.commit()
+    >>> print log_info
+    lovely.responsecache.purge INFO
+      directory '...' purged 'http://domain/purge_expression1'
+    lovely.responsecache.purge INFO
+      directory '...' purged 'http://domain/purge_expression2'
+
+Now call purge with duplicated expressions::
+
+    >>> log_info.clear()
+    >>> util.purge('http://domain/purge_expression1')
+    >>> util.purge('http://domain/purge_expression2')
+    >>> util.purge('http://domain/purge_expression1')
+    >>> transaction.commit()
+    >>> print log_info
+    lovely.responsecache.purge INFO
+      directory '...' purged 'http://domain/purge_expression1'
+    lovely.responsecache.purge INFO
+      directory '...' purged 'http://domain/purge_expression2'
+
+If there are multiple paths to purge it has to work this way::
+
+    >>> log_info.clear()
+    >>> cachedir2 = tempfile.mkdtemp()
+    >>> util.paths.append(cachedir2)
+    >>> util.purge('http://domain/purge_expression1')
+    >>> util.purge('http://domain/purge_expression2')
+    >>> transaction.commit()
+    >>> print log_info
+    lovely.responsecache.purge INFO
+      directory '...' purged 'http://domain/purge_expression1'
+    lovely.responsecache.purge INFO
+      directory '...' purged 'http://domain/purge_expression2'
+    lovely.responsecache.purge INFO
+      directory '...' purged 'http://domain/purge_expression1'
+    lovely.responsecache.purge INFO
+      directory '...' purged 'http://domain/purge_expression2'
+
+Let's see if files in the folder get deleted::
+
+    >>> log_info.clear()
+    >>> util.paths = [cachedir]
+
+    >>> cachefile = tempfile.mktemp(dir=cachedir)
+    >>> expr = cachefile.replace(cachedir+'/', "")
+    >>> open(cachefile, "w").write("<h1>holla</h1>")
+    >>> import os
+    >>> os.path.exists(cachefile)
+    True
+
+    >>> util.purge(".*%s.*" % expr)
+    >>> transaction.commit()
+    >>> print log_info
+    lovely.responsecache.purge INFO
+      directory '...' purged '...'
+
+    >>> os.path.exists(cachefile)
+    False


Property changes on: lovely.responsecache/trunk/src/lovely/responsecache/PURGEDISK.txt
___________________________________________________________________
Name: svn:eol-style
   + native

Modified: lovely.responsecache/trunk/src/lovely/responsecache/PURGEVIEW.txt
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/PURGEVIEW.txt	2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/src/lovely/responsecache/PURGEVIEW.txt	2008-06-20 16:11:25 UTC (rev 87599)
@@ -27,7 +27,7 @@
   >>> component.provideUtility(util)
 
   >>> browser.open('http://localhost/@@contents.html')
-  >>> browser.getLink('purge').click()
+  >>> browser.getLink('Purge').click()
 
 The expression is a required field::
 

Modified: lovely.responsecache/trunk/src/lovely/responsecache/configure.zcml
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/configure.zcml	2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/src/lovely/responsecache/configure.zcml	2008-06-20 16:11:25 UTC (rev 87599)
@@ -26,7 +26,7 @@
       />
 
   <browser:menuItem
-      title="purge"
+      title="Purge"
       for="zope.app.folder.interfaces.IRootFolder"
       menu="zmi_views"
       action="purge.html"

Modified: lovely.responsecache/trunk/src/lovely/responsecache/event.py
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/event.py	2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/src/lovely/responsecache/event.py	2008-06-20 16:11:25 UTC (rev 87599)
@@ -19,7 +19,7 @@
 from zope.app.publication.interfaces import IEndRequestEvent
 from zope import component
 from zope.app.publication.interfaces import IBeforeTraverseEvent
-from interfaces import IResponseCacheSettings
+from interfaces import IResponseCacheSettings, IPurge
 from zope.app.security.interfaces import IUnauthenticatedPrincipal
 from zope.contentprovider.interfaces import IBeforeUpdateEvent
 from zope.contentprovider.interfaces import IContentProvider
@@ -106,4 +106,3 @@
                                       str(auth),
                                       path = '/',
                                       expires = 'Tue, 19 Jan 2038 00:00:00 GMT')
-

Modified: lovely.responsecache/trunk/src/lovely/responsecache/meta.zcml
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/meta.zcml	2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/src/lovely/responsecache/meta.zcml	2008-06-20 16:11:25 UTC (rev 87599)
@@ -16,6 +16,12 @@
         handler=".zcml.purgeDirective"
         />
 
+    <meta:directive
+        name="purgeDisk"
+        schema=".zcml.IPurgeDiskDirective"
+        handler=".zcml.purgeDiskDirective"
+        />
+
   </meta:directives>
 
 </configure>

Modified: lovely.responsecache/trunk/src/lovely/responsecache/purge.py
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/purge.py	2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/src/lovely/responsecache/purge.py	2008-06-20 16:11:25 UTC (rev 87599)
@@ -17,11 +17,13 @@
 """
 __docformat__ = "reStructuredText"
 
+import os
 import logging
 import threading
 import urlparse
 import httplib
 import transaction
+import subprocess
 
 from time import time
 from transaction.interfaces import IDataManager
@@ -33,14 +35,36 @@
 
 
 storage = threading.local()
-EXPRS_ATTR='varnish_purgeurls'
 
 log = logging.getLogger(__name__)
 
-class PurgeUtil(object):
+class BasePurgeUtil(object):
+
+    @property
+    def EXPRS_ATTR(self):
+        # override this in descendant
+        raise NotImplemented
+
+    def purge(self, expr, escapes='+-'):
+        for esc in escapes:
+            expr = expr.replace(esc, '\\' + esc)
+        if not hasattr(storage, self.EXPRS_ATTR):
+            transaction.get().join(PurgeDataManager(self))
+            setattr(storage, self.EXPRS_ATTR, set([expr]))
+        else:
+            getattr(storage, self.EXPRS_ATTR).add(expr)
+
+    def _clearPurge(self):
+        if hasattr(storage, self.EXPRS_ATTR):
+            delattr(storage, self.EXPRS_ATTR)
+
+
+class PurgeUtil(BasePurgeUtil):
     """Utilty to purge mutliple caches"""
     interface.implements(interfaces.IPurge)
 
+    EXPRS_ATTR='varnish_purgeurls'
+
     def __init__(self, hosts, timeout, retryDelay):
         self.hosts = hosts
         self.timeout = timeout
@@ -48,17 +72,8 @@
         self.failedHosts = {}
         self.failLock = threading.Lock()
 
-    def purge(self, expr, escapes='+-'):
-        for esc in escapes:
-            expr = expr.replace(esc, '\\' + esc)
-        if not hasattr(storage, EXPRS_ATTR):
-            transaction.get().join(PurgeDataManager(self))
-            setattr(storage, EXPRS_ATTR, set([expr]))
-        else:
-            getattr(storage, EXPRS_ATTR).add(expr)
-
     def doPurge(self):
-        exprs = getattr(storage, EXPRS_ATTR, None)
+        exprs = getattr(storage, self.EXPRS_ATTR, None)
         if exprs is None:
             return
         for host in self.hosts:
@@ -80,10 +95,6 @@
                 self.failLock.release()
         self._clearPurge()
 
-    def _clearPurge(self):
-        if hasattr(storage, EXPRS_ATTR):
-            delattr(storage, EXPRS_ATTR)
-
     def _expr2URL(self, expr):
         #URL: scheme://netloc/path;parameters?query#fragment
         expr = str(expr)
@@ -180,3 +191,25 @@
     def sortKey(self):
         return "purge_%d" % id(self)
 
+
+class PurgeDiskUtil(BasePurgeUtil):
+    """Utilty to execute purge expressions on multiple disk locations."""
+    interface.implements(interfaces.IPurge)
+
+    EXPRS_ATTR='disk_purgeurls'
+
+    def __init__(self, paths):
+        self.paths = [p.rstrip(os.path.sep) + os.path.sep for p in paths]
+
+    def doPurge(self):
+        exprs = getattr(storage, self.EXPRS_ATTR, None)
+        if exprs is None:
+            return
+        for path in self.paths:
+            if not os.path.exists(path):
+                log.error('path %s not found' % path)
+                return
+            for expr in exprs:
+                subprocess.Popen(["find", path, "-regex", expr, "-delete"])
+                log.info('directory %r purged %r' % (path, expr))
+        self._clearPurge()

Modified: lovely.responsecache/trunk/src/lovely/responsecache/tests.py
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/tests.py	2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/src/lovely/responsecache/tests.py	2008-06-20 16:11:25 UTC (rev 87599)
@@ -60,7 +60,6 @@
     log_info = InstalledHandler('lovely.responsecache.purge')
     test.globs['log_info'] = log_info
 
-
 def setUpHTTPLib(test):
     # setup the PurgeUtil to use httplib instead of pyCurl
     setUp(test)
@@ -87,11 +86,11 @@
             optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS,
             ),
         DocFileSuite(
-            'PURGE.txt', setUp=setUp, tearDown=tearDown,
+            'PURGE.txt', setUp=setUpHTTPLib, tearDown=tearDown,
             optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS,
             ),
         DocFileSuite(
-            'PURGE.txt', setUp=setUpHTTPLib, tearDown=tearDown,
+            'PURGEDISK.txt', setUp=setUp, tearDown=tearDown,
             optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS,
             ),
         fsuite,

Modified: lovely.responsecache/trunk/src/lovely/responsecache/view.py
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/view.py	2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/src/lovely/responsecache/view.py	2008-06-20 16:11:25 UTC (rev 87599)
@@ -71,11 +71,19 @@
 
     form_fields = form.FormFields(IPurgeView)
 
-    @form.action(u'Purge')
+    form_reset = False
+
+    @form.action(u'purge')
     def handle_purge_action(self, action, data):
-        util = component.getUtility(IPurge)
-        util.purge(data['expression'])
+        util = component.queryUtility(IPurge)
+        if util is not None:
+            util.purge(data['expression'])
 
+        diskutil = component.queryUtility(IPurge, 'disk')
+        if diskutil is not None:
+            diskutil.purge(data['expression'])
 
+
 def canPurge(context):
-    return component.queryUtility(IPurge) is not None
+    return (   component.queryUtility(IPurge) is not None
+            or component.queryUtility(IPurge, 'disk') is not None)

Modified: lovely.responsecache/trunk/src/lovely/responsecache/zcml.py
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/zcml.py	2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/src/lovely/responsecache/zcml.py	2008-06-20 16:11:25 UTC (rev 87599)
@@ -31,7 +31,7 @@
 
 from interfaces import IResponseCacheSettings, IPurge
 from view import ResponseCacheSettings
-from purge import PurgeUtil
+from purge import PurgeUtil, PurgeDiskUtil
 
 from zope.i18nmessageid import MessageFactory
 _ = MessageFactory('lovely.responseheader')
@@ -236,3 +236,30 @@
                  name='')
 
 
+class IPurgeDiskDirective(interface.Interface):
+    """Parameters for the purge disk directive."""
+
+    paths = Tokens(
+        title=u'Paths',
+        description=u'Lists of paths where to execute purge expressions',
+        required=True,
+        value_type=schema.TextLine(title=u"path"))
+
+    permission = Permission(
+        title=_("Permission"),
+        description=_("Permission required to use this component."),
+        required=False,
+        )
+
+
+def purgeDiskDirective(_context, paths, permission=None):
+    """Function to create a purge disk utility"""
+
+    util = PurgeDiskUtil(paths)
+    zcml.utility(_context,
+                 provides=IPurge,
+                 component=util,
+                 permission=permission,
+                 name='disk')
+
+



More information about the Checkins mailing list