[Checkins] SVN: lovely.responsecache/trunk/ - added a disk purge
utility to remove files from disk storage by using
Bernd Roessl
bernd.roessl at lovelysystems.com
Fri Jun 20 12:11:29 EDT 2008
Log message for revision 87599:
- added a disk purge utility to remove files from disk storage by using
regular expressions. just get the util by getUtility(IPurge, 'disk')
and call purge(expr) on it.
Changed:
U lovely.responsecache/trunk/CHANGES.txt
U lovely.responsecache/trunk/setup.py
A lovely.responsecache/trunk/src/lovely/responsecache/PURGEDISK.txt
U lovely.responsecache/trunk/src/lovely/responsecache/PURGEVIEW.txt
U lovely.responsecache/trunk/src/lovely/responsecache/configure.zcml
U lovely.responsecache/trunk/src/lovely/responsecache/event.py
U lovely.responsecache/trunk/src/lovely/responsecache/meta.zcml
U lovely.responsecache/trunk/src/lovely/responsecache/purge.py
U lovely.responsecache/trunk/src/lovely/responsecache/tests.py
U lovely.responsecache/trunk/src/lovely/responsecache/view.py
U lovely.responsecache/trunk/src/lovely/responsecache/zcml.py
-=-
Modified: lovely.responsecache/trunk/CHANGES.txt
===================================================================
--- lovely.responsecache/trunk/CHANGES.txt 2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/CHANGES.txt 2008-06-20 16:11:25 UTC (rev 87599)
@@ -2,6 +2,14 @@
Changes for lovely.responsecache
================================
+
+2008/06/20 1.1.0
+================
+
+ - added a disk purge utility to remove files from disk storage by using
+ regular expressions. just get the util by getUtility(IPurge, 'disk')
+ and call purge(expr) on it.
+
2008/06/16 1.0.0a1
==================
Modified: lovely.responsecache/trunk/setup.py
===================================================================
--- lovely.responsecache/trunk/setup.py 2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/setup.py 2008-06-20 16:11:25 UTC (rev 87599)
@@ -23,7 +23,7 @@
setup(
name = 'lovely.responsecache',
- version = '1.0.0a1',
+ version = '1.1.0',
author = "Lovely Systems",
author_email = "office at lovelysystems.com",
description = "Cache results of ContentProviders",
Added: lovely.responsecache/trunk/src/lovely/responsecache/PURGEDISK.txt
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/PURGEDISK.txt (rev 0)
+++ lovely.responsecache/trunk/src/lovely/responsecache/PURGEDISK.txt 2008-06-20 16:11:25 UTC (rev 87599)
@@ -0,0 +1,105 @@
+==========================
+Purging a filesystem cache
+==========================
+
+ - multiple caches
+ - zmi UI to manually purge
+ - implemented as utility
+ - configured by zcml
+
+
+Let's create a purge utility. To ::
+
+ >>> from zope import component
+ >>> from lovely.responsecache.purge import PurgeDiskUtil
+ >>> import tempfile
+ >>> cachedir = tempfile.mkdtemp()
+ >>> util = PurgeDiskUtil([cachedir])
+ >>> component.provideUtility(util, name='disk')
+
+Let's purge an expression::
+
+ >>> util.purge('purge_expression1.*')
+ >>> print log_info
+ <BLANKLINE>
+
+Now lets commit the transaction::
+
+ >>> import transaction
+ >>> log_info.clear()
+ >>> transaction.commit()
+ >>> print log_info
+ lovely.responsecache.purge INFO
+ directory ... purged 'purge_expression1.*'
+
+If the transaction was commited get called twice there should not be more
+to purge::
+
+ >>> log_info.clear()
+ >>> transaction.commit()
+ >>> print log_info
+ <BLANKLINE>
+
+Now call purge with more than one expressions::
+
+ >>> log_info.clear()
+ >>> util.purge('http://domain/purge_expression1')
+ >>> util.purge('http://domain/purge_expression2')
+ >>> transaction.commit()
+ >>> print log_info
+ lovely.responsecache.purge INFO
+ directory '...' purged 'http://domain/purge_expression1'
+ lovely.responsecache.purge INFO
+ directory '...' purged 'http://domain/purge_expression2'
+
+Now call purge with duplicated expressions::
+
+ >>> log_info.clear()
+ >>> util.purge('http://domain/purge_expression1')
+ >>> util.purge('http://domain/purge_expression2')
+ >>> util.purge('http://domain/purge_expression1')
+ >>> transaction.commit()
+ >>> print log_info
+ lovely.responsecache.purge INFO
+ directory '...' purged 'http://domain/purge_expression1'
+ lovely.responsecache.purge INFO
+ directory '...' purged 'http://domain/purge_expression2'
+
+If there are multiple paths to purge it has to work this way::
+
+ >>> log_info.clear()
+ >>> cachedir2 = tempfile.mkdtemp()
+ >>> util.paths.append(cachedir2)
+ >>> util.purge('http://domain/purge_expression1')
+ >>> util.purge('http://domain/purge_expression2')
+ >>> transaction.commit()
+ >>> print log_info
+ lovely.responsecache.purge INFO
+ directory '...' purged 'http://domain/purge_expression1'
+ lovely.responsecache.purge INFO
+ directory '...' purged 'http://domain/purge_expression2'
+ lovely.responsecache.purge INFO
+ directory '...' purged 'http://domain/purge_expression1'
+ lovely.responsecache.purge INFO
+ directory '...' purged 'http://domain/purge_expression2'
+
+Let's see if files in the folder get deleted::
+
+ >>> log_info.clear()
+ >>> util.paths = [cachedir]
+
+ >>> cachefile = tempfile.mktemp(dir=cachedir)
+ >>> expr = cachefile.replace(cachedir+'/', "")
+ >>> open(cachefile, "w").write("<h1>holla</h1>")
+ >>> import os
+ >>> os.path.exists(cachefile)
+ True
+
+ >>> util.purge(".*%s.*" % expr)
+ >>> transaction.commit()
+ >>> print log_info
+ lovely.responsecache.purge INFO
+ directory '...' purged '...'
+
+ >>> os.path.exists(cachefile)
+ False
Property changes on: lovely.responsecache/trunk/src/lovely/responsecache/PURGEDISK.txt
___________________________________________________________________
Name: svn:eol-style
+ native
Modified: lovely.responsecache/trunk/src/lovely/responsecache/PURGEVIEW.txt
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/PURGEVIEW.txt 2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/src/lovely/responsecache/PURGEVIEW.txt 2008-06-20 16:11:25 UTC (rev 87599)
@@ -27,7 +27,7 @@
>>> component.provideUtility(util)
>>> browser.open('http://localhost/@@contents.html')
- >>> browser.getLink('purge').click()
+ >>> browser.getLink('Purge').click()
The expression is a required field::
Modified: lovely.responsecache/trunk/src/lovely/responsecache/configure.zcml
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/configure.zcml 2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/src/lovely/responsecache/configure.zcml 2008-06-20 16:11:25 UTC (rev 87599)
@@ -26,7 +26,7 @@
/>
<browser:menuItem
- title="purge"
+ title="Purge"
for="zope.app.folder.interfaces.IRootFolder"
menu="zmi_views"
action="purge.html"
Modified: lovely.responsecache/trunk/src/lovely/responsecache/event.py
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/event.py 2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/src/lovely/responsecache/event.py 2008-06-20 16:11:25 UTC (rev 87599)
@@ -19,7 +19,7 @@
from zope.app.publication.interfaces import IEndRequestEvent
from zope import component
from zope.app.publication.interfaces import IBeforeTraverseEvent
-from interfaces import IResponseCacheSettings
+from interfaces import IResponseCacheSettings, IPurge
from zope.app.security.interfaces import IUnauthenticatedPrincipal
from zope.contentprovider.interfaces import IBeforeUpdateEvent
from zope.contentprovider.interfaces import IContentProvider
@@ -106,4 +106,3 @@
str(auth),
path = '/',
expires = 'Tue, 19 Jan 2038 00:00:00 GMT')
-
Modified: lovely.responsecache/trunk/src/lovely/responsecache/meta.zcml
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/meta.zcml 2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/src/lovely/responsecache/meta.zcml 2008-06-20 16:11:25 UTC (rev 87599)
@@ -16,6 +16,12 @@
handler=".zcml.purgeDirective"
/>
+ <meta:directive
+ name="purgeDisk"
+ schema=".zcml.IPurgeDiskDirective"
+ handler=".zcml.purgeDiskDirective"
+ />
+
</meta:directives>
</configure>
Modified: lovely.responsecache/trunk/src/lovely/responsecache/purge.py
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/purge.py 2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/src/lovely/responsecache/purge.py 2008-06-20 16:11:25 UTC (rev 87599)
@@ -17,11 +17,13 @@
"""
__docformat__ = "reStructuredText"
+import os
import logging
import threading
import urlparse
import httplib
import transaction
+import subprocess
from time import time
from transaction.interfaces import IDataManager
@@ -33,14 +35,36 @@
storage = threading.local()
-EXPRS_ATTR='varnish_purgeurls'
log = logging.getLogger(__name__)
-class PurgeUtil(object):
+class BasePurgeUtil(object):
+
+ @property
+ def EXPRS_ATTR(self):
+ # override this in descendant
+ raise NotImplemented
+
+ def purge(self, expr, escapes='+-'):
+ for esc in escapes:
+ expr = expr.replace(esc, '\\' + esc)
+ if not hasattr(storage, self.EXPRS_ATTR):
+ transaction.get().join(PurgeDataManager(self))
+ setattr(storage, self.EXPRS_ATTR, set([expr]))
+ else:
+ getattr(storage, self.EXPRS_ATTR).add(expr)
+
+ def _clearPurge(self):
+ if hasattr(storage, self.EXPRS_ATTR):
+ delattr(storage, self.EXPRS_ATTR)
+
+
+class PurgeUtil(BasePurgeUtil):
"""Utilty to purge mutliple caches"""
interface.implements(interfaces.IPurge)
+ EXPRS_ATTR='varnish_purgeurls'
+
def __init__(self, hosts, timeout, retryDelay):
self.hosts = hosts
self.timeout = timeout
@@ -48,17 +72,8 @@
self.failedHosts = {}
self.failLock = threading.Lock()
- def purge(self, expr, escapes='+-'):
- for esc in escapes:
- expr = expr.replace(esc, '\\' + esc)
- if not hasattr(storage, EXPRS_ATTR):
- transaction.get().join(PurgeDataManager(self))
- setattr(storage, EXPRS_ATTR, set([expr]))
- else:
- getattr(storage, EXPRS_ATTR).add(expr)
-
def doPurge(self):
- exprs = getattr(storage, EXPRS_ATTR, None)
+ exprs = getattr(storage, self.EXPRS_ATTR, None)
if exprs is None:
return
for host in self.hosts:
@@ -80,10 +95,6 @@
self.failLock.release()
self._clearPurge()
- def _clearPurge(self):
- if hasattr(storage, EXPRS_ATTR):
- delattr(storage, EXPRS_ATTR)
-
def _expr2URL(self, expr):
#URL: scheme://netloc/path;parameters?query#fragment
expr = str(expr)
@@ -180,3 +191,25 @@
def sortKey(self):
return "purge_%d" % id(self)
+
+class PurgeDiskUtil(BasePurgeUtil):
+ """Utilty to execute purge expressions on multiple disk locations."""
+ interface.implements(interfaces.IPurge)
+
+ EXPRS_ATTR='disk_purgeurls'
+
+ def __init__(self, paths):
+ self.paths = [p.rstrip(os.path.sep) + os.path.sep for p in paths]
+
+ def doPurge(self):
+ exprs = getattr(storage, self.EXPRS_ATTR, None)
+ if exprs is None:
+ return
+ for path in self.paths:
+ if not os.path.exists(path):
+ log.error('path %s not found' % path)
+ return
+ for expr in exprs:
+ subprocess.Popen(["find", path, "-regex", expr, "-delete"])
+ log.info('directory %r purged %r' % (path, expr))
+ self._clearPurge()
Modified: lovely.responsecache/trunk/src/lovely/responsecache/tests.py
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/tests.py 2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/src/lovely/responsecache/tests.py 2008-06-20 16:11:25 UTC (rev 87599)
@@ -60,7 +60,6 @@
log_info = InstalledHandler('lovely.responsecache.purge')
test.globs['log_info'] = log_info
-
def setUpHTTPLib(test):
# setup the PurgeUtil to use httplib instead of pyCurl
setUp(test)
@@ -87,11 +86,11 @@
optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS,
),
DocFileSuite(
- 'PURGE.txt', setUp=setUp, tearDown=tearDown,
+ 'PURGE.txt', setUp=setUpHTTPLib, tearDown=tearDown,
optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS,
),
DocFileSuite(
- 'PURGE.txt', setUp=setUpHTTPLib, tearDown=tearDown,
+ 'PURGEDISK.txt', setUp=setUp, tearDown=tearDown,
optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS,
),
fsuite,
Modified: lovely.responsecache/trunk/src/lovely/responsecache/view.py
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/view.py 2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/src/lovely/responsecache/view.py 2008-06-20 16:11:25 UTC (rev 87599)
@@ -71,11 +71,19 @@
form_fields = form.FormFields(IPurgeView)
- @form.action(u'Purge')
+ form_reset = False
+
+ @form.action(u'purge')
def handle_purge_action(self, action, data):
- util = component.getUtility(IPurge)
- util.purge(data['expression'])
+ util = component.queryUtility(IPurge)
+ if util is not None:
+ util.purge(data['expression'])
+ diskutil = component.queryUtility(IPurge, 'disk')
+ if diskutil is not None:
+ diskutil.purge(data['expression'])
+
def canPurge(context):
- return component.queryUtility(IPurge) is not None
+ return ( component.queryUtility(IPurge) is not None
+ or component.queryUtility(IPurge, 'disk') is not None)
Modified: lovely.responsecache/trunk/src/lovely/responsecache/zcml.py
===================================================================
--- lovely.responsecache/trunk/src/lovely/responsecache/zcml.py 2008-06-20 12:36:41 UTC (rev 87598)
+++ lovely.responsecache/trunk/src/lovely/responsecache/zcml.py 2008-06-20 16:11:25 UTC (rev 87599)
@@ -31,7 +31,7 @@
from interfaces import IResponseCacheSettings, IPurge
from view import ResponseCacheSettings
-from purge import PurgeUtil
+from purge import PurgeUtil, PurgeDiskUtil
from zope.i18nmessageid import MessageFactory
_ = MessageFactory('lovely.responseheader')
@@ -236,3 +236,30 @@
name='')
+class IPurgeDiskDirective(interface.Interface):
+ """Parameters for the purge disk directive."""
+
+ paths = Tokens(
+ title=u'Paths',
+ description=u'Lists of paths where to execute purge expressions',
+ required=True,
+ value_type=schema.TextLine(title=u"path"))
+
+ permission = Permission(
+ title=_("Permission"),
+ description=_("Permission required to use this component."),
+ required=False,
+ )
+
+
+def purgeDiskDirective(_context, paths, permission=None):
+ """Function to create a purge disk utility"""
+
+ util = PurgeDiskUtil(paths)
+ zcml.utility(_context,
+ provides=IPurge,
+ component=util,
+ permission=permission,
+ name='disk')
+
+
More information about the Checkins
mailing list