[Checkins] SVN: Sandbox/J1m/zookeeperrecipes/ Recipe to set up zookeeper trees for development
Jim Fulton
jim at zope.com
Fri Jan 27 21:57:17 UTC 2012
Log message for revision 124224:
Recipe to set up zookeeper trees for development
Changed:
D Sandbox/J1m/zookeeperrecipes/README.txt
A Sandbox/J1m/zookeeperrecipes/README.txt
U Sandbox/J1m/zookeeperrecipes/buildout.cfg
U Sandbox/J1m/zookeeperrecipes/setup.py
A Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/
A Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/README.txt
A Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/__init__.py
A Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/tests.py
-=-
Deleted: Sandbox/J1m/zookeeperrecipes/README.txt
===================================================================
--- Sandbox/J1m/zookeeperrecipes/README.txt 2012-01-27 21:53:11 UTC (rev 124223)
+++ Sandbox/J1m/zookeeperrecipes/README.txt 2012-01-27 21:57:17 UTC (rev 124224)
@@ -1,14 +0,0 @@
-Title Here
-**********
-
-
-To learn more, see
-
-
-Changes
-*******
-
-0.1.0 (yyyy-mm-dd)
-==================
-
-Initial release
Added: Sandbox/J1m/zookeeperrecipes/README.txt
===================================================================
--- Sandbox/J1m/zookeeperrecipes/README.txt (rev 0)
+++ Sandbox/J1m/zookeeperrecipes/README.txt 2012-01-27 21:57:17 UTC (rev 124224)
@@ -0,0 +1 @@
+link ./src/zc/zookeeperrecipes/README.txt
\ No newline at end of file
Property changes on: Sandbox/J1m/zookeeperrecipes/README.txt
___________________________________________________________________
Added: svn:special
+ *
Modified: Sandbox/J1m/zookeeperrecipes/buildout.cfg
===================================================================
--- Sandbox/J1m/zookeeperrecipes/buildout.cfg 2012-01-27 21:53:11 UTC (rev 124223)
+++ Sandbox/J1m/zookeeperrecipes/buildout.cfg 2012-01-27 21:57:17 UTC (rev 124224)
@@ -4,7 +4,7 @@
[test]
recipe = zc.recipe.testrunner
-eggs =
+eggs = zc.zookeeperrecipes [test]
[py]
recipe = zc.recipe.egg
Modified: Sandbox/J1m/zookeeperrecipes/setup.py
===================================================================
--- Sandbox/J1m/zookeeperrecipes/setup.py 2012-01-27 21:53:11 UTC (rev 124223)
+++ Sandbox/J1m/zookeeperrecipes/setup.py 2012-01-27 21:57:17 UTC (rev 124224)
@@ -11,12 +11,16 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
-name, version = 'zc.', '0'
+name, version = 'zc.zookeeperrecipes', '0'
-install_requires = ['setuptools']
-extras_require = dict(test=['zope.testing'])
+install_requires = ['setuptools', 'zc.zk [static]']
+extras_require = dict(test=[
+ 'zope.testing', 'zc.buildout', 'zc.zk [test]'
+ ])
entry_points = """
+[zc.buildout]
+devtree = zc.zookeeperrecipes:DevTree
"""
from setuptools import setup
Added: Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/README.txt
===================================================================
--- Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/README.txt (rev 0)
+++ Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/README.txt 2012-01-27 21:57:17 UTC (rev 124224)
@@ -0,0 +1,395 @@
+=================
+ZooKeeper Recipes
+=================
+
+devtree
+=======
+
+The devtree recipe sets up temporary ZooKeeper tree for a buildout::
+
+ [myproject]
+ recipe = zc.zookeeperrecipes:devtree
+ import-file = tree.txt
+
+.. -> conf
+
+ *** Basics, default path, ***
+
+ >>> def write(name, text):
+ ... with open(name, 'w') as f: f.write(text)
+
+ >>> write('tree.txt', """
+ ... x=1
+ ... type = 'foo'
+ ... /a
+ ... /b
+ ... /c
+ ... """)
+
+ >>> import ConfigParser, StringIO, os
+ >>> from zc.zookeeperrecipes import DevTree
+
+ >>> here = os.getcwd()
+ >>> buildoutbuildout = {
+ ... 'directory': here,
+ ... 'parts-directory': os.path.join(here, 'parts'),
+ ... }
+
+ >>> def buildout():
+ ... parser = ConfigParser.RawConfigParser()
+ ... parser.readfp(StringIO.StringIO(conf))
+ ... buildout = dict((name, dict(parser.items(name)))
+ ... for name in parser.sections())
+ ... [name] = buildout.keys()
+ ... buildout['buildout'] = buildoutbuildout
+ ... options = buildout[name]
+ ... recipe = DevTree(buildout, name, options)
+ ... return recipe, options
+
+
+ >>> import zc.zookeeperrecipes, mock
+ >>> with mock.patch('zc.zookeeperrecipes.timestamp') as ts:
+ ... ts.return_value = '2012-01-26T14:50:14.864772'
+ ... recipe, options = buildout()
+
+
+ >>> from pprint import pprint
+ >>> pprint(options)
+ {'effective-path': '/myproject2012-01-26T14:50:14.864772',
+ 'import-file': 'tree.txt',
+ 'import-text': "\nx=1\ntype = 'foo'\n/a\n /b\n/c\n",
+ 'location': '/testdirectory/parts/myproject',
+ 'path': '/myproject',
+ 'recipe': 'zc.zookeeperrecipes:devtree'}
+
+ >>> recipe.install()
+ ()
+
+ >>> def cat(*path):
+ ... with open(os.path.join(*path)) as f:
+ ... return f.read()
+
+ >>> cat('parts', 'myproject')
+ '/myproject2012-01-26T14:50:14.864772'
+
+ *** Test node name is persistent ***
+
+ Updating doesn't change the name:
+
+ >>> recipe, options = buildout()
+ >>> recipe.update()
+ ()
+ >>> options['effective-path'] == '/myproject2012-01-26T14:50:14.864772'
+ True
+ >>> cat('parts', 'myproject')
+ '/myproject2012-01-26T14:50:14.864772'
+
+ >>> import zc.zk
+ >>> zk = zc.zk.ZooKeeper('127.0.0.1:2181')
+ >>> zk.print_tree()
+ /myproject2012-01-26T14:50:14.864772 : foo
+ buildout:location = u'/testdirectory/parts/myproject'
+ x = 1
+ /a
+ /b
+ /c
+
+ *** Test updating tree source ***
+
+ If there are changes, we see them
+
+ >>> write('tree.txt', """
+ ... /a
+ ... /d
+ ... /c
+ ... """)
+
+ >>> buildout()[0].install()
+ ()
+ >>> zk.print_tree()
+ /myproject2012-01-26T14:50:14.864772
+ buildout:location = u'/testdirectory/parts/myproject'
+ /a
+ /d
+ /c
+
+ Now, if there are ephemeral nodes:
+
+ >>> with mock.patch('os.getpid') as getpid:
+ ... getpid.return_value = 42
+ ... zk.register_server('/myproject2012-01-26T14:50:14.864772/a/d',
+ ... 'x:y')
+
+ >>> write('tree.txt', """
+ ... /a
+ ... /b
+ ... /c
+ ... """)
+
+ >>> buildout()[0].install() # doctest: +NORMALIZE_WHITESPACE
+ Not deleting /myproject2012-01-26T14:50:14.864772/a/d/x:y
+ because it's ephemeral.
+ /myproject2012-01-26T14:50:14.864772/a/d
+ not deleted due to ephemeral descendent.
+ ()
+
+ >>> zk.print_tree()
+ /myproject2012-01-26T14:50:14.864772
+ buildout:location = u'/testdirectory/parts/myproject'
+ /a
+ /b
+ /d
+ /x:y
+ pid = 42
+ /c
+
+ The ephemeral node, and the node containing it is left, but a
+ warning is issued.
+
+ *** Cleanup w different part name ***
+
+ Now, let's change out buildout to use a different part name:
+
+ >>> conf = """
+ ... [myproj]
+ ... recipe = zc.zookeeperrecipes:devtree
+ ... import-file = tree.txt
+ ... """
+
+ >>> os.remove(os.path.join('parts', 'myproject'))
+
+ Now, when we rerun the buildout, the old tree will get cleaned up:
+
+ >>> import signal
+ >>> with mock.patch('os.kill') as kill:
+ ... with mock.patch('zc.zookeeperrecipes.timestamp') as ts:
+ ... ts.return_value = '2012-01-26T14:50:15.864772'
+ ... recipe, options = buildout()
+ ... recipe.install()
+ ... kill.assert_called_with(42, signal.SIGTERM)
+ ()
+
+ >>> pprint(options)
+ {'effective-path': '/myproj2012-01-26T14:50:15.864772',
+ 'import-file': 'tree.txt',
+ 'import-text': '\n/a\n /b\n/c\n',
+ 'location': '/testdirectory/parts/myproj',
+ 'path': '/myproj',
+ 'recipe': 'zc.zookeeperrecipes:devtree'}
+
+ >>> with mock.patch('os.getpid') as getpid:
+ ... getpid.return_value = 42
+ ... zk.register_server('/myproj2012-01-26T14:50:15.864772/a/b',
+ ... 'x:y')
+
+ >>> zk.print_tree()
+ /myproj2012-01-26T14:50:15.864772
+ buildout:location = u'/testdirectory/parts/myproj'
+ /a
+ /b
+ /x:y
+ pid = 42
+ /c
+
+
+ *** Cleanup w different path and explicit path, and creation of base nodes ***
+
+ >>> conf = """
+ ... [myproj]
+ ... recipe = zc.zookeeperrecipes:devtree
+ ... import-file = tree.txt
+ ... path = /ztest/path
+ ... """
+
+ >>> with mock.patch('os.kill') as kill:
+ ... with mock.patch('zc.zookeeperrecipes.timestamp') as ts:
+ ... ts.return_value = '2012-01-26T14:50:16.864772'
+ ... recipe, options = buildout()
+ ... recipe.install()
+ ... kill.assert_called_with(42, signal.SIGTERM)
+ ()
+
+ >>> pprint(options)
+ {'effective-path': '/ztest/path2012-01-26T14:50:16.864772',
+ 'import-file': 'tree.txt',
+ 'import-text': '\n/a\n /b\n/c\n',
+ 'location': '/tmp/tmpZ3mohq/testdirectory/parts/myproj',
+ 'path': '/ztest/path',
+ 'recipe': 'zc.zookeeperrecipes:devtree'}
+
+ >>> with mock.patch('os.getpid') as getpid:
+ ... getpid.return_value = 42
+ ... zk.register_server('/ztest/path2012-01-26T14:50:16.864772/a/b',
+ ... 'x:y')
+
+ >>> zk.print_tree()
+ /ztest
+ /path2012-01-26T14:50:16.864772
+ buildout:location = u'/tmp/tmpZ3mohq/testdirectory/parts/myproj'
+ /a
+ /b
+ /x:y
+ pid = 42
+ /c
+
+ *** explicit effective-path ***
+
+ We can control the effective-path directly:
+
+ >>> conf = """
+ ... [myproj]
+ ... recipe = zc.zookeeperrecipes:devtree
+ ... effective-path = /my/path
+ ... import-file = tree.txt
+ ... """
+
+ This time, we'll also check
+ that kill fail handlers are handled properly.
+
+ >>> with mock.patch('os.kill') as kill:
+ ... def noway(pid, sig):
+ ... raise OSError
+ ... kill.side_effect = noway
+ ... recipe, options = buildout()
+ ... recipe.install()
+ ... kill.assert_called_with(42, signal.SIGTERM)
+ ()
+
+ >>> pprint(options)
+ {'effective-path': '/my/path',
+ 'import-file': 'tree.txt',
+ 'import-text': '\n/a\n /b\n/c\n',
+ 'location': '/testdirectory/parts/myproj',
+ 'recipe': 'zc.zookeeperrecipes:devtree'}
+
+ >>> zk.print_tree() # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
+ /my
+ /path
+ buildout:location = u'/tmp/tmpiKIi2U/testdirectory/parts/myproj'
+ /a
+ /b
+ /c
+ /ztest
+
+ >>> zk.close()
+
+ *** Non-local zookeeper no cleanup no/explicit import text ***
+
+ **Note** because of the way zookeeper testing works, there's
+ really only one zookeeper "server", so even though we're using a
+ different connection string, we get the same db.
+
+ >>> conf = """
+ ... [myproj]
+ ... recipe = zc.zookeeperrecipes:devtree
+ ... effective-path = /my/path
+ ... zookeeper = zookeeper.example.com:2181
+ ... """
+
+ >>> recipe, options = buildout()
+ >>> recipe.install()
+ ()
+
+ >>> pprint(options)
+ {'effective-path': '/my/path',
+ 'location': '/tmp/tmpUAkJkK/testdirectory/parts/myproj',
+ 'recipe': 'zc.zookeeperrecipes:devtree',
+ 'zookeeper': 'zookeeper.example.com:2181'}
+
+ >>> zk = zc.zk.ZooKeeper('zookeeper.example.com:2181')
+ >>> with mock.patch('os.getpid') as getpid:
+ ... getpid.return_value = 42
+ ... zk.register_server('/my/path', 'a:b')
+ >>> zk.print_tree()
+ /my
+ /path
+ buildout:location = u'/tmp/tmp2Qp4qX/testdirectory/parts/myproj'
+ /a:b
+ pid = 42
+ /ztest
+
+ >>> conf = """
+ ... [myproj]
+ ... recipe = zc.zookeeperrecipes:devtree
+ ... import-text = /a
+ ... zookeeper = zookeeper.example.com:2181
+ ... path =
+ ... """
+
+
+ >>> with mock.patch('os.kill') as kill:
+ ... def noway(pid, sig):
+ ... print 'wtf killed'
+ ... kill.side_effect = noway
+ ... with mock.patch('zc.zookeeperrecipes.timestamp') as ts:
+ ... ts.return_value = '2012-01-26T14:50:24.864772'
+ ... recipe, options = buildout()
+ ... recipe.install()
+ ()
+
+ >>> zk.print_tree()
+ /2012-01-26T14:50:24.864772
+ buildout:location = u'/tmp/tmpxh1XPP/testdirectory/parts/myproj'
+ /a
+ /my
+ /path
+ buildout:location = u'/tmp/tmpxh1XPP/testdirectory/parts/myproj'
+ /a:b
+ pid = 42
+ /ztest
+
+
+In this example, we're creating a ZooKeeper tree at the path
+``/myprojectYYYY-MM-DDTHH:MM:SS.SSSSSS`` with data imported from the
+buildout-local file ``tree.txt``, where YYYY-MM-DDTHH:MM:SS.SSSSSS is
+the ISO date-time when the node was created.
+
+The ``tree`` recipe options are:
+
+zookeeper
+ Optional ZooKeeper connection string.
+
+ It defaults to '127.0.0.1:2181'.
+
+path
+ Optional path at which to create the tree.
+
+ If not provided, the part name is used, with a leading ``/`` added.
+
+ When a ``devtree`` part is installed, a path is created at a path
+ derived from the given (or implied) path by adding the current date
+ and time to the path in ISO date-time format
+ (YYYY-MM-DDTHH:MM:SS.SSSSSS). The derived path is stored a file in
+ the buildout parts directory with a name equal to the section name.
+
+effective-path
+ Optional path to be used as is.
+
+ This option is normally computed by the recipe and is queryable
+ from other recipes, but it may also be set explicitly.
+
+import-file
+ Optional import file.
+
+ This is the name of a file containing tree-definition text. See the
+ ``zc.zk`` documentation for information on the format of this file.
+
+import-text
+ Optional import text.
+
+ Unfortunately, because of the way buildout parsers configuration
+ files, leading whitespace is stripped, making this option hard to
+ specify.
+
+Cleanup
+-------
+
+We don't want trees to accumulate indefinately. When using a local
+zookeeper (default), when the recipe is run, the entire tree is
+scanned looking for nodes that have ``buildout:location`` properties
+with paths that no-longer exist in the local file system paths that
+contain different ZooKeeper paths.
+
+If such nodes are found, then the nodes are removed and, if the nodes
+had any ephemeral subnodes with pids, those pids are sent a SIGTERM
+signal.
Property changes on: Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/README.txt
___________________________________________________________________
Added: svn:eol-style
+ native
Added: Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/__init__.py
===================================================================
--- Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/__init__.py (rev 0)
+++ Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/__init__.py 2012-01-27 21:57:17 UTC (rev 124224)
@@ -0,0 +1,127 @@
+##############################################################################
+#
+# Copyright (c) 2011 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+import datetime
+import logging
+import os
+import re
+import signal
+import textwrap
+import zc.buildout
+import zc.zk
+
+logger = logging.getLogger(__name__)
+
+def timestamp():
+ return datetime.datetime.now().isoformat()
+
+timestamped = re.compile(r'(.+)\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+$').match
+
+class DevTree:
+
+ def __init__(self, buildout, name, options):
+ self.name, self.options = name, options
+ if 'import-file' in options:
+ options['import-text'] = open(os.path.join(
+ buildout['buildout']['directory'],
+ options['import-file'])).read()
+
+ options['location'] = location = os.path.join(
+ buildout['buildout']['parts-directory'], name)
+
+ if 'effective-path' not in options:
+ path = options.get('path', name)
+ if not path.startswith('/'):
+ path = '/' + path
+ options['path'] = path
+
+ if os.path.exists(location):
+ with open(location) as f:
+ epath = f.read()
+ m = timestamped(epath)
+ if m and m.group(1) == path:
+ options['effective-path'] = epath
+ else:
+ options['effective-path'] = path + timestamp()
+ else:
+ options['effective-path'] = path + timestamp()
+
+ if not options.get('clean', 'auto') in ('auto', 'yes', 'no'):
+ raise zc.buildout.UserError(
+ 'clean must be one of "auto", "yes", or "no"')
+
+
+ def install(self):
+ options = self.options
+ connection = options.get('zookeeper', '127.0.0.1:2181')
+ zk = zc.zk.ZooKeeper(connection)
+ location = options['location']
+ path = options['effective-path']
+ base, name = path.rsplit('/', 1)
+ if base:
+ zk.create_recursive(base, '', zc.zk.OPEN_ACL_UNSAFE)
+ zk.import_tree(
+ '/'+name+'\n buildout:location = %r\n ' % location +
+ textwrap.dedent(
+ self.options.get('import-text', '')
+ ).replace('\n', '\n '),
+ base, trim=True)
+
+ with open(location, 'w') as f:
+ f.write(path)
+
+ clean = options.get('clean', 'auto')
+ if (clean == 'yes' or
+ (clean == 'auto' and (
+ connection.startswith('localhost:')
+ or
+ connection.startswith('127.')
+ )
+ )
+ ):
+ self.clean(zk)
+
+ return ()
+
+ update = install
+
+ def clean(self, zk):
+ for name in zk.get_children('/'):
+ if name == 'zookeeper':
+ continue
+ self._clean(zk, '/'+name)
+
+ def _clean(self, zk, path):
+ location = zk.get_properties(path).get('buildout:location')
+ if location is not None:
+ if not os.path.exists(location) or readfile(location) != path:
+ pids = []
+ for spath in zk.walk(path):
+ if spath != path:
+ if zk.is_ephemeral(spath):
+ pid = zk.get_properties(spath).get('pid')
+ if pid:
+ pids.append(pid)
+ zk.delete_recursive(path, force=True)
+ for pid in pids:
+ try:
+ os.kill(pid, signal.SIGTERM)
+ except OSError:
+ logger.warn('Failed to kill')
+ else:
+ for name in zk.get_children(path):
+ self._clean(zk, path + '/' + name)
+
+def readfile(path):
+ with open(path) as f:
+ return f.read()
Property changes on: Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/__init__.py
___________________________________________________________________
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
Added: Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/tests.py
===================================================================
--- Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/tests.py (rev 0)
+++ Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/tests.py 2012-01-27 21:57:17 UTC (rev 124224)
@@ -0,0 +1,52 @@
+##############################################################################
+#
+# Copyright (c) 2010 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+import manuel.capture
+import manuel.doctest
+import manuel.testing
+import mock
+import os
+import re
+import unittest
+import zc.zk.testing
+import zope.testing.setupstack
+import zope.testing.renormalizing
+
+def setUp(test):
+ zope.testing.setupstack.setUpDirectory(test)
+ os.mkdir('testdirectory')
+ os.chdir('testdirectory')
+ os.mkdir('parts')
+ zc.zk.testing.setUp(test, tree='/zookeeper\n buildout:path="/xxxxxxxxx"')
+ test.globs['ZooKeeper'].connection_strings.add('127.0.0.1:2181')
+ test.globs['ZooKeeper'].connection_strings.add('localhost:2181')
+
+
+def tearDown(test):
+ zc.zk.testing.tearDown(test)
+ zope.testing.setupstack.tearDown(test)
+
+def test_suite():
+ checker = zope.testing.renormalizing.RENormalizing([
+ (re.compile(r'(/\w+)+/testdirectory/'), '/testdirectory/'),
+ # (re.compile(r''), ''),
+ # (re.compile(r''), ''),
+ ])
+ return unittest.TestSuite((
+ manuel.testing.TestSuite(
+ manuel.doctest.Manuel(checker=checker) + manuel.capture.Manuel(),
+ 'README.txt',
+ setUp=setUp, tearDown=tearDown,
+ ),
+ ))
+
Property changes on: Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/tests.py
___________________________________________________________________
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
More information about the checkins
mailing list