[Zope3-checkins] SVN: Zope3/trunk/src/zope/app/twisted/ Updated the
twisted support to run the asyncore main loop to make sure
Jim Fulton
jim at zope.com
Wed Dec 14 14:43:17 EST 2005
Log message for revision 40781:
Updated the twisted support to run the asyncore main loop to make sure
that ZEO runs correctly. Note that the tests for this are only run at
run-level 2.
Changed:
A Zope3/trunk/src/zope/app/twisted/asyncore_main_loop.py
U Zope3/trunk/src/zope/app/twisted/main.py
A Zope3/trunk/src/zope/app/twisted/tests/test_asyncore_main_loop.py
A Zope3/trunk/src/zope/app/twisted/tests/test_zeo.py
-=-
Added: Zope3/trunk/src/zope/app/twisted/asyncore_main_loop.py
===================================================================
--- Zope3/trunk/src/zope/app/twisted/asyncore_main_loop.py 2005-12-14 18:49:09 UTC (rev 40780)
+++ Zope3/trunk/src/zope/app/twisted/asyncore_main_loop.py 2005-12-14 19:43:16 UTC (rev 40781)
@@ -0,0 +1,78 @@
+##############################################################################
+#
+# Copyright (c) 2004 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Run the asyncore main loop
+
+This module provides a function that tries to run the asyncore main loop.
+
+If the asyncore socket map is empty when the function is called, then the
+function exits immediately:
+
+ >>> run(None)
+
+If the loop dies due to an exception, then a panic is logged anf the failure
+handler passed is called:
+
+ >>> def failed():
+ ... print "FAILED!"
+
+ >>> import asyncore
+ >>> class BadDispatcher(asyncore.dispatcher):
+ ... _fileno = 42
+ ... def readable(self):
+ ... raise SystemError("I am evil")
+
+ >>> import zope.testing.loggingsupport
+ >>> handler = zope.testing.loggingsupport.InstalledHandler('ZEO.twisted')
+
+ >>> BadDispatcher().add_channel()
+
+ >>> run(failed)
+ FAILED!
+
+ >>> print handler
+ ZEO.twisted CRITICAL
+ The asyncore main loop died unexpectedly!
+
+ >>> print handler.records[0].exc_info[1]
+ I am evil
+
+$Id$
+"""
+
+import logging
+import sys
+import threading
+
+import ThreadedAsync
+logger = logging.getLogger('ZEO.twisted')
+
+def run(onerror):
+ try:
+ ThreadedAsync.loop()
+ except:
+ exc_info = sys.exc_info()
+ logger.critical("The asyncore main loop died unexpectedly!",
+ exc_info = exc_info,
+ )
+ onerror()
+
+def run_in_thread(reactor):
+ thread = threading.Thread(
+ target=run,
+ args=(reactor, ),
+ )
+ thread.setDaemon(True)
+ thread.start()
+
+
Property changes on: Zope3/trunk/src/zope/app/twisted/asyncore_main_loop.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
Modified: Zope3/trunk/src/zope/app/twisted/main.py
===================================================================
--- Zope3/trunk/src/zope/app/twisted/main.py 2005-12-14 18:49:09 UTC (rev 40780)
+++ Zope3/trunk/src/zope/app/twisted/main.py 2005-12-14 19:43:16 UTC (rev 40781)
@@ -33,6 +33,7 @@
import zope.app.appsetup.interfaces
from zope.app import wsgi
from zope.app.twisted import log
+from zope.app.twisted import asyncore_main_loop
CONFIG_FILENAME = "zope.conf"
@@ -79,6 +80,14 @@
c1 = time.clock()
logging.info("Startup time: %.3f sec real, %.3f sec CPU", t1-t0, c1-c0)
+ # Start the ThreadedAsync main loop. This will either end immediately,
+ # or keep going if ZEO is around. We don't actually care which.
+ def failed():
+ global RESTART_ON_SHUTDOWN
+ RESTART_ON_SHUTDOWN = True
+ reactor.callFromThread(reactor.stop)
+ reactor.callWhenRunning(asyncore_main_loop.run_in_thread, failed)
+
reactor.run()
if RESTART_ON_SHUTDOWN:
Added: Zope3/trunk/src/zope/app/twisted/tests/test_asyncore_main_loop.py
===================================================================
--- Zope3/trunk/src/zope/app/twisted/tests/test_asyncore_main_loop.py 2005-12-14 18:49:09 UTC (rev 40780)
+++ Zope3/trunk/src/zope/app/twisted/tests/test_asyncore_main_loop.py 2005-12-14 19:43:16 UTC (rev 40781)
@@ -0,0 +1,38 @@
+##############################################################################
+#
+# Copyright (c) 2004 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""XXX short summary goes here.
+
+$Id$
+"""
+
+import asyncore
+import unittest
+from zope.testing import doctest
+
+def setUp(test):
+ test.globs['saved-socket-map'] = asyncore.socket_map.copy()
+ asyncore.socket_map.clear()
+
+def tearDown(test):
+ test.globs['handler'].uninstall()
+ asyncore.socket_map.clear()
+ asyncore.socket_map.update(test.globs['saved-socket-map'])
+
+def test_suite():
+ return doctest.DocTestSuite('zope.app.twisted.asyncore_main_loop',
+ setUp=setUp, tearDown=tearDown)
+
+if __name__ == '__main__':
+ unittest.main(defaultTest='test_suite')
+
Property changes on: Zope3/trunk/src/zope/app/twisted/tests/test_asyncore_main_loop.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
Added: Zope3/trunk/src/zope/app/twisted/tests/test_zeo.py
===================================================================
--- Zope3/trunk/src/zope/app/twisted/tests/test_zeo.py 2005-12-14 18:49:09 UTC (rev 40780)
+++ Zope3/trunk/src/zope/app/twisted/tests/test_zeo.py 2005-12-14 19:43:16 UTC (rev 40781)
@@ -0,0 +1,364 @@
+##############################################################################
+#
+# Copyright (c) 2004 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+r"""Test that ZEO is handled correctly.
+
+This is a rather evil test that involves setting up a real ZEO server
+ans some clients. We'll borrow some evil infrastructure from ZEO to do
+this.
+
+We'll start by setting up and starting a ZEO server.
+
+ >>> zeo_port = ZEO.tests.testZEO.get_port()
+ >>> zconf = ZEO.tests.forker.ZEOConfig(('', zeo_port))
+ >>> zport, adminaddr, pid, path = ZEO.tests.forker.start_zeo_server(
+ ... '<demostorage 1>\n</demostorage>\n', zconf, zeo_port)
+
+We'll set up and start two Zope servers that are clients of the
+storage server:
+
+ >>> instance_dir = tempfile.mkdtemp('zeo', 'test')
+ >>> inst1 = Instance(instance_dir, '1', zeo_port)
+ >>> inst1.start()
+ >>> inst2 = Instance(instance_dir, '2', zeo_port)
+ >>> inst2.start()
+
+Lets visit the public view of the top folder for both servers.
+Before we make each of these calls, we'll wait for the server to
+come up:
+
+ >>> inst1.wait()
+ >>> print inst1.main_page()
+ <...
+ <table ...
+ <thead>
+ <tr>
+ ...
+ <th>Name</th>
+ <th>Title</th>
+ <th>Created</th>
+ <th>Modified</th>
+ ...
+
+ >>> inst2.wait()
+ >>> print inst2.main_page()
+ <...
+ <table ...
+ <thead>
+ <tr>
+ ...
+ <th>Name</th>
+ <th>Title</th>
+ <th>Created</th>
+ <th>Modified</th>
+ ...
+
+Now, if we add a folder on one server, it should appear on the other:
+
+ >>> from zope.testbrowser.browser import Browser
+ >>> browser = Browser()
+ >>> browser.open(inst1.url + '@@contents.html')
+ >>> browser.getLink('Folder').click()
+ >>> browser.getControl(name='new_value').value = 'newfolder'
+ >>> browser.getControl('Apply').click()
+
+ >>> 'newfolder' in inst1.main_page()
+ True
+
+ >>> 'newfolder' in inst2.main_page()
+ True
+
+Note that we use main_page because testbrowser, unfortunately, asks for
+robots.txt on every request. This defeats the test we are trying to
+do here. Why? The original symptom was that, when a change was made
+on one ZEO client, then the change wouldn't be seen on the other
+client on the first request. Subsequent requests would see the change
+because ZODB syncs the storages at the end of each transaction. The
+request for robots.txt causes the database to sync, which prevents us
+from seeing the bug.
+
+Cleanup:
+
+ >>> browser.mech_browser.close() # TODO: Browser needs close.
+ >>> inst1.stop()
+ >>> inst2.stop()
+ >>> ZEO.tests.forker.shutdown_zeo_server(('localhost', zeo_port))
+ >>> shutil.rmtree(instance_dir)
+
+$Id$
+"""
+import asyncore
+import errno
+import httplib
+import os
+import sys
+import shutil
+import socket
+import tempfile
+import time
+import unittest
+from zope.testing import doctest
+import ZEO.tests.testZEO
+import ZEO.tests.forker
+
+class Instance:
+
+ def __init__(self, dir=None, name=None, zeo_port=1):
+ if dir is None:
+ self.dir = tempfile.mkdtemp('zeo', 'test')
+ else:
+ self.dir = os.path.join(dir, name)
+ os.mkdir(self.dir)
+
+ self.path = sys.path
+ self.python = sys.executable
+ self.config = os.path.join(self.dir, 'zope.conf')
+ self.zeo_port = zeo_port
+ self.port = ZEO.tests.testZEO.get_port()
+ #print >> sys.stderr, 'port', self.port
+ self.socket = os.path.join(self.dir, 'socket')
+ self.z3log = os.path.join(self.dir, 'z3.log')
+ self.accesslog = os.path.join(self.dir, 'access.log')
+ self.sitezcml = os.path.join(self.dir, 'site.zcml')
+ for file in self.files:
+ getattr(self, file)()
+
+ files = 'runzope', 'site_zcml', 'zdaemon_conf', 'zope_conf', 'zopectl'
+
+ def runzope(self):
+ template = """
+ import sys
+ sys.path[:] = %(path)r
+ from zope.app.twisted.main import main
+ main(["-C", %(config)r] + sys.argv[1:])
+ """
+ template = '\n'.join([l.strip() for l in template.split('\n')])
+ mkfile(self.dir, "runzope", template, self.__dict__)
+
+ def site_zcml(self):
+ template = """
+ <configure xmlns="http://namespaces.zope.org/zope">
+
+ <include package="zope.app" />
+ <include package="zope.app.twisted" />
+ <securityPolicy
+ component="zope.security.simplepolicies.PermissiveSecurityPolicy" />
+
+ <unauthenticatedPrincipal
+ id="zope.anybody"
+ title="Unauthenticated User" />
+
+ <principal
+ id="zope.manager"
+ title="Manager"
+ login="jim"
+ password="123"
+ />
+
+ </configure>
+ """
+ mkfile(self.dir, "site.zcml", template, self.__dict__)
+
+ def zdaemon_conf(self):
+ self.runzope = os.path.join(self.dir, 'runzope')
+ template = """
+ <runner>
+ program %(python)s %(runzope)s
+ daemon on
+ socket-name %(socket)s
+ </runner>
+ <eventlog>
+ <logfile>
+ path %(z3log)s
+ </logfile>
+ </eventlog>
+ """
+ mkfile(self.dir, "zdaemon.conf", template, self.__dict__)
+
+ def zope_conf(self):
+ template = """
+ site-definition %(sitezcml)s
+ threads 1
+ <server>
+ type HTTP
+ address localhost:%(port)s
+ </server>
+ <zodb>
+ <zeoclient>
+ server localhost:%(zeo_port)s
+ storage 1
+ cache-size 20MB
+ </zeoclient>
+ </zodb>
+ <accesslog>
+ <logfile>
+ path %(accesslog)s
+ </logfile>
+ </accesslog>
+ <eventlog>
+ <logfile>
+ path %(z3log)s
+ </logfile>
+ </eventlog>
+ """
+ mkfile(self.dir, "zope.conf", template, self.__dict__)
+
+ def zopectl(self):
+ template = """
+ import os, sys, StringIO
+ CONFIG_FILE = os.path.join(%(dir)r, "zdaemon.conf")
+ sys.path[:] = %(path)r
+ import zope.app.twisted.controller
+ zope.app.twisted.controller.INSTANCE_HOME = %(dir)r
+ sys.stdout = StringIO.StringIO()
+ zope.app.twisted.controller.main(["-C", CONFIG_FILE] + sys.argv[1:])
+ """
+ template = '\n'.join([l.strip() for l in template.split('\n')])
+ mkfile(self.dir, "zopectl", template, self.__dict__)
+
+ def start(self):
+ os.spawnv(os.P_NOWAIT, sys.executable,
+ (sys.executable,
+ os.path.join(self.dir, "zopectl"),
+ 'start'),
+ )
+
+ def stop(self):
+ os.spawnv(os.P_WAIT, sys.executable,
+ (sys.executable,
+ os.path.join(self.dir, "zopectl"),
+ 'stop'),
+ )
+
+ def main_page(self):
+ connection = httplib.HTTPConnection('localhost', self.port)
+ connection.request('GET', self.url)
+ response = connection.getresponse()
+ if response.status != 200:
+ raise AssertionError(response.status)
+ body = response.read()
+ connection.close()
+ return body
+
+ def wait(self):
+ addr = 'localhost', self.port
+ for i in range(120):
+ time.sleep(0.25)
+ try:
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.connect(addr)
+ s.close()
+ break
+ except socket.error, e:
+ if e[0] not in (errno.ECONNREFUSED, errno.ECONNRESET):
+ raise
+ s.close()
+
+ url = property(lambda self: 'http://localhost:%d/' % self.port)
+
+def mkfile(dir, name, template, kw):
+ f = open(os.path.join(dir, name), 'w')
+ f.write(template % kw)
+ f.close()
+
+class BadInstance(Instance):
+
+ files = 'runzope', 'site_zcml', 'zope_conf'
+
+ def runzope(self):
+ template = """
+ import sys
+ sys.path[:] = %(path)r
+ from zope.app.twisted.main import main
+ from zope.app.twisted.tests.test_zeo import BadDispatcher
+ BadDispatcher().add_channel()
+ main(["-C", %(config)r] + sys.argv[1:])
+ """
+ template = '\n'.join([l.strip() for l in template.split('\n')])
+ mkfile(self.dir, "runzope", template, self.__dict__)
+
+ def zope_conf(self):
+ template = """
+ site-definition %(sitezcml)s
+ threads 1
+ <server>
+ type HTTP
+ address localhost:%(port)s
+ </server>
+ <zodb>
+ <demostorage>
+ </demostorage>
+ </zodb>
+ <accesslog>
+ <logfile>
+ path %(accesslog)s
+ </logfile>
+ </accesslog>
+ <eventlog>
+ <logfile>
+ path %(z3log)s
+ </logfile>
+ </eventlog>
+ """
+ mkfile(self.dir, "zope.conf", template, self.__dict__)
+
+ def start(self):
+ return os.spawnv(os.P_WAIT, sys.executable,
+ (sys.executable,
+ os.path.join(self.dir, "runzope"),
+ )
+ )
+
+class BadDispatcher(asyncore.dispatcher):
+ _fileno = 42
+ def readable(self):
+ raise SystemError("I am evil")
+
+def test_asyncore_failure_causes_zope_failure():
+ """
+
+A failure of the asyncore mail loop should cause a zope process to fail:
+
+ >>> bad = BadInstance()
+ >>> bad.start()
+ 1
+
+And puts a panic in the event log:
+
+ >>> print open(bad.z3log).read()
+ ------
+ ... CRITICAL ZEO.twisted The asyncore main loop died unexpectedly!
+ Traceback (most recent call last):
+ ...
+ raise SystemError("I am evil")
+ SystemError: I am evil
+ <BLANKLINE>
+
+
+Cleanup:
+
+ >>> shutil.rmtree(bad.dir)
+
+"""
+
+def test_suite():
+ suite = doctest.DocTestSuite(
+ optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+ )
+ suite.level = 2
+ return suite
+
+
+if __name__ == '__main__':
+ unittest.main(defaultTest='test_suite')
+
Property changes on: Zope3/trunk/src/zope/app/twisted/tests/test_zeo.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
More information about the Zope3-Checkins
mailing list