[Zope3-checkins] SVN: Zope3/trunk/src/zope/app/twisted/ Updated the twisted support to run the asyncore main loop to make sure

Jim Fulton jim at zope.com
Wed Dec 14 14:43:17 EST 2005


Log message for revision 40781:
  Updated the twisted support to run the asyncore main loop to make sure
  that ZEO runs correctly.  Note that the tests for this are only run at
  run-level 2.
  

Changed:
  A   Zope3/trunk/src/zope/app/twisted/asyncore_main_loop.py
  U   Zope3/trunk/src/zope/app/twisted/main.py
  A   Zope3/trunk/src/zope/app/twisted/tests/test_asyncore_main_loop.py
  A   Zope3/trunk/src/zope/app/twisted/tests/test_zeo.py

-=-
Added: Zope3/trunk/src/zope/app/twisted/asyncore_main_loop.py
===================================================================
--- Zope3/trunk/src/zope/app/twisted/asyncore_main_loop.py	2005-12-14 18:49:09 UTC (rev 40780)
+++ Zope3/trunk/src/zope/app/twisted/asyncore_main_loop.py	2005-12-14 19:43:16 UTC (rev 40781)
@@ -0,0 +1,78 @@
+##############################################################################
+#
+# Copyright (c) 2004 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Run the asyncore main loop
+
+This module provides a function that tries to run the asyncore main loop.
+
+If the asyncore socket map is empty when the function is called, then the
+function exits immediately:
+
+    >>> run(None)
+
+If the loop dies due to an exception, then a panic is logged anf the failure
+handler passed is called:
+
+    >>> def failed():
+    ...     print "FAILED!"
+
+    >>> import asyncore
+    >>> class BadDispatcher(asyncore.dispatcher):
+    ...     _fileno = 42
+    ...     def readable(self):
+    ...         raise SystemError("I am evil")
+
+    >>> import zope.testing.loggingsupport
+    >>> handler = zope.testing.loggingsupport.InstalledHandler('ZEO.twisted')
+
+    >>> BadDispatcher().add_channel()
+
+    >>> run(failed)
+    FAILED!
+
+    >>> print handler
+    ZEO.twisted CRITICAL
+      The asyncore main loop died unexpectedly!
+
+    >>> print handler.records[0].exc_info[1]
+    I am evil
+
+$Id$
+"""
+
+import logging
+import sys
+import threading
+
+import ThreadedAsync
+logger = logging.getLogger('ZEO.twisted')
+
+def run(onerror):
+    try:
+        ThreadedAsync.loop()
+    except:
+        exc_info = sys.exc_info()
+        logger.critical("The asyncore main loop died unexpectedly!",
+                        exc_info = exc_info,
+                        )
+        onerror()
+    
+def run_in_thread(reactor):
+    thread = threading.Thread(
+        target=run,
+        args=(reactor, ),
+        )
+    thread.setDaemon(True)
+    thread.start()
+
+        


Property changes on: Zope3/trunk/src/zope/app/twisted/asyncore_main_loop.py
___________________________________________________________________
Name: svn:keywords
   + Id
Name: svn:eol-style
   + native

Modified: Zope3/trunk/src/zope/app/twisted/main.py
===================================================================
--- Zope3/trunk/src/zope/app/twisted/main.py	2005-12-14 18:49:09 UTC (rev 40780)
+++ Zope3/trunk/src/zope/app/twisted/main.py	2005-12-14 19:43:16 UTC (rev 40781)
@@ -33,6 +33,7 @@
 import zope.app.appsetup.interfaces
 from zope.app import wsgi
 from zope.app.twisted import log
+from zope.app.twisted import asyncore_main_loop
 
 CONFIG_FILENAME = "zope.conf"
 
@@ -79,6 +80,14 @@
     c1 = time.clock()
     logging.info("Startup time: %.3f sec real, %.3f sec CPU", t1-t0, c1-c0)
 
+    # Start the ThreadedAsync main loop.  This will either end immediately,
+    # or keep going if ZEO is around.  We don't actually care which.
+    def failed():
+        global RESTART_ON_SHUTDOWN
+        RESTART_ON_SHUTDOWN = True
+        reactor.callFromThread(reactor.stop)
+    reactor.callWhenRunning(asyncore_main_loop.run_in_thread, failed)
+
     reactor.run()
 
     if RESTART_ON_SHUTDOWN:

Added: Zope3/trunk/src/zope/app/twisted/tests/test_asyncore_main_loop.py
===================================================================
--- Zope3/trunk/src/zope/app/twisted/tests/test_asyncore_main_loop.py	2005-12-14 18:49:09 UTC (rev 40780)
+++ Zope3/trunk/src/zope/app/twisted/tests/test_asyncore_main_loop.py	2005-12-14 19:43:16 UTC (rev 40781)
@@ -0,0 +1,38 @@
+##############################################################################
+#
+# Copyright (c) 2004 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""XXX short summary goes here.
+
+$Id$
+"""
+
+import asyncore
+import unittest
+from zope.testing import doctest
+
+def setUp(test):
+    test.globs['saved-socket-map'] = asyncore.socket_map.copy()
+    asyncore.socket_map.clear()
+
+def tearDown(test):
+    test.globs['handler'].uninstall()
+    asyncore.socket_map.clear()
+    asyncore.socket_map.update(test.globs['saved-socket-map'])
+
+def test_suite():
+    return doctest.DocTestSuite('zope.app.twisted.asyncore_main_loop',
+                                setUp=setUp, tearDown=tearDown)
+
+if __name__ == '__main__':
+    unittest.main(defaultTest='test_suite')
+


Property changes on: Zope3/trunk/src/zope/app/twisted/tests/test_asyncore_main_loop.py
___________________________________________________________________
Name: svn:keywords
   + Id
Name: svn:eol-style
   + native

Added: Zope3/trunk/src/zope/app/twisted/tests/test_zeo.py
===================================================================
--- Zope3/trunk/src/zope/app/twisted/tests/test_zeo.py	2005-12-14 18:49:09 UTC (rev 40780)
+++ Zope3/trunk/src/zope/app/twisted/tests/test_zeo.py	2005-12-14 19:43:16 UTC (rev 40781)
@@ -0,0 +1,364 @@
+##############################################################################
+#
+# Copyright (c) 2004 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+r"""Test that ZEO is handled correctly.
+
+This is a rather evil test that involves setting up a real ZEO server
+ans some clients. We'll borrow some evil infrastructure from ZEO to do
+this.
+
+We'll start by setting up and starting a ZEO server.  
+
+    >>> zeo_port = ZEO.tests.testZEO.get_port()
+    >>> zconf = ZEO.tests.forker.ZEOConfig(('', zeo_port))
+    >>> zport, adminaddr, pid, path = ZEO.tests.forker.start_zeo_server(
+    ...     '<demostorage 1>\n</demostorage>\n', zconf, zeo_port)
+
+We'll set up and start two Zope servers that are clients of the
+storage server:
+
+    >>> instance_dir = tempfile.mkdtemp('zeo', 'test')
+    >>> inst1 = Instance(instance_dir, '1', zeo_port)
+    >>> inst1.start()
+    >>> inst2 = Instance(instance_dir, '2', zeo_port)
+    >>> inst2.start()
+
+Lets visit the public view of the top folder for both servers.
+Before we make each of these calls, we'll wait for the server to
+come up:
+
+    >>> inst1.wait()
+    >>> print inst1.main_page()
+    <...
+    <table ...
+        <thead> 
+          <tr>
+            ...
+            <th>Name</th>
+            <th>Title</th>
+            <th>Created</th>
+            <th>Modified</th>
+            ...
+
+    >>> inst2.wait()
+    >>> print inst2.main_page()
+    <...
+    <table ...
+        <thead> 
+          <tr>
+            ...
+            <th>Name</th>
+            <th>Title</th>
+            <th>Created</th>
+            <th>Modified</th>
+            ...
+
+Now, if we add a folder on one server, it should appear on the other:
+
+    >>> from zope.testbrowser.browser import Browser
+    >>> browser = Browser()
+    >>> browser.open(inst1.url + '@@contents.html')
+    >>> browser.getLink('Folder').click()
+    >>> browser.getControl(name='new_value').value = 'newfolder'
+    >>> browser.getControl('Apply').click()
+
+    >>> 'newfolder' in inst1.main_page()
+    True
+
+    >>> 'newfolder' in inst2.main_page()
+    True
+
+Note that we use main_page because testbrowser, unfortunately, asks for
+robots.txt on every request.  This defeats the test we are trying to
+do here. Why?  The original symptom was that, when a change was made
+on one ZEO client, then the change wouldn't be seen on the other
+client on the first request. Subsequent requests would see the change
+because ZODB syncs the storages at the end of each transaction.  The
+request for robots.txt causes the database to sync, which prevents us
+from seeing the bug.
+
+Cleanup:
+
+    >>> browser.mech_browser.close() # TODO: Browser needs close.
+    >>> inst1.stop()
+    >>> inst2.stop()
+    >>> ZEO.tests.forker.shutdown_zeo_server(('localhost', zeo_port))
+    >>> shutil.rmtree(instance_dir)
+
+$Id$
+"""
+import asyncore
+import errno
+import httplib
+import os
+import sys
+import shutil
+import socket
+import tempfile
+import time
+import unittest
+from zope.testing import doctest
+import ZEO.tests.testZEO
+import ZEO.tests.forker
+
+class Instance:
+
+    def __init__(self, dir=None, name=None, zeo_port=1):
+        if dir is None:
+            self.dir = tempfile.mkdtemp('zeo', 'test')
+        else:
+            self.dir = os.path.join(dir, name)
+            os.mkdir(self.dir)
+
+        self.path = sys.path
+        self.python = sys.executable
+        self.config = os.path.join(self.dir, 'zope.conf')
+        self.zeo_port = zeo_port
+        self.port = ZEO.tests.testZEO.get_port()
+        #print >> sys.stderr, 'port', self.port
+        self.socket = os.path.join(self.dir, 'socket')
+        self.z3log = os.path.join(self.dir, 'z3.log')
+        self.accesslog = os.path.join(self.dir, 'access.log')
+        self.sitezcml = os.path.join(self.dir, 'site.zcml')
+        for file in self.files:
+            getattr(self, file)()
+
+    files = 'runzope', 'site_zcml', 'zdaemon_conf', 'zope_conf', 'zopectl'
+
+    def runzope(self):
+        template = """
+        import sys
+        sys.path[:] = %(path)r
+        from zope.app.twisted.main import main
+        main(["-C", %(config)r] + sys.argv[1:])
+        """
+        template = '\n'.join([l.strip() for l in template.split('\n')])
+        mkfile(self.dir, "runzope", template, self.__dict__)
+
+    def site_zcml(self):
+        template = """
+        <configure xmlns="http://namespaces.zope.org/zope">
+
+        <include package="zope.app" />
+        <include package="zope.app.twisted" />
+        <securityPolicy
+            component="zope.security.simplepolicies.PermissiveSecurityPolicy" />
+        
+        <unauthenticatedPrincipal
+            id="zope.anybody"
+            title="Unauthenticated User" />
+        
+        <principal
+            id="zope.manager"
+            title="Manager"
+            login="jim"
+            password="123"
+            />
+        
+        </configure>
+        """
+        mkfile(self.dir, "site.zcml", template, self.__dict__)
+
+    def zdaemon_conf(self):
+        self.runzope = os.path.join(self.dir, 'runzope')
+        template = """
+        <runner>
+          program %(python)s %(runzope)s
+          daemon on
+          socket-name %(socket)s
+        </runner>
+        <eventlog>
+          <logfile>
+            path %(z3log)s
+          </logfile>
+        </eventlog>
+        """
+        mkfile(self.dir, "zdaemon.conf", template, self.__dict__)
+
+    def zope_conf(self):
+        template = """
+        site-definition %(sitezcml)s
+        threads 1
+        <server>
+          type HTTP
+          address localhost:%(port)s
+        </server>
+        <zodb>
+        <zeoclient>
+          server localhost:%(zeo_port)s
+          storage 1
+          cache-size 20MB
+        </zeoclient>
+        </zodb>
+        <accesslog>
+          <logfile>
+            path %(accesslog)s
+          </logfile>
+        </accesslog>
+        <eventlog>
+          <logfile>
+            path %(z3log)s
+          </logfile>
+        </eventlog>
+        """
+        mkfile(self.dir, "zope.conf", template, self.__dict__)
+
+    def zopectl(self):
+        template = """
+        import os, sys, StringIO
+        CONFIG_FILE = os.path.join(%(dir)r, "zdaemon.conf")
+        sys.path[:] = %(path)r
+        import zope.app.twisted.controller
+        zope.app.twisted.controller.INSTANCE_HOME = %(dir)r
+        sys.stdout = StringIO.StringIO()
+        zope.app.twisted.controller.main(["-C", CONFIG_FILE] + sys.argv[1:])
+        """
+        template = '\n'.join([l.strip() for l in template.split('\n')])
+        mkfile(self.dir, "zopectl", template, self.__dict__)
+
+    def start(self):
+        os.spawnv(os.P_NOWAIT, sys.executable,
+                  (sys.executable,
+                   os.path.join(self.dir, "zopectl"),
+                   'start'),
+                  )
+
+    def stop(self):
+        os.spawnv(os.P_WAIT, sys.executable,
+                  (sys.executable,
+                   os.path.join(self.dir, "zopectl"),
+                   'stop'),
+                  )
+
+    def main_page(self):
+        connection = httplib.HTTPConnection('localhost', self.port)
+        connection.request('GET', self.url)
+        response = connection.getresponse()
+        if response.status != 200:
+            raise AssertionError(response.status)
+        body = response.read()
+        connection.close()
+        return body
+
+    def wait(self):
+        addr = 'localhost', self.port
+        for i in range(120):
+            time.sleep(0.25)
+            try:
+                s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                s.connect(addr)
+                s.close()
+                break
+            except socket.error, e:
+                if e[0] not in (errno.ECONNREFUSED, errno.ECONNRESET):
+                    raise
+                s.close()
+
+    url = property(lambda self: 'http://localhost:%d/' % self.port)
+    
+def mkfile(dir, name, template, kw):
+    f = open(os.path.join(dir, name), 'w')
+    f.write(template % kw)
+    f.close()
+
+class BadInstance(Instance):
+
+    files = 'runzope', 'site_zcml', 'zope_conf'
+
+    def runzope(self):
+        template = """
+        import sys
+        sys.path[:] = %(path)r
+        from zope.app.twisted.main import main
+        from zope.app.twisted.tests.test_zeo import BadDispatcher
+        BadDispatcher().add_channel()
+        main(["-C", %(config)r] + sys.argv[1:])
+        """
+        template = '\n'.join([l.strip() for l in template.split('\n')])
+        mkfile(self.dir, "runzope", template, self.__dict__)
+
+    def zope_conf(self):
+        template = """
+        site-definition %(sitezcml)s
+        threads 1
+        <server>
+          type HTTP
+          address localhost:%(port)s
+        </server>
+        <zodb>
+        <demostorage>
+        </demostorage>
+        </zodb>
+        <accesslog>
+          <logfile>
+            path %(accesslog)s
+          </logfile>
+        </accesslog>
+        <eventlog>
+          <logfile>
+            path %(z3log)s
+          </logfile>
+        </eventlog>
+        """
+        mkfile(self.dir, "zope.conf", template, self.__dict__)
+
+    def start(self):
+        return os.spawnv(os.P_WAIT, sys.executable,
+                         (sys.executable,
+                          os.path.join(self.dir, "runzope"),
+                          )
+                         )
+
+class BadDispatcher(asyncore.dispatcher):
+    _fileno = 42
+    def readable(self):
+        raise SystemError("I am evil")
+
+def test_asyncore_failure_causes_zope_failure():
+    """
+
+A failure of the asyncore mail loop should cause a zope process to fail:
+
+    >>> bad = BadInstance()
+    >>> bad.start()
+    1
+
+And puts a panic in the event log:
+
+    >>> print open(bad.z3log).read()
+    ------
+    ... CRITICAL ZEO.twisted The asyncore main loop died unexpectedly!
+    Traceback (most recent call last):
+    ...
+        raise SystemError("I am evil")
+    SystemError: I am evil
+    <BLANKLINE>
+
+
+Cleanup:
+
+    >>> shutil.rmtree(bad.dir)
+
+"""
+        
+def test_suite():
+    suite = doctest.DocTestSuite(
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
+        )
+    suite.level = 2
+    return suite
+
+
+if __name__ == '__main__':
+    unittest.main(defaultTest='test_suite')
+


Property changes on: Zope3/trunk/src/zope/app/twisted/tests/test_zeo.py
___________________________________________________________________
Name: svn:keywords
   + Id
Name: svn:eol-style
   + native



More information about the Zope3-Checkins mailing list