[Zope3-checkins] SVN: zope.testing/trunk/src/zope/testing/testrunner Added a test for -1 option.

Jim Fulton jim at zope.com
Fri Jun 24 16:39:59 EDT 2005


Log message for revision 30913:
  Added a test for -1 option.
  
  In doing this, I realized that the tests could be made to fail if the
  test runner running the tests used the -1 option.  Added code to reset
  doctest formatting options before running tests, restoring the old
  values afterwards.
  
  Removed entries for the test runner in import-error tracebacks.  This
  was to avoid changeable output (testrunner implementation line
  numbers) but also makes test-module-import errors less cluttered.
  

Changed:
  A   zope.testing/trunk/src/zope/testing/testrunner-ex/sample2/sampletests_1.py
  U   zope.testing/trunk/src/zope/testing/testrunner.py
  U   zope.testing/trunk/src/zope/testing/testrunner.txt

-=-
Added: zope.testing/trunk/src/zope/testing/testrunner-ex/sample2/sampletests_1.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner-ex/sample2/sampletests_1.py	2005-06-24 18:52:48 UTC (rev 30912)
+++ zope.testing/trunk/src/zope/testing/testrunner-ex/sample2/sampletests_1.py	2005-06-24 20:39:59 UTC (rev 30913)
@@ -0,0 +1,29 @@
+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+from zope.testing import doctest
+
+def eek(self):
+    """
+    >>> x = y
+
+    >>> x
+
+
+    >>> z = x + 1
+
+    """
+        
+def test_suite():
+    return doctest.DocTestSuite()


Property changes on: zope.testing/trunk/src/zope/testing/testrunner-ex/sample2/sampletests_1.py
___________________________________________________________________
Name: svn:keywords
   + Id
Name: svn:eol-style
   + native

Modified: zope.testing/trunk/src/zope/testing/testrunner.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner.py	2005-06-24 18:52:48 UTC (rev 30912)
+++ zope.testing/trunk/src/zope/testing/testrunner.py	2005-06-24 20:39:59 UTC (rev 30913)
@@ -34,9 +34,14 @@
 
 
 def run(defaults=None, args=None):
+    # Control reporting flags during run
+    old_reporting_flags = doctest.set_unittest_reportflags(0)
+    
     options = get_options(args, defaults)
     run_with_options(options)
 
+    doctest.set_unittest_reportflags(old_reporting_flags)
+
 def run_with_options(options):
 
     if options.verbose:
@@ -64,6 +69,15 @@
     failures = []
     errors = []
     nlayers = 0
+    import_errors = tests_by_layer_name.pop(None, None)
+
+    if import_errors:
+        print "Test-module import failures:"
+        for error in import_errors:
+            print_traceback("Module: %s\n" % error.module, error.exc_info),
+        print
+            
+    
     if 'unit' in tests_by_layer_name:
         tests = tests_by_layer_name.pop('unit')
         if not options.non_unit:
@@ -118,6 +132,12 @@
         print "Total: %s tests, %s failures, %s errors" % (
             ran, len(failures), len(errors))
 
+    if import_errors:
+        print
+        print "Test-modules with import problems:"
+        for test in import_errors:
+            print "  " + test.module
+            
 
 def run_tests(options, tests, name, failures, errors):
     repeat = options.repeat or 1
@@ -274,18 +294,6 @@
         self.test_width = self.last_width = 0
 
 
-    def _print_traceback(self, msg, exc_info):
-        print
-        print msg
-
-        v = exc_info[1]
-        if isinstance(v, doctest.DocTestFailureException):
-            tb = v.args[0]
-        else:
-            tb = "".join(traceback.format_exception(*exc_info))
-
-        print tb
-
     def stopTest(self, test):
         if self.options.progress:
             sys.stdout.write(' ' * (self.last_width - self.test_width) + "\r")
@@ -312,6 +320,22 @@
 
         sys.stdout.flush()
 
+
+    def _print_traceback(self, msg, exc_info):
+        print_traceback(msg, exc_info)
+
+def print_traceback(msg, exc_info):
+    print
+    print msg
+
+    v = exc_info[1]
+    if isinstance(v, doctest.DocTestFailureException):
+        tb = v.args[0]
+    else:
+        tb = "".join(traceback.format_exception(*exc_info))
+        
+    print tb
+
 def post_mortem(exc_info):
     err = exc_info[1]
     if isinstance(err, (doctest.UnexpectedException, doctest.DocTestFailure)):
@@ -363,7 +387,6 @@
     layer_module, module_layer_name = layer_names[:-1], layer_names[-1]
     return getattr(import_name('.'.join(layer_module)), module_layer_name)
 
-
 def order_by_bases(layers):
     """Order the layers from least to most specific (bottom to top)
     """
@@ -405,6 +428,8 @@
         for possible_suite in suite:
             for r in tests_from_suite(possible_suite, options, level, layer):
                 yield r
+    elif isinstance(suite, StartUpFailure):
+        yield (suite, None)
     else:
         if level <= options.at_level:
             for pat in options.test:
@@ -420,33 +445,30 @@
                 module_name = fpath[len(prefix):-3].replace(os.path.sep, '.')
                 try:
                     module = import_name(module_name)
-                    suite = getattr(module, options.suite_name)()
                 except:
                     suite = StartUpFailure(
-                        options,
-                        "Couldn't get suite for %s" % module_name,
-                        sys.exc_info()
+                        options, module_name,
+                        sys.exc_info()[:2]
+                        + (sys.exc_info()[2].tb_next.tb_next,),
                         )
-
+                else:
+                    try:
+                        suite = getattr(module, options.suite_name)()
+                    except:
+                        suite = StartUpFailure(
+                            options, module_name, sys.exc_info()[:2]+(None,))
+                
                 yield suite
                 break
 
-class StartUpFailure(unittest.TestCase):
+class StartUpFailure:
 
-    def __init__(self, options, message, exc_info):
+    def __init__(self, options, module, exc_info):
         if options.post_mortem:
             post_mortem(exc_info)
-        unittest.TestCase.__init__(self)
-        self.message = message
+        self.module = module
         self.exc_info = exc_info
 
-    def __str__(self):
-        return "Startup failure: %s" % self.message
-
-    def runTest(self):
-        raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
-
-
 def find_test_files(options):
     found = {}
     for f in find_test_files_(options):
@@ -945,6 +967,7 @@
         (re.compile(r'0[.]\d\d\d seconds'), '0.NNN seconds'),
         (re.compile(r'\d+[.]\d\d\d ms'), 'N.NNN ms'),
         (re.compile('( |")[^\n]+testrunner-ex'), r'\1testrunner-ex'),
+        (re.compile('( |")[^\n]+testrunner.py'), r'\1testrunner.py'),
 
         # omit traceback entries for unittest.py or doctest.py from
         # output:

Modified: zope.testing/trunk/src/zope/testing/testrunner.txt
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner.txt	2005-06-24 18:52:48 UTC (rev 30912)
+++ zope.testing/trunk/src/zope/testing/testrunner.txt	2005-06-24 20:39:59 UTC (rev 30913)
@@ -1330,3 +1330,169 @@
        eek (sample2.sampletests_e)
        testrunner-ex/sample2/e.txt
        test (sample2.sampletests_f.Test)
+
+Suppressing multiple doctest errors
+-----------------------------------
+
+Often, when a doctest example fails, the failure will cause later
+examples in the same test to fail.  Each failure is reported:
+
+    >>> sys.argv = 'test --tests-pattern ^sampletests_1$'.split()
+    >>> testrunner.run(defaults) # doctest: +NORMALIZE_WHITESPACE
+    Running unit tests:
+    <BLANKLINE>
+    <BLANKLINE>
+    Failure in test eek (sample2.sampletests_1)
+    Failed doctest test for sample2.sampletests_1.eek
+      File "testrunner-ex/sample2/sampletests_1.py", line 17, in eek
+    <BLANKLINE>
+    ----------------------------------------------------------------------
+    File "testrunner-ex/sample2/sampletests_1.py", line 19, 
+         in sample2.sampletests_1.eek
+    Failed example:
+        x = y
+    Exception raised:
+        Traceback (most recent call last):
+          File "src/zope/testing/doctest.py", line 1256, in __run
+            compileflags, 1) in test.globs
+          File "<doctest sample2.sampletests_1.eek[0]>", line 1, in ?
+            x = y
+        NameError: name 'y' is not defined
+    ----------------------------------------------------------------------
+    File "testrunner-ex/sample2/sampletests_1.py", line 21, 
+         in sample2.sampletests_1.eek
+    Failed example:
+        x
+    Exception raised:
+        Traceback (most recent call last):
+          File "src/zope/testing/doctest.py", line 1256, in __run
+            compileflags, 1) in test.globs
+          File "<doctest sample2.sampletests_1.eek[1]>", line 1, in ?
+            x
+        NameError: name 'x' is not defined
+    ----------------------------------------------------------------------
+    File "testrunner-ex/sample2/sampletests_1.py", line 24, 
+         in sample2.sampletests_1.eek
+    Failed example:
+        z = x + 1
+    Exception raised:
+        Traceback (most recent call last):
+          File "src/zope/testing/doctest.py", line 1256, in __run
+            compileflags, 1) in test.globs
+          File "<doctest sample2.sampletests_1.eek[2]>", line 1, in ?
+            z = x + 1
+        NameError: name 'x' is not defined
+    <BLANKLINE>
+      Ran 1 tests with 1 failures and 0 errors in 0.002 seconds.
+
+This can be a bid confusing, especially when there are enough tests
+that they scroll off a screen.  Often you just want to see the first
+failure.  This can be accomplished with the -1 option (for "just show
+me the first failed example in a doctest" :)
+
+    >>> sys.argv = 'test --tests-pattern ^sampletests_1$ -1'.split()
+    >>> testrunner.run(defaults) # doctest: +NORMALIZE_WHITESPACE
+    Running unit tests:
+    <BLANKLINE>
+    <BLANKLINE>
+    Failure in test eek (sample2.sampletests_1)
+    Failed doctest test for sample2.sampletests_1.eek
+      File "testrunner-ex/sample2/sampletests_1.py", line 17, in eek
+    <BLANKLINE>
+    ----------------------------------------------------------------------
+    File "testrunner-ex/sample2/sampletests_1.py", line 19, 
+         in sample2.sampletests_1.eek
+    Failed example:
+        x = y
+    Exception raised:
+        Traceback (most recent call last):
+          File "src/zope/testing/doctest.py", line 1256, in __run
+            compileflags, 1) in test.globs
+          File "<doctest sample2.sampletests_1.eek[0]>", line 1, in ?
+            x = y
+        NameError: name 'y' is not defined
+    <BLANKLINE>
+      Ran 1 tests with 1 failures and 0 errors in 0.001 seconds.
+
+
+Testing-Module Import Errors
+----------------------------
+
+If there are errors when importing a test module, these errors are
+reported:
+
+    >>> sys.argv = ('test --tests-pattern ^sampletests(f|_i)?$ --layer 1 '
+    ...            ).split()
+    >>> testrunner.run(defaults)
+    ... # doctest: +NORMALIZE_WHITESPACE +REPORT_NDIFF
+    Test-module import failures:
+    <BLANKLINE>
+    Module: sample2.sampletests_i
+    <BLANKLINE>
+      File "testrunner-ex/sample2/sampletests_i.py", line 15
+        importx unittest
+                       ^
+    SyntaxError: invalid syntax
+    <BLANKLINE>
+    <BLANKLINE>
+    Module: sample2.sample21.sampletests_i
+    <BLANKLINE>
+    Traceback (most recent call last):
+      File "testrunner-ex/sample2/sample21/sampletests_i.py", line 15, in ?
+        import zope.testing.huh
+    ImportError: No module named huh
+    <BLANKLINE>
+    <BLANKLINE>
+    Module: sample2.sample22.sampletests_i
+    <BLANKLINE>
+    AttributeError: 'module' object has no attribute 'test_suite'
+    <BLANKLINE>
+    <BLANKLINE>
+    Module: sample2.sample23.sampletests_i
+    <BLANKLINE>
+    Traceback (most recent call last):
+      File "testrunner-ex/sample2/sample23/sampletests_i.py", line 18, in ?
+        class Test(unittest.TestCase):
+      File "testrunner-ex/sample2/sample23/sampletests_i.py", line 23, in Test
+        raise TypeError('eek')
+    TypeError: eek
+    <BLANKLINE>
+    <BLANKLINE>
+    Running samplelayers.Layer1 tests:
+      Set up samplelayers.Layer1 in 0.000 seconds.
+      Ran 9 tests with 0 failures and 0 errors in 0.000 seconds.
+    Running samplelayers.Layer12 tests:
+      Set up samplelayers.Layer12 in 0.000 seconds.
+      Ran 34 tests with 0 failures and 0 errors in 0.007 seconds.
+    Running samplelayers.Layer122 tests:
+      Set up samplelayers.Layer122 in 0.000 seconds.
+      Ran 34 tests with 0 failures and 0 errors in 0.007 seconds.
+    Running samplelayers.Layer121 tests:
+      Tear down samplelayers.Layer122 in 0.000 seconds.
+      Set up samplelayers.Layer121 in 0.000 seconds.
+      Ran 34 tests with 0 failures and 0 errors in 0.007 seconds.
+    Running samplelayers.Layer11 tests:
+      Tear down samplelayers.Layer121 in 0.000 seconds.
+      Tear down samplelayers.Layer12 in 0.000 seconds.
+      Set up samplelayers.Layer11 in 0.000 seconds.
+      Ran 34 tests with 0 failures and 0 errors in 0.007 seconds.
+    Running samplelayers.Layer112 tests:
+      Set up samplelayers.Layerx in 0.000 seconds.
+      Set up samplelayers.Layer112 in 0.000 seconds.
+      Ran 34 tests with 0 failures and 0 errors in 0.007 seconds.
+    Running samplelayers.Layer111 tests:
+      Tear down samplelayers.Layer112 in 0.000 seconds.
+      Set up samplelayers.Layer111 in 0.000 seconds.
+      Ran 34 tests with 0 failures and 0 errors in 0.007 seconds.
+    Tearing down left over layers:
+      Tear down samplelayers.Layer111 in 0.000 seconds.
+      Tear down samplelayers.Layer11 in 0.000 seconds.
+      Tear down samplelayers.Layer1 in 0.000 seconds.
+      Tear down samplelayers.Layerx in 0.000 seconds.
+    Total: 213 tests, 0 failures, 0 errors
+    <BLANKLINE>
+    Test-modules with import problems:
+      sample2.sampletests_i
+      sample2.sample21.sampletests_i
+      sample2.sample22.sampletests_i
+      sample2.sample23.sampletests_i



More information about the Zope3-Checkins mailing list