[Zope3-checkins]
SVN: zope.testing/trunk/src/zope/testing/testrunner
Added a test for -1 option.
Jim Fulton
jim at zope.com
Fri Jun 24 16:39:59 EDT 2005
Log message for revision 30913:
Added a test for -1 option.
In doing this, I realized that the tests could be made to fail if the
test runner running the tests used the -1 option. Added code to reset
doctest formatting options before running tests, restoring the old
values afterwards.
Removed entries for the test runner in import-error tracebacks. This
was to avoid changeable output (testrunner implementation line
numbers) but also makes test-module-import errors less cluttered.
Changed:
A zope.testing/trunk/src/zope/testing/testrunner-ex/sample2/sampletests_1.py
U zope.testing/trunk/src/zope/testing/testrunner.py
U zope.testing/trunk/src/zope/testing/testrunner.txt
-=-
Added: zope.testing/trunk/src/zope/testing/testrunner-ex/sample2/sampletests_1.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner-ex/sample2/sampletests_1.py 2005-06-24 18:52:48 UTC (rev 30912)
+++ zope.testing/trunk/src/zope/testing/testrunner-ex/sample2/sampletests_1.py 2005-06-24 20:39:59 UTC (rev 30913)
@@ -0,0 +1,29 @@
+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+from zope.testing import doctest
+
+def eek(self):
+ """
+ >>> x = y
+
+ >>> x
+
+
+ >>> z = x + 1
+
+ """
+
+def test_suite():
+ return doctest.DocTestSuite()
Property changes on: zope.testing/trunk/src/zope/testing/testrunner-ex/sample2/sampletests_1.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
Modified: zope.testing/trunk/src/zope/testing/testrunner.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner.py 2005-06-24 18:52:48 UTC (rev 30912)
+++ zope.testing/trunk/src/zope/testing/testrunner.py 2005-06-24 20:39:59 UTC (rev 30913)
@@ -34,9 +34,14 @@
def run(defaults=None, args=None):
+ # Control reporting flags during run
+ old_reporting_flags = doctest.set_unittest_reportflags(0)
+
options = get_options(args, defaults)
run_with_options(options)
+ doctest.set_unittest_reportflags(old_reporting_flags)
+
def run_with_options(options):
if options.verbose:
@@ -64,6 +69,15 @@
failures = []
errors = []
nlayers = 0
+ import_errors = tests_by_layer_name.pop(None, None)
+
+ if import_errors:
+ print "Test-module import failures:"
+ for error in import_errors:
+ print_traceback("Module: %s\n" % error.module, error.exc_info),
+ print
+
+
if 'unit' in tests_by_layer_name:
tests = tests_by_layer_name.pop('unit')
if not options.non_unit:
@@ -118,6 +132,12 @@
print "Total: %s tests, %s failures, %s errors" % (
ran, len(failures), len(errors))
+ if import_errors:
+ print
+ print "Test-modules with import problems:"
+ for test in import_errors:
+ print " " + test.module
+
def run_tests(options, tests, name, failures, errors):
repeat = options.repeat or 1
@@ -274,18 +294,6 @@
self.test_width = self.last_width = 0
- def _print_traceback(self, msg, exc_info):
- print
- print msg
-
- v = exc_info[1]
- if isinstance(v, doctest.DocTestFailureException):
- tb = v.args[0]
- else:
- tb = "".join(traceback.format_exception(*exc_info))
-
- print tb
-
def stopTest(self, test):
if self.options.progress:
sys.stdout.write(' ' * (self.last_width - self.test_width) + "\r")
@@ -312,6 +320,22 @@
sys.stdout.flush()
+
+ def _print_traceback(self, msg, exc_info):
+ print_traceback(msg, exc_info)
+
+def print_traceback(msg, exc_info):
+ print
+ print msg
+
+ v = exc_info[1]
+ if isinstance(v, doctest.DocTestFailureException):
+ tb = v.args[0]
+ else:
+ tb = "".join(traceback.format_exception(*exc_info))
+
+ print tb
+
def post_mortem(exc_info):
err = exc_info[1]
if isinstance(err, (doctest.UnexpectedException, doctest.DocTestFailure)):
@@ -363,7 +387,6 @@
layer_module, module_layer_name = layer_names[:-1], layer_names[-1]
return getattr(import_name('.'.join(layer_module)), module_layer_name)
-
def order_by_bases(layers):
"""Order the layers from least to most specific (bottom to top)
"""
@@ -405,6 +428,8 @@
for possible_suite in suite:
for r in tests_from_suite(possible_suite, options, level, layer):
yield r
+ elif isinstance(suite, StartUpFailure):
+ yield (suite, None)
else:
if level <= options.at_level:
for pat in options.test:
@@ -420,33 +445,30 @@
module_name = fpath[len(prefix):-3].replace(os.path.sep, '.')
try:
module = import_name(module_name)
- suite = getattr(module, options.suite_name)()
except:
suite = StartUpFailure(
- options,
- "Couldn't get suite for %s" % module_name,
- sys.exc_info()
+ options, module_name,
+ sys.exc_info()[:2]
+ + (sys.exc_info()[2].tb_next.tb_next,),
)
-
+ else:
+ try:
+ suite = getattr(module, options.suite_name)()
+ except:
+ suite = StartUpFailure(
+ options, module_name, sys.exc_info()[:2]+(None,))
+
yield suite
break
-class StartUpFailure(unittest.TestCase):
+class StartUpFailure:
- def __init__(self, options, message, exc_info):
+ def __init__(self, options, module, exc_info):
if options.post_mortem:
post_mortem(exc_info)
- unittest.TestCase.__init__(self)
- self.message = message
+ self.module = module
self.exc_info = exc_info
- def __str__(self):
- return "Startup failure: %s" % self.message
-
- def runTest(self):
- raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
-
-
def find_test_files(options):
found = {}
for f in find_test_files_(options):
@@ -945,6 +967,7 @@
(re.compile(r'0[.]\d\d\d seconds'), '0.NNN seconds'),
(re.compile(r'\d+[.]\d\d\d ms'), 'N.NNN ms'),
(re.compile('( |")[^\n]+testrunner-ex'), r'\1testrunner-ex'),
+ (re.compile('( |")[^\n]+testrunner.py'), r'\1testrunner.py'),
# omit traceback entries for unittest.py or doctest.py from
# output:
Modified: zope.testing/trunk/src/zope/testing/testrunner.txt
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner.txt 2005-06-24 18:52:48 UTC (rev 30912)
+++ zope.testing/trunk/src/zope/testing/testrunner.txt 2005-06-24 20:39:59 UTC (rev 30913)
@@ -1330,3 +1330,169 @@
eek (sample2.sampletests_e)
testrunner-ex/sample2/e.txt
test (sample2.sampletests_f.Test)
+
+Suppressing multiple doctest errors
+-----------------------------------
+
+Often, when a doctest example fails, the failure will cause later
+examples in the same test to fail. Each failure is reported:
+
+ >>> sys.argv = 'test --tests-pattern ^sampletests_1$'.split()
+ >>> testrunner.run(defaults) # doctest: +NORMALIZE_WHITESPACE
+ Running unit tests:
+ <BLANKLINE>
+ <BLANKLINE>
+ Failure in test eek (sample2.sampletests_1)
+ Failed doctest test for sample2.sampletests_1.eek
+ File "testrunner-ex/sample2/sampletests_1.py", line 17, in eek
+ <BLANKLINE>
+ ----------------------------------------------------------------------
+ File "testrunner-ex/sample2/sampletests_1.py", line 19,
+ in sample2.sampletests_1.eek
+ Failed example:
+ x = y
+ Exception raised:
+ Traceback (most recent call last):
+ File "src/zope/testing/doctest.py", line 1256, in __run
+ compileflags, 1) in test.globs
+ File "<doctest sample2.sampletests_1.eek[0]>", line 1, in ?
+ x = y
+ NameError: name 'y' is not defined
+ ----------------------------------------------------------------------
+ File "testrunner-ex/sample2/sampletests_1.py", line 21,
+ in sample2.sampletests_1.eek
+ Failed example:
+ x
+ Exception raised:
+ Traceback (most recent call last):
+ File "src/zope/testing/doctest.py", line 1256, in __run
+ compileflags, 1) in test.globs
+ File "<doctest sample2.sampletests_1.eek[1]>", line 1, in ?
+ x
+ NameError: name 'x' is not defined
+ ----------------------------------------------------------------------
+ File "testrunner-ex/sample2/sampletests_1.py", line 24,
+ in sample2.sampletests_1.eek
+ Failed example:
+ z = x + 1
+ Exception raised:
+ Traceback (most recent call last):
+ File "src/zope/testing/doctest.py", line 1256, in __run
+ compileflags, 1) in test.globs
+ File "<doctest sample2.sampletests_1.eek[2]>", line 1, in ?
+ z = x + 1
+ NameError: name 'x' is not defined
+ <BLANKLINE>
+ Ran 1 tests with 1 failures and 0 errors in 0.002 seconds.
+
+This can be a bid confusing, especially when there are enough tests
+that they scroll off a screen. Often you just want to see the first
+failure. This can be accomplished with the -1 option (for "just show
+me the first failed example in a doctest" :)
+
+ >>> sys.argv = 'test --tests-pattern ^sampletests_1$ -1'.split()
+ >>> testrunner.run(defaults) # doctest: +NORMALIZE_WHITESPACE
+ Running unit tests:
+ <BLANKLINE>
+ <BLANKLINE>
+ Failure in test eek (sample2.sampletests_1)
+ Failed doctest test for sample2.sampletests_1.eek
+ File "testrunner-ex/sample2/sampletests_1.py", line 17, in eek
+ <BLANKLINE>
+ ----------------------------------------------------------------------
+ File "testrunner-ex/sample2/sampletests_1.py", line 19,
+ in sample2.sampletests_1.eek
+ Failed example:
+ x = y
+ Exception raised:
+ Traceback (most recent call last):
+ File "src/zope/testing/doctest.py", line 1256, in __run
+ compileflags, 1) in test.globs
+ File "<doctest sample2.sampletests_1.eek[0]>", line 1, in ?
+ x = y
+ NameError: name 'y' is not defined
+ <BLANKLINE>
+ Ran 1 tests with 1 failures and 0 errors in 0.001 seconds.
+
+
+Testing-Module Import Errors
+----------------------------
+
+If there are errors when importing a test module, these errors are
+reported:
+
+ >>> sys.argv = ('test --tests-pattern ^sampletests(f|_i)?$ --layer 1 '
+ ... ).split()
+ >>> testrunner.run(defaults)
+ ... # doctest: +NORMALIZE_WHITESPACE +REPORT_NDIFF
+ Test-module import failures:
+ <BLANKLINE>
+ Module: sample2.sampletests_i
+ <BLANKLINE>
+ File "testrunner-ex/sample2/sampletests_i.py", line 15
+ importx unittest
+ ^
+ SyntaxError: invalid syntax
+ <BLANKLINE>
+ <BLANKLINE>
+ Module: sample2.sample21.sampletests_i
+ <BLANKLINE>
+ Traceback (most recent call last):
+ File "testrunner-ex/sample2/sample21/sampletests_i.py", line 15, in ?
+ import zope.testing.huh
+ ImportError: No module named huh
+ <BLANKLINE>
+ <BLANKLINE>
+ Module: sample2.sample22.sampletests_i
+ <BLANKLINE>
+ AttributeError: 'module' object has no attribute 'test_suite'
+ <BLANKLINE>
+ <BLANKLINE>
+ Module: sample2.sample23.sampletests_i
+ <BLANKLINE>
+ Traceback (most recent call last):
+ File "testrunner-ex/sample2/sample23/sampletests_i.py", line 18, in ?
+ class Test(unittest.TestCase):
+ File "testrunner-ex/sample2/sample23/sampletests_i.py", line 23, in Test
+ raise TypeError('eek')
+ TypeError: eek
+ <BLANKLINE>
+ <BLANKLINE>
+ Running samplelayers.Layer1 tests:
+ Set up samplelayers.Layer1 in 0.000 seconds.
+ Ran 9 tests with 0 failures and 0 errors in 0.000 seconds.
+ Running samplelayers.Layer12 tests:
+ Set up samplelayers.Layer12 in 0.000 seconds.
+ Ran 34 tests with 0 failures and 0 errors in 0.007 seconds.
+ Running samplelayers.Layer122 tests:
+ Set up samplelayers.Layer122 in 0.000 seconds.
+ Ran 34 tests with 0 failures and 0 errors in 0.007 seconds.
+ Running samplelayers.Layer121 tests:
+ Tear down samplelayers.Layer122 in 0.000 seconds.
+ Set up samplelayers.Layer121 in 0.000 seconds.
+ Ran 34 tests with 0 failures and 0 errors in 0.007 seconds.
+ Running samplelayers.Layer11 tests:
+ Tear down samplelayers.Layer121 in 0.000 seconds.
+ Tear down samplelayers.Layer12 in 0.000 seconds.
+ Set up samplelayers.Layer11 in 0.000 seconds.
+ Ran 34 tests with 0 failures and 0 errors in 0.007 seconds.
+ Running samplelayers.Layer112 tests:
+ Set up samplelayers.Layerx in 0.000 seconds.
+ Set up samplelayers.Layer112 in 0.000 seconds.
+ Ran 34 tests with 0 failures and 0 errors in 0.007 seconds.
+ Running samplelayers.Layer111 tests:
+ Tear down samplelayers.Layer112 in 0.000 seconds.
+ Set up samplelayers.Layer111 in 0.000 seconds.
+ Ran 34 tests with 0 failures and 0 errors in 0.007 seconds.
+ Tearing down left over layers:
+ Tear down samplelayers.Layer111 in 0.000 seconds.
+ Tear down samplelayers.Layer11 in 0.000 seconds.
+ Tear down samplelayers.Layer1 in 0.000 seconds.
+ Tear down samplelayers.Layerx in 0.000 seconds.
+ Total: 213 tests, 0 failures, 0 errors
+ <BLANKLINE>
+ Test-modules with import problems:
+ sample2.sampletests_i
+ sample2.sample21.sampletests_i
+ sample2.sample22.sampletests_i
+ sample2.sample23.sampletests_i
More information about the Zope3-Checkins
mailing list