[Zope3-checkins] SVN: zope.testing/trunk/ Merge sdouche's work on shuffling tests.

Christian Theune ct at gocept.com
Fri Dec 18 03:23:21 EST 2009


Log message for revision 106739:
  Merge sdouche's work on shuffling tests.
  
  Add some more comments, make some of the language clearer.:
  

Changed:
  U   zope.testing/trunk/CHANGES.txt
  U   zope.testing/trunk/src/zope/testing/testrunner/options.py
  U   zope.testing/trunk/src/zope/testing/testrunner/runner.py
  A   zope.testing/trunk/src/zope/testing/testrunner/shuffle.py
  A   zope.testing/trunk/src/zope/testing/testrunner/testrunner-shuffle.txt
  U   zope.testing/trunk/src/zope/testing/testrunner/tests.py

-=-
Modified: zope.testing/trunk/CHANGES.txt
===================================================================
--- zope.testing/trunk/CHANGES.txt	2009-12-18 08:18:39 UTC (rev 106738)
+++ zope.testing/trunk/CHANGES.txt	2009-12-18 08:23:21 UTC (rev 106739)
@@ -9,6 +9,9 @@
 
 - Cleaned up unused imports reported by pyflakes.
 
+- Added two new options to generate randomly ordered list of tests and to
+  select a specific order of tests.
+
 - RENormalizing checkers can be combined via `` + `` now: ``checker1 +
 checker2`` creates a checker with the transformations of both checkers.
 

Modified: zope.testing/trunk/src/zope/testing/testrunner/options.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/options.py	2009-12-18 08:18:39 UTC (rev 106738)
+++ zope.testing/trunk/src/zope/testing/testrunner/options.py	2009-12-18 08:23:21 UTC (rev 106739)
@@ -398,6 +398,19 @@
 Specifies the name of a directory to ignore when looking for tests.
 """)
 
+setup.add_option(
+    '--shuffle', action="store_true", dest='shuffle',
+    help="""\
+Shuffles the order in which tests are ran.
+""")
+
+setup.add_option(
+    '--shuffle-seed', action="store", dest='shuffle_seed', type="int",
+    help="""\
+Value used to initialize the tests shuffler. Specify a value to create
+repeatable random ordered tests.
+""")
+
 parser.add_option_group(setup)
 
 ######################################################################

Modified: zope.testing/trunk/src/zope/testing/testrunner/runner.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/runner.py	2009-12-18 08:18:39 UTC (rev 106738)
+++ zope.testing/trunk/src/zope/testing/testrunner/runner.py	2009-12-18 08:23:21 UTC (rev 106739)
@@ -46,6 +46,7 @@
 import zope.testing.testrunner.interfaces
 import zope.testing.testrunner.debug
 import zope.testing.testrunner.tb_format
+import zope.testing.testrunner.shuffle
 
 
 PYREFCOUNT_PATTERN = re.compile('\[[0-9]+ refs\]')
@@ -188,6 +189,7 @@
                 zope.testing.testrunner.garbagecollection.Debug(self))
 
         self.features.append(zope.testing.testrunner.find.Find(self))
+        self.features.append(zope.testing.testrunner.shuffle.Shuffle(self))
         self.features.append(zope.testing.testrunner.process.SubProcess(self))
         self.features.append(zope.testing.testrunner.filter.Filter(self))
         self.features.append(zope.testing.testrunner.listing.Listing(self))

Copied: zope.testing/trunk/src/zope/testing/testrunner/shuffle.py (from rev 105102, zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/shuffle.py)
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/shuffle.py	                        (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/shuffle.py	2009-12-18 08:23:21 UTC (rev 106739)
@@ -0,0 +1,51 @@
+##############################################################################
+#
+# Copyright (c) 2004-2008 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Shuffle tests discovered before executing them.
+
+$Id$
+"""
+
+import time
+import random
+import unittest
+import zope.testing.testrunner.feature
+
+
+class Shuffle(zope.testing.testrunner.feature.Feature):
+    """Take the tests found so far and shuffle them."""
+
+    def __init__(self, runner):
+        super(Shuffle, self).__init__(runner)
+        self.active = runner.options.shuffle
+        self.seed = runner.options.shuffle_seed
+        if self.seed is None:
+            # We can't rely on the random modules seed initialization because
+            # we can't introspect the seed later for reporting.  This is a
+            # simple emulation of what random.Random.seed does anyway.
+            self.seed = long(time.time() * 256) # use fractional seconds
+
+    def global_setup(self):
+        rng = random.Random(self.seed)
+        for layer, suite in list(self.runner.tests_by_layer_name.items()):
+            # Test suites cannot be modified through a public API.  We thus
+            # take a mutable copy of the list of tests of that suite, shuffle
+            # that and replace the test suite instance with a new one of the
+            # same class.
+            tests = list(suite)
+            rng.shuffle(tests)
+            self.runner.tests_by_layer_name[layer] = suite.__class__(tests)
+
+    def report(self):
+        msg = "Tests were shuffled using seed number %d." % self.seed
+        self.runner.options.output.info(msg)

Copied: zope.testing/trunk/src/zope/testing/testrunner/testrunner-shuffle.txt (from rev 105102, zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/testrunner-shuffle.txt)
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/testrunner-shuffle.txt	                        (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/testrunner-shuffle.txt	2009-12-18 08:23:21 UTC (rev 106739)
@@ -0,0 +1,134 @@
+Shuffling tests
+===============
+
+By default, every time you launch the testrunner it will run the tests in a
+specific order.  However, if you want to ensure that your tests are well
+isolated then running them in a varying order can be helpful. (Proper
+isolation meaning your tests don't depend on each others outcomes or side
+effects and thus the setup and tear down methods are written properly.)
+
+The ``--shuffle`` option tells the test runner to shuffle the list of tests
+before running them:
+
+    >>> import os.path, sys
+    >>> directory_with_tests = os.path.join(this_directory, 'testrunner-ex')
+    >>> defaults = [
+    ...     '--path', directory_with_tests,
+    ...     '--tests-pattern', '^sampletestsf?$',
+    ...     ]
+
+    >>> from zope.testing import testrunner
+    >>> default_argv = 'test -u -m sample1 -t test_y0 --list-tests '
+
+Running shuffled tests
+----------------------
+
+When specifying the ``--shuffle`` option tests are ordered differently each
+time you run the tests:
+
+    >>> argv = (default_argv + '--shuffle').split()
+    >>> testrunner.run_internal(defaults, argv)
+    Tests were shuffled using seed number ...
+    Listing zope.testing.testrunner.layer.UnitTests tests:
+    ...
+    False
+
+Note: The runner prints out a new piece of information which is the seed number used
+to generate the shuffled list of tests. This seed number can later be used to
+re-run the tests in exactly the same order to support debugging.
+
+Specifying a seed number to control tests shuffling
+---------------------------------------------------
+
+Along with the ``--shuffle`` option comes the ``--shuffle-seed`` option which
+takes a seed number as an argument. If you want to reproduce test failures
+that happened during a randomly shuffled test run then simply write down the
+seed number that was printed out and run it again using the ``--shuffle-seed``
+option. The order is guaranteed to be the same.
+
+For example, using the seed number 0 will give us the following, stable, list of
+tests:
+
+    >>> argv = (default_argv + '--shuffle --shuffle-seed 0').split()
+    >>> testrunner.run_internal(defaults, argv)
+    Tests were shuffled using seed number 0.
+    Listing zope.testing.testrunner.layer.UnitTests tests:
+      test_y0 (sample1.sampletestsf.TestA)
+      test_y0 (sample1.sampletests.test_one)
+      test_y0 (sample1.sampletests.test1.TestA)
+      test_y0 (sample1.sampletestsf)
+      test_y0 (sample1.sampletests.test_one.TestA)
+      test_y0 (sample1.sample13.sampletests)
+      test_y0 (sample1.sample13.sampletests.TestA)
+      test_y0 (sample1.sample11.sampletests)
+      test_y0 (sample1.sample11.sampletests.TestA)
+      test_y0 (sample1.sampletests.test1)
+    False
+    >>> testrunner.run_internal(defaults, argv)
+    Tests were shuffled using seed number 0.
+    Listing zope.testing.testrunner.layer.UnitTests tests:
+      test_y0 (sample1.sampletestsf.TestA)
+      test_y0 (sample1.sampletests.test_one)
+      test_y0 (sample1.sampletests.test1.TestA)
+      test_y0 (sample1.sampletestsf)
+      test_y0 (sample1.sampletests.test_one.TestA)
+      test_y0 (sample1.sample13.sampletests)
+      test_y0 (sample1.sample13.sampletests.TestA)
+      test_y0 (sample1.sample11.sampletests)
+      test_y0 (sample1.sample11.sampletests.TestA)
+      test_y0 (sample1.sampletests.test1)
+    False
+
+Whereas using the seed number 42 will give us the following, different but
+stable, list of tests:
+
+    >>> argv = (default_argv + '--shuffle --shuffle-seed 42').split()
+    >>> testrunner.run_internal(defaults, argv)
+    Tests were shuffled using seed number 42.
+    Listing zope.testing.testrunner.layer.UnitTests tests:
+      test_y0 (sample1.sample13.sampletests.TestA)
+      test_y0 (sample1.sample13.sampletests)
+      test_y0 (sample1.sampletests.test1)
+      test_y0 (sample1.sampletests.test1.TestA)
+      test_y0 (sample1.sample11.sampletests.TestA)
+      test_y0 (sample1.sampletestsf)
+      test_y0 (sample1.sampletests.test_one)
+      test_y0 (sample1.sample11.sampletests)
+      test_y0 (sample1.sampletestsf.TestA)
+      test_y0 (sample1.sampletests.test_one.TestA)
+    False
+    >>> testrunner.run_internal(defaults, argv)
+    Tests were shuffled using seed number 42.
+    Listing zope.testing.testrunner.layer.UnitTests tests:
+      test_y0 (sample1.sample13.sampletests.TestA)
+      test_y0 (sample1.sample13.sampletests)
+      test_y0 (sample1.sampletests.test1)
+      test_y0 (sample1.sampletests.test1.TestA)
+      test_y0 (sample1.sample11.sampletests.TestA)
+      test_y0 (sample1.sampletestsf)
+      test_y0 (sample1.sampletests.test_one)
+      test_y0 (sample1.sample11.sampletests)
+      test_y0 (sample1.sampletestsf.TestA)
+      test_y0 (sample1.sampletests.test_one.TestA)
+    False
+
+Selecting a seed number without ``--shuffle``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Note that the ``--shuffle-seed`` option must be used along with ``--shuffle``
+option or tests will not be re-ordered:
+
+    >>> argv = (default_argv + '--shuffle-seed 42').split()
+    >>> testrunner.run_internal(defaults, argv)
+    Listing zope.testing.testrunner.layer.UnitTests tests:
+      test_y0 (sample1.sampletestsf.TestA)
+      test_y0 (sample1.sampletestsf)
+      test_y0 (sample1.sample11.sampletests.TestA)
+      test_y0 (sample1.sample11.sampletests)
+      test_y0 (sample1.sample13.sampletests.TestA)
+      test_y0 (sample1.sample13.sampletests)
+      test_y0 (sample1.sampletests.test1.TestA)
+      test_y0 (sample1.sampletests.test1)
+      test_y0 (sample1.sampletests.test_one.TestA)
+      test_y0 (sample1.sampletests.test_one)
+    False

Modified: zope.testing/trunk/src/zope/testing/testrunner/tests.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/tests.py	2009-12-18 08:18:39 UTC (rev 106738)
+++ zope.testing/trunk/src/zope/testing/testrunner/tests.py	2009-12-18 08:23:21 UTC (rev 106739)
@@ -163,6 +163,7 @@
         'testrunner-repeat.txt',
         'testrunner-gc.txt',
         'testrunner-knit.txt',
+        'testrunner-shuffle.txt',
         setUp=setUp, tearDown=tearDown,
         optionflags=doctest.ELLIPSIS+doctest.NORMALIZE_WHITESPACE,
         checker=checker),



More information about the Zope3-Checkins mailing list