[Zope3-checkins] SVN: zope.testing/trunk/ Added import checker that warns when re-importing symbols and not

Christian Theune ct at gocept.com
Wed Jan 28 08:55:40 EST 2009


Log message for revision 95313:
  Added import checker that warns when re-importing symbols and not
  getting them from their original location, applying some heuristics to
  filter out regular re-import scenarios.
  

Changed:
  U   zope.testing/trunk/CHANGES.txt
  U   zope.testing/trunk/src/zope/testing/module.txt
  A   zope.testing/trunk/src/zope/testing/testrunner/importcheck.py
  A   zope.testing/trunk/src/zope/testing/testrunner/importcheck.txt
  A   zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/
  A   zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0.py
  A   zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0a.py
  A   zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0b.py
  A   zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/
  A   zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/__init__.py
  A   zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub1.py
  A   zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub2.py
  A   zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/
  A   zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/__init__.py
  A   zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/sub3a.py
  A   zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck2/
  U   zope.testing/trunk/src/zope/testing/testrunner/options.py
  U   zope.testing/trunk/src/zope/testing/testrunner/runner.py
  U   zope.testing/trunk/src/zope/testing/testrunner/testrunner-debugging-layer-setup.test
  U   zope.testing/trunk/src/zope/testing/testrunner/testrunner-edge-cases.txt
  U   zope.testing/trunk/src/zope/testing/testrunner/testrunner-errors.txt
  U   zope.testing/trunk/src/zope/testing/testrunner/tests.py

-=-
Modified: zope.testing/trunk/CHANGES.txt
===================================================================
--- zope.testing/trunk/CHANGES.txt	2009-01-28 13:26:06 UTC (rev 95312)
+++ zope.testing/trunk/CHANGES.txt	2009-01-28 13:55:39 UTC (rev 95313)
@@ -1,6 +1,13 @@
 zope.testing Changelog
 **********************
 
+3.7.2 (unreleased)
+==================
+
+- Added feature that warns about re-imports of symbols that originate
+  from a different location. This allows detecting when classes and
+  functions get moved from one module to another.
+
 3.7.1 (2008-10-17)
 ==================
 

Modified: zope.testing/trunk/src/zope/testing/module.txt
===================================================================
--- zope.testing/trunk/src/zope/testing/module.txt	2009-01-28 13:26:06 UTC (rev 95312)
+++ zope.testing/trunk/src/zope/testing/module.txt	2009-01-28 13:55:39 UTC (rev 95313)
@@ -89,7 +89,7 @@
   >>> import zope.testing.unlikelymodulename
   Traceback (most recent call last):
     ...
-  ImportError: No module named unlikelymodulename
+  ImportError: No module named zope.testing.unlikelymodulename
 
 This only works for packages that already exist::
  

Added: zope.testing/trunk/src/zope/testing/testrunner/importcheck.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importcheck.py	                        (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importcheck.py	2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1,113 @@
+# vim:fileencoding=utf-8
+# Copyright (c) 2008 gocept gmbh & co. kg
+# See also LICENSE.txt
+
+import ihooks
+import os.path
+import sys
+import types
+import zope.testing.testrunner.feature
+import inspect
+
+
+WHITELIST = [('re', 'match', 'sre'),
+             ('os', 'error', 'exceptions')]
+
+
+class IndirectAttributeAccessChecker(types.ModuleType):
+
+    def __init__(self, module):
+        self.__import_checker_module = module
+
+    def __setattr__(self, name, value):
+        if name.startswith('_IndirectAttributeAccessChecker__import_checker_'):
+            object.__setattr__(self, name.replace('_IndirectAttributeAccessChecker', ''), value)
+        else:
+            module = self.__import_checker_module
+            setattr(module, name, value)
+
+    def __getattribute__(self, name):
+        if name.startswith('_IndirectAttributeAccessChecker__import_checker_'):
+            return object.__getattribute__(self, name.replace('_IndirectAttributeAccessChecker', ''))
+        module = self.__import_checker_module
+        attr = getattr(module, name)
+        if getattr(attr, '__module__', None) is None:
+            return attr
+        if attr.__module__ != module.__name__:
+            import_mod, real_mod = module.__name__, attr.__module__
+            if (import_mod, name, real_mod) in WHITELIST:
+                # No warning for things on the whitelist.
+                pass
+            elif import_mod == 'sys' and name in ['stdin', 'stdout']:
+                # Those are redirected regularly.
+                pass
+            elif import_mod == 'os' and real_mod == 'posix':
+                pass
+            elif name == '__class__' and real_mod == '__builtin__':
+                pass
+            elif (import_mod == 'types' or
+                  import_mod.startswith('xml.dom')) and real_mod == '__builtin__':
+                # No warning for the types from the type module that
+                # really originate from builtins
+                pass
+            elif import_mod.split('.') == real_mod.split('.')[:-1]:
+                # Don't warn if the package of a module re-exports a
+                # symbol.
+                pass
+            elif real_mod in sys.modules and hasattr(sys.modules[real_mod], '__file__') and not sys.modules[real_mod].__file__.endswith('.py'):
+                # The real module seems to be a C-Module which is
+                # regularly masked using another Python module in front
+                # of it.
+                pass
+            elif real_mod.split('.')[-1] == '_' + import_mod.split('.')[-1]:
+                # Looks like an internal module, prefixed with _ was
+                # exported to the same module name without an _
+                pass
+            elif real_mod.startswith('_') and real_mod == import_mod.split('.')[-1]:
+                # Looks like a C-module which doesn't declare its
+                # package path correctly.
+                pass
+            else:
+                attr_type = type(attr).__name__
+                print ("WARNING: indirect import of %s `%s.%s` (originally defined at `%s`)"
+                        % (attr_type, import_mod, name, real_mod))
+                frame = sys._getframe(1)
+                file = frame.f_code.co_filename
+                line = frame.f_lineno
+                print "caused at %s:%s" % (file, line)
+        return attr
+
+
+class IndirectImportWarner(ihooks.ModuleImporter):
+
+    def import_module(self, name, globals=None, locals=None,
+            fromlist=None):
+        result = ihooks.ModuleImporter.import_module(
+            self, name, globals=globals, locals=locals, fromlist=fromlist)
+        checker = IndirectAttributeAccessChecker(result)
+        if not hasattr(result, '__all__'):
+            checker.__all__ = [x for x in dir(result) if not
+                    x.startswith('_')]
+        return checker
+
+    def import_it(self, partname, fqname, parent, force_load=0):
+        result = ihooks.ModuleImporter.import_it(self, partname, fqname,
+                parent, force_load)
+        if result is not None:
+            if hasattr(result, '__file__') and not '.' in os.path.basename(result.__file__):
+                # Smells like a package which didn't get __init__.py
+                # attached to its path.
+                result.__file__ = os.path.join(result.__file__, '__init__.py')
+        return result
+
+
+class ImportChecker(zope.testing.testrunner.feature.Feature):
+    """Monitor indirect imports and warn about them."""
+
+    active = True
+
+    def global_setup(self):
+        ihooks.install(IndirectImportWarner())
+
+    def global_teardown(self):
+        ihooks.uninstall()


Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importcheck.py
___________________________________________________________________
Added: svn:eol-style
   + native

Added: zope.testing/trunk/src/zope/testing/testrunner/importcheck.txt
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importcheck.txt	                        (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importcheck.txt	2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1,107 @@
+=============
+Import checks
+=============
+
+Import checks are intended to help finding imports of classes and
+functions that have been moved and are still referred to by their old
+location. We usually re-import them at their old location to keep code
+working, but for cleaning up, it's nice to get a heads-up where we still
+refer to the old locations.
+
+Note: Only objects that support `__module__` can be checked for indirect
+reports. This is especially *not* true for instances.
+
+Test-runner integration
+=======================
+
+This test (as it is a test for the test runner) assumes that the test
+runner has the import check enabled. We extend Python's search path to
+include some example modules that trigger the import checker:
+
+>>> import sys, os.path
+>>> import zope.testing.testrunner
+>>> sys.path.insert(0, os.path.join(os.path.dirname(
+...     zope.testing.testrunner.__file__), 'importchecker-fixtures'))
+
+When importing a function or a class from a module that isn't its
+original place of definition, a warning will be printed:
+
+>>> import impcheck0
+>>> import impcheck0a
+>>> import impcheck0b
+WARNING: indirect import of type `impcheck0a.X` (originally defined at `impcheck0`)
+caused at ...impcheck0b.py:NNN
+WARNING: indirect import of function `impcheck0a.y` (originally defined at `impcheck0`)
+caused at ...impcheck0b.py:NNN
+
+
+Heuristics for ignoring re-imports
+==================================
+
+Indirect imports in the standard library
+----------------------------------------
+
+The standard library uses reimports to hide some lower level modules which is
+ok for us.
+
+The types in the `type` module come originally from `__builtin__`. This is ok
+for us and we ignore that:
+
+>>> from types import NoneType
+>>> NoneType.__module__
+'__builtin__'
+
+The `os` module tends to reimport from locations like `posix`:
+
+>>> from os import unlink
+>>> unlink.__module__
+'posix'
+
+The `re` module imports its content from the `sre` module:
+
+>>> from re import match
+
+
+Indirect imports due to the testing environment
+-----------------------------------------------
+
+The test runner and `doctest` redirect the stdout temporarily, so sys.stdout
+can be ignored as well:
+
+>>> from sys import stdout
+>>> stdout.__module__
+'zope.testing.doctest'
+
+
+Indirect imports from packages and their sub-modules
+----------------------------------------------------
+
+We allow indirect imports for the sake of providing an API within a package
+that hides its (sub-)modules. E.g. the following structure is ok, as X is
+defined in module `impcheck1.sub1` and re-imported from `impcheck1`:
+
+>>> from impcheck1 import X
+>>> from impcheck1.sub1 import X
+
+
+However, we do not allow re-imports from sibling modules:
+
+>>> from impcheck1.sub2 import X
+WARNING: indirect import of type `impcheck1.sub2.X` (originally defined at `impcheck1.sub1`)
+caused at <doctest importcheck.txt[15]>:1
+
+
+Also, we only allow one level of re-importing via modules:
+
+>>> from impcheck1 import Y
+WARNING: indirect import of type `impcheck1.Y` (originally defined at `impcheck1.sub3.sub3a`)
+caused at <doctest importcheck.txt[16]>:1
+
+
+Indirect imports from C-modules
+-------------------------------
+
+Often there are indirect imports from modules that are written in C to provide
+a nicer API for it. We ignore symbols that are re-imported from C:
+
+>>> from curses import setupterm


Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importcheck.txt
___________________________________________________________________
Added: svn:eol-style
   + native

Added: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0.py	                        (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0.py	2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1,6 @@
+
+def y():
+    pass
+
+class X(object):
+    pass


Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0.py
___________________________________________________________________
Added: svn:eol-style
   + native

Added: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0a.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0a.py	                        (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0a.py	2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1,3 @@
+# This import is ok, because X originated in impcheck0
+from impcheck0 import X, y
+


Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0a.py
___________________________________________________________________
Added: svn:eol-style
   + native

Added: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0b.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0b.py	                        (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0b.py	2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1,2 @@
+# This import gives a warning as X originally came from impcheck0
+from impcheck0a import X, y


Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck0b.py
___________________________________________________________________
Added: svn:eol-style
   + native

Added: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/__init__.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/__init__.py	                        (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/__init__.py	2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1,3 @@
+# Make this a Python package
+from impcheck1.sub1 import X
+from impcheck1.sub3.sub3a import Y


Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/__init__.py
___________________________________________________________________
Added: svn:eol-style
   + native

Added: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub1.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub1.py	                        (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub1.py	2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1,2 @@
+class X(object):
+    pass


Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub1.py
___________________________________________________________________
Added: svn:eol-style
   + native

Added: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub2.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub2.py	                        (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub2.py	2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1 @@
+from impcheck1.sub1 import X


Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub2.py
___________________________________________________________________
Added: svn:eol-style
   + native

Added: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/__init__.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/__init__.py	                        (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/__init__.py	2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1 @@
+# Make this a Python package


Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/__init__.py
___________________________________________________________________
Added: svn:eol-style
   + native

Added: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/sub3a.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/sub3a.py	                        (rev 0)
+++ zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/sub3a.py	2009-01-28 13:55:39 UTC (rev 95313)
@@ -0,0 +1,2 @@
+class Y(object):
+    pass


Property changes on: zope.testing/trunk/src/zope/testing/testrunner/importchecker-fixtures/impcheck1/sub3/sub3a.py
___________________________________________________________________
Added: svn:eol-style
   + native

Modified: zope.testing/trunk/src/zope/testing/testrunner/options.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/options.py	2009-01-28 13:26:06 UTC (rev 95312)
+++ zope.testing/trunk/src/zope/testing/testrunner/options.py	2009-01-28 13:55:39 UTC (rev 95313)
@@ -517,6 +517,7 @@
         args = sys.argv
 
     options, positional = parser.parse_args(args[1:], defaults)
+
     options.original_testrunner_args = args
 
     if options.color:

Modified: zope.testing/trunk/src/zope/testing/testrunner/runner.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/runner.py	2009-01-28 13:26:06 UTC (rev 95312)
+++ zope.testing/trunk/src/zope/testing/testrunner/runner.py	2009-01-28 13:55:39 UTC (rev 95313)
@@ -46,7 +46,9 @@
 import zope.testing.testrunner.subprocess
 import zope.testing.testrunner.interfaces
 import zope.testing.testrunner.debug
+import zope.testing.testrunner.importcheck
 
+
 PYREFCOUNT_PATTERN = re.compile('\[[0-9]+ refs\]')
 
 is_jython = sys.platform.startswith('java')
@@ -191,6 +193,7 @@
         self.features.append(zope.testing.testrunner.filter.Filter(self))
         self.features.append(zope.testing.testrunner.listing.Listing(self))
         self.features.append(zope.testing.testrunner.statistics.Statistics(self))
+        self.features.append(zope.testing.testrunner.importcheck.ImportChecker(self))
 
         # Remove all features that aren't activated
         self.features = [f for f in self.features if f.active]

Modified: zope.testing/trunk/src/zope/testing/testrunner/testrunner-debugging-layer-setup.test
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/testrunner-debugging-layer-setup.test	2009-01-28 13:26:06 UTC (rev 95312)
+++ zope.testing/trunk/src/zope/testing/testrunner/testrunner-debugging-layer-setup.test	2009-01-28 13:55:39 UTC (rev 95313)
@@ -101,7 +101,6 @@
     ...     zope.testing.testrunner.run(
     ...       ['--path', dir, '-Dvv', '--tests-pattern', 'tests2'])
     ... finally: sys.stdin = real_stdin
-    ... # doctest: +ELLIPSIS +REPORT_NDIFF
     Running tests at level 1
     Running tests2.Layer1 tests:
       Set up tests2.Layer1 in 0.000 seconds.

Modified: zope.testing/trunk/src/zope/testing/testrunner/testrunner-edge-cases.txt
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/testrunner-edge-cases.txt	2009-01-28 13:26:06 UTC (rev 95312)
+++ zope.testing/trunk/src/zope/testing/testrunner/testrunner-edge-cases.txt	2009-01-28 13:55:39 UTC (rev 95313)
@@ -28,6 +28,8 @@
     <BLANKLINE>
     Module: sampletestsf
     <BLANKLINE>
+    Traceback (most recent call last):
+    ...
     ImportError: No module named sampletestsf
     ...
 

Modified: zope.testing/trunk/src/zope/testing/testrunner/testrunner-errors.txt
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/testrunner-errors.txt	2009-01-28 13:26:06 UTC (rev 95312)
+++ zope.testing/trunk/src/zope/testing/testrunner/testrunner-errors.txt	2009-01-28 13:55:39 UTC (rev 95313)
@@ -703,11 +703,12 @@
     >>> sys.argv = ('test --tests-pattern ^sampletests(f|_i)?$ --layer 1 '
     ...            ).split()
     >>> testrunner.run(defaults)
-    ... # doctest: +NORMALIZE_WHITESPACE
+    ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
     Test-module import failures:
     <BLANKLINE>
     Module: sample2.sampletests_i
     <BLANKLINE>
+    ...
       File "testrunner-ex/sample2/sampletests_i.py", line 1
         importx unittest
                        ^
@@ -717,9 +718,11 @@
     Module: sample2.sample21.sampletests_i
     <BLANKLINE>
     Traceback (most recent call last):
+      ...
       File "testrunner-ex/sample2/sample21/sampletests_i.py", line 15, in ?
         import zope.testing.huh
-    ImportError: No module named huh
+      ...
+    ImportError: No module named zope.testing.huh
     <BLANKLINE>
     <BLANKLINE>
     Module: sample2.sample22.sampletests_i
@@ -730,6 +733,7 @@
     Module: sample2.sample23.sampletests_i
     <BLANKLINE>
     Traceback (most recent call last):
+      ...
       File "testrunner-ex/sample2/sample23/sampletests_i.py", line 18, in ?
         class Test(unittest.TestCase):
       File "testrunner-ex/sample2/sample23/sampletests_i.py", line 23, in Test

Modified: zope.testing/trunk/src/zope/testing/testrunner/tests.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner/tests.py	2009-01-28 13:26:06 UTC (rev 95312)
+++ zope.testing/trunk/src/zope/testing/testrunner/tests.py	2009-01-28 13:55:39 UTC (rev 95313)
@@ -104,6 +104,7 @@
         'testrunner-repeat.txt',
         'testrunner-gc.txt',
         'testrunner-knit.txt',
+        'importcheck.txt',
         setUp=setUp, tearDown=tearDown,
         optionflags=doctest.ELLIPSIS+doctest.NORMALIZE_WHITESPACE,
         checker=checker),



More information about the Zope3-Checkins mailing list