Fix the stubgen test flake. (#4811)

Fix the stubgen test flake.

The core problem is that `importlib.invalidate_caches()` is full of
lies and does not actually invalidate all of the relevant
caches. Python maintains sys.path_importer_cache, which maps from
directory names on the sys.path to "importer" objects for that path
entry. `importlib.invalidate_caches()` invalidates the caches inside
each of the per-directory importer objects, but there is an additional
negative cache that is *not* cleared: sys.path entries whose directory
doesn't exist have their importer set to None, which prevents that
path entry from ever being searched unless it is manually cleared.

This failure comes up rarely, because it only occurs if the following
events occur in sequence:

 1. 'stubgen-test-path' is added to sys.path, but no new import is
    done while there is a 'stubgen-test-path' subdirectory in the working
    directory. This is done by all the stubgen tests that don't end with
    `_import`.
 2. Some non-stubgen test does an import of a previously unimported
    module. This will cause
    sys.path_importer_cache['stubgen-test-path'] to be set to None
    when the directory doesn't exist during the module search.
    This can happen the *first* time that a `_python2` test is run
    since `parse.parse()` dynamically imports `mypy.fastparse2`
    when asked to parse python 2.
 3. A stubgen test tries to use importlib to import a module in
    'stubgen-test-path'. All of the `_import` tests do this.

We fix this by clearing out the relevant entry from
`sys.path_importer_cache` ourselves.

Fixes #4635

Author: msullivan

mypy/stubgen.py

@@ -153,13 +153,6 @@ def find_module_path_and_all(module: str, pyversion: Tuple[int, int],
             try:
                 mod = importlib.import_module(module)
             except Exception:
-                # Print some debugging output that might help diagnose problems.
-                print('=== debug dump follows ===')
-                traceback.print_exc()
-                dump_sys_path()
-                print('PYTHONPATH: %s' % os.getenv("PYTHONPATH"))
-                dump_dir(os.getcwd())
-                print('=== end of debug dump ===')
                 raise CantImport(module)
             if is_c_module(mod):
                 return None
@@ -175,31 +168,6 @@ def find_module_path_and_all(module: str, pyversion: Tuple[int, int],
     return module_path, module_all
 
 
-def dump_sys_path() -> None:
-    print('sys.path:')
-    for entry in sys.path:
-        print('    %r' % entry)
-        if entry in sys.path_importer_cache:
-            x = sys.path_importer_cache[entry]
-            print('      path_importer: %r / %r' % (x, getattr(x, '__dict__', None)))
-            try:
-                print('      stat: %s ' % str(os.stat(entry)))
-            except Exception as e:
-                pass
-
-
-def dump_dir(path: str) -> None:
-    for root, dirs, files in os.walk(os.getcwd()):
-        print('%s:' % root)
-        for d in dirs:
-            print('    %s/' % d)
-        for f in files:
-            path = os.path.join(root, f)
-            print('    %s (%d bytes)' % (f, os.path.getsize(path)))
-        if not dirs and not files:
-            print('    (empty)')
-
-
 def load_python_module_info(module: str, interpreter: str) -> Tuple[str, Optional[List[str]]]:
     """Return tuple (module path, module __all__) for a Python 2 module.
 

mypy/test/teststubgen.py

@@ -1,5 +1,4 @@
 import glob
-import importlib
 import os.path
 import shutil
 import sys
@@ -125,7 +124,7 @@ def test_stubgen(testcase: DataDrivenTestCase) -> None:
         handle.close()
         # Without this we may sometimes be unable to import the module below, as importlib
         # caches os.listdir() results in Python 3.3+ (Guido explained this to me).
-        reset_importlib_caches()
+        reset_importlib_cache('stubgen-test-path')
         try:
             if testcase.name.endswith('_import'):
                 generate_stub_for_module(name, out_dir, quiet=True,
@@ -145,11 +144,16 @@ def test_stubgen(testcase: DataDrivenTestCase) -> None:
         shutil.rmtree(out_dir)
 
 
-def reset_importlib_caches() -> None:
-    try:
-        importlib.invalidate_caches()
-    except (ImportError, AttributeError):
-        pass
+def reset_importlib_cache(entry: str) -> None:
+    # importlib.invalidate_caches() is insufficient, since it doesn't
+    # clear cache entries that indicate that a directory on the path
+    # does not exist, which can cause failures.  Just directly clear
+    # the sys.path_importer_cache entry ourselves.  Other possible
+    # workarounds include always using different paths in the sys.path
+    # (perhaps by using the full path name) or removing the entry from
+    # sys.path after each run.
+    if entry in sys.path_importer_cache:
+        del sys.path_importer_cache[entry]
 
 
 def load_output(dirname: str) -> List[str]: