fix restoring Python state after in-process pytest runs

Now each in-process pytest run saves a snapshot of important global Python
state and restores it after the test completes, including the list of loaded
modules & the Python path settings.

Previously only the loaded package data was getting restored, but that was
also reverting any loaded package changes done in the test triggering the
pytest runs, and not only those done by the pytest runs themselves.

Updated acceptance tests broken by this change, which were only passing before
by accident as they were making multiple pytest runs with later ones depending
on sys.path changes left behind by the initial one.

_pytest/pytester.py

@@ -477,9 +477,6 @@ def preserve_module(name):
             return name.startswith("zope")
         return SysModulesSnapshot(preserve=preserve_module)
 
-    def delete_loaded_modules(self):
-        self._sys_modules_snapshot.restore()
-
     def make_hook_recorder(self, pluginmanager):
         """Create a new :py:class:`HookRecorder` for a PluginManager."""
         assert not hasattr(pluginmanager, "reprec")
@@ -708,42 +705,58 @@ def inline_run(self, *args, **kwargs):
         :return: a :py:class:`HookRecorder` instance
 
         """
-        # When running py.test inline any plugins active in the main test
-        # process are already imported.  So this disables the warning which
-        # will trigger to say they can no longer be rewritten, which is fine as
-        # they have already been rewritten.
-        orig_warn = AssertionRewritingHook._warn_already_imported
-
-        def revert():
-            AssertionRewritingHook._warn_already_imported = orig_warn
-
-        self.request.addfinalizer(revert)
-        AssertionRewritingHook._warn_already_imported = lambda *a: None
-
-        rec = []
-
-        class Collect:
-            def pytest_configure(x, config):
-                rec.append(self.make_hook_recorder(config.pluginmanager))
-
-        plugins = kwargs.get("plugins") or []
-        plugins.append(Collect())
-        ret = pytest.main(list(args), plugins=plugins)
-        self.delete_loaded_modules()
-        if len(rec) == 1:
-            reprec = rec.pop()
-        else:
-            class reprec:
-                pass
-        reprec.ret = ret
-
-        # typically we reraise keyboard interrupts from the child run because
-        # it's our user requesting interruption of the testing
-        if ret == 2 and not kwargs.get("no_reraise_ctrlc"):
-            calls = reprec.getcalls("pytest_keyboard_interrupt")
-            if calls and calls[-1].excinfo.type == KeyboardInterrupt:
-                raise KeyboardInterrupt()
-        return reprec
+        finalizers = []
+        try:
+            # When running py.test inline any plugins active in the main test
+            # process are already imported.  So this disables the warning which
+            # will trigger to say they can no longer be rewritten, which is
+            # fine as they have already been rewritten.
+            orig_warn = AssertionRewritingHook._warn_already_imported
+
+            def revert_warn_already_imported():
+                AssertionRewritingHook._warn_already_imported = orig_warn
+            finalizers.append(revert_warn_already_imported)
+            AssertionRewritingHook._warn_already_imported = lambda *a: None
+
+            # Any sys.module or sys.path changes done while running py.test
+            # inline should be reverted after the test run completes to avoid
+            # clashing with later inline tests run within the same pytest test,
+            # e.g. just because they use matching test module names.
+            finalizers.append(self.__take_sys_modules_snapshot().restore)
+            finalizers.append(SysPathsSnapshot().restore)
+
+            # Important note:
+            # - our tests should not leave any other references/registrations
+            #   laying around other than possibly loaded test modules
+            #   referenced from sys.modules, as nothing will clean those up
+            #   automatically
+
+            rec = []
+
+            class Collect:
+                def pytest_configure(x, config):
+                    rec.append(self.make_hook_recorder(config.pluginmanager))
+
+            plugins = kwargs.get("plugins") or []
+            plugins.append(Collect())
+            ret = pytest.main(list(args), plugins=plugins)
+            if len(rec) == 1:
+                reprec = rec.pop()
+            else:
+                class reprec:
+                    pass
+            reprec.ret = ret
+
+            # typically we reraise keyboard interrupts from the child run
+            # because it's our user requesting interruption of the testing
+            if ret == 2 and not kwargs.get("no_reraise_ctrlc"):
+                calls = reprec.getcalls("pytest_keyboard_interrupt")
+                if calls and calls[-1].excinfo.type == KeyboardInterrupt:
+                    raise KeyboardInterrupt()
+            return reprec
+        finally:
+            for finalizer in finalizers:
+                finalizer()
 
     def runpytest_inprocess(self, *args, **kwargs):
         """Return result of running pytest in-process, providing a similar

testing/acceptance_test.py

@@ -533,7 +533,7 @@ def test_pyargs_importerror(self, testdir, monkeypatch):
         path = testdir.mkpydir("tpkg")
         path.join("test_hello.py").write('raise ImportError')
 
-        result = testdir.runpytest_subprocess("--pyargs", "tpkg.test_hello")
+        result = testdir.runpytest("--pyargs", "tpkg.test_hello", syspathinsert=True)
         assert result.ret != 0
 
         result.stdout.fnmatch_lines([
@@ -551,7 +551,7 @@ def test_cmdline_python_package(self, testdir, monkeypatch):
         result.stdout.fnmatch_lines([
             "*2 passed*"
         ])
-        result = testdir.runpytest("--pyargs", "tpkg.test_hello")
+        result = testdir.runpytest("--pyargs", "tpkg.test_hello", syspathinsert=True)
         assert result.ret == 0
         result.stdout.fnmatch_lines([
             "*1 passed*"
@@ -575,7 +575,7 @@ def join_pythonpath(what):
         ])
 
         monkeypatch.setenv('PYTHONPATH', join_pythonpath(testdir))
-        result = testdir.runpytest("--pyargs", "tpkg.test_missing")
+        result = testdir.runpytest("--pyargs", "tpkg.test_missing", syspathinsert=True)
         assert result.ret != 0
         result.stderr.fnmatch_lines([
             "*not*found*test_missing*",

testing/deprecated_test.py

@@ -58,7 +58,6 @@ def pytest_logwarning(self, message):
             warnings.append(message)
 
     ret = pytest.main("%s -x" % tmpdir, plugins=[Collect()])
-    testdir.delete_loaded_modules()
     msg = ('passing a string to pytest.main() is deprecated, '
            'pass a list of arguments instead.')
     assert msg in warnings