fix(testing): Set the warning filter around each test

oscarbenjamin · oscarbenjamin · commit 585569682a74 · 2020-05-03T16:01:12.000+01:00
When running tests/doctests with sympy's own test runners a warning
filter is used to turn DeprecationWarning/SymPyDeprecationWarning into
an error. The filter was only set one at the start of running the tests.
This means that if the filter is modified by one test then it will still
be modified when subsequent tests run.

This commit adds a raise_on_deprecated context manager in runtests.py
and uses it to set the warning filter in the test runners. The filter is
set immediately around running the test and is reset before each
individual test. This has picked up on a couple of failures that will
need to be fixed in subsequent commits.
diff --git a/sympy/testing/runtests.py b/sympy/testing/runtests.py
@@ -32,13 +32,14 @@
 import signal
 import stat
 import tempfile
+import warnings
+from contextlib import contextmanager
 
 from sympy.core.cache import clear_cache
 from sympy.core.compatibility import (exec_, PY3, unwrap,
         unicode)
 from sympy.utilities.misc import find_executable
 from sympy.external import import_module
-from sympy.utilities.exceptions import SymPyDeprecationWarning
 
 IS_WINDOWS = (os.name == 'nt')
 ON_TRAVIS = os.getenv('TRAVIS_BUILD_NUMBER', None)
@@ -156,6 +157,21 @@ def setup_pprint():
     interactive_printing.NO_GLOBAL = True
     return use_unicode_prev
 
+
+@contextmanager
+def raise_on_deprecated():
+    """Context manager to make DeprecationWarning raise an error
+
+    This is to catch SymPyDeprecationWarning from library code while running
+    tests and doctests. It is important to use this context manager around
+    each individual test/doctest in case some tests modify the warning
+    filters.
+    """
+    with warnings.catch_warnings():
+        warnings.filterwarnings('error', '.*', DeprecationWarning, module='sympy.*')
+        yield
+
+
 def run_in_subprocess_with_hash_randomization(
         function, function_args=(),
         function_kwargs=None, command=sys.executable,
@@ -537,11 +553,6 @@ def _test(*paths, **kwargs):
                    fast_threshold=fast_threshold,
                    slow_threshold=slow_threshold)
 
-    # Show deprecation warnings
-    import warnings
-    warnings.simplefilter("error", SymPyDeprecationWarning)
-    warnings.filterwarnings('error', '.*', DeprecationWarning, module='sympy.*')
-
     test_files = t.get_test_files('sympy')
 
     not_blacklisted = [f for f in test_files
@@ -786,11 +797,6 @@ def _doctest(*paths, **kwargs):
     from sympy.plotting.plot import unset_show
     unset_show()
 
-    # Show deprecation warnings
-    import warnings
-    warnings.simplefilter("error", SymPyDeprecationWarning)
-    warnings.filterwarnings('error', '.*', DeprecationWarning, module='sympy.*')
-
     r = PyTestReporter(verbose, split=split, colors=colors,\
                        force_colors=force_colors)
     t = SymPyDocTests(r, normal)
@@ -1278,11 +1284,12 @@ def test_file(self, filename, sort=True, timeout=False, slow=False,
             try:
                 if getattr(f, '_slow', False) and not slow:
                     raise Skipped("Slow")
-                if timeout:
-                    self._timeout(f, timeout, fail_on_timeout)
-                else:
-                    random.seed(self._seed)
-                    f()
+                with raise_on_deprecated():
+                    if timeout:
+                        self._timeout(f, timeout, fail_on_timeout)
+                    else:
+                        random.seed(self._seed)
+                        f()
             except KeyboardInterrupt:
                 if getattr(f, '_slow', False):
                     reporter.test_skip("KeyboardInterrupt")
@@ -1828,15 +1835,17 @@ def run(self, test, compileflags=None, out=None, clear_globs=True):
         self.save_linecache_getlines = pdoctest.linecache.getlines
         linecache.getlines = self.__patched_linecache_getlines
 
-        try:
-            test.globs['print_function'] = print_function
-            return self.__run(test, compileflags, out)
-        finally:
-            sys.stdout = save_stdout
-            pdb.set_trace = save_set_trace
-            linecache.getlines = self.save_linecache_getlines
-            if clear_globs:
-                test.globs.clear()
+        # Fail for deprecation warnings
+        with raise_on_deprecated():
+            try:
+                test.globs['print_function'] = print_function
+                return self.__run(test, compileflags, out)
+            finally:
+                sys.stdout = save_stdout
+                pdb.set_trace = save_set_trace
+                linecache.getlines = self.save_linecache_getlines
+                if clear_globs:
+                    test.globs.clear()
 
 
 # We have to override the name mangled methods.
