Handle multiple encodings for WSL error messages

EliahKagan · EliahKagan · commit 496acaac5d2c · 2023-11-26T00:27:57.000-05:00
This affects the test suite only. It improves _WinBashStatus.

When bash.exe is the WSL wrapper, and it reports an error that
prevents bash in a WSL system from ever actually being run, the
message may be encoded in a narrow or multibyte encoding, or may
instead be in Windows's native UTF-16LE encoding. This was partly
handled before, by assuming the message indicating the absence of
any installed WSL distribuions could be interpreted as UTF-8, but
matching against bytes and not actually decoding it or other error
messages. That presented a few problems, which are rememedied here:

- It turns out that this "Windows Subsystem for Linux has no
  installed distributions" message actually can be in UTF-16LE too.
  This happens when it is part of a message that also reports a
  textual error code like:

  Bash/Service/CreateInstance/GetDefaultDistro/WSL_E_DEFAULT_DISTRO_NOT_FOUND

  Therefore, narrow-encoding matching was not sufficient.

- Logged messages were hard to understand, because they were
  printing the reprs of bytes objects, which sometimes were
  UTF-16LE and thus had many "\x00" interspersed in them.

- The cases were not broken down elegantly. The ErrorWhileChecking
  case could hold a CompletedProcess, a CalledProcessError, or an
  OSError. This is now replaced with a WinError case for the rare
  scenario where CreateProcessW fails in a completely unexpected
  way, and a CheckError case for when the exit status or output of
  bash.exe indicates an error other than one we want to handle.
  CheckError has two attributes: `process` for the CompletedProcess
  (CalledProcessError is no longer used, and CompletedProcess has
  `returncode` and other attributes that provide the same info),
  and `message` for the decoded output.
diff --git a/test/test_index.py b/test/test_index.py
@@ -61,13 +61,11 @@ class _WinBashStatus:
     WslNoDistro = constructor()
     """Running ``bash.exe` tries to run bash on a WSL distribution, but none exists."""
 
-    ErrorWhileChecking = constructor("error_or_process")
-    """Could not determine the status.
+    CheckError = constructor("process", "message")
+    """Running ``bash.exe`` fails in an unexpected error or gives unexpected output."""
 
-    This should not trigger a skip or xfail, as it typically indicates either a fixable
-    problem on the test machine, such as an "Insufficient system resources exist to
-    complete the requested service" error starting WSL, or a bug in this detection code.
-    """
+    WinError = constructor("exception")
+    """``bash.exe`` may exist but can't run. ``CreateProcessW`` fails unexpectedly."""
 
     @classmethod
     def check(cls):
@@ -94,36 +92,34 @@ def check(cls):
         if os.name != "nt":
             return cls.Inapplicable()
 
-        # Use bytes because messages for different WSL errors use different encodings.
-        no_distro_message = b"Windows Subsystem for Linux has no installed distributions."
-
-        def error_running_bash(error):
-            log.error("Error running bash.exe to check WSL status: %r", error)
-            return cls.ErrorWhileChecking(error)
+        no_distro_message = "Windows Subsystem for Linux has no installed distributions."
 
         try:
             # Output rather than forwarding the test command's exit status so that if a
             # failure occurs before we even get to this point, we will detect it. For
             # information on ways to check for WSL, see https://superuser.com/a/1749811.
             script = 'test -e /proc/sys/fs/binfmt_misc/WSLInterop; echo "$?"'
             command = ["bash.exe", "-c", script]
-            proc = subprocess.run(command, capture_output=True, check=True)
+            process = subprocess.run(command, capture_output=True)
         except FileNotFoundError:
             return cls.Absent()
         except OSError as error:
-            return error_running_bash(error)
-        except subprocess.CalledProcessError as error:
-            if error.returncode == 1 and error.stdout.startswith(no_distro_message):
-                return cls.WslNoDistro()
-            return error_running_bash(error)
-
-        status = proc.stdout.rstrip()
-        if status == b"0":
+            return cls.WinError(error)
+
+        encoding = "utf-16le" if b"\r\0\n\0" in process.stdout else "utf-8"
+        text = process.stdout.decode(encoding).rstrip()  # stdout includes WSL errors.
+
+        if process.returncode == 1 and text.startswith(no_distro_message):
+            return cls.WslNoDistro()
+        if process.returncode != 0:
+            log.error("Error running bash.exe to check WSL status: %s", text)
+            return cls.CheckError(process, text)
+        if text == "0":
             return cls.Wsl()
-        if status == b"1":
+        if text == "1":
             return cls.Native()
-        log.error("Strange output checking WSL status: %r", proc.stdout)
-        return cls.ErrorWhileChecking(proc)
+        log.error("Strange output checking WSL status: %s", text)
+        return cls.CheckError(process, text)
 
 
 _win_bash_status = _WinBashStatus.check()
