Use universal_newlines=True for all Popen calls

Instead of adding it everywhere manually, create a wrapper called
mesonlib.Popen_safe and use that everywhere that we call an executable
and extract its output.

This will also allow us to tweak it to do more/different things if
needed for some locales and/or systems.

Closes #1079

4 files changed, 23 insertions, 32 deletions

diff --git a/mesonbuild/scripts/gtkdochelper.py b/mesonbuild/scripts/gtkdochelper.py
index 14de486..0cfd644 100755
--- a/mesonbuild/scripts/gtkdochelper.py
+++ b/mesonbuild/scripts/gtkdochelper.py
@@ -17,7 +17,7 @@ import sys, os
 import subprocess
 import shutil
 import argparse
-from ..mesonlib import MesonException
+from ..mesonlib import MesonException, Popen_safe
 from . import destdir_join
 
 parser = argparse.ArgumentParser()
@@ -46,15 +46,13 @@ parser.add_argument('--mode', dest='mode', default='')
 parser.add_argument('--installdir', dest='install_dir')
 
 def gtkdoc_run_check(cmd, cwd):
-    p = subprocess.Popen(cmd, cwd=cwd,
-                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (stde, stdo) = p.communicate()
+    # Put stderr into stdout since we want to print it out anyway.
+    # This preserves the order of messages.
+    p, out = Popen_safe(cmd, cwd=cwd, stderr=subprocess.STDOUT)[0:2]
     if p.returncode != 0:
         err_msg = ["{!r} failed with status {:d}".format(cmd[0], p.returncode)]
-        if stde:
-            err_msg.append(stde.decode(errors='ignore'))
-        if stdo:
-            err_msg.append(stdo.decode(errors='ignore'))
+        if out:
+            err_msg.append(out)
         raise MesonException('\n'.join(err_msg))
 
 def build_gtkdoc(source_root, build_root, doc_subdir, src_subdirs,
diff --git a/mesonbuild/scripts/meson_exe.py b/mesonbuild/scripts/meson_exe.py
index 3ea3926..d412e01 100755
--- a/mesonbuild/scripts/meson_exe.py
+++ b/mesonbuild/scripts/meson_exe.py
@@ -21,7 +21,7 @@ import pickle
 import platform
 import subprocess
 
-import mesonbuild
+from ..mesonlib import MesonException, Popen_safe
 
 options = None
 
@@ -45,7 +45,7 @@ def run_exe(exe):
     else:
         if exe.is_cross:
             if exe.exe_runner is None:
-                raise Exception('BUG: Trying to run cross-compiled exes with no wrapper')
+                raise AssertionError('BUG: Trying to run cross-compiled exes with no wrapper')
             else:
                 cmd = [exe.exe_runner] + exe.fname
         else:
@@ -55,17 +55,12 @@ def run_exe(exe):
     if len(exe.extra_paths) > 0:
         child_env['PATH'] = (os.pathsep.join(exe.extra_paths + ['']) +
                              child_env['PATH'])
-    p = subprocess.Popen(cmd + exe.cmd_args,
-                         stdout=subprocess.PIPE,
-                         stderr=subprocess.PIPE,
-                         env=child_env,
-                         cwd=exe.workdir)
-    stdout, stderr = p.communicate()
+    p, stdout, stderr = Popen_safe(cmd + exe.cmd_args, env=child_env, cwd=exe.workdir)
     if exe.capture and p.returncode == 0:
-        with open(exe.capture, 'wb') as output:
+        with open(exe.capture, 'w') as output:
             output.write(stdout)
     if stderr:
-        sys.stderr.buffer.write(stderr)
+        sys.stderr.write(stderr)
     return p.returncode
 
 def run(args):
diff --git a/mesonbuild/scripts/meson_install.py b/mesonbuild/scripts/meson_install.py
index 14539e0..3d22022 100755
--- a/mesonbuild/scripts/meson_install.py
+++ b/mesonbuild/scripts/meson_install.py
@@ -18,6 +18,7 @@ import sys, pickle, os, shutil, subprocess, gzip, platform
 from glob import glob
 from . import depfixer
 from . import destdir_join
+from ..mesonlib import MesonException, Popen_safe
 
 install_log_file = None
 
@@ -205,12 +206,11 @@ def install_targets(d):
         do_copy(fname, outname)
         if should_strip:
             print('Stripping target')
-            ps = subprocess.Popen(['strip', outname], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            (stdo, stde) = ps.communicate()
+            ps, stdo, stde = Popen_safe(['strip', outname])
             if ps.returncode != 0:
                 print('Could not strip file.\n')
-                print('Stdout:\n%s\n' % stdo.decode())
-                print('Stderr:\n%s\n' % stde.decode())
+                print('Stdout:\n%s\n' % stdo)
+                print('Stderr:\n%s\n' % stde)
                 sys.exit(1)
         printed_symlink_error = False
         for alias in aliases:
diff --git a/mesonbuild/scripts/symbolextractor.py b/mesonbuild/scripts/symbolextractor.py
index 9d28028..2df3a0c 100755
--- a/mesonbuild/scripts/symbolextractor.py
+++ b/mesonbuild/scripts/symbolextractor.py
@@ -23,7 +23,8 @@
 # http://cgit.freedesktop.org/libreoffice/core/commit/?id=3213cd54b76bc80a6f0516aac75a48ff3b2ad67c
 
 import os, sys, subprocess
-from mesonbuild import mesonlib
+from .. import mesonlib
+from ..mesonlib import MesonException, Popen_safe
 import argparse
 
 parser = argparse.ArgumentParser()
@@ -59,23 +60,21 @@ def linux_syms(libfilename, outfilename):
         nmbin = os.environ[evar].strip()
     else:
         nmbin = 'nm'
-    pe = subprocess.Popen([readelfbin, '-d', libfilename], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    output = pe.communicate()[0].decode()
+    pe, output = Popen_safe([readelfbin, '-d', libfilename])[0:2]
     if pe.returncode != 0:
         raise RuntimeError('Readelf does not work')
     result = [x for x in output.split('\n') if 'SONAME' in x]
     assert(len(result) <= 1)
-    pnm = subprocess.Popen([nmbin, '--dynamic', '--extern-only', '--defined-only', '--format=posix', libfilename],
-                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    output = pnm.communicate()[0].decode()
+    pnm, output = Popen_safe([nmbin, '--dynamic', '--extern-only',
+                              '--defined-only', '--format=posix',
+                              libfilename])[0:2]
     if pnm.returncode != 0:
         raise RuntimeError('nm does not work.')
     result += [' '.join(x.split()[0:2]) for x in output.split('\n') if len(x) > 0]
     write_if_changed('\n'.join(result) + '\n', outfilename)
 
 def osx_syms(libfilename, outfilename):
-    pe = subprocess.Popen(['otool', '-l', libfilename], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    output = pe.communicate()[0].decode()
+    pe, output = Popen_safe(['otool', '-l', libfilename])[0:2]
     if pe.returncode != 0:
         raise RuntimeError('Otool does not work.')
     arr = output.split('\n')
@@ -84,8 +83,7 @@ def osx_syms(libfilename, outfilename):
             match = i
             break
     result = [arr[match+2], arr[match+5]] # Libreoffice stores all 5 lines but the others seem irrelevant.
-    pnm = subprocess.Popen(['nm', '-g', '-P', libfilename], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    output = pnm.communicate()[0].decode()
+    pnm, output = Popen_safe(['nm', '-g', '-P', libfilename])[0:2]
     if pnm.returncode != 0:
         raise RuntimeError('nm does not work.')
     result += [' '.join(x.split()[0:2]) for x in output.split('\n') if len(x) > 0 and not x.endswith('U')]