use a more sane check instead of run_custom_lint

Unfortunately, checking for strings without context is exceedingly prone
to false positives, while missing anything that indirectly opens a file.

Python 3.10 has a feature to warn about this though -- and it uses a
runtime check which runs at the same time that the code fails to open
files in the broken Windows locale. Set this up automatically when
running the testsuite.

Sadly, Python's builtin feature to change the warning level, e.g. by
setting EncodingWarning to error at startup, is utterly broken if you
want to limit it to only certain modules. This is tracked in order to be
more efficiently ignored at https://bugs.python.org/issue34624 and
https://github.com/python/cpython/pull/9358

It is also very trigger happy and passing stuff around via environment
variable either messes with the testsuite, or with thirdparty programs
which are implemented in python *such as lots of gnome*, or perhaps
both.

Instead, add runtime code to meson itself, to add a hidden "feature".
In the application source code, running the 'warnings' module, you can
actually get the expected behavior that $PYTHONWARNINGS doesn't have. So
check for a magic testsuite variable every time meson starts up, and if
it does, then go ahead and initialize a warnings filter that makes
EncodingWarning fatal, but *only* when triggered via Meson and not
arbitrary subprocess scripts.

5 files changed, 13 insertions, 86 deletions

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index ee4b1db..4afbc84 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -26,15 +26,6 @@ jobs:
     - run: python -m pip install pylint
     - run: pylint mesonbuild
 
-  custom_lint:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-      with:
-        python-version: '3.x'
-    - run: python ./run_custom_lint.py
-
   mypy:
     runs-on: ubuntu-latest
     steps:
diff --git a/mesonbuild/mesonmain.py b/mesonbuild/mesonmain.py
index 7a4e421..93cb8b0 100644
--- a/mesonbuild/mesonmain.py
+++ b/mesonbuild/mesonmain.py
@@ -220,6 +220,11 @@ def run(original_args, mainfile):
         print('Please update your environment')
         return 1
 
+    if sys.version_info >= (3, 10) and os.environ.get('MESON_RUNNING_IN_PROJECT_TESTS'):
+        # workaround for https://bugs.python.org/issue34624
+        import warnings
+        warnings.filterwarnings('error', category=EncodingWarning, module='mesonbuild')
+
     # Meson gets confused if stdout can't output Unicode, if the
     # locale isn't Unicode, just force stdout to accept it. This tries
     # to emulate enough of PEP 540 to work elsewhere.
diff --git a/run_custom_lint.py b/run_custom_lint.py
deleted file mode 100755
index 89de950..0000000
--- a/run_custom_lint.py
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/usr/bin/env python3
-
-from pathlib import Path
-import typing as T
-
-root = Path(__file__).absolute().parent
-mesonbuild = root / 'mesonbuild'
-
-whitelist = ['mesonbuild/', 'run_', 'ci/', 'tools/', 'docs/']
-
-def check_missing_encoding(lines: T.List[str], path: str) -> int:
-    errors = 0
-    functions = ['read_text', 'write_text', 'open']
-    for num, line in enumerate(lines):
-        for func in functions:
-            l = line
-
-            # Skip ignored lines
-            if '[ignore encoding]' in l:
-                continue
-
-            # Do we have a match?
-            loc = l.find(func + '(')
-            if loc < 0:
-                continue
-            if loc > 0 and ord(l[loc-1].lower()) in [*range(ord('a'), ord('z')), *range(ord('0'), ord('9')), '_']:
-                continue
-            loc += len(func) + 1
-            # Some preprocessign to make parsing easier
-            l = l[loc:]
-            l = l.replace(' ', '')
-            l = l.replace('\t', '')
-            l = l.replace('\n', '')
-            l = l.replace('\'', '"')
-
-            # Parameter begin
-            args = ''
-            b_open = 1
-            while l:
-                c = l[0]
-                l = l[1:]
-                if c == ')':
-                    b_open -= 1
-                if b_open == 0:
-                    break
-                elif b_open == 1:
-                    args += c
-                if c == '(':
-                    b_open += 1
-
-            binary_modes = ['rb', 'br', 'r+b', 'wb', 'bw', 'ab', 'ba']
-            is_binary = any([f'"{x}"' in args for x in binary_modes])
-            if 'encoding=' not in args and not (func == 'open' and is_binary):
-                location = f'\x1b[33;1m[\x1b[0;1m{path}:{num+1}\x1b[33m]\x1b[0m'
-                #print(f'{location:<64}: \x1b[31;1mERROR:\x1b[0m Missing `encoding=` parameter in "{line.strip()}"')
-                print(f'{location:<72}: \x1b[31;1mERROR:\x1b[0m Missing `encoding=` parameter in `{func}` call')
-                errors += 1
-    return errors
-
-def main() -> int:
-    print('Scanning mesonbuild...')
-    errors = 0
-    for i in sorted(root.glob('**/*.py')):
-        raw = i.read_text(encoding='utf-8')
-        lines = raw.splitlines()
-        filename = i.relative_to(root).as_posix()
-
-        if not any([filename.startswith(x) for x in whitelist]):
-            continue
-
-        errors += check_missing_encoding(lines, filename)
-    print(f'Found {errors} errors while scanning mesonbuild')
-    return 0 if errors == 0 else 1
-
-if __name__ == '__main__':
-    raise SystemExit(main())
diff --git a/run_mypy.py b/run_mypy.py
index 2d7232b..d4fde4f 100755
--- a/run_mypy.py
+++ b/run_mypy.py
@@ -56,7 +56,6 @@ modules = [
     'mesonbuild/optinterpreter.py',
     'mesonbuild/programs.py',
 
-    'run_custom_lint.py',
     'run_mypy.py',
     'run_project_tests.py',
     'run_single_test.py',
diff --git a/run_tests.py b/run_tests.py
index 6867ce6..dc7a8f9 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -68,6 +68,14 @@ if NINJA_CMD is not None:
 else:
     raise RuntimeError('Could not find Ninja v1.7 or newer')
 
+# Emulate running meson with -X utf8 by making sure all open() calls have a
+# sane encoding. This should be a python default, but PEP 540 considered it not
+# backwards compatible. Instead, much line noise in diffs to update this, and in
+# python 3.10 we can also make it a warning when absent.
+os.environ['PYTHONWARNDEFAULTENCODING'] = '1'
+# work around https://bugs.python.org/issue34624
+os.environ['MESON_RUNNING_IN_PROJECT_TESTS'] = '1'
+
 def guess_backend(backend_str: str, msbuild_exe: str) -> T.Tuple['Backend', T.List[str]]:
     # Auto-detect backend if unspecified
     backend_flags = []