From 52e1b0a3c9097d3fc647785d3cee180de0272a5c Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 28 Mar 2017 12:17:38 +0530 Subject: project tests: DummyExecutor for MSYS2 and OpenBSD Added and tested on MSYS2/MinGW which doesn't implement the required semaphore locks in the multiprocessing module: Traceback (most recent call last): File "C:/msys64/mingw64/lib/python3.5\multiprocessing\synchronize.py", line 29, in from _multiprocessing import SemLock, sem_unlink ImportError: cannot import name 'sem_unlink' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "run_project_tests.py", line 560, in (passing_tests, failing_tests, skipped_tests) = run_tests(all_tests, 'meson-test-run', options.extra_args) File "run_project_tests.py", line 406, in run_tests executor = conc.ProcessPoolExecutor(max_workers=num_workers) File "F:/msys64/mingw64/lib/python3.5\concurrent\futures\process.py", line 390, in __init__ EXTRA_QUEUED_CALLS) File "F:/msys64/mingw64/lib/python3.5\multiprocessing\context.py", line 101, in Queue return Queue(maxsize, ctx=self.get_context()) File "F:/msys64/mingw64/lib/python3.5\multiprocessing\queues.py", line 42, in __init__ self._rlock = ctx.Lock() File "F:/msys64/mingw64/lib/python3.5\multiprocessing\context.py", line 65, in Lock from .synchronize import Lock File "F:/msys64/mingw64/lib/python3.5\multiprocessing\synchronize.py", line 34, in " function, see issue 3770.") ImportError: This platform lacks a functioning sem_open implementation, therefore, the required synchronization primitives needed will not function, see issue 3770. See also: https://bugs.python.org/issue3770 https://github.com/mesonbuild/meson/issues/1323 According to 3770, the same problem also exists on OpenBSD, so this will potentially also be useful there. --- run_project_tests.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) (limited to 'run_project_tests.py') diff --git a/run_project_tests.py b/run_project_tests.py index 3684de5..a487da2 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -34,6 +34,7 @@ import concurrent.futures as conc from mesonbuild.coredata import backendlist + class BuildStep(Enum): configure = 1 build = 2 @@ -41,6 +42,7 @@ class BuildStep(Enum): install = 4 clean = 5 + class TestResult: def __init__(self, msg, step, stdo, stde, mlog, conftime=0, buildtime=0, testtime=0): self.msg = msg @@ -52,6 +54,54 @@ class TestResult: self.buildtime = buildtime self.testtime = testtime +class DummyFuture(conc.Future): + ''' + Dummy Future implementation that executes the provided function when you + ask for the result. Used on platforms where sem_open() is not available: + MSYS2, OpenBSD, etc: https://bugs.python.org/issue3770 + ''' + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def set_function(self, fn, *args, **kwargs): + self.fn = fn + self.fn_args = args + self.fn_kwargs = kwargs + + def result(self, **kwargs): + try: + result = self.fn(*self.fn_args, **self.fn_kwargs) + except BaseException as e: + self.set_exception(e) + else: + self.set_result(result) + return super().result(**kwargs) + + +class DummyExecutor(conc.Executor): + ''' + Dummy single-thread 'concurrent' executor for use on platforms where + sem_open is not available: https://bugs.python.org/issue3770 + ''' + + def __init__(self): + from threading import Lock + self._shutdown = False + self._shutdownLock = Lock() + + def submit(self, fn, *args, **kwargs): + with self._shutdownLock: + if self._shutdown: + raise RuntimeError('Cannot schedule new futures after shutdown') + f = DummyFuture() + f.set_function(fn, *args, **kwargs) + return f + + def shutdown(self, wait=True): + with self._shutdownLock: + self._shutdown = True + + class AutoDeletedDir: def __init__(self, d): self.dir = d @@ -421,7 +471,11 @@ def run_tests(all_tests, log_name_base, extra_args): print('Could not determine number of CPUs due to the following reason:' + str(e)) print('Defaulting to using only one process') num_workers = 1 - executor = conc.ProcessPoolExecutor(max_workers=num_workers) + try: + executor = conc.ProcessPoolExecutor(max_workers=num_workers) + except ImportError: + print('Platform doesn\'t ProcessPoolExecutor, falling back to single-threaded testing\n') + executor = DummyExecutor() for name, test_cases, skipped in all_tests: current_suite = ET.SubElement(junit_root, 'testsuite', {'name': name, 'tests': str(len(test_cases))}) @@ -441,6 +495,7 @@ def run_tests(all_tests, log_name_base, extra_args): result = executor.submit(run_test, skipped, t, extra_args, unity_flags + backend_flags, compile_commands, should_fail) futures.append((testname, t, result)) for (testname, t, result) in futures: + sys.stdout.flush() result = result.result() if result is None or 'MESON_SKIP_TEST' in result.stdo: print('Skipping:', t) -- cgit v1.1 From f4f9272e87a5e21cbc255f47234f31212b7a8338 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 28 Mar 2017 12:55:39 +0530 Subject: project tests: Don't look for PDB files on MinGW/GCC --- run_project_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'run_project_tests.py') diff --git a/run_project_tests.py b/run_project_tests.py index a487da2..28de638 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -244,7 +244,7 @@ def validate_install(srcdir, installdir): # Windows-specific tests check for the existence of installed PDB # files, but common tests do not, for obvious reasons. Ignore any # extra PDB files found. - if fname not in expected and not fname.endswith('.pdb'): + if fname not in expected and not fname.endswith('.pdb') and compiler == 'cl': ret_msg += 'Extra file {0} found.\n'.format(fname) return ret_msg @@ -587,6 +587,7 @@ def generate_pb_static(compiler, object_suffix, static_suffix): return stlibfile def generate_prebuilt(): + global compiler static_suffix = 'a' if shutil.which('cl'): compiler = 'cl' -- cgit v1.1