From fc9b0cbb7fe718cbbd63e3b51839b90b3e558037 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 26 Oct 2020 06:13:38 -0400 Subject: stabilize sets that are converted to lists The order of elements in sets cannot be relied upon, because the hash values are randomized by Python. Whenever sets are converted to lists we need to keep their order stable, or random changes in the command line cause ninja to rebuild a lot of files unnecessarily. To stabilize them, use either sort or OrderedSet. Sorting is not always applicable, but it can be faster because it's done in C and it can produce slightly nicer output. --- mesonbuild/depfile.py | 2 +- mesonbuild/modules/sourceset.py | 4 ++-- run_unittests.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mesonbuild/depfile.py b/mesonbuild/depfile.py index 7a896cd..62cbe81 100644 --- a/mesonbuild/depfile.py +++ b/mesonbuild/depfile.py @@ -82,4 +82,4 @@ class DepFile: deps.update(target.deps) for dep in target.deps: deps.update(self.get_all_dependencies(dep, visited)) - return deps + return sorted(deps) diff --git a/mesonbuild/modules/sourceset.py b/mesonbuild/modules/sourceset.py index e23e12e..e49a548 100644 --- a/mesonbuild/modules/sourceset.py +++ b/mesonbuild/modules/sourceset.py @@ -14,7 +14,7 @@ from collections import namedtuple from .. import mesonlib -from ..mesonlib import listify +from ..mesonlib import listify, OrderedSet from . import ExtensionModule from ..interpreterbase import ( noPosargs, noKwargs, permittedKwargs, @@ -111,7 +111,7 @@ class SourceSetHolder(MutableInterpreterObject, ObjectHolder): def collect(self, enabled_fn, all_sources, into=None): if not into: - into = SourceFiles(set(), set()) + into = SourceFiles(OrderedSet(), OrderedSet()) for entry in self.held_object: if all(x.found() for x in entry.dependencies) and \ all(enabled_fn(key) for key in entry.keys): diff --git a/run_unittests.py b/run_unittests.py index 7f7df36..9815058 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -1187,7 +1187,7 @@ class InternalTests(unittest.TestCase): ]: d = mesonbuild.depfile.DepFile(f) deps = d.get_all_dependencies(target) - self.assertEqual(deps, expdeps) + self.assertEqual(sorted(deps), sorted(expdeps)) def test_log_once(self): f = io.StringIO() -- cgit v1.1 From 4caa0577ecfdc9b4d7101cfb07154384fdd191a9 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 26 Oct 2020 06:14:54 -0400 Subject: stabilize iteration order for dictionaries The order of keys in dictionaries cannot be relied upon, because the hash values are randomized by Python. Whenever we iterate on dictionaries and meson.build generates a list during the iteration, the different iteration orders may cause random changes in the command line and cause ninja to rebuild a lot of files unnecessarily. --- mesonbuild/interpreterbase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mesonbuild/interpreterbase.py b/mesonbuild/interpreterbase.py index d3f8181..e57580b 100644 --- a/mesonbuild/interpreterbase.py +++ b/mesonbuild/interpreterbase.py @@ -823,7 +823,7 @@ The result of this is undefined and will become a hard error in a future Meson r elif isinstance(items, dict): if len(node.varnames) != 2: raise InvalidArguments('Foreach on dict unpacks key and value') - for key, value in items.items(): + for key, value in sorted(items.items()): self.set_variable(node.varnames[0], key) self.set_variable(node.varnames[1], value) try: @@ -1166,7 +1166,7 @@ The result of this is undefined and will become a hard error in a future Meson r if method_name == 'keys': if len(posargs) != 0: raise InterpreterException('keys() takes no arguments.') - return list(obj.keys()) + return sorted(obj.keys()) raise InterpreterException('Dictionaries do not have a method called "%s".' % method_name) -- cgit v1.1 From 0b2865e8b95ef119271011c5854836589e8866ad Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 26 Oct 2020 06:21:48 -0400 Subject: ninjabackend: stabilize order of dependencies and order-only dependencies These do not go into the command line, and therefore do not matter for the purpose of avoiding unnecessary rebuilds after meson is rerun. However, they complicate the task of finding differences between build lines across meson reruns. So take the easy way out and sort everything after | and ||. With this change, there is absolutely no change in QEMU's 40000-line build.ninja file after meson is rerun. --- mesonbuild/backend/ninjabackend.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index c3c5705..15218c1 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -359,9 +359,9 @@ class NinjaBuildElement: rulename = self.rulename line = 'build {}{}: {} {}'.format(outs, implicit_outs, rulename, ins) if len(self.deps) > 0: - line += ' | ' + ' '.join([ninja_quote(x, True) for x in self.deps]) + line += ' | ' + ' '.join([ninja_quote(x, True) for x in sorted(self.deps)]) if len(self.orderdeps) > 0: - line += ' || ' + ' '.join([ninja_quote(x, True) for x in self.orderdeps]) + line += ' || ' + ' '.join([ninja_quote(x, True) for x in sorted(self.orderdeps)]) line += '\n' # This is the only way I could find to make this work on all # platforms including Windows command shell. Slash is a dir separator -- cgit v1.1