Add option to limit maximum number of concurrent link processes.

5 files changed, 84 insertions, 12 deletions

diff --git a/docs/markdown/Release-notes-for-0.42.0.md b/docs/markdown/Release-notes-for-0.42.0.md
index 9b2df63..ff0771f 100644
--- a/docs/markdown/Release-notes-for-0.42.0.md
+++ b/docs/markdown/Release-notes-for-0.42.0.md
@@ -88,3 +88,11 @@ other. This warning will become a hard error in some future release.
 Vulkan can now be used as native dependency. The dependency module will detect 
 the VULKAN_SDK environment variable or otherwise try to receive the vulkan
 library and header via pkgconfig or from the system.
+
+## Limiting the maximum number of linker processes
+
+With the Ninja backend it is now possible to limit the maximum number of
+concurrent linker processes. This is usually only needed for projects
+that have many large link steps that cause the system to run out of
+memory if they are run in parallel. This limit can be set with the
+new `backend_max_links` option.
diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py
index 9048a78..f477a2b 100644
--- a/mesonbuild/backend/ninjabackend.py
+++ b/mesonbuild/backend/ninjabackend.py
@@ -842,6 +842,12 @@ int dummy;
         outfile.write('# Rules for compiling.\n\n')
         self.generate_compile_rules(outfile)
         outfile.write('# Rules for linking.\n\n')
+        num_pools = self.environment.coredata.backend_options['backend_max_links'].value
+        if num_pools > 0:
+            outfile.write('''pool link_pool
+  depth = %d
+
+''' % num_pools)
         if self.environment.is_cross_build():
             self.generate_static_link_rules(True, outfile)
         self.generate_static_link_rules(False, outfile)
@@ -1369,6 +1375,7 @@ int dummy;
             raise MesonException('Swift supports only executable and static library targets.')
 
     def generate_static_link_rules(self, is_cross, outfile):
+        num_pools = self.environment.coredata.backend_options['backend_max_links'].value
         if 'java' in self.build.compilers:
             if not is_cross:
                 self.generate_java_link(outfile)
@@ -1412,9 +1419,12 @@ int dummy;
         description = ' description = Linking static target $out.\n\n'
         outfile.write(rule)
         outfile.write(command)
+        if num_pools > 0:
+            outfile.write(' pool = link_pool\n')
         outfile.write(description)
 
     def generate_dynamic_link_rules(self, outfile):
+        num_pools = self.environment.coredata.backend_options['backend_max_links'].value
         ctypes = [(self.build.compilers, False)]
         if self.environment.is_cross_build():
             if self.environment.cross_info.need_cross_compiler():
@@ -1452,9 +1462,11 @@ int dummy;
                     cross_args=' '.join(cross_args),
                     output_args=' '.join(compiler.get_linker_output_args('$out'))
                 )
-                description = ' description = Linking target $out.'
+                description = ' description = Linking target $out.\n'
                 outfile.write(rule)
                 outfile.write(command)
+                if num_pools > 0:
+                    outfile.write(' pool = link_pool\n')
                 outfile.write(description)
                 outfile.write('\n')
         outfile.write('\n')
diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py
index 80ad94f..d39270b 100644
--- a/mesonbuild/coredata.py
+++ b/mesonbuild/coredata.py
@@ -85,6 +85,36 @@ class UserBooleanOption(UserOption):
     def validate_value(self, value):
         return self.tobool(value)
 
+class UserIntegerOption(UserOption):
+    def __init__(self, name, description, min_value, max_value, value):
+        super().__init__(name, description, [True, False])
+        self.min_value = min_value
+        self.max_value = max_value
+        self.set_value(value)
+
+    def set_value(self, newvalue):
+        if isinstance(newvalue, str):
+            newvalue = self.toint(newvalue)
+        if not isinstance(newvalue, int):
+            raise MesonException('New value for integer option is not an integer.')
+        if self.min_value is not None and newvalue < self.min_value:
+            raise MesonException('New value %d is less than minimum value %d.' % (newvalue, self.min_value))
+        if self.max_value is not None and newvalue > self.max_value:
+            raise MesonException('New value %d is more than maximum value %d.' % (newvalue, self.max_value))
+        self.value = newvalue
+
+    def toint(self, valuestring):
+        try:
+            return int(valuestring)
+        except:
+            raise MesonException('Value string "%s" is not convertable to an integer.' % valuestring)
+
+    def parse_string(self, valuestring):
+        return self.toint(valuestring)
+
+    def validate_value(self, value):
+        return self.toint(value)
+
 class UserComboOption(UserOption):
     def __init__(self, name, description, choices, value):
         super().__init__(name, description, choices)
@@ -145,6 +175,7 @@ class CoreData:
         self.target_guids = {}
         self.version = version
         self.init_builtins(options)
+        self.init_backend_options(self.builtins['backend'].value)
         self.user_options = {}
         self.compiler_options = {}
         self.base_options = {}
@@ -218,6 +249,13 @@ class CoreData:
             args = [key] + builtin_options[key][1:-1] + [value]
             self.builtins[key] = builtin_options[key][0](*args)
 
+    def init_backend_options(self, backend_name):
+        self.backend_options = {}
+        if backend_name == 'ninja':
+            self.backend_options['backend_max_links'] = UserIntegerOption('backend_max_links',
+                                                                          'Maximum number of linker processes to run or 0 for no limit',
+                                                                          0, None, 0)
+
     def get_builtin_option(self, optname):
         if optname in self.builtins:
             return self.builtins[optname].value
diff --git a/mesonbuild/mconf.py b/mesonbuild/mconf.py
index 14eddf5..ee4dbf8 100644
--- a/mesonbuild/mconf.py
+++ b/mesonbuild/mconf.py
@@ -122,6 +122,9 @@ class Conf:
             (k, v) = o.split('=', 1)
             if coredata.is_builtin_option(k):
                 self.coredata.set_builtin_option(k, v)
+            elif k in self.coredata.backend_options:
+                tgt = self.coredata.backend_options[k]
+                tgt.set_value(v)
             elif k in self.coredata.user_options:
                 tgt = self.coredata.user_options[k]
                 tgt.set_value(v)
@@ -163,6 +166,16 @@ class Conf:
                          'choices': coredata.get_builtin_option_choices(key)})
         self.print_aligned(carr)
         print('')
+        bekeys = sorted(self.coredata.backend_options.keys())
+        if not bekeys:
+            print('  No backend options\n')
+        else:
+            bearr = []
+            for k in bekeys:
+                o = self.coredata.backend_options[k]
+                bearr.append({'name': k, 'descr': o.description, 'value': o.value, 'choices': ''})
+            self.print_aligned(bearr)
+        print('')
         print('Base options:')
         okeys = sorted(self.coredata.base_options.keys())
         if not okeys:
diff --git a/mesonbuild/optinterpreter.py b/mesonbuild/optinterpreter.py
index f9e7f26..ac27269 100644
--- a/mesonbuild/optinterpreter.py
+++ b/mesonbuild/optinterpreter.py
@@ -18,17 +18,18 @@ from . import mesonlib
 import os, re
 
 forbidden_option_names = coredata.get_builtin_options()
-forbidden_prefixes = {'c_': True,
-                      'cpp_': True,
-                      'd_': True,
-                      'rust_': True,
-                      'fortran_': True,
-                      'objc_': True,
-                      'objcpp_': True,
-                      'vala_': True,
-                      'csharp_': True,
-                      'swift_': True,
-                      'b_': True,
+forbidden_prefixes = {'c_',
+                      'cpp_',
+                      'd_',
+                      'rust_',
+                      'fortran_',
+                      'objc_',
+                      'objcpp_',
+                      'vala_',
+                      'csharp_',
+                      'swift_',
+                      'b_',
+                      'backend_',
                       }
 
 def is_invalid_name(name):