Merge pull request #4743 from dcbaker/native-file-extended

Extend native files to store install path information

19 files changed, 252 insertions, 24 deletions

diff --git a/docs/markdown/Cross-compilation.md b/docs/markdown/Cross-compilation.md
index 7d316ed..36620eb 100644
--- a/docs/markdown/Cross-compilation.md
+++ b/docs/markdown/Cross-compilation.md
@@ -150,7 +150,7 @@ binaries are not actually compatible. In such cases you may use the
 needs_exe_wrapper = true
 ```
 
-The last bit is the definition of host and target machines. Every
+The next bit is the definition of host and target machines. Every
 cross build definition must have one or both of them. If it had
 neither, the build would not be a cross build but a native build. You
 do not need to define the build machine, as all necessary information
@@ -186,6 +186,20 @@ If you do not define your host machine, it is assumed to be the build
 machine. Similarly if you do not specify target machine, it is assumed
 to be the host machine.
 
+Additionally, you can define the paths that you want to install to in your
+cross file. This may be especially useful when cross compiling an entire
+operating system, or for operating systems to use internally for consistency.
+
+```ini
+[paths]
+prefix = '/my/prefix'
+libdir = 'lib/i386-linux-gnu'
+bindir = 'bin'
+```
+
+This will be overwritten by any options passed on the command line.
+
+
 ## Starting a cross build
 
 
diff --git a/docs/markdown/Native-environments.md b/docs/markdown/Native-environments.md
index a9719a7..f0d41eb 100644
--- a/docs/markdown/Native-environments.md
+++ b/docs/markdown/Native-environments.md
@@ -43,6 +43,23 @@ rust = '/usr/local/bin/rust'
 llvm-config = '/usr/local/llvm-svn/bin/llvm-config'
 ```
 
+### Paths and Directories
+
+As of 0.50.0 paths and directories such as libdir can be defined in the native
+file in a paths section
+
+```ini
+[paths]
+libdir = 'mylibdir'
+prefix = '/my prefix'
+```
+
+These values will only be loaded when not cross compiling. Any arguments on the
+command line will override any options in the native file. For example, passing
+`--libdir=otherlibdir` would result in a prefix of `/my prefix` and a libdir of
+`otherlibdir`.
+
+
 ## Loading multiple native files
 
 Unlike cross file, native files allow layering. More than one native file can be
diff --git a/docs/markdown/snippets/native-file-paths.md b/docs/markdown/snippets/native-file-paths.md
new file mode 100644
index 0000000..b091c40
--- /dev/null
+++ b/docs/markdown/snippets/native-file-paths.md
@@ -0,0 +1,4 @@
+## Native and Cross File Paths and Directories
+
+A new `[paths]` section has been added to native and cross files. This
+can be used to set paths such a prefix and libdir in a persistent way.
diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py
index 5d0ec5a..4a03e98 100644
--- a/mesonbuild/ast/introspection.py
+++ b/mesonbuild/ast/introspection.py
@@ -86,7 +86,7 @@ class IntrospectionInterpreter(AstInterpreter):
         self.project_default_options = mesonlib.stringlistify(def_opts)
         self.project_default_options = cdata.create_options_dict(self.project_default_options)
         self.default_options.update(self.project_default_options)
-        self.coredata.set_default_options(self.default_options, self.subproject, self.environment.cmd_line_options)
+        self.coredata.set_default_options(self.default_options, self.subproject, self.environment)
 
         if not self.is_subproject() and 'subproject_dir' in kwargs:
             spdirname = kwargs['subproject_dir']
diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py
index 3ce272e..139dd6e 100644
--- a/mesonbuild/coredata.py
+++ b/mesonbuild/coredata.py
@@ -525,7 +525,13 @@ class CoreData:
             sub = 'In subproject {}: '.format(subproject) if subproject else ''
             mlog.warning('{}Unknown options: "{}"'.format(sub, unknown_options))
 
-    def set_default_options(self, default_options, subproject, cmd_line_options):
+    def set_default_options(self, default_options, subproject, env):
+        # Set defaults first from conf files (cross or native), then
+        # override them as nec as necessary.
+        for k, v in env.paths.host:
+            if v is not None:
+                env.cmd_line_options.setdefault(k, v)
+
         # Set default options as if they were passed to the command line.
         # Subprojects can only define default for user options.
         from . import optinterpreter
@@ -534,7 +540,7 @@ class CoreData:
                 if optinterpreter.is_invalid_name(k):
                     continue
                 k = subproject + ':' + k
-            cmd_line_options.setdefault(k, v)
+            env.cmd_line_options.setdefault(k, v)
 
         # Create a subset of cmd_line_options, keeping only options for this
         # subproject. Also take builtin options if it's the main project.
@@ -542,7 +548,7 @@ class CoreData:
         # languages and setting the backend (builtin options must be set first
         # to know which backend we'll use).
         options = {}
-        for k, v in cmd_line_options.items():
+        for k, v in env.cmd_line_options.items():
             if subproject:
                 if not k.startswith(subproject + ':'):
                     continue
diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py
index b23509a..b2cc657 100644
--- a/mesonbuild/environment.py
+++ b/mesonbuild/environment.py
@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import configparser, os, platform, re, sys, shlex, shutil, subprocess, typing
+import configparser, os, platform, re, sys, shlex, shutil, subprocess
+import typing
 
 from . import coredata
 from .linkers import ArLinker, ArmarLinker, VisualStudioLinker, DLinker, CcrxLinker
@@ -371,6 +372,7 @@ class Environment:
         # Similar to coredata.compilers and build.compilers, but lower level in
         # that there is no meta data, only names/paths.
         self.binaries = PerMachineDefaultable()
+
         # Just uses hard-coded defaults and environment variables. Might be
         # overwritten by a native file.
         self.binaries.build = BinaryTable({})
@@ -378,10 +380,16 @@ class Environment:
         # Misc other properties about each machine.
         self.properties = PerMachine(Properties(), Properties(), Properties())
 
+        # Store paths for native and cross build files. There is no target
+        # machine information here because nothing is installed for the target
+        # architecture, just the build and host architectures
+        self.paths = PerMachineDefaultable()
+
         if self.coredata.config_files is not None:
             config = MesonConfigFile.from_config_parser(
                 coredata.load_configs(self.coredata.config_files))
             self.binaries.build = BinaryTable(config.get('binaries', {}))
+            self.paths.build = Directories(**config.get('paths', {}))
 
         if self.coredata.cross_file is not None:
             config = MesonConfigFile.parse_datafile(self.coredata.cross_file)
@@ -391,9 +399,11 @@ class Environment:
                 self.machines.host = MachineInfo.from_literal(config['host_machine'])
             if 'target_machine' in config:
                 self.machines.target = MachineInfo.from_literal(config['target_machine'])
+            self.paths.host = Directories(**config.get('paths', {}))
 
         self.machines.default_missing()
         self.binaries.default_missing()
+        self.paths.default_missing()
 
         exe_wrapper = self.binaries.host.lookup_entry('exe_wrapper')
         if exe_wrapper is not None:
@@ -1172,46 +1182,46 @@ class Environment:
     def get_exe_suffix(self):
         return self.exe_suffix
 
-    def get_import_lib_dir(self):
+    def get_import_lib_dir(self) -> str:
         "Install dir for the import library (library used for linking)"
         return self.get_libdir()
 
-    def get_shared_module_dir(self):
+    def get_shared_module_dir(self) -> str:
         "Install dir for shared modules that are loaded at runtime"
         return self.get_libdir()
 
-    def get_shared_lib_dir(self):
+    def get_shared_lib_dir(self) -> str:
         "Install dir for the shared library"
         if self.win_libdir_layout:
             return self.get_bindir()
         return self.get_libdir()
 
-    def get_static_lib_dir(self):
+    def get_static_lib_dir(self) -> str:
         "Install dir for the static library"
         return self.get_libdir()
 
     def get_object_suffix(self):
         return self.object_suffix
 
-    def get_prefix(self):
+    def get_prefix(self) -> str:
         return self.coredata.get_builtin_option('prefix')
 
-    def get_libdir(self):
+    def get_libdir(self) -> str:
         return self.coredata.get_builtin_option('libdir')
 
-    def get_libexecdir(self):
+    def get_libexecdir(self) -> str:
         return self.coredata.get_builtin_option('libexecdir')
 
-    def get_bindir(self):
+    def get_bindir(self) -> str:
         return self.coredata.get_builtin_option('bindir')
 
-    def get_includedir(self):
+    def get_includedir(self) -> str:
         return self.coredata.get_builtin_option('includedir')
 
-    def get_mandir(self):
+    def get_mandir(self) -> str:
         return self.coredata.get_builtin_option('mandir')
 
-    def get_datadir(self):
+    def get_datadir(self) -> str:
         return self.coredata.get_builtin_option('datadir')
 
     def get_compiler_system_dirs(self):
@@ -1581,3 +1591,42 @@ This is probably wrong, it should always point to the native compiler.''' % evar
             if command is not None:
                 command = shlex.split(command)
         return command
+
+class Directories:
+
+    """Data class that holds information about directories for native and cross
+    builds.
+    """
+
+    def __init__(self, bindir: typing.Optional[str] = None, datadir: typing.Optional[str] = None,
+                 includedir: typing.Optional[str] = None, infodir: typing.Optional[str] = None,
+                 libdir: typing.Optional[str] = None, libexecdir: typing.Optional[str] = None,
+                 localedir: typing.Optional[str] = None, localstatedir: typing.Optional[str] = None,
+                 mandir: typing.Optional[str] = None, prefix: typing.Optional[str] = None,
+                 sbindir: typing.Optional[str] = None, sharedstatedir: typing.Optional[str] = None,
+                 sysconfdir: typing.Optional[str] = None):
+        self.bindir = bindir
+        self.datadir = datadir
+        self.includedir = includedir
+        self.infodir = infodir
+        self.libdir = libdir
+        self.libexecdir = libexecdir
+        self.localedir = localedir
+        self.localstatedir = localstatedir
+        self.mandir = mandir
+        self.prefix = prefix
+        self.sbindir = sbindir
+        self.sharedstatedir = sharedstatedir
+        self.sysconfdir = sysconfdir
+
+    def __contains__(self, key: str) -> str:
+        return hasattr(self, key)
+
+    def __getitem__(self, key: str) -> str:
+        return getattr(self, key)
+
+    def __setitem__(self, key: str, value: typing.Optional[str]) -> None:
+        setattr(self, key, value)
+
+    def __iter__(self) -> typing.Iterator[typing.Tuple[str, str]]:
+        return iter(self.__dict__.items())
diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py
index 2eb0720..c03bae1 100644
--- a/mesonbuild/interpreter.py
+++ b/mesonbuild/interpreter.py
@@ -2458,8 +2458,6 @@ external dependencies (including libraries) must go to "dependencies".''')
         return self.subprojects[dirname]
 
     def get_option_internal(self, optname):
-        # Some base options are not defined in some environments, return the
-        # default value from compilers.base_options in that case.
         for d in chain(
                 [self.coredata.base_options, compilers.base_options, self.coredata.builtins],
                 self.coredata.get_all_compiler_options()):
@@ -2576,7 +2574,7 @@ external dependencies (including libraries) must go to "dependencies".''')
             default_options.update(self.default_project_options)
         else:
             default_options = {}
-        self.coredata.set_default_options(default_options, self.subproject, self.environment.cmd_line_options)
+        self.coredata.set_default_options(default_options, self.subproject, self.environment)
         self.set_backend()
 
         if not self.is_subproject():
diff --git a/mesonbuild/mconf.py b/mesonbuild/mconf.py
index fbb528c..18bb37b 100644
--- a/mesonbuild/mconf.py
+++ b/mesonbuild/mconf.py
@@ -122,8 +122,7 @@ class Conf:
         if not options:
             print('  No {}\n'.format(title.lower()))
         arr = []
-        for k in sorted(options):
-            o = options[k]
+        for k, o in sorted(options.items()):
             d = o.description
             v = o.printable_value()
             c = o.choices
diff --git a/mesonbuild/mesonlib.py b/mesonbuild/mesonlib.py
index 540fcdc..9a55c26 100644
--- a/mesonbuild/mesonlib.py
+++ b/mesonbuild/mesonlib.py
@@ -308,7 +308,15 @@ class OrderedEnum(Enum):
             return self.value < other.value
         return NotImplemented
 
-MachineChoice = OrderedEnum('MachineChoice', ['BUILD', 'HOST', 'TARGET'])
+class MachineChoice(OrderedEnum):
+
+    """Enum class representing one of the three possible values for binaries,
+    the build, host, and target machines.
+    """
+
+    BUILD = 0
+    HOST = 1
+    TARGET = 2
 
 class PerMachine:
     def __init__(self, build, host, target):
diff --git a/run_project_tests.py b/run_project_tests.py
index d10f3a2..c35928f 100755
--- a/run_project_tests.py
+++ b/run_project_tests.py
@@ -345,6 +345,12 @@ def _run_test(testdir, test_build_dir, install_dir, extra_args, compiler, backen
     if pass_libdir_to_test(testdir):
         gen_args += ['--libdir', 'lib']
     gen_args += [testdir, test_build_dir] + flags + test_args + extra_args
+    nativefile = os.path.join(testdir, 'nativefile.ini')
+    if os.path.exists(nativefile):
+        gen_args.extend(['--native-file', nativefile])
+    crossfile = os.path.join(testdir, 'crossfile.ini')
+    if os.path.exists(crossfile):
+        gen_args.extend(['--cross-file', crossfile])
     (returncode, stdo, stde) = run_configure(gen_args)
     try:
         logfile = Path(test_build_dir, 'meson-logs', 'meson-log.txt')
diff --git a/run_unittests.py b/run_unittests.py
index a244bbd..9dd734a 100755
--- a/run_unittests.py
+++ b/run_unittests.py
@@ -5438,6 +5438,62 @@ class NativeFileTests(BasePlatformTests):
         compiler = env.detect_swift_compiler()
         self.assertEqual(compiler.version, '1.2345')
 
+    def test_native_file_dirs(self):
+        testcase = os.path.join(self.unit_test_dir, '54 native file override')
+        self.init(testcase, default_args=False,
+                  extra_args=['--native-file', os.path.join(testcase, 'nativefile')])
+
+    def test_native_file_dirs_overriden(self):
+        testcase = os.path.join(self.unit_test_dir, '54 native file override')
+        self.init(testcase, default_args=False,
+                  extra_args=['--native-file', os.path.join(testcase, 'nativefile'),
+                              '-Ddef_libdir=liblib', '-Dlibdir=liblib'])
+
+
+class CrossFileTests(BasePlatformTests):
+
+    """Tests for cross file functioality not directly related to
+    cross compiling.
+
+    This is mainly aimed to testing overrides from cross files.
+    """
+
+    def test_cross_file_dirs(self):
+        testcase = os.path.join(self.unit_test_dir, '54 native file override')
+        self.init(testcase, default_args=False,
+                  extra_args=['--native-file', os.path.join(testcase, 'nativefile'),
+                              '--cross-file', os.path.join(testcase, 'crossfile'),
+                              '-Ddef_bindir=binbar',
+                              '-Ddef_datadir=databar',
+                              '-Ddef_includedir=includebar',
+                              '-Ddef_infodir=infobar',
+                              '-Ddef_libdir=libbar',
+                              '-Ddef_libexecdir=libexecbar',
+                              '-Ddef_localedir=localebar',
+                              '-Ddef_localstatedir=localstatebar',
+                              '-Ddef_mandir=manbar',
+                              '-Ddef_sbindir=sbinbar',
+                              '-Ddef_sharedstatedir=sharedstatebar',
+                              '-Ddef_sysconfdir=sysconfbar'])
+
+    def test_cross_file_dirs_overriden(self):
+        testcase = os.path.join(self.unit_test_dir, '54 native file override')
+        self.init(testcase, default_args=False,
+                  extra_args=['--native-file', os.path.join(testcase, 'nativefile'),
+                              '--cross-file', os.path.join(testcase, 'crossfile'),
+                              '-Ddef_libdir=liblib', '-Dlibdir=liblib',
+                              '-Ddef_bindir=binbar',
+                              '-Ddef_datadir=databar',
+                              '-Ddef_includedir=includebar',
+                              '-Ddef_infodir=infobar',
+                              '-Ddef_libexecdir=libexecbar',
+                              '-Ddef_localedir=localebar',
+                              '-Ddef_localstatedir=localstatebar',
+                              '-Ddef_mandir=manbar',
+                              '-Ddef_sbindir=sbinbar',
+                              '-Ddef_sharedstatedir=sharedstatebar',
+                              '-Ddef_sysconfdir=sysconfbar'])
+
 
 def unset_envs():
     # For unit tests we must fully control all command lines
@@ -5457,7 +5513,7 @@ def should_run_cross_mingw_tests():
 def main():
     unset_envs()
     cases = ['InternalTests', 'DataTests', 'AllPlatformTests', 'FailureTests',
-             'PythonTests', 'NativeFileTests', 'RewriterTests']
+             'PythonTests', 'NativeFileTests', 'RewriterTests', 'CrossFileTests']
     if not is_windows():
         cases += ['LinuxlikeTests']
         if should_run_cross_arm_tests():
diff --git a/test cases/common/212 native file path override/installed_files.txt b/test cases/common/212 native file path override/installed_files.txt
new file mode 100644
index 0000000..0044d40
--- /dev/null
+++ b/test cases/common/212 native file path override/installed_files.txt
@@ -0,0 +1,2 @@
+usr/custom_bindir/main?exe
+?msvc:usr/custom_bindir/main.pdb
diff --git a/test cases/common/212 native file path override/main.cpp b/test cases/common/212 native file path override/main.cpp
new file mode 100644
index 0000000..d65cab2
--- /dev/null
+++ b/test cases/common/212 native file path override/main.cpp
@@ -0,0 +1,5 @@
+#include <iostream>
+
+int main() {
+    std::cout << "Hello world!" << std::endl;
+}
diff --git a/test cases/common/212 native file path override/meson.build b/test cases/common/212 native file path override/meson.build
new file mode 100644
index 0000000..142ca1c
--- /dev/null
+++ b/test cases/common/212 native file path override/meson.build
@@ -0,0 +1,7 @@
+project('native file install dir override', 'cpp')
+
+if meson.is_cross_build()
+  error('MESON_SKIP_TEST cannot test native build rules in cross build')
+endif
+
+executable('main', 'main.cpp', install : true)
diff --git a/test cases/common/212 native file path override/nativefile.ini b/test cases/common/212 native file path override/nativefile.ini
new file mode 100644
index 0000000..1c295c7
--- /dev/null
+++ b/test cases/common/212 native file path override/nativefile.ini
@@ -0,0 +1,2 @@
+[paths]
+bindir = 'custom_bindir'
diff --git a/test cases/unit/54 native file override/crossfile b/test cases/unit/54 native file override/crossfile
new file mode 100644
index 0000000..9dc4fbc
--- /dev/null
+++ b/test cases/unit/54 native file override/crossfile
@@ -0,0 +1,16 @@
+[paths]
+bindir = 'binbar'
+datadir = 'databar'
+includedir = 'includebar'
+infodir = 'infobar'
+libdir = 'libbar'
+libexecdir = 'libexecbar'
+localedir = 'localebar'
+localstatedir = 'localstatebar'
+mandir = 'manbar'
+prefix = '/prefix'
+sbindir = 'sbinbar'
+sharedstatedir = 'sharedstatebar'
+sysconfdir = 'sysconfbar'
+
+; vim: ft=dosini
diff --git a/test cases/unit/54 native file override/meson.build b/test cases/unit/54 native file override/meson.build
new file mode 100644
index 0000000..8318aba
--- /dev/null
+++ b/test cases/unit/54 native file override/meson.build
@@ -0,0 +1,10 @@
+project('native file overrides')
+
+foreach o : ['bindir', 'datadir', 'includedir', 'infodir', 'libdir',
+             'libexecdir', 'localedir', 'localstatedir', 'mandir', 'prefix',
+             'sbindir', 'sharedstatedir', 'sysconfdir']
+  expected = get_option('def_' + o)
+  actual = get_option(o) 
+  assert(expected == actual,
+         '@0@ should have been @1@, but was @2@!'.format(o, expected, actual))
+endforeach
diff --git a/test cases/unit/54 native file override/meson_options.txt b/test cases/unit/54 native file override/meson_options.txt
new file mode 100644
index 0000000..4d2abf9
--- /dev/null
+++ b/test cases/unit/54 native file override/meson_options.txt
@@ -0,0 +1,13 @@
+option('def_bindir', type: 'string', value : 'binfoo',)
+option('def_datadir', type: 'string', value : 'datafoo',)
+option('def_includedir', type: 'string', value : 'includefoo',)
+option('def_infodir', type: 'string', value : 'infofoo',)
+option('def_libdir', type: 'string', value : 'libfoo',)
+option('def_libexecdir', type: 'string', value : 'libexecfoo',)
+option('def_localedir', type: 'string', value : 'localefoo',)
+option('def_localstatedir', type: 'string', value : 'localstatefoo',)
+option('def_mandir', type: 'string', value : 'manfoo',)
+option('def_prefix', type: 'string', value : '/prefix',)
+option('def_sbindir', type: 'string', value : 'sbinfoo',)
+option('def_sharedstatedir', type: 'string', value : 'sharedstatefoo',)
+option('def_sysconfdir', type: 'string', value : 'sysconffoo',)
diff --git a/test cases/unit/54 native file override/nativefile b/test cases/unit/54 native file override/nativefile
new file mode 100644
index 0000000..a390725
--- /dev/null
+++ b/test cases/unit/54 native file override/nativefile
@@ -0,0 +1,16 @@
+[paths]
+bindir = 'binfoo'
+datadir = 'datafoo'
+includedir = 'includefoo'
+infodir = 'infofoo'
+libdir = 'libfoo'
+libexecdir = 'libexecfoo'
+localedir = 'localefoo'
+localstatedir = 'localstatefoo'
+mandir = 'manfoo'
+prefix = '/prefix'
+sbindir = 'sbinfoo'
+sharedstatedir = 'sharedstatefoo'
+sysconfdir = 'sysconffoo'
+
+; vim: ft=dosini