mdist: use git archive with git repositories for correctness

Fixes various inconsistencies:
- gitattributes is respected
  - export-subst
  - export-ignore
- submodules with relative paths are not checked out relative to the
  local clone (which does not work anyway)
- no need to manually remove gitfiles with inaccurate heuristics

Fixes #2287
Fixes #3081
Fixes #8144

1 files changed, 35 insertions, 29 deletions

diff --git a/mesonbuild/mdist.py b/mesonbuild/mdist.py
index f1040e9..43bc66f 100644
--- a/mesonbuild/mdist.py
+++ b/mesonbuild/mdist.py
@@ -18,6 +18,8 @@ import os
 import sys
 import shutil
 import subprocess
+import tarfile
+import tempfile
 import hashlib
 import json
 from glob import glob
@@ -56,30 +58,36 @@ def create_hash(fname):
         f.write('{} *{}\n'.format(m.hexdigest(), os.path.basename(fname)))
 
 
-def del_gitfiles(dirname):
-    gitfiles = ('.git', '.gitattributes', '.gitignore', '.gitmodules')
-    for f in glob(os.path.join(dirname, '.git*')):
-        if os.path.split(f)[1] in gitfiles:
-            if os.path.isdir(f) and not os.path.islink(f):
-                windows_proof_rmtree(f)
-            else:
-                os.unlink(f)
-
-def process_submodules(dirname):
-    module_file = os.path.join(dirname, '.gitmodules')
+def copy_git(src, distdir, revision='HEAD', prefix=None, subdir=None):
+    cmd = ['git', 'archive', '--format', 'tar', revision]
+    if prefix is not None:
+        cmd.insert(2, f'--prefix={prefix}/')
+    if subdir is not None:
+        cmd.extend(['--', subdir])
+    with tempfile.TemporaryFile() as f:
+        subprocess.check_call(cmd, cwd=src, stdout=f)
+        f.seek(0)
+        t = tarfile.open(fileobj=f) # [ignore encoding]
+        t.extractall(path=distdir)
+
+def process_submodules(src, distdir):
+    module_file = os.path.join(src, '.gitmodules')
     if not os.path.exists(module_file):
         return
-    subprocess.check_call(['git', 'submodule', 'update', '--init', '--recursive'], cwd=dirname)
-    for line in open(module_file, encoding='utf-8'):
-        line = line.strip()
-        if '=' not in line:
-            continue
-        k, v = line.split('=', 1)
-        k = k.strip()
-        v = v.strip()
-        if k != 'path':
+    cmd = ['git', 'submodule', 'status', '--cached', '--recursive']
+    modlist = subprocess.check_output(cmd, cwd=src, universal_newlines=True).splitlines()
+    for submodule in modlist:
+        status = submodule[:1]
+        sha1, rest = submodule[1:].split(' ', 1)
+        subpath = rest.rsplit(' ', 1)[0]
+
+        if status == '-':
+            mlog.warning(f'Submodule {subpath!r} is not checked out and cannot be added to the dist')
             continue
-        del_gitfiles(os.path.join(dirname, v))
+        elif status in {'+', 'U'}:
+            mlog.warning(f'Submodule {subpath!r} has uncommitted changes that will not be included in the dist tarball')
+
+        copy_git(os.path.join(src, subpath), distdir, revision=sha1, prefix=subpath)
 
 
 def run_dist_scripts(src_root, bld_root, dist_root, dist_scripts, subprojects):
@@ -126,7 +134,7 @@ def git_have_dirty_index(src_root):
     ret = subprocess.call(['git', '-C', src_root, 'diff-index', '--quiet', 'HEAD'])
     return ret == 1
 
-def git_clone(src_root, distdir):
+def process_git_project(src_root, distdir):
     if git_have_dirty_index(src_root):
         mlog.warning('Repository has uncommitted changes that will not be included in the dist tarball')
     if os.path.exists(distdir):
@@ -134,30 +142,28 @@ def git_clone(src_root, distdir):
     repo_root = git_root(src_root)
     if repo_root.samefile(src_root):
         os.makedirs(distdir)
-        subprocess.check_call(['git', 'clone', '--shared', src_root, distdir])
+        copy_git(src_root, distdir)
     else:
         subdir = Path(src_root).relative_to(repo_root)
         tmp_distdir = distdir + '-tmp'
         if os.path.exists(tmp_distdir):
             windows_proof_rmtree(tmp_distdir)
         os.makedirs(tmp_distdir)
-        subprocess.check_call(['git', 'clone', '--shared', '--no-checkout', str(repo_root), tmp_distdir])
-        subprocess.check_call(['git', 'checkout', 'HEAD', '--', str(subdir)], cwd=tmp_distdir)
+        copy_git(repo_root, tmp_distdir, subdir=str(subdir))
         Path(tmp_distdir, subdir).rename(distdir)
         windows_proof_rmtree(tmp_distdir)
-    process_submodules(distdir)
-    del_gitfiles(distdir)
+    process_submodules(src_root, distdir)
 
 def create_dist_git(dist_name, archives, src_root, bld_root, dist_sub, dist_scripts, subprojects):
     distdir = os.path.join(dist_sub, dist_name)
-    git_clone(src_root, distdir)
+    process_git_project(src_root, distdir)
     for path in subprojects.values():
         sub_src_root = os.path.join(src_root, path)
         sub_distdir = os.path.join(distdir, path)
         if os.path.exists(sub_distdir):
             continue
         if is_git(sub_src_root):
-            git_clone(sub_src_root, sub_distdir)
+            process_git_project(sub_src_root, sub_distdir)
         else:
             shutil.copytree(sub_src_root, sub_distdir)
     run_dist_scripts(src_root, bld_root, distdir, dist_scripts, subprojects)