Merge pull request #4389 from xclaesse/subprojects-cmd

Add 'meson subprojects update' command

8 files changed, 301 insertions, 9 deletions

diff --git a/docs/markdown/Subprojects.md b/docs/markdown/Subprojects.md
index 2e3e2ea..ed11852 100644
--- a/docs/markdown/Subprojects.md
+++ b/docs/markdown/Subprojects.md
@@ -179,7 +179,7 @@ the following command-line options:
     This is useful (mostly for distros) when you want to only use the
     sources provided by a software release, and want to manually handle
     or provide missing dependencies.
-    
+
 * **--wrap-mode=nofallback**
 
     Meson will not use subproject fallbacks for any dependency
@@ -196,6 +196,51 @@ the following command-line options:
     want to specifically build against the library sources provided by
     your subprojects.
 
+## Download subprojects
+
+*Since 0.49.0*
+
+Meson will automatically download needed subprojects during configure, unless
+**--wrap-mode=nodownload** option is passed. It is sometimes preferable to
+download all subprojects in advance, so the meson configure can be performed
+offline. The command-line `meson subprojects download` can be used for that, it
+will download all missing subprojects, but will not update already fetched
+subprojects.
+
+## Update subprojects
+
+*Since 0.49.0*
+
+Once a subproject has been fetched, Meson will not update it automatically.
+For example if the wrap file tracks a git branch, it won't pull latest commits.
+
+To pull latest version of all your subprojects at once, just run the command:
+`meson subprojects update`.
+- If the wrap file comes from wrapdb, the latest version of the wrap file will
+  be pulled and used next time meson reconfigure the project. This can be
+  triggered using `meson --reconfigure`. Previous source tree is not deleted, to
+  prevent from any loss of local changes.
+- If the wrap file points to a git commit or tag, a checkout of that commit is
+  performed.
+- If the wrap file points to a git branch, and the current branch has the same
+  name, a `git pull` is performed.
+- If the wrap file points to a git branch, and the current branch is different,
+  it is skipped. Unless `--rebase` option is passed in which case
+  `git pull --rebase` is performed.
+
+## Start a topic branch across all git subprojects
+
+*Since 0.49.0*
+
+The command-line `meson subprojects checkout <branch_name>` will checkout a
+branch, or create one with `-b` argument, in every git subprojects. This is
+useful when starting local changes across multiple subprojects. It is still your
+responsability to commit and push in each repository where you made local
+changes.
+
+To come back to the revision set in wrap file (i.e. master), just run
+`meson subprojects checkout` with no branch name.
+
 ## Why must all subprojects be inside a single directory?
 
 There are several reasons.
diff --git a/docs/markdown/snippets/subprojects_cmd.md b/docs/markdown/snippets/subprojects_cmd.md
new file mode 100644
index 0000000..20fef5c
--- /dev/null
+++ b/docs/markdown/snippets/subprojects_cmd.md
@@ -0,0 +1,7 @@
+## Subprojects download, checkout, update command-line
+
+New command-line tool has been added to manage subprojects:
+
+- `meson subprojects download` to download all subprojects that have a wrap file.
+- `meson subprojects update` to update all subprojects to latest version.
+- `meson subprojects checkout` to checkout or create a branch in all git subprojects.
diff --git a/mesonbuild/mesonmain.py b/mesonbuild/mesonmain.py
index ebe2c8e..c11d044 100644
--- a/mesonbuild/mesonmain.py
+++ b/mesonbuild/mesonmain.py
@@ -20,7 +20,7 @@ import argparse
 
 from . import mesonlib
 from . import mlog
-from . import mconf, minit, minstall, mintro, msetup, mtest, rewriter
+from . import mconf, minit, minstall, mintro, msetup, mtest, rewriter, msubprojects
 from .mesonlib import MesonException
 from .environment import detect_msys2_arch
 from .wrap import wraptool
@@ -47,6 +47,8 @@ class CommandLineParser:
                          help='Run tests')
         self.add_command('wrap', wraptool.add_arguments, wraptool.run,
                          help='Wrap tools')
+        self.add_command('subprojects', msubprojects.add_arguments, msubprojects.run,
+                         help='Manage subprojects')
         self.add_command('help', self.add_help_arguments, self.run_help_command,
                          help='Print help of a subcommand')
 
diff --git a/mesonbuild/mlog.py b/mesonbuild/mlog.py
index 890cb46..ea99d09 100644
--- a/mesonbuild/mlog.py
+++ b/mesonbuild/mlog.py
@@ -96,6 +96,9 @@ def green(text):
 def yellow(text):
     return AnsiDecorator(text, "\033[1;33m")
 
+def blue(text):
+    return AnsiDecorator(text, "\033[1;34m")
+
 def cyan(text):
     return AnsiDecorator(text, "\033[1;36m")
 
diff --git a/mesonbuild/msubprojects.py b/mesonbuild/msubprojects.py
new file mode 100644
index 0000000..1536d96
--- /dev/null
+++ b/mesonbuild/msubprojects.py
@@ -0,0 +1,226 @@
+import os, subprocess
+
+from . import mlog
+from .mesonlib import Popen_safe
+from .wrap.wrap import API_ROOT, PackageDefinition, Resolver, WrapException
+from .wrap import wraptool
+
+def update_wrapdb_file(wrap, repo_dir, options):
+    patch_url = wrap.get('patch_url')
+    branch, revision = wraptool.parse_patch_url(patch_url)
+    new_branch, new_revision = wraptool.get_latest_version(wrap.name)
+    if new_branch == branch and new_revision == revision:
+        mlog.log('  -> Up to date.')
+        return
+    wraptool.update_wrap_file(wrap.filename, wrap.name, new_branch, new_revision)
+    msg = ['  -> New wrap file downloaded.']
+    # Meson reconfigure won't use the new wrap file as long as the source
+    # directory exists. We don't delete it ourself to avoid data loss in case
+    # user has changes in their copy.
+    if os.path.isdir(repo_dir):
+        msg += ['To use it, delete', mlog.bold(repo_dir), 'and run', mlog.bold('meson --reconfigure')]
+    mlog.log(*msg)
+
+def update_file(wrap, repo_dir, options):
+    patch_url = wrap.values.get('patch_url', '')
+    if patch_url.startswith(API_ROOT):
+        update_wrapdb_file(wrap, repo_dir, options)
+    elif not os.path.isdir(repo_dir):
+        # The subproject is not needed, or it is a tarball extracted in
+        # 'libfoo-1.0' directory and the version has been bumped and the new
+        # directory is 'libfoo-2.0'. In that case forcing a meson
+        # reconfigure will download and use the new tarball.
+        mlog.log('  -> Subproject has not been checked out. Run', mlog.bold('meson --reconfigure'), 'to fetch it if needed.')
+    else:
+        # The subproject has not changed, or the new source and/or patch
+        # tarballs should be extracted in the same directory than previous
+        # version.
+        mlog.log('  -> Subproject has not changed, or the new source/patch needs to be extracted on the same location.\n' +
+                 '     In that case, delete', mlog.bold(repo_dir), 'and run', mlog.bold('meson --reconfigure'))
+
+def git(cmd, workingdir):
+    return subprocess.check_output(['git', '-C', workingdir] + cmd,
+                                   stderr=subprocess.STDOUT).decode()
+
+def git_show(repo_dir):
+    commit_message = git(['show', '--quiet', '--pretty=format:%h%n%d%n%s%n[%an]'], repo_dir)
+    parts = [s.strip() for s in commit_message.split('\n')]
+    mlog.log('  ->', mlog.yellow(parts[0]), mlog.red(parts[1]), parts[2], mlog.blue(parts[3]))
+
+def update_git(wrap, repo_dir, options):
+    if not os.path.isdir(repo_dir):
+        mlog.log('  -> Not used.')
+        return
+    revision = wrap.get('revision')
+    ret = git(['rev-parse', '--abbrev-ref', 'HEAD'], repo_dir).strip()
+    if ret == 'HEAD':
+        try:
+            # We are currently in detached mode, just checkout the new revision
+            git(['fetch'], repo_dir)
+            git(['checkout', revision], repo_dir)
+        except subprocess.CalledProcessError as e:
+            out = e.output.decode().strip()
+            mlog.log('  -> Could not checkout revision', mlog.cyan(revision))
+            mlog.log(mlog.red(out))
+            mlog.log(mlog.red(str(e)))
+            return
+    elif ret == revision:
+        try:
+            # We are in the same branch, pull latest commits
+            git(['-c', 'rebase.autoStash=true', 'pull', '--rebase'], repo_dir)
+        except subprocess.CalledProcessError as e:
+            out = e.output.decode().strip()
+            mlog.log('  -> Could not rebase', mlog.bold(repo_dir), 'please fix and try again.')
+            mlog.log(mlog.red(out))
+            mlog.log(mlog.red(str(e)))
+            return
+    else:
+        # We are in another branch, probably user created their own branch and
+        # we should rebase it on top of wrap's branch.
+        if options.rebase:
+            try:
+                git(['fetch'], repo_dir)
+                git(['-c', 'rebase.autoStash=true', 'rebase', revision], repo_dir)
+            except subprocess.CalledProcessError as e:
+                out = e.output.decode().strip()
+                mlog.log('  -> Could not rebase', mlog.bold(repo_dir), 'please fix and try again.')
+                mlog.log(mlog.red(out))
+                mlog.log(mlog.red(str(e)))
+                return
+        else:
+            mlog.log('  -> Target revision is', mlog.bold(revision), 'but currently in branch is', mlog.bold(ret), '\n' +
+                     '     To rebase your branch on top of', mlog.bold(revision), 'use', mlog.bold('--rebase'), 'option.')
+            return
+
+    git(['submodule', 'update'], repo_dir)
+    git_show(repo_dir)
+
+def update_hg(wrap, repo_dir, options):
+    if not os.path.isdir(repo_dir):
+        mlog.log('  -> Not used.')
+        return
+    revno = wrap.get('revision')
+    if revno.lower() == 'tip':
+        # Failure to do pull is not a fatal error,
+        # because otherwise you can't develop without
+        # a working net connection.
+        subprocess.call(['hg', 'pull'], cwd=repo_dir)
+    else:
+        if subprocess.call(['hg', 'checkout', revno], cwd=repo_dir) != 0:
+            subprocess.check_call(['hg', 'pull'], cwd=repo_dir)
+            subprocess.check_call(['hg', 'checkout', revno], cwd=repo_dir)
+
+def update_svn(wrap, repo_dir, options):
+    if not os.path.isdir(repo_dir):
+        mlog.log('  -> Not used.')
+        return
+    revno = wrap.get('revision')
+    p, out = Popen_safe(['svn', 'info', '--show-item', 'revision', repo_dir])
+    current_revno = out
+    if current_revno == revno:
+        return
+    if revno.lower() == 'head':
+        # Failure to do pull is not a fatal error,
+        # because otherwise you can't develop without
+        # a working net connection.
+        subprocess.call(['svn', 'update'], cwd=repo_dir)
+    else:
+        subprocess.check_call(['svn', 'update', '-r', revno], cwd=repo_dir)
+
+def update(wrap, repo_dir, options):
+    mlog.log('Updating %s...' % wrap.name)
+    if wrap.type == 'file':
+        update_file(wrap, repo_dir, options)
+    elif wrap.type == 'git':
+        update_git(wrap, repo_dir, options)
+    elif wrap.type == 'hg':
+        update_hg(wrap, repo_dir, options)
+    elif wrap.type == 'svn':
+        update_svn(wrap, repo_dir, options)
+    else:
+        mlog.log('  -> Cannot update', wrap.type, 'subproject')
+
+def checkout(wrap, repo_dir, options):
+    if wrap.type != 'git' or not os.path.isdir(repo_dir):
+        return
+    branch_name = options.branch_name if options.branch_name else wrap.get('revision')
+    cmd = ['checkout', branch_name, '--']
+    if options.b:
+        cmd.insert(1, '-b')
+    mlog.log('Checkout %s in %s...' % (branch_name, wrap.name))
+    try:
+        git(cmd, repo_dir)
+        git_show(repo_dir)
+    except subprocess.CalledProcessError as e:
+        out = e.output.decode().strip()
+        mlog.log('  -> ', mlog.red(out))
+
+def download(wrap, repo_dir, options):
+    mlog.log('Download %s...' % wrap.name)
+    if os.path.isdir(repo_dir):
+        mlog.log('  -> Already downloaded')
+        return
+    try:
+        r = Resolver(os.path.dirname(repo_dir))
+        r.resolve(wrap.name)
+        mlog.log('  -> done')
+    except WrapException as e:
+        mlog.log('  ->', mlog.red(str(e)))
+
+def add_common_arguments(p):
+    p.add_argument('--sourcedir', default='.',
+                   help='Path to source directory')
+    p.add_argument('subprojects', nargs='*',
+                   help='List of subprojects (default: all)')
+
+def add_arguments(parser):
+    subparsers = parser.add_subparsers(title='Commands', dest='command')
+    subparsers.required = True
+
+    p = subparsers.add_parser('update', help='Update all subprojects from wrap files')
+    p.add_argument('--rebase', default=False, action='store_true',
+                   help='Rebase your branch on top of wrap\'s revision (git only)')
+    add_common_arguments(p)
+    p.set_defaults(subprojects_func=update)
+
+    p = subparsers.add_parser('checkout', help='Checkout a branch (git only)')
+    p.add_argument('-b', default=False, action='store_true',
+                   help='Create a new branch')
+    p.add_argument('branch_name', nargs='?',
+                   help='Name of the branch to checkout or create (default: revision set in wrap file)')
+    add_common_arguments(p)
+    p.set_defaults(subprojects_func=checkout)
+
+    p = subparsers.add_parser('download', help='Ensure subprojects are fetched, even if not in use. ' +
+                                               'Already downloaded subprojects are not modified. ' +
+                                               'This can be used to pre-fetch all subprojects and avoid downloads during configure.')
+    add_common_arguments(p)
+    p.set_defaults(subprojects_func=download)
+
+def run(options):
+    src_dir = os.path.relpath(os.path.realpath(options.sourcedir))
+    if not os.path.isfile(os.path.join(src_dir, 'meson.build')):
+        mlog.error('Directory', mlog.bold(src_dir), 'does not seem to be a Meson source directory.')
+        return 1
+    subprojects_dir = os.path.join(src_dir, 'subprojects')
+    if not os.path.isdir(subprojects_dir):
+        mlog.log('Directory', mlog.bold(src_dir), 'does not seem to have subprojects.')
+        return 0
+    files = []
+    for name in options.subprojects:
+        f = os.path.join(subprojects_dir, name + '.wrap')
+        if not os.path.isfile(f):
+            mlog.error('Subproject', mlog.bold(name), 'not found.')
+            return 1
+        else:
+            files.append(f)
+    if not files:
+        for f in os.listdir(subprojects_dir):
+            if f.endswith('.wrap'):
+                files.append(os.path.join(subprojects_dir, f))
+    for f in files:
+        wrap = PackageDefinition(f)
+        directory = wrap.values.get('directory', wrap.name)
+        repo_dir = os.path.join(subprojects_dir, directory)
+        options.subprojects_func(wrap, repo_dir, options)
+    return 0
diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py
index 7cad904..f4134d3 100644
--- a/mesonbuild/wrap/wrap.py
+++ b/mesonbuild/wrap/wrap.py
@@ -78,7 +78,9 @@ class WrapNotFoundException(WrapException):
 
 class PackageDefinition:
     def __init__(self, fname):
+        self.filename = fname
         self.basename = os.path.basename(fname)
+        self.name = self.basename[:-5]
         try:
             self.config = configparser.ConfigParser(interpolation=None)
             self.config.read(fname)
diff --git a/mesonbuild/wrap/wraptool.py b/mesonbuild/wrap/wraptool.py
index bb64b5b..132decf 100644
--- a/mesonbuild/wrap/wraptool.py
+++ b/mesonbuild/wrap/wraptool.py
@@ -104,16 +104,24 @@ def install(options):
         f.write(data)
     print('Installed', name, 'branch', branch, 'revision', revision)
 
+def parse_patch_url(patch_url):
+    arr = patch_url.split('/')
+    return arr[-3], int(arr[-2])
+
 def get_current_version(wrapfile):
     cp = configparser.ConfigParser()
     cp.read(wrapfile)
     cp = cp['wrap-file']
     patch_url = cp['patch_url']
-    arr = patch_url.split('/')
-    branch = arr[-3]
-    revision = int(arr[-2])
+    branch, revision = parse_patch_url(patch_url)
     return branch, revision, cp['directory'], cp['source_filename'], cp['patch_filename']
 
+def update_wrap_file(wrapfile, name, new_branch, new_revision):
+    u = open_wrapdburl(API_ROOT + 'projects/%s/%s/%d/get_wrap' % (name, new_branch, new_revision))
+    data = u.read()
+    with open(wrapfile, 'wb') as f:
+        f.write(data)
+
 def update(options):
     name = options.name
     if not os.path.isdir('subprojects'):
@@ -128,8 +136,7 @@ def update(options):
     if new_branch == branch and new_revision == revision:
         print('Project', name, 'is already up to date.')
         sys.exit(0)
-    u = open_wrapdburl(API_ROOT + 'projects/%s/%s/%d/get_wrap' % (name, new_branch, new_revision))
-    data = u.read()
+    update_wrap_file(wrapfile, name, new_branch, new_revision)
     shutil.rmtree(os.path.join('subprojects', subdir), ignore_errors=True)
     try:
         os.unlink(os.path.join('subprojects/packagecache', src_file))
@@ -139,8 +146,6 @@ def update(options):
         os.unlink(os.path.join('subprojects/packagecache', patch_file))
     except FileNotFoundError:
         pass
-    with open(wrapfile, 'wb') as f:
-        f.write(data)
     print('Updated', name, 'to branch', new_branch, 'revision', new_revision)
 
 def info(options):
diff --git a/setup.cfg b/setup.cfg
index 65a3c64..7a94d85 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -26,4 +26,6 @@ ignore =
     E741
     # E722: do not use bare except'
     E722
+    # W504: line break after binary operator
+    W504
 max-line-length = 120