aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJoseph Myers <joseph@codesourcery.com>2016-11-24 22:25:58 +0000
committerJoseph Myers <joseph@codesourcery.com>2016-11-24 22:25:58 +0000
commit02c78f02a941461efa566fed8a0209592b0943fc (patch)
tree424eda074045526f18cc7d7658d08ab3412ea06f
parentc579f48edba88380635ab98cb612030e3ed8691e (diff)
downloadglibc-02c78f02a941461efa566fed8a0209592b0943fc.zip
glibc-02c78f02a941461efa566fed8a0209592b0943fc.tar.gz
glibc-02c78f02a941461efa566fed8a0209592b0943fc.tar.bz2
Make build-many-glibcs.py track component versions requested and used.
This patch makes build-many-glibcs.py record the component versions checked out, and whether those versions were explicitly requested or defaults, in a file versions.json in the source directory. The preferred version of a component is the first of: one explicitly specified on the command line; one explicitly specified on the command line in a previous run of build-many-glibcs.py; a default version for that component. Thus, once you've run build-many-glibcs.py checkout once with the updated script (making sure to specify the right versions of any components previously checked out with a non-default version), in future you can just run it without version specifiers and it will know when a default-version component has changed its default version and so should be checked out again. Because you might have local changes and not want a default-version component checkout replaced, you need to pass the --replace-sources option to allow the script to delete and replace a component source directory automatically; otherwise, it will give an error if a version has changed. The script does not try to change branches of git or SVN checkouts without checking out from scratch; if the version number requested has changed and --replace-sources is used, the relevant source directory will be removed completely and a new one checked out from scratch. Apart from allowing automatic updates of components with default versions, this also facilitates bots reporting on the versions used in a given build. versions.json contains not just information on the version number and whether that was requested explicitly, but also git or SVN revision information intended to be used in email reports from bots. * scripts/build-many-glibcs.py: Import json module. (Context.__init__): Take replace_sources argument. Load versions.json. (Context.load_versions_json): New function. (Context.store_json): Likewise. (Context.store_versions_json): Likewise. (Context.set_component_version): Likewise. (Context.checkout): Update versions.json. Check for and handle changes of version. Prefer previously explicitly specified version to default version. (Context.checkout_vcs): Return a revision identifier. (Context.git_checkout): Likewise. (Context.gcc_checkout): Likewise. (get_parser): Add --replace-sources option. (main): Pass replace_sources argument to Context call.
-rw-r--r--ChangeLog18
-rwxr-xr-xscripts/build-many-glibcs.py83
2 files changed, 91 insertions, 10 deletions
diff --git a/ChangeLog b/ChangeLog
index 6985be2..03597a7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,21 @@
+2016-11-24 Joseph Myers <joseph@codesourcery.com>
+
+ * scripts/build-many-glibcs.py: Import json module.
+ (Context.__init__): Take replace_sources argument. Load
+ versions.json.
+ (Context.load_versions_json): New function.
+ (Context.store_json): Likewise.
+ (Context.store_versions_json): Likewise.
+ (Context.set_component_version): Likewise.
+ (Context.checkout): Update versions.json. Check for and handle
+ changes of version. Prefer previously explicitly specified
+ version to default version.
+ (Context.checkout_vcs): Return a revision identifier.
+ (Context.git_checkout): Likewise.
+ (Context.gcc_checkout): Likewise.
+ (get_parser): Add --replace-sources option.
+ (main): Pass replace_sources argument to Context call.
+
2016-11-24 Adhemerval Zanella <adhemerval.zanella@linaro.org>
* sysdeps/nptl/fork.c (__libc_fork): Remove pid cache setting.
diff --git a/scripts/build-many-glibcs.py b/scripts/build-many-glibcs.py
index 1447e22..72f22a8 100755
--- a/scripts/build-many-glibcs.py
+++ b/scripts/build-many-glibcs.py
@@ -32,6 +32,7 @@ configurations for which compilers or glibc are to be built.
"""
import argparse
+import json
import os
import re
import shutil
@@ -44,12 +45,14 @@ import urllib.request
class Context(object):
"""The global state associated with builds in a given directory."""
- def __init__(self, topdir, parallelism, keep, action):
+ def __init__(self, topdir, parallelism, keep, replace_sources, action):
"""Initialize the context."""
self.topdir = topdir
self.parallelism = parallelism
self.keep = keep
+ self.replace_sources = replace_sources
self.srcdir = os.path.join(topdir, 'src')
+ self.versions_json = os.path.join(self.srcdir, 'versions.json')
self.installdir = os.path.join(topdir, 'install')
self.host_libraries_installdir = os.path.join(self.installdir,
'host-libraries')
@@ -65,6 +68,7 @@ class Context(object):
self.glibc_configs = {}
self.makefile_pieces = ['.PHONY: all\n']
self.add_all_configs()
+ self.load_versions_json()
def get_build_triplet(self):
"""Determine the build triplet with config.guess."""
@@ -559,6 +563,32 @@ class Context(object):
for c in configs:
self.glibc_configs[c].build()
+ def load_versions_json(self):
+ """Load information about source directory versions."""
+ if not os.access(self.versions_json, os.F_OK):
+ self.versions = {}
+ return
+ with open(self.versions_json, 'r') as f:
+ self.versions = json.load(f)
+
+ def store_json(self, data, filename):
+ """Store information in a JSON file."""
+ filename_tmp = filename + '.tmp'
+ with open(filename_tmp, 'w') as f:
+ json.dump(data, f, indent=2, sort_keys=True)
+ os.rename(filename_tmp, filename)
+
+ def store_versions_json(self):
+ """Store information about source directory versions."""
+ self.store_json(self.versions, self.versions_json)
+
+ def set_component_version(self, component, version, explicit, revision):
+ """Set the version information for a component."""
+ self.versions[component] = {'version': version,
+ 'explicit': explicit,
+ 'revision': revision}
+ self.store_versions_json()
+
def checkout(self, versions):
"""Check out the desired component versions."""
default_versions = {'binutils': 'vcs-2.27',
@@ -569,6 +599,7 @@ class Context(object):
'mpc': '1.0.3',
'mpfr': '3.1.5'}
use_versions = {}
+ explicit_versions = {}
for v in versions:
found_v = False
for k in default_versions.keys():
@@ -579,6 +610,7 @@ class Context(object):
print('error: multiple versions for %s' % k)
exit(1)
use_versions[k] = vx
+ explicit_versions[k] = True
found_v = True
break
if not found_v:
@@ -586,19 +618,36 @@ class Context(object):
exit(1)
for k in default_versions.keys():
if k not in use_versions:
- use_versions[k] = default_versions[k]
+ if k in self.versions and self.versions[k]['explicit']:
+ use_versions[k] = self.versions[k]['version']
+ explicit_versions[k] = True
+ else:
+ use_versions[k] = default_versions[k]
+ explicit_versions[k] = False
os.makedirs(self.srcdir, exist_ok=True)
for k in sorted(default_versions.keys()):
update = os.access(self.component_srcdir(k), os.F_OK)
v = use_versions[k]
+ if (update and
+ k in self.versions and
+ v != self.versions[k]['version']):
+ if not self.replace_sources:
+ print('error: version of %s has changed from %s to %s, '
+ 'use --replace-sources to check out again' %
+ (k, self.versions[k]['version'], v))
+ exit(1)
+ shutil.rmtree(self.component_srcdir(k))
+ update = False
if v.startswith('vcs-'):
- self.checkout_vcs(k, v[4:], update)
+ revision = self.checkout_vcs(k, v[4:], update)
else:
self.checkout_tar(k, v, update)
+ revision = v
+ self.set_component_version(k, v, explicit_versions[k], revision)
def checkout_vcs(self, component, version, update):
"""Check out the given version of the given component from version
- control."""
+ control. Return a revision identifier."""
if component == 'binutils':
git_url = 'git://sourceware.org/git/binutils-gdb.git'
if version == 'mainline':
@@ -606,7 +655,7 @@ class Context(object):
else:
trans = str.maketrans({'.': '_'})
git_branch = 'binutils-%s-branch' % version.translate(trans)
- self.git_checkout(component, git_url, git_branch, update)
+ return self.git_checkout(component, git_url, git_branch, update)
elif component == 'gcc':
if version == 'mainline':
branch = 'trunk'
@@ -614,21 +663,22 @@ class Context(object):
trans = str.maketrans({'.': '_'})
branch = 'branches/gcc-%s-branch' % version.translate(trans)
svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
- self.gcc_checkout(svn_url, update)
+ return self.gcc_checkout(svn_url, update)
elif component == 'glibc':
git_url = 'git://sourceware.org/git/glibc.git'
if version == 'mainline':
git_branch = 'master'
else:
git_branch = 'release/%s/master' % version
- self.git_checkout(component, git_url, git_branch, update)
+ r = self.git_checkout(component, git_url, git_branch, update)
self.fix_glibc_timestamps()
+ return r
else:
print('error: component %s coming from VCS' % component)
exit(1)
def git_checkout(self, component, git_url, git_branch, update):
- """Check out a component from git."""
+ """Check out a component from git. Return a commit identifier."""
if update:
subprocess.run(['git', 'remote', 'prune', 'origin'],
cwd=self.component_srcdir(component), check=True)
@@ -637,6 +687,11 @@ class Context(object):
else:
subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
self.component_srcdir(component)], check=True)
+ r = subprocess.run(['git', 'rev-parse', 'HEAD'],
+ cwd=self.component_srcdir(component),
+ stdout=subprocess.PIPE,
+ check=True, universal_newlines=True).stdout
+ return r.rstrip()
def fix_glibc_timestamps(self):
"""Fix timestamps in a glibc checkout."""
@@ -652,12 +707,16 @@ class Context(object):
subprocess.run(['touch', to_touch], check=True)
def gcc_checkout(self, svn_url, update):
- """Check out GCC from SVN."""
+ """Check out GCC from SVN. Return the revision number."""
if not update:
subprocess.run(['svn', 'co', '-q', svn_url,
self.component_srcdir('gcc')], check=True)
subprocess.run(['contrib/gcc_update', '--silent'],
cwd=self.component_srcdir('gcc'), check=True)
+ r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
+ stdout=subprocess.PIPE,
+ check=True, universal_newlines=True).stdout
+ return r.rstrip()
def checkout_tar(self, component, version, update):
"""Check out the given version of the given component from a
@@ -1125,6 +1184,9 @@ def get_parser():
help='Whether to keep all build directories, '
'none or only those from failed builds',
default='none', choices=('none', 'all', 'failed'))
+ parser.add_argument('--replace-sources', action='store_true',
+ help='Remove and replace source directories '
+ 'with the wrong version of a component')
parser.add_argument('topdir',
help='Toplevel working directory')
parser.add_argument('action',
@@ -1142,7 +1204,8 @@ def main(argv):
parser = get_parser()
opts = parser.parse_args(argv)
topdir = os.path.abspath(opts.topdir)
- ctx = Context(topdir, opts.parallelism, opts.keep, opts.action)
+ ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
+ opts.action)
ctx.run_builds(opts.action, opts.configs)