From 4251fbc6fb19cc7406ea450145d307d4b2ec19e9 Mon Sep 17 00:00:00 2001 From: Cristian Ciocaltea Date: Sun, 24 Nov 2019 22:30:26 +0200 Subject: buildman: Improve [make-flags] section parser to allow quoted strings The parser responsible for the '[make-flags]' section in the '.buildman' settings file is currently not able to handle quoted strings, as given in the sample bellow: [make-flags] qemu_arm=HOSTCC="cc -isystem /add/include" HOSTLDFLAGS="-L/add/lib" This patch replaces the simple string splitter based on the delimiter with a regex tokenizer that preserves spaces inside double quoted strings. Signed-off-by: Cristian Ciocaltea --- tools/buildman/toolchain.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index cc26e2e..086a4d4 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -435,9 +435,10 @@ class Toolchains: self._make_flags['target'] = board.target arg_str = self.ResolveReferences(self._make_flags, self._make_flags.get(board.target, '')) - args = arg_str.split(' ') + args = re.findall("(?:\".*?\"|\S)+", arg_str) i = 0 while i < len(args): + args[i] = args[i].replace('"', '') if not args[i]: del args[i] else: -- cgit v1.1 From 69bbdd112bd3d2d6650cb2d81cd583400cef49c7 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 5 Dec 2019 15:59:11 -0700 Subject: genboardcfg: Support a quiet mode We don't really need buildman to print this every time it runs. Add a flag to run quietly, that buildman can use. Signed-off-by: Simon Glass --- tools/genboardscfg.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'tools') diff --git a/tools/genboardscfg.py b/tools/genboardscfg.py index 4ff0bff..24df13e 100755 --- a/tools/genboardscfg.py +++ b/tools/genboardscfg.py @@ -403,18 +403,20 @@ def format_and_output(params_list, output): with open(output, 'w', encoding="utf-8") as f: f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n') -def gen_boards_cfg(output, jobs=1, force=False): +def gen_boards_cfg(output, jobs=1, force=False, quiet=False): """Generate a board database file. Arguments: output: The name of the output file jobs: The number of jobs to run simultaneously force: Force to generate the output even if it is new + quiet: True to avoid printing a message if nothing needs doing """ check_top_directory() if not force and output_is_new(output): - print("%s is up to date. Nothing to do." % output) + if not quiet: + print("%s is up to date. Nothing to do." % output) sys.exit(0) params_list = scan_defconfigs(jobs) @@ -435,9 +437,11 @@ def main(): help='the number of jobs to run simultaneously') parser.add_option('-o', '--output', default=OUTPUT_FILE, help='output file [default=%s]' % OUTPUT_FILE) + parser.add_option('-q', '--quiet', action="store_true", help='run silently') (options, args) = parser.parse_args() - gen_boards_cfg(options.output, jobs=options.jobs, force=options.force) + gen_boards_cfg(options.output, jobs=options.jobs, force=options.force, + quiet=options.quiet) if __name__ == '__main__': main() -- cgit v1.1 From 6a3fc91ea7fdf3f6e35c92eb02c9c73fb9f42d04 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 5 Dec 2019 15:59:12 -0700 Subject: buildman: Ask genboardscfg to be quiet Now that this tool has a 'quiet' flag, use it. Signed-off-by: Simon Glass --- tools/buildman/control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/buildman/control.py b/tools/buildman/control.py index c55a65d..3b41d7b 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -205,7 +205,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, os.makedirs(options.output_dir) board_file = os.path.join(options.output_dir, 'boards.cfg') genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py') - status = subprocess.call([genboardscfg, '-o', board_file]) + status = subprocess.call([genboardscfg, '-q', '-o', board_file]) if status != 0: sys.exit("Failed to generate boards.cfg") -- cgit v1.1 From 7c66ead45267122c48bc854d1d5bd0d9a99bf943 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 5 Dec 2019 15:59:13 -0700 Subject: buildman: Figure out boards before commits At present buildman looks at toolchains, then commits and then boards. Move the board processing up above the commit processing, since it relates to the toolchain code. This will make it easier to check the toolchains needed for a board without processing commits first. Signed-off-by: Simon Glass --- tools/buildman/control.py | 59 +++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 30 deletions(-) (limited to 'tools') diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 3b41d7b..a9c5022 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -170,35 +170,6 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, print() return 0 - # Work out how many commits to build. We want to build everything on the - # branch. We also build the upstream commit as a control so we can see - # problems introduced by the first commit on the branch. - count = options.count - has_range = options.branch and '..' in options.branch - if count == -1: - if not options.branch: - count = 1 - else: - if has_range: - count, msg = gitutil.CountCommitsInRange(options.git_dir, - options.branch) - else: - count, msg = gitutil.CountCommitsInBranch(options.git_dir, - options.branch) - if count is None: - sys.exit(col.Color(col.RED, msg)) - elif count == 0: - sys.exit(col.Color(col.RED, "Range '%s' has no commits" % - options.branch)) - if msg: - print(col.Color(col.YELLOW, msg)) - count += 1 # Build upstream commit also - - if not count: - str = ("No commits found to process in branch '%s': " - "set branch's upstream or use -c flag" % options.branch) - sys.exit(col.Color(col.RED, str)) - # Work out what subset of the boards we are building if not boards: if not os.path.exists(options.output_dir): @@ -217,7 +188,6 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, for arg in options.exclude: exclude += arg.split(',') - if options.boards: requested_boards = [] for b in options.boards: @@ -230,6 +200,35 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, if not len(selected): sys.exit(col.Color(col.RED, 'No matching boards found')) + # Work out how many commits to build. We want to build everything on the + # branch. We also build the upstream commit as a control so we can see + # problems introduced by the first commit on the branch. + count = options.count + has_range = options.branch and '..' in options.branch + if count == -1: + if not options.branch: + count = 1 + else: + if has_range: + count, msg = gitutil.CountCommitsInRange(options.git_dir, + options.branch) + else: + count, msg = gitutil.CountCommitsInBranch(options.git_dir, + options.branch) + if count is None: + sys.exit(col.Color(col.RED, msg)) + elif count == 0: + sys.exit(col.Color(col.RED, "Range '%s' has no commits" % + options.branch)) + if msg: + print(col.Color(col.YELLOW, msg)) + count += 1 # Build upstream commit also + + if not count: + str = ("No commits found to process in branch '%s': " + "set branch's upstream or use -c flag" % options.branch) + sys.exit(col.Color(col.RED, str)) + # Read the metadata from the commits. First look at the upstream commit, # then the ones in the branch. We would like to do something like # upstream/master~..branch but that isn't possible if upstream/master is -- cgit v1.1 From 57cb9d52397c9051d7b3d27bd107cce04a16846f Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 5 Dec 2019 15:59:14 -0700 Subject: buildman: Add options to get the arch and toolchain info Sometimes it is useful for external tools to use buildman to provide the toolchain information. Add an -a option which shows the value to use for the ARCH environment variable, and -A which does the same for CROSS_COMPILE Signed-off-by: Simon Glass --- tools/buildman/README | 3 +++ tools/buildman/cmdline.py | 4 ++++ tools/buildman/control.py | 35 +++++++++++++++++++++++++++++++++++ tools/buildman/test.py | 18 ++++++++++++++++++ tools/buildman/toolchain.py | 26 ++++++++++++++++++++++++++ 5 files changed, 86 insertions(+) (limited to 'tools') diff --git a/tools/buildman/README b/tools/buildman/README index e366192..c1ac0d0 100644 --- a/tools/buildman/README +++ b/tools/buildman/README @@ -1061,6 +1061,9 @@ Other options Buildman has various other command line options. Try --help to see them. +To find out what architecture or toolchain prefix buildman will use for a build, +see the -a and -A options. + When doing builds, Buildman's return code will reflect the overall result: 0 (success) No errors or warnings found diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index 832a514..b412093 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -13,6 +13,10 @@ def ParseArgs(): args: command lin arguments """ parser = OptionParser() + parser.add_option('-a', '--print-arch', action='store_true', + help='Print the architecture for a board (ARCH=)') + parser.add_option('-A', '--print-prefix', action='store_true', + help='Print the tool-chain prefix for a board (CROSS_COMPILE=)') parser.add_option('-b', '--branch', type='string', help='Branch name to build, or range of commits to build') parser.add_option('-B', '--bloat', dest='show_bloat', diff --git a/tools/buildman/control.py b/tools/buildman/control.py index a9c5022..969d866 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -107,6 +107,34 @@ def CheckOutputDir(output_dir): break path = parent +def ShowToolchainInfo(boards, toolchains, print_arch, print_prefix): + """Show information about a the tool chain used by one or more boards + + The function checks that all boards use the same toolchain. + + Args: + boards: Boards object containing selected boards + toolchains: Toolchains object containing available toolchains + print_arch: True to print ARCH value + print_prefix: True to print CROSS_COMPILE value + + Return: + None on success, string error message otherwise + """ + boards = boards.GetSelectedDict() + tc_set = set() + for brd in boards.values(): + tc_set.add(toolchains.Select(brd.arch)) + if len(tc_set) != 1: + return 'Supplied boards must share one toolchain' + return False + tc = tc_set.pop() + if print_arch: + print(tc.GetEnvArgs(toolchain.VAR_ARCH)) + if print_prefix: + print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) + return None + def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, clean_dir=False): """The main control code for buildman @@ -200,6 +228,13 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, if not len(selected): sys.exit(col.Color(col.RED, 'No matching boards found')) + if options.print_arch or options.print_prefix: + err = ShowToolchainInfo(boards, toolchains, options.print_arch, + options.print_prefix) + if err: + sys.exit(col.Color(col.RED, err)) + return 0 + # Work out how many commits to build. We want to build everything on the # branch. We also build the upstream commit as a control so we can see # problems introduced by the first commit on the branch. diff --git a/tools/buildman/test.py b/tools/buildman/test.py index b4e28d6..acd862b 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -451,6 +451,24 @@ class TestBuild(unittest.TestCase): 'crosstool/files/bin/x86_64/.*/' 'x86_64-gcc-.*-nolibc_arm-.*linux-gnueabi.tar.xz') + def testGetEnvArgs(self): + """Test the GetEnvArgs() function""" + tc = self.toolchains.Select('arm') + self.assertEqual('arm-linux-', + tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) + self.assertEqual('', tc.GetEnvArgs(toolchain.VAR_PATH)) + self.assertEqual('arm', + tc.GetEnvArgs(toolchain.VAR_ARCH)) + self.assertEqual('', tc.GetEnvArgs(toolchain.VAR_MAKE_ARGS)) + + self.toolchains.Add('/path/to/x86_64-linux-gcc', test=False) + tc = self.toolchains.Select('x86') + self.assertEqual('/path/to', + tc.GetEnvArgs(toolchain.VAR_PATH)) + tc.override_toolchain = 'clang' + self.assertEqual('HOSTCC=clang CC=clang', + tc.GetEnvArgs(toolchain.VAR_MAKE_ARGS)) + if __name__ == "__main__": unittest.main() diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index 086a4d4..4f39bfd 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -18,6 +18,8 @@ import tools (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH, PRIORITY_CALC) = list(range(4)) +(VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4) + # Simple class to collect links from a page class MyHTMLParser(HTMLParser): def __init__(self, arch): @@ -145,6 +147,30 @@ class Toolchain: return value + def GetEnvArgs(self, which): + """Get an environment variable/args value based on the the toolchain + + Args: + which: VAR_... value to get + + Returns: + Value of that environment variable or arguments + """ + wrapper = self.GetWrapper() + if which == VAR_CROSS_COMPILE: + return wrapper + os.path.join(self.path, self.cross) + elif which == VAR_PATH: + return self.path + elif which == VAR_ARCH: + return self.arch + elif which == VAR_MAKE_ARGS: + args = self.MakeArgs() + if args: + return ' '.join(args) + return '' + else: + raise ValueError('Unknown arg to GetEnvArgs (%d)' % which) + def MakeEnvironment(self, full_path): """Returns an environment for using the toolchain. -- cgit v1.1 From d08c38c32f64498946230059bd4ec1431c23476a Mon Sep 17 00:00:00 2001 From: Tom Rini Date: Fri, 6 Dec 2019 15:31:31 -0500 Subject: buildman: Ignore blank lines during size checks Today when parsing the .sizes files we get a warning about an invalid line in the file as it's blank. Solve this by checking that we have a non-blank line prior to processing. Cc: Simon Glass Signed-off-by: Tom Rini Reviewed-by: Simon Glass --- tools/buildman/builder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index cfbe4c2..784c641 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -577,7 +577,8 @@ class Builder: sym = {} for line in fd.readlines(): try: - size, type, name = line[:-1].split() + if line.strip(): + size, type, name = line[:-1].split() except: Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) continue -- cgit v1.1