diff options
Diffstat (limited to 'tools/buildman/builderthread.py')
| -rw-r--r-- | tools/buildman/builderthread.py | 652 |
1 files changed, 403 insertions, 249 deletions
diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index 635865c..25f460c 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -2,8 +2,15 @@ # Copyright (c) 2014 Google, Inc # +"""Implementation the bulider threads + +This module provides the BuilderThread class, which handles calling the builder +based on the jobs provided. +""" + import errno import glob +import io import os import shutil import sys @@ -16,11 +23,15 @@ from u_boot_pylib import command RETURN_CODE_RETRY = -1 BASE_ELF_FILENAMES = ['u-boot', 'spl/u-boot-spl', 'tpl/u-boot-tpl'] -def Mkdir(dirname, parents = False): +def mkdir(dirname, parents=False): """Make a directory if it doesn't already exist. Args: - dirname: Directory to create + dirname (str): Directory to create + parents (bool): True to also make parent directories + + Raises: + OSError: File already exists """ try: if parents: @@ -30,12 +41,51 @@ def Mkdir(dirname, parents = False): except OSError as err: if err.errno == errno.EEXIST: if os.path.realpath('.') == os.path.realpath(dirname): - print("Cannot create the current working directory '%s'!" % dirname) + print(f"Cannot create the current working directory '{dirname}'!") sys.exit(1) - pass else: raise + +def _remove_old_outputs(out_dir): + """Remove any old output-target files + + Args: + out_dir (str): Output directory for the build + + Since we use a build directory that was previously used by another + board, it may have produced an SPL image. If we don't remove it (i.e. + see do_config and self.mrproper below) then it will appear to be the + output of this build, even if it does not produce SPL images. + """ + for elf in BASE_ELF_FILENAMES: + fname = os.path.join(out_dir, elf) + if os.path.exists(fname): + os.remove(fname) + + +def copy_files(out_dir, build_dir, dirname, patterns): + """Copy files from the build directory to the output. + + Args: + out_dir (str): Path to output directory containing the files + build_dir (str): Place to copy the files + dirname (str): Source directory, '' for normal U-Boot, 'spl' for SPL + patterns (list of str): A list of filenames to copy, each relative + to the build directory + """ + for pattern in patterns: + file_list = glob.glob(os.path.join(out_dir, dirname, pattern)) + for fname in file_list: + target = os.path.basename(fname) + if dirname: + base, ext = os.path.splitext(target) + if ext: + target = f'{base}-{dirname}{ext}' + shutil.copy(fname, os.path.join(build_dir, target)) + + +# pylint: disable=R0903 class BuilderJob: """Holds information about a job to be performed by a thread @@ -77,7 +127,7 @@ class ResultThread(threading.Thread): """ while True: result = self.builder.out_queue.get() - self.builder.ProcessResult(result) + self.builder.process_result(result) self.builder.out_queue.task_done() @@ -107,22 +157,25 @@ class BuilderThread(threading.Thread): self.mrproper = mrproper self.per_board_out_dir = per_board_out_dir self.test_exception = test_exception + self.toolchain = None - def Make(self, commit, brd, stage, cwd, *args, **kwargs): + def make(self, commit, brd, stage, cwd, *args, **kwargs): """Run 'make' on a particular commit and board. The source code will already be checked out, so the 'commit' argument is only for information. Args: - commit: Commit object that is being built - brd: Board object that is being built - stage: Stage of the build. Valid stages are: + commit (Commit): Commit that is being built + brd (Board): Board that is being built + stage (str): Stage of the build. Valid stages are: mrproper - can be called to clean source config - called to configure for a board build - the main make invocation - it does the build - args: A list of arguments to pass to 'make' - kwargs: A list of keyword arguments to pass to command.run_pipe() + cwd (str): Working directory to set, or None to leave it alone + *args (list of str): Arguments to pass to 'make' + **kwargs (dict): A list of keyword arguments to pass to + command.run_pipe() Returns: CommandResult object @@ -130,61 +183,140 @@ class BuilderThread(threading.Thread): return self.builder.do_make(commit, brd, stage, cwd, *args, **kwargs) - def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only, - force_build, force_build_failures, work_in_output, - adjust_cfg): - """Build a particular commit. - - If the build is already done, and we are not forcing a build, we skip - the build and just return the previously-saved results. + def _build_args(self, brd, out_dir, out_rel_dir, work_dir, commit_upto): + """Set up arguments to the args list based on the settings Args: - commit_upto: Commit number to build (0...n-1) - brd: Board object to build - work_dir: Directory to which the source will be checked out - do_config: True to run a make <board>_defconfig on the source - config_only: Only configure the source, do not build it - force_build: Force a build even if one was previously done - force_build_failures: Force a bulid if the previous result showed - failure - work_in_output: Use the output directory as the work directory and - don't write to a separate output directory. - adjust_cfg (list of str): List of changes to make to .config file - before building. Each is one of (where C is either CONFIG_xxx - or just xxx): - C to enable C - ~C to disable C - C=val to set the value of C (val must have quotes if C is - a string Kconfig + brd (Board): Board to create arguments for + out_dir (str): Path to output directory containing the files + out_rel_dir (str): Output directory relative to the current dir + work_dir (str): Directory to which the source will be checked out + commit_upto (int): Commit number to build (0...n-1) Returns: - tuple containing: - - CommandResult object containing the results of the build - - boolean indicating whether 'make config' is still needed + tuple: + list of str: Arguments to pass to make + str: Current working directory, or None if no commit + str: Source directory (typically the work directory) """ - # Create a default result - it will be overwritte by the call to - # self.Make() below, in the event that we do a build. - result = command.CommandResult() - result.return_code = 0 - if work_in_output or self.builder.in_tree: - out_dir = work_dir - else: - if self.per_board_out_dir: - out_rel_dir = os.path.join('..', brd.target) + args = [] + cwd = work_dir + src_dir = os.path.realpath(work_dir) + if not self.builder.in_tree: + if commit_upto is None: + # In this case we are building in the original source directory + # (i.e. the current directory where buildman is invoked. The + # output directory is set to this thread's selected work + # directory. + # + # Symlinks can confuse U-Boot's Makefile since we may use '..' + # in our path, so remove them. + real_dir = os.path.realpath(out_dir) + args.append(f'O={real_dir}') + cwd = None + src_dir = os.getcwd() else: - out_rel_dir = 'build' - out_dir = os.path.join(work_dir, out_rel_dir) + args.append(f'O={out_rel_dir}') + if self.builder.verbose_build: + args.append('V=1') + else: + args.append('-s') + if self.builder.num_jobs is not None: + args.extend(['-j', str(self.builder.num_jobs)]) + if self.builder.warnings_as_errors: + args.append('KCFLAGS=-Werror') + args.append('HOSTCFLAGS=-Werror') + if self.builder.allow_missing: + args.append('BINMAN_ALLOW_MISSING=1') + if self.builder.no_lto: + args.append('NO_LTO=1') + if self.builder.reproducible_builds: + args.append('SOURCE_DATE_EPOCH=0') + args.extend(self.builder.toolchains.GetMakeArguments(brd)) + args.extend(self.toolchain.MakeArgs()) + return args, cwd, src_dir + + def _reconfigure(self, commit, brd, cwd, args, env, config_args, config_out, + cmd_list): + """Reconfigure the build - # Check if the job was already completed last time - done_file = self.builder.GetDoneFile(commit_upto, brd.target) + Args: + commit (Commit): Commit only being built + brd (Board): Board being built + cwd (str): Current working directory + args (list of str): Arguments to pass to make + env (dict): Environment strings + config_args (list of str): defconfig arg for this board + cmd_list (list of str): List to add the commands to, for logging + + Returns: + CommandResult object + """ + if self.mrproper: + result = self.make(commit, brd, 'mrproper', cwd, 'mrproper', *args, + env=env) + config_out.write(result.combined) + cmd_list.append([self.builder.gnu_make, 'mrproper', *args]) + result = self.make(commit, brd, 'config', cwd, *(args + config_args), + env=env) + cmd_list.append([self.builder.gnu_make] + args + config_args) + config_out.write(result.combined) + return result + + def _build(self, commit, brd, cwd, args, env, cmd_list, config_only): + """Perform the build + + Args: + commit (Commit): Commit only being built + brd (Board): Board being built + cwd (str): Current working directory + args (list of str): Arguments to pass to make + env (dict): Environment strings + cmd_list (list of str): List to add the commands to, for logging + config_only (bool): True if this is a config-only build (using the + 'make cfg' target) + + Returns: + CommandResult object + """ + if config_only: + args.append('cfg') + result = self.make(commit, brd, 'build', cwd, *args, env=env) + cmd_list.append([self.builder.gnu_make] + args) + if (result.return_code == 2 and + ('Some images are invalid' in result.stderr)): + # This is handled later by the check for output in stderr + result.return_code = 0 + return result + + def _read_done_file(self, commit_upto, brd, force_build, + force_build_failures): + """Check the 'done' file and see if this commit should be built + + Args: + commit (Commit): Commit only being built + brd (Board): Board being built + force_build (bool): Force a build even if one was previously done + force_build_failures (bool): Force a bulid if the previous result + showed failure + + Returns: + tuple: + bool: True if build should be built + CommandResult: if there was a previous run: + - already_done set to True + - return_code set to return code + - result.stderr set to 'bad' if stderr output was recorded + """ + result = command.CommandResult() + done_file = self.builder.get_done_file(commit_upto, brd.target) result.already_done = os.path.exists(done_file) will_build = (force_build or force_build_failures or not result.already_done) if result.already_done: - # Get the return code from that build and use it - with open(done_file, 'r') as fd: + with open(done_file, 'r', encoding='utf-8') as outf: try: - result.return_code = int(fd.readline()) + result.return_code = int(outf.readline()) except ValueError: # The file may be empty due to running out of disk space. # Try a rebuild @@ -194,12 +326,155 @@ class BuilderThread(threading.Thread): if result.return_code == RETURN_CODE_RETRY: will_build = True elif will_build: - err_file = self.builder.GetErrFile(commit_upto, brd.target) + err_file = self.builder.get_err_file(commit_upto, brd.target) if os.path.exists(err_file) and os.stat(err_file).st_size: result.stderr = 'bad' elif not force_build: # The build passed, so no need to build it again will_build = False + return will_build, result + + def _decide_dirs(self, brd, work_dir, work_in_output): + """Decide the output directory to use + + Args: + work_dir (str): Directory to which the source will be checked out + work_in_output (bool): Use the output directory as the work + directory and don't write to a separate output directory. + + Returns: + tuple: + out_dir (str): Output directory for the build + out_rel_dir (str): Output directory relatie to the current dir + """ + if work_in_output or self.builder.in_tree: + out_rel_dir = None + out_dir = work_dir + else: + if self.per_board_out_dir: + out_rel_dir = os.path.join('..', brd.target) + else: + out_rel_dir = 'build' + out_dir = os.path.join(work_dir, out_rel_dir) + return out_dir, out_rel_dir + + def _checkout(self, commit_upto, work_dir): + """Checkout the right commit + + Args: + commit_upto (int): Commit number to build (0...n-1) + work_dir (str): Directory to which the source will be checked out + + Returns: + Commit: Commit being built, or 'current' for current source + """ + if self.builder.commits: + commit = self.builder.commits[commit_upto] + if self.builder.checkout: + git_dir = os.path.join(work_dir, '.git') + gitutil.checkout(commit.hash, git_dir, work_dir, force=True) + else: + commit = 'current' + return commit + + def _config_and_build(self, commit_upto, brd, work_dir, do_config, + config_only, adjust_cfg, commit, out_dir, out_rel_dir, + result): + """Do the build, configuring first if necessary + + Args: + commit_upto (int): Commit number to build (0...n-1) + brd (Board): Board to create arguments for + work_dir (str): Directory to which the source will be checked out + do_config (bool): True to run a make <board>_defconfig on the source + config_only (bool): Only configure the source, do not build it + adjust_cfg (list of str): See the cfgutil module and run_commit() + commit (Commit): Commit only being built + out_dir (str): Output directory for the build + out_rel_dir (str): Output directory relatie to the current dir + result (CommandResult): Previous result + + Returns: + tuple: + result (CommandResult): Result of the build + do_config (bool): indicates whether 'make config' is needed on + the next incremental build + """ + # Set up the environment and command line + env = self.toolchain.MakeEnvironment(self.builder.full_path) + mkdir(out_dir) + + args, cwd, src_dir = self._build_args(brd, out_dir, out_rel_dir, + work_dir, commit_upto) + config_args = [f'{brd.target}_defconfig'] + config_out = io.StringIO() + + _remove_old_outputs(out_dir) + + # If we need to reconfigure, do that now + cfg_file = os.path.join(out_dir, '.config') + cmd_list = [] + if do_config or adjust_cfg: + result = self._reconfigure( + commit, brd, cwd, args, env, config_args, config_out, cmd_list) + do_config = False # No need to configure next time + if adjust_cfg: + cfgutil.adjust_cfg_file(cfg_file, adjust_cfg) + + # Now do the build, if everything looks OK + if result.return_code == 0: + result = self._build(commit, brd, cwd, args, env, cmd_list, + config_only) + if adjust_cfg: + errs = cfgutil.check_cfg_file(cfg_file, adjust_cfg) + if errs: + result.stderr += errs + result.return_code = 1 + result.stderr = result.stderr.replace(src_dir + '/', '') + if self.builder.verbose_build: + result.stdout = config_out.getvalue() + result.stdout + result.cmd_list = cmd_list + return result, do_config + + def run_commit(self, commit_upto, brd, work_dir, do_config, config_only, + force_build, force_build_failures, work_in_output, + adjust_cfg): + """Build a particular commit. + + If the build is already done, and we are not forcing a build, we skip + the build and just return the previously-saved results. + + Args: + commit_upto (int): Commit number to build (0...n-1) + brd (Board): Board to build + work_dir (str): Directory to which the source will be checked out + do_config (bool): True to run a make <board>_defconfig on the source + config_only (bool): Only configure the source, do not build it + force_build (bool): Force a build even if one was previously done + force_build_failures (bool): Force a bulid if the previous result + showed failure + work_in_output (bool) : Use the output directory as the work + directory and don't write to a separate output directory. + adjust_cfg (list of str): List of changes to make to .config file + before building. Each is one of (where C is either CONFIG_xxx + or just xxx): + C to enable C + ~C to disable C + C=val to set the value of C (val must have quotes if C is + a string Kconfig + + Returns: + tuple containing: + - CommandResult object containing the results of the build + - boolean indicating whether 'make config' is still needed + """ + # Create a default result - it will be overwritte by the call to + # self.make() below, in the event that we do a build. + out_dir, out_rel_dir = self._decide_dirs(brd, work_dir, work_in_output) + + # Check if the job was already completed last time + will_build, result = self._read_done_file(commit_upto, brd, force_build, + force_build_failures) if will_build: # We are going to have to build it. First, get a toolchain @@ -209,115 +484,13 @@ class BuilderThread(threading.Thread): except ValueError as err: result.return_code = 10 result.stdout = '' - result.stderr = str(err) - # TODO(sjg@chromium.org): This gets swallowed, but needs - # to be reported. + result.stderr = f'Tool chain error for {brd.arch}: {str(err)}' if self.toolchain: - # Checkout the right commit - if self.builder.commits: - commit = self.builder.commits[commit_upto] - if self.builder.checkout: - git_dir = os.path.join(work_dir, '.git') - gitutil.checkout(commit.hash, git_dir, work_dir, - force=True) - else: - commit = 'current' - - # Set up the environment and command line - env = self.toolchain.MakeEnvironment(self.builder.full_path) - Mkdir(out_dir) - args = [] - cwd = work_dir - src_dir = os.path.realpath(work_dir) - if not self.builder.in_tree: - if commit_upto is None: - # In this case we are building in the original source - # directory (i.e. the current directory where buildman - # is invoked. The output directory is set to this - # thread's selected work directory. - # - # Symlinks can confuse U-Boot's Makefile since - # we may use '..' in our path, so remove them. - out_dir = os.path.realpath(out_dir) - args.append('O=%s' % out_dir) - cwd = None - src_dir = os.getcwd() - else: - args.append('O=%s' % out_rel_dir) - if self.builder.verbose_build: - args.append('V=1') - else: - args.append('-s') - if self.builder.num_jobs is not None: - args.extend(['-j', str(self.builder.num_jobs)]) - if self.builder.warnings_as_errors: - args.append('KCFLAGS=-Werror') - args.append('HOSTCFLAGS=-Werror') - if self.builder.allow_missing: - args.append('BINMAN_ALLOW_MISSING=1') - if self.builder.no_lto: - args.append('NO_LTO=1') - if self.builder.reproducible_builds: - args.append('SOURCE_DATE_EPOCH=0') - config_args = ['%s_defconfig' % brd.target] - config_out = '' - args.extend(self.builder.toolchains.GetMakeArguments(brd)) - args.extend(self.toolchain.MakeArgs()) - - # Remove any output targets. Since we use a build directory that - # was previously used by another board, it may have produced an - # SPL image. If we don't remove it (i.e. see do_config and - # self.mrproper below) then it will appear to be the output of - # this build, even if it does not produce SPL images. - build_dir = self.builder.GetBuildDir(commit_upto, brd.target) - for elf in BASE_ELF_FILENAMES: - fname = os.path.join(out_dir, elf) - if os.path.exists(fname): - os.remove(fname) - - # If we need to reconfigure, do that now - cfg_file = os.path.join(out_dir, '.config') - cmd_list = [] - if do_config or adjust_cfg: - config_out = '' - if self.mrproper: - result = self.Make(commit, brd, 'mrproper', cwd, - 'mrproper', *args, env=env) - config_out += result.combined - cmd_list.append([self.builder.gnu_make, 'mrproper', - *args]) - result = self.Make(commit, brd, 'config', cwd, - *(args + config_args), env=env) - cmd_list.append([self.builder.gnu_make] + args + - config_args) - config_out += result.combined - do_config = False # No need to configure next time - if adjust_cfg: - cfgutil.adjust_cfg_file(cfg_file, adjust_cfg) - if result.return_code == 0: - if config_only: - args.append('cfg') - result = self.Make(commit, brd, 'build', cwd, *args, - env=env) - cmd_list.append([self.builder.gnu_make] + args) - if (result.return_code == 2 and - ('Some images are invalid' in result.stderr)): - # This is handled later by the check for output in - # stderr - result.return_code = 0 - if adjust_cfg: - errs = cfgutil.check_cfg_file(cfg_file, adjust_cfg) - if errs: - result.stderr += errs - result.return_code = 1 - result.stderr = result.stderr.replace(src_dir + '/', '') - if self.builder.verbose_build: - result.stdout = config_out + result.stdout - result.cmd_list = cmd_list - else: - result.return_code = 1 - result.stderr = 'No tool chain for %s\n' % brd.arch + commit = self._checkout(commit_upto, work_dir) + result, do_config = self._config_and_build( + commit_upto, brd, work_dir, do_config, config_only, + adjust_cfg, commit, out_dir, out_rel_dir, result) result.already_done = False result.toolchain = self.toolchain @@ -326,15 +499,15 @@ class BuilderThread(threading.Thread): result.out_dir = out_dir return result, do_config - def _WriteResult(self, result, keep_outputs, work_in_output): + def _write_result(self, result, keep_outputs, work_in_output): """Write a built result to the output directory. Args: - result: CommandResult object containing result to write - keep_outputs: True to store the output binaries, False + result (CommandResult): result to write + keep_outputs (bool): True to store the output binaries, False to delete them - work_in_output: Use the output directory as the work directory and - don't write to a separate output directory. + work_in_output (bool): Use the output directory as the work + directory and don't write to a separate output directory. """ # If we think this might have been aborted with Ctrl-C, record the # failure but not that we are 'done' with this board. A retry may fix @@ -345,22 +518,22 @@ class BuilderThread(threading.Thread): return # Write the output and stderr - output_dir = self.builder._GetOutputDir(result.commit_upto) - Mkdir(output_dir) - build_dir = self.builder.GetBuildDir(result.commit_upto, + output_dir = self.builder.get_output_dir(result.commit_upto) + mkdir(output_dir) + build_dir = self.builder.get_build_dir(result.commit_upto, result.brd.target) - Mkdir(build_dir) + mkdir(build_dir) outfile = os.path.join(build_dir, 'log') - with open(outfile, 'w') as fd: + with open(outfile, 'w', encoding='utf-8') as outf: if result.stdout: - fd.write(result.stdout) + outf.write(result.stdout) - errfile = self.builder.GetErrFile(result.commit_upto, + errfile = self.builder.get_err_file(result.commit_upto, result.brd.target) if result.stderr: - with open(errfile, 'w') as fd: - fd.write(result.stderr) + with open(errfile, 'w', encoding='utf-8') as outf: + outf.write(result.stderr) elif os.path.exists(errfile): os.remove(errfile) @@ -370,60 +543,61 @@ class BuilderThread(threading.Thread): if result.toolchain: # Write the build result and toolchain information. - done_file = self.builder.GetDoneFile(result.commit_upto, + done_file = self.builder.get_done_file(result.commit_upto, result.brd.target) - with open(done_file, 'w') as fd: + with open(done_file, 'w', encoding='utf-8') as outf: if maybe_aborted: # Special code to indicate we need to retry - fd.write('%s' % RETURN_CODE_RETRY) + outf.write(f'{RETURN_CODE_RETRY}') else: - fd.write('%s' % result.return_code) - with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: - print('gcc', result.toolchain.gcc, file=fd) - print('path', result.toolchain.path, file=fd) - print('cross', result.toolchain.cross, file=fd) - print('arch', result.toolchain.arch, file=fd) - fd.write('%s' % result.return_code) + outf.write(f'{result.return_code}') + with open(os.path.join(build_dir, 'toolchain'), 'w', + encoding='utf-8') as outf: + print('gcc', result.toolchain.gcc, file=outf) + print('path', result.toolchain.path, file=outf) + print('cross', result.toolchain.cross, file=outf) + print('arch', result.toolchain.arch, file=outf) + outf.write(f'{result.return_code}') # Write out the image and function size information and an objdump env = result.toolchain.MakeEnvironment(self.builder.full_path) - with open(os.path.join(build_dir, 'out-env'), 'wb') as fd: + with open(os.path.join(build_dir, 'out-env'), 'wb') as outf: for var in sorted(env.keys()): - fd.write(b'%s="%s"' % (var, env[var])) + outf.write(b'%s="%s"' % (var, env[var])) with open(os.path.join(build_dir, 'out-cmd'), 'w', - encoding='utf-8') as fd: + encoding='utf-8') as outf: for cmd in result.cmd_list: - print(' '.join(cmd), file=fd) + print(' '.join(cmd), file=outf) lines = [] for fname in BASE_ELF_FILENAMES: - cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] + cmd = [f'{self.toolchain.cross}nm', '--size-sort', fname] nm_result = command.run_pipe([cmd], capture=True, capture_stderr=True, cwd=result.out_dir, raise_on_error=False, env=env) if nm_result.stdout: - nm = self.builder.GetFuncSizesFile(result.commit_upto, - result.brd.target, fname) - with open(nm, 'w') as fd: - print(nm_result.stdout, end=' ', file=fd) + nm_fname = self.builder.get_func_sizes_file( + result.commit_upto, result.brd.target, fname) + with open(nm_fname, 'w', encoding='utf-8') as outf: + print(nm_result.stdout, end=' ', file=outf) - cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] + cmd = [f'{self.toolchain.cross}objdump', '-h', fname] dump_result = command.run_pipe([cmd], capture=True, capture_stderr=True, cwd=result.out_dir, raise_on_error=False, env=env) rodata_size = '' if dump_result.stdout: - objdump = self.builder.GetObjdumpFile(result.commit_upto, + objdump = self.builder.get_objdump_file(result.commit_upto, result.brd.target, fname) - with open(objdump, 'w') as fd: - print(dump_result.stdout, end=' ', file=fd) + with open(objdump, 'w', encoding='utf-8') as outf: + print(dump_result.stdout, end=' ', file=outf) for line in dump_result.stdout.splitlines(): fields = line.split() if len(fields) > 5 and fields[1] == '.rodata': rodata_size = fields[2] - cmd = ['%ssize' % self.toolchain.cross, fname] + cmd = [f'{self.toolchain.cross}size', fname] size_result = command.run_pipe([cmd], capture=True, capture_stderr=True, cwd=result.out_dir, raise_on_error=False, env=env) @@ -432,30 +606,29 @@ class BuilderThread(threading.Thread): rodata_size) # Extract the environment from U-Boot and dump it out - cmd = ['%sobjcopy' % self.toolchain.cross, '-O', 'binary', + cmd = [f'{self.toolchain.cross}objcopy', '-O', 'binary', '-j', '.rodata.default_environment', 'env/built-in.o', 'uboot.env'] command.run_pipe([cmd], capture=True, capture_stderr=True, cwd=result.out_dir, raise_on_error=False, env=env) - ubootenv = os.path.join(result.out_dir, 'uboot.env') if not work_in_output: - self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env']) + copy_files(result.out_dir, build_dir, '', ['uboot.env']) # Write out the image sizes file. This is similar to the output # of binutil's 'size' utility, but it omits the header line and # adds an additional hex value at the end of each line for the # rodata size - if len(lines): - sizes = self.builder.GetSizesFile(result.commit_upto, + if lines: + sizes = self.builder.get_sizes_file(result.commit_upto, result.brd.target) - with open(sizes, 'w') as fd: - print('\n'.join(lines), file=fd) + with open(sizes, 'w', encoding='utf-8') as outf: + print('\n'.join(lines), file=outf) if not work_in_output: # Write out the configuration files, with a special case for SPL for dirname in ['', 'spl', 'tpl']: - self.CopyFiles( + copy_files( result.out_dir, build_dir, dirname, ['u-boot.cfg', 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config', 'include/autoconf.mk', @@ -463,60 +636,40 @@ class BuilderThread(threading.Thread): # Now write the actual build output if keep_outputs: - self.CopyFiles( + copy_files( result.out_dir, build_dir, '', ['u-boot*', '*.bin', '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk', 'spl/u-boot-spl*']) - def CopyFiles(self, out_dir, build_dir, dirname, patterns): - """Copy files from the build directory to the output. - - Args: - out_dir: Path to output directory containing the files - build_dir: Place to copy the files - dirname: Source directory, '' for normal U-Boot, 'spl' for SPL - patterns: A list of filenames (strings) to copy, each relative - to the build directory - """ - for pattern in patterns: - file_list = glob.glob(os.path.join(out_dir, dirname, pattern)) - for fname in file_list: - target = os.path.basename(fname) - if dirname: - base, ext = os.path.splitext(target) - if ext: - target = '%s-%s%s' % (base, dirname, ext) - shutil.copy(fname, os.path.join(build_dir, target)) - - def _SendResult(self, result): + def _send_result(self, result): """Send a result to the builder for processing Args: - result: CommandResult object containing the results of the build + result (CommandResult): results of the build Raises: - ValueError if self.test_exception is true (for testing) + ValueError: self.test_exception is true (for testing) """ if self.test_exception: raise ValueError('test exception') if self.thread_num != -1: self.builder.out_queue.put(result) else: - self.builder.ProcessResult(result) + self.builder.process_result(result) - def RunJob(self, job): + def run_job(self, job): """Run a single job A job consists of a building a list of commits for a particular board. Args: - job: Job to build + job (Job): Job to build - Returns: - List of Result objects + Raises: + ValueError: Thread was interrupted """ brd = job.brd - work_dir = self.builder.GetThreadDir(self.thread_num) + work_dir = self.builder.get_thread_dir(self.thread_num) self.toolchain = None if job.commits: # Run 'make board_defconfig' on the first commit @@ -524,7 +677,7 @@ class BuilderThread(threading.Thread): commit_upto = 0 force_build = False for commit_upto in range(0, len(job.commits), job.step): - result, request_config = self.RunCommit(commit_upto, brd, + result, request_config = self.run_commit(commit_upto, brd, work_dir, do_config, self.builder.config_only, force_build or self.builder.force_build, self.builder.force_build_failures, @@ -535,7 +688,7 @@ class BuilderThread(threading.Thread): # If our incremental build failed, try building again # with a reconfig. if self.builder.force_config_on_failure: - result, request_config = self.RunCommit(commit_upto, + result, request_config = self.run_commit(commit_upto, brd, work_dir, True, False, True, False, job.work_in_output, job.adjust_cfg) did_config = True @@ -576,17 +729,17 @@ class BuilderThread(threading.Thread): raise ValueError('Interrupt') # We have the build results, so output the result - self._WriteResult(result, job.keep_outputs, job.work_in_output) - self._SendResult(result) + self._write_result(result, job.keep_outputs, job.work_in_output) + self._send_result(result) else: # Just build the currently checked-out build - result, request_config = self.RunCommit(None, brd, work_dir, True, + result, request_config = self.run_commit(None, brd, work_dir, True, self.builder.config_only, True, self.builder.force_build_failures, job.work_in_output, job.adjust_cfg) result.commit_upto = 0 - self._WriteResult(result, job.keep_outputs, job.work_in_output) - self._SendResult(result) + self._write_result(result, job.keep_outputs, job.work_in_output) + self._send_result(result) def run(self): """Our thread's run function @@ -597,8 +750,9 @@ class BuilderThread(threading.Thread): while True: job = self.builder.queue.get() try: - self.RunJob(job) - except Exception as e: - print('Thread exception (use -T0 to run without threads):', e) - self.builder.thread_exceptions.append(e) + self.run_job(job) + except Exception as exc: + print('Thread exception (use -T0 to run without threads):', + exc) + self.builder.thread_exceptions.append(exc) self.builder.queue.task_done() |