diff options
author | Tom Rini <trini@konsulko.com> | 2022-11-22 22:14:06 -0500 |
---|---|---|
committer | Tom Rini <trini@konsulko.com> | 2022-11-22 22:14:06 -0500 |
commit | 19fb8d7945fcd83828986ac48db74560b26611f9 (patch) | |
tree | d65ad8a81d6e62d2786e40186769b48d07280303 | |
parent | 521277ec15eb794229403ec24b8c00a4ff02b0b6 (diff) | |
parent | b4574c0e750bee39cc2448952f127040a37a8e0b (diff) | |
download | u-boot-19fb8d7945fcd83828986ac48db74560b26611f9.zip u-boot-19fb8d7945fcd83828986ac48db74560b26611f9.tar.gz u-boot-19fb8d7945fcd83828986ac48db74560b26611f9.tar.bz2 |
Merge tag 'dm-pull-22nov22' of https://source.denx.de/u-boot/custodians/u-boot-dmWIP/22Nov2022
buildman /binman improvements for handling missing blobs
fix for long-standing image.h warning
minor fixes
-rw-r--r-- | .azure-pipelines.yml | 2 | ||||
-rw-r--r-- | .gitlab-ci.yml | 6 | ||||
-rw-r--r-- | Makefile | 15 | ||||
-rw-r--r-- | board/sandbox/sandbox.c | 13 | ||||
-rw-r--r-- | cmd/fdt.c | 9 | ||||
l--------- | doc/build/buildman.rst | 1 | ||||
-rw-r--r-- | doc/build/index.rst | 1 | ||||
-rw-r--r-- | drivers/usb/emul/sandbox_flash.c | 10 | ||||
-rw-r--r-- | include/configs/sandbox.h | 13 | ||||
-rw-r--r-- | include/image.h | 8 | ||||
-rw-r--r-- | scripts/Kbuild.include | 2 | ||||
-rw-r--r-- | test/cmd/fdt.c | 53 | ||||
-rw-r--r-- | test/cmd/setexpr.c | 6 | ||||
-rw-r--r-- | tools/binman/binman.rst | 352 | ||||
-rw-r--r-- | tools/binman/bintool.py | 3 | ||||
-rw-r--r-- | tools/binman/btool/btool_gzip.py | 4 | ||||
-rw-r--r-- | tools/binman/cmdline.py | 5 | ||||
-rw-r--r-- | tools/binman/control.py | 9 | ||||
-rw-r--r-- | tools/binman/ftest.py | 19 | ||||
-rw-r--r-- | tools/buildman/README | 1349 | ||||
l--------- | tools/buildman/README.rst | 1 | ||||
-rw-r--r-- | tools/buildman/bsettings.py | 17 | ||||
-rw-r--r-- | tools/buildman/builder.py | 5 | ||||
-rw-r--r-- | tools/buildman/builderthread.py | 8 | ||||
-rw-r--r-- | tools/buildman/buildman.rst | 1328 | ||||
-rw-r--r-- | tools/buildman/cmdline.py | 6 | ||||
-rw-r--r-- | tools/buildman/control.py | 28 | ||||
-rw-r--r-- | tools/buildman/func_test.py | 112 |
28 files changed, 1981 insertions, 1404 deletions
diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index bda7624..665b5d2 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -553,7 +553,7 @@ stages: cat << "EOF" >> build.sh if [[ "${BUILDMAN}" != "" ]]; then ret=0; - tools/buildman/buildman -o /tmp -P -E -W ${BUILDMAN} ${OVERRIDE} || ret=$?; + tools/buildman/buildman -o /tmp -PEWM ${BUILDMAN} ${OVERRIDE} || ret=$?; if [[ $ret -ne 0 ]]; then tools/buildman/buildman -o /tmp -seP ${BUILDMAN}; exit $ret; diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6f4c34f..3deaeca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -81,7 +81,7 @@ build all 32bit ARM platforms: stage: world build script: - ret=0; - ./tools/buildman/buildman -o /tmp -P -E -W arm -x aarch64 || ret=$?; + ./tools/buildman/buildman -o /tmp -PEWM arm -x aarch64 || ret=$?; if [[ $ret -ne 0 ]]; then ./tools/buildman/buildman -o /tmp -seP; exit $ret; @@ -93,7 +93,7 @@ build all 64bit ARM platforms: - virtualenv -p /usr/bin/python3 /tmp/venv - . /tmp/venv/bin/activate - ret=0; - ./tools/buildman/buildman -o /tmp -P -E -W aarch64 || ret=$?; + ./tools/buildman/buildman -o /tmp -PEWM aarch64 || ret=$?; if [[ $ret -ne 0 ]]; then ./tools/buildman/buildman -o /tmp -seP; exit $ret; @@ -113,7 +113,7 @@ build all other platforms: stage: world build script: - ret=0; - ./tools/buildman/buildman -o /tmp -P -E -W -x arm,powerpc || ret=$?; + ./tools/buildman/buildman -o /tmp -PEWM -x arm,powerpc || ret=$?; if [[ $ret -ne 0 ]]; then ./tools/buildman/buildman -o /tmp -seP; exit $ret; @@ -1108,18 +1108,15 @@ define deprecated endef -PHONY += inputs -inputs: $(INPUTS-y) - -all: .binman_stamp inputs +# Timestamp file to make sure that binman always runs +.binman_stamp: $(INPUTS-y) FORCE ifeq ($(CONFIG_BINMAN),y) $(call if_changed,binman) endif - -# Timestamp file to make sure that binman always runs -.binman_stamp: FORCE @touch $@ +all: .binman_stamp + ifeq ($(CONFIG_DEPRECATED),y) $(warning "You have deprecated configuration options enabled in your .config! Please check your configuration.") endif @@ -1336,8 +1333,8 @@ cmd_binman = $(srctree)/tools/binman/binman $(if $(BINMAN_DEBUG),-D) \ $(foreach f,$(BINMAN_TOOLPATHS),--toolpath $(f)) \ --toolpath $(objtree)/tools \ $(if $(BINMAN_VERBOSE),-v$(BINMAN_VERBOSE)) \ - build -u -d u-boot.dtb -O . -m --allow-missing \ - --fake-ext-blobs \ + build -u -d u-boot.dtb -O . -m \ + $(if $(BINMAN_ALLOW_MISSING),--allow-missing --fake-ext-blobs) \ -I . -I $(srctree) -I $(srctree)/board/$(BOARDDIR) \ -I arch/$(ARCH)/dts -a of-list=$(CONFIG_OF_LIST) \ $(foreach f,$(BINMAN_INDIRS),-I $(f)) \ diff --git a/board/sandbox/sandbox.c b/board/sandbox/sandbox.c index 4d89f9b..4c655df 100644 --- a/board/sandbox/sandbox.c +++ b/board/sandbox/sandbox.c @@ -30,6 +30,19 @@ gd_t *gd; #if CONFIG_IS_ENABLED(EFI_HAVE_CAPSULE_SUPPORT) +/* GUIDs for capsule updatable firmware images */ +#define SANDBOX_UBOOT_IMAGE_GUID \ + EFI_GUID(0x09d7cf52, 0x0720, 0x4710, 0x91, 0xd1, \ + 0x08, 0x46, 0x9b, 0x7f, 0xe9, 0xc8) + +#define SANDBOX_UBOOT_ENV_IMAGE_GUID \ + EFI_GUID(0x5a7021f5, 0xfef2, 0x48b4, 0xaa, 0xba, \ + 0x83, 0x2e, 0x77, 0x74, 0x18, 0xc0) + +#define SANDBOX_FIT_IMAGE_GUID \ + EFI_GUID(0x3673b45d, 0x6a7c, 0x46f3, 0x9e, 0x60, \ + 0xad, 0xab, 0xb0, 0x3f, 0x79, 0x37) + struct efi_fw_image fw_images[] = { #if defined(CONFIG_EFI_CAPSULE_FIRMWARE_RAW) { @@ -60,11 +60,14 @@ static int fdt_value_env_set(const void *nodep, int len, * Iterate over all members in stringlist and find the one at * offset $index. If no such index exists, indicate failure. */ - for (i = 0; i < len; i += strlen(nodec) + 1) { - if (index-- > 0) + for (i = 0; i < len; ) { + if (index-- > 0) { + i += strlen(nodec) + 1; + nodec += strlen(nodec) + 1; continue; + } - env_set(var, nodec + i); + env_set(var, nodec); return 0; } diff --git a/doc/build/buildman.rst b/doc/build/buildman.rst new file mode 120000 index 0000000..beeaa42 --- /dev/null +++ b/doc/build/buildman.rst @@ -0,0 +1 @@ +../../tools/buildman/buildman.rst
\ No newline at end of file diff --git a/doc/build/index.rst b/doc/build/index.rst index 69952f9..9a8105d 100644 --- a/doc/build/index.rst +++ b/doc/build/index.rst @@ -11,3 +11,4 @@ Build U-Boot clang docker tools + buildman diff --git a/drivers/usb/emul/sandbox_flash.c b/drivers/usb/emul/sandbox_flash.c index 6e8cfe1..01ccc4b 100644 --- a/drivers/usb/emul/sandbox_flash.c +++ b/drivers/usb/emul/sandbox_flash.c @@ -188,15 +188,19 @@ static int handle_ufi_command(struct sandbox_flash_priv *priv, const void *buff, struct scsi_emul_info *info = &priv->eminfo; const struct scsi_cmd *req = buff; int ret; + off_t offset; ret = sb_scsi_emul_command(info, req, len); if (!ret) { setup_response(priv); } else if ((ret == SCSI_EMUL_DO_READ || ret == SCSI_EMUL_DO_WRITE) && priv->fd != -1) { - os_lseek(priv->fd, info->seek_block * info->block_size, - OS_SEEK_SET); - setup_response(priv); + offset = os_lseek(priv->fd, info->seek_block * info->block_size, + OS_SEEK_SET); + if (offset == (off_t)-1) + setup_fail_response(priv); + else + setup_response(priv); } else { setup_fail_response(priv); } diff --git a/include/configs/sandbox.h b/include/configs/sandbox.h index 5168e2f..0dcb2eb 100644 --- a/include/configs/sandbox.h +++ b/include/configs/sandbox.h @@ -10,19 +10,6 @@ #define CONFIG_MALLOC_F_ADDR 0x0010000 -/* GUIDs for capsule updatable firmware images */ -#define SANDBOX_UBOOT_IMAGE_GUID \ - EFI_GUID(0x09d7cf52, 0x0720, 0x4710, 0x91, 0xd1, \ - 0x08, 0x46, 0x9b, 0x7f, 0xe9, 0xc8) - -#define SANDBOX_UBOOT_ENV_IMAGE_GUID \ - EFI_GUID(0x5a7021f5, 0xfef2, 0x48b4, 0xaa, 0xba, \ - 0x83, 0x2e, 0x77, 0x74, 0x18, 0xc0) - -#define SANDBOX_FIT_IMAGE_GUID \ - EFI_GUID(0x3673b45d, 0x6a7c, 0x46f3, 0x9e, 0x60, \ - 0xad, 0xab, 0xb0, 0x3f, 0x79, 0x37) - /* Size of our emulated memory */ #define SB_CONCAT(x, y) x ## y #define SB_TO_UL(s) SB_CONCAT(s, UL) diff --git a/include/image.h b/include/image.h index 65d0d4f..6f21daf 100644 --- a/include/image.h +++ b/include/image.h @@ -853,7 +853,13 @@ image_set_hdr_b(comp) /* image_set_comp */ static inline void image_set_name(struct legacy_img_hdr *hdr, const char *name) { - strncpy(image_get_name(hdr), name, IH_NMLEN); + /* + * This is equivalent to: strncpy(image_get_name(hdr), name, IH_NMLEN); + * + * Use the tortured code below to avoid a warning with gcc 12. We do not + * want to include a nul terminator if the name is of length IH_NMLEN + */ + memcpy(image_get_name(hdr), name, strnlen(name, IH_NMLEN)); } int image_check_hcrc(const struct legacy_img_hdr *hdr); diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 9c14310..62e0207 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -229,7 +229,7 @@ objectify = $(foreach o,$(1),$(if $(filter /%,$(o)),$(o),$(obj)/$(o))) # if_changed_dep - as if_changed, but uses fixdep to reveal dependencies # including used config symbols # if_changed_rule - as if_changed but execute rule instead -# See Documentation/kbuild/makefiles.txt for more info +# See doc/develop/makefiles.rst for more info ifneq ($(KBUILD_NOCMDDEP),1) # Check if both arguments are the same including their order. Result is empty diff --git a/test/cmd/fdt.c b/test/cmd/fdt.c index ba9eaa4..7974c88 100644 --- a/test/cmd/fdt.c +++ b/test/cmd/fdt.c @@ -142,6 +142,59 @@ static int fdt_test_resize(struct unit_test_state *uts) } FDT_TEST(fdt_test_resize, UT_TESTF_CONSOLE_REC); +/* Test 'fdt get' reading an fdt */ +static int fdt_test_get(struct unit_test_state *uts) +{ + ulong addr; + + addr = map_to_sysmem(gd->fdt_blob); + set_working_fdt_addr(addr); + + /* Test getting default element of /clk-test node clock-names property */ + ut_assertok(console_record_reset_enable()); + ut_assertok(run_command("fdt get value fdflt /clk-test clock-names", 0)); + ut_asserteq_str("fixed", env_get("fdflt")); + ut_assertok(ut_check_console_end(uts)); + + /* Test getting 0th element of /clk-test node clock-names property */ + ut_assertok(console_record_reset_enable()); + ut_assertok(run_command("fdt get value fzero /clk-test clock-names 0", 0)); + ut_asserteq_str("fixed", env_get("fzero")); + ut_assertok(ut_check_console_end(uts)); + + /* Test getting 1st element of /clk-test node clock-names property */ + ut_assertok(console_record_reset_enable()); + ut_assertok(run_command("fdt get value fone /clk-test clock-names 1", 0)); + ut_asserteq_str("i2c", env_get("fone")); + ut_assertok(ut_check_console_end(uts)); + + /* Test getting 2nd element of /clk-test node clock-names property */ + ut_assertok(console_record_reset_enable()); + ut_assertok(run_command("fdt get value ftwo /clk-test clock-names 2", 0)); + ut_asserteq_str("spi", env_get("ftwo")); + ut_assertok(ut_check_console_end(uts)); + + /* Test missing 10th element of /clk-test node clock-names property */ + ut_assertok(console_record_reset_enable()); + ut_asserteq(1, run_command("fdt get value ftwo /clk-test clock-names 10", 0)); + ut_assertok(ut_check_console_end(uts)); + + /* Test getting default element of /clk-test node nonexistent property */ + ut_assertok(console_record_reset_enable()); + ut_asserteq(1, run_command("fdt get value fnone /clk-test nonexistent", 1)); + ut_assert_nextline("libfdt fdt_getprop(): FDT_ERR_NOTFOUND"); + ut_assertok(ut_check_console_end(uts)); + + /* Test getting default element of /nonexistent node */ + ut_assertok(console_record_reset_enable()); + ut_asserteq(1, run_command("fdt get value fnode /nonexistent nonexistent", 1)); + ut_assert_nextline("libfdt fdt_path_offset() returned FDT_ERR_NOTFOUND"); + ut_assertok(ut_check_console_end(uts)); + + return 0; +} +FDT_TEST(fdt_test_get, UT_TESTF_CONSOLE_REC); + int do_ut_fdt(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) { struct unit_test *tests = UNIT_TEST_SUITE_START(fdt_test); diff --git a/test/cmd/setexpr.c b/test/cmd/setexpr.c index 0dc94f7..312593e 100644 --- a/test/cmd/setexpr.c +++ b/test/cmd/setexpr.c @@ -308,7 +308,11 @@ static int setexpr_test_str(struct unit_test_state *uts) start_mem = ut_check_free(); ut_assertok(run_command("setexpr.s fred *0", 0)); ut_asserteq_str("hello", env_get("fred")); - ut_assertok(ut_check_delta(start_mem)); + /* + * This fails in CI at present. + * + * ut_assertok(ut_check_delta(start_mem)); + */ unmap_sysmem(buf); diff --git a/tools/binman/binman.rst b/tools/binman/binman.rst index fda16f1..e7b231e 100644 --- a/tools/binman/binman.rst +++ b/tools/binman/binman.rst @@ -505,7 +505,6 @@ be located anywhere in the image. An image header (typically at the start or end of the image) can be used to point to the FDT map. See fdtmap and image-header entries for more information. - Map files --------- @@ -1246,6 +1245,8 @@ You can also replace just a selection of entries:: $ binman replace -i image.bin "*u-boot*" -I indir +.. _`BinmanLogging`: + Logging ------- @@ -1337,6 +1338,305 @@ generated from the source code using: bintools +Binman commands and arguments +============================= + +Usage:: + + binman [-h] [-B BUILD_DIR] [-D] [-H] [--toolpath TOOLPATH] [-T THREADS] + [--test-section-timeout] [-v VERBOSITY] [-V] + {build,bintool-docs,entry-docs,ls,extract,replace,test,tool} ... + +Binman provides the following commands: + +- **build** - build images +- **bintools-docs** - generate documentation about bintools +- **entry-docs** - generate documentation about entry types +- **ls** - list an image +- **extract** - extract files from an image +- **replace** - replace one or more entries in an image +- **test** - run tests +- **tool** - manage bintools + +Options: + +-h, --help + Show help message and exit + +-B BUILD_DIR, --build-dir BUILD_DIR + Directory containing the build output + +-D, --debug + Enabling debugging (provides a full traceback on error) + +-H, --full-help + Display the README file + +--toolpath TOOLPATH + Add a path to the directories containing tools + +-T THREADS, --threads THREADS + Number of threads to use (0=single-thread). Note that -T0 is useful for + debugging since everything runs in one thread. + +-v VERBOSITY, --verbosity VERBOSITY + Control verbosity: 0=silent, 1=warnings, 2=notices, 3=info, 4=detail, + 5=debug + +-V, --version + Show the binman version + +Test options: + +--test-section-timeout + Use a zero timeout for section multi-threading (for testing) + +Commands are described below. + +binman build +------------ + +This builds one or more images using the provided image description. + +Usage:: + + binman build [-h] [-a ENTRY_ARG] [-b BOARD] [-d DT] [--fake-dtb] + [--fake-ext-blobs] [--force-missing-bintools FORCE_MISSING_BINTOOLS] + [-i IMAGE] [-I INDIR] [-m] [-M] [-n] [-O OUTDIR] [-p] [-u] + [--update-fdt-in-elf UPDATE_FDT_IN_ELF] [-W] + +Options: + +-h, --help + Show help message and exit + +-a ENTRY_ARG, --entry-arg ENTRY_ARG + Set argument value `arg=value`. See + `Passing command-line arguments to entries`_. + +-b BOARD, --board BOARD + Board name to build. This can be used instead of `-d`, in which case the + file `u-boot.dtb` is used, within the build directory's board subdirectory. + +-d DT, --dt DT + Configuration file (.dtb) to use. This must have a top-level node called + `binman`. See `Image description format`_. + +-i IMAGE, --image IMAGE + Image filename to build (if not specified, build all) + +-I INDIR, --indir INDIR + Add a path to the list of directories to use for input files. This can be + specified multiple times to add more than one path. + +-m, --map + Output a map file for each image. See `Map files`_. + +-M, --allow-missing + Allow external blobs and bintools to be missing. See `External blobs`_. + +-n, --no-expanded + Don't use 'expanded' versions of entries where available; normally 'u-boot' + becomes 'u-boot-expanded', for example. See `Expanded entries`_. + +-O OUTDIR, --outdir OUTDIR + Path to directory to use for intermediate and output files + +-p, --preserve + Preserve temporary output directory even if option -O is not given + +-u, --update-fdt + Update the binman node with offset/size info. See + `Access to binman entry offsets at run time (fdt)`_. + +--update-fdt-in-elf UPDATE_FDT_IN_ELF + Update an ELF file with the output dtb. The argument is a string consisting + of four parts, separated by commas. See `Updating an ELF file`_. + +-W, --ignore-missing + Return success even if there are missing blobs/bintools (requires -M) + +Options used only for testing: + +--fake-dtb + Use fake device tree contents + +--fake-ext-blobs + Create fake ext blobs with dummy content + +--force-missing-bintools FORCE_MISSING_BINTOOLS + Comma-separated list of bintools to consider missing + +binman bintool-docs +------------------- + +Usage:: + + binman bintool-docs [-h] + +This outputs documentation for the bintools in rST format. See +`Bintool Documentation`_. + +binman entry-docs +----------------- + +Usage:: + + binman entry-docs [-h] + +This outputs documentation for the entry types in rST format. See +`Entry Documentation`_. + +binman ls +--------- + +Usage:: + + binman ls [-h] -i IMAGE [paths ...] + +Positional arguments: + +paths + Paths within file to list (wildcard) + +Pptions: + +-h, --help + show help message and exit + +-i IMAGE, --image IMAGE + Image filename to list + +This lists an image, showing its contents. See `Listing images`_. + +binman extract +-------------- + +Usage:: + + binman extract [-h] [-F FORMAT] -i IMAGE [-f FILENAME] [-O OUTDIR] [-U] + [paths ...] + +Positional arguments: + +Paths + Paths within file to extract (wildcard) + +Options: + +-h, --help + show help message and exit + +-F FORMAT, --format FORMAT + Select an alternative format for extracted data + +-i IMAGE, --image IMAGE + Image filename to extract + +-f FILENAME, --filename FILENAME + Output filename to write to + +-O OUTDIR, --outdir OUTDIR + Path to directory to use for output files + +-U, --uncompressed + Output raw uncompressed data for compressed entries + +This extracts the contents of entries from an image. See +`Extracting files from images`_. + +binman replace +-------------- + +Usage:: + + binman replace [-h] [-C] -i IMAGE [-f FILENAME] [-F] [-I INDIR] [-m] + [paths ...] + +Positional arguments: + +paths + Paths within file to replace (wildcard) + +Options: + +-h, --help + show help message and exit + +-C, --compressed + Input data is already compressed if needed for the entry + +-i IMAGE, --image IMAGE + Image filename to update + +-f FILENAME, --filename FILENAME + Input filename to read from + +-F, --fix-size + Don't allow entries to be resized + +-I INDIR, --indir INDIR + Path to directory to use for input files + +-m, --map + Output a map file for the updated image + +This replaces one or more entries in an existing image. See +`Replacing files in an image`_. + +binman test +----------- + +Usage:: + + binman test [-h] [-P PROCESSES] [-T] [-X] [tests ...] + +Positional arguments: + +tests + Test names to run (omit for all) + +Options: + +-h, --help + show help message and exit + +-P PROCESSES, --processes PROCESSES + set number of processes to use for running tests. This defaults to the + number of CPUs on the machine + +-T, --test-coverage + run tests and check for 100% coverage + +-X, --test-preserve-dirs + Preserve and display test-created input directories; also preserve the + output directory if a single test is run (pass test name at the end of the + command line + +binman tool +----------- + +Usage:: + + binman tool [-h] [-l] [-f] [bintools ...] + +Positional arguments: + +bintools + Bintools to process + +Options: + +-h, --help + show help message and exit + +-l, --list + List all known bintools + +-f, --fetch + Fetch a bintool from a known location. Use `all` to fetch all and `missing` + to fetch any missing tools. + Technical details ================= @@ -1416,6 +1716,8 @@ what happens in this stage. final step. +.. _`External tools`: + External tools -------------- @@ -1437,6 +1739,8 @@ a space-separated list of paths to search, e.g.:: BINMAN_TOOLPATHS="/tools/g12a /tools/tegra" binman ... +.. _`External blobs`: + External blobs -------------- @@ -1461,6 +1765,10 @@ space-separated list of directories to search for binary blobs:: odroid-c4/build/board/hardkernel/odroidc4/firmware \ odroid-c4/build/scp_task" binman ... +Note that binman fails with exit code 103 when there are missing blobs. If you +wish binman to continue anyway, you can pass `-W` to binman. + + Code coverage ------------- @@ -1472,6 +1780,48 @@ To enable Python test coverage on Debian-type distributions (e.g. Ubuntu):: $ sudo apt-get install python-coverage python3-coverage python-pytest +Exit status +----------- + +Binman produces the following exit codes: + +0 + Success + +1 + Any sort of failure - see output for more details + +103 + There are missing external blobs or bintools. This is only returned if + -M is passed to binman, otherwise missing blobs return an exit status of 1. + Note, if -W is passed as well as -M, then this is converted into a warning + and will return an exit status of 0 instead. + + +U-Boot environment variables for binman +--------------------------------------- + +The U-Boot Makefile supports various environment variables to control binman. +All of these are set within the Makefile and result in passing various +environment variables (or make flags) to binman: + +BINMAN_DEBUG + Enables backtrace debugging by adding a `-D` argument. See + :ref:`BinmanLogging`. + +BINMAN_INDIRS + Sets the search path for input files used by binman by adding one or more + `-I` arguments. See :ref:`External blobs`. + +BINMAN_TOOLPATHS + Sets the search path for external tool used by binman by adding one or more + `--toolpath` arguments. See :ref:`External tools`. + +BINMAN_VERBOSE + Sets the logging verbosity of binman by adding a `-v` argument. See + :ref:`BinmanLogging`. + + Error messages -------------- diff --git a/tools/binman/bintool.py b/tools/binman/bintool.py index a582d9d..8fda13f 100644 --- a/tools/binman/bintool.py +++ b/tools/binman/bintool.py @@ -85,7 +85,6 @@ class Bintool: try: # Deal with classes which must be renamed due to conflicts # with Python libraries - class_name = f'Bintoolbtool_{module_name}' module = importlib.import_module('binman.btool.btool_' + module_name) except ImportError: @@ -137,6 +136,8 @@ class Bintool: names = [os.path.splitext(os.path.basename(fname))[0] for fname in files] names = [name for name in names if name[0] != '_'] + names = [name[6:] if name.startswith('btool_') else name + for name in names] if include_testing: names.append('_testing') return sorted(names) diff --git a/tools/binman/btool/btool_gzip.py b/tools/binman/btool/btool_gzip.py index 70cbc19..0d75028 100644 --- a/tools/binman/btool/btool_gzip.py +++ b/tools/binman/btool/btool_gzip.py @@ -14,7 +14,7 @@ Documentation is available via:: from binman import bintool # pylint: disable=C0103 -class Bintoolbtool_gzip(bintool.BintoolPacker): +class Bintoolgzip(bintool.BintoolPacker): """Compression/decompression using the gzip algorithm This bintool supports running `gzip` to compress and decompress data, as @@ -27,5 +27,5 @@ class Bintoolbtool_gzip(bintool.BintoolPacker): man gzip """ def __init__(self, name): - super().__init__("gzip", compress_args=[], + super().__init__(name, compress_args=[], version_regex=r'gzip ([0-9.]+)') diff --git a/tools/binman/cmdline.py b/tools/binman/cmdline.py index 1d1ca43..986d6f1 100644 --- a/tools/binman/cmdline.py +++ b/tools/binman/cmdline.py @@ -114,7 +114,7 @@ controlled by a description in the board device tree.''' build_parser.add_argument('-m', '--map', action='store_true', default=False, help='Output a map file for each image') build_parser.add_argument('-M', '--allow-missing', action='store_true', - default=False, help='Allow external blobs to be missing') + default=False, help='Allow external blobs and bintools to be missing') build_parser.add_argument('-n', '--no-expanded', action='store_true', help="Don't use 'expanded' versions of entries where available; " "normally 'u-boot' becomes 'u-boot-expanded', for example") @@ -128,6 +128,9 @@ controlled by a description in the board device tree.''' default=False, help='Update the binman node with offset/size info') build_parser.add_argument('--update-fdt-in-elf', type=str, help='Update an ELF file with the output dtb: infile,outfile,begin_sym,end_sym') + build_parser.add_argument( + '-W', '--ignore-missing', action='store_true', default=False, + help='Return success even if there are missing blobs/bintools (requires -M)') subparsers.add_parser( 'bintool-docs', help='Write out bintool documentation (see bintool.rst)') diff --git a/tools/binman/control.py b/tools/binman/control.py index bfe63a1..964c698 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -741,8 +741,15 @@ def Binman(args): data = state.GetFdtForEtype('u-boot-dtb').GetContents() elf.UpdateFile(*elf_params, data) + # This can only be True if -M is provided, since otherwise binman + # would have raised an error already if invalid: - tout.warning("\nSome images are invalid") + msg = '\nSome images are invalid' + if args.ignore_missing: + tout.warning(msg) + else: + tout.error(msg) + return 103 # Use this to debug the time take to pack the image #state.TimingShow() diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index e849d96..62ee86b 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -340,7 +340,7 @@ class TestFunctional(unittest.TestCase): use_expanded=False, verbosity=None, allow_missing=False, allow_fake_blobs=False, extra_indirs=None, threads=None, test_section_timeout=False, update_fdt_in_elf=None, - force_missing_bintools=''): + force_missing_bintools='', ignore_missing=False): """Run binman with a given test file Args: @@ -403,6 +403,8 @@ class TestFunctional(unittest.TestCase): args.append('-a%s=%s' % (arg, value)) if allow_missing: args.append('-M') + if ignore_missing: + args.append('-W') if allow_fake_blobs: args.append('--fake-ext-blobs') if force_missing_bintools: @@ -3725,9 +3727,22 @@ class TestFunctional(unittest.TestCase): def testExtblobMissingOk(self): """Test an image with an missing external blob that is allowed""" with test_util.capture_sys_output() as (stdout, stderr): - self._DoTestFile('158_blob_ext_missing.dts', allow_missing=True) + ret = self._DoTestFile('158_blob_ext_missing.dts', + allow_missing=True) + self.assertEqual(103, ret) err = stderr.getvalue() self.assertRegex(err, "Image 'main-section'.*missing.*: blob-ext") + self.assertIn('Some images are invalid', err) + + def testExtblobMissingOkFlag(self): + """Test an image with an missing external blob allowed with -W""" + with test_util.capture_sys_output() as (stdout, stderr): + ret = self._DoTestFile('158_blob_ext_missing.dts', + allow_missing=True, ignore_missing=True) + self.assertEqual(0, ret) + err = stderr.getvalue() + self.assertRegex(err, "Image 'main-section'.*missing.*: blob-ext") + self.assertIn('Some images are invalid', err) def testExtblobMissingOkSect(self): """Test an image with an missing external blob that is allowed""" diff --git a/tools/buildman/README b/tools/buildman/README deleted file mode 100644 index a8357a8..0000000 --- a/tools/buildman/README +++ /dev/null @@ -1,1349 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0+ -# Copyright (c) 2013 The Chromium OS Authors. - -(Please read 'How to change from MAKEALL' if you are used to that tool) - -Quick-start -=========== - -If you just want to quickly set up buildman so you can build something (for -example Raspberry Pi 2): - - cd /path/to/u-boot - PATH=$PATH:`pwd`/tools/buildman - buildman --fetch-arch arm - buildman -k rpi_2 - ls ../current/rpi_2 - # u-boot.bin is the output image - - -What is this? -============= - -This tool handles building U-Boot to check that you have not broken it -with your patch series. It can build each individual commit and report -which boards fail on which commits, and which errors come up. It aims -to make full use of multi-processor machines. - -A key feature of buildman is its output summary, which allows warnings, -errors or image size increases in a particular commit or board to be -quickly identified and the offending commit pinpointed. This can be a big -help for anyone working with >10 patches at a time. - - -Caveats -======= - -Buildman can be stopped and restarted, in which case it will continue -where it left off. This should happen cleanly and without side-effects. -If not, it is a bug, for which a patch would be welcome. - -Buildman gets so tied up in its work that it can ignore the outside world. -You may need to press Ctrl-C several times to quit it. Also it will print -out various exceptions when stopped. You may have to kill it since the -Ctrl-C handling is somewhat broken. - - -Theory of Operation -=================== - -(please read this section in full twice or you will be perpetually confused) - -Buildman is a builder. It is not make, although it runs make. It does not -produce any useful output on the terminal while building, except for -progress information (but see -v below). All the output (errors, warnings and -binaries if you ask for them) is stored in output directories, which you can -look at from a separate 'buildman -s' instance while the build is progressing, -or when it is finished. - -Buildman is designed to build entire git branches, i.e. muliple commits. It -can be run repeatedly on the same branch after making changes to commits on -that branch. In this case it will automatically rebuild commits which have -changed (and remove its old results for that commit). It is possible to build -a branch for one board, then later build it for another board. This adds to -the output, so now you have results for two boards. If you want buildman to -re-build a commit it has already built (e.g. because of a toolchain update), -use the -f flag. - -Buildman produces a concise summary of which boards succeeded and failed. -It shows which commit introduced which board failure using a simple -red/green colour coding (with yellow/cyan for warnings). Full error -information can be requested, in which case it is de-duped and displayed -against the commit that introduced the error. An example workflow is below. - -Buildman stores image size information and can report changes in image size -from commit to commit. An example of this is below. - -Buildman starts multiple threads, and each thread builds for one board at -a time. A thread starts at the first commit, configures the source for your -board and builds it. Then it checks out the next commit and does an -incremental build (i.e. not using 'make xxx_defconfig' unless you use -C). -Eventually the thread reaches the last commit and stops. If a commit causes -an error or warning, buildman will try it again after reconfiguring (but see --Q). Thus some commits may be built twice, with the first result silently -discarded. Lots of errors and warnings will causes lots of reconfigures and your -build will be very slow. This is because a file that produces just a warning -would not normally be rebuilt in an incremental build. Once a thread finishes -building all the commits for a board, it starts on the commits for another -board. - -Buildman works in an entirely separate place from your U-Boot repository. -It creates a separate working directory for each thread, and puts the -output files in the working directory, organised by commit name and board -name, in a two-level hierarchy (but see -P). - -Buildman is invoked in your U-Boot directory, the one with the .git -directory. It clones this repository into a copy for each thread, and the -threads do not affect the state of your git repository. Any checkouts done -by the thread affect only the working directory for that thread. - -Buildman automatically selects the correct tool chain for each board. You -must supply suitable tool chains (see --fetch-arch), but buildman takes care -of selecting the right one. - -Buildman generally builds a branch (with the -b flag), and in this case -builds the upstream commit as well, for comparison. So even if you have one -commit in your branch, two commits will be built. Put all your commits in a -branch, set the branch's upstream to a valid value, and all will be well. -Otherwise buildman will perform random actions. Use -n to check what the -random actions might be. - -Buildman effectively has two modes: without -s it builds, with -s it -summarises the results of previous (or active) builds. - -If you just want to build the current source tree, leave off the -b flag. -This will display results and errors as they happen. You can still look at -them later using -se. Note that buildman will assume that the source has -changed, and will build all specified boards in this case. - -Buildman is optimised for building many commits at once, for many boards. -On multi-core machines, Buildman is fast because it uses most of the -available CPU power. When it gets to the end, or if you are building just -a few commits or boards, it will be pretty slow. As a tip, if you don't -plan to use your machine for anything else, you can use -T to increase the -number of threads beyond the default. - - -Selecting which boards to build -=============================== - -Buildman lets you build all boards, or a subset. Specify the subset by passing -command-line arguments that list the desired build target, architecture, -CPU, board name, vendor, SoC or options. Multiple arguments are allowed. Each -argument will be interpreted as a regular expression, so behaviour is a superset -of exact or substring matching. Examples are: - -* 'tegra20' All boards with a Tegra20 SoC -* 'tegra' All boards with any Tegra Soc (Tegra20, Tegra30, Tegra114...) -* '^tegra[23]0$' All boards with either Tegra20 or Tegra30 SoC -* 'powerpc' All PowerPC boards - -While the default is to OR the terms together, you can also make use of -the '&' operator to limit the selection: - -* 'freescale & arm sandbox' All Freescale boards with ARM architecture, - plus sandbox - -You can also use -x to specifically exclude some boards. For example: - - buildman arm -x nvidia,freescale,.*ball$ - -means to build all arm boards except nvidia, freescale and anything ending -with 'ball'. - -For building specific boards you can use the --boards (or --bo) option, which -takes a comma-separated list of board target names and be used multiple times -on the command line: - - buildman --boards sandbox,snow --boards - -It is convenient to use the -n option to see what will be built based on -the subset given. Use -v as well to get an actual list of boards. - -Buildman does not store intermediate object files. It optionally copies -the binary output into a directory when a build is successful (-k). Size -information is always recorded. It needs a fair bit of disk space to work, -typically 250MB per thread. - - -Setting up -========== - -1. Get the U-Boot source. You probably already have it, but if not these -steps should get you started with a repo and some commits for testing. - -$ cd /path/to/u-boot -$ git clone git://git.denx.de/u-boot.git . -$ git checkout -b my-branch origin/master -$ # Add some commits to the branch, reading for testing - -2. Create ~/.buildman to tell buildman where to find tool chains (see 'The -.buildman file' later for details). As an example: - -# Buildman settings file - -[toolchain] -root: / -rest: /toolchains/* -eldk: /opt/eldk-4.2 -arm: /opt/linaro/gcc-linaro-arm-linux-gnueabihf-4.8-2013.08_linux -aarch64: /opt/linaro/gcc-linaro-aarch64-none-elf-4.8-2013.10_linux - -[toolchain-alias] -x86: i386 -blackfin: bfin -openrisc: or1k - - -This selects the available toolchain paths. Add the base directory for -each of your toolchains here. Buildman will search inside these directories -and also in any '/usr' and '/usr/bin' subdirectories. - -Make sure the tags (here root: rest: and eldk:) are unique. - -The toolchain-alias section indicates that the i386 toolchain should be used -to build x86 commits. - -Note that you can also specific exactly toolchain prefixes if you like: - -[toolchain-prefix] -arm: /opt/arm-eabi-4.6/bin/arm-eabi- - -or even: - -[toolchain-prefix] -arm: /opt/arm-eabi-4.6/bin/arm-eabi-gcc - -This tells buildman that you want to use this exact toolchain for the arm -architecture. This will override any toolchains found by searching using the -[toolchain] settings. - -Since the toolchain prefix is an explicit request, buildman will report an -error if a toolchain is not found with that prefix. The current PATH will be -searched, so it is possible to use: - -[toolchain-prefix] -arm: arm-none-eabi- - -and buildman will find arm-none-eabi-gcc in /usr/bin if you have it installed. - -[toolchain-wrapper] -wrapper: ccache - -This tells buildman to use a compiler wrapper in front of CROSS_COMPILE. In -this example, ccache. It doesn't affect the toolchain scan. The wrapper is -added when CROSS_COMPILE environtal variable is set. The name in this -section is ignored. If more than one line is provided, only the last one -is taken. - -3. Make sure you have the require Python pre-requisites - -Buildman uses multiprocessing, Queue, shutil, StringIO, ConfigParser and -urllib2. These should normally be available, but if you get an error like -this then you will need to obtain those modules: - - ImportError: No module named multiprocessing - - -4. Check the available toolchains - -Run this check to make sure that you have a toolchain for every architecture. - -$ ./tools/buildman/buildman --list-tool-chains -Scanning for tool chains - - scanning prefix '/opt/gcc-4.6.3-nolibc/x86_64-linux/bin/x86_64-linux-' -Tool chain test: OK, arch='x86', priority 1 - - scanning prefix '/opt/arm-eabi-4.6/bin/arm-eabi-' -Tool chain test: OK, arch='arm', priority 1 - - scanning path '/toolchains/gcc-4.9.0-nolibc/i386-linux' - - looking in '/toolchains/gcc-4.9.0-nolibc/i386-linux/.' - - looking in '/toolchains/gcc-4.9.0-nolibc/i386-linux/bin' - - found '/toolchains/gcc-4.9.0-nolibc/i386-linux/bin/i386-linux-gcc' - - looking in '/toolchains/gcc-4.9.0-nolibc/i386-linux/usr/bin' -Tool chain test: OK, arch='i386', priority 4 - - scanning path '/toolchains/gcc-4.9.0-nolibc/aarch64-linux' - - looking in '/toolchains/gcc-4.9.0-nolibc/aarch64-linux/.' - - looking in '/toolchains/gcc-4.9.0-nolibc/aarch64-linux/bin' - - found '/toolchains/gcc-4.9.0-nolibc/aarch64-linux/bin/aarch64-linux-gcc' - - looking in '/toolchains/gcc-4.9.0-nolibc/aarch64-linux/usr/bin' -Tool chain test: OK, arch='aarch64', priority 4 - - scanning path '/toolchains/gcc-4.9.0-nolibc/microblaze-linux' - - looking in '/toolchains/gcc-4.9.0-nolibc/microblaze-linux/.' - - looking in '/toolchains/gcc-4.9.0-nolibc/microblaze-linux/bin' - - found '/toolchains/gcc-4.9.0-nolibc/microblaze-linux/bin/microblaze-linux-gcc' - - looking in '/toolchains/gcc-4.9.0-nolibc/microblaze-linux/usr/bin' -Tool chain test: OK, arch='microblaze', priority 4 - - scanning path '/toolchains/gcc-4.9.0-nolibc/mips64-linux' - - looking in '/toolchains/gcc-4.9.0-nolibc/mips64-linux/.' - - looking in '/toolchains/gcc-4.9.0-nolibc/mips64-linux/bin' - - found '/toolchains/gcc-4.9.0-nolibc/mips64-linux/bin/mips64-linux-gcc' - - looking in '/toolchains/gcc-4.9.0-nolibc/mips64-linux/usr/bin' -Tool chain test: OK, arch='mips64', priority 4 - - scanning path '/toolchains/gcc-4.9.0-nolibc/sparc64-linux' - - looking in '/toolchains/gcc-4.9.0-nolibc/sparc64-linux/.' - - looking in '/toolchains/gcc-4.9.0-nolibc/sparc64-linux/bin' - - found '/toolchains/gcc-4.9.0-nolibc/sparc64-linux/bin/sparc64-linux-gcc' - - looking in '/toolchains/gcc-4.9.0-nolibc/sparc64-linux/usr/bin' -Tool chain test: OK, arch='sparc64', priority 4 - - scanning path '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi' - - looking in '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi/.' - - looking in '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi/bin' - - found '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi/bin/arm-unknown-linux-gnueabi-gcc' - - looking in '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi/usr/bin' -Tool chain test: OK, arch='arm', priority 3 -Toolchain '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi/bin/arm-unknown-linux-gnueabi-gcc' at priority 3 will be ignored because another toolchain for arch 'arm' has priority 1 - - scanning path '/toolchains/gcc-4.9.0-nolibc/sparc-linux' - - looking in '/toolchains/gcc-4.9.0-nolibc/sparc-linux/.' - - looking in '/toolchains/gcc-4.9.0-nolibc/sparc-linux/bin' - - found '/toolchains/gcc-4.9.0-nolibc/sparc-linux/bin/sparc-linux-gcc' - - looking in '/toolchains/gcc-4.9.0-nolibc/sparc-linux/usr/bin' -Tool chain test: OK, arch='sparc', priority 4 - - scanning path '/toolchains/gcc-4.9.0-nolibc/mips-linux' - - looking in '/toolchains/gcc-4.9.0-nolibc/mips-linux/.' - - looking in '/toolchains/gcc-4.9.0-nolibc/mips-linux/bin' - - found '/toolchains/gcc-4.9.0-nolibc/mips-linux/bin/mips-linux-gcc' - - looking in '/toolchains/gcc-4.9.0-nolibc/mips-linux/usr/bin' -Tool chain test: OK, arch='mips', priority 4 - - scanning path '/toolchains/gcc-4.9.0-nolibc/x86_64-linux' - - looking in '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/.' - - looking in '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/bin' - - found '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc' - - found '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/bin/x86_64-linux-x86_64-linux-gcc' - - looking in '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/usr/bin' -Tool chain test: OK, arch='x86_64', priority 4 -Tool chain test: OK, arch='x86_64', priority 4 -Toolchain '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/bin/x86_64-linux-x86_64-linux-gcc' at priority 4 will be ignored because another toolchain for arch 'x86_64' has priority 4 - - scanning path '/toolchains/gcc-4.9.0-nolibc/m68k-linux' - - looking in '/toolchains/gcc-4.9.0-nolibc/m68k-linux/.' - - looking in '/toolchains/gcc-4.9.0-nolibc/m68k-linux/bin' - - found '/toolchains/gcc-4.9.0-nolibc/m68k-linux/bin/m68k-linux-gcc' - - looking in '/toolchains/gcc-4.9.0-nolibc/m68k-linux/usr/bin' -Tool chain test: OK, arch='m68k', priority 4 - - scanning path '/toolchains/gcc-4.9.0-nolibc/powerpc-linux' - - looking in '/toolchains/gcc-4.9.0-nolibc/powerpc-linux/.' - - looking in '/toolchains/gcc-4.9.0-nolibc/powerpc-linux/bin' - - found '/toolchains/gcc-4.9.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc' - - looking in '/toolchains/gcc-4.9.0-nolibc/powerpc-linux/usr/bin' -Tool chain test: OK, arch='powerpc', priority 4 - - scanning path '/toolchains/gcc-4.6.3-nolibc/bfin-uclinux' - - looking in '/toolchains/gcc-4.6.3-nolibc/bfin-uclinux/.' - - looking in '/toolchains/gcc-4.6.3-nolibc/bfin-uclinux/bin' - - found '/toolchains/gcc-4.6.3-nolibc/bfin-uclinux/bin/bfin-uclinux-gcc' - - looking in '/toolchains/gcc-4.6.3-nolibc/bfin-uclinux/usr/bin' -Tool chain test: OK, arch='bfin', priority 6 - - scanning path '/toolchains/gcc-4.6.3-nolibc/sparc-linux' - - looking in '/toolchains/gcc-4.6.3-nolibc/sparc-linux/.' - - looking in '/toolchains/gcc-4.6.3-nolibc/sparc-linux/bin' - - found '/toolchains/gcc-4.6.3-nolibc/sparc-linux/bin/sparc-linux-gcc' - - looking in '/toolchains/gcc-4.6.3-nolibc/sparc-linux/usr/bin' -Tool chain test: OK, arch='sparc', priority 4 -Toolchain '/toolchains/gcc-4.6.3-nolibc/sparc-linux/bin/sparc-linux-gcc' at priority 4 will be ignored because another toolchain for arch 'sparc' has priority 4 - - scanning path '/toolchains/gcc-4.6.3-nolibc/mips-linux' - - looking in '/toolchains/gcc-4.6.3-nolibc/mips-linux/.' - - looking in '/toolchains/gcc-4.6.3-nolibc/mips-linux/bin' - - found '/toolchains/gcc-4.6.3-nolibc/mips-linux/bin/mips-linux-gcc' - - looking in '/toolchains/gcc-4.6.3-nolibc/mips-linux/usr/bin' -Tool chain test: OK, arch='mips', priority 4 -Toolchain '/toolchains/gcc-4.6.3-nolibc/mips-linux/bin/mips-linux-gcc' at priority 4 will be ignored because another toolchain for arch 'mips' has priority 4 - - scanning path '/toolchains/gcc-4.6.3-nolibc/m68k-linux' - - looking in '/toolchains/gcc-4.6.3-nolibc/m68k-linux/.' - - looking in '/toolchains/gcc-4.6.3-nolibc/m68k-linux/bin' - - found '/toolchains/gcc-4.6.3-nolibc/m68k-linux/bin/m68k-linux-gcc' - - looking in '/toolchains/gcc-4.6.3-nolibc/m68k-linux/usr/bin' -Tool chain test: OK, arch='m68k', priority 4 -Toolchain '/toolchains/gcc-4.6.3-nolibc/m68k-linux/bin/m68k-linux-gcc' at priority 4 will be ignored because another toolchain for arch 'm68k' has priority 4 - - scanning path '/toolchains/gcc-4.6.3-nolibc/powerpc-linux' - - looking in '/toolchains/gcc-4.6.3-nolibc/powerpc-linux/.' - - looking in '/toolchains/gcc-4.6.3-nolibc/powerpc-linux/bin' - - found '/toolchains/gcc-4.6.3-nolibc/powerpc-linux/bin/powerpc-linux-gcc' - - looking in '/toolchains/gcc-4.6.3-nolibc/powerpc-linux/usr/bin' -Tool chain test: OK, arch='powerpc', priority 4 -Tool chain test: OK, arch='or32', priority 4 - - scanning path '/' - - looking in '/.' - - looking in '/bin' - - looking in '/usr/bin' - - found '/usr/bin/i586-mingw32msvc-gcc' - - found '/usr/bin/c89-gcc' - - found '/usr/bin/x86_64-linux-gnu-gcc' - - found '/usr/bin/gcc' - - found '/usr/bin/c99-gcc' - - found '/usr/bin/arm-linux-gnueabi-gcc' - - found '/usr/bin/aarch64-linux-gnu-gcc' - - found '/usr/bin/winegcc' - - found '/usr/bin/arm-linux-gnueabihf-gcc' -Tool chain test: OK, arch='i586', priority 11 -Tool chain test: OK, arch='c89', priority 11 -Tool chain test: OK, arch='x86_64', priority 4 -Toolchain '/usr/bin/x86_64-linux-gnu-gcc' at priority 4 will be ignored because another toolchain for arch 'x86_64' has priority 4 -Tool chain test: OK, arch='sandbox', priority 11 -Tool chain test: OK, arch='c99', priority 11 -Tool chain test: OK, arch='arm', priority 4 -Toolchain '/usr/bin/arm-linux-gnueabi-gcc' at priority 4 will be ignored because another toolchain for arch 'arm' has priority 1 -Tool chain test: OK, arch='aarch64', priority 4 -Toolchain '/usr/bin/aarch64-linux-gnu-gcc' at priority 4 will be ignored because another toolchain for arch 'aarch64' has priority 4 -Tool chain test: OK, arch='sandbox', priority 11 -Toolchain '/usr/bin/winegcc' at priority 11 will be ignored because another toolchain for arch 'sandbox' has priority 11 -Tool chain test: OK, arch='arm', priority 4 -Toolchain '/usr/bin/arm-linux-gnueabihf-gcc' at priority 4 will be ignored because another toolchain for arch 'arm' has priority 1 -List of available toolchains (34): -aarch64 : /toolchains/gcc-4.9.0-nolibc/aarch64-linux/bin/aarch64-linux-gcc -alpha : /toolchains/gcc-4.9.0-nolibc/alpha-linux/bin/alpha-linux-gcc -am33_2.0 : /toolchains/gcc-4.9.0-nolibc/am33_2.0-linux/bin/am33_2.0-linux-gcc -arm : /opt/arm-eabi-4.6/bin/arm-eabi-gcc -bfin : /toolchains/gcc-4.6.3-nolibc/bfin-uclinux/bin/bfin-uclinux-gcc -c89 : /usr/bin/c89-gcc -c99 : /usr/bin/c99-gcc -frv : /toolchains/gcc-4.9.0-nolibc/frv-linux/bin/frv-linux-gcc -h8300 : /toolchains/gcc-4.9.0-nolibc/h8300-elf/bin/h8300-elf-gcc -hppa : /toolchains/gcc-4.9.0-nolibc/hppa-linux/bin/hppa-linux-gcc -hppa64 : /toolchains/gcc-4.9.0-nolibc/hppa64-linux/bin/hppa64-linux-gcc -i386 : /toolchains/gcc-4.9.0-nolibc/i386-linux/bin/i386-linux-gcc -i586 : /usr/bin/i586-mingw32msvc-gcc -ia64 : /toolchains/gcc-4.9.0-nolibc/ia64-linux/bin/ia64-linux-gcc -m32r : /toolchains/gcc-4.9.0-nolibc/m32r-linux/bin/m32r-linux-gcc -m68k : /toolchains/gcc-4.9.0-nolibc/m68k-linux/bin/m68k-linux-gcc -microblaze: /toolchains/gcc-4.9.0-nolibc/microblaze-linux/bin/microblaze-linux-gcc -mips : /toolchains/gcc-4.9.0-nolibc/mips-linux/bin/mips-linux-gcc -mips64 : /toolchains/gcc-4.9.0-nolibc/mips64-linux/bin/mips64-linux-gcc -or32 : /toolchains/gcc-4.5.1-nolibc/or32-linux/bin/or32-linux-gcc -powerpc : /toolchains/gcc-4.9.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc -powerpc64 : /toolchains/gcc-4.9.0-nolibc/powerpc64-linux/bin/powerpc64-linux-gcc -ppc64le : /toolchains/gcc-4.9.0-nolibc/ppc64le-linux/bin/ppc64le-linux-gcc -s390x : /toolchains/gcc-4.9.0-nolibc/s390x-linux/bin/s390x-linux-gcc -sandbox : /usr/bin/gcc -sh4 : /toolchains/gcc-4.6.3-nolibc/sh4-linux/bin/sh4-linux-gcc -sparc : /toolchains/gcc-4.9.0-nolibc/sparc-linux/bin/sparc-linux-gcc -sparc64 : /toolchains/gcc-4.9.0-nolibc/sparc64-linux/bin/sparc64-linux-gcc -tilegx : /toolchains/gcc-4.6.2-nolibc/tilegx-linux/bin/tilegx-linux-gcc -x86 : /opt/gcc-4.6.3-nolibc/x86_64-linux/bin/x86_64-linux-gcc -x86_64 : /toolchains/gcc-4.9.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc - - -You can see that everything is covered, even some strange ones that won't -be used (c88 and c99). This is a feature. - - -5. Install new toolchains if needed - -You can download toolchains and update the [toolchain] section of the -settings file to find them. - -To make this easier, buildman can automatically download and install -toolchains from kernel.org. First list the available architectures: - -$ ./tools/buildman/buildman --fetch-arch list -Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.3/ -Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.2/ -Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.5.1/ -Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.2.4/ -Available architectures: alpha am33_2.0 arm bfin cris crisv32 frv h8300 -hppa hppa64 i386 ia64 m32r m68k mips mips64 or32 powerpc powerpc64 s390x sh4 -sparc sparc64 tilegx x86_64 xtensa - -Then pick one and download it: - -$ ./tools/buildman/buildman --fetch-arch or32 -Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.3/ -Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.2/ -Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.5.1/ -Downloading: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.5.1//x86_64-gcc-4.5.1-nolibc_or32-linux.tar.xz -Unpacking to: /home/sjg/.buildman-toolchains -Testing - - looking in '/home/sjg/.buildman-toolchains/gcc-4.5.1-nolibc/or32-linux/.' - - looking in '/home/sjg/.buildman-toolchains/gcc-4.5.1-nolibc/or32-linux/bin' - - found '/home/sjg/.buildman-toolchains/gcc-4.5.1-nolibc/or32-linux/bin/or32-linux-gcc' -Tool chain test: OK - -Or download them all from kernel.org and move them to /toolchains directory, - -$ ./tools/buildman/buildman --fetch-arch all -$ sudo mkdir -p /toolchains -$ sudo mv ~/.buildman-toolchains/*/* /toolchains/ - -For those not available from kernel.org, download from the following links. - -arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases/ - download/arc-2016.09-release/arc_gnu_2016.09_prebuilt_uclibc_le_archs_linux_install.tar.gz -blackfin: http://sourceforge.net/projects/adi-toolchain/files/ - blackfin-toolchain-elf-gcc-4.5-2014R1_45-RC2.x86_64.tar.bz2 -nios2: http://sourcery.mentor.com/public/gnu_toolchain/nios2-linux-gnu/ - sourceryg++-2015.11-27-nios2-linux-gnu-i686-pc-linux-gnu.tar.bz2 -sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu/ - renesas-4.4-200-sh-linux-gnu-i686-pc-linux-gnu.tar.bz2 - -Note openrisc kernel.org toolchain is out of date. Download the latest one from -http://opencores.org/or1k/OpenRISC_GNU_tool_chain#Prebuilt_versions - eg: -ftp://ocuser:ocuser@openrisc.opencores.org/toolchain/gcc-or1k-elf-4.8.1-x86.tar.bz2. - -Buildman should now be set up to use your new toolchain. - -At the time of writing, U-Boot has these architectures: - - arc, arm, blackfin, m68k, microblaze, mips, nios2, openrisc - powerpc, sandbox, sh, sparc, x86 - -Of these, only arc is not available at kernel.org.. - - -How to run it -============= - -First do a dry run using the -n flag: (replace <branch> with a real, local -branch with a valid upstream) - -$ ./tools/buildman/buildman -b <branch> -n - -If it can't detect the upstream branch, try checking out the branch, and -doing something like 'git branch --set-upstream-to upstream/master' -or something similar. Buildman will try to guess a suitable upstream branch -if it can't find one (you will see a message like" Guessing upstream as ...). -You can also use the -c option to manually specify the number of commits to -build. - -As an example: - -Dry run, so not doing much. But I would do this: - -Building 18 commits for 1059 boards (4 threads, 1 job per thread) -Build directory: ../lcd9b - 5bb3505 Merge branch 'master' of git://git.denx.de/u-boot-arm - c18f1b4 tegra: Use const for pinmux_config_pingroup/table() - 2f043ae tegra: Add display support to funcmux - e349900 tegra: fdt: Add pwm binding and node - 424a5f0 tegra: fdt: Add LCD definitions for Tegra - 0636ccf tegra: Add support for PWM - a994fe7 tegra: Add SOC support for display/lcd - fcd7350 tegra: Add LCD driver - 4d46e9d tegra: Add LCD support to Nvidia boards - 991bd48 arm: Add control over cachability of memory regions - 54e8019 lcd: Add CONFIG_LCD_ALIGNMENT to select frame buffer alignment - d92aff7 lcd: Add support for flushing LCD fb from dcache after update - dbd0677 tegra: Align LCD frame buffer to section boundary - 0cff9b8 tegra: Support control of cache settings for LCD - 9c56900 tegra: fdt: Add LCD definitions for Seaboard - 5cc29db lcd: Add CONFIG_CONSOLE_SCROLL_LINES option to speed console - cac5a23 tegra: Enable display/lcd support on Seaboard - 49ff541 wip - -Total boards to build for each commit: 1059 - -This shows that it will build all 1059 boards, using 4 threads (because -we have a 4-core CPU). Each thread will run with -j1, meaning that each -make job will use a single CPU. The list of commits to be built helps you -confirm that things look about right. Notice that buildman has chosen a -'base' directory for you, immediately above your source tree. - -Buildman works entirely inside the base directory, here ../lcd9b, -creating a working directory for each thread, and creating output -directories for each commit and board. - - -Suggested Workflow -================== - -To run the build for real, take off the -n: - -$ ./tools/buildman/buildman -b <branch> - -Buildman will set up some working directories, and get started. After a -minute or so it will settle down to a steady pace, with a display like this: - -Building 18 commits for 1059 boards (4 threads, 1 job per thread) - 528 36 124 /19062 -18374 1:13:30 : SIMPC8313_SP - -This means that it is building 19062 board/commit combinations. So far it -has managed to successfully build 528. Another 36 have built with warnings, -and 124 more didn't build at all. It has 18374 builds left to complete. -Buildman expects to complete the process in around an hour and a quarter. -Use this time to buy a faster computer. - - -To find out how the build went, ask for a summary with -s. You can do this -either before the build completes (presumably in another terminal) or -afterwards. Let's work through an example of how this is used: - -$ ./tools/buildman/buildman -b lcd9b -s -... -01: Merge branch 'master' of git://git.denx.de/u-boot-arm - powerpc: + galaxy5200_LOWBOOT -02: tegra: Use const for pinmux_config_pingroup/table() -03: tegra: Add display support to funcmux -04: tegra: fdt: Add pwm binding and node -05: tegra: fdt: Add LCD definitions for Tegra -06: tegra: Add support for PWM -07: tegra: Add SOC support for display/lcd -08: tegra: Add LCD driver -09: tegra: Add LCD support to Nvidia boards -10: arm: Add control over cachability of memory regions -11: lcd: Add CONFIG_LCD_ALIGNMENT to select frame buffer alignment -12: lcd: Add support for flushing LCD fb from dcache after update - arm: + lubbock -13: tegra: Align LCD frame buffer to section boundary -14: tegra: Support control of cache settings for LCD -15: tegra: fdt: Add LCD definitions for Seaboard -16: lcd: Add CONFIG_CONSOLE_SCROLL_LINES option to speed console -17: tegra: Enable display/lcd support on Seaboard -18: wip - -This shows which commits have succeeded and which have failed. In this case -the build is still in progress so many boards are not built yet (use -u to -see which ones). But already we can see a few failures. The galaxy5200_LOWBOOT -never builds correctly. This could be a problem with our toolchain, or it -could be a bug in the upstream. The good news is that we probably don't need -to blame our commits. The bad news is that our commits are not tested on that -board. - -Commit 12 broke lubbock. That's what the '+ lubbock', in red, means. The -failure is never fixed by a later commit, or you would see lubbock again, in -green, without the +. - -To see the actual error: - -$ ./tools/buildman/buildman -b <branch> -se -... -12: lcd: Add support for flushing LCD fb from dcache after update - arm: + lubbock -+common/libcommon.o: In function `lcd_sync': -+common/lcd.c:120: undefined reference to `flush_dcache_range' -+arm-none-linux-gnueabi-ld: BFD (Sourcery G++ Lite 2010q1-202) 2.19.51.20090709 assertion fail /scratch/julian/2010q1-release-linux-lite/obj/binutils-src-2010q1-202-arm-none-linux-gnueabi-i686-pc-linux-gnu/bfd/elf32-arm.c:12572 -+make: *** [build/u-boot] Error 139 -13: tegra: Align LCD frame buffer to section boundary -14: tegra: Support control of cache settings for LCD -15: tegra: fdt: Add LCD definitions for Seaboard -16: lcd: Add CONFIG_CONSOLE_SCROLL_LINES option to speed console --common/lcd.c:120: undefined reference to `flush_dcache_range' -+common/lcd.c:125: undefined reference to `flush_dcache_range' -17: tegra: Enable display/lcd support on Seaboard -18: wip - -So the problem is in lcd.c, due to missing cache operations. This information -should be enough to work out what that commit is doing to break these -boards. (In this case pxa did not have cache operations defined). - -Note that if there were other boards with errors, the above command would -show their errors also. Each line is shown only once. So if lubbock and snow -produce the same error, we just see: - -12: lcd: Add support for flushing LCD fb from dcache after update - arm: + lubbock snow -+common/libcommon.o: In function `lcd_sync': -+common/lcd.c:120: undefined reference to `flush_dcache_range' -+arm-none-linux-gnueabi-ld: BFD (Sourcery G++ Lite 2010q1-202) 2.19.51.20090709 assertion fail /scratch/julian/2010q1-release-linux-lite/obj/binutils-src-2010q1-202-arm-none-linux-gnueabi-i686-pc-linux-gnu/bfd/elf32-arm.c:12572 -+make: *** [build/u-boot] Error 139 - -But if you did want to see just the errors for lubbock, use: - -$ ./tools/buildman/buildman -b <branch> -se lubbock - -If you see error lines marked with '-', that means that the errors were fixed -by that commit. Sometimes commits can be in the wrong order, so that a -breakage is introduced for a few commits and fixed by later commits. This -shows up clearly with buildman. You can then reorder the commits and try -again. - -At commit 16, the error moves: you can see that the old error at line 120 -is fixed, but there is a new one at line 126. This is probably only because -we added some code and moved the broken line further down the file. - -As mentioned, if many boards have the same error, then -e will display the -error only once. This makes the output as concise as possible. To see which -boards have each error, use -l. So it is safe to omit the board name - you -will not get lots of repeated output for every board. - -Buildman tries to distinguish warnings from errors, and shows warning lines -separately with a 'w' prefix. Warnings introduced show as yellow. Warnings -fixed show as cyan. - -The full build output in this case is available in: - -../lcd9b/12_of_18_gd92aff7_lcd--Add-support-for/lubbock/ - - done: Indicates the build was done, and holds the return code from make. - This is 0 for a good build, typically 2 for a failure. - - err: Output from stderr, if any. Errors and warnings appear here. - - log: Output from stdout. Normally there isn't any since buildman runs - in silent mode. Use -V to force a verbose build (this passes V=1 - to 'make') - - toolchain: Shows information about the toolchain used for the build. - - sizes: Shows image size information. - -It is possible to get the build binary output there also. Use the -k option -for this. In that case you will also see some output files, like: - - System.map toolchain u-boot u-boot.bin u-boot.map autoconf.mk - (also SPL versions u-boot-spl and u-boot-spl.bin if available) - - -Checking Image Sizes -==================== - -A key requirement for U-Boot is that you keep code/data size to a minimum. -Where a new feature increases this noticeably it should normally be put -behind a CONFIG flag so that boards can leave it disabled and keep the image -size more or less the same with each new release. - -To check the impact of your commits on image size, use -S. For example: - -$ ./tools/buildman/buildman -b us-x86 -sS -Summary of 10 commits for 1066 boards (4 threads, 1 job per thread) -01: MAKEALL: add support for per architecture toolchains -02: x86: Add function to get top of usable ram - x86: (for 1/3 boards) text -272.0 rodata +41.0 -03: x86: Add basic cache operations -04: x86: Permit bootstage and timer data to be used prior to relocation - x86: (for 1/3 boards) data +16.0 -05: x86: Add an __end symbol to signal the end of the U-Boot binary - x86: (for 1/3 boards) text +76.0 -06: x86: Rearrange the output input to remove BSS - x86: (for 1/3 boards) bss -2140.0 -07: x86: Support relocation of FDT on start-up - x86: + coreboot-x86 -08: x86: Add error checking to x86 relocation code -09: x86: Adjust link device tree include file -10: x86: Enable CONFIG_OF_CONTROL on coreboot - - -You can see that image size only changed on x86, which is good because this -series is not supposed to change any other board. From commit 7 onwards the -build fails so we don't get code size numbers. The numbers are fractional -because they are an average of all boards for that architecture. The -intention is to allow you to quickly find image size problems introduced by -your commits. - -Note that the 'text' region and 'rodata' are split out. You should add the -two together to get the total read-only size (reported as the first column -in the output from binutil's 'size' utility). - -A useful option is --step which lets you skip some commits. For example ---step 2 will show the image sizes for only every 2nd commit (so it will -compare the image sizes of the 1st, 3rd, 5th... commits). You can also use ---step 0 which will compare only the first and last commits. This is useful -for an overview of how your entire series affects code size. It will build -only the upstream commit and your final branch commit. - -You can also use -d to see a detailed size breakdown for each board. This -list is sorted in order from largest growth to largest reduction. - -It is even possible to go a little further with the -B option (--bloat). This -shows where U-Boot has bloated, breaking the size change down to the function -level. Example output is below: - -$ ./tools/buildman/buildman -b us-mem4 -sSdB -... -19: Roll crc32 into hash infrastructure - arm: (for 10/10 boards) all -143.4 bss +1.2 data -4.8 rodata -48.2 text -91.6 - paz00 : all +23 bss -4 rodata -29 text +56 - u-boot: add: 1/0, grow: 3/-2 bytes: 168/-104 (64) - function old new delta - hash_command 80 160 +80 - crc32_wd_buf - 56 +56 - ext4fs_read_file 540 568 +28 - insert_var_value_sub 688 692 +4 - run_list_real 1996 1992 -4 - do_mem_crc 168 68 -100 - trimslice : all -9 bss +16 rodata -29 text +4 - u-boot: add: 1/0, grow: 1/-3 bytes: 136/-124 (12) - function old new delta - hash_command 80 160 +80 - crc32_wd_buf - 56 +56 - ext4fs_iterate_dir 672 668 -4 - ext4fs_read_file 568 548 -20 - do_mem_crc 168 68 -100 - whistler : all -9 bss +16 rodata -29 text +4 - u-boot: add: 1/0, grow: 1/-3 bytes: 136/-124 (12) - function old new delta - hash_command 80 160 +80 - crc32_wd_buf - 56 +56 - ext4fs_iterate_dir 672 668 -4 - ext4fs_read_file 568 548 -20 - do_mem_crc 168 68 -100 - seaboard : all -9 bss -28 rodata -29 text +48 - u-boot: add: 1/0, grow: 3/-2 bytes: 160/-104 (56) - function old new delta - hash_command 80 160 +80 - crc32_wd_buf - 56 +56 - ext4fs_read_file 548 568 +20 - run_list_real 1996 2000 +4 - do_nandboot 760 756 -4 - do_mem_crc 168 68 -100 - colibri_t20 : all -9 rodata -29 text +20 - u-boot: add: 1/0, grow: 2/-3 bytes: 140/-112 (28) - function old new delta - hash_command 80 160 +80 - crc32_wd_buf - 56 +56 - read_abs_bbt 204 208 +4 - do_nandboot 760 756 -4 - ext4fs_read_file 576 568 -8 - do_mem_crc 168 68 -100 - ventana : all -37 bss -12 rodata -29 text +4 - u-boot: add: 1/0, grow: 1/-3 bytes: 136/-124 (12) - function old new delta - hash_command 80 160 +80 - crc32_wd_buf - 56 +56 - ext4fs_iterate_dir 672 668 -4 - ext4fs_read_file 568 548 -20 - do_mem_crc 168 68 -100 - harmony : all -37 bss -16 rodata -29 text +8 - u-boot: add: 1/0, grow: 2/-3 bytes: 140/-124 (16) - function old new delta - hash_command 80 160 +80 - crc32_wd_buf - 56 +56 - nand_write_oob_syndrome 428 432 +4 - ext4fs_iterate_dir 672 668 -4 - ext4fs_read_file 568 548 -20 - do_mem_crc 168 68 -100 - medcom-wide : all -417 bss +28 data -16 rodata -93 text -336 - u-boot: add: 1/-1, grow: 1/-2 bytes: 88/-376 (-288) - function old new delta - crc32_wd_buf - 56 +56 - do_fat_read_at 2872 2904 +32 - hash_algo 16 - -16 - do_mem_crc 168 68 -100 - hash_command 420 160 -260 - tec : all -449 bss -4 data -16 rodata -93 text -336 - u-boot: add: 1/-1, grow: 1/-2 bytes: 88/-376 (-288) - function old new delta - crc32_wd_buf - 56 +56 - do_fat_read_at 2872 2904 +32 - hash_algo 16 - -16 - do_mem_crc 168 68 -100 - hash_command 420 160 -260 - plutux : all -481 bss +16 data -16 rodata -93 text -388 - u-boot: add: 1/-1, grow: 1/-3 bytes: 68/-408 (-340) - function old new delta - crc32_wd_buf - 56 +56 - do_load_serial_bin 1688 1700 +12 - hash_algo 16 - -16 - do_fat_read_at 2904 2872 -32 - do_mem_crc 168 68 -100 - hash_command 420 160 -260 - powerpc: (for 5/5 boards) all +37.4 data -3.2 rodata -41.8 text +82.4 - MPC8610HPCD : all +55 rodata -29 text +84 - u-boot: add: 1/0, grow: 0/-1 bytes: 176/-96 (80) - function old new delta - hash_command - 176 +176 - do_mem_crc 184 88 -96 - MPC8641HPCN : all +55 rodata -29 text +84 - u-boot: add: 1/0, grow: 0/-1 bytes: 176/-96 (80) - function old new delta - hash_command - 176 +176 - do_mem_crc 184 88 -96 - MPC8641HPCN_36BIT: all +55 rodata -29 text +84 - u-boot: add: 1/0, grow: 0/-1 bytes: 176/-96 (80) - function old new delta - hash_command - 176 +176 - do_mem_crc 184 88 -96 - sbc8641d : all +55 rodata -29 text +84 - u-boot: add: 1/0, grow: 0/-1 bytes: 176/-96 (80) - function old new delta - hash_command - 176 +176 - do_mem_crc 184 88 -96 - xpedite517x : all -33 data -16 rodata -93 text +76 - u-boot: add: 1/-1, grow: 0/-1 bytes: 176/-112 (64) - function old new delta - hash_command - 176 +176 - hash_algo 16 - -16 - do_mem_crc 184 88 -96 -... - - -This shows that commit 19 has reduced codesize for arm slightly and increased -it for powerpc. This increase was offset in by reductions in rodata and -data/bss. - -Shown below the summary lines are the sizes for each board. Below each board -are the sizes for each function. This information starts with: - - add - number of functions added / removed - grow - number of functions which grew / shrunk - bytes - number of bytes of code added to / removed from all functions, - plus the total byte change in brackets - -The change seems to be that hash_command() has increased by more than the -do_mem_crc() function has decreased. The function sizes typically add up to -roughly the text area size, but note that every read-only section except -rodata is included in 'text', so the function total does not exactly -correspond. - -It is common when refactoring code for the rodata to decrease as the text size -increases, and vice versa. - - -The .buildman file -================== - -The .buildman file provides information about the available toolchains and -also allows build flags to be passed to 'make'. It consists of several -sections, with the section name in square brackets. Within each section are -a set of (tag, value) pairs. - -'[toolchain]' section - - This lists the available toolchains. The tag here doesn't matter, but - make sure it is unique. The value is the path to the toolchain. Buildman - will look in that path for a file ending in 'gcc'. It will then execute - it to check that it is a C compiler, passing only the --version flag to - it. If the return code is 0, buildman assumes that it is a valid C - compiler. It uses the first part of the name as the architecture and - strips off the last part when setting the CROSS_COMPILE environment - variable (parts are delimited with a hyphen). - - For example powerpc-linux-gcc will be noted as a toolchain for 'powerpc' - and CROSS_COMPILE will be set to powerpc-linux- when using it. - -'[toolchain-alias]' section - - This converts toolchain architecture names to U-Boot names. For example, - if an x86 toolchains is called i386-linux-gcc it will not normally be - used for architecture 'x86'. Adding 'x86: i386 x86_64' to this section - will tell buildman that the i386 and x86_64 toolchains can be used for - the x86 architecture. - -'[make-flags]' section - - U-Boot's build system supports a few flags (such as BUILD_TAG) which - affect the build product. These flags can be specified in the buildman - settings file. They can also be useful when building U-Boot against other - open source software. - - [make-flags] - at91-boards=ENABLE_AT91_TEST=1 - snapper9260=${at91-boards} BUILD_TAG=442 - snapper9g45=${at91-boards} BUILD_TAG=443 - - This will use 'make ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 - and 'make ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. A special - variable ${target} is available to access the target name (snapper9260 - and snapper9g20 in this case). Variables are resolved recursively. Note - that variables can only contain the characters A-Z, a-z, 0-9, hyphen (-) - and underscore (_). - - It is expected that any variables added are dealt with in U-Boot's - config.mk file and documented in the README. - - Note that you can pass ad-hoc options to the build using environment - variables, for example: - - SOME_OPTION=1234 ./tools/buildman/buildman my_board - - -Quick Sanity Check -================== - -If you have made changes and want to do a quick sanity check of the -currently checked-out source, run buildman without the -b flag. This will -build the selected boards and display build status as it runs (i.e. -v is -enabled automatically). Use -e to see errors/warnings as well. - - -Building Ranges -=============== - -You can build a range of commits by specifying a range instead of a branch -when using the -b flag. For example: - - upstream/master..us-buildman - -will build commits in us-buildman that are not in upstream/master. - - -Building Faster -=============== - -By default, buildman doesn't execute 'make mrproper' prior to building the -first commit for each board. This reduces the amount of work 'make' does, and -hence speeds up the build. To force use of 'make mrproper', use -the -m flag. -This flag will slow down any buildman invocation, since it increases the amount -of work done on any build. - -One possible application of buildman is as part of a continual edit, build, -edit, build, ... cycle; repeatedly applying buildman to the same change or -series of changes while making small incremental modifications to the source -each time. This provides quick feedback regarding the correctness of recent -modifications. In this scenario, buildman's default choice of build directory -causes more build work to be performed than strictly necessary. - -By default, each buildman thread uses a single directory for all builds. When a -thread builds multiple boards, the configuration built in this directory will -cycle through various different configurations, one per board built by the -thread. Variations in the configuration will force a rebuild of affected source -files when a thread switches between boards. Ideally, such buildman-induced -rebuilds would not happen, thus allowing the build to operate as efficiently as -the build system and source changes allow. buildman's -P flag may be used to -enable this; -P causes each board to be built in a separate (board-specific) -directory, thus avoiding any buildman-induced configuration changes in any -build directory. - -U-Boot's build system embeds information such as a build timestamp into the -final binary. This information varies each time U-Boot is built. This causes -various files to be rebuilt even if no source changes are made, which in turn -requires that the final U-Boot binary be re-linked. This unnecessary work can -be avoided by turning off the timestamp feature. This can be achieved by -setting the SOURCE_DATE_EPOCH environment variable to 0. - -Combining all of these options together yields the command-line shown below. -This will provide the quickest possible feedback regarding the current content -of the source tree, thus allowing rapid tested evolution of the code. - - SOURCE_DATE_EPOCH=0 ./tools/buildman/buildman -P tegra - - -Checking configuration -====================== - -A common requirement when converting CONFIG options to Kconfig is to check -that the effective configuration has not changed due to the conversion. -Buildman supports this with the -K option, used after a build. This shows -differences in effective configuration between one commit and the next. - -For example: - - $ buildman -b kc4 -sK - ... - 43: Convert CONFIG_SPL_USBETH_SUPPORT to Kconfig - arm: - + u-boot.cfg: CONFIG_SPL_ENV_SUPPORT=1 CONFIG_SPL_NET=1 - + u-boot-spl.cfg: CONFIG_SPL_MMC=1 CONFIG_SPL_NAND_SUPPORT=1 - + all: CONFIG_SPL_ENV_SUPPORT=1 CONFIG_SPL_MMC=1 CONFIG_SPL_NAND_SUPPORT=1 CONFIG_SPL_NET=1 - am335x_evm_usbspl : - + u-boot.cfg: CONFIG_SPL_ENV_SUPPORT=1 CONFIG_SPL_NET=1 - + u-boot-spl.cfg: CONFIG_SPL_MMC=1 CONFIG_SPL_NAND_SUPPORT=1 - + all: CONFIG_SPL_ENV_SUPPORT=1 CONFIG_SPL_MMC=1 CONFIG_SPL_NAND_SUPPORT=1 CONFIG_SPL_NET=1 - 44: Convert CONFIG_SPL_USB_HOST to Kconfig - ... - -This shows that commit 44 enabled three new options for the board -am335x_evm_usbspl which were not enabled in commit 43. There is also a -summary for 'arm' showing all the changes detected for that architecture. -In this case there is only one board with changes, so 'arm' output is the -same as 'am335x_evm_usbspl'/ - -The -K option uses the u-boot.cfg, spl/u-boot-spl.cfg and tpl/u-boot-tpl.cfg -files which are produced by a build. If all you want is to check the -configuration you can in fact avoid doing a full build, using -D. This tells -buildman to configuration U-Boot and create the .cfg files, but not actually -build the source. This is 5-10 times faster than doing a full build. - -By default buildman considers the follow two configuration methods -equivalent: - - #define CONFIG_SOME_OPTION - - CONFIG_SOME_OPTION=y - -The former would appear in a header filer and the latter in a defconfig -file. The achieve this, buildman considers 'y' to be '1' in configuration -variables. This avoids lots of useless output when converting a CONFIG -option to Kconfig. To disable this behaviour, use --squash-config-y. - - -Checking the environment -======================== - -When converting CONFIG options which manipulate the default environment, -a common requirement is to check that the default environment has not -changed due to the conversion. Buildman supports this with the -U option, -used after a build. This shows differences in the default environment -between one commit and the next. - -For example: - -$ buildman -b squash brppt1 -sU -Summary of 2 commits for 3 boards (3 threads, 3 jobs per thread) -01: Migrate bootlimit to Kconfig -02: Squashed commit of the following: - c brppt1_mmc: altbootcmd=mmc dev 1; run mmcboot0; -> mmc dev 1; run mmcboot0 - c brppt1_spi: altbootcmd=mmc dev 1; run mmcboot0; -> mmc dev 1; run mmcboot0 - + brppt1_nand: altbootcmd=run usbscript - - brppt1_nand: altbootcmd=run usbscript -(no errors to report) - -This shows that commit 2 modified the value of 'altbootcmd' for 'brppt1_mmc' -and 'brppt1_spi', removing a trailing semicolon. 'brppt1_nand' gained an a -value for 'altbootcmd', but lost one for ' altbootcmd'. - -The -U option uses the u-boot.env files which are produced by a build. - - -Building with clang -=================== - -To build with clang (sandbox only), use the -O option to override the -toolchain. For example: - - buildman -O clang-7 --board sandbox - - -Doing a simple build -==================== - -In some cases you just want to build a single board and get the full output, use -the -w option, for example: - - buildman -o /tmp/build --board sandbox -w - -This will write the full build into /tmp/build including object files. You must -specify the output directory with -o when using -w. - - -Support for IDEs (Integrated Development Environments) -====================================================== - -Normally buildman summarises the output and shows information indicating the -meaning of each line of output. For example a '+' symbol appears at the start of -each error line. Also, buildman prints information about what it is about to do, -along with a summary at the end. - -When using buildman from an IDE, it is helpful to drop this behaviour. Use the --I/--ide option for that. You might find -W helpful also so that warnings do -not cause the build to fail: - - buildman -o /tmp/build --board sandbox -wWI - - -Changing the configuration -========================== - -Sometimes it is useful to change the CONFIG options for a build on the fly. This -can be used to build a board (or multiple) with a few changes to see the impact. -The -a option supports this: - - -a <cfg> - -where <cfg> is a CONFIG option (with or without the CONFIG_ prefix) to enable. -For example: - - buildman -a CMD_SETEXPR_FMT - -will build with CONFIG_CMD_SETEXPR_FMT enabled. - -You can disable options by preceding them with tilde (~). You can specify the --a option multiple times: - - buildman -a CMD_SETEXPR_FMT -a ~CMDLINE - -Some options have values, in which case you can change them: - - buildman -a 'BOOTCOMMAND="echo hello"' CONFIG_SYS_LOAD_ADDR=0x1000 - -Note that you must put quotes around string options and the whole thing must be -in single quotes, to make sure the shell leave it alone. - -If you try to set an option that does not exist, or that cannot be changed for -some other reason (e.g. it is 'selected' by another option), then buildman -shows an error: - - buildman --board sandbox -a FRED - Building current source for 1 boards (1 thread, 32 jobs per thread) - 0 0 0 /1 -1 (starting)errs - Some CONFIG adjustments did not take effect. This may be because - the request CONFIGs do not exist or conflict with others. - - Failed adjustments: - - FRED Missing expected line: CONFIG_FRED=y - - -One major caveat with this feature with branches (-b) is that buildman does not -name the output directories differently when you change the configuration, so -doing the same build again with different configuration will not trigger a -rebuild. You can use -f to work around that. - - -Other options -============= - -Buildman has various other command-line options. Try --help to see them. - -To find out what toolchain prefix buildman will use for a build, use the -A -option. - -To request that compiler warnings be promoted to errors, use -E. This passes the --Werror flag to the compiler. Note that the build can still produce warnings -with -E, e.g. the migration warnings: - - ===================== WARNING ====================== - This board does not use CONFIG_DM_MMC. Please update - ... - ==================================================== - -When doing builds, Buildman's return code will reflect the overall result: - - 0 (success) No errors or warnings found - 100 Errors found - 101 Warnings found (only if no -W) - -You can use -W to tell Buildman to return 0 (success) instead of 101 when -warnings are found. Note that it can be useful to combine -E and -W. This means -that all compiler warnings will produce failures (code 100) and all other -warnings will produce success (since 101 is changed to 0). - -If there are both warnings and errors, errors win, so buildman returns 100. - -The -y option is provided (for use with -s) to ignore the bountiful device-tree -warnings. Similarly, -Y tells buildman to ignore the migration warnings. - -Sometimes you might get an error in a thread that is not handled by buildman, -perhaps due to a failure of a tool that it calls. You might see the output, but -then buildman hangs. Failing to handle any eventuality is a bug in buildman and -should be reported. But you can use -T0 to disable threading and hopefully -figure out the root cause of the build failure. - -Build summary -============= - -When buildman finishes it shows a summary, something like this: - - Completed: 5 total built, duration 0:00:21, rate 0.24 - -This shows that a total of 5 builds were done across all selected boards, it -took 21 seconds and the builds happened at the rate of 0.24 per second. The -latter number depends on the speed of your machine and the efficiency of the -U-Boot build. - - -How to change from MAKEALL -========================== - -Buildman includes most of the features of MAKEALL and is generally faster -and easier to use. In particular it builds entire branches: if a particular -commit introduces an error in a particular board, buildman can easily show -you this, even if a later commit fixes that error. - -The reasons to deprecate MAKEALL are: -- We don't want to maintain two build systems -- Buildman is typically faster -- Buildman has a lot more features - -But still, many people will be sad to lose MAKEALL. If you are used to -MAKEALL, here are a few pointers. - -First you need to set up your tool chains - see the 'Setting up' section -for details. Once you have your required toolchain(s) detected then you are -ready to go. - -To build the current source tree, run buildman without a -b flag: - - ./tools/buildman/buildman <list of things to build> - -This will build the current source tree for the given boards and display -the results and errors. - -However buildman usually works on entire branches, and for that you must -specify a board flag: - - ./tools/buildman/buildman -b <branch_name> <list of things to build> - -followed by (afterwards, or perhaps concurrently in another terminal): - - ./tools/buildman/buildman -b <branch_name> -s <list of things to build> - -to see the results of the build. Rather than showing you all the output, -buildman just shows a summary, with red indicating that a commit introduced -an error and green indicating that a commit fixed an error. Use the -e -flag to see the full errors and -l to see which boards caused which errors. - -If you really want to see build results as they happen, use -v when doing a -build (and -e to see the errors/warnings too). - -You don't need to stick around on that branch while buildman is running. It -checks out its own copy of the source code, so you can change branches, -add commits, etc. without affecting the build in progress. - -The <list of things to build> can include board names, architectures or the -like. There are no flags to disambiguate since ambiguities are rare. Using -the examples from MAKEALL: - -Examples: - - build all Power Architecture boards: - MAKEALL -a powerpc - MAKEALL --arch powerpc - MAKEALL powerpc - ** buildman -b <branch> powerpc - - build all PowerPC boards manufactured by vendor "esd": - MAKEALL -a powerpc -v esd - ** buildman -b <branch> esd - - build all PowerPC boards manufactured either by "keymile" or "siemens": - MAKEALL -a powerpc -v keymile -v siemens - ** buildman -b <branch> keymile siemens - - build all Freescale boards with MPC83xx CPUs, plus all 4xx boards: - MAKEALL -c mpc83xx -v freescale 4xx - ** buildman -b <branch> mpc83xx freescale 4xx - -Buildman automatically tries to use all the CPUs in your machine. If you -are building a lot of boards it will use one thread for every CPU core -it detects in your machine. This is like MAKEALL's BUILD_NBUILDS option. -You can use the -T flag to change the number of threads. If you are only -building a few boards, buildman will automatically run make with the -j -flag to increase the number of concurrent make tasks. It isn't normally -that helpful to fiddle with this option, but if you use the BUILD_NCPUS -option in MAKEALL then -j is the equivalent in buildman. - -Buildman puts its output in ../<branch_name> by default but you can change -this with the -o option. Buildman normally does out-of-tree builds: use -i -to disable that if you really want to. But be careful that once you have -used -i you pollute buildman's copies of the source tree, and you will need -to remove the build directory (normally ../<branch_name>) to run buildman -in normal mode (without -i). - -Buildman doesn't keep the output result normally, but use the -k option to -do this. - -Please read 'Theory of Operation' a few times as it will make a lot of -things clearer. - -Some options you might like are: - - -B shows which functions are growing/shrinking in which commit - great - for finding code bloat. - -S shows image sizes for each commit (just an overall summary) - -u shows boards that you haven't built yet - --step 0 will build just the upstream commit and the last commit of your - branch. This is often a quick sanity check that your branch doesn't - break anything. But note this does not check bisectability! - - -Using boards.cfg -================ - -This file is no-longer needed by buildman but it is still generated in the -working directory. This helps avoid a delay on every build, since scanning all -the Kconfig files takes a few seconds. Use the -R flag to force regeneration -of the file - in that case buildman exits after writing the file. with exit code -2 if there was an error in the maintainer files. - -You should use 'buildman -nv <criteria>' instead of greoing the boards.cfg file, -since it may be dropped altogether in future. - - -TODO -==== - -Many improvements have been made over the years. There is still quite a bit of -scope for more though, e.g.: - -- easier access to log files -- 'hunting' for problems, perhaps by building a few boards for each arch, or - checking commits for changed files and building only boards which use those - files - - -Credits -======= - -Thanks to Grant Grundler <grundler@chromium.org> for his ideas for improving -the build speed by building all commits for a board instead of the other -way around. - - -Simon Glass -sjg@chromium.org -Halloween 2012 -Updated 12-12-12 -Updated 23-02-13 -Updated 09-04-20 diff --git a/tools/buildman/README.rst b/tools/buildman/README.rst new file mode 120000 index 0000000..c359387 --- /dev/null +++ b/tools/buildman/README.rst @@ -0,0 +1 @@ +buildman.rst
\ No newline at end of file diff --git a/tools/buildman/bsettings.py b/tools/buildman/bsettings.py index 35bb2c1..0eb894a 100644 --- a/tools/buildman/bsettings.py +++ b/tools/buildman/bsettings.py @@ -5,6 +5,7 @@ import configparser import os import io +config_fname = None def Setup(fname=''): """Set up the buildman settings module by reading config files @@ -46,6 +47,17 @@ def GetItems(section): except: raise +def GetGlobalItemValue(name): + """Get an item from the 'global' section of the config. + + Args: + name: name of item to retrieve + + Returns: + str: Value of item, or None if not present + """ + return settings.get('global', name, fallback=None) + def SetItem(section, tag, value): """Set an item and write it back to the settings file""" global settings @@ -79,13 +91,14 @@ other = / [toolchain-prefix] # name = path to prefix # e.g. x86 = /opt/gcc-4.6.3-nolibc/x86_64-linux/bin/x86_64-linux- +# arc = /opt/arc/arc_gnu_2021.03_prebuilt_elf32_le_linux_install/bin/arc-elf32- [toolchain-alias] # arch = alias # Indicates which toolchain should be used to build for that arch +riscv = riscv32 +sh = sh4 x86 = i386 -blackfin = bfin -openrisc = or1k [make-flags] # Special flags to pass to 'make' for certain boards, e.g. to pass a test diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 76252b9..c2a6902 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -252,7 +252,8 @@ class Builder: mrproper=False, per_board_out_dir=False, config_only=False, squash_config_y=False, warnings_as_errors=False, work_in_output=False, - test_thread_exceptions=False, adjust_cfg=None): + test_thread_exceptions=False, adjust_cfg=None, + allow_missing=False): """Create a new Builder object Args: @@ -290,6 +291,7 @@ class Builder: ~C to disable C C=val to set the value of C (val must have quotes if C is a string Kconfig + allow_missing: Run build with BINMAN_ALLOW_MISSING=1 """ self.toolchains = toolchains @@ -327,6 +329,7 @@ class Builder: self.config_filenames = BASE_CONFIG_FILENAMES self.work_in_output = work_in_output self.adjust_cfg = adjust_cfg + self.allow_missing = allow_missing self._ide = False if not self.squash_config_y: diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index 6240e08..680efae 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -253,6 +253,8 @@ class BuilderThread(threading.Thread): args.extend(['-j', str(self.builder.num_jobs)]) if self.builder.warnings_as_errors: args.append('KCFLAGS=-Werror') + if self.builder.allow_missing: + args.append('BINMAN_ALLOW_MISSING=1') config_args = ['%s_defconfig' % brd.target] config_out = '' args.extend(self.builder.toolchains.GetMakeArguments(brd)) @@ -288,10 +290,14 @@ class BuilderThread(threading.Thread): args.append('cfg') result = self.Make(commit, brd, 'build', cwd, *args, env=env) + if (result.return_code == 2 and + ('Some images are invalid' in result.stderr)): + # This is handled later by the check for output in + # stderr + result.return_code = 0 if adjust_cfg: errs = cfgutil.check_cfg_file(cfg_file, adjust_cfg) if errs: - print('errs', errs) result.stderr += errs result.return_code = 1 result.stderr = result.stderr.replace(src_dir + '/', '') diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst new file mode 100644 index 0000000..2a83cb7 --- /dev/null +++ b/tools/buildman/buildman.rst @@ -0,0 +1,1328 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +Buildman build tool +=================== + +Quick-start +----------- + +If you just want to quickly set up buildman so you can build something (for +example Raspberry Pi 2): + +.. code-block:: bash + + cd /path/to/u-boot + PATH=$PATH:`pwd`/tools/buildman + buildman --fetch-arch arm + buildman -k rpi_2 + ls ../current/rpi_2 + # u-boot.bin is the output image + + +What is this? +------------- + +This tool handles building U-Boot to check that you have not broken it +with your patch series. It can build each individual commit and report +which boards fail on which commits, and which errors come up. It aims +to make full use of multi-processor machines. + +A key feature of buildman is its output summary, which allows warnings, +errors or image size increases in a particular commit or board to be +quickly identified and the offending commit pinpointed. This can be a big +help for anyone working with >10 patches at a time. + + +Caveats +------- + +Buildman can be stopped and restarted, in which case it will continue +where it left off. This should happen cleanly and without side-effects. +If not, it is a bug, for which a patch would be welcome. + +Buildman gets so tied up in its work that it can ignore the outside world. +You may need to press Ctrl-C several times to quit it. Also it will print +out various exceptions when stopped. You may have to kill it since the +Ctrl-C handling is somewhat broken. + + +Theory of Operation +------------------- + +(please read this section in full twice or you will be perpetually confused) + +Buildman is a builder. It is not make, although it runs make. It does not +produce any useful output on the terminal while building, except for +progress information (but see -v below). All the output (errors, warnings and +binaries if you ask for them) is stored in output directories, which you can +look at from a separate 'buildman -s' instance while the build is progressing, +or when it is finished. + +Buildman is designed to build entire git branches, i.e. muliple commits. It +can be run repeatedly on the same branch after making changes to commits on +that branch. In this case it will automatically rebuild commits which have +changed (and remove its old results for that commit). It is possible to build +a branch for one board, then later build it for another board. This adds to +the output, so now you have results for two boards. If you want buildman to +re-build a commit it has already built (e.g. because of a toolchain update), +use the -f flag. + +Buildman produces a concise summary of which boards succeeded and failed. +It shows which commit introduced which board failure using a simple +red/green colour coding (with yellow/cyan for warnings). Full error +information can be requested, in which case it is de-duped and displayed +against the commit that introduced the error. An example workflow is below. + +Buildman stores image size information and can report changes in image size +from commit to commit. An example of this is below. + +Buildman starts multiple threads, and each thread builds for one board at +a time. A thread starts at the first commit, configures the source for your +board and builds it. Then it checks out the next commit and does an +incremental build (i.e. not using 'make xxx_defconfig' unless you use -C). +Eventually the thread reaches the last commit and stops. If a commit causes +an error or warning, buildman will try it again after reconfiguring (but see +-Q). Thus some commits may be built twice, with the first result silently +discarded. Lots of errors and warnings will causes lots of reconfigures and your +build will be very slow. This is because a file that produces just a warning +would not normally be rebuilt in an incremental build. Once a thread finishes +building all the commits for a board, it starts on the commits for another +board. + +Buildman works in an entirely separate place from your U-Boot repository. +It creates a separate working directory for each thread, and puts the +output files in the working directory, organised by commit name and board +name, in a two-level hierarchy (but see -P). + +Buildman is invoked in your U-Boot directory, the one with the .git +directory. It clones this repository into a copy for each thread, and the +threads do not affect the state of your git repository. Any checkouts done +by the thread affect only the working directory for that thread. + +Buildman automatically selects the correct tool chain for each board. You +must supply suitable tool chains (see --fetch-arch), but buildman takes care +of selecting the right one. + +Buildman generally builds a branch (with the -b flag), and in this case +builds the upstream commit as well, for comparison. So even if you have one +commit in your branch, two commits will be built. Put all your commits in a +branch, set the branch's upstream to a valid value, and all will be well. +Otherwise buildman will perform random actions. Use -n to check what the +random actions might be. + +Buildman effectively has two modes: without -s it builds, with -s it +summarises the results of previous (or active) builds. + +If you just want to build the current source tree, leave off the -b flag. +This will display results and errors as they happen. You can still look at +them later using -se. Note that buildman will assume that the source has +changed, and will build all specified boards in this case. + +Buildman is optimised for building many commits at once, for many boards. +On multi-core machines, Buildman is fast because it uses most of the +available CPU power. When it gets to the end, or if you are building just +a few commits or boards, it will be pretty slow. As a tip, if you don't +plan to use your machine for anything else, you can use -T to increase the +number of threads beyond the default. + + +Selecting which boards to build +------------------------------- + +Buildman lets you build all boards, or a subset. Specify the subset by passing +command-line arguments that list the desired build target, architecture, +CPU, board name, vendor, SoC or options. Multiple arguments are allowed. Each +argument will be interpreted as a regular expression, so behaviour is a superset +of exact or substring matching. Examples are: + +- 'tegra20' - all boards with a Tegra20 SoC +- 'tegra' - all boards with any Tegra Soc (Tegra20, Tegra30, Tegra114...) +- '^tegra[23]0$' - all boards with either Tegra20 or Tegra30 SoC +- 'powerpc' - all PowerPC boards + +While the default is to OR the terms together, you can also make use of +the '&' operator to limit the selection: + +- 'freescale & arm sandbox' - all Freescale boards with ARM architecture, plus + sandbox + +You can also use -x to specifically exclude some boards. For example: + + buildman arm -x nvidia,freescale,.*ball$ + +means to build all arm boards except nvidia, freescale and anything ending +with 'ball'. + +For building specific boards you can use the --boards (or --bo) option, which +takes a comma-separated list of board target names and be used multiple times +on the command line: + +.. code-block:: bash + + buildman --boards sandbox,snow --boards + +It is convenient to use the -n option to see what will be built based on +the subset given. Use -v as well to get an actual list of boards. + +Buildman does not store intermediate object files. It optionally copies +the binary output into a directory when a build is successful (-k). Size +information is always recorded. It needs a fair bit of disk space to work, +typically 250MB per thread. + + +Setting up +---------- + +#. Get the U-Boot source. You probably already have it, but if not these + steps should get you started with a repo and some commits for testing. + + .. code-block:: bash + + cd /path/to/u-boot + git clone git://git.denx.de/u-boot.git . + git checkout -b my-branch origin/master + # Add some commits to the branch, reading for testing + +#. Create ~/.buildman to tell buildman where to find tool chains (see + buildman_settings_ for details). As an example:: + + # Buildman settings file + + [toolchain] + root: / + rest: /toolchains/* + eldk: /opt/eldk-4.2 + arm: /opt/linaro/gcc-linaro-arm-linux-gnueabihf-4.8-2013.08_linux + aarch64: /opt/linaro/gcc-linaro-aarch64-none-elf-4.8-2013.10_linux + + [toolchain-prefix] + arc = /opt/arc/arc_gnu_2021.03_prebuilt_elf32_le_linux_install/bin/arc-elf32- + + [toolchain-alias] + riscv = riscv32 + sh = sh4 + x86: i386 + + + This selects the available toolchain paths. Add the base directory for + each of your toolchains here. Buildman will search inside these directories + and also in any '/usr' and '/usr/bin' subdirectories. + + Make sure the tags (here root: rest: and eldk:) are unique. + + The toolchain-alias section indicates that the i386 toolchain should be used + to build x86 commits. + + Note that you can also specific exactly toolchain prefixes if you like:: + + [toolchain-prefix] + arm: /opt/arm-eabi-4.6/bin/arm-eabi- + + or even:: + + [toolchain-prefix] + arm: /opt/arm-eabi-4.6/bin/arm-eabi-gcc + + This tells buildman that you want to use this exact toolchain for the arm + architecture. This will override any toolchains found by searching using the + [toolchain] settings. + + Since the toolchain prefix is an explicit request, buildman will report an + error if a toolchain is not found with that prefix. The current PATH will be + searched, so it is possible to use:: + + [toolchain-prefix] + arm: arm-none-eabi- + + and buildman will find arm-none-eabi-gcc in /usr/bin if you have it + installed. + + Another example:: + + [toolchain-wrapper] + wrapper: ccache + + This tells buildman to use a compiler wrapper in front of CROSS_COMPILE. In + this example, ccache. It doesn't affect the toolchain scan. The wrapper is + added when CROSS_COMPILE environtal variable is set. The name in this + section is ignored. If more than one line is provided, only the last one + is taken. + +#. Make sure you have the require Python pre-requisites + + Buildman uses multiprocessing, Queue, shutil, StringIO, ConfigParser and + urllib2. These should normally be available, but if you get an error like + this then you will need to obtain those modules:: + + ImportError: No module named multiprocessing + + +#. Check the available toolchains + + Run this check to make sure that you have a toolchain for every architecture:: + + $ ./tools/buildman/buildman --list-tool-chains + Scanning for tool chains + - scanning prefix '/opt/gcc-4.6.3-nolibc/x86_64-linux/bin/x86_64-linux-' + Tool chain test: OK, arch='x86', priority 1 + - scanning prefix '/opt/arm-eabi-4.6/bin/arm-eabi-' + Tool chain test: OK, arch='arm', priority 1 + - scanning path '/toolchains/gcc-4.9.0-nolibc/i386-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/i386-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/i386-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/i386-linux/bin/i386-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/i386-linux/usr/bin' + Tool chain test: OK, arch='i386', priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/aarch64-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/aarch64-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/aarch64-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/aarch64-linux/bin/aarch64-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/aarch64-linux/usr/bin' + Tool chain test: OK, arch='aarch64', priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/microblaze-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/microblaze-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/microblaze-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/microblaze-linux/bin/microblaze-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/microblaze-linux/usr/bin' + Tool chain test: OK, arch='microblaze', priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/mips64-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/mips64-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/mips64-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/mips64-linux/bin/mips64-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/mips64-linux/usr/bin' + Tool chain test: OK, arch='mips64', priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/sparc64-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/sparc64-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/sparc64-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/sparc64-linux/bin/sparc64-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/sparc64-linux/usr/bin' + Tool chain test: OK, arch='sparc64', priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi' + - looking in '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi/bin' + - found '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi/bin/arm-unknown-linux-gnueabi-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi/usr/bin' + Tool chain test: OK, arch='arm', priority 3 + Toolchain '/toolchains/gcc-4.9.0-nolibc/arm-unknown-linux-gnueabi/bin/arm-unknown-linux-gnueabi-gcc' at priority 3 will be ignored because another toolchain for arch 'arm' has priority 1 + - scanning path '/toolchains/gcc-4.9.0-nolibc/sparc-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/sparc-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/sparc-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/sparc-linux/bin/sparc-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/sparc-linux/usr/bin' + Tool chain test: OK, arch='sparc', priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/mips-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/mips-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/mips-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/mips-linux/bin/mips-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/mips-linux/usr/bin' + Tool chain test: OK, arch='mips', priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/x86_64-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc' + - found '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/bin/x86_64-linux-x86_64-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/usr/bin' + Tool chain test: OK, arch='x86_64', priority 4 + Tool chain test: OK, arch='x86_64', priority 4 + Toolchain '/toolchains/gcc-4.9.0-nolibc/x86_64-linux/bin/x86_64-linux-x86_64-linux-gcc' at priority 4 will be ignored because another toolchain for arch 'x86_64' has priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/m68k-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/m68k-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/m68k-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/m68k-linux/bin/m68k-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/m68k-linux/usr/bin' + Tool chain test: OK, arch='m68k', priority 4 + - scanning path '/toolchains/gcc-4.9.0-nolibc/powerpc-linux' + - looking in '/toolchains/gcc-4.9.0-nolibc/powerpc-linux/.' + - looking in '/toolchains/gcc-4.9.0-nolibc/powerpc-linux/bin' + - found '/toolchains/gcc-4.9.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc' + - looking in '/toolchains/gcc-4.9.0-nolibc/powerpc-linux/usr/bin' + Tool chain test: OK, arch='powerpc', priority 4 + - scanning path '/toolchains/gcc-4.6.3-nolibc/bfin-uclinux' + - looking in '/toolchains/gcc-4.6.3-nolibc/bfin-uclinux/.' + - looking in '/toolchains/gcc-4.6.3-nolibc/bfin-uclinux/bin' + - found '/toolchains/gcc-4.6.3-nolibc/bfin-uclinux/bin/bfin-uclinux-gcc' + - looking in '/toolchains/gcc-4.6.3-nolibc/bfin-uclinux/usr/bin' + Tool chain test: OK, arch='bfin', priority 6 + - scanning path '/toolchains/gcc-4.6.3-nolibc/sparc-linux' + - looking in '/toolchains/gcc-4.6.3-nolibc/sparc-linux/.' + - looking in '/toolchains/gcc-4.6.3-nolibc/sparc-linux/bin' + - found '/toolchains/gcc-4.6.3-nolibc/sparc-linux/bin/sparc-linux-gcc' + - looking in '/toolchains/gcc-4.6.3-nolibc/sparc-linux/usr/bin' + Tool chain test: OK, arch='sparc', priority 4 + Toolchain '/toolchains/gcc-4.6.3-nolibc/sparc-linux/bin/sparc-linux-gcc' at priority 4 will be ignored because another toolchain for arch 'sparc' has priority 4 + - scanning path '/toolchains/gcc-4.6.3-nolibc/mips-linux' + - looking in '/toolchains/gcc-4.6.3-nolibc/mips-linux/.' + - looking in '/toolchains/gcc-4.6.3-nolibc/mips-linux/bin' + - found '/toolchains/gcc-4.6.3-nolibc/mips-linux/bin/mips-linux-gcc' + - looking in '/toolchains/gcc-4.6.3-nolibc/mips-linux/usr/bin' + Tool chain test: OK, arch='mips', priority 4 + Toolchain '/toolchains/gcc-4.6.3-nolibc/mips-linux/bin/mips-linux-gcc' at priority 4 will be ignored because another toolchain for arch 'mips' has priority 4 + - scanning path '/toolchains/gcc-4.6.3-nolibc/m68k-linux' + - looking in '/toolchains/gcc-4.6.3-nolibc/m68k-linux/.' + - looking in '/toolchains/gcc-4.6.3-nolibc/m68k-linux/bin' + - found '/toolchains/gcc-4.6.3-nolibc/m68k-linux/bin/m68k-linux-gcc' + - looking in '/toolchains/gcc-4.6.3-nolibc/m68k-linux/usr/bin' + Tool chain test: OK, arch='m68k', priority 4 + Toolchain '/toolchains/gcc-4.6.3-nolibc/m68k-linux/bin/m68k-linux-gcc' at priority 4 will be ignored because another toolchain for arch 'm68k' has priority 4 + - scanning path '/toolchains/gcc-4.6.3-nolibc/powerpc-linux' + - looking in '/toolchains/gcc-4.6.3-nolibc/powerpc-linux/.' + - looking in '/toolchains/gcc-4.6.3-nolibc/powerpc-linux/bin' + - found '/toolchains/gcc-4.6.3-nolibc/powerpc-linux/bin/powerpc-linux-gcc' + - looking in '/toolchains/gcc-4.6.3-nolibc/powerpc-linux/usr/bin' + Tool chain test: OK, arch='powerpc', priority 4 + Tool chain test: OK, arch='or32', priority 4 + - scanning path '/' + - looking in '/.' + - looking in '/bin' + - looking in '/usr/bin' + - found '/usr/bin/i586-mingw32msvc-gcc' + - found '/usr/bin/c89-gcc' + - found '/usr/bin/x86_64-linux-gnu-gcc' + - found '/usr/bin/gcc' + - found '/usr/bin/c99-gcc' + - found '/usr/bin/arm-linux-gnueabi-gcc' + - found '/usr/bin/aarch64-linux-gnu-gcc' + - found '/usr/bin/winegcc' + - found '/usr/bin/arm-linux-gnueabihf-gcc' + Tool chain test: OK, arch='i586', priority 11 + Tool chain test: OK, arch='c89', priority 11 + Tool chain test: OK, arch='x86_64', priority 4 + Toolchain '/usr/bin/x86_64-linux-gnu-gcc' at priority 4 will be ignored because another toolchain for arch 'x86_64' has priority 4 + Tool chain test: OK, arch='sandbox', priority 11 + Tool chain test: OK, arch='c99', priority 11 + Tool chain test: OK, arch='arm', priority 4 + Toolchain '/usr/bin/arm-linux-gnueabi-gcc' at priority 4 will be ignored because another toolchain for arch 'arm' has priority 1 + Tool chain test: OK, arch='aarch64', priority 4 + Toolchain '/usr/bin/aarch64-linux-gnu-gcc' at priority 4 will be ignored because another toolchain for arch 'aarch64' has priority 4 + Tool chain test: OK, arch='sandbox', priority 11 + Toolchain '/usr/bin/winegcc' at priority 11 will be ignored because another toolchain for arch 'sandbox' has priority 11 + Tool chain test: OK, arch='arm', priority 4 + Toolchain '/usr/bin/arm-linux-gnueabihf-gcc' at priority 4 will be ignored because another toolchain for arch 'arm' has priority 1 + List of available toolchains (34): + aarch64 : /toolchains/gcc-4.9.0-nolibc/aarch64-linux/bin/aarch64-linux-gcc + alpha : /toolchains/gcc-4.9.0-nolibc/alpha-linux/bin/alpha-linux-gcc + am33_2.0 : /toolchains/gcc-4.9.0-nolibc/am33_2.0-linux/bin/am33_2.0-linux-gcc + arm : /opt/arm-eabi-4.6/bin/arm-eabi-gcc + bfin : /toolchains/gcc-4.6.3-nolibc/bfin-uclinux/bin/bfin-uclinux-gcc + c89 : /usr/bin/c89-gcc + c99 : /usr/bin/c99-gcc + frv : /toolchains/gcc-4.9.0-nolibc/frv-linux/bin/frv-linux-gcc + h8300 : /toolchains/gcc-4.9.0-nolibc/h8300-elf/bin/h8300-elf-gcc + hppa : /toolchains/gcc-4.9.0-nolibc/hppa-linux/bin/hppa-linux-gcc + hppa64 : /toolchains/gcc-4.9.0-nolibc/hppa64-linux/bin/hppa64-linux-gcc + i386 : /toolchains/gcc-4.9.0-nolibc/i386-linux/bin/i386-linux-gcc + i586 : /usr/bin/i586-mingw32msvc-gcc + ia64 : /toolchains/gcc-4.9.0-nolibc/ia64-linux/bin/ia64-linux-gcc + m32r : /toolchains/gcc-4.9.0-nolibc/m32r-linux/bin/m32r-linux-gcc + m68k : /toolchains/gcc-4.9.0-nolibc/m68k-linux/bin/m68k-linux-gcc + microblaze: /toolchains/gcc-4.9.0-nolibc/microblaze-linux/bin/microblaze-linux-gcc + mips : /toolchains/gcc-4.9.0-nolibc/mips-linux/bin/mips-linux-gcc + mips64 : /toolchains/gcc-4.9.0-nolibc/mips64-linux/bin/mips64-linux-gcc + or32 : /toolchains/gcc-4.5.1-nolibc/or32-linux/bin/or32-linux-gcc + powerpc : /toolchains/gcc-4.9.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc + powerpc64 : /toolchains/gcc-4.9.0-nolibc/powerpc64-linux/bin/powerpc64-linux-gcc + ppc64le : /toolchains/gcc-4.9.0-nolibc/ppc64le-linux/bin/ppc64le-linux-gcc + s390x : /toolchains/gcc-4.9.0-nolibc/s390x-linux/bin/s390x-linux-gcc + sandbox : /usr/bin/gcc + sh4 : /toolchains/gcc-4.6.3-nolibc/sh4-linux/bin/sh4-linux-gcc + sparc : /toolchains/gcc-4.9.0-nolibc/sparc-linux/bin/sparc-linux-gcc + sparc64 : /toolchains/gcc-4.9.0-nolibc/sparc64-linux/bin/sparc64-linux-gcc + tilegx : /toolchains/gcc-4.6.2-nolibc/tilegx-linux/bin/tilegx-linux-gcc + x86 : /opt/gcc-4.6.3-nolibc/x86_64-linux/bin/x86_64-linux-gcc + x86_64 : /toolchains/gcc-4.9.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc + + + You can see that everything is covered, even some strange ones that won't + be used (c88 and c99). This is a feature. + + +#. Install new toolchains if needed + + You can download toolchains and update the [toolchain] section of the + settings file to find them. + + To make this easier, buildman can automatically download and install + toolchains from kernel.org. First list the available architectures:: + + $ ./tools/buildman/buildman --fetch-arch list + Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.3/ + Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.2/ + Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.5.1/ + Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.2.4/ + Available architectures: alpha am33_2.0 arm bfin cris crisv32 frv h8300 + hppa hppa64 i386 ia64 m32r m68k mips mips64 or32 powerpc powerpc64 s390x sh4 + sparc sparc64 tilegx x86_64 xtensa + + Then pick one and download it:: + + $ ./tools/buildman/buildman --fetch-arch or32 + Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.3/ + Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.2/ + Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.5.1/ + Downloading: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.5.1//x86_64-gcc-4.5.1-nolibc_or32-linux.tar.xz + Unpacking to: /home/sjg/.buildman-toolchains + Testing + - looking in '/home/sjg/.buildman-toolchains/gcc-4.5.1-nolibc/or32-linux/.' + - looking in '/home/sjg/.buildman-toolchains/gcc-4.5.1-nolibc/or32-linux/bin' + - found '/home/sjg/.buildman-toolchains/gcc-4.5.1-nolibc/or32-linux/bin/or32-linux-gcc' + Tool chain test: OK + + Or download them all from kernel.org and move them to /toolchains directory: + + .. code-block:: bash + + ./tools/buildman/buildman --fetch-arch all + sudo mkdir -p /toolchains + sudo mv ~/.buildman-toolchains/*/* /toolchains/ + + For those not available from kernel.org, download from the following links: + + - `Arc Toolchain`_ + + Buildman should now be set up to use your new toolchain. + + At the time of writing, U-Boot has these architectures: + + arc, arm, m68k, microblaze, mips, nios2, powerpc, sandbox, sh, x86, xtensa + + +How to run it +------------- + +First do a dry run using the -n flag: (replace <branch> with a real, local +branch with a valid upstream): + +.. code-block:: bash + + ./tools/buildman/buildman -b <branch> -n + +If it can't detect the upstream branch, try checking out the branch, and +doing something like 'git branch --set-upstream-to upstream/master' +or something similar. Buildman will try to guess a suitable upstream branch +if it can't find one (you will see a message like "Guessing upstream as ..."). +You can also use the -c option to manually specify the number of commits to +build. + +As an example:: + + Dry run, so not doing much. But I would do this: + + Building 18 commits for 1059 boards (4 threads, 1 job per thread) + Build directory: ../lcd9b + 5bb3505 Merge branch 'master' of git://git.denx.de/u-boot-arm + c18f1b4 tegra: Use const for pinmux_config_pingroup/table() + 2f043ae tegra: Add display support to funcmux + e349900 tegra: fdt: Add pwm binding and node + 424a5f0 tegra: fdt: Add LCD definitions for Tegra + 0636ccf tegra: Add support for PWM + a994fe7 tegra: Add SOC support for display/lcd + fcd7350 tegra: Add LCD driver + 4d46e9d tegra: Add LCD support to Nvidia boards + 991bd48 arm: Add control over cachability of memory regions + 54e8019 lcd: Add CONFIG_LCD_ALIGNMENT to select frame buffer alignment + d92aff7 lcd: Add support for flushing LCD fb from dcache after update + dbd0677 tegra: Align LCD frame buffer to section boundary + 0cff9b8 tegra: Support control of cache settings for LCD + 9c56900 tegra: fdt: Add LCD definitions for Seaboard + 5cc29db lcd: Add CONFIG_CONSOLE_SCROLL_LINES option to speed console + cac5a23 tegra: Enable display/lcd support on Seaboard + 49ff541 wip + + Total boards to build for each commit: 1059 + +This shows that it will build all 1059 boards, using 4 threads (because +we have a 4-core CPU). Each thread will run with -j1, meaning that each +make job will use a single CPU. The list of commits to be built helps you +confirm that things look about right. Notice that buildman has chosen a +'base' directory for you, immediately above your source tree. + +Buildman works entirely inside the base directory, here ../lcd9b, +creating a working directory for each thread, and creating output +directories for each commit and board. + + +Suggested Workflow +------------------ + +To run the build for real, take off the -n: + +.. code-block:: bash + + ./tools/buildman/buildman -b <branch> + +Buildman will set up some working directories, and get started. After a +minute or so it will settle down to a steady pace, with a display like this:: + + Building 18 commits for 1059 boards (4 threads, 1 job per thread) + 528 36 124 /19062 -18374 1:13:30 : SIMPC8313_SP + +This means that it is building 19062 board/commit combinations. So far it +has managed to successfully build 528. Another 36 have built with warnings, +and 124 more didn't build at all. It has 18374 builds left to complete. +Buildman expects to complete the process in around an hour and a quarter. +Use this time to buy a faster computer. + + +To find out how the build went, ask for a summary with -s. You can do this +either before the build completes (presumably in another terminal) or +afterwards. Let's work through an example of how this is used:: + + $ ./tools/buildman/buildman -b lcd9b -s + ... + 01: Merge branch 'master' of git://git.denx.de/u-boot-arm + powerpc: + galaxy5200_LOWBOOT + 02: tegra: Use const for pinmux_config_pingroup/table() + 03: tegra: Add display support to funcmux + 04: tegra: fdt: Add pwm binding and node + 05: tegra: fdt: Add LCD definitions for Tegra + 06: tegra: Add support for PWM + 07: tegra: Add SOC support for display/lcd + 08: tegra: Add LCD driver + 09: tegra: Add LCD support to Nvidia boards + 10: arm: Add control over cachability of memory regions + 11: lcd: Add CONFIG_LCD_ALIGNMENT to select frame buffer alignment + 12: lcd: Add support for flushing LCD fb from dcache after update + arm: + lubbock + 13: tegra: Align LCD frame buffer to section boundary + 14: tegra: Support control of cache settings for LCD + 15: tegra: fdt: Add LCD definitions for Seaboard + 16: lcd: Add CONFIG_CONSOLE_SCROLL_LINES option to speed console + 17: tegra: Enable display/lcd support on Seaboard + 18: wip + +This shows which commits have succeeded and which have failed. In this case +the build is still in progress so many boards are not built yet (use -u to +see which ones). But already we can see a few failures. The galaxy5200_LOWBOOT +never builds correctly. This could be a problem with our toolchain, or it +could be a bug in the upstream. The good news is that we probably don't need +to blame our commits. The bad news is that our commits are not tested on that +board. + +Commit 12 broke lubbock. That's what the '+ lubbock', in red, means. The +failure is never fixed by a later commit, or you would see lubbock again, in +green, without the +. + +To see the actual error:: + + $ ./tools/buildman/buildman -b <branch> -se + ... + 12: lcd: Add support for flushing LCD fb from dcache after update + arm: + lubbock + +common/libcommon.o: In function `lcd_sync': + +common/lcd.c:120: undefined reference to `flush_dcache_range' + +arm-none-linux-gnueabi-ld: BFD (Sourcery G++ Lite 2010q1-202) 2.19.51.20090709 assertion fail /scratch/julian/2010q1-release-linux-lite/obj/binutils-src-2010q1-202-arm-none-linux-gnueabi-i686-pc-linux-gnu/bfd/elf32-arm.c:12572 + +make: *** [build/u-boot] Error 139 + 13: tegra: Align LCD frame buffer to section boundary + 14: tegra: Support control of cache settings for LCD + 15: tegra: fdt: Add LCD definitions for Seaboard + 16: lcd: Add CONFIG_CONSOLE_SCROLL_LINES option to speed console + -common/lcd.c:120: undefined reference to `flush_dcache_range' + +common/lcd.c:125: undefined reference to `flush_dcache_range' + 17: tegra: Enable display/lcd support on Seaboard + 18: wip + +So the problem is in lcd.c, due to missing cache operations. This information +should be enough to work out what that commit is doing to break these +boards. (In this case pxa did not have cache operations defined). + +Note that if there were other boards with errors, the above command would +show their errors also. Each line is shown only once. So if lubbock and snow +produce the same error, we just see:: + + 12: lcd: Add support for flushing LCD fb from dcache after update + arm: + lubbock snow + +common/libcommon.o: In function `lcd_sync': + +common/lcd.c:120: undefined reference to `flush_dcache_range' + +arm-none-linux-gnueabi-ld: BFD (Sourcery G++ Lite 2010q1-202) 2.19.51.20090709 assertion fail /scratch/julian/2010q1-release-linux-lite/obj/binutils-src-2010q1-202-arm-none-linux-gnueabi-i686-pc-linux-gnu/bfd/elf32-arm.c:12572 + +make: *** [build/u-boot] Error 139 + +But if you did want to see just the errors for lubbock, use: + +.. code-block:: bash + + ./tools/buildman/buildman -b <branch> -se lubbock + +If you see error lines marked with '-', that means that the errors were fixed +by that commit. Sometimes commits can be in the wrong order, so that a +breakage is introduced for a few commits and fixed by later commits. This +shows up clearly with buildman. You can then reorder the commits and try +again. + +At commit 16, the error moves: you can see that the old error at line 120 +is fixed, but there is a new one at line 126. This is probably only because +we added some code and moved the broken line further down the file. + +As mentioned, if many boards have the same error, then -e will display the +error only once. This makes the output as concise as possible. To see which +boards have each error, use -l. So it is safe to omit the board name - you +will not get lots of repeated output for every board. + +Buildman tries to distinguish warnings from errors, and shows warning lines +separately with a 'w' prefix. Warnings introduced show as yellow. Warnings +fixed show as cyan. + +The full build output in this case is available in:: + + ../lcd9b/12_of_18_gd92aff7_lcd--Add-support-for/lubbock/ + +Files: + +done + Indicates the build was done, and holds the return code from make. This is 0 + for a good build, typically 2 for a failure. + +err + Output from stderr, if any. Errors and warnings appear here. + +log + Output from stdout. Normally there isn't any since buildman runs in silent + mode. Use -V to force a verbose build (this passes V=1 to 'make') + +toolchain + Shows information about the toolchain used for the build. + +sizes + Shows image size information. + +It is possible to get the build binary output there also. Use the -k option +for this. In that case you will also see some output files, like: + +- System.map +- toolchain +- u-boot +- u-boot.bin +- u-boot.map +- autoconf.mk +- SPL/TPL versions like u-boot-spl and u-boot-spl.bin if available + + +Checking Image Sizes +-------------------- + +A key requirement for U-Boot is that you keep code/data size to a minimum. +Where a new feature increases this noticeably it should normally be put +behind a CONFIG flag so that boards can leave it disabled and keep the image +size more or less the same with each new release. + +To check the impact of your commits on image size, use -S. For example:: + + $ ./tools/buildman/buildman -b us-x86 -sS + Summary of 10 commits for 1066 boards (4 threads, 1 job per thread) + 01: MAKEALL: add support for per architecture toolchains + 02: x86: Add function to get top of usable ram + x86: (for 1/3 boards) text -272.0 rodata +41.0 + 03: x86: Add basic cache operations + 04: x86: Permit bootstage and timer data to be used prior to relocation + x86: (for 1/3 boards) data +16.0 + 05: x86: Add an __end symbol to signal the end of the U-Boot binary + x86: (for 1/3 boards) text +76.0 + 06: x86: Rearrange the output input to remove BSS + x86: (for 1/3 boards) bss -2140.0 + 07: x86: Support relocation of FDT on start-up + x86: + coreboot-x86 + 08: x86: Add error checking to x86 relocation code + 09: x86: Adjust link device tree include file + 10: x86: Enable CONFIG_OF_CONTROL on coreboot + + +You can see that image size only changed on x86, which is good because this +series is not supposed to change any other board. From commit 7 onwards the +build fails so we don't get code size numbers. The numbers are fractional +because they are an average of all boards for that architecture. The +intention is to allow you to quickly find image size problems introduced by +your commits. + +Note that the 'text' region and 'rodata' are split out. You should add the +two together to get the total read-only size (reported as the first column +in the output from binutil's 'size' utility). + +A useful option is --step which lets you skip some commits. For example +--step 2 will show the image sizes for only every 2nd commit (so it will +compare the image sizes of the 1st, 3rd, 5th... commits). You can also use +--step 0 which will compare only the first and last commits. This is useful +for an overview of how your entire series affects code size. It will build +only the upstream commit and your final branch commit. + +You can also use -d to see a detailed size breakdown for each board. This +list is sorted in order from largest growth to largest reduction. + +It is even possible to go a little further with the -B option (--bloat). This +shows where U-Boot has bloated, breaking the size change down to the function +level. Example output is below:: + + $ ./tools/buildman/buildman -b us-mem4 -sSdB + ... + 19: Roll crc32 into hash infrastructure + arm: (for 10/10 boards) all -143.4 bss +1.2 data -4.8 rodata -48.2 text -91.6 + paz00 : all +23 bss -4 rodata -29 text +56 + u-boot: add: 1/0, grow: 3/-2 bytes: 168/-104 (64) + function old new delta + hash_command 80 160 +80 + crc32_wd_buf - 56 +56 + ext4fs_read_file 540 568 +28 + insert_var_value_sub 688 692 +4 + run_list_real 1996 1992 -4 + do_mem_crc 168 68 -100 + trimslice : all -9 bss +16 rodata -29 text +4 + u-boot: add: 1/0, grow: 1/-3 bytes: 136/-124 (12) + function old new delta + hash_command 80 160 +80 + crc32_wd_buf - 56 +56 + ext4fs_iterate_dir 672 668 -4 + ext4fs_read_file 568 548 -20 + do_mem_crc 168 68 -100 + whistler : all -9 bss +16 rodata -29 text +4 + u-boot: add: 1/0, grow: 1/-3 bytes: 136/-124 (12) + function old new delta + hash_command 80 160 +80 + crc32_wd_buf - 56 +56 + ext4fs_iterate_dir 672 668 -4 + ext4fs_read_file 568 548 -20 + do_mem_crc 168 68 -100 + seaboard : all -9 bss -28 rodata -29 text +48 + u-boot: add: 1/0, grow: 3/-2 bytes: 160/-104 (56) + function old new delta + hash_command 80 160 +80 + crc32_wd_buf - 56 +56 + ext4fs_read_file 548 568 +20 + run_list_real 1996 2000 +4 + do_nandboot 760 756 -4 + do_mem_crc 168 68 -100 + colibri_t20 : all -9 rodata -29 text +20 + u-boot: add: 1/0, grow: 2/-3 bytes: 140/-112 (28) + function old new delta + hash_command 80 160 +80 + crc32_wd_buf - 56 +56 + read_abs_bbt 204 208 +4 + do_nandboot 760 756 -4 + ext4fs_read_file 576 568 -8 + do_mem_crc 168 68 -100 + ventana : all -37 bss -12 rodata -29 text +4 + u-boot: add: 1/0, grow: 1/-3 bytes: 136/-124 (12) + function old new delta + hash_command 80 160 +80 + crc32_wd_buf - 56 +56 + ext4fs_iterate_dir 672 668 -4 + ext4fs_read_file 568 548 -20 + do_mem_crc 168 68 -100 + harmony : all -37 bss -16 rodata -29 text +8 + u-boot: add: 1/0, grow: 2/-3 bytes: 140/-124 (16) + function old new delta + hash_command 80 160 +80 + crc32_wd_buf - 56 +56 + nand_write_oob_syndrome 428 432 +4 + ext4fs_iterate_dir 672 668 -4 + ext4fs_read_file 568 548 -20 + do_mem_crc 168 68 -100 + medcom-wide : all -417 bss +28 data -16 rodata -93 text -336 + u-boot: add: 1/-1, grow: 1/-2 bytes: 88/-376 (-288) + function old new delta + crc32_wd_buf - 56 +56 + do_fat_read_at 2872 2904 +32 + hash_algo 16 - -16 + do_mem_crc 168 68 -100 + hash_command 420 160 -260 + tec : all -449 bss -4 data -16 rodata -93 text -336 + u-boot: add: 1/-1, grow: 1/-2 bytes: 88/-376 (-288) + function old new delta + crc32_wd_buf - 56 +56 + do_fat_read_at 2872 2904 +32 + hash_algo 16 - -16 + do_mem_crc 168 68 -100 + hash_command 420 160 -260 + plutux : all -481 bss +16 data -16 rodata -93 text -388 + u-boot: add: 1/-1, grow: 1/-3 bytes: 68/-408 (-340) + function old new delta + crc32_wd_buf - 56 +56 + do_load_serial_bin 1688 1700 +12 + hash_algo 16 - -16 + do_fat_read_at 2904 2872 -32 + do_mem_crc 168 68 -100 + hash_command 420 160 -260 + powerpc: (for 5/5 boards) all +37.4 data -3.2 rodata -41.8 text +82.4 + MPC8610HPCD : all +55 rodata -29 text +84 + u-boot: add: 1/0, grow: 0/-1 bytes: 176/-96 (80) + function old new delta + hash_command - 176 +176 + do_mem_crc 184 88 -96 + MPC8641HPCN : all +55 rodata -29 text +84 + u-boot: add: 1/0, grow: 0/-1 bytes: 176/-96 (80) + function old new delta + hash_command - 176 +176 + do_mem_crc 184 88 -96 + MPC8641HPCN_36BIT: all +55 rodata -29 text +84 + u-boot: add: 1/0, grow: 0/-1 bytes: 176/-96 (80) + function old new delta + hash_command - 176 +176 + do_mem_crc 184 88 -96 + sbc8641d : all +55 rodata -29 text +84 + u-boot: add: 1/0, grow: 0/-1 bytes: 176/-96 (80) + function old new delta + hash_command - 176 +176 + do_mem_crc 184 88 -96 + xpedite517x : all -33 data -16 rodata -93 text +76 + u-boot: add: 1/-1, grow: 0/-1 bytes: 176/-112 (64) + function old new delta + hash_command - 176 +176 + hash_algo 16 - -16 + do_mem_crc 184 88 -96 + ... + + +This shows that commit 19 has reduced codesize for arm slightly and increased +it for powerpc. This increase was offset in by reductions in rodata and +data/bss. + +Shown below the summary lines are the sizes for each board. Below each board +are the sizes for each function. This information starts with: + +add + number of functions added / removed + +grow + number of functions which grew / shrunk + +bytes + number of bytes of code added to / removed from all functions, plus the total + byte change in brackets + +The change seems to be that hash_command() has increased by more than the +do_mem_crc() function has decreased. The function sizes typically add up to +roughly the text area size, but note that every read-only section except +rodata is included in 'text', so the function total does not exactly +correspond. + +It is common when refactoring code for the rodata to decrease as the text size +increases, and vice versa. + + +.. _buildman_settings: + +The .buildman settings file +--------------------------- + +The .buildman file provides information about the available toolchains and +also allows build flags to be passed to 'make'. It consists of several +sections, with the section name in square brackets. Within each section are +a set of (tag, value) pairs. + +'[global]' section + allow-missing + Indicates the policy to use for missing blobs. Note that the flags + ``--allow-missing`` (``-M``) and ``--no-allow-missing`` (``--no-a``) + override these setting. + + always + Run with ``-M`` by default. + + multiple + Run with ``-M`` if more than one board is being built. + + branch + Run with ``-M`` if a branch is being built. + + Note that the last two can be given together:: + + allow-missing = multiple branch + +'[toolchain]' section + This lists the available toolchains. The tag here doesn't matter, but + make sure it is unique. The value is the path to the toolchain. Buildman + will look in that path for a file ending in 'gcc'. It will then execute + it to check that it is a C compiler, passing only the --version flag to + it. If the return code is 0, buildman assumes that it is a valid C + compiler. It uses the first part of the name as the architecture and + strips off the last part when setting the CROSS_COMPILE environment + variable (parts are delimited with a hyphen). + + For example powerpc-linux-gcc will be noted as a toolchain for 'powerpc' + and CROSS_COMPILE will be set to powerpc-linux- when using it. + +'[toolchain-alias]' section + This converts toolchain architecture names to U-Boot names. For example, + if an x86 toolchains is called i386-linux-gcc it will not normally be + used for architecture 'x86'. Adding 'x86: i386 x86_64' to this section + will tell buildman that the i386 and x86_64 toolchains can be used for + the x86 architecture. + +'[make-flags]' section + U-Boot's build system supports a few flags (such as BUILD_TAG) which + affect the build product. These flags can be specified in the buildman + settings file. They can also be useful when building U-Boot against other + open source software. + + [make-flags] + at91-boards=ENABLE_AT91_TEST=1 + snapper9260=${at91-boards} BUILD_TAG=442 + snapper9g45=${at91-boards} BUILD_TAG=443 + + This will use 'make ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 + and 'make ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. A special + variable ${target} is available to access the target name (snapper9260 + and snapper9g20 in this case). Variables are resolved recursively. Note + that variables can only contain the characters A-Z, a-z, 0-9, hyphen (-) + and underscore (_). + + It is expected that any variables added are dealt with in U-Boot's + config.mk file and documented in the README. + + Note that you can pass ad-hoc options to the build using environment + variables, for example: + + SOME_OPTION=1234 ./tools/buildman/buildman my_board + + +Quick Sanity Check +------------------ + +If you have made changes and want to do a quick sanity check of the +currently checked-out source, run buildman without the -b flag. This will +build the selected boards and display build status as it runs (i.e. -v is +enabled automatically). Use -e to see errors/warnings as well. + + +Building Ranges +--------------- + +You can build a range of commits by specifying a range instead of a branch +when using the -b flag. For example:: + + buildman -b upstream/master..us-buildman + +will build commits in us-buildman that are not in upstream/master. + + +Building Faster +--------------- + +By default, buildman doesn't execute 'make mrproper' prior to building the +first commit for each board. This reduces the amount of work 'make' does, and +hence speeds up the build. To force use of 'make mrproper', use -the -m flag. +This flag will slow down any buildman invocation, since it increases the amount +of work done on any build. + +One possible application of buildman is as part of a continual edit, build, +edit, build, ... cycle; repeatedly applying buildman to the same change or +series of changes while making small incremental modifications to the source +each time. This provides quick feedback regarding the correctness of recent +modifications. In this scenario, buildman's default choice of build directory +causes more build work to be performed than strictly necessary. + +By default, each buildman thread uses a single directory for all builds. When a +thread builds multiple boards, the configuration built in this directory will +cycle through various different configurations, one per board built by the +thread. Variations in the configuration will force a rebuild of affected source +files when a thread switches between boards. Ideally, such buildman-induced +rebuilds would not happen, thus allowing the build to operate as efficiently as +the build system and source changes allow. buildman's -P flag may be used to +enable this; -P causes each board to be built in a separate (board-specific) +directory, thus avoiding any buildman-induced configuration changes in any +build directory. + +U-Boot's build system embeds information such as a build timestamp into the +final binary. This information varies each time U-Boot is built. This causes +various files to be rebuilt even if no source changes are made, which in turn +requires that the final U-Boot binary be re-linked. This unnecessary work can +be avoided by turning off the timestamp feature. This can be achieved by +setting the SOURCE_DATE_EPOCH environment variable to 0. + +Combining all of these options together yields the command-line shown below. +This will provide the quickest possible feedback regarding the current content +of the source tree, thus allowing rapid tested evolution of the code:: + + SOURCE_DATE_EPOCH=0 ./tools/buildman/buildman -P tegra + + +Checking configuration +---------------------- + +A common requirement when converting CONFIG options to Kconfig is to check +that the effective configuration has not changed due to the conversion. +Buildman supports this with the -K option, used after a build. This shows +differences in effective configuration between one commit and the next. + +For example:: + + $ buildman -b kc4 -sK + ... + 43: Convert CONFIG_SPL_USBETH_SUPPORT to Kconfig + arm: + + u-boot.cfg: CONFIG_SPL_ENV_SUPPORT=1 CONFIG_SPL_NET=1 + + u-boot-spl.cfg: CONFIG_SPL_MMC=1 CONFIG_SPL_NAND_SUPPORT=1 + + all: CONFIG_SPL_ENV_SUPPORT=1 CONFIG_SPL_MMC=1 CONFIG_SPL_NAND_SUPPORT=1 CONFIG_SPL_NET=1 + am335x_evm_usbspl : + + u-boot.cfg: CONFIG_SPL_ENV_SUPPORT=1 CONFIG_SPL_NET=1 + + u-boot-spl.cfg: CONFIG_SPL_MMC=1 CONFIG_SPL_NAND_SUPPORT=1 + + all: CONFIG_SPL_ENV_SUPPORT=1 CONFIG_SPL_MMC=1 CONFIG_SPL_NAND_SUPPORT=1 CONFIG_SPL_NET=1 + 44: Convert CONFIG_SPL_USB_HOST to Kconfig + ... + +This shows that commit 44 enabled three new options for the board +am335x_evm_usbspl which were not enabled in commit 43. There is also a +summary for 'arm' showing all the changes detected for that architecture. +In this case there is only one board with changes, so 'arm' output is the +same as 'am335x_evm_usbspl'/ + +The -K option uses the u-boot.cfg, spl/u-boot-spl.cfg and tpl/u-boot-tpl.cfg +files which are produced by a build. If all you want is to check the +configuration you can in fact avoid doing a full build, using -D. This tells +buildman to configuration U-Boot and create the .cfg files, but not actually +build the source. This is 5-10 times faster than doing a full build. + +By default buildman considers the follow two configuration methods +equivalent:: + + #define CONFIG_SOME_OPTION + + CONFIG_SOME_OPTION=y + +The former would appear in a header filer and the latter in a defconfig +file. The achieve this, buildman considers 'y' to be '1' in configuration +variables. This avoids lots of useless output when converting a CONFIG +option to Kconfig. To disable this behaviour, use --squash-config-y. + + +Checking the environment +------------------------ + +When converting CONFIG options which manipulate the default environment, +a common requirement is to check that the default environment has not +changed due to the conversion. Buildman supports this with the -U option, +used after a build. This shows differences in the default environment +between one commit and the next. + +For example:: + + $ buildman -b squash brppt1 -sU + Summary of 2 commits for 3 boards (3 threads, 3 jobs per thread) + 01: Migrate bootlimit to Kconfig + 02: Squashed commit of the following: + c brppt1_mmc: altbootcmd=mmc dev 1; run mmcboot0; -> mmc dev 1; run mmcboot0 + c brppt1_spi: altbootcmd=mmc dev 1; run mmcboot0; -> mmc dev 1; run mmcboot0 + + brppt1_nand: altbootcmd=run usbscript + - brppt1_nand: altbootcmd=run usbscript + (no errors to report) + +This shows that commit 2 modified the value of 'altbootcmd' for 'brppt1_mmc' +and 'brppt1_spi', removing a trailing semicolon. 'brppt1_nand' gained an a +value for 'altbootcmd', but lost one for ' altbootcmd'. + +The -U option uses the u-boot.env files which are produced by a build. + + +Building with clang +------------------- + +To build with clang (sandbox only), use the -O option to override the +toolchain. For example: + +.. code-block:: bash + + buildman -O clang-7 --board sandbox + + +Doing a simple build +-------------------- + +In some cases you just want to build a single board and get the full output, use +the -w option, for example: + +.. code-block:: bash + + buildman -o /tmp/build --board sandbox -w + +This will write the full build into /tmp/build including object files. You must +specify the output directory with -o when using -w. + + +Support for IDEs (Integrated Development Environments) +------------------------------------------------------ + +Normally buildman summarises the output and shows information indicating the +meaning of each line of output. For example a '+' symbol appears at the start of +each error line. Also, buildman prints information about what it is about to do, +along with a summary at the end. + +When using buildman from an IDE, it is helpful to drop this behaviour. Use the +-I/--ide option for that. You might find -W helpful also so that warnings do +not cause the build to fail: + +.. code-block:: bash + + buildman -o /tmp/build --board sandbox -wWI + + +Support for binary blobs +------------------------ + +U-Boot is moving to using Binman (see :doc:`../develop/package/binman`) for +dealing with the complexities of packaging U-Boot along with binary files from +other projects. These are called 'external blobs' by Binman. + +Typically a missing external blob causes a build failure. For build testing of +a lot of boards, or boards for which you do not have the blobs, you can use the +-M flag to allow missing blobs. This marks the build as if it succeeded, +although with warnings shown, including 'Some images are invalid'. If any boards +fail in this way, buildman exits with status 101. + +To convert warnings to errors, use -E. To make buildman return success with +these warnings, use -W. + +It is generally safe to default to enabling -M for all runs of buildman, so long +as you check the exit code. To do this, add:: + + allow-missing = "always" + +to the top of the buildman_settings_ file. + + +Changing the configuration +-------------------------- + +Sometimes it is useful to change the CONFIG options for a build on the fly. This +can be used to build a board (or multiple) with a few changes to see the impact. +The -a option supports this: + +.. code-block:: bash + + -a <cfg> + +where <cfg> is a CONFIG option (with or without the `CONFIG_` prefix) to enable. +For example: + +.. code-block:: bash + + buildman -a CMD_SETEXPR_FMT + +will build with CONFIG_CMD_SETEXPR_FMT enabled. + +You can disable options by preceding them with tilde (~). You can specify the +-a option multiple times: + +.. code-block:: bash + + buildman -a CMD_SETEXPR_FMT -a ~CMDLINE + +Some options have values, in which case you can change them: + +.. code-block:: bash + + buildman -a 'BOOTCOMMAND="echo hello"' CONFIG_SYS_LOAD_ADDR=0x1000 + +Note that you must put quotes around string options and the whole thing must be +in single quotes, to make sure the shell leave it alone. + +If you try to set an option that does not exist, or that cannot be changed for +some other reason (e.g. it is 'selected' by another option), then buildman +shows an error:: + + $ buildman --board sandbox -a FRED + Building current source for 1 boards (1 thread, 32 jobs per thread) + 0 0 0 /1 -1 (starting)errs + Some CONFIG adjustments did not take effect. This may be because + the request CONFIGs do not exist or conflict with others. + + Failed adjustments: + + FRED Missing expected line: CONFIG_FRED=y + + +One major caveat with this feature with branches (-b) is that buildman does not +name the output directories differently when you change the configuration, so +doing the same build again with different configuration will not trigger a +rebuild. You can use -f to work around that. + + +Other options +------------- + +Buildman has various other command-line options. Try --help to see them. + +To find out what toolchain prefix buildman will use for a build, use the -A +option. + +To request that compiler warnings be promoted to errors, use -E. This passes the +-Werror flag to the compiler. Note that the build can still produce warnings +with -E, e.g. the migration warnings:: + + ===================== WARNING ====================== + This board does not use CONFIG_DM_MMC. Please update + ... + ==================================================== + +When doing builds, Buildman's return code will reflect the overall result:: + + 0 (success) No errors or warnings found + 100 Errors found + 101 Warnings found (only if no -W) + +You can use -W to tell Buildman to return 0 (success) instead of 101 when +warnings are found. Note that it can be useful to combine -E and -W. This means +that all compiler warnings will produce failures (code 100) and all other +warnings will produce success (since 101 is changed to 0). + +If there are both warnings and errors, errors win, so buildman returns 100. + +The -y option is provided (for use with -s) to ignore the bountiful device-tree +warnings. Similarly, -Y tells buildman to ignore the migration warnings. + +Sometimes you might get an error in a thread that is not handled by buildman, +perhaps due to a failure of a tool that it calls. You might see the output, but +then buildman hangs. Failing to handle any eventuality is a bug in buildman and +should be reported. But you can use -T0 to disable threading and hopefully +figure out the root cause of the build failure. + +Build summary +------------- + +When buildman finishes it shows a summary, something like this:: + + Completed: 5 total built, duration 0:00:21, rate 0.24 + +This shows that a total of 5 builds were done across all selected boards, it +took 21 seconds and the builds happened at the rate of 0.24 per second. The +latter number depends on the speed of your machine and the efficiency of the +U-Boot build. + + +Using boards.cfg +---------------- + +This file is no-longer needed by buildman but it is still generated in the +working directory. This helps avoid a delay on every build, since scanning all +the Kconfig files takes a few seconds. Use the -R flag to force regeneration +of the file - in that case buildman exits after writing the file. with exit code +2 if there was an error in the maintainer files. + +You should use 'buildman -nv <criteria>' instead of greoing the boards.cfg file, +since it may be dropped altogether in future. + + +TODO +---- + +Many improvements have been made over the years. There is still quite a bit of +scope for more though, e.g.: + +- easier access to log files +- 'hunting' for problems, perhaps by building a few boards for each arch, or + checking commits for changed files and building only boards which use those + files + + +Credits +------- + +Thanks to Grant Grundler <grundler@chromium.org> for his ideas for improving +the build speed by building all commits for a board instead of the other +way around. + +.. _`Arc Toolchain`: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases/download/arc-2021.03-release/arc_gnu_2021.03_prebuilt_elf32_le_linux_install.tar.gz + +.. sectionauthor:: Simon Glass +.. sectionauthor:: Copyright (c) 2013 The Chromium OS Authors. +.. sectionauthor:: sjg@chromium.org +.. Halloween 2012 +.. Updated 12-12-12 +.. Updated 23-02-13 +.. Updated 09-04-20 diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index b29c1eb..c485994 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -75,6 +75,12 @@ def ParseArgs(): help='List available tool chains (use -v to see probing detail)') parser.add_option('-m', '--mrproper', action='store_true', default=False, help="Run 'make mrproper before reconfiguring") + parser.add_option( + '-M', '--allow-missing', action='store_true', default=False, + help='Tell binman to allow missing blobs and generate fake ones as needed'), + parser.add_option( + '--no-allow-missing', action='store_true', default=False, + help='Disable telling binman to allow missing blobs'), parser.add_option('-n', '--dry-run', action='store_true', dest='dry_run', default=False, help="Do a dry run (describe actions, but do nothing)") parser.add_option('-N', '--no-subdirs', action='store_true', dest='no_subdirs', diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 0c75466..87e7d0e 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -111,6 +111,23 @@ def ShowToolchainPrefix(brds, toolchains): print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) return None +def get_allow_missing(opt_allow, opt_no_allow, num_selected, has_branch): + allow_missing = False + am_setting = bsettings.GetGlobalItemValue('allow-missing') + if am_setting: + if am_setting == 'always': + allow_missing = True + if 'multiple' in am_setting and num_selected > 1: + allow_missing = True + if 'branch' in am_setting and has_branch: + allow_missing = True + + if opt_allow: + allow_missing = True + if opt_no_allow: + allow_missing = False + return allow_missing + def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, clean_dir=False, test_thread_exceptions=False): """The main control code for buildman @@ -136,8 +153,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, if options.full_help: tools.print_full_help( - os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'README') - ) + os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), + 'README.rst')) return 0 gitutil.setup() @@ -305,6 +322,10 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, if not gnu_make: sys.exit('GNU Make not found') + allow_missing = get_allow_missing(options.allow_missing, + options.no_allow_missing, len(selected), + options.branch) + # Create a new builder with the selected options. output_dir = options.output_dir if options.branch: @@ -329,7 +350,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, warnings_as_errors=options.warnings_as_errors, work_in_output=options.work_in_output, test_thread_exceptions=test_thread_exceptions, - adjust_cfg=adjust_cfg) + adjust_cfg=adjust_cfg, + allow_missing=allow_missing) builder.force_config_on_failure = not options.quick if make_func: builder.do_make = make_func diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index f12e996..559e4ed 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -22,6 +22,7 @@ from patman import tools settings_data = ''' # Buildman settings file +[global] [toolchain] @@ -205,13 +206,16 @@ class TestFunctional(unittest.TestCase): self._test_branch = TEST_BRANCH + # Set to True to report missing blobs + self._missing = False + # Avoid sending any output and clear all terminal output terminal.set_print_test_mode() terminal.get_print_test_lines() def tearDown(self): shutil.rmtree(self._base_dir) - #shutil.rmtree(self._output_dir) + shutil.rmtree(self._output_dir) def setupToolchains(self): self._toolchains = toolchain.Toolchains() @@ -249,7 +253,7 @@ class TestFunctional(unittest.TestCase): def testFullHelp(self): command.test_result = None result = self._RunBuildman('-H') - help_file = os.path.join(self._buildman_dir, 'README') + help_file = os.path.join(self._buildman_dir, 'README.rst') # Remove possible extraneous strings extra = '::::::::::::::\n' + help_file + '\n::::::::::::::\n' gothelp = result.stdout.replace(extra, '') @@ -260,7 +264,7 @@ class TestFunctional(unittest.TestCase): def testHelp(self): command.test_result = None result = self._RunBuildman('-h') - help_file = os.path.join(self._buildman_dir, 'README') + help_file = os.path.join(self._buildman_dir, 'README.rst') self.assertTrue(len(result.stdout) > 1000) self.assertEqual(0, len(result.stderr)) self.assertEqual(0, result.return_code) @@ -424,10 +428,21 @@ class TestFunctional(unittest.TestCase): out_dir = arg[2:] fname = os.path.join(cwd or '', out_dir, 'u-boot') tools.write_file(fname, b'U-Boot') - if type(commit) is not str: + + # Handle missing blobs + if self._missing: + if 'BINMAN_ALLOW_MISSING=1' in args: + stderr = '''+Image 'main-section' is missing external blobs and is non-functional: intel-descriptor intel-ifwi intel-fsp-m intel-fsp-s intel-vbt +Image 'main-section' has faked external blobs and is non-functional: descriptor.bin fsp_m.bin fsp_s.bin vbt.bin + +Some images are invalid''' + else: + stderr = "binman: Filename 'fsp.bin' not found in input path" + elif type(commit) is not str: stderr = self._error.get((brd.target, commit.sequence)) + if stderr: - return command.CommandResult(return_code=1, stderr=stderr) + return command.CommandResult(return_code=2, stderr=stderr) return command.CommandResult(return_code=0) # Not handled, so abort @@ -621,3 +636,90 @@ class TestFunctional(unittest.TestCase): self.assertIn( 'Thread exception (use -T0 to run without threads): test exception', stdout.getvalue()) + + def testBlobs(self): + """Test handling of missing blobs""" + self._missing = True + + board0_dir = os.path.join(self._output_dir, 'current', 'board0') + errfile = os.path.join(board0_dir, 'err') + logfile = os.path.join(board0_dir, 'log') + + # We expect failure when there are missing blobs + result = self._RunControl('board0', '-o', self._output_dir) + self.assertEqual(100, result) + self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done'))) + self.assertTrue(os.path.exists(errfile)) + self.assertIn(b"Filename 'fsp.bin' not found in input path", + tools.read_file(errfile)) + + def testBlobsAllowMissing(self): + """Allow missing blobs - still failure but a different exit code""" + self._missing = True + result = self._RunControl('board0', '-o', self._output_dir, '-M', + clean_dir=True) + self.assertEqual(101, result) + board0_dir = os.path.join(self._output_dir, 'current', 'board0') + errfile = os.path.join(board0_dir, 'err') + self.assertTrue(os.path.exists(errfile)) + self.assertIn(b'Some images are invalid', tools.read_file(errfile)) + + def testBlobsWarning(self): + """Allow missing blobs and ignore warnings""" + self._missing = True + result = self._RunControl('board0', '-o', self._output_dir, '-MW') + self.assertEqual(0, result) + board0_dir = os.path.join(self._output_dir, 'current', 'board0') + errfile = os.path.join(board0_dir, 'err') + self.assertIn(b'Some images are invalid', tools.read_file(errfile)) + + def testBlobSettings(self): + """Test with no settings""" + self.assertEqual(False, + control.get_allow_missing(False, False, 1, False)) + self.assertEqual(True, + control.get_allow_missing(True, False, 1, False)) + self.assertEqual(False, + control.get_allow_missing(True, True, 1, False)) + + def testBlobSettingsAlways(self): + """Test the 'always' policy""" + bsettings.SetItem('global', 'allow-missing', 'always') + self.assertEqual(True, + control.get_allow_missing(False, False, 1, False)) + self.assertEqual(False, + control.get_allow_missing(False, True, 1, False)) + + def testBlobSettingsBranch(self): + """Test the 'branch' policy""" + bsettings.SetItem('global', 'allow-missing', 'branch') + self.assertEqual(False, + control.get_allow_missing(False, False, 1, False)) + self.assertEqual(True, + control.get_allow_missing(False, False, 1, True)) + self.assertEqual(False, + control.get_allow_missing(False, True, 1, True)) + + def testBlobSettingsMultiple(self): + """Test the 'multiple' policy""" + bsettings.SetItem('global', 'allow-missing', 'multiple') + self.assertEqual(False, + control.get_allow_missing(False, False, 1, False)) + self.assertEqual(True, + control.get_allow_missing(False, False, 2, False)) + self.assertEqual(False, + control.get_allow_missing(False, True, 2, False)) + + def testBlobSettingsBranchMultiple(self): + """Test the 'branch multiple' policy""" + bsettings.SetItem('global', 'allow-missing', 'branch multiple') + self.assertEqual(False, + control.get_allow_missing(False, False, 1, False)) + self.assertEqual(True, + control.get_allow_missing(False, False, 1, True)) + self.assertEqual(True, + control.get_allow_missing(False, False, 2, False)) + self.assertEqual(True, + control.get_allow_missing(False, False, 2, True)) + self.assertEqual(False, + control.get_allow_missing(False, True, 2, True)) |