diff options
| author | Tom Rini <trini@konsulko.com> | 2023-04-03 16:45:41 -0400 |
|---|---|---|
| committer | Tom Rini <trini@konsulko.com> | 2023-04-03 16:45:41 -0400 |
| commit | 288fe30a2367b8d0e3f416493150a38ebaa88459 (patch) | |
| tree | 1f841eb95d9ceeda4aa3255fb1132a0342f9b19a /tools | |
| parent | fd4ed6b7e83ec3aea9a2ce21baea8ca9676f40dd (diff) | |
| parent | 9876c8c147144db2c120fcc9ffa6de27f6894441 (diff) | |
| download | u-boot-288fe30a2367b8d0e3f416493150a38ebaa88459.zip u-boot-288fe30a2367b8d0e3f416493150a38ebaa88459.tar.gz u-boot-288fe30a2367b8d0e3f416493150a38ebaa88459.tar.bz2 | |
Merge branch 'next'
Signed-off-by: Tom Rini <trini@konsulko.com>
Diffstat (limited to 'tools')
152 files changed, 3082 insertions, 935 deletions
diff --git a/tools/.gitignore b/tools/.gitignore index 788ea26..cda3ea6 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -6,6 +6,7 @@ /dumpimage /easylogo/easylogo /envcrc +/fdt_add_pubkey /fdtgrep /file2include /fit_check_sign diff --git a/tools/Makefile b/tools/Makefile index e13effb..38699b0 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -65,6 +65,7 @@ mkenvimage-objs := mkenvimage.o os_support.o lib/crc32.o hostprogs-y += dumpimage mkimage hostprogs-$(CONFIG_TOOLS_LIBCRYPTO) += fit_info fit_check_sign +hostprogs-$(CONFIG_TOOLS_LIBCRYPTO) += fdt_add_pubkey ifneq ($(CONFIG_CMD_BOOTEFI_SELFTEST)$(CONFIG_FWU_MDATA_GPT_BLK),) hostprogs-y += file2include @@ -150,6 +151,7 @@ dumpimage-objs := $(dumpimage-mkimage-objs) dumpimage.o mkimage-objs := $(dumpimage-mkimage-objs) mkimage.o fit_info-objs := $(dumpimage-mkimage-objs) fit_info.o fit_check_sign-objs := $(dumpimage-mkimage-objs) fit_check_sign.o +fdt_add_pubkey-objs := $(dumpimage-mkimage-objs) fdt_add_pubkey.o file2include-objs := file2include.o ifneq ($(CONFIG_MX23)$(CONFIG_MX28)$(CONFIG_TOOLS_LIBCRYPTO),) @@ -187,6 +189,7 @@ HOSTCFLAGS_fit_image.o += -DMKIMAGE_DTC=\"$(CONFIG_MKIMAGE_DTC_PATH)\" HOSTLDLIBS_dumpimage := $(HOSTLDLIBS_mkimage) HOSTLDLIBS_fit_info := $(HOSTLDLIBS_mkimage) HOSTLDLIBS_fit_check_sign := $(HOSTLDLIBS_mkimage) +HOSTLDLIBS_fdt_add_pubkey := $(HOSTLDLIBS_mkimage) hostprogs-$(CONFIG_EXYNOS5250) += mkexynosspl hostprogs-$(CONFIG_EXYNOS5420) += mkexynosspl diff --git a/tools/binman/binman.rst b/tools/binman/binman.rst index 0921e31..23cbb99 100644 --- a/tools/binman/binman.rst +++ b/tools/binman/binman.rst @@ -95,6 +95,19 @@ Binman uses the following terms: - binary - an input binary that goes into the image +Installation +------------ + +You can install binman using:: + + pip install binary-manager + +The name is chosen since binman conflicts with an existing package. + +If you are using binman within the U-Boot tree, it may be easiest to add a +symlink from your local `~/.bin` directory to `/path/to/tools/binman/binman`. + + Relationship to FIT ------------------- @@ -838,6 +851,14 @@ offset-from-elf: is the symbol to lookup (relative to elf-base-sym) and <offset> is an offset to add to that value. +preserve: + Indicates that this entry should be preserved by any firmware updates. This + flag should be checked by the updater when it is deciding which entries to + update. This flag is normally attached to sections but can be attached to + a single entry in a section if the updater supports it. Not that binman + itself has no control over the updater's behaviour, so this is just a + signal. It is not enforced by binman. + Examples of the above options can be found in the tests. See the tools/binman/test directory. @@ -1122,8 +1143,7 @@ It is sometimes inconvenient to add a 'binman' node to the .dts file for each board. This can be done by using #include to bring in a common file. Another approach supported by the U-Boot build system is to automatically include a common header. You can then put the binman node (and anything else that is -specific to U-Boot, such as u-boot,dm-pre-reloc properies) in that header -file. +specific to U-Boot, such as bootph-all properies) in that header file. Binman will search for the following files in arch/<arch>/dts:: @@ -1327,9 +1347,43 @@ You can also replace just a selection of entries:: $ binman replace -i image.bin "*u-boot*" -I indir +It is possible to replace whole sections as well, but in that case any +information about entries within the section may become outdated. This is +because Binman cannot know whether things have moved around or resized within +the section, once you have updated its data. + +Technical note: With 'allow-repack', Binman writes information about the +original offset and size properties of each entry, if any were specified, in +the 'orig-offset' and 'orig-size' properties. This allows Binman to distinguish +between an entry which ended up being packed at an offset (or assigned a size) +and an entry which had a particular offset / size requested in the Binman +configuration. Where are particular offset / size was requested, this is treated +as set in stone, so Binman will ensure it doesn't change. Without this feature, +repacking an entry might cause it to disobey the original constraints provided +when it was created. + + Repacking an image involves .. _`BinmanLogging`: +Signing FIT container with private key in an image +-------------------------------------------------- + +You can sign FIT container with private key in your image. +For example:: + + $ binman sign -i image.bin -k privatekey -a sha256,rsa4096 fit + +binman will extract FIT container, sign and replace it immediately. + +If you want to sign and replace FIT container in place:: + + $ binman sign -i image.bin -k privatekey -a sha256,rsa4096 -f fit.fit fit + +which will sign FIT container with private key and replace it immediately +inside your image. + + Logging ------- @@ -1408,6 +1462,16 @@ You can also use `--fetch all` to fetch all tools or `--fetch <tool>` to fetch a particular tool. Some tools are built from source code, in which case you will need to have at least the `build-essential` and `git` packages installed. +Tools are fetched into the `~/.binman-tools` directory. This directory is +automatically added to the toolpath so there is no need to use `--toolpath` to +specify it. If you want to use these tools outside binman, you may want to +add this directory to your `PATH`. For example, if you use bash, add this to +the end of `.bashrc`:: + + PATH="$HOME/.binman-tools:$PATH" + +To select a custom directory, use the `--tooldir` option. + Bintool Documentation ===================== @@ -1426,8 +1490,9 @@ Binman commands and arguments Usage:: - binman [-h] [-B BUILD_DIR] [-D] [-H] [--toolpath TOOLPATH] [-T THREADS] - [--test-section-timeout] [-v VERBOSITY] [-V] + binman [-h] [-B BUILD_DIR] [-D] [--tooldir TOOLDIR] [-H] + [--toolpath TOOLPATH] [-T THREADS] [--test-section-timeout] + [-v VERBOSITY] [-V] {build,bintool-docs,entry-docs,ls,extract,replace,test,tool} ... Binman provides the following commands: @@ -1452,11 +1517,13 @@ Options: -D, --debug Enabling debugging (provides a full traceback on error) +--tooldir TOOLDIR Set the directory to store tools + -H, --full-help Display the README file --toolpath TOOLPATH - Add a path to the directories containing tools + Add a path to the list of directories containing tools -T THREADS, --threads THREADS Number of threads to use (0=single-thread). Note that -T0 is useful for @@ -1664,6 +1731,12 @@ Options: -m, --map Output a map file for the updated image +-O OUTDIR, --outdir OUTDIR + Path to directory to use for intermediate and output files + +-p, --preserve + Preserve temporary output directory even if option -O is not given + This replaces one or more entries in an existing image. See `Replacing files in an image`_. @@ -1696,6 +1769,35 @@ Options: output directory if a single test is run (pass test name at the end of the command line +binman sign +----------- + +Usage:: + + binman sign [-h] -a ALGO [-f FILE] -i IMAGE -k KEY [paths ...] + +positional arguments: + +paths + Paths within file to sign (wildcard) + +options: + +-h, --help + show this help message and exit + +-a ALGO, --algo ALGO + Hash algorithm e.g. sha256,rsa4096 + +-f FILE, --file FILE + Input filename to sign + +-i IMAGE, --image IMAGE + Image filename to update + +-k KEY, --key KEY + Private key file for signing + binman tool ----------- diff --git a/tools/binman/bintool.py b/tools/binman/bintool.py index 8fda13f..8162968 100644 --- a/tools/binman/bintool.py +++ b/tools/binman/bintool.py @@ -18,10 +18,10 @@ import shutil import tempfile import urllib.error -from patman import command -from patman import terminal -from patman import tools -from patman import tout +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib import tools +from u_boot_pylib import tout BINMAN_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -43,8 +43,6 @@ FETCH_NAMES = { # Status of tool fetching FETCHED, FAIL, PRESENT, STATUS_COUNT = range(4) -DOWNLOAD_DESTDIR = os.path.join(os.getenv('HOME'), 'bin') - class Bintool: """Tool which operates on binaries to help produce entry contents @@ -53,6 +51,10 @@ class Bintool: # List of bintools to regard as missing missing_list = [] + # Directory to store tools. Note that this set up by set_tool_dir() which + # must be called before this class is used. + tooldir = '' + def __init__(self, name, desc, version_regex=None, version_args='-V'): self.name = name self.desc = desc @@ -112,6 +114,11 @@ class Bintool: obj = cls(name) return obj + @classmethod + def set_tool_dir(cls, pathname): + """Set the path to use to store and find tools""" + cls.tooldir = pathname + def show(self): """Show a line of information about a bintool""" if self.is_present(): @@ -208,7 +215,8 @@ class Bintool: return FAIL if result is not True: fname, tmpdir = result - dest = os.path.join(DOWNLOAD_DESTDIR, self.name) + dest = os.path.join(self.tooldir, self.name) + os.makedirs(self.tooldir, exist_ok=True) print(f"- writing to '{dest}'") shutil.move(fname, dest) if tmpdir: @@ -389,7 +397,7 @@ class Bintool: @classmethod def apt_install(cls, package): - """Install a bintool using the 'aot' tool + """Install a bintool using the 'apt' tool This requires use of servo so may request a password diff --git a/tools/binman/bintool_test.py b/tools/binman/bintool_test.py index 7efb839..f9b16d4 100644 --- a/tools/binman/bintool_test.py +++ b/tools/binman/bintool_test.py @@ -16,10 +16,10 @@ import urllib.error from binman import bintool from binman.bintool import Bintool -from patman import command -from patman import terminal -from patman import test_util -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib import test_util +from u_boot_pylib import tools # pylint: disable=R0904 class TestBintool(unittest.TestCase): @@ -134,12 +134,14 @@ class TestBintool(unittest.TestCase): dirname = os.path.join(self._indir, 'download_dir') os.mkdir(dirname) fname = os.path.join(dirname, 'downloaded') + + # Rely on bintool to create this directory destdir = os.path.join(self._indir, 'dest_dir') - os.mkdir(destdir) + dest_fname = os.path.join(destdir, '_testing') self.seq = 0 - with unittest.mock.patch.object(bintool, 'DOWNLOAD_DESTDIR', destdir): + with unittest.mock.patch.object(bintool.Bintool, 'tooldir', destdir): with unittest.mock.patch.object(tools, 'download', side_effect=handle_download): with test_util.capture_sys_output() as (stdout, _): @@ -250,7 +252,7 @@ class TestBintool(unittest.TestCase): btest = Bintool.create('_testing') col = terminal.Color() self.fname = None - with unittest.mock.patch.object(bintool, 'DOWNLOAD_DESTDIR', + with unittest.mock.patch.object(bintool.Bintool, 'tooldir', self._indir): with unittest.mock.patch.object(tools, 'run', side_effect=fake_run): with test_util.capture_sys_output() as (stdout, _): @@ -344,8 +346,11 @@ class TestBintool(unittest.TestCase): def test_failed_command(self): """Check that running a command that does not exist returns None""" - btool = Bintool.create('_testing') - result = btool.run_cmd_result('fred') + destdir = os.path.join(self._indir, 'dest_dir') + os.mkdir(destdir) + with unittest.mock.patch.object(bintool.Bintool, 'tooldir', destdir): + btool = Bintool.create('_testing') + result = btool.run_cmd_result('fred') self.assertIsNone(result) diff --git a/tools/binman/bintools.rst b/tools/binman/bintools.rst index edb373a..c30e7eb 100644 --- a/tools/binman/bintools.rst +++ b/tools/binman/bintools.rst @@ -10,6 +10,20 @@ binaries. It is fairly easy to create new bintools. Just add a new file to the +Bintool: bzip2: Compression/decompression using the bzip2 algorithm +------------------------------------------------------------------- + +This bintool supports running `bzip2` to compress and decompress data, as +used by binman. + +It is also possible to fetch the tool, which uses `apt` to install it. + +Documentation is available via:: + + man bzip2 + + + Bintool: cbfstool: Coreboot filesystem (CBFS) tool -------------------------------------------------- @@ -58,6 +72,20 @@ See `Chromium OS vboot documentation`_ for more information. +Bintool: gzip: Compression/decompression using the gzip algorithm +----------------------------------------------------------------- + +This bintool supports running `gzip` to compress and decompress data, as +used by binman. + +It is also possible to fetch the tool, which uses `apt` to install it. + +Documentation is available via:: + + man gzip + + + Bintool: ifwitool: Handles the 'ifwitool' tool ---------------------------------------------- @@ -101,6 +129,20 @@ Documentation is available via:: +Bintool: lzop: Compression/decompression using the lzop algorithm +----------------------------------------------------------------- + +This bintool supports running `lzop` to compress and decompress data, as +used by binman. + +It is also possible to fetch the tool, which uses `apt` to install it. + +Documentation is available via:: + + man lzop + + + Bintool: mkimage: Image generation for U-Boot --------------------------------------------- @@ -113,3 +155,31 @@ Support is provided for fetching this on Debian-like systems, using apt. +Bintool: xz: Compression/decompression using the xz algorithm +------------------------------------------------------------- + +This bintool supports running `xz` to compress and decompress data, as +used by binman. + +It is also possible to fetch the tool, which uses `apt` to install it. + +Documentation is available via:: + + man xz + + + +Bintool: zstd: Compression/decompression using the zstd algorithm +----------------------------------------------------------------- + +This bintool supports running `zstd` to compress and decompress data, as +used by binman. + +It is also possible to fetch the tool, which uses `apt` to install it. + +Documentation is available via:: + + man zstd + + + diff --git a/tools/binman/btool/lz4.py b/tools/binman/btool/lz4.py index dc9e379..fd520d1 100644 --- a/tools/binman/btool/lz4.py +++ b/tools/binman/btool/lz4.py @@ -60,7 +60,7 @@ import re import tempfile from binman import bintool -from patman import tools +from u_boot_pylib import tools # pylint: disable=C0103 class Bintoollz4(bintool.Bintool): diff --git a/tools/binman/btool/lzma_alone.py b/tools/binman/btool/lzma_alone.py index 52a960f..1fda2f6 100644 --- a/tools/binman/btool/lzma_alone.py +++ b/tools/binman/btool/lzma_alone.py @@ -37,7 +37,7 @@ import re import tempfile from binman import bintool -from patman import tools +from u_boot_pylib import tools # pylint: disable=C0103 class Bintoollzma_alone(bintool.Bintool): diff --git a/tools/binman/btool/openssl.py b/tools/binman/btool/openssl.py new file mode 100644 index 0000000..3a4dbdd --- /dev/null +++ b/tools/binman/btool/openssl.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2022 Google LLC +# +"""Bintool implementation for openssl + +openssl provides a number of features useful for signing images + +Documentation is at https://www.coreboot.org/CBFS + +Source code is at https://www.openssl.org/ +""" + +import hashlib + +from binman import bintool +from u_boot_pylib import tools + +class Bintoolopenssl(bintool.Bintool): + """openssl tool + + This bintool supports creating new openssl certificates. + + It also supports fetching a binary openssl + + Documentation about openssl is at https://www.openssl.org/ + """ + def __init__(self, name): + super().__init__( + name, 'openssl cryptography toolkit', + version_regex=r'OpenSSL (.*) \(', version_args='version') + + def x509_cert(self, cert_fname, input_fname, key_fname, cn, revision, + config_fname): + """Create a certificate + + Args: + cert_fname (str): Filename of certificate to create + input_fname (str): Filename containing data to sign + key_fname (str): Filename of .pem file + cn (str): Common name + revision (int): Revision number + config_fname (str): Filename to write fconfig into + + Returns: + str: Tool output + """ + indata = tools.read_file(input_fname) + hashval = hashlib.sha512(indata).hexdigest() + with open(config_fname, 'w', encoding='utf-8') as outf: + print(f'''[ req ] +distinguished_name = req_distinguished_name +x509_extensions = v3_ca +prompt = no +dirstring_type = nobmp + +[ req_distinguished_name ] +CN = {cert_fname} + +[ v3_ca ] +basicConstraints = CA:true +1.3.6.1.4.1.294.1.3 = ASN1:SEQUENCE:swrv +1.3.6.1.4.1.294.1.34 = ASN1:SEQUENCE:sysfw_image_integrity + +[ swrv ] +swrv = INTEGER:{revision} + +[ sysfw_image_integrity ] +shaType = OID:2.16.840.1.101.3.4.2.3 +shaValue = FORMAT:HEX,OCT:{hashval} +imageSize = INTEGER:{len(indata)} +''', file=outf) + args = ['req', '-new', '-x509', '-key', key_fname, '-nodes', + '-outform', 'DER', '-out', cert_fname, '-config', config_fname, + '-sha512'] + return self.run_cmd(*args) + + def fetch(self, method): + """Fetch handler for openssl + + This installs the openssl package using the apt utility. + + Args: + method (FETCH_...): Method to use + + Returns: + True if the file was fetched and now installed, None if a method + other than FETCH_BIN was requested + + Raises: + Valuerror: Fetching could not be completed + """ + if method != bintool.FETCH_BIN: + return None + return self.apt_install('openssl') diff --git a/tools/binman/cbfs_util.py b/tools/binman/cbfs_util.py index 7bd3d89..fc56b40 100644 --- a/tools/binman/cbfs_util.py +++ b/tools/binman/cbfs_util.py @@ -22,8 +22,8 @@ import sys from binman import bintool from binman import elf -from patman import command -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import tools # Set to True to enable printing output while working DEBUG = False diff --git a/tools/binman/cbfs_util_test.py b/tools/binman/cbfs_util_test.py index e0f792f..ee951d1 100755 --- a/tools/binman/cbfs_util_test.py +++ b/tools/binman/cbfs_util_test.py @@ -20,8 +20,8 @@ from binman import bintool from binman import cbfs_util from binman.cbfs_util import CbfsWriter from binman import elf -from patman import test_util -from patman import tools +from u_boot_pylib import test_util +from u_boot_pylib import tools U_BOOT_DATA = b'1234' U_BOOT_DTB_DATA = b'udtb' diff --git a/tools/binman/cmdline.py b/tools/binman/cmdline.py index 986d6f1..4b875a9 100644 --- a/tools/binman/cmdline.py +++ b/tools/binman/cmdline.py @@ -7,7 +7,13 @@ import argparse from argparse import ArgumentParser +import os from binman import state +import os +import pathlib + +BINMAN_DIR = pathlib.Path(__file__).parent +HAS_TESTS = (BINMAN_DIR / "ftest.py").exists() def make_extract_parser(subparsers): """make_extract_parser: Make a subparser for the 'extract' command @@ -67,6 +73,14 @@ def ParseArgs(argv): options provides access to the options (e.g. option.debug) args is a list of string arguments """ + def _AddPreserve(pars): + pars.add_argument('-O', '--outdir', type=str, + action='store', help='Path to directory to use for intermediate ' + 'and output files') + pars.add_argument('-p', '--preserve', action='store_true',\ + help='Preserve temporary output directory even if option -O is not ' + 'given') + if '-H' in argv: argv.append('build') @@ -80,8 +94,11 @@ controlled by a description in the board device tree.''' help='Enabling debugging (provides a full traceback on error)') parser.add_argument('-H', '--full-help', action='store_true', default=False, help='Display the README file') + parser.add_argument('--tooldir', type=str, + default=os.path.join(os.getenv('HOME'), '.binman-tools'), + help='Set the directory to store tools') parser.add_argument('--toolpath', type=str, action='append', - help='Add a path to the directories containing tools') + help='Add a path to the list of directories containing tools') parser.add_argument('-T', '--threads', type=int, default=None, help='Number of threads to use (0=single-thread)') parser.add_argument('--test-section-timeout', action='store_true', @@ -118,12 +135,7 @@ controlled by a description in the board device tree.''' build_parser.add_argument('-n', '--no-expanded', action='store_true', help="Don't use 'expanded' versions of entries where available; " "normally 'u-boot' becomes 'u-boot-expanded', for example") - build_parser.add_argument('-O', '--outdir', type=str, - action='store', help='Path to directory to use for intermediate and ' - 'output files') - build_parser.add_argument('-p', '--preserve', action='store_true',\ - help='Preserve temporary output directory even if option -O is not ' - 'given') + _AddPreserve(build_parser) build_parser.add_argument('-u', '--update-fdt', action='store_true', default=False, help='Update the binman node with offset/size info') build_parser.add_argument('--update-fdt-in-elf', type=str, @@ -160,26 +172,43 @@ controlled by a description in the board device tree.''' help='Path to directory to use for input files') replace_parser.add_argument('-m', '--map', action='store_true', default=False, help='Output a map file for the updated image') + _AddPreserve(replace_parser) replace_parser.add_argument('paths', type=str, nargs='*', help='Paths within file to replace (wildcard)') - test_parser = subparsers.add_parser('test', help='Run tests') - test_parser.add_argument('-P', '--processes', type=int, - help='set number of processes to use for running tests') - test_parser.add_argument('-T', '--test-coverage', action='store_true', - default=False, help='run tests and check for 100%% coverage') - test_parser.add_argument('-X', '--test-preserve-dirs', action='store_true', - help='Preserve and display test-created input directories; also ' - 'preserve the output directory if a single test is run (pass test ' - 'name at the end of the command line') - test_parser.add_argument('tests', nargs='*', - help='Test names to run (omit for all)') + sign_parser = subparsers.add_parser('sign', + help='Sign entries in image') + sign_parser.add_argument('-a', '--algo', type=str, required=True, + help='Hash algorithm e.g. sha256,rsa4096') + sign_parser.add_argument('-f', '--file', type=str, required=False, + help='Input filename to sign') + sign_parser.add_argument('-i', '--image', type=str, required=True, + help='Image filename to update') + sign_parser.add_argument('-k', '--key', type=str, required=True, + help='Private key file for signing') + sign_parser.add_argument('paths', type=str, nargs='*', + help='Paths within file to sign (wildcard)') + + if HAS_TESTS: + test_parser = subparsers.add_parser('test', help='Run tests') + test_parser.add_argument('-P', '--processes', type=int, + help='set number of processes to use for running tests') + test_parser.add_argument('-T', '--test-coverage', action='store_true', + default=False, help='run tests and check for 100%% coverage') + test_parser.add_argument( + '-X', '--test-preserve-dirs', action='store_true', + help='Preserve and display test-created input directories; also ' + 'preserve the output directory if a single test is run (pass ' + 'test name at the end of the command line') + test_parser.add_argument('tests', nargs='*', + help='Test names to run (omit for all)') tool_parser = subparsers.add_parser('tool', help='Check bintools') tool_parser.add_argument('-l', '--list', action='store_true', help='List all known bintools') - tool_parser.add_argument('-f', '--fetch', action='store_true', - help='fetch a bintool from a known location (or: all/missing)') + tool_parser.add_argument( + '-f', '--fetch', action='store_true', + help='fetch a bintool from a known location (or: all/missing)') tool_parser.add_argument('bintools', type=str, nargs='*') return parser.parse_args(argv) diff --git a/tools/binman/control.py b/tools/binman/control.py index e647400..0febcb7 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -7,19 +7,20 @@ from collections import OrderedDict import glob +import importlib.resources import os import pkg_resources import re import sys -from patman import tools from binman import bintool from binman import cbfs_util -from patman import command from binman import elf from binman import entry -from patman import tout +from u_boot_pylib import command +from u_boot_pylib import tools +from u_boot_pylib import tout # These are imported if needed since they import libfdt state = None @@ -402,6 +403,8 @@ def ReplaceEntries(image_fname, input_fname, indir, entry_paths, image_fname = os.path.abspath(image_fname) image = Image.FromFile(image_fname) + image.mark_build_done() + # Replace an entry from a single file, as a special case if input_fname: if not entry_paths: @@ -445,6 +448,31 @@ def ReplaceEntries(image_fname, input_fname, indir, entry_paths, AfterReplace(image, allow_resize=allow_resize, write_map=write_map) return image +def SignEntries(image_fname, input_fname, privatekey_fname, algo, entry_paths, + write_map=False): + """Sign and replace the data from one or more entries from input files + + Args: + image_fname: Image filename to process + input_fname: Single input filename to use if replacing one file, None + otherwise + algo: Hashing algorithm + entry_paths: List of entry paths to sign + privatekey_fname: Private key filename + write_map (bool): True to write the map file + """ + image_fname = os.path.abspath(image_fname) + image = Image.FromFile(image_fname) + + image.mark_build_done() + + BeforeReplace(image, allow_resize=True) + + for entry_path in entry_paths: + entry = image.FindEntryPath(entry_path) + entry.UpdateSignatures(privatekey_fname, algo, input_fname) + + AfterReplace(image, allow_resize=True, write_map=write_map) def PrepareImagesAndDtbs(dtb_fname, select_images, update_fdt, use_expanded): """Prepare the images to be processed and select the device tree @@ -641,19 +669,29 @@ def Binman(args): global state if args.full_help: - tools.print_full_help( - os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'README.rst') - ) + with importlib.resources.path('binman', 'README.rst') as readme: + tools.print_full_help(str(readme)) return 0 # Put these here so that we can import this module without libfdt from binman.image import Image from binman import state - if args.cmd in ['ls', 'extract', 'replace', 'tool']: + tool_paths = [] + if args.toolpath: + tool_paths += args.toolpath + if args.tooldir: + tool_paths.append(args.tooldir) + tools.set_tool_paths(tool_paths or None) + bintool.Bintool.set_tool_dir(args.tooldir) + + if args.cmd in ['ls', 'extract', 'replace', 'tool', 'sign']: try: tout.init(args.verbosity) - tools.prepare_output_dir(None) + if args.cmd == 'replace': + tools.prepare_output_dir(args.outdir, args.preserve) + else: + tools.prepare_output_dir(None) if args.cmd == 'ls': ListEntries(args.image, args.paths) @@ -666,8 +704,10 @@ def Binman(args): do_compress=not args.compressed, allow_resize=not args.fix_size, write_map=args.map) + if args.cmd == 'sign': + SignEntries(args.image, args.file, args.key, args.algo, args.paths) + if args.cmd == 'tool': - tools.set_tool_paths(args.toolpath) if args.list: bintool.Bintool.list_all() elif args.fetch: @@ -719,7 +759,6 @@ def Binman(args): try: tools.set_input_dirs(args.indir) tools.prepare_output_dir(args.outdir, args.preserve) - tools.set_tool_paths(args.toolpath) state.SetEntryArgs(args.entry_arg) state.SetThreads(args.threads) diff --git a/tools/binman/elf.py b/tools/binman/elf.py index 3cc8a38..5816284 100644 --- a/tools/binman/elf.py +++ b/tools/binman/elf.py @@ -13,9 +13,9 @@ import shutil import struct import tempfile -from patman import command -from patman import tools -from patman import tout +from u_boot_pylib import command +from u_boot_pylib import tools +from u_boot_pylib import tout ELF_TOOLS = True try: diff --git a/tools/binman/elf_test.py b/tools/binman/elf_test.py index 8cb55eb..c980839 100644 --- a/tools/binman/elf_test.py +++ b/tools/binman/elf_test.py @@ -12,10 +12,10 @@ import tempfile import unittest from binman import elf -from patman import command -from patman import test_util -from patman import tools -from patman import tout +from u_boot_pylib import command +from u_boot_pylib import test_util +from u_boot_pylib import tools +from u_boot_pylib import tout binman_dir = os.path.dirname(os.path.realpath(sys.argv[0])) diff --git a/tools/binman/entries.rst b/tools/binman/entries.rst index e177860..b71af80 100644 --- a/tools/binman/entries.rst +++ b/tools/binman/entries.rst @@ -887,6 +887,11 @@ before its contents, so that it is possible to reconstruct the hierarchy from the FMAP by using the offset information. This convention does not seem to be documented, but is used in Chromium OS. +To mark an area as preserved, use the normal 'preserved' flag in the entry. +This will result in the corresponding FMAP area having the +FMAP_AREA_PRESERVE flag. This flag does not automatically propagate down to +child entries. + CBFS entries appear as a single entry, i.e. the sub-entries are ignored. @@ -2285,6 +2290,24 @@ and kernel are genuine. +.. _etype_x509_cert: + +Entry: x509-cert: An entry which contains an X509 certificate +------------------------------------------------------------- + +Properties / Entry arguments: + - content: List of phandles to entries to sign + +Output files: + - input.<unique_name> - input file passed to openssl + - cert.<unique_name> - output file generated by openssl (which is + used as the entry contents) + +openssl signs the provided data, writing the signature in this entry. This +allows verification that the data is genuine + + + .. _etype_x86_reset16: Entry: x86-reset16: x86 16-bit reset code for U-Boot diff --git a/tools/binman/entry.py b/tools/binman/entry.py index 5eacc5f..3945690 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -14,9 +14,9 @@ import time from binman import bintool from binman import elf from dtoc import fdt_util -from patman import tools -from patman.tools import to_hex, to_hex_size -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib.tools import to_hex, to_hex_size +from u_boot_pylib import tout modules = {} @@ -100,6 +100,14 @@ class Entry(object): appear in the map optional (bool): True if this entry contains an optional external blob overlap (bool): True if this entry overlaps with others + preserve (bool): True if this entry should be preserved when updating + firmware. This means that it will not be changed by the update. + This is just a signal: enforcement of this is up to the updater. + This flag does not automatically propagate down to child entries. + build_done (bool): Indicates that the entry data has been built and does + not need to be done again. This is only used with 'binman replace', + to stop sections from being rebuilt if their entries have not been + replaced """ fake_dir = None @@ -148,6 +156,8 @@ class Entry(object): self.overlap = False self.elf_base_sym = None self.offset_from_elf = None + self.preserve = False + self.build_done = False @staticmethod def FindEntryClass(etype, expanded): @@ -310,6 +320,8 @@ class Entry(object): self.offset_from_elf = fdt_util.GetPhandleNameOffset(self._node, 'offset-from-elf') + self.preserve = fdt_util.GetBool(self._node, 'preserve') + def GetDefaultFilename(self): return None @@ -1006,6 +1018,7 @@ features to produce new behaviours. else: self.contents_size = self.pre_reset_size ok = self.ProcessContentsUpdate(data) + self.build_done = False self.Detail('WriteData: size=%x, ok=%s' % (len(data), ok)) section_ok = self.section.WriteChildData(self) return ok and section_ok @@ -1027,6 +1040,14 @@ features to produce new behaviours. True if the section could be updated successfully, False if the data is such that the section could not update """ + self.build_done = False + entry = self.section + + # Now we must rebuild all sections above this one + while entry and entry != entry.section: + self.build_done = False + entry = entry.section + return True def GetSiblingOrder(self): @@ -1104,7 +1125,7 @@ features to produce new behaviours. If there are faked blobs, the entries are added to the list Args: - fake_blobs_list: List of Entry objects to be added to + faked_blobs_list: List of Entry objects to be added to """ # This is meaningless for anything other than blobs pass @@ -1349,3 +1370,14 @@ features to produce new behaviours. val = elf.GetSymbolOffset(entry.elf_fname, sym_name, entry.elf_base_sym) return val + offset + + def mark_build_done(self): + """Mark an entry as already built""" + self.build_done = True + entries = self.GetEntries() + if entries: + for entry in entries.values(): + entry.mark_build_done() + + def UpdateSignatures(self, privatekey_fname, algo, input_fname): + self.Raise('Updating signatures is not supported with this entry type') diff --git a/tools/binman/entry_test.py b/tools/binman/entry_test.py index a6fbf62..ac6582c 100644 --- a/tools/binman/entry_test.py +++ b/tools/binman/entry_test.py @@ -14,7 +14,7 @@ from binman import entry from binman.etype.blob import Entry_blob from dtoc import fdt from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class TestEntry(unittest.TestCase): def setUp(self): diff --git a/tools/binman/etype/_testing.py b/tools/binman/etype/_testing.py index 1c1efb2..e092d98 100644 --- a/tools/binman/etype/_testing.py +++ b/tools/binman/etype/_testing.py @@ -9,7 +9,7 @@ from collections import OrderedDict from binman.entry import Entry, EntryArg from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry__testing(Entry): diff --git a/tools/binman/etype/atf_fip.py b/tools/binman/etype/atf_fip.py index 6ecd95b..73a3f85 100644 --- a/tools/binman/etype/atf_fip.py +++ b/tools/binman/etype/atf_fip.py @@ -11,7 +11,7 @@ from binman.entry import Entry from binman.etype.section import Entry_section from binman.fip_util import FIP_TYPES, FipReader, FipWriter, UUID_LEN from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_atf_fip(Entry_section): """ARM Trusted Firmware's Firmware Image Package (FIP) @@ -270,4 +270,4 @@ class Entry_atf_fip(Entry_section): # Recreate the data structure, leaving the data for this child alone, # so that child.data is used to pack into the FIP. self.ObtainContents(skip_entry=child) - return True + return super().WriteChildData(child) diff --git a/tools/binman/etype/blob.py b/tools/binman/etype/blob.py index c7ddced..064fae5 100644 --- a/tools/binman/etype/blob.py +++ b/tools/binman/etype/blob.py @@ -8,8 +8,8 @@ from binman.entry import Entry from binman import state from dtoc import fdt_util -from patman import tools -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib import tout class Entry_blob(Entry): """Arbitrary binary blob @@ -102,7 +102,7 @@ class Entry_blob(Entry): If there are faked blobs, the entries are added to the list Args: - fake_blobs_list: List of Entry objects to be added to + faked_blobs_list: List of Entry objects to be added to """ if self.faked: faked_blobs_list.append(self) diff --git a/tools/binman/etype/blob_ext.py b/tools/binman/etype/blob_ext.py index fba6271..ca26530 100644 --- a/tools/binman/etype/blob_ext.py +++ b/tools/binman/etype/blob_ext.py @@ -9,8 +9,8 @@ import os from binman.etype.blob import Entry_blob from dtoc import fdt_util -from patman import tools -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib import tout class Entry_blob_ext(Entry_blob): """Externally built binary blob @@ -26,11 +26,3 @@ class Entry_blob_ext(Entry_blob): def __init__(self, section, etype, node): Entry_blob.__init__(self, section, etype, node) self.external = True - - def SetAllowFakeBlob(self, allow_fake): - """Set whether the entry allows to create a fake blob - - Args: - allow_fake_blob: True if allowed, False if not allowed - """ - self.allow_fake = allow_fake diff --git a/tools/binman/etype/blob_ext_list.py b/tools/binman/etype/blob_ext_list.py index f00202e..1bfcf67 100644 --- a/tools/binman/etype/blob_ext_list.py +++ b/tools/binman/etype/blob_ext_list.py @@ -9,8 +9,8 @@ import os from binman.etype.blob import Entry_blob from dtoc import fdt_util -from patman import tools -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib import tout class Entry_blob_ext_list(Entry_blob): """List of externally built binary blobs diff --git a/tools/binman/etype/cbfs.py b/tools/binman/etype/cbfs.py index 832f8d0..575aa62 100644 --- a/tools/binman/etype/cbfs.py +++ b/tools/binman/etype/cbfs.py @@ -295,7 +295,7 @@ class Entry_cbfs(Entry): # Recreate the data structure, leaving the data for this child alone, # so that child.data is used to pack into the FIP. self.ObtainContents(skip_entry=child) - return True + return super().WriteChildData(child) def AddBintools(self, btools): super().AddBintools(btools) diff --git a/tools/binman/etype/fdtmap.py b/tools/binman/etype/fdtmap.py index 33c9d03..f1f6217 100644 --- a/tools/binman/etype/fdtmap.py +++ b/tools/binman/etype/fdtmap.py @@ -9,8 +9,8 @@ image. """ from binman.entry import Entry -from patman import tools -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib import tout FDTMAP_MAGIC = b'_FDTMAP_' FDTMAP_HDR_LEN = 16 diff --git a/tools/binman/etype/files.py b/tools/binman/etype/files.py index 2081bc7..c8757ea 100644 --- a/tools/binman/etype/files.py +++ b/tools/binman/etype/files.py @@ -11,7 +11,7 @@ import os from binman.etype.section import Entry_section from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools # This is imported if needed state = None diff --git a/tools/binman/etype/fill.py b/tools/binman/etype/fill.py index c91d015..7c93d4e 100644 --- a/tools/binman/etype/fill.py +++ b/tools/binman/etype/fill.py @@ -5,7 +5,7 @@ from binman.entry import Entry from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_fill(Entry): """An entry which is filled to a particular byte value diff --git a/tools/binman/etype/fit.py b/tools/binman/etype/fit.py index cd29435..c395706 100644 --- a/tools/binman/etype/fit.py +++ b/tools/binman/etype/fit.py @@ -12,7 +12,7 @@ from binman.etype.section import Entry_section from binman import elf from dtoc import fdt_util from dtoc.fdt import Fdt -from patman import tools +from u_boot_pylib import tools # Supported operations, with the fit,operation property OP_GEN_FDT_NODES, OP_SPLIT_ELF = range(2) @@ -453,6 +453,8 @@ class Entry_fit(Entry_section): args.update({'align': fdt_util.fdt32_to_cpu(align.value)}) if self.mkimage.run(reset_timestamp=True, output_fname=output_fname, **args) is None: + if not self.GetAllowMissing(): + self.Raise("Missing tool: 'mkimage'") # Bintool is missing; just use empty data as the output self.record_missing_bintool(self.mkimage) return tools.get_bytes(0, 1024) @@ -775,6 +777,8 @@ class Entry_fit(Entry_section): Args: image_pos (int): Position of this entry in the image """ + if self.build_done: + return super().SetImagePos(image_pos) # If mkimage is missing we'll have empty data, @@ -823,8 +827,27 @@ class Entry_fit(Entry_section): self.mkimage = self.AddBintool(btools, 'mkimage') def CheckMissing(self, missing_list): - # We must use our private entry list for this since generator notes + # We must use our private entry list for this since generator nodes # which are removed from self._entries will otherwise not show up as # missing for entry in self._priv_entries.values(): entry.CheckMissing(missing_list) + + def CheckEntries(self): + pass + + def UpdateSignatures(self, privatekey_fname, algo, input_fname): + uniq = self.GetUniqueName() + args = [ '-G', privatekey_fname, '-r', '-o', algo, '-F' ] + if input_fname: + fname = input_fname + else: + fname = tools.get_output_filename('%s.fit' % uniq) + tools.write_file(fname, self.GetData()) + args.append(fname) + + if self.mkimage.run_cmd(*args) is None: + self.Raise("Missing tool: 'mkimage'") + + data = tools.read_file(fname) + self.WriteData(data) diff --git a/tools/binman/etype/fmap.py b/tools/binman/etype/fmap.py index 0c57620..3669d91 100644 --- a/tools/binman/etype/fmap.py +++ b/tools/binman/etype/fmap.py @@ -7,9 +7,9 @@ from binman.entry import Entry from binman import fmap_util -from patman import tools -from patman.tools import to_hex_size -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib.tools import to_hex_size +from u_boot_pylib import tout class Entry_fmap(Entry): @@ -33,6 +33,11 @@ class Entry_fmap(Entry): from the FMAP by using the offset information. This convention does not seem to be documented, but is used in Chromium OS. + To mark an area as preserved, use the normal 'preserved' flag in the entry. + This will result in the corresponding FMAP area having the + FMAP_AREA_PRESERVE flag. This flag does not automatically propagate down to + child entries. + CBFS entries appear as a single entry, i.e. the sub-entries are ignored. """ def __init__(self, section, etype, node): @@ -48,6 +53,12 @@ class Entry_fmap(Entry): entries = entry.GetEntries() tout.debug("fmap: Add entry '%s' type '%s' (%s subentries)" % (entry.GetPath(), entry.etype, to_hex_size(entries))) + + # Collect any flag (separate lines to ensure code coverage) + flags = 0 + if entry.preserve: + flags = fmap_util.FMAP_AREA_PRESERVE + if entries and entry.etype != 'cbfs': # Create an area for the section, which encompasses all entries # within it @@ -59,7 +70,7 @@ class Entry_fmap(Entry): # Drop @ symbols in name name = entry.name.replace('@', '') areas.append( - fmap_util.FmapArea(pos, entry.size or 0, name, 0)) + fmap_util.FmapArea(pos, entry.size or 0, name, flags)) for subentry in entries.values(): _AddEntries(areas, subentry) else: @@ -67,7 +78,7 @@ class Entry_fmap(Entry): if pos is not None: pos -= entry.section.GetRootSkipAtStart() areas.append(fmap_util.FmapArea(pos or 0, entry.size or 0, - entry.name, 0)) + entry.name, flags)) entries = self.GetImage().GetEntries() areas = [] diff --git a/tools/binman/etype/gbb.py b/tools/binman/etype/gbb.py index ba2a362..cca18af 100644 --- a/tools/binman/etype/gbb.py +++ b/tools/binman/etype/gbb.py @@ -8,11 +8,11 @@ from collections import OrderedDict -from patman import command +from u_boot_pylib import command from binman.entry import Entry, EntryArg from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools # Build GBB flags. # (src/platform/vboot_reference/firmware/include/gbb_header.h) diff --git a/tools/binman/etype/intel_ifwi.py b/tools/binman/etype/intel_ifwi.py index 04fad40..6513b97 100644 --- a/tools/binman/etype/intel_ifwi.py +++ b/tools/binman/etype/intel_ifwi.py @@ -10,7 +10,7 @@ from collections import OrderedDict from binman.entry import Entry from binman.etype.blob_ext import Entry_blob_ext from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_intel_ifwi(Entry_blob_ext): """Intel Integrated Firmware Image (IFWI) file diff --git a/tools/binman/etype/mkimage.py b/tools/binman/etype/mkimage.py index 8a13d5e..e028c44 100644 --- a/tools/binman/etype/mkimage.py +++ b/tools/binman/etype/mkimage.py @@ -9,7 +9,7 @@ from collections import OrderedDict from binman.entry import Entry from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_mkimage(Entry): """Binary produced by mkimage diff --git a/tools/binman/etype/null.py b/tools/binman/etype/null.py index c10d482..263fb52 100644 --- a/tools/binman/etype/null.py +++ b/tools/binman/etype/null.py @@ -5,7 +5,7 @@ from binman.entry import Entry from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_null(Entry): """An entry which has no contents of its own diff --git a/tools/binman/etype/pre_load.py b/tools/binman/etype/pre_load.py index b622281..bd3545b 100644 --- a/tools/binman/etype/pre_load.py +++ b/tools/binman/etype/pre_load.py @@ -8,7 +8,7 @@ import os import struct from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools from binman.entry import Entry from binman.etype.collection import Entry_collection diff --git a/tools/binman/etype/section.py b/tools/binman/etype/section.py index 57b91ff..c36edd1 100644 --- a/tools/binman/etype/section.py +++ b/tools/binman/etype/section.py @@ -16,9 +16,9 @@ import sys from binman.entry import Entry from binman import state from dtoc import fdt_util -from patman import tools -from patman import tout -from patman.tools import to_hex_size +from u_boot_pylib import tools +from u_boot_pylib import tout +from u_boot_pylib.tools import to_hex_size class Entry_section(Entry): @@ -172,7 +172,7 @@ class Entry_section(Entry): def IsSpecialSubnode(self, node): """Check if a node is a special one used by the section itself - Some notes are used for hashing / signatures and do not add entries to + Some nodes are used for hashing / signatures and do not add entries to the actual section. Returns: @@ -397,10 +397,13 @@ class Entry_section(Entry): This excludes any padding. If the section is compressed, the compressed data is returned """ - data = self.BuildSectionData(required) - if data is None: - return None - self.SetContents(data) + if not self.build_done: + data = self.BuildSectionData(required) + if data is None: + return None + self.SetContents(data) + else: + data = self.data if self._filename: tools.write_file(tools.get_output_filename(self._filename), data) return data @@ -427,8 +430,11 @@ class Entry_section(Entry): self._SortEntries() self._extend_entries() - data = self.BuildSectionData(True) - self.SetContents(data) + if self.build_done: + self.size = None + else: + data = self.BuildSectionData(True) + self.SetContents(data) self.CheckSize() @@ -810,6 +816,9 @@ class Entry_section(Entry): def LoadData(self, decomp=True): for entry in self._entries.values(): entry.LoadData(decomp) + data = self.ReadData(decomp) + self.contents_size = len(data) + self.ProcessContentsUpdate(data) self.Detail('Loaded data') def GetImage(self): @@ -866,10 +875,15 @@ class Entry_section(Entry): return data def WriteData(self, data, decomp=True): - self.Raise("Replacing sections is not implemented yet") + ok = super().WriteData(data, decomp) + + # The section contents are now fixed and cannot be rebuilt from the + # containing entries. + self.mark_build_done() + return ok def WriteChildData(self, child): - return True + return super().WriteChildData(child) def SetAllowMissing(self, allow_missing): """Set whether a section allows missing external blobs @@ -885,7 +899,7 @@ class Entry_section(Entry): """Set whether a section allows to create a fake blob Args: - allow_fake_blob: True if allowed, False if not allowed + allow_fake: True if allowed, False if not allowed """ super().SetAllowFakeBlob(allow_fake) for entry in self._entries.values(): @@ -909,7 +923,7 @@ class Entry_section(Entry): If there are faked blobs, the entries are added to the list Args: - fake_blobs_list: List of Entry objects to be added to + faked_blobs_list: List of Entry objects to be added to """ for entry in self._entries.values(): entry.CheckFakedBlobs(faked_blobs_list) diff --git a/tools/binman/etype/text.py b/tools/binman/etype/text.py index c55e023..e4deb4a 100644 --- a/tools/binman/etype/text.py +++ b/tools/binman/etype/text.py @@ -7,7 +7,7 @@ from collections import OrderedDict from binman.entry import Entry, EntryArg from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_text(Entry): diff --git a/tools/binman/etype/u_boot_dtb_with_ucode.py b/tools/binman/etype/u_boot_dtb_with_ucode.py index 047d310..f7225ce 100644 --- a/tools/binman/etype/u_boot_dtb_with_ucode.py +++ b/tools/binman/etype/u_boot_dtb_with_ucode.py @@ -7,7 +7,7 @@ from binman.entry import Entry from binman.etype.blob_dtb import Entry_blob_dtb -from patman import tools +from u_boot_pylib import tools # This is imported if needed state = None diff --git a/tools/binman/etype/u_boot_elf.py b/tools/binman/etype/u_boot_elf.py index 3ec774f..f4d86aa 100644 --- a/tools/binman/etype/u_boot_elf.py +++ b/tools/binman/etype/u_boot_elf.py @@ -9,7 +9,7 @@ from binman.entry import Entry from binman.etype.blob import Entry_blob from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_elf(Entry_blob): """U-Boot ELF image diff --git a/tools/binman/etype/u_boot_env.py b/tools/binman/etype/u_boot_env.py index c38340b..c027e93 100644 --- a/tools/binman/etype/u_boot_env.py +++ b/tools/binman/etype/u_boot_env.py @@ -8,7 +8,7 @@ import zlib from binman.etype.blob import Entry_blob from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_env(Entry_blob): """An entry which contains a U-Boot environment diff --git a/tools/binman/etype/u_boot_spl_bss_pad.py b/tools/binman/etype/u_boot_spl_bss_pad.py index 680d198..1ffeb39 100644 --- a/tools/binman/etype/u_boot_spl_bss_pad.py +++ b/tools/binman/etype/u_boot_spl_bss_pad.py @@ -10,7 +10,7 @@ from binman import elf from binman.entry import Entry from binman.etype.blob import Entry_blob -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_spl_bss_pad(Entry_blob): """U-Boot SPL binary padded with a BSS region diff --git a/tools/binman/etype/u_boot_spl_expanded.py b/tools/binman/etype/u_boot_spl_expanded.py index 319f670..fcd0dd1 100644 --- a/tools/binman/etype/u_boot_spl_expanded.py +++ b/tools/binman/etype/u_boot_spl_expanded.py @@ -5,7 +5,7 @@ # Entry-type module for expanded U-Boot SPL binary # -from patman import tout +from u_boot_pylib import tout from binman import state from binman.etype.blob_phase import Entry_blob_phase diff --git a/tools/binman/etype/u_boot_tpl_bss_pad.py b/tools/binman/etype/u_boot_tpl_bss_pad.py index 47f4b23..29c6a95 100644 --- a/tools/binman/etype/u_boot_tpl_bss_pad.py +++ b/tools/binman/etype/u_boot_tpl_bss_pad.py @@ -10,7 +10,7 @@ from binman import elf from binman.entry import Entry from binman.etype.blob import Entry_blob -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_tpl_bss_pad(Entry_blob): """U-Boot TPL binary padded with a BSS region diff --git a/tools/binman/etype/u_boot_tpl_expanded.py b/tools/binman/etype/u_boot_tpl_expanded.py index 55fde3c..58db4f3 100644 --- a/tools/binman/etype/u_boot_tpl_expanded.py +++ b/tools/binman/etype/u_boot_tpl_expanded.py @@ -5,7 +5,7 @@ # Entry-type module for expanded U-Boot TPL binary # -from patman import tout +from u_boot_pylib import tout from binman import state from binman.etype.blob_phase import Entry_blob_phase diff --git a/tools/binman/etype/u_boot_tpl_with_ucode_ptr.py b/tools/binman/etype/u_boot_tpl_with_ucode_ptr.py index c7f3f9d..86f9578 100644 --- a/tools/binman/etype/u_boot_tpl_with_ucode_ptr.py +++ b/tools/binman/etype/u_boot_tpl_with_ucode_ptr.py @@ -7,11 +7,11 @@ import struct -from patman import command from binman.entry import Entry from binman.etype.blob import Entry_blob from binman.etype.u_boot_with_ucode_ptr import Entry_u_boot_with_ucode_ptr -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import tools class Entry_u_boot_tpl_with_ucode_ptr(Entry_u_boot_with_ucode_ptr): """U-Boot TPL with embedded microcode pointer diff --git a/tools/binman/etype/u_boot_ucode.py b/tools/binman/etype/u_boot_ucode.py index 6945411..97ed7d7 100644 --- a/tools/binman/etype/u_boot_ucode.py +++ b/tools/binman/etype/u_boot_ucode.py @@ -7,7 +7,7 @@ from binman.entry import Entry from binman.etype.blob import Entry_blob -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_ucode(Entry_blob): """U-Boot microcode block diff --git a/tools/binman/etype/u_boot_vpl_bss_pad.py b/tools/binman/etype/u_boot_vpl_bss_pad.py index b2ce2a3..bba38cc 100644 --- a/tools/binman/etype/u_boot_vpl_bss_pad.py +++ b/tools/binman/etype/u_boot_vpl_bss_pad.py @@ -10,7 +10,7 @@ from binman import elf from binman.entry import Entry from binman.etype.blob import Entry_blob -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_vpl_bss_pad(Entry_blob): """U-Boot VPL binary padded with a BSS region diff --git a/tools/binman/etype/u_boot_vpl_expanded.py b/tools/binman/etype/u_boot_vpl_expanded.py index 92c64f0..deff5a3 100644 --- a/tools/binman/etype/u_boot_vpl_expanded.py +++ b/tools/binman/etype/u_boot_vpl_expanded.py @@ -5,7 +5,7 @@ # Entry-type module for expanded U-Boot VPL binary # -from patman import tout +from u_boot_pylib import tout from binman import state from binman.etype.blob_phase import Entry_blob_phase diff --git a/tools/binman/etype/u_boot_with_ucode_ptr.py b/tools/binman/etype/u_boot_with_ucode_ptr.py index e275698..41731fd 100644 --- a/tools/binman/etype/u_boot_with_ucode_ptr.py +++ b/tools/binman/etype/u_boot_with_ucode_ptr.py @@ -11,8 +11,8 @@ from binman import elf from binman.entry import Entry from binman.etype.blob import Entry_blob from dtoc import fdt_util -from patman import tools -from patman import command +from u_boot_pylib import tools +from u_boot_pylib import command class Entry_u_boot_with_ucode_ptr(Entry_blob): """U-Boot with embedded microcode pointer diff --git a/tools/binman/etype/vblock.py b/tools/binman/etype/vblock.py index 04cb722..4adb9a4 100644 --- a/tools/binman/etype/vblock.py +++ b/tools/binman/etype/vblock.py @@ -13,7 +13,7 @@ from binman.entry import EntryArg from binman.etype.collection import Entry_collection from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_vblock(Entry_collection): """An entry which contains a Chromium OS verified boot block diff --git a/tools/binman/etype/x509_cert.py b/tools/binman/etype/x509_cert.py new file mode 100644 index 0000000..f80a6ec --- /dev/null +++ b/tools/binman/etype/x509_cert.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2023 Google LLC +# Written by Simon Glass <sjg@chromium.org> +# + +# Support for an X509 certificate, used to sign a set of entries + +from collections import OrderedDict +import os + +from binman.entry import EntryArg +from binman.etype.collection import Entry_collection + +from dtoc import fdt_util +from u_boot_pylib import tools + +class Entry_x509_cert(Entry_collection): + """An entry which contains an X509 certificate + + Properties / Entry arguments: + - content: List of phandles to entries to sign + + Output files: + - input.<unique_name> - input file passed to openssl + - cert.<unique_name> - output file generated by openssl (which is + used as the entry contents) + + openssl signs the provided data, writing the signature in this entry. This + allows verification that the data is genuine + """ + def __init__(self, section, etype, node): + super().__init__(section, etype, node) + self.openssl = None + + def ReadNode(self): + super().ReadNode() + self._cert_ca = fdt_util.GetString(self._node, 'cert-ca') + self._cert_rev = fdt_util.GetInt(self._node, 'cert-revision-int', 0) + self.key_fname = self.GetEntryArgsOrProps([ + EntryArg('keyfile', str)], required=True)[0] + + def GetCertificate(self, required): + """Get the contents of this entry + + Args: + required: True if the data must be present, False if it is OK to + return None + + Returns: + bytes content of the entry, which is the signed vblock for the + provided data + """ + # Join up the data files to be signed + input_data = self.GetContents(required) + if input_data is None: + return None + + uniq = self.GetUniqueName() + output_fname = tools.get_output_filename('cert.%s' % uniq) + input_fname = tools.get_output_filename('input.%s' % uniq) + config_fname = tools.get_output_filename('config.%s' % uniq) + tools.write_file(input_fname, input_data) + stdout = self.openssl.x509_cert( + cert_fname=output_fname, + input_fname=input_fname, + key_fname=self.key_fname, + cn=self._cert_ca, + revision=self._cert_rev, + config_fname=config_fname) + if stdout is not None: + data = tools.read_file(output_fname) + else: + # Bintool is missing; just use 4KB of zero data + self.record_missing_bintool(self.openssl) + data = tools.get_bytes(0, 4096) + return data + + def ObtainContents(self): + data = self.GetCertificate(False) + if data is None: + return False + self.SetContents(data) + return True + + def ProcessContents(self): + # The blob may have changed due to WriteSymbols() + data = self.GetCertificate(True) + return self.ProcessContentsUpdate(data) + + def AddBintools(self, btools): + super().AddBintools(btools) + self.openssl = self.AddBintool(btools, 'openssl') diff --git a/tools/binman/fdt_test.py b/tools/binman/fdt_test.py index 94347b1..7ef8729 100644 --- a/tools/binman/fdt_test.py +++ b/tools/binman/fdt_test.py @@ -12,7 +12,7 @@ import unittest from dtoc import fdt from dtoc import fdt_util from dtoc.fdt import FdtScan -from patman import tools +from u_boot_pylib import tools class TestFdt(unittest.TestCase): @classmethod diff --git a/tools/binman/fip_util.py b/tools/binman/fip_util.py index 95eee32..b5caab2 100755 --- a/tools/binman/fip_util.py +++ b/tools/binman/fip_util.py @@ -37,8 +37,8 @@ OUR_PATH = os.path.dirname(OUR_FILE) sys.path.insert(2, os.path.join(OUR_PATH, '..')) # pylint: disable=C0413 -from patman import command -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import tools # The TOC header, at the start of the FIP HEADER_FORMAT = '<IIQ' diff --git a/tools/binman/fip_util_test.py b/tools/binman/fip_util_test.py index cf6d000..56aa56f 100755 --- a/tools/binman/fip_util_test.py +++ b/tools/binman/fip_util_test.py @@ -20,10 +20,10 @@ OUR_PATH = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(2, os.path.join(OUR_PATH, '..')) # pylint: disable=C0413 -from patman import test_util -from patman import tools from binman import bintool from binman import fip_util +from u_boot_pylib import test_util +from u_boot_pylib import tools FIPTOOL = bintool.Bintool.create('fiptool') HAVE_FIPTOOL = FIPTOOL.is_present() diff --git a/tools/binman/fmap_util.py b/tools/binman/fmap_util.py index 1ce63d1..40f2dbf 100644 --- a/tools/binman/fmap_util.py +++ b/tools/binman/fmap_util.py @@ -10,7 +10,7 @@ import collections import struct import sys -from patman import tools +from u_boot_pylib import tools # constants imported from lib/fmap.h FMAP_SIGNATURE = b'__FMAP__' @@ -45,6 +45,9 @@ FMAP_AREA_NAMES = ( 'flags', ) +# Flags supported by areas (bits 2:0 are unused so not included here) +FMAP_AREA_PRESERVE = 1 << 3 # Preserved by any firmware updates + # These are the two data structures supported by flashrom, a header (which # appears once at the start) and an area (which is repeated until the end of # the list of areas) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index d74aa90..43b4f85 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -34,10 +34,10 @@ from dtoc import fdt_util from binman.etype import fdtmap from binman.etype import image_header from binman.image import Image -from patman import command -from patman import test_util -from patman import tools -from patman import tout +from u_boot_pylib import command +from u_boot_pylib import test_util +from u_boot_pylib import tools +from u_boot_pylib import tout # Contents of test files, corresponding to different entry types U_BOOT_DATA = b'1234' @@ -709,6 +709,14 @@ class TestFunctional(unittest.TestCase): AddNode(dtb.GetRoot(), '') return tree + def _CheckSign(self, fit, key): + try: + tools.run('fit_check_sign', '-k', key, '-f', fit) + except: + self.fail('Expected signed FIT container') + return False + return True + def testRun(self): """Test a basic run with valid args""" result = self._RunBinman('-h') @@ -1704,7 +1712,7 @@ class TestFunctional(unittest.TestCase): self.assertEqual(b'SECTION0', fentry.name) self.assertEqual(0, fentry.offset) self.assertEqual(16, fentry.size) - self.assertEqual(0, fentry.flags) + self.assertEqual(fmap_util.FMAP_AREA_PRESERVE, fentry.flags) fentry = next(fiter) self.assertEqual(b'RO_U_BOOT', fentry.name) @@ -1752,7 +1760,7 @@ class TestFunctional(unittest.TestCase): def _HandleGbbCommand(self, pipe_list): """Fake calls to the futility utility""" - if pipe_list[0][0] == 'futility': + if 'futility' in pipe_list[0][0]: fname = pipe_list[0][-1] # Append our GBB data to the file, which will happen every time the # futility command is called. @@ -1814,7 +1822,7 @@ class TestFunctional(unittest.TestCase): self._hash_data is False, it writes VBLOCK_DATA, else it writes a hash of the input data (here, 'input.vblock'). """ - if pipe_list[0][0] == 'futility': + if 'futility' in pipe_list[0][0]: fname = pipe_list[0][3] with open(fname, 'wb') as fd: if self._hash_data: @@ -4001,9 +4009,17 @@ class TestFunctional(unittest.TestCase): self.assertEqual(expected, data[image_pos:image_pos+size]) def testFitMissing(self): + """Test that binman complains if mkimage is missing""" + with self.assertRaises(ValueError) as e: + self._DoTestFile('162_fit_external.dts', + force_missing_bintools='mkimage') + self.assertIn("Node '/binman/fit': Missing tool: 'mkimage'", + str(e.exception)) + + def testFitMissingOK(self): """Test that binman still produces a FIT image if mkimage is missing""" with test_util.capture_sys_output() as (_, stderr): - self._DoTestFile('162_fit_external.dts', + self._DoTestFile('162_fit_external.dts', allow_missing=True, force_missing_bintools='mkimage') err = stderr.getvalue() self.assertRegex(err, "Image 'image'.*missing bintools.*: mkimage") @@ -5813,13 +5829,61 @@ fdt fdtmap Extract the devicetree blob from the fdtmap self.assertEqual(expected_fdtmap, fdtmap) def testReplaceSectionSimple(self): - """Test replacing a simple section with arbitrary data""" + """Test replacing a simple section with same-sized data""" new_data = b'w' * len(COMPRESS_DATA + U_BOOT_DATA) - with self.assertRaises(ValueError) as exc: - self._RunReplaceCmd('section', new_data, - dts='241_replace_section_simple.dts') + data, expected_fdtmap, image = self._RunReplaceCmd('section', + new_data, dts='241_replace_section_simple.dts') + self.assertEqual(new_data, data) + + entries = image.GetEntries() + self.assertIn('section', entries) + entry = entries['section'] + self.assertEqual(len(new_data), entry.size) + + def testReplaceSectionLarger(self): + """Test replacing a simple section with larger data""" + new_data = b'w' * (len(COMPRESS_DATA + U_BOOT_DATA) + 1) + data, expected_fdtmap, image = self._RunReplaceCmd('section', + new_data, dts='241_replace_section_simple.dts') + self.assertEqual(new_data, data) + + entries = image.GetEntries() + self.assertIn('section', entries) + entry = entries['section'] + self.assertEqual(len(new_data), entry.size) + fentry = entries['fdtmap'] + self.assertEqual(entry.offset + entry.size, fentry.offset) + + def testReplaceSectionSmaller(self): + """Test replacing a simple section with smaller data""" + new_data = b'w' * (len(COMPRESS_DATA + U_BOOT_DATA) - 1) + b'\0' + data, expected_fdtmap, image = self._RunReplaceCmd('section', + new_data, dts='241_replace_section_simple.dts') + self.assertEqual(new_data, data) + + # The new size is the same as the old, just with a pad byte at the end + entries = image.GetEntries() + self.assertIn('section', entries) + entry = entries['section'] + self.assertEqual(len(new_data), entry.size) + + def testReplaceSectionSmallerAllow(self): + """Test failing to replace a simple section with smaller data""" + new_data = b'w' * (len(COMPRESS_DATA + U_BOOT_DATA) - 1) + try: + state.SetAllowEntryContraction(True) + with self.assertRaises(ValueError) as exc: + self._RunReplaceCmd('section', new_data, + dts='241_replace_section_simple.dts') + finally: + state.SetAllowEntryContraction(False) + + # Since we have no information about the position of things within the + # section, we cannot adjust the position of /section-u-boot so it ends + # up outside the section self.assertIn( - "Node '/section': Replacing sections is not implemented yet", + "Node '/section/u-boot': Offset 0x24 (36) size 0x4 (4) is outside " + "the section '/section' starting at 0x0 (0) of size 0x27 (39)", str(exc.exception)) def testMkimageImagename(self): @@ -6355,10 +6419,11 @@ fdt fdtmap Extract the devicetree blob from the fdtmap 'tee-os-path': 'missing.bin', } test_subdir = os.path.join(self._indir, TEST_FDT_SUBDIR) - data = self._DoReadFileDtb( - '276_fit_firmware_loadables.dts', - entry_args=entry_args, - extra_indirs=[test_subdir])[0] + with test_util.capture_sys_output() as (stdout, stderr): + data = self._DoReadFileDtb( + '276_fit_firmware_loadables.dts', + entry_args=entry_args, + extra_indirs=[test_subdir])[0] dtb = fdt.Fdt.FromData(data) dtb.Scan() @@ -6388,6 +6453,128 @@ fdt fdtmap Extract the devicetree blob from the fdtmap self.assertEqual(['u-boot', 'atf-2'], fdt_util.GetStringList(node, 'loadables')) + def testTooldir(self): + """Test that we can specify the tooldir""" + with test_util.capture_sys_output() as (stdout, stderr): + self.assertEqual(0, self._DoBinman('--tooldir', 'fred', + 'tool', '-l')) + self.assertEqual('fred', bintool.Bintool.tooldir) + + # Check that the toolpath is updated correctly + self.assertEqual(['fred'], tools.tool_search_paths) + + # Try with a few toolpaths; the tooldir should be at the end + with test_util.capture_sys_output() as (stdout, stderr): + self.assertEqual(0, self._DoBinman( + '--toolpath', 'mary', '--toolpath', 'anna', '--tooldir', 'fred', + 'tool', '-l')) + self.assertEqual(['mary', 'anna', 'fred'], tools.tool_search_paths) + + def testReplaceSectionEntry(self): + """Test replacing an entry in a section""" + expect_data = b'w' * len(U_BOOT_DATA + COMPRESS_DATA) + entry_data, expected_fdtmap, image = self._RunReplaceCmd('section/blob', + expect_data, dts='241_replace_section_simple.dts') + self.assertEqual(expect_data, entry_data) + + entries = image.GetEntries() + self.assertIn('section', entries) + section = entries['section'] + + sect_entries = section.GetEntries() + self.assertIn('blob', sect_entries) + entry = sect_entries['blob'] + self.assertEqual(len(expect_data), entry.size) + + fname = tools.get_output_filename('image-updated.bin') + data = tools.read_file(fname) + + new_blob_data = data[entry.image_pos:entry.image_pos + len(expect_data)] + self.assertEqual(expect_data, new_blob_data) + + self.assertEqual(U_BOOT_DATA, + data[entry.image_pos + len(expect_data):] + [:len(U_BOOT_DATA)]) + + def testReplaceSectionDeep(self): + """Test replacing an entry in two levels of sections""" + expect_data = b'w' * len(U_BOOT_DATA + COMPRESS_DATA) + entry_data, expected_fdtmap, image = self._RunReplaceCmd( + 'section/section/blob', expect_data, + dts='278_replace_section_deep.dts') + self.assertEqual(expect_data, entry_data) + + entries = image.GetEntries() + self.assertIn('section', entries) + section = entries['section'] + + subentries = section.GetEntries() + self.assertIn('section', subentries) + section = subentries['section'] + + sect_entries = section.GetEntries() + self.assertIn('blob', sect_entries) + entry = sect_entries['blob'] + self.assertEqual(len(expect_data), entry.size) + + fname = tools.get_output_filename('image-updated.bin') + data = tools.read_file(fname) + + new_blob_data = data[entry.image_pos:entry.image_pos + len(expect_data)] + self.assertEqual(expect_data, new_blob_data) + + self.assertEqual(U_BOOT_DATA, + data[entry.image_pos + len(expect_data):] + [:len(U_BOOT_DATA)]) + + def testReplaceFitSibling(self): + """Test an image with a FIT inside where we replace its sibling""" + fname = TestFunctional._MakeInputFile('once', b'available once') + self._DoReadFileRealDtb('277_replace_fit_sibling.dts') + os.remove(fname) + + try: + tmpdir, updated_fname = self._SetupImageInTmpdir() + + fname = os.path.join(tmpdir, 'update-blob') + expected = b'w' * (len(COMPRESS_DATA + U_BOOT_DATA) + 1) + tools.write_file(fname, expected) + + self._DoBinman('replace', '-i', updated_fname, 'blob', '-f', fname) + data = tools.read_file(updated_fname) + start = len(U_BOOT_DTB_DATA) + self.assertEqual(expected, data[start:start + len(expected)]) + map_fname = os.path.join(tmpdir, 'image-updated.map') + self.assertFalse(os.path.exists(map_fname)) + finally: + shutil.rmtree(tmpdir) + + def testX509Cert(self): + """Test creating an X509 certificate""" + keyfile = self.TestFile('key.key') + entry_args = { + 'keyfile': keyfile, + } + data = self._DoReadFileDtb('279_x509_cert.dts', + entry_args=entry_args)[0] + cert = data[:-4] + self.assertEqual(U_BOOT_DATA, data[-4:]) + + # TODO: verify the signature + + def testX509CertMissing(self): + """Test that binman still produces an image if openssl is missing""" + keyfile = self.TestFile('key.key') + entry_args = { + 'keyfile': 'keyfile', + } + with test_util.capture_sys_output() as (_, stderr): + self._DoTestFile('279_x509_cert.dts', + force_missing_bintools='openssl', + entry_args=entry_args) + err = stderr.getvalue() + self.assertRegex(err, "Image 'image'.*missing bintools.*: openssl") + def testPackRockchipTpl(self): """Test that an image with a Rockchip TPL binary can be created""" data = self._DoReadFile('277_rockchip_tpl.dts') @@ -6404,6 +6591,91 @@ fdt fdtmap Extract the devicetree blob from the fdtmap self._DoTestFile('278_mkimage_missing_multiple.dts', allow_missing=False) self.assertIn("not found in input path", str(e.exception)) + def _PrepareSignEnv(self, dts='280_fit_sign.dts'): + """Prepare sign environment + + Create private and public keys, add pubkey into dtb. + + Returns: + Tuple: + FIT container + Image name + Private key + DTB + """ + + data = self._DoReadFileRealDtb(dts) + updated_fname = tools.get_output_filename('image-updated.bin') + tools.write_file(updated_fname, data) + dtb = tools.get_output_filename('source.dtb') + private_key = tools.get_output_filename('test_key.key') + public_key = tools.get_output_filename('test_key.crt') + fit = tools.get_output_filename('fit.fit') + key_dir = tools.get_output_dir() + + tools.run('openssl', 'req', '-batch' , '-newkey', 'rsa:4096', + '-sha256', '-new', '-nodes', '-x509', '-keyout', + private_key, '-out', public_key) + tools.run('fdt_add_pubkey', '-a', 'sha256,rsa4096', '-k', key_dir, + '-n', 'test_key', '-r', 'conf', dtb) + + return fit, updated_fname, private_key, dtb + + def testSignSimple(self): + """Test that a FIT container can be signed in image""" + is_signed = False + fit, fname, private_key, dtb = self._PrepareSignEnv() + + # do sign with private key + control.SignEntries(fname, None, private_key, 'sha256,rsa4096', + ['fit']) + is_signed = self._CheckSign(fit, dtb) + + self.assertEqual(is_signed, True) + + def testSignExactFIT(self): + """Test that a FIT container can be signed and replaced in image""" + is_signed = False + fit, fname, private_key, dtb = self._PrepareSignEnv() + + # Make sure we propagate the toolpath, since mkimage may not be on PATH + args = [] + if self.toolpath: + for path in self.toolpath: + args += ['--toolpath', path] + + # do sign with private key + self._DoBinman(*args, 'sign', '-i', fname, '-k', private_key, '-a', + 'sha256,rsa4096', '-f', fit, 'fit') + is_signed = self._CheckSign(fit, dtb) + + self.assertEqual(is_signed, True) + + def testSignNonFit(self): + """Test a non-FIT entry cannot be signed""" + is_signed = False + fit, fname, private_key, _ = self._PrepareSignEnv( + '281_sign_non_fit.dts') + + # do sign with private key + with self.assertRaises(ValueError) as e: + self._DoBinman('sign', '-i', fname, '-k', private_key, '-a', + 'sha256,rsa4096', '-f', fit, 'u-boot') + self.assertIn( + "Node '/u-boot': Updating signatures is not supported with this entry type", + str(e.exception)) + + def testSignMissingMkimage(self): + """Test that FIT signing handles a missing mkimage tool""" + fit, fname, private_key, _ = self._PrepareSignEnv() + + # try to sign with a missing mkimage tool + bintool.Bintool.set_missing_list(['mkimage']) + with self.assertRaises(ValueError) as e: + control.SignEntries(fname, None, private_key, 'sha256,rsa4096', + ['fit']) + self.assertIn("Node '/fit': Missing tool: 'mkimage'", str(e.exception)) + if __name__ == "__main__": unittest.main() diff --git a/tools/binman/image.py b/tools/binman/image.py index 9415963..8ebf71d 100644 --- a/tools/binman/image.py +++ b/tools/binman/image.py @@ -18,8 +18,8 @@ from binman.etype import image_header from binman.etype import section from dtoc import fdt from dtoc import fdt_util -from patman import tools -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib import tout class Image(section.Entry_section): """A Image, representing an output from binman diff --git a/tools/binman/image_test.py b/tools/binman/image_test.py index e351fa8..bd51c1e 100644 --- a/tools/binman/image_test.py +++ b/tools/binman/image_test.py @@ -7,7 +7,7 @@ import unittest from binman.image import Image -from patman.test_util import capture_sys_output +from u_boot_pylib.test_util import capture_sys_output class TestImage(unittest.TestCase): def testInvalidFormat(self): diff --git a/tools/binman/main.py b/tools/binman/main.py index 14432a8..92d2431 100755 --- a/tools/binman/main.py +++ b/tools/binman/main.py @@ -34,7 +34,7 @@ sys.pycache_prefix = os.path.relpath(our_path, srctree) sys.path.insert(2, our1_path) from binman import bintool -from patman import test_util +from u_boot_pylib import test_util # Bring in the libfdt module sys.path.insert(2, 'scripts/dtc/pylibfdt') @@ -44,7 +44,7 @@ sys.path.insert(2, os.path.join(srctree, 'build-sandbox_spl/scripts/dtc/pylibfdt from binman import cmdline from binman import control -from patman import test_util +from u_boot_pylib import test_util def RunTests(debug, verbosity, processes, test_preserve_dirs, args, toolpath): """Run the functional tests and any embedded doctests @@ -85,7 +85,7 @@ def RunTests(debug, verbosity, processes, test_preserve_dirs, args, toolpath): return (0 if result.wasSuccessful() else 1) -def RunTestCoverage(toolpath): +def RunTestCoverage(toolpath, build_dir): """Run the tests and check that we get 100% coverage""" glob_list = control.GetEntryModules(False) all_set = set([os.path.splitext(os.path.basename(item))[0] @@ -95,8 +95,9 @@ def RunTestCoverage(toolpath): for path in toolpath: extra_args += ' --toolpath %s' % path test_util.run_test_coverage('tools/binman/binman', None, - ['*test*', '*main.py', 'tools/patman/*', 'tools/dtoc/*'], - args.build_dir, all_set, extra_args or None) + ['*test*', '*main.py', 'tools/patman/*', 'tools/dtoc/*', + 'tools/u_boot_pylib/*'], + build_dir, all_set, extra_args or None) def RunBinman(args): """Main entry point to binman once arguments are parsed @@ -116,7 +117,7 @@ def RunBinman(args): if args.cmd == 'test': if args.test_coverage: - RunTestCoverage(args.toolpath) + RunTestCoverage(args.toolpath, args.build_dir) else: ret_code = RunTests(args.debug, args.verbosity, args.processes, args.test_preserve_dirs, args.tests, @@ -140,8 +141,12 @@ def RunBinman(args): return ret_code -if __name__ == "__main__": +def start_binman(): args = cmdline.ParseArgs(sys.argv[1:]) ret_code = RunBinman(args) sys.exit(ret_code) + + +if __name__ == "__main__": + start_binman() diff --git a/tools/binman/missing-blob-help b/tools/binman/missing-blob-help index f3a44d0..f013367 100644 --- a/tools/binman/missing-blob-help +++ b/tools/binman/missing-blob-help @@ -21,13 +21,15 @@ Please read the section on SCP firmware in board/sunxi/README.sunxi64 iot2050-seboot: See the documentation for IOT2050 board. Your image is missing SEBoot which is mandatory for board startup. Prebuilt SEBoot located at -meta-iot2050/tree/master/recipes-bsp/u-boot/files/prebuild/tiboot3.bin. - -iot2050-sysfw: -See the documentation for IOT2050 board. Your image is missing system -firmware which is mandatory for board startup. Prebuilt system firmware -located at meta-iot2050/tree/master/recipes-bsp/u-boot/files/prebuild/ -with sysfw prefix. +meta-iot2050/tree/master/recipes-bsp/u-boot/files/prebuild/seboot_pg*.bin. + +iot2050-otpcmd: +See the documentation for IOT2050 board. Your image is missing OTP command data +block which is used for provisioning the customer keys to the board. +Please refer to +meta-iot2050/tree/master/recipes-bsp/secure-boot-otp-provisioning/files/make-otpcmd.sh +for how to generate this binary. If you are not using secure boot or do not +intend to provision the keys, disable CONFIG_IOT2050_EMBED_OTPCMD. k3-rti-wdt-firmware: If CONFIG_WDT_K3_RTI_LOAD_FW is enabled, a firmware image is needed for diff --git a/tools/binman/pyproject.toml b/tools/binman/pyproject.toml new file mode 100644 index 0000000..b4b54fb --- /dev/null +++ b/tools/binman/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "binary-manager" +version = "0.0.2" +authors = [ + { name="Simon Glass", email="sjg@chromium.org" }, +] +dependencies = ["pylibfdt", "u_boot_pylib", "dtoc"] +description = "Binman firmware-packaging tool" +readme = "README.rst" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://u-boot.readthedocs.io/en/latest/develop/package/index.html" +"Bug Tracker" = "https://source.denx.de/groups/u-boot/-/issues" + +[project.scripts] +binman = "binman.main:start_binman" + +[tool.setuptools.package-data] +patman = ["*.rst"] diff --git a/tools/binman/state.py b/tools/binman/state.py index 56e5bf8..3e78cf3 100644 --- a/tools/binman/state.py +++ b/tools/binman/state.py @@ -13,8 +13,8 @@ import threading from dtoc import fdt import os -from patman import tools -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib import tout OUR_PATH = os.path.dirname(os.path.realpath(__file__)) @@ -306,7 +306,7 @@ def GetUpdateNodes(node, for_repack=False): """Yield all the nodes that need to be updated in all device trees The property referenced by this node is added to any device trees which - have the given node. Due to removable of unwanted notes, SPL and TPL may + have the given node. Due to removable of unwanted nodes, SPL and TPL may not have this node. Args: diff --git a/tools/binman/test/067_fmap.dts b/tools/binman/test/067_fmap.dts index 9c0e293..24fa635 100644 --- a/tools/binman/test/067_fmap.dts +++ b/tools/binman/test/067_fmap.dts @@ -11,6 +11,7 @@ name-prefix = "ro-"; size = <0x10>; pad-byte = <0x21>; + preserve; u-boot { }; diff --git a/tools/binman/test/277_replace_fit_sibling.dts b/tools/binman/test/277_replace_fit_sibling.dts new file mode 100644 index 0000000..fc941a8 --- /dev/null +++ b/tools/binman/test/277_replace_fit_sibling.dts @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0+ +/dts-v1/; + +/ { + binman { + allow-repack; + + u-boot { + }; + + blob { + filename = "compress"; + }; + + fit { + description = "test-desc"; + #address-cells = <1>; + + images { + kernel { + description = "Vanilla Linux kernel"; + type = "kernel"; + arch = "ppc"; + os = "linux"; + compression = "gzip"; + load = <00000000>; + entry = <00000000>; + hash-1 { + algo = "crc32"; + }; + blob-ext { + filename = "once"; + }; + }; + fdt-1 { + description = "Flattened Device Tree blob"; + type = "flat_dt"; + arch = "ppc"; + compression = "none"; + hash-1 { + algo = "crc32"; + }; + u-boot-spl-dtb { + }; + }; + }; + + configurations { + default = "conf-1"; + conf-1 { + description = "Boot Linux kernel with FDT blob"; + kernel = "kernel"; + fdt = "fdt-1"; + }; + }; + }; + + fdtmap { + }; + }; +}; diff --git a/tools/binman/test/278_replace_section_deep.dts b/tools/binman/test/278_replace_section_deep.dts new file mode 100644 index 0000000..fba2d7d --- /dev/null +++ b/tools/binman/test/278_replace_section_deep.dts @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0+ +/dts-v1/; + +/ { + binman { + allow-repack; + + u-boot-dtb { + }; + + section { + section { + blob { + filename = "compress"; + }; + }; + + u-boot { + }; + }; + + fdtmap { + }; + }; +}; diff --git a/tools/binman/test/279_x509_cert.dts b/tools/binman/test/279_x509_cert.dts new file mode 100644 index 0000000..7123817 --- /dev/null +++ b/tools/binman/test/279_x509_cert.dts @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + x509-cert { + cert-ca = "IOT2050 Firmware Signature"; + cert-revision-int = <0>; + content = <&u_boot>; + }; + + u_boot: u-boot { + }; + }; +}; diff --git a/tools/binman/test/280_fit_sign.dts b/tools/binman/test/280_fit_sign.dts new file mode 100644 index 0000000..b9f17dc --- /dev/null +++ b/tools/binman/test/280_fit_sign.dts @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + size = <0x100000>; + allow-repack; + + fit { + description = "U-Boot"; + offset = <0x10000>; + images { + u-boot-1 { + description = "U-Boot"; + type = "standalone"; + arch = "arm64"; + os = "u-boot"; + compression = "none"; + hash-1 { + algo = "sha256"; + }; + u-boot { + }; + }; + + fdt-1 { + description = "test.dtb"; + type = "flat_dt"; + arch = "arm64"; + compression = "none"; + hash-1 { + algo = "sha256"; + }; + u-boot-spl-dtb { + }; + }; + + }; + + configurations { + default = "conf-1"; + conf-1 { + description = "u-boot with fdt"; + firmware = "u-boot-1"; + fdt = "fdt-1"; + signature-1 { + algo = "sha256,rsa4096"; + key-name-hint = "test_key"; + sign-images = "firmware", "fdt"; + }; + + }; + }; + }; + + fdtmap { + }; + }; +}; diff --git a/tools/binman/test/281_sign_non_fit.dts b/tools/binman/test/281_sign_non_fit.dts new file mode 100644 index 0000000..e16c954 --- /dev/null +++ b/tools/binman/test/281_sign_non_fit.dts @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + size = <0x100000>; + allow-repack; + + u-boot { + }; + fit { + description = "U-Boot"; + offset = <0x10000>; + images { + u-boot-1 { + description = "U-Boot"; + type = "standalone"; + arch = "arm64"; + os = "u-boot"; + compression = "none"; + hash-1 { + algo = "sha256"; + }; + u-boot { + }; + }; + + fdt-1 { + description = "test.dtb"; + type = "flat_dt"; + arch = "arm64"; + compression = "none"; + hash-1 { + algo = "sha256"; + }; + u-boot-spl-dtb { + }; + }; + + }; + + configurations { + default = "conf-1"; + conf-1 { + description = "u-boot with fdt"; + firmware = "u-boot-1"; + fdt = "fdt-1"; + signature-1 { + algo = "sha256,rsa4096"; + key-name-hint = "test_key"; + sign-images = "firmware", "fdt"; + }; + + }; + }; + }; + + fdtmap { + }; + }; +}; diff --git a/tools/binman/test/key.key b/tools/binman/test/key.key new file mode 100644 index 0000000..9de3be1 --- /dev/null +++ b/tools/binman/test/key.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCSDLMHq1Jw3U+G +H2wutSGrT4Xhs5Yy7uhR/rDOiuKTW3zkVdfSIliye3Nnwrl/nNUFzEJ+4t/AiDaJ +Qk5KddTAJnOkw5SYBvFsTDhMR4HH6AyfzaaVl+AAGOg4LXwZzGYKncgOY5u6ZyMB +SzHxozJmmoqYaCIi4Iv2VZRZw1YPBoT6sv38RQSET5ci/g+89Sfb85ZPHPu6PLlz +ZTufG+yzAhIDsIvNpt2YlCnQ1TqoZxXsztxN1bKIP68xvlAQHSAB8+x4y0tYPE1I +UT1DK22FMgz5iyBp6ksFaqI06fITtJjPKG13z8sXXgb4/rJ5I0lhsn1ySsHQ0zLw +/CX4La2/VMA0Bw6GLFRhu/rOycqKfmwLm25bExV8xL6lwFohxbzBQgYr93ujGFyQ +AXBDOphvZcdXP3CHAcEViVRjrsBWNz8wyf7X8h2FIU16kAd30WuspjmnGuvRZ6Gn +SNDVO2tbEKvwkg6liYWy4IXtWcvooMtkhYyvFudcxRPgxEUTQ00biYfJ59ukqD7I +hyT7pq1bZDCVnAt6dUUPWZutrbBacsyITs01hyiPxvAAQ7XRoInmW1DLqHZ+gCJU +YJ0TaiAI8AmnjypMWRUo19l0zIgPdva8EJ+mz+kKFsZszo1nwuxQL7oUSUCb0hfB +2k3WxNthBi3QpspUKPtKweIg9ITtIwIDAQABAoICAA9dS6ZGZTVfauLKwnhFcOXT +R1vfpzDzhjg+CX6pCL4E1WY2C67dEySvrQvg5d/hcV2bR/GOT4izK72T3qWhsMCI +KwlN0/+MV3CTsiaALUyJAm77VQeOwy9vb1qdml0ibie2wpmU7AiXmgykSvxHNWGq +52KyLckqgz7mcOVikdah0nKHSwXzgs6iit1RCfnQdqGChjELdQX6Jm5X24ZZCzUn +xhpiQ8reP5iyGZYRIIsf0SQo/O8pSI9h173tbgHL9paOATYR+Pqu2Vh+x2meE3b8 +NXY5Jy9NSRgoSCk15VQiXyMH90Av+YcbSrN+I+tvhWREQUM5Txt3ZHgKprntoEYE +XLHAr9cvmIzLNeApt2z/g4t80xFBpIvTG3+SV/rthmq0KGCLW2kPkdujOiIwdNFF +6fJ6ikphKAbx2NgUY+6AM5AoOh5QPMqvCdsPwO21YG1WoxmiUpNTaYMlR1fDofr/ +A/z2bFH4SiJPkHXRT2KBiJh4ZZWNzP6hOqGy+jreOpWh5IAyn7cKx6t3I28Q9df0 +tK/1PLgR8WWu6G4uHtF5lKL+LgqFCTbSu9JtLQVQntD7Qyd98sF5o23QQWyA19uU +TVGxtkVaP1y7v+gtC+xMTW9MbGIeJiqMZuZ3xXJVvUNg1/2BDd+VAfPCOq6xGHC7 +s9MFqwUsLCAFFebXC8oVAoIBAQDKGc/o21Ags2t61IJaJjU7YwrsRywhZR+vUz5F +xtqH4jt9AkdWpDkKbO7xNMQ2OFdnobq5mkM+iW6Jvc1fi4gm1HDyP296nPKZdFrJ +UgGfTxOhxFLp7gsJ2F0GX5eDJYvqUTBeYB3wrQkCc+t7fLg2oS+gKGIIn2CP07Mx +Bist3eCcDvL6QIxYS43u+ptTyAItyUYn8KwvCxlIEfjxowsxfhRWuU+Mr4A4hfGB +64xSI1YU1AYZLMucOtK/mmlscfO8isdcyfea0GJn4VLRnNvAKL5g627IdErWHs3u +KgYWAXtVKzHrf4hO8dpVgIzO69wAsqZEvKYGmTJhfyvBN9DdAoIBAQC5AA7s2XOX +raVymhPwEy4I/2w9NuMFmTavOREBp/gA9uaWBdqAWn1rRJiJ5plgdcnOBFPSGBnc +thkuWBRqkklQ0YPKhNBT48CZGBN7VUsvyTZD1+IXLW1TmY5UGT0p6/dAYkoJHnvX +TAHl1tfmeHxVCJWV6Shf5LfJJwsAiykxzetkzmeaycy2s9GKCnkc2uFxyhKnfM0/ +SLwTuXQIJvHuErTYA4jjVOG9EGYW2/uKScPBLpB1YTliAUIvByDy6suCN5pVZGT8 +xVLTYec9lXjhfyhysOAjhD3w77Jh7Exft91fEK50k2ZkqYYnh+mYZcnR52msVSBS +3YL9kK/9dNX/AoIBABcEaZFzqOSQiqUqns31nApvdUcDtBr5kWo+aNE5nJntQiky +oT1U5soxLeV6xP4H3KyI1uNcllwA+v3lCAbhtVf2ygZNAz1LsrWXct+K33RtZSb/ +XRIXclpksfOP34moNQ8yv/d/qulGS8hju2YNBk3yfaIX91JUFINM8ROcSD6pDnO3 +oCSwRUupDzkwgZBBLz5Xtg3Gc1XIRdDXeyrKDvRMD7Tw1gaH1mqZlq/dS9XvAFbO +7wLe/zGD4YzA4VDgiYnnpF0FA5Y2NX7vQqds3fo8qbIQHkXmOL+6Mmn1j0viT1Gb +4cuYcsXK9brXMTI/2oaZ0iXx9la6C+reuPUAjmECggEBAInEvlips0hgW1ZV4cUm +M2El/dA0YKoZqDyjDcQi9zCYra1JXKe7O603XzVK0iugbBGM7XMG2bOgtG3r0ABx +QkH6VN/rOk1OzW31HQT6xswmVs/9I/TIsqLQNsrwJLlkbTO4PpQ97FGv27Xy4cNT +NJwKkYMbKCMJa8hT2ACmoZ3iUIs4nrUJ1Pa2QLRBCmJvqfYYWv35lcur+cvijsNH +ZWE68wvuzfEllBo87RnW5qLcPfhOGewf5CDU+RmWgHYGXllx2PAAnKgUtpKOVStq +daPQEyoeCDzKzWnwxvHfjBy4CxYxkQllf5o1GJ+1ukLwgnRbljltB25OYa89IaJp +cLcCggEAa5vbegzMKYPjR3zcVjnvhRsLXQi1vMtbUqOQ5wYMwGIef4v3QHNoF7EA +aNpWQ/qgCTQUzl3qoQCkRiVmVBBr60Fs5y7sfA92eBxQIV5hxJftH3vmiKqeWeqm +ila9DNw84MNAIqI2u6R3K/ur9fkSswDr3nzvFjuheW5V/M/6zAUtJZXr4iUih929 +uhf2dn6pSLR+epJ5023CVaI2zwz+U6PDEATKy9HjeKab3tQMHxQkT/5IWcLqrVTs +0rMobIgONzQqYDi2sO05YvgNBxvX3pUvqNlthcOtauT8BoE6wxLYm7ZcWYLPn15A +wR0+2mDpx+HDyu76q3M+KxXG2U8sJg== +-----END PRIVATE KEY----- diff --git a/tools/binman/test/key.pem b/tools/binman/test/key.pem new file mode 100644 index 0000000..7a7b84a --- /dev/null +++ b/tools/binman/test/key.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFcTCCA1kCFB/17qhcvpyKhG+jfS2c0qG1yjruMA0GCSqGSIb3DQEBCwUAMHUx +CzAJBgNVBAYTAk5aMRMwEQYDVQQIDApDYW50ZXJidXJ5MRUwEwYDVQQHDAxDaHJp +c3RjaHVyY2gxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEXMBUG +A1UEAwwOTXkgQ29tbW9uIE5hbWUwHhcNMjMwMjEzMDM1MzMzWhcNMjQwMjEzMDM1 +MzMzWjB1MQswCQYDVQQGEwJOWjETMBEGA1UECAwKQ2FudGVyYnVyeTEVMBMGA1UE +BwwMQ2hyaXN0Y2h1cmNoMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBM +dGQxFzAVBgNVBAMMDk15IENvbW1vbiBOYW1lMIICIjANBgkqhkiG9w0BAQEFAAOC +Ag8AMIICCgKCAgEAkgyzB6tScN1Phh9sLrUhq0+F4bOWMu7oUf6wzorik1t85FXX +0iJYsntzZ8K5f5zVBcxCfuLfwIg2iUJOSnXUwCZzpMOUmAbxbEw4TEeBx+gMn82m +lZfgABjoOC18GcxmCp3IDmObumcjAUsx8aMyZpqKmGgiIuCL9lWUWcNWDwaE+rL9 +/EUEhE+XIv4PvPUn2/OWTxz7ujy5c2U7nxvsswISA7CLzabdmJQp0NU6qGcV7M7c +TdWyiD+vMb5QEB0gAfPseMtLWDxNSFE9QytthTIM+YsgaepLBWqiNOnyE7SYzyht +d8/LF14G+P6yeSNJYbJ9ckrB0NMy8Pwl+C2tv1TANAcOhixUYbv6zsnKin5sC5tu +WxMVfMS+pcBaIcW8wUIGK/d7oxhckAFwQzqYb2XHVz9whwHBFYlUY67AVjc/MMn+ +1/IdhSFNepAHd9FrrKY5pxrr0Wehp0jQ1TtrWxCr8JIOpYmFsuCF7VnL6KDLZIWM +rxbnXMUT4MRFE0NNG4mHyefbpKg+yIck+6atW2QwlZwLenVFD1mbra2wWnLMiE7N +NYcoj8bwAEO10aCJ5ltQy6h2foAiVGCdE2ogCPAJp48qTFkVKNfZdMyID3b2vBCf +ps/pChbGbM6NZ8LsUC+6FElAm9IXwdpN1sTbYQYt0KbKVCj7SsHiIPSE7SMCAwEA +ATANBgkqhkiG9w0BAQsFAAOCAgEAJAJoia6Vq4vXP/0bCgW3o9TOMmFYhI/xPxoh +Gd7was9R7BOrMGO+/3E7DZtjycZYL0r9nOtr9S/BBreuZ4vkk/PSoGaSnG8ST4jC +Ajk7ew/32RGOgA/oIzgKj1SPkBtvW+x+76sjUkGKsxmABBUhycIY7K0U8McTTfJ7 +gJ164VXmdG7qFMWmRy4Ry9QGXkDsbMSOZ485X7zbphjK5OZXEujP7GMUgg1lP479 +NqC1g+1m/A3PIB767lVYA7APQsrckHdRqOTkK9TYRQ3mvyE2wruhqE6lx8G/UyFh +RZjZ3lh2bx07UWIlyMabnGDMrM4FCnesqVyVAc8VAbkdXkeJI9r6DdFw+dzIY0P1 +il+MlYpZNwRyNv2W5SCPilyuhuPOSrSnsSHx64puCIvwG/4xA30Jw8nviJuyGSef +7uE+W7SD9E/hQHi/S9KRsYVoo7a6X9ADiwNsRNzVnuqc7K3mv/C5E9s6uFTNoObe +fUBA7pL3Fmvc5pYatxTFI85ajBpe/la6AA+7HX/8PXEphmp6GhFCcfsq+DL03vTM +DqIJL1i/JXggwqvvdcfaSeMDIOIzO89yUGGwwuj9rqMeEY99qDtljgy1EljjrB5i +0j4Jg4O0OEd2KIOD7nz4do1tLNlRcpysDZeXIiwAI7Dd3wWMsgpOQxs0zqWyqDVq +mCKa5Tw= +-----END CERTIFICATE----- diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index c2a6902..d81752e 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -19,10 +19,10 @@ import time from buildman import builderthread from buildman import toolchain -from patman import command from patman import gitutil -from patman import terminal -from patman.terminal import tprint +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib.terminal import tprint # This indicates an new int or hex Kconfig property with no default # It hangs the build since the 'conf' tool cannot proceed without valid input. @@ -194,6 +194,8 @@ class Builder: work_in_output: Use the output directory as the work directory and don't write to a separate output directory. thread_exceptions: List of exceptions raised by thread jobs + no_lto (bool): True to set the NO_LTO flag when building + reproducible_builds (bool): True to set SOURCE_DATE_EPOCH=0 for builds Private members: _base_board_dict: Last-summarised Dict of boards @@ -253,7 +255,7 @@ class Builder: config_only=False, squash_config_y=False, warnings_as_errors=False, work_in_output=False, test_thread_exceptions=False, adjust_cfg=None, - allow_missing=False): + allow_missing=False, no_lto=False, reproducible_builds=False): """Create a new Builder object Args: @@ -292,6 +294,7 @@ class Builder: C=val to set the value of C (val must have quotes if C is a string Kconfig allow_missing: Run build with BINMAN_ALLOW_MISSING=1 + no_lto (bool): True to set the NO_LTO flag when building """ self.toolchains = toolchains @@ -331,6 +334,8 @@ class Builder: self.adjust_cfg = adjust_cfg self.allow_missing = allow_missing self._ide = False + self.no_lto = no_lto + self.reproducible_builds = reproducible_builds if not self.squash_config_y: self.config_filenames += EXTRA_CONFIG_FILENAMES diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index 680efae..879ff138 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -10,8 +10,8 @@ import sys import threading from buildman import cfgutil -from patman import command from patman import gitutil +from u_boot_pylib import command RETURN_CODE_RETRY = -1 BASE_ELF_FILENAMES = ['u-boot', 'spl/u-boot-spl', 'tpl/u-boot-tpl'] @@ -255,6 +255,10 @@ class BuilderThread(threading.Thread): args.append('KCFLAGS=-Werror') if self.builder.allow_missing: args.append('BINMAN_ALLOW_MISSING=1') + if self.builder.no_lto: + args.append('NO_LTO=1') + if self.builder.reproducible_builds: + args.append('SOURCE_DATE_EPOCH=0') config_args = ['%s_defconfig' % brd.target] config_out = '' args.extend(self.builder.toolchains.GetMakeArguments(brd)) @@ -273,14 +277,19 @@ class BuilderThread(threading.Thread): # If we need to reconfigure, do that now cfg_file = os.path.join(out_dir, '.config') + cmd_list = [] if do_config or adjust_cfg: config_out = '' if self.mrproper: result = self.Make(commit, brd, 'mrproper', cwd, 'mrproper', *args, env=env) config_out += result.combined + cmd_list.append([self.builder.gnu_make, 'mrproper', + *args]) result = self.Make(commit, brd, 'config', cwd, *(args + config_args), env=env) + cmd_list.append([self.builder.gnu_make] + args + + config_args) config_out += result.combined do_config = False # No need to configure next time if adjust_cfg: @@ -290,6 +299,7 @@ class BuilderThread(threading.Thread): args.append('cfg') result = self.Make(commit, brd, 'build', cwd, *args, env=env) + cmd_list.append([self.builder.gnu_make] + args) if (result.return_code == 2 and ('Some images are invalid' in result.stderr)): # This is handled later by the check for output in @@ -303,6 +313,7 @@ class BuilderThread(threading.Thread): result.stderr = result.stderr.replace(src_dir + '/', '') if self.builder.verbose_build: result.stdout = config_out + result.stdout + result.cmd_list = cmd_list else: result.return_code = 1 result.stderr = 'No tool chain for %s\n' % brd.arch @@ -378,6 +389,12 @@ class BuilderThread(threading.Thread): with open(os.path.join(build_dir, 'out-env'), 'wb') as fd: for var in sorted(env.keys()): fd.write(b'%s="%s"' % (var, env[var])) + + with open(os.path.join(build_dir, 'out-cmd'), 'w', + encoding='utf-8') as fd: + for cmd in result.cmd_list: + print(' '.join(cmd), file=fd) + lines = [] for fname in BASE_ELF_FILENAMES: cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst index 2a83cb7..c8b0db3 100644 --- a/tools/buildman/buildman.rst +++ b/tools/buildman/buildman.rst @@ -1023,14 +1023,15 @@ U-Boot's build system embeds information such as a build timestamp into the final binary. This information varies each time U-Boot is built. This causes various files to be rebuilt even if no source changes are made, which in turn requires that the final U-Boot binary be re-linked. This unnecessary work can -be avoided by turning off the timestamp feature. This can be achieved by -setting the SOURCE_DATE_EPOCH environment variable to 0. +be avoided by turning off the timestamp feature. This can be achieved using +the `-r` flag, which enables reproducible builds by setting +`SOURCE_DATE_EPOCH=0` when building. Combining all of these options together yields the command-line shown below. This will provide the quickest possible feedback regarding the current content of the source tree, thus allowing rapid tested evolution of the code:: - SOURCE_DATE_EPOCH=0 ./tools/buildman/buildman -P tegra + ./tools/buildman/buildman -Pr tegra Checking configuration @@ -1108,6 +1109,8 @@ and 'brppt1_spi', removing a trailing semicolon. 'brppt1_nand' gained an a value for 'altbootcmd', but lost one for ' altbootcmd'. The -U option uses the u-boot.env files which are produced by a build. +Internally, buildman writes out an out-env file into the build directory for +later comparison. Building with clang @@ -1121,6 +1124,20 @@ toolchain. For example: buildman -O clang-7 --board sandbox +Building without LTO +-------------------- + +Link-time optimisation (LTO) is designed to reduce code size by globally +optimising the U-Boot build. Unfortunately this can dramatically slow down +builds. This is particularly noticeable when running a lot of builds. + +Use the -L (--no-lto) flag to disable LTO. + +.. code-block:: bash + + buildman -L --board sandbox + + Doing a simple build -------------------- @@ -1298,6 +1315,14 @@ You should use 'buildman -nv <criteria>' instead of greoing the boards.cfg file, since it may be dropped altogether in future. +Checking the command +-------------------- + +Buildman writes out the toolchain information to a `toolchain` file within the +output directory. It also writes the commands used to build U-Boot in an +`out-cmd` file. You can check these if you suspect something strange is +happening. + TODO ---- diff --git a/tools/buildman/cfgutil.py b/tools/buildman/cfgutil.py index ab74a8e..a340e01 100644 --- a/tools/buildman/cfgutil.py +++ b/tools/buildman/cfgutil.py @@ -7,7 +7,7 @@ import re -from patman import tools +from u_boot_pylib import tools RE_LINE = re.compile(r'(# )?CONFIG_([A-Z0-9_]+)(=(.*)| is not set)') RE_CFG = re.compile(r'(~?)(CONFIG_)?([A-Z0-9_]+)(=.*)?') diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index c485994..a9cda24 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -3,6 +3,11 @@ # from optparse import OptionParser +import os +import pathlib + +BUILDMAN_DIR = pathlib.Path(__file__).parent +HAS_TESTS = os.path.exists(BUILDMAN_DIR / "test.py") def ParseArgs(): """Parse command line arguments from sys.argv[] @@ -71,6 +76,8 @@ def ParseArgs(): default=False, help="Don't convert y to 1 in configs") parser.add_option('-l', '--list-error-boards', action='store_true', default=False, help='Show a list of boards next to each error/warning') + parser.add_option('-L', '--no-lto', action='store_true', + default=False, help='Disable Link-time Optimisation (LTO) for builds') parser.add_option('--list-tool-chains', action='store_true', default=False, help='List available tool chains (use -v to see probing detail)') parser.add_option('-m', '--mrproper', action='store_true', @@ -95,18 +102,21 @@ def ParseArgs(): default=False, help="Use full toolchain path in CROSS_COMPILE") parser.add_option('-P', '--per-board-out-dir', action='store_true', default=False, help="Use an O= (output) directory per board rather than per thread") + parser.add_option('-r', '--reproducible-builds', action='store_true', + help='Set SOURCE_DATE_EPOCH=0 to suuport a reproducible build') parser.add_option('-R', '--regen-board-list', action='store_true', help='Force regeneration of the list of boards, like the old boards.cfg file') parser.add_option('-s', '--summary', action='store_true', default=False, help='Show a build summary') parser.add_option('-S', '--show-sizes', action='store_true', default=False, help='Show image size variation in summary') - parser.add_option('--skip-net-tests', action='store_true', default=False, - help='Skip tests which need the network') parser.add_option('--step', type='int', default=1, help='Only build every n commits (0=just first and last)') - parser.add_option('-t', '--test', action='store_true', dest='test', - default=False, help='run tests') + if HAS_TESTS: + parser.add_option('--skip-net-tests', action='store_true', default=False, + help='Skip tests which need the network') + parser.add_option('-t', '--test', action='store_true', dest='test', + default=False, help='run tests') parser.add_option('-T', '--threads', type='int', default=None, help='Number of builder threads to use (0=single-thread)') diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 87e7d0e..35f44c0 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -3,6 +3,7 @@ # import multiprocessing +import importlib.resources import os import shutil import subprocess @@ -13,12 +14,12 @@ from buildman import bsettings from buildman import cfgutil from buildman import toolchain from buildman.builder import Builder -from patman import command from patman import gitutil from patman import patchstream -from patman import terminal -from patman import tools -from patman.terminal import tprint +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib import tools +from u_boot_pylib.terminal import tprint def GetPlural(count): """Returns a plural 's' if count is not 1""" @@ -152,9 +153,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, global builder if options.full_help: - tools.print_full_help( - os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), - 'README.rst')) + with importlib.resources.path('buildman', 'README.rst') as readme: + tools.print_full_help(str(readme)) return 0 gitutil.setup() @@ -261,9 +261,9 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, count += 1 # Build upstream commit also if not count: - str = ("No commits found to process in branch '%s': " + msg = ("No commits found to process in branch '%s': " "set branch's upstream or use -c flag" % options.branch) - sys.exit(col.build(col.RED, str)) + sys.exit(col.build(col.RED, msg)) if options.work_in_output: if len(selected) != 1: sys.exit(col.build(col.RED, @@ -338,6 +338,14 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, shutil.rmtree(output_dir) adjust_cfg = cfgutil.convert_list_to_dict(options.adjust_cfg) + # Drop LOCALVERSION_AUTO since it changes the version string on every commit + if options.reproducible_builds: + # If these are mentioned, leave the local version alone + if 'LOCALVERSION' in adjust_cfg or 'LOCALVERSION_AUTO' in adjust_cfg: + print('Not dropping LOCALVERSION_AUTO for reproducible build') + else: + adjust_cfg['LOCALVERSION_AUTO'] = '~' + builder = Builder(toolchains, output_dir, options.git_dir, options.threads, options.jobs, gnu_make=gnu_make, checkout=True, show_unknown=options.show_unknown, step=options.step, @@ -351,7 +359,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, work_in_output=options.work_in_output, test_thread_exceptions=test_thread_exceptions, adjust_cfg=adjust_cfg, - allow_missing=allow_missing) + allow_missing=allow_missing, no_lto=options.no_lto, + reproducible_builds=options.reproducible_builds) builder.force_config_on_failure = not options.quick if make_func: builder.do_make = make_func diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 559e4ed..ebd78f2 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -14,11 +14,11 @@ from buildman import bsettings from buildman import cmdline from buildman import control from buildman import toolchain -from patman import command from patman import gitutil -from patman import terminal -from patman import test_util -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib import test_util +from u_boot_pylib import tools settings_data = ''' # Buildman settings file @@ -415,17 +415,19 @@ class TestFunctional(unittest.TestCase): kwargs: Arguments to pass to command.run_pipe() """ self._make_calls += 1 + out_dir = '' + for arg in args: + if arg.startswith('O='): + out_dir = arg[2:] if stage == 'mrproper': return command.CommandResult(return_code=0) elif stage == 'config': + fname = os.path.join(cwd or '', out_dir, '.config') + tools.write_file(fname, b'CONFIG_SOMETHING=1') return command.CommandResult(return_code=0, combined='Test configuration complete') elif stage == 'build': stderr = '' - out_dir = '' - for arg in args: - if arg.startswith('O='): - out_dir = arg[2:] fname = os.path.join(cwd or '', out_dir, 'u-boot') tools.write_file(fname, b'U-Boot') @@ -723,3 +725,57 @@ Some images are invalid''' control.get_allow_missing(False, False, 2, True)) self.assertEqual(False, control.get_allow_missing(False, True, 2, True)) + + def check_command(self, *extra_args): + """Run a command with the extra arguments and return the commands used + + Args: + extra_args (list of str): List of extra arguments + + Returns: + list of str: Lines returned in the out-cmd file + """ + self._RunControl('-o', self._output_dir, *extra_args) + board0_dir = os.path.join(self._output_dir, 'current', 'board0') + self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done'))) + cmd_fname = os.path.join(board0_dir, 'out-cmd') + self.assertTrue(os.path.exists(cmd_fname)) + data = tools.read_file(cmd_fname) + + config_fname = os.path.join(board0_dir, '.config') + self.assertTrue(os.path.exists(config_fname)) + cfg_data = tools.read_file(config_fname) + + return data.splitlines(), cfg_data + + def testCmdFile(self): + """Test that the -cmd-out file is produced""" + lines = self.check_command()[0] + self.assertEqual(2, len(lines)) + self.assertRegex(lines[0], b'make O=/.*board0_defconfig') + self.assertRegex(lines[0], b'make O=/.*-s.*') + + def testNoLto(self): + """Test that the --no-lto flag works""" + lines = self.check_command('-L')[0] + self.assertIn(b'NO_LTO=1', lines[0]) + + def testReproducible(self): + """Test that the -r flag works""" + lines, cfg_data = self.check_command('-r') + self.assertIn(b'SOURCE_DATE_EPOCH=0', lines[0]) + + # We should see CONFIG_LOCALVERSION_AUTO unset + self.assertEqual(b'''CONFIG_SOMETHING=1 +# CONFIG_LOCALVERSION_AUTO is not set +''', cfg_data) + + with test_util.capture_sys_output() as (stdout, stderr): + lines, cfg_data = self.check_command('-r', '-a', 'LOCALVERSION') + self.assertIn(b'SOURCE_DATE_EPOCH=0', lines[0]) + + # We should see CONFIG_LOCALVERSION_AUTO unset + self.assertEqual(b'''CONFIG_SOMETHING=1 +CONFIG_LOCALVERSION=y +''', cfg_data) + self.assertIn('Not dropping LOCALVERSION_AUTO', stdout.getvalue()) diff --git a/tools/buildman/main.py b/tools/buildman/main.py index 67c560c..5e1f68d 100755 --- a/tools/buildman/main.py +++ b/tools/buildman/main.py @@ -25,8 +25,8 @@ from buildman import control from buildman import toolchain from patman import patchstream from patman import gitutil -from patman import terminal -from patman import test_util +from u_boot_pylib import terminal +from u_boot_pylib import test_util def RunTests(skip_net_tests, verboose, args): from buildman import func_test @@ -46,17 +46,22 @@ def RunTests(skip_net_tests, verboose, args): return (0 if result.wasSuccessful() else 1) -options, args = cmdline.ParseArgs() +def run_buildman(): + options, args = cmdline.ParseArgs() -if not options.debug: - sys.tracebacklimit = 0 + if not options.debug: + sys.tracebacklimit = 0 -# Run our meagre tests -if options.test: - RunTests(options.skip_net_tests, options.verbose, args) + # Run our meagre tests + if cmdline.HAS_TESTS and options.test: + RunTests(options.skip_net_tests, options.verbose, args) -# Build selected commits for selected boards -else: - bsettings.Setup(options.config_file) - ret_code = control.DoBuildman(options, args) - sys.exit(ret_code) + # Build selected commits for selected boards + else: + bsettings.Setup(options.config_file) + ret_code = control.DoBuildman(options, args) + sys.exit(ret_code) + + +if __name__ == "__main__": + run_buildman() diff --git a/tools/buildman/pyproject.toml b/tools/buildman/pyproject.toml new file mode 100644 index 0000000..4d75e77 --- /dev/null +++ b/tools/buildman/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "buildman" +version = "0.0.2" +authors = [ + { name="Simon Glass", email="sjg@chromium.org" }, +] +dependencies = ["u_boot_pylib", "patch-manager"] +description = "Buildman build tool for U-Boot" +readme = "README.rst" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://u-boot.readthedocs.io/en/latest/build/buildman.html" +"Bug Tracker" = "https://source.denx.de/groups/u-boot/-/issues" + +[project.scripts] +buildman = "buildman.main:run_buildman" + +[tool.setuptools.package-data] +buildman = ["*.rst"] diff --git a/tools/buildman/test.py b/tools/buildman/test.py index daf5467..9fa6445 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -17,10 +17,10 @@ from buildman import cfgutil from buildman import control from buildman import toolchain from patman import commit -from patman import command -from patman import terminal -from patman import test_util -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib import test_util +from u_boot_pylib import tools use_network = True diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index 6bae913..0ecd845 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -11,9 +11,9 @@ import tempfile import urllib.request, urllib.error, urllib.parse from buildman import bsettings -from patman import command -from patman import terminal -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib import tools (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH, PRIORITY_CALC) = list(range(4)) @@ -156,9 +156,10 @@ class Toolchain: Returns: Value of that environment variable or arguments """ - wrapper = self.GetWrapper() if which == VAR_CROSS_COMPILE: - return wrapper + os.path.join(self.path, self.cross) + wrapper = self.GetWrapper() + base = '' if self.arch == 'sandbox' else self.path + return wrapper + os.path.join(base, self.cross) elif which == VAR_PATH: return self.path elif which == VAR_ARCH: diff --git a/tools/concurrencytest/.gitignore b/tools/concurrencytest/.gitignore deleted file mode 100644 index 0d20b64..0000000 --- a/tools/concurrencytest/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.pyc diff --git a/tools/concurrencytest/README.md b/tools/concurrencytest/README.md deleted file mode 100644 index 2d7fe75..0000000 --- a/tools/concurrencytest/README.md +++ /dev/null @@ -1,74 +0,0 @@ -concurrencytest -=============== - - - -Python testtools extension for running unittest suites concurrently. - ----- - -Install from PyPI: -``` -pip install concurrencytest -``` - ----- - -Requires: - -* [testtools](https://pypi.python.org/pypi/testtools) : `pip install testtools` -* [python-subunit](https://pypi.python.org/pypi/python-subunit) : `pip install python-subunit` - ----- - -Example: - -```python -import time -import unittest - -from concurrencytest import ConcurrentTestSuite, fork_for_tests - - -class SampleTestCase(unittest.TestCase): - """Dummy tests that sleep for demo.""" - - def test_me_1(self): - time.sleep(0.5) - - def test_me_2(self): - time.sleep(0.5) - - def test_me_3(self): - time.sleep(0.5) - - def test_me_4(self): - time.sleep(0.5) - - -# Load tests from SampleTestCase defined above -suite = unittest.TestLoader().loadTestsFromTestCase(SampleTestCase) -runner = unittest.TextTestRunner() - -# Run tests sequentially -runner.run(suite) - -# Run same tests across 4 processes -suite = unittest.TestLoader().loadTestsFromTestCase(SampleTestCase) -concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(4)) -runner.run(concurrent_suite) -``` -Output: - -``` -.... ----------------------------------------------------------------------- -Ran 4 tests in 2.003s - -OK -.... ----------------------------------------------------------------------- -Ran 4 tests in 0.504s - -OK -``` diff --git a/tools/concurrencytest/__init__.py b/tools/concurrencytest/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/tools/concurrencytest/__init__.py +++ /dev/null diff --git a/tools/concurrencytest/concurrencytest.py b/tools/concurrencytest/concurrencytest.py deleted file mode 100644 index 1c4f03f..0000000 --- a/tools/concurrencytest/concurrencytest.py +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env python -# SPDX-License-Identifier: GPL-2.0+ -# -# Modified by: Corey Goldberg, 2013 -# -# Original code from: -# Bazaar (bzrlib.tests.__init__.py, v2.6, copied Jun 01 2013) -# Copyright (C) 2005-2011 Canonical Ltd - -"""Python testtools extension for running unittest suites concurrently. - -The `testtools` project provides a ConcurrentTestSuite class, but does -not provide a `make_tests` implementation needed to use it. - -This allows you to parallelize a test run across a configurable number -of worker processes. While this can speed up CPU-bound test runs, it is -mainly useful for IO-bound tests that spend most of their time waiting for -data to arrive from someplace else and can benefit from cocncurrency. - -Unix only. -""" - -import os -import sys -import traceback -import unittest -from itertools import cycle -from multiprocessing import cpu_count - -from subunit import ProtocolTestCase, TestProtocolClient -from subunit.test_results import AutoTimingTestResultDecorator - -from testtools import ConcurrentTestSuite, iterate_tests -from testtools.content import TracebackContent, text_content - - -_all__ = [ - 'ConcurrentTestSuite', - 'fork_for_tests', - 'partition_tests', -] - - -CPU_COUNT = cpu_count() - - -class BufferingTestProtocolClient(TestProtocolClient): - """A TestProtocolClient which can buffer the test outputs - - This class captures the stdout and stderr output streams of the - tests as it runs them, and includes the output texts in the subunit - stream as additional details. - - Args: - stream: A file-like object to write a subunit stream to - buffer (bool): True to capture test stdout/stderr outputs and - include them in the test details - """ - def __init__(self, stream, buffer=True): - super().__init__(stream) - self.buffer = buffer - - def _addOutcome(self, outcome, test, error=None, details=None, - error_permitted=True): - """Report a test outcome to the subunit stream - - The parent class uses this function as a common implementation - for various methods that report successes, errors, failures, etc. - - This version automatically upgrades the error tracebacks to the - new 'details' format by wrapping them in a Content object, so - that we can include the captured test output in the test result - details. - - Args: - outcome: A string describing the outcome - used as the - event name in the subunit stream. - test: The test case whose outcome is to be reported - error: Standard unittest positional argument form - an - exc_info tuple. - details: New Testing-in-python drafted API; a dict from - string to subunit.Content objects. - error_permitted: If True then one and only one of error or - details must be supplied. If False then error must not - be supplied and details is still optional. - """ - if details is None: - details = {} - - # Parent will raise an exception if error_permitted is False but - # error is not None. We want that exception in that case, so - # don't touch error when error_permitted is explicitly False. - if error_permitted and error is not None: - # Parent class prefers error over details - details['traceback'] = TracebackContent(error, test) - error_permitted = False - error = None - - if self.buffer: - stdout = sys.stdout.getvalue() - if stdout: - details['stdout'] = text_content(stdout) - - stderr = sys.stderr.getvalue() - if stderr: - details['stderr'] = text_content(stderr) - - return super()._addOutcome(outcome, test, error=error, - details=details, error_permitted=error_permitted) - - -def fork_for_tests(concurrency_num=CPU_COUNT, buffer=False): - """Implementation of `make_tests` used to construct `ConcurrentTestSuite`. - - :param concurrency_num: number of processes to use. - """ - if buffer: - test_protocol_client_class = BufferingTestProtocolClient - else: - test_protocol_client_class = TestProtocolClient - - def do_fork(suite): - """Take suite and start up multiple runners by forking (Unix only). - - :param suite: TestSuite object. - - :return: An iterable of TestCase-like objects which can each have - run(result) called on them to feed tests to result. - """ - result = [] - test_blocks = partition_tests(suite, concurrency_num) - # Clear the tests from the original suite so it doesn't keep them alive - suite._tests[:] = [] - for process_tests in test_blocks: - process_suite = unittest.TestSuite(process_tests) - # Also clear each split list so new suite has only reference - process_tests[:] = [] - c2pread, c2pwrite = os.pipe() - pid = os.fork() - if pid == 0: - try: - stream = os.fdopen(c2pwrite, 'wb') - os.close(c2pread) - # Leave stderr and stdout open so we can see test noise - # Close stdin so that the child goes away if it decides to - # read from stdin (otherwise its a roulette to see what - # child actually gets keystrokes for pdb etc). - sys.stdin.close() - subunit_result = AutoTimingTestResultDecorator( - test_protocol_client_class(stream) - ) - process_suite.run(subunit_result) - except: - # Try and report traceback on stream, but exit with error - # even if stream couldn't be created or something else - # goes wrong. The traceback is formatted to a string and - # written in one go to avoid interleaving lines from - # multiple failing children. - try: - stream.write(traceback.format_exc()) - finally: - os._exit(1) - os._exit(0) - else: - os.close(c2pwrite) - stream = os.fdopen(c2pread, 'rb') - # If we don't pass the second argument here, it defaults - # to sys.stdout.buffer down the line. But if we don't - # pass it *now*, it may be resolved after sys.stdout is - # replaced with a StringIO (to capture tests' outputs) - # which doesn't have a buffer attribute and can end up - # occasionally causing a 'broken-runner' error. - test = ProtocolTestCase(stream, sys.stdout.buffer) - result.append(test) - return result - return do_fork - - -def partition_tests(suite, count): - """Partition suite into count lists of tests.""" - # This just assigns tests in a round-robin fashion. On one hand this - # splits up blocks of related tests that might run faster if they shared - # resources, but on the other it avoids assigning blocks of slow tests to - # just one partition. So the slowest partition shouldn't be much slower - # than the fastest. - partitions = [list() for _ in range(count)] - tests = iterate_tests(suite) - for partition, test in zip(cycle(partitions), tests): - partition.append(test) - return partitions - - -if __name__ == '__main__': - import time - - class SampleTestCase(unittest.TestCase): - """Dummy tests that sleep for demo.""" - - def test_me_1(self): - time.sleep(0.5) - - def test_me_2(self): - time.sleep(0.5) - - def test_me_3(self): - time.sleep(0.5) - - def test_me_4(self): - time.sleep(0.5) - - # Load tests from SampleTestCase defined above - suite = unittest.TestLoader().loadTestsFromTestCase(SampleTestCase) - runner = unittest.TextTestRunner() - - # Run tests sequentially - runner.run(suite) - - # Run same tests across 4 processes - suite = unittest.TestLoader().loadTestsFromTestCase(SampleTestCase) - concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(4)) - runner.run(concurrent_suite) diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index 33e2bd2..bd02531 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -2,7 +2,7 @@ # This Dockerfile is used to build an image containing basic stuff to be used # to build U-Boot and run our test suites. -FROM ubuntu:jammy-20230126 +FROM ubuntu:jammy-20230308 MAINTAINER Tom Rini <trini@konsulko.com> LABEL Description=" This image is for building U-Boot inside a container" @@ -182,10 +182,31 @@ RUN git clone https://gitlab.com/qemu-project/qemu.git /tmp/qemu && \ git config user.email u-boot@denx.de && \ # manually apply the bug fix for QEMU 6.1.0 Xilinx Zynq UART emulation codes wget -O - http://patchwork.ozlabs.org/project/qemu-devel/patch/20210823020813.25192-2-bmeng.cn@gmail.com/mbox/ | git am && \ - ./configure --prefix=/opt/qemu --target-list="aarch64-softmmu,arm-softmmu,i386-softmmu,mips-softmmu,mips64-softmmu,mips64el-softmmu,mipsel-softmmu,ppc-softmmu,riscv32-softmmu,riscv64-softmmu,sh4-softmmu,x86_64-softmmu,xtensa-softmmu" && \ + ./configure --prefix=/opt/qemu --target-list="aarch64-softmmu,arm-softmmu,i386-softmmu,m68k-softmmu,mips-softmmu,mips64-softmmu,mips64el-softmmu,mipsel-softmmu,ppc-softmmu,riscv32-softmmu,riscv64-softmmu,sh4-softmmu,x86_64-softmmu,xtensa-softmmu" && \ make -j$(nproc) all install && \ rm -rf /tmp/qemu +# Build QEMU supporting Nokia n900 emulation +RUN mkdir -p /opt/nokia && \ + cd /tmp && \ + git clone https://git.linaro.org/qemu/qemu-linaro.git && \ + cd /tmp/qemu-linaro && \ + git checkout 8f8d8e0796efe1a6f34cdd83fb798f3c41217ec1 && \ + ./configure --enable-system --target-list=arm-softmmu \ + --python=/usr/bin/python2.7 --disable-sdl --disable-gtk \ + --disable-curses --audio-drv-list= --audio-card-list= \ + --disable-werror --disable-xen --disable-xen-pci-passthrough \ + --disable-brlapi --disable-vnc --disable-curl --disable-slirp \ + --disable-kvm --disable-user --disable-linux-user --disable-bsd-user \ + --disable-guest-base --disable-uuid --disable-vde --disable-linux-aio \ + --disable-cap-ng --disable-attr --disable-blobs --disable-docs \ + --disable-spice --disable-libiscsi --disable-smartcard-nss \ + --disable-usb-redir --disable-guest-agent --disable-seccomp \ + --disable-glusterfs --disable-nptl --disable-fdt && \ + make -j$(nproc) && \ + cp /tmp/qemu-linaro/arm-softmmu/qemu-system-arm /opt/nokia && \ + rm -rf /tmp/qemu-linaro + # Build genimage (required by some targets to generate disk images) RUN wget -O - https://github.com/pengutronix/genimage/releases/download/v14/genimage-14.tar.xz | tar -C /tmp -xJ && \ cd /tmp/genimage-14 && \ @@ -229,6 +250,16 @@ RUN mkdir /tmp/trace && \ sudo make install && \ rm -rf /tmp/trace +# Files to run Nokia RX-51 (aka N900) tests +RUN mkdir -p /opt/nokia && \ + cd /opt/nokia && \ + wget https://raw.githubusercontent.com/pali/u-boot-maemo/master/debian/u-boot-gen-combined && \ + chmod 0755 u-boot-gen-combined && \ + wget http://repository.maemo.org/qemu-n900/qemu-n900.tar.gz && \ + wget http://repository.maemo.org/pool/maemo5.0/free/k/kernel/kernel_2.6.28-20103103+0m5_armel.deb && \ + wget http://repository.maemo.org/pool/maemo5.0/free/g/glibc/libc6_2.5.1-1eglibc27+0m5_armel.deb && \ + wget http://repository.maemo.org/pool/maemo5.0/free/b/busybox/busybox_1.10.2.legal-1osso30+0m5_armel.deb + # Create our user/group RUN echo uboot ALL=NOPASSWD: ALL > /etc/sudoers.d/uboot RUN useradd -m -U uboot diff --git a/tools/dtoc/README.rst b/tools/dtoc/README.rst new file mode 100644 index 0000000..92b3975 --- /dev/null +++ b/tools/dtoc/README.rst @@ -0,0 +1,15 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +Devicetree-to-C generator +========================= + +This is a Python program and associated utilities, which supports converting +devicetree files into C code. It generates header files containing struct +definitions, as well as C files containing the data. It does not require any +modification of the devicetree files. + +Some high-level libraries are provided for working with devicetree. These may +be useful in other projects. + +This package also includes some U-Boot-specific features, such as creating +`struct udevice` and `struct uclass` entries for devicetree nodes. diff --git a/tools/dtoc/dtb_platdata.py b/tools/dtoc/dtb_platdata.py index a69a788..39f416c 100644 --- a/tools/dtoc/dtb_platdata.py +++ b/tools/dtoc/dtb_platdata.py @@ -35,9 +35,9 @@ PROP_IGNORE_LIST = [ 'linux,phandle', "status", 'phandle', - 'u-boot,dm-pre-reloc', - 'u-boot,dm-tpl', - 'u-boot,dm-spl', + 'bootph-all', + 'bootph-pre-sram', + 'bootph-pre-ram', ] # C type declarations for the types we support @@ -442,7 +442,7 @@ class DtbPlatdata(): """ parent = node.parent if parent and not parent.props: - raise ValueError("Parent node '%s' has no properties - do you need u-boot,dm-spl or similar?" % + raise ValueError("Parent node '%s' has no properties - do you need bootph-pre-ram or similar?" % parent.path) num_addr, num_size = 2, 2 if parent: @@ -754,7 +754,7 @@ class DtbPlatdata(): # This might indicate that the parent node is not in the # SPL/TPL devicetree but the child is. For example if we are # dealing with of-platdata in TPL, the parent has a - # u-boot,dm-tpl tag but the child has u-boot,dm-pre-reloc. In + # bootph-pre-sram tag but the child has bootph-all. In # this case the child node exists in TPL but the parent does # not. raise ValueError("Node '%s' requires parent node '%s' but it is not in the valid list" % diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index d933972..a8e0534 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -12,7 +12,7 @@ import sys from dtoc import fdt_util import libfdt from libfdt import QUIET_NOTFOUND -from patman import tools +from u_boot_pylib import tools # This deals with a device tree, presenting it as an assortment of Node and # Prop objects, representing nodes and properties, respectively. This file diff --git a/tools/dtoc/fdt_util.py b/tools/dtoc/fdt_util.py index f343166..f1f7056 100644 --- a/tools/dtoc/fdt_util.py +++ b/tools/dtoc/fdt_util.py @@ -13,8 +13,8 @@ import struct import sys import tempfile -from patman import command -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import tools def fdt32_to_cpu(val): """Convert a device tree cell to an integer diff --git a/tools/dtoc/main.py b/tools/dtoc/main.py index 5508759..6c91450 100755 --- a/tools/dtoc/main.py +++ b/tools/dtoc/main.py @@ -23,6 +23,7 @@ see doc/driver-model/of-plat.rst from argparse import ArgumentParser import os +import pathlib import sys # Bring in the patman libraries @@ -35,7 +36,10 @@ sys.path.insert(0, os.path.join(our_path, '../../build-sandbox_spl/scripts/dtc/pylibfdt')) from dtoc import dtb_platdata -from patman import test_util +from u_boot_pylib import test_util + +DTOC_DIR = pathlib.Path(__file__).parent +HAVE_TESTS = (DTOC_DIR / 'test_dtoc.py').exists() def run_tests(processes, args): """Run all the test we have for dtoc @@ -61,54 +65,62 @@ def run_tests(processes, args): return (0 if result.wasSuccessful() else 1) -def RunTestCoverage(): +def RunTestCoverage(build_dir): """Run the tests and check that we get 100% coverage""" sys.argv = [sys.argv[0]] test_util.run_test_coverage('tools/dtoc/dtoc', '/main.py', - ['tools/patman/*.py', '*/fdt*', '*test*'], args.build_dir) - - -if __name__ != '__main__': - sys.exit(1) - -epilog = '''Generate C code from devicetree files. See of-plat.rst for details''' - -parser = ArgumentParser(epilog=epilog) -parser.add_argument('-B', '--build-dir', type=str, default='b', - help='Directory containing the build output') -parser.add_argument('-c', '--c-output-dir', action='store', - help='Select output directory for C files') -parser.add_argument('-C', '--h-output-dir', action='store', - help='Select output directory for H files (defaults to --c-output-di)') -parser.add_argument('-d', '--dtb-file', action='store', - help='Specify the .dtb input file') -parser.add_argument('-i', '--instantiate', action='store_true', default=False, - help='Instantiate devices to avoid needing device_bind()') -parser.add_argument('--include-disabled', action='store_true', - help='Include disabled nodes') -parser.add_argument('-o', '--output', action='store', - help='Select output filename') -parser.add_argument('-p', '--phase', type=str, - help='set phase of U-Boot this invocation is for (spl/tpl)') -parser.add_argument('-P', '--processes', type=int, - help='set number of processes to use for running tests') -parser.add_argument('-t', '--test', action='store_true', dest='test', - default=False, help='run tests') -parser.add_argument('-T', '--test-coverage', action='store_true', - default=False, help='run tests and check for 100%% coverage') -parser.add_argument('files', nargs='*') -args = parser.parse_args() - -# Run our meagre tests -if args.test: - ret_code = run_tests(args.processes, args) - sys.exit(ret_code) - -elif args.test_coverage: - RunTestCoverage() - -else: - dtb_platdata.run_steps(args.files, args.dtb_file, args.include_disabled, - args.output, - [args.c_output_dir, args.h_output_dir], - args.phase, instantiate=args.instantiate) + ['tools/patman/*.py', 'tools/u_boot_pylib/*','*/fdt*', '*test*'], + build_dir) + + +def run_dtoc(): + epilog = 'Generate C code from devicetree files. See of-plat.rst for details' + + parser = ArgumentParser(epilog=epilog) + parser.add_argument('-B', '--build-dir', type=str, default='b', + help='Directory containing the build output') + parser.add_argument('-c', '--c-output-dir', action='store', + help='Select output directory for C files') + parser.add_argument( + '-C', '--h-output-dir', action='store', + help='Select output directory for H files (defaults to --c-output-di)') + parser.add_argument('-d', '--dtb-file', action='store', + help='Specify the .dtb input file') + parser.add_argument( + '-i', '--instantiate', action='store_true', default=False, + help='Instantiate devices to avoid needing device_bind()') + parser.add_argument('--include-disabled', action='store_true', + help='Include disabled nodes') + parser.add_argument('-o', '--output', action='store', + help='Select output filename') + parser.add_argument( + '-p', '--phase', type=str, + help='set phase of U-Boot this invocation is for (spl/tpl)') + parser.add_argument('-P', '--processes', type=int, + help='set number of processes to use for running tests') + if HAVE_TESTS: + parser.add_argument('-t', '--test', action='store_true', dest='test', + default=False, help='run tests') + parser.add_argument( + '-T', '--test-coverage', action='store_true', + default=False, help='run tests and check for 100%% coverage') + parser.add_argument('files', nargs='*') + args = parser.parse_args() + + # Run our meagre tests + if HAVE_TESTS and args.test: + ret_code = run_tests(args.processes, args) + sys.exit(ret_code) + + elif HAVE_TESTS and args.test_coverage: + RunTestCoverage(args.build_dir) + + else: + dtb_platdata.run_steps(args.files, args.dtb_file, args.include_disabled, + args.output, + [args.c_output_dir, args.h_output_dir], + args.phase, instantiate=args.instantiate) + + +if __name__ == '__main__': + run_dtoc() diff --git a/tools/dtoc/pyproject.toml b/tools/dtoc/pyproject.toml new file mode 100644 index 0000000..77fe4da --- /dev/null +++ b/tools/dtoc/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "dtoc" +version = "0.0.2" +authors = [ + { name="Simon Glass", email="sjg@chromium.org" }, +] +dependencies = ["pylibfdt", "u_boot_pylib"] +description = "Devicetree-to-C generator" +readme = "README.rst" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://u-boot.readthedocs.io/en/latest/develop/driver-model/of-plat.html" +"Bug Tracker" = "https://source.denx.de/groups/u-boot/-/issues" + +[project.scripts] +dtoc = "dtoc.main:run_dtoc" diff --git a/tools/dtoc/test/dtoc_test_add_prop.dts b/tools/dtoc/test/dtoc_test_add_prop.dts index fa296e5..8225de3 100644 --- a/tools/dtoc/test/dtoc_test_add_prop.dts +++ b/tools/dtoc/test/dtoc_test_add_prop.dts @@ -11,13 +11,13 @@ #address-cells = <1>; #size-cells = <1>; spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; intval = <1>; }; spl-test2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; intarray = <5>; }; diff --git a/tools/dtoc/test/dtoc_test_addr32.dts b/tools/dtoc/test/dtoc_test_addr32.dts index 2390454..3e7dc56 100644 --- a/tools/dtoc/test/dtoc_test_addr32.dts +++ b/tools/dtoc/test/dtoc_test_addr32.dts @@ -12,13 +12,13 @@ #size-cells = <1>; test1 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test1"; reg = <0x1234 0x5678>; }; test2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test2"; reg = <0x12345678 0x98765432 2 3>; }; diff --git a/tools/dtoc/test/dtoc_test_addr32_64.dts b/tools/dtoc/test/dtoc_test_addr32_64.dts index 7599d5b..7ce16fe 100644 --- a/tools/dtoc/test/dtoc_test_addr32_64.dts +++ b/tools/dtoc/test/dtoc_test_addr32_64.dts @@ -12,19 +12,19 @@ #size-cells = <2>; test1 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test1"; reg = <0x1234 0x5678 0x0>; }; test2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test2"; reg = <0x12345678 0x98765432 0x10987654>; }; test3 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test3"; reg = <0x12345678 0x98765432 0x10987654 2 0 3>; }; diff --git a/tools/dtoc/test/dtoc_test_addr64.dts b/tools/dtoc/test/dtoc_test_addr64.dts index 263d251..5f8c23f 100644 --- a/tools/dtoc/test/dtoc_test_addr64.dts +++ b/tools/dtoc/test/dtoc_test_addr64.dts @@ -12,19 +12,19 @@ #size-cells = <2>; test1 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test1"; reg = /bits/ 64 <0x1234 0x5678>; }; test2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test2"; reg = /bits/ 64 <0x1234567890123456 0x9876543210987654>; }; test3 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test3"; reg = /bits/ 64 <0x1234567890123456 0x9876543210987654 2 3>; }; diff --git a/tools/dtoc/test/dtoc_test_addr64_32.dts b/tools/dtoc/test/dtoc_test_addr64_32.dts index 85e4f5f..bfbfd87 100644 --- a/tools/dtoc/test/dtoc_test_addr64_32.dts +++ b/tools/dtoc/test/dtoc_test_addr64_32.dts @@ -12,19 +12,19 @@ #size-cells = <1>; test1 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test1"; reg = <0x1234 0x0 0x5678>; }; test2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test2"; reg = <0x12345678 0x90123456 0x98765432>; }; test3 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test3"; reg = <0x12345678 0x90123456 0x98765432 0 2 3>; }; diff --git a/tools/dtoc/test/dtoc_test_alias_bad.dts b/tools/dtoc/test/dtoc_test_alias_bad.dts index d4f502a..69761f9 100644 --- a/tools/dtoc/test/dtoc_test_alias_bad.dts +++ b/tools/dtoc/test/dtoc_test_alias_bad.dts @@ -18,20 +18,20 @@ }; spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; boolval; intval = <1>; }; i2c: i2c { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,i2c"; intval = <3>; }; spl-test3 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; stringarray = "one"; longbytearray = [09 0a 0b 0c 0d 0e 0f 10]; diff --git a/tools/dtoc/test/dtoc_test_alias_bad_path.dts b/tools/dtoc/test/dtoc_test_alias_bad_path.dts index 0beca4f..6f566fe 100644 --- a/tools/dtoc/test/dtoc_test_alias_bad_path.dts +++ b/tools/dtoc/test/dtoc_test_alias_bad_path.dts @@ -18,20 +18,20 @@ }; spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; boolval; intval = <1>; }; i2c: i2c { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,i2c"; intval = <3>; }; spl-test3 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; stringarray = "one"; longbytearray = [09 0a 0b 0c 0d 0e 0f 10]; diff --git a/tools/dtoc/test/dtoc_test_alias_bad_uc.dts b/tools/dtoc/test/dtoc_test_alias_bad_uc.dts index ae64f5b..5d23c63 100644 --- a/tools/dtoc/test/dtoc_test_alias_bad_uc.dts +++ b/tools/dtoc/test/dtoc_test_alias_bad_uc.dts @@ -18,20 +18,20 @@ }; spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; boolval; intval = <1>; }; i2c: i2c { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,i2c"; intval = <3>; }; spl-test3 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; stringarray = "one"; longbytearray = [09 0a 0b 0c 0d 0e 0f 10]; diff --git a/tools/dtoc/test/dtoc_test_aliases.dts b/tools/dtoc/test/dtoc_test_aliases.dts index ae33716..018b834 100644 --- a/tools/dtoc/test/dtoc_test_aliases.dts +++ b/tools/dtoc/test/dtoc_test_aliases.dts @@ -9,13 +9,13 @@ / { spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "compat1", "compat2.1-fred", "compat3"; intval = <1>; }; spl-test2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "compat1", "simple_bus"; intval = <1>; }; diff --git a/tools/dtoc/test/dtoc_test_driver_alias.dts b/tools/dtoc/test/dtoc_test_driver_alias.dts index da7973b..22369a4 100644 --- a/tools/dtoc/test/dtoc_test_driver_alias.dts +++ b/tools/dtoc/test/dtoc_test_driver_alias.dts @@ -9,7 +9,7 @@ / { gpio_a: gpios@0 { - u-boot,dm-pre-reloc; + bootph-all; gpio-controller; compatible = "sandbox_gpio_alias"; #gpio-cells = <1>; diff --git a/tools/dtoc/test/dtoc_test_inst.dts b/tools/dtoc/test/dtoc_test_inst.dts index b8177fc..9689be3 100644 --- a/tools/dtoc/test/dtoc_test_inst.dts +++ b/tools/dtoc/test/dtoc_test_inst.dts @@ -18,20 +18,20 @@ }; spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; boolval; intval = <1>; }; i2c: i2c { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,i2c"; intval = <3>; }; spl-test3 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; stringarray = "one"; longbytearray = [09 0a 0b 0c 0d 0e 0f 10]; diff --git a/tools/dtoc/test/dtoc_test_invalid_driver.dts b/tools/dtoc/test/dtoc_test_invalid_driver.dts index 914ac3e..042a325 100644 --- a/tools/dtoc/test/dtoc_test_invalid_driver.dts +++ b/tools/dtoc/test/dtoc_test_invalid_driver.dts @@ -9,7 +9,7 @@ / { spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "invalid"; }; }; diff --git a/tools/dtoc/test/dtoc_test_noparent.dts b/tools/dtoc/test/dtoc_test_noparent.dts index e976dd2..0efb17e 100644 --- a/tools/dtoc/test/dtoc_test_noparent.dts +++ b/tools/dtoc/test/dtoc_test_noparent.dts @@ -12,18 +12,18 @@ #size-cells = <1>; i2c@0 { compatible = "sandbox,i2c"; - u-boot,dm-tpl; + bootph-pre-sram; #address-cells = <1>; #size-cells = <0>; spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; #address-cells = <1>; #size-cells = <0>; status = "disabled"; pmic@9 { compatible = "sandbox,pmic"; - u-boot,dm-pre-reloc; + bootph-all; reg = <9>; low-power; }; diff --git a/tools/dtoc/test/dtoc_test_noprops.dts b/tools/dtoc/test/dtoc_test_noprops.dts index e6fdd11..75296be 100644 --- a/tools/dtoc/test/dtoc_test_noprops.dts +++ b/tools/dtoc/test/dtoc_test_noprops.dts @@ -13,7 +13,7 @@ i2c@0 { pmic@9 { compatible = "sandbox,pmic"; - u-boot,dm-pre-reloc; + bootph-all; reg = <9>; low-power; }; diff --git a/tools/dtoc/test/dtoc_test_phandle.dts b/tools/dtoc/test/dtoc_test_phandle.dts index d9aa433..74a146b 100644 --- a/tools/dtoc/test/dtoc_test_phandle.dts +++ b/tools/dtoc/test/dtoc_test_phandle.dts @@ -9,34 +9,34 @@ / { phandle: phandle-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; intval = <0>; #clock-cells = <0>; }; phandle_1: phandle2-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; intval = <1>; #clock-cells = <1>; }; phandle_2: phandle3-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; intval = <2>; #clock-cells = <2>; }; phandle-source { - u-boot,dm-pre-reloc; + bootph-all; compatible = "source"; clocks = <&phandle &phandle_1 11 &phandle_2 12 13 &phandle>; phandle-name-offset = <&phandle_2>, "fred", <123>; }; phandle-source2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "source"; clocks = <&phandle>; }; diff --git a/tools/dtoc/test/dtoc_test_phandle_bad.dts b/tools/dtoc/test/dtoc_test_phandle_bad.dts index a3ddc59..94cfada 100644 --- a/tools/dtoc/test/dtoc_test_phandle_bad.dts +++ b/tools/dtoc/test/dtoc_test_phandle_bad.dts @@ -9,7 +9,7 @@ / { phandle-source { - u-boot,dm-pre-reloc; + bootph-all; compatible = "source"; clocks = <20>; /* Invalid phandle */ }; diff --git a/tools/dtoc/test/dtoc_test_phandle_bad2.dts b/tools/dtoc/test/dtoc_test_phandle_bad2.dts index fe25f56..4d24b96 100644 --- a/tools/dtoc/test/dtoc_test_phandle_bad2.dts +++ b/tools/dtoc/test/dtoc_test_phandle_bad2.dts @@ -9,13 +9,13 @@ / { phandle: phandle-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; intval = <0>; }; phandle-source2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "source"; clocks = <&phandle>; }; diff --git a/tools/dtoc/test/dtoc_test_phandle_cd_gpios.dts b/tools/dtoc/test/dtoc_test_phandle_cd_gpios.dts index 241743e..6ad8006 100644 --- a/tools/dtoc/test/dtoc_test_phandle_cd_gpios.dts +++ b/tools/dtoc/test/dtoc_test_phandle_cd_gpios.dts @@ -9,33 +9,33 @@ / { phandle: phandle-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; intval = <0>; #gpio-cells = <0>; }; phandle_1: phandle2-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; intval = <1>; #gpio-cells = <1>; }; phandle_2: phandle3-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; intval = <2>; #gpio-cells = <2>; }; phandle-source { - u-boot,dm-pre-reloc; + bootph-all; compatible = "source"; cd-gpios = <&phandle &phandle_1 11 &phandle_2 12 13 &phandle>; }; phandle-source2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "source"; cd-gpios = <&phandle>; }; diff --git a/tools/dtoc/test/dtoc_test_phandle_reorder.dts b/tools/dtoc/test/dtoc_test_phandle_reorder.dts index aa71d56..573a4f6 100644 --- a/tools/dtoc/test/dtoc_test_phandle_reorder.dts +++ b/tools/dtoc/test/dtoc_test_phandle_reorder.dts @@ -10,13 +10,13 @@ / { phandle-source2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "source"; clocks = <&phandle>; }; phandle: phandle-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; #clock-cells = <0>; }; diff --git a/tools/dtoc/test/dtoc_test_phandle_single.dts b/tools/dtoc/test/dtoc_test_phandle_single.dts index aacd0b1..1b17639 100644 --- a/tools/dtoc/test/dtoc_test_phandle_single.dts +++ b/tools/dtoc/test/dtoc_test_phandle_single.dts @@ -9,14 +9,14 @@ / { phandle: phandle-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; intval = <0>; #clock-cells = <0>; }; phandle-source2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "source"; clocks = <&phandle>; }; diff --git a/tools/dtoc/test/dtoc_test_simple.dts b/tools/dtoc/test/dtoc_test_simple.dts index aef07ef..08f667e 100644 --- a/tools/dtoc/test/dtoc_test_simple.dts +++ b/tools/dtoc/test/dtoc_test_simple.dts @@ -11,7 +11,7 @@ #address-cells = <1>; #size-cells = <1>; spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; boolval; maybe-empty-int = <>; @@ -27,7 +27,7 @@ }; spl-test2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; intval = <3>; intarray = <5>; @@ -40,7 +40,7 @@ }; spl-test3 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; stringarray = "one"; longbytearray = [09 0a 0b 0c 0d 0e 0f 10]; @@ -49,12 +49,12 @@ i2c@0 { compatible = "sandbox,i2c"; - u-boot,dm-pre-reloc; + bootph-all; #address-cells = <1>; #size-cells = <0>; pmic@9 { compatible = "sandbox,pmic"; - u-boot,dm-pre-reloc; + bootph-all; reg = <9>; low-power; }; diff --git a/tools/dtoc/test/dtoc_test_single_reg.dts b/tools/dtoc/test/dtoc_test_single_reg.dts index 804b678..035937c 100644 --- a/tools/dtoc/test/dtoc_test_single_reg.dts +++ b/tools/dtoc/test/dtoc_test_single_reg.dts @@ -13,12 +13,12 @@ i2c@0 { compatible = "sandbox,i2c"; - u-boot,dm-pre-reloc; + bootph-all; #address-cells = <1>; #size-cells = <0>; pmic@9 { compatible = "sandbox,pmic"; - u-boot,dm-pre-reloc; + bootph-all; reg = <9>; low-power; diff --git a/tools/dtoc/test_dtoc.py b/tools/dtoc/test_dtoc.py index c62fcba..597c93e 100755 --- a/tools/dtoc/test_dtoc.py +++ b/tools/dtoc/test_dtoc.py @@ -13,6 +13,7 @@ import collections import copy import glob import os +import pathlib import struct import unittest @@ -25,10 +26,11 @@ from dtoc.dtb_platdata import get_value from dtoc.dtb_platdata import tab_to from dtoc.src_scan import conv_name_to_c from dtoc.src_scan import get_compat_name -from patman import test_util -from patman import tools +from u_boot_pylib import test_util +from u_boot_pylib import tools -OUR_PATH = os.path.dirname(os.path.realpath(__file__)) +DTOC_DIR = pathlib.Path(__file__).parent +TEST_DATA_DIR = DTOC_DIR / 'test/' HEADER = '''/* @@ -91,7 +93,7 @@ def get_dtb_file(dts_fname, capture_stderr=False): Returns: str: Filename of compiled file in output directory """ - return fdt_util.EnsureCompiled(os.path.join(OUR_PATH, 'test', dts_fname), + return fdt_util.EnsureCompiled(str(TEST_DATA_DIR / dts_fname), capture_stderr=capture_stderr) diff --git a/tools/dtoc/test_fdt.py b/tools/dtoc/test_fdt.py index 3b8ee00..32fa69c 100755 --- a/tools/dtoc/test_fdt.py +++ b/tools/dtoc/test_fdt.py @@ -30,8 +30,8 @@ from dtoc import fdt_util from dtoc.fdt_util import fdt32_to_cpu, fdt64_to_cpu from dtoc.fdt import Type, BytesToValue import libfdt -from patman import test_util -from patman import tools +from u_boot_pylib import test_util +from u_boot_pylib import tools #pylint: disable=protected-access @@ -132,10 +132,10 @@ class TestFdt(unittest.TestCase): """Tests obtaining a list of properties""" node = self.dtb.GetNode('/spl-test') props = self.dtb.GetProps(node) - self.assertEqual(['boolval', 'bytearray', 'byteval', 'compatible', - 'int64val', 'intarray', 'intval', 'longbytearray', - 'maybe-empty-int', 'notstring', 'stringarray', - 'stringval', 'u-boot,dm-pre-reloc'], + self.assertEqual(['boolval', 'bootph-all', 'bytearray', 'byteval', + 'compatible', 'int64val', 'intarray', 'intval', + 'longbytearray', 'maybe-empty-int', 'notstring', + 'stringarray', 'stringval', ], sorted(props.keys())) def test_check_error(self): @@ -814,7 +814,8 @@ def run_test_coverage(build_dir): build_dir (str): Directory containing the build output """ test_util.run_test_coverage('tools/dtoc/test_fdt.py', None, - ['tools/patman/*.py', '*test_fdt.py'], build_dir) + ['tools/patman/*.py', 'tools/u_boot_pylib/*', '*test_fdt.py'], + build_dir) def run_tests(names, processes): diff --git a/tools/dtoc/test_src_scan.py b/tools/dtoc/test_src_scan.py index f93cd7f..64b7408 100644 --- a/tools/dtoc/test_src_scan.py +++ b/tools/dtoc/test_src_scan.py @@ -15,8 +15,8 @@ import unittest from unittest import mock from dtoc import src_scan -from patman import test_util -from patman import tools +from u_boot_pylib import test_util +from u_boot_pylib import tools OUR_PATH = os.path.dirname(os.path.realpath(__file__)) diff --git a/tools/fdt_add_pubkey.c b/tools/fdt_add_pubkey.c new file mode 100644 index 0000000..999f5a7 --- /dev/null +++ b/tools/fdt_add_pubkey.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0+ +#include <image.h> +#include "fit_common.h" + +static const char *cmdname; + +static const char *algo_name = "sha1,rsa2048"; /* -a <algo> */ +static const char *keydir = "."; /* -k <keydir> */ +static const char *keyname = "key"; /* -n <keyname> */ +static const char *require_keys; /* -r <conf|image> */ +static const char *keydest; /* argv[n] */ + +static void print_usage(const char *msg) +{ + fprintf(stderr, "Error: %s\n", msg); + fprintf(stderr, "Usage: %s [-a <algo>] [-k <keydir>] [-n <keyname>] [-r <conf|image>]" + " <fdt blob>\n", cmdname); + fprintf(stderr, "Help information: %s [-h]\n", cmdname); + exit(EXIT_FAILURE); +} + +static void print_help(void) +{ + fprintf(stderr, "Options:\n" + "\t-a <algo> Cryptographic algorithm. Optional parameter, default value: sha1,rsa2048\n" + "\t-k <keydir> Directory with public key. Optional parameter, default value: .\n" + "\t-n <keyname> Public key name. Optional parameter, default value: key\n" + "\t-r <conf|image> Required: If present this indicates that the key must be verified for the image / configuration to be considered valid.\n" + "\t<fdt blob> FDT blob file for adding of the public key. Required parameter.\n"); + exit(EXIT_FAILURE); +} + +static void process_args(int argc, char *argv[]) +{ + int opt; + + while ((opt = getopt(argc, argv, "a:k:n:r:h")) != -1) { + switch (opt) { + case 'k': + keydir = optarg; + break; + case 'a': + algo_name = optarg; + break; + case 'n': + keyname = optarg; + break; + case 'r': + require_keys = optarg; + break; + case 'h': + print_help(); + default: + print_usage("Invalid option"); + } + } + /* The last parameter is expected to be the .dtb to add the public key to */ + if (optind < argc) + keydest = argv[optind]; + + if (!keydest) + print_usage("Missing dtb file to update"); +} + +static void reset_info(struct image_sign_info *info) +{ + if (!info) + fprintf(stderr, "Error: info is NULL in %s\n", __func__); + + memset(info, 0, sizeof(struct image_sign_info)); + + info->keydir = keydir; + info->keyname = keyname; + info->name = algo_name; + info->require_keys = require_keys; + info->crypto = image_get_crypto_algo(algo_name); + + if (!info->crypto) { + fprintf(stderr, "Unsupported signature algorithm '%s'\n", + algo_name); + exit(EXIT_FAILURE); + } +} + +static int add_pubkey(struct image_sign_info *info) +{ + int destfd = -1, ret; + void *dest_blob = NULL; + struct stat dest_sbuf; + size_t size_inc = 0; + + if (!info) + fprintf(stderr, "Error: info is NULL in %s\n", __func__); + + do { + if (destfd >= 0) { + munmap(dest_blob, dest_sbuf.st_size); + close(destfd); + + fprintf(stderr, ".dtb too small, increasing size by 1024 bytes\n"); + size_inc = 1024; + } + + destfd = mmap_fdt(cmdname, keydest, size_inc, &dest_blob, + &dest_sbuf, false, false); + if (destfd < 0) + exit(EXIT_FAILURE); + + ret = info->crypto->add_verify_data(info, dest_blob); + if (ret == -ENOSPC) + continue; + else if (ret < 0) + break; + } while (ret == -ENOSPC); + + return ret; +} + +int main(int argc, char *argv[]) +{ + struct image_sign_info info; + int ret; + + cmdname = argv[0]; + + process_args(argc, argv); + reset_info(&info); + ret = add_pubkey(&info); + + if (ret < 0) { + fprintf(stderr, "%s: Cannot add public key to FIT blob: %s\n", + cmdname, strerror(ret)); + exit(EXIT_FAILURE); + } + + exit(EXIT_SUCCESS); +} + diff --git a/tools/imx8image.c b/tools/imx8image.c index 395d5c6..c25ea84 100644 --- a/tools/imx8image.c +++ b/tools/imx8image.c @@ -829,7 +829,6 @@ static int build_container(soc_type_t soc, uint32_t sector_size, int ret; int container = -1; - int cont_img_count = 0; /* indexes to arrange the container */ memset((char *)&imx_header, 0, sizeof(imx_header_v3_t)); @@ -879,7 +878,6 @@ static int build_container(soc_type_t soc, uint32_t sector_size, img_sp->src = file_off; file_off += ALIGN(sbuf.st_size, sector_size); - cont_img_count++; break; case SECO: @@ -899,7 +897,6 @@ static int build_container(soc_type_t soc, uint32_t sector_size, img_sp->src = file_off; file_off += sbuf.st_size; - cont_img_count++; break; case NEW_CONTAINER: @@ -908,8 +905,6 @@ static int build_container(soc_type_t soc, uint32_t sector_size, CONTAINER_ALIGNMENT, CONTAINER_FLAGS_DEFAULT, fuse_version); - /* reset img count when moving to new container */ - cont_img_count = 0; scfw_flags = 0; break; diff --git a/tools/iot2050-sign-fw.sh b/tools/iot2050-sign-fw.sh new file mode 100755 index 0000000..4d1d794 --- /dev/null +++ b/tools/iot2050-sign-fw.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +if [ -z "$1" ]; then + echo "Usage: $0 KEY" + exit 1 +fi + +TEMP_X509=$(mktemp XXXXXXXX.temp) + +REVISION=${2:-0} +SHA_VAL=$(openssl dgst -sha512 -hex tispl.bin | sed -e "s/^.*= //g") +BIN_SIZE=$(stat -c %s tispl.bin) + +cat <<EOF >$TEMP_X509 +[ req ] +distinguished_name = req_distinguished_name +x509_extensions = v3_ca +prompt = no +dirstring_type = nobmp + +[ req_distinguished_name ] +CN = IOT2050 Firmware Signature + +[ v3_ca ] +basicConstraints = CA:true +1.3.6.1.4.1.294.1.3 = ASN1:SEQUENCE:swrv +1.3.6.1.4.1.294.1.34 = ASN1:SEQUENCE:sysfw_image_integrity + +[ swrv ] +swrv = INTEGER:$REVISION + +[ sysfw_image_integrity ] +shaType = OID:2.16.840.1.101.3.4.2.3 +shaValue = FORMAT:HEX,OCT:$SHA_VAL +imageSize = INTEGER:$BIN_SIZE +EOF + +CERT_X509=$(mktemp XXXXXXXX.crt) + +openssl req -new -x509 -key $1 -nodes -outform DER -out $CERT_X509 -config $TEMP_X509 -sha512 +cat $CERT_X509 tispl.bin > tispl.bin_signed +# currently broken in upstream +#source/tools/binman/binman replace -i flash.bin -f tispl.bin_signed blob@0x180000 +dd if=tispl.bin_signed of=flash.bin bs=$((0x1000)) seek=$((0x180000/0x1000)) conv=notrunc + +rm $TEMP_X509 $CERT_X509 + +tools/mkimage -G $1 -r -o sha256,rsa4096 -F fit@0x380000.fit +# currently broken in upstream +#source/tools/binman/binman replace -i flash.bin -f fit@0x380000.fit fit@0x380000 +dd if=fit@0x380000.fit of=flash.bin bs=$((0x1000)) seek=$((0x380000/0x1000)) conv=notrunc diff --git a/tools/key2dtsi.py b/tools/key2dtsi.py new file mode 100755 index 0000000..1dbb2cc --- /dev/null +++ b/tools/key2dtsi.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# +# Public key to dtsi converter. +# +# Copyright (c) Siemens AG, 2022 +# + +from argparse import ArgumentParser, FileType +from os.path import basename, splitext +from Cryptodome.PublicKey import RSA +from Cryptodome.Util.number import inverse + +def int_to_bytestr(n, length=None): + if not length: + length = (n.bit_length() + 7) // 8 + byte_array = n.to_bytes(length, 'big') + return ' '.join(['{:02x}'.format(byte) for byte in byte_array]) + +ap = ArgumentParser(description='Public key to dtsi converter') + +ap.add_argument('--hash', '-H', default='sha256', + help='hash to be used with key (default: sha256)') +ap.add_argument('--required-conf', '-c', action='store_true', + help='mark key required for configuration') +ap.add_argument('--required-image', '-i', action='store_true', + help='mark key required for image') +ap.add_argument('--spl', '-s', action='store_true', + help='mark key for usage in SPL') +ap.add_argument('key_file', metavar='KEY_FILE', type=FileType('r'), + help='key file (formats: X.509, PKCS#1, OpenSSH)') +ap.add_argument('dtsi_file', metavar='DTSI_FILE', type=FileType('w'), + help='dtsi output file') + +args = ap.parse_args() + +key_name, _ = splitext(basename(args.key_file.name)) + +key_data = args.key_file.read() +key = RSA.importKey(key_data) + +r_squared = (2**key.size_in_bits())**2 % key.n +n0_inverse = 2**32 - inverse(key.n, 2**32) + +out = args.dtsi_file +out.write('/ {\n') +out.write('\tsignature {\n') +out.write('\t\tkey-{} {{\n'.format(key_name)) +out.write('\t\t\tkey-name-hint = "{}";\n'.format(key_name)) +out.write('\t\t\talgo = "{},rsa{}";\n'.format(args.hash, key.size_in_bits())) +out.write('\t\t\trsa,num-bits = <{}>;\n'.format(key.size_in_bits())) +out.write('\t\t\trsa,modulus = [{}];\n'.format(int_to_bytestr(key.n))) +out.write('\t\t\trsa,exponent = [{}];\n'.format(int_to_bytestr(key.e, 8))) +out.write('\t\t\trsa,r-squared = [{}];\n'.format(int_to_bytestr(r_squared))) +out.write('\t\t\trsa,n0-inverse = <0x{:x}>;\n'.format(n0_inverse)) +if args.required_conf: + out.write('\t\t\trequired = "conf";\n') +elif args.required_image: + out.write('\t\t\trequired = "image";\n') +if args.spl: + out.write('\t\t\tu-boot,dm-spl;\n') +out.write('\t\t};\n') +out.write('\t};\n') +out.write('};\n') diff --git a/tools/kwbimage.c b/tools/kwbimage.c index 6abb9f2..177084a 100644 --- a/tools/kwbimage.c +++ b/tools/kwbimage.c @@ -927,6 +927,71 @@ done: return ret; } +static int image_fill_xip_header(void *image, struct image_tool_params *params) +{ + struct main_hdr_v1 *main_hdr = image; /* kwbimage v0 and v1 have same XIP members */ + int version = kwbimage_version(image); + uint32_t srcaddr = le32_to_cpu(main_hdr->srcaddr); + uint32_t startaddr = 0; + + if (main_hdr->blockid != IBR_HDR_SPI_ID) { + fprintf(stderr, "XIP is supported only for SPI images\n"); + return 0; + } + + if (version == 0 && + params->addr >= 0xE8000000 && params->addr < 0xEFFFFFFF && + params->ep >= 0xE8000000 && params->ep < 0xEFFFFFFF) { + /* Load and Execute address is in SPI address space (kwbimage v0) */ + startaddr = 0xE8000000; + } else if (version != 0 && + params->addr >= 0xD4000000 && params->addr < 0xD7FFFFFF && + params->ep >= 0xD4000000 && params->ep < 0xD7FFFFFF) { + /* Load and Execute address is in SPI address space (kwbimage v1) */ + startaddr = 0xD4000000; + } else if (version != 0 && + params->addr >= 0xD8000000 && params->addr < 0xDFFFFFFF && + params->ep >= 0xD8000000 && params->ep < 0xDFFFFFFF) { + /* Load and Execute address is in Device bus space (kwbimage v1) */ + startaddr = 0xD8000000; + } else if (params->addr != 0x0) { + /* Load address is non-zero */ + if (version == 0) + fprintf(stderr, "XIP Load Address or XIP Entry Point is not in SPI address space\n"); + else + fprintf(stderr, "XIP Load Address or XIP Entry Point is not in SPI nor in Device bus address space\n"); + return 0; + } + + /* + * For XIP destaddr must be set to 0xFFFFFFFF and + * execaddr relative to the start of XIP memory address space. + */ + main_hdr->destaddr = cpu_to_le32(0xFFFFFFFF); + + if (startaddr == 0) { + /* + * mkimage's --load-address 0x0 means that binary is Position + * Independent and in this case mkimage's --entry-point address + * is relative offset from beginning of the data part of image. + */ + main_hdr->execaddr = cpu_to_le32(srcaddr + params->ep); + } else { + /* The lowest possible load address is after the header at srcaddr. */ + if (params->addr - startaddr < srcaddr) { + fprintf(stderr, + "Invalid XIP Load Address 0x%08x.\n" + "The lowest address for this configuration is 0x%08x.\n", + params->addr, (unsigned)(startaddr + srcaddr)); + return 0; + } + main_hdr->srcaddr = cpu_to_le32(params->addr - startaddr); + main_hdr->execaddr = cpu_to_le32(params->ep - startaddr); + } + + return 1; +} + static size_t image_headersz_align(size_t headersz, uint8_t blockid) { /* @@ -959,10 +1024,10 @@ static size_t image_headersz_v0(int *hasext) *hasext = 1; } - return image_headersz_align(headersz, image_get_bootfrom()); + return headersz; } -static void *image_create_v0(size_t *imagesz, struct image_tool_params *params, +static void *image_create_v0(size_t *dataoff, struct image_tool_params *params, int payloadsz) { struct image_cfg_element *e; @@ -972,10 +1037,11 @@ static void *image_create_v0(size_t *imagesz, struct image_tool_params *params, int has_ext = 0; /* - * Calculate the size of the header and the size of the + * Calculate the size of the header and the offset of the * payload */ headersz = image_headersz_v0(&has_ext); + *dataoff = image_headersz_align(headersz, image_get_bootfrom()); image = malloc(headersz); if (!image) { @@ -990,7 +1056,7 @@ static void *image_create_v0(size_t *imagesz, struct image_tool_params *params, /* Fill in the main header */ main_hdr->blocksize = cpu_to_le32(payloadsz); - main_hdr->srcaddr = cpu_to_le32(headersz); + main_hdr->srcaddr = cpu_to_le32(*dataoff); main_hdr->ext = has_ext; main_hdr->version = 0; main_hdr->destaddr = cpu_to_le32(params->addr); @@ -1009,31 +1075,26 @@ static void *image_create_v0(size_t *imagesz, struct image_tool_params *params, e = image_find_option(IMAGE_CFG_NAND_BADBLK_LOCATION); if (e) main_hdr->nandbadblklocation = e->nandbadblklocation; - main_hdr->checksum = image_checksum8(image, - sizeof(struct main_hdr_v0)); /* - * For SATA srcaddr is specified in number of sectors starting from - * sector 0. The main header is stored at sector number 1. + * For SATA srcaddr is specified in number of sectors. * This expects the sector size to be 512 bytes. - * Header size is already aligned. */ if (main_hdr->blockid == IBR_HDR_SATA_ID) - main_hdr->srcaddr = cpu_to_le32(headersz / 512 + 1); - - /* - * For SDIO srcaddr is specified in number of sectors starting from - * sector 0. The main header is stored at sector number 0. - * This expects sector size to be 512 bytes. - * Header size is already aligned. - */ - if (main_hdr->blockid == IBR_HDR_SDIO_ID) - main_hdr->srcaddr = cpu_to_le32(headersz / 512); + main_hdr->srcaddr = cpu_to_le32(le32_to_cpu(main_hdr->srcaddr) / 512); /* For PCIe srcaddr is not used and must be set to 0xFFFFFFFF. */ if (main_hdr->blockid == IBR_HDR_PEX_ID) main_hdr->srcaddr = cpu_to_le32(0xFFFFFFFF); + if (params->xflag) { + if (!image_fill_xip_header(main_hdr, params)) { + free(image); + return NULL; + } + *dataoff = le32_to_cpu(main_hdr->srcaddr); + } + /* Generate the ext header */ if (has_ext) { struct ext_hdr_v0 *ext_hdr; @@ -1059,7 +1120,9 @@ static void *image_create_v0(size_t *imagesz, struct image_tool_params *params, sizeof(struct ext_hdr_v0)); } - *imagesz = headersz; + main_hdr->checksum = image_checksum8(image, + sizeof(struct main_hdr_v0)); + return image; } @@ -1073,10 +1136,6 @@ static size_t image_headersz_v1(int *hasext) int cfgi; int ret; - /* - * Calculate the size of the header and the size of the - * payload - */ headersz = sizeof(struct main_hdr_v1); if (image_get_csk_index() >= 0) { @@ -1172,7 +1231,17 @@ static size_t image_headersz_v1(int *hasext) if (count > 0) headersz += sizeof(struct register_set_hdr_v1) + 8 * count + 4; - return image_headersz_align(headersz, image_get_bootfrom()); + /* + * For all images except UART, headersz stored in header itself should + * contains header size without padding. For UART image BootROM rounds + * down headersz to multiply of 128 bytes. Therefore align UART headersz + * to multiply of 128 bytes to ensure that remaining UART header bytes + * are not ignored by BootROM. + */ + if (image_get_bootfrom() == IBR_HDR_UART_ID) + headersz = ALIGN(headersz, 128); + + return headersz; } static int add_binary_header_v1(uint8_t **cur, uint8_t **next_ext, @@ -1331,16 +1400,14 @@ static int kwb_sign_csk_with_kak(struct image_tool_params *params, return 0; } -static int add_secure_header_v1(struct image_tool_params *params, uint8_t *ptr, - int payloadsz, size_t headersz, uint8_t *image, +static int add_secure_header_v1(struct image_tool_params *params, uint8_t *image_ptr, + size_t image_size, uint8_t *header_ptr, size_t headersz, struct secure_hdr_v1 *secure_hdr) { struct image_cfg_element *e_jtagdelay; struct image_cfg_element *e_boxid; struct image_cfg_element *e_flashid; RSA *csk = NULL; - unsigned char *image_ptr; - size_t image_size; struct sig_v1 tmp_sig; bool specialized_img = image_get_spezialized_img(); @@ -1366,14 +1433,11 @@ static int add_secure_header_v1(struct image_tool_params *params, uint8_t *ptr, if (kwb_sign_csk_with_kak(params, secure_hdr, csk)) return 1; - image_ptr = ptr + headersz; - image_size = payloadsz - headersz; - - if (kwb_sign_and_verify(csk, image_ptr, image_size, + if (kwb_sign_and_verify(csk, image_ptr, image_size - 4, &secure_hdr->imgsig, "image") < 0) return 1; - if (kwb_sign_and_verify(csk, image, headersz, &tmp_sig, "header") < 0) + if (kwb_sign_and_verify(csk, header_ptr, headersz, &tmp_sig, "header") < 0) return 1; secure_hdr->hdrsig = tmp_sig; @@ -1399,12 +1463,11 @@ static void finish_register_set_header_v1(uint8_t **cur, uint8_t **next_ext, *datai = 0; } -static void *image_create_v1(size_t *imagesz, struct image_tool_params *params, +static void *image_create_v1(size_t *dataoff, struct image_tool_params *params, uint8_t *ptr, int payloadsz) { struct image_cfg_element *e; struct main_hdr_v1 *main_hdr; - struct opt_hdr_v1 *ohdr; struct register_set_hdr_v1 *register_set_hdr; struct secure_hdr_v1 *secure_hdr = NULL; size_t headersz; @@ -1415,12 +1478,13 @@ static void *image_create_v1(size_t *imagesz, struct image_tool_params *params, uint8_t delay; /* - * Calculate the size of the header and the size of the + * Calculate the size of the header and the offset of the * payload */ headersz = image_headersz_v1(&hasext); if (headersz == 0) return NULL; + *dataoff = image_headersz_align(headersz, image_get_bootfrom()); image = malloc(headersz); if (!image) { @@ -1442,7 +1506,7 @@ static void *image_create_v1(size_t *imagesz, struct image_tool_params *params, main_hdr->headersz_msb = (headersz & 0xFFFF0000) >> 16; main_hdr->destaddr = cpu_to_le32(params->addr); main_hdr->execaddr = cpu_to_le32(params->ep); - main_hdr->srcaddr = cpu_to_le32(headersz); + main_hdr->srcaddr = cpu_to_le32(*dataoff); main_hdr->ext = hasext; main_hdr->version = 1; main_hdr->blockid = image_get_bootfrom(); @@ -1470,27 +1534,24 @@ static void *image_create_v1(size_t *imagesz, struct image_tool_params *params, main_hdr->flags = e->debug ? 0x1 : 0; /* - * For SATA srcaddr is specified in number of sectors starting from - * sector 0. The main header is stored at sector number 1. + * For SATA srcaddr is specified in number of sectors. * This expects the sector size to be 512 bytes. - * Header size is already aligned. */ if (main_hdr->blockid == IBR_HDR_SATA_ID) - main_hdr->srcaddr = cpu_to_le32(headersz / 512 + 1); - - /* - * For SDIO srcaddr is specified in number of sectors starting from - * sector 0. The main header is stored at sector number 0. - * This expects sector size to be 512 bytes. - * Header size is already aligned. - */ - if (main_hdr->blockid == IBR_HDR_SDIO_ID) - main_hdr->srcaddr = cpu_to_le32(headersz / 512); + main_hdr->srcaddr = cpu_to_le32(le32_to_cpu(main_hdr->srcaddr) / 512); /* For PCIe srcaddr is not used and must be set to 0xFFFFFFFF. */ if (main_hdr->blockid == IBR_HDR_PEX_ID) main_hdr->srcaddr = cpu_to_le32(0xFFFFFFFF); + if (params->xflag) { + if (!image_fill_xip_header(main_hdr, params)) { + free(image); + return NULL; + } + *dataoff = le32_to_cpu(main_hdr->srcaddr); + } + if (image_get_csk_index() >= 0) { /* * only reserve the space here; we fill the header later since @@ -1552,19 +1613,10 @@ static void *image_create_v1(size_t *imagesz, struct image_tool_params *params, &datai, delay); } - if (secure_hdr && add_secure_header_v1(params, ptr, payloadsz + headersz, - headersz, image, secure_hdr)) + if (secure_hdr && add_secure_header_v1(params, ptr + *dataoff, payloadsz, + image, headersz, secure_hdr)) return NULL; - *imagesz = headersz; - - /* Fill the real header size without padding into the main header */ - headersz = sizeof(*main_hdr); - for_each_opt_hdr_v1 (ohdr, main_hdr) - headersz += opt_hdr_v1_size(ohdr); - main_hdr->headersz_lsb = cpu_to_le16(headersz & 0xFFFF); - main_hdr->headersz_msb = (headersz & 0xFFFF0000) >> 16; - /* Calculate and set the header checksum */ main_hdr->checksum = image_checksum8(main_hdr, headersz); @@ -1835,7 +1887,7 @@ static void kwbimage_set_header(void *ptr, struct stat *sbuf, int ifd, FILE *fcfg; void *image = NULL; int version; - size_t headersz = 0; + size_t dataoff = 0; size_t datasz; uint32_t checksum; struct stat s; @@ -1845,7 +1897,9 @@ static void kwbimage_set_header(void *ptr, struct stat *sbuf, int ifd, * Do not use sbuf->st_size as it contains size with padding. * We need original image data size, so stat original file. */ - if (stat(params->datafile, &s)) { + if (params->skipcpy) { + s.st_size = 0; + } else if (stat(params->datafile, &s)) { fprintf(stderr, "Could not stat data file %s: %s\n", params->datafile, strerror(errno)); exit(EXIT_FAILURE); @@ -1886,11 +1940,11 @@ static void kwbimage_set_header(void *ptr, struct stat *sbuf, int ifd, */ case -1: case 0: - image = image_create_v0(&headersz, params, datasz + 4); + image = image_create_v0(&dataoff, params, datasz + 4); break; case 1: - image = image_create_v1(&headersz, params, ptr, datasz + 4); + image = image_create_v1(&dataoff, params, ptr, datasz + 4); break; default: @@ -1908,12 +1962,12 @@ static void kwbimage_set_header(void *ptr, struct stat *sbuf, int ifd, free(image_cfg); /* Build and add image data checksum */ - checksum = cpu_to_le32(image_checksum32((uint8_t *)ptr + headersz, + checksum = cpu_to_le32(image_checksum32((uint8_t *)ptr + dataoff, datasz)); - memcpy((uint8_t *)ptr + headersz + datasz, &checksum, sizeof(uint32_t)); + memcpy((uint8_t *)ptr + dataoff + datasz, &checksum, sizeof(uint32_t)); /* Finally copy the header into the image area */ - memcpy(ptr, image, headersz); + memcpy(ptr, image, kwbheader_size(image)); free(image); } @@ -1933,9 +1987,9 @@ static void kwbimage_print_header(const void *ptr) printf("BIN Img Size: "); genimg_print_size(opt_hdr_v1_size(ohdr) - 12 - 4 * ohdr->data[0]); - printf("BIN Img Offs: %08x\n", - (unsigned)((uint8_t *)ohdr - (uint8_t *)mhdr) + - 8 + 4 * ohdr->data[0]); + printf("BIN Img Offs: "); + genimg_print_size(((uint8_t *)ohdr - (uint8_t *)mhdr) + + 8 + 4 * ohdr->data[0]); } } @@ -1947,9 +2001,20 @@ static void kwbimage_print_header(const void *ptr) } printf("Data Size: "); - genimg_print_size(mhdr->blocksize - sizeof(uint32_t)); - printf("Load Address: %08x\n", mhdr->destaddr); - printf("Entry Point: %08x\n", mhdr->execaddr); + genimg_print_size(le32_to_cpu(mhdr->blocksize) - sizeof(uint32_t)); + printf("Data Offset: "); + if (mhdr->blockid == IBR_HDR_SATA_ID) + printf("%u Sector%s (LBA)\n", le32_to_cpu(mhdr->srcaddr), + le32_to_cpu(mhdr->srcaddr) != 1 ? "s" : ""); + else + genimg_print_size(le32_to_cpu(mhdr->srcaddr)); + if (mhdr->blockid == IBR_HDR_SPI_ID && le32_to_cpu(mhdr->destaddr) == 0xFFFFFFFF) { + printf("Load Address: XIP\n"); + printf("Execute Offs: %08x\n", le32_to_cpu(mhdr->execaddr)); + } else { + printf("Load Address: %08x\n", le32_to_cpu(mhdr->destaddr)); + printf("Entry Point: %08x\n", le32_to_cpu(mhdr->execaddr)); + } } static int kwbimage_check_image_types(uint8_t type) @@ -2028,23 +2093,9 @@ static int kwbimage_verify_header(unsigned char *ptr, int image_size, /* * For SATA srcaddr is specified in number of sectors. - * The main header is must be stored at sector number 1. - * This expects that sector size is 512 bytes and recalculates - * data offset to bytes relative to the main header. + * This expects that sector size is 512 bytes. */ - if (blockid == IBR_HDR_SATA_ID) { - if (offset < 1) - return -FDT_ERR_BADSTRUCTURE; - offset -= 1; - offset *= 512; - } - - /* - * For SDIO srcaddr is specified in number of sectors. - * This expects that sector size is 512 bytes and recalculates - * data offset to bytes. - */ - if (blockid == IBR_HDR_SDIO_ID) + if (blockid == IBR_HDR_SATA_ID) offset *= 512; /* @@ -2067,6 +2118,8 @@ static int kwbimage_verify_header(unsigned char *ptr, int image_size, return 0; } +static int kwbimage_align_size(int bootfrom, int alloc_len, struct stat s); + static int kwbimage_generate(struct image_tool_params *params, struct image_type_params *tparams) { @@ -2085,7 +2138,9 @@ static int kwbimage_generate(struct image_tool_params *params, exit(EXIT_FAILURE); } - if (stat(params->datafile, &s)) { + if (params->skipcpy) { + s.st_size = 0; + } else if (stat(params->datafile, &s)) { fprintf(stderr, "Could not stat data file %s: %s\n", params->datafile, strerror(errno)); exit(EXIT_FAILURE); @@ -2141,6 +2196,8 @@ static int kwbimage_generate(struct image_tool_params *params, exit(EXIT_FAILURE); } + alloc_len = image_headersz_align(alloc_len, image_get_bootfrom()); + free(image_cfg); hdr = malloc(alloc_len); @@ -2155,6 +2212,22 @@ static int kwbimage_generate(struct image_tool_params *params, tparams->hdr = hdr; /* + * This function should return aligned size of the datafile. + * When skipcpy is set (datafile is skipped) then return value of this + * function is ignored, so we have to put required kwbimage aligning + * into the preallocated header size. + */ + if (params->skipcpy) { + tparams->header_size += kwbimage_align_size(bootfrom, alloc_len, s); + return 0; + } else { + return kwbimage_align_size(bootfrom, alloc_len, s); + } +} + +static int kwbimage_align_size(int bootfrom, int alloc_len, struct stat s) +{ + /* * The resulting image needs to be 4-byte aligned. At least * the Marvell hdrparser tool complains if its unaligned. * After the image data is stored 4-byte checksum. @@ -2182,6 +2255,7 @@ static int kwbimage_generate_config(void *ptr, struct image_tool_params *params) struct ext_hdr_v0 *ehdr0; struct bin_hdr_v0 *bhdr0; struct opt_hdr_v1 *ohdr; + int regset_count; int params_count; unsigned offset; int is_v0_ext; @@ -2215,12 +2289,18 @@ static int kwbimage_generate_config(void *ptr, struct image_tool_params *params) fprintf(f, "NAND_ECC_MODE %s\n", image_nand_ecc_mode_name(mhdr0->nandeccmode)); if (mhdr->blockid == IBR_HDR_NAND_ID) - fprintf(f, "NAND_PAGE_SIZE 0x%x\n", (unsigned)mhdr->nandpagesize); + fprintf(f, "NAND_PAGE_SIZE 0x%x\n", (unsigned)le16_to_cpu(mhdr->nandpagesize)); - if (version != 0 && mhdr->blockid == IBR_HDR_NAND_ID) - fprintf(f, "NAND_BLKSZ 0x%x\n", (unsigned)mhdr->nandblocksize); + if (mhdr->blockid == IBR_HDR_NAND_ID && (version != 0 || is_v0_ext || mhdr->nandblocksize != 0)) { + if (mhdr->nandblocksize != 0) /* block size explicitly set in 64 kB unit */ + fprintf(f, "NAND_BLKSZ 0x%x\n", (unsigned)mhdr->nandblocksize * 64*1024); + else if (le16_to_cpu(mhdr->nandpagesize) > 512) + fprintf(f, "NAND_BLKSZ 0x10000\n"); /* large page NAND flash = 64 kB block size */ + else + fprintf(f, "NAND_BLKSZ 0x4000\n"); /* small page NAND flash = 16 kB block size */ + } - if (mhdr->blockid == IBR_HDR_NAND_ID && (mhdr->nandbadblklocation != 0 || is_v0_ext)) + if (mhdr->blockid == IBR_HDR_NAND_ID && (version != 0 || is_v0_ext)) fprintf(f, "NAND_BADBLK_LOCATION 0x%x\n", (unsigned)mhdr->nandbadblklocation); if (version == 0 && mhdr->blockid == IBR_HDR_SATA_ID) @@ -2266,18 +2346,20 @@ static int kwbimage_generate_config(void *ptr, struct image_tool_params *params) cur_idx++; } else if (ohdr->headertype == OPT_HDR_V1_REGISTER_TYPE) { regset_hdr = (struct register_set_hdr_v1 *)ohdr; - for (i = 0; - i < opt_hdr_v1_size(ohdr) - sizeof(struct opt_hdr_v1) - - sizeof(regset_hdr->data[0].last_entry); - i++) + if (opt_hdr_v1_size(ohdr) > sizeof(*ohdr)) + regset_count = (opt_hdr_v1_size(ohdr) - sizeof(*ohdr)) / + sizeof(regset_hdr->data[0].entry); + else + regset_count = 0; + for (i = 0; i < regset_count; i++) fprintf(f, "DATA 0x%08x 0x%08x\n", le32_to_cpu(regset_hdr->data[i].entry.address), le32_to_cpu(regset_hdr->data[i].entry.value)); - if (opt_hdr_v1_size(ohdr) - sizeof(struct opt_hdr_v1) >= - sizeof(regset_hdr->data[0].last_entry)) { - if (regset_hdr->data[0].last_entry.delay) + if (regset_count > 0) { + if (regset_hdr->data[regset_count-1].last_entry.delay != + REGISTER_SET_HDR_OPT_DELAY_SDRAM_SETUP) fprintf(f, "DATA_DELAY %u\n", - (unsigned)regset_hdr->data[0].last_entry.delay); + (unsigned)regset_hdr->data[regset_count-1].last_entry.delay); else fprintf(f, "DATA_DELAY SDRAM_SETUP\n"); } @@ -2403,12 +2485,7 @@ static int kwbimage_extract_subimage(void *ptr, struct image_tool_params *params /* Extract data image when -p is not specified or when '-p 0' is specified */ offset = le32_to_cpu(mhdr->srcaddr); - if (mhdr->blockid == IBR_HDR_SATA_ID) { - offset -= 1; - offset *= 512; - } - - if (mhdr->blockid == IBR_HDR_SDIO_ID) + if (mhdr->blockid == IBR_HDR_SATA_ID) offset *= 512; if (mhdr->blockid == IBR_HDR_PEX_ID && offset == 0xFFFFFFFF) @@ -2455,9 +2532,6 @@ static int kwbimage_extract_subimage(void *ptr, struct image_tool_params *params return imagetool_save_subimage(params->outfile, image, size); } -/* - * Report Error if xflag is set in addition to default - */ static int kwbimage_check_params(struct image_tool_params *params) { if (!params->lflag && !params->iflag && !params->pflag && @@ -2468,10 +2542,9 @@ static int kwbimage_check_params(struct image_tool_params *params) return 1; } - return (params->dflag && (params->fflag || params->lflag)) || - (params->fflag && (params->dflag || params->lflag)) || - (params->lflag && (params->dflag || params->fflag)) || - (params->xflag); + return (params->dflag && (params->fflag || params->lflag || params->skipcpy)) || + (params->fflag) || + (params->lflag && (params->dflag || params->fflag)); } /* diff --git a/tools/kwboot.c b/tools/kwboot.c index da4fe32..348a320 100644 --- a/tools/kwboot.c +++ b/tools/kwboot.c @@ -15,6 +15,12 @@ * Processor, and High-Definition Video Decoder: Functional Specifications" * August 3, 2011. Chapter 5 "BootROM Firmware" * https://web.archive.org/web/20120130172443/https://www.marvell.com/application-processors/armada-500/assets/Armada-510-Functional-Spec.pdf + * - "88F6665, 88F6660, 88F6658, 88F6655, 88F6655F, 88F6650, 88F6650F, 88F6610, + * and 88F6610F Avanta LP Family Integrated Single/Dual CPU Ecosystem for + * Gateway (GW), Home Gateway Unit (HGU), and Single Family Unit (SFU) + * Functional Specifications" Doc. No. MV-S108952-00, Rev. A. November 7, 2013. + * Chapter 7 "Boot Flow" + * CONFIDENTIAL, no public documentation available * - "88F6710, 88F6707, and 88F6W11: ARMADA(R) 370 SoC: Functional Specifications" * May 26, 2014. Chapter 6 "BootROM Firmware". * https://web.archive.org/web/20140617183701/https://www.marvell.com/embedded-processors/armada-300/assets/ARMADA370-FunctionalSpec-datasheet.pdf @@ -22,6 +28,15 @@ * Multi-Core ARMv7 Based SoC Processors: Functional Specifications" * May 29, 2014. Chapter 6 "BootROM Firmware". * https://web.archive.org/web/20180829171131/https://www.marvell.com/embedded-processors/armada-xp/assets/ARMADA-XP-Functional-SpecDatasheet.pdf + * - "BobCat2 Control and Management Subsystem Functional Specifications" + * Doc. No. MV-S109400-00, Rev. A. December 4, 2014. + * Chapter 1.6 BootROM Firmware + * CONFIDENTIAL, no public documentation available + * - "AlleyCat3 and PONCat3 Highly Integrated 1/10 Gigabit Ethernet Switch + * Control and Management Subsystem: Functional Specifications" + * Doc. No. MV-S109693-00, Rev. A. May 20, 2014. + * Chapter 1.6 BootROM Firmware + * CONFIDENTIAL, no public documentation available * - "ARMADA(R) 375 Value-Performance Dual Core CPU System on Chip: Functional * Specifications" Doc. No. MV-S109377-00, Rev. A. September 18, 2013. * Chapter 7 "Boot Sequence" @@ -35,6 +50,85 @@ * System on Chip Functional Specifications" Doc. No. MV-S109896-00, Rev. B. * December 22, 2015. Chapter 7 "Boot Flow" * CONFIDENTIAL, no public documentation available + * - "Marvell boot image parser", Marvell U-Boot 2013.01, version 18.06. September 17, 2015. + * https://github.com/MarvellEmbeddedProcessors/u-boot-marvell/blob/u-boot-2013.01-armada-18.06/tools/marvell/doimage_mv/hdrparser.c + * - "Marvell doimage Tool", Marvell U-Boot 2013.01, version 18.06. August 30, 2015. + * https://github.com/MarvellEmbeddedProcessors/u-boot-marvell/blob/u-boot-2013.01-armada-18.06/tools/marvell/doimage_mv/doimage.c + * + * Storage location / offset of different image types: + * - IBR_HDR_SPI_ID (0x5A): + * SPI image can be stored at any 2 MB aligned offset in the first 16 MB of + * SPI-NOR or parallel-NOR. Despite the type name it really can be stored on + * parallel-NOR and cannot be stored on other SPI devices, like SPI-NAND. + * So it should have been named NOR image, not SPI image. This image type + * supports XIP - Execute In Place directly from NOR memory. Destination + * address of the XIP image is set to 0xFFFFFFFF and execute address to the + * absolute offset in bytes from the beginning of NOR memory. + * + * - IBR_HDR_NAND_ID (0x8B): + * NAND image can be stored either at any 2 MB aligned offset in the first + * 16 MB of SPI-NAND or at any blocksize aligned offset in the first 64 MB + * of parallel-NAND. + * + * - IBR_HDR_PEX_ID (0x9C): + * PEX image is used for booting from PCI Express device. Source address + * stored in image is ignored by BootROM. It is not the BootROM who parses + * or loads data part of the PEX image. BootROM just configures SoC to the + * PCIe endpoint mode and let the PCIe device on the other end of the PCIe + * link (which must be in Root Complex mode) to load kwbimage into SoC's + * memory and tell BootROM physical address. + * + * - IBR_HDR_UART_ID (0x69): + * UART image can be transfered via xmodem protocol over first UART. + * Unlike all other image types, header size stored in the image must be + * multiply of the 128 bytes (for all other image types it can be any size) + * and data part of the image does not have to contain 32-bit checksum + * (all other image types must have valid 32-bit checksum in its data part). + * And data size stored in the image is ignored. A38x BootROM determinates + * size of the data part implicitly by the end of the xmodem transfer. + * A38x BootROM has a bug which cause that BootROM loads data part of UART + * image into RAM target address increased by one byte when source address + * and header size stored in the image header are not same. So UART image + * should be constructed in a way that there is no gap between header and + * data part. + * + * - IBR_HDR_I2C_ID (0x4D): + * It is unknown for what kind of storage is used this image. It is not + * specified in any document from References section. + * + * - IBR_HDR_SATA_ID (0x78): + * SATA image can be stored at sector 1 (after the MBR table), sector 34 + * (after the GPT table) or at any next sector which is aligned to 2 MB and + * is in the first 16 MB of SATA disk. Note that source address in SATA image + * is stored in sector unit and not in bytes like for any other images. + * Unfortunately sector size is disk specific, in most cases it is 512 bytes + * but there are also Native 4K SATA disks which have 4096 bytes long sectors. + * + * - IBR_HDR_SDIO_ID (0xAE): + * SDIO image can be stored on different medias: + * - SD(SC) card + * - SDHC/SDXC card + * - eMMC HW boot partition + * - eMMC user data partition / MMC card + * It cannot be stored on SDIO card despite the image name. + * + * For SD(SC)/SDHC/SDXC cards, image can be stored at the same locations as + * the SATA image (sector 1, sector 34 or any 2 MB aligned sector) but within + * the first 64 MB. SDHC and SDXC cards have fixed 512 bytes long sector size. + * Old SD(SC) cards unfortunately can have also different sector sizes, mostly + * 1024 bytes long sector sizes and also can be changed at runtime. + * + * For MMC-compatible devices, image can be stored at offset 0 or at offset + * 2 MB. If MMC device supports HW boot partitions then image must be stored + * on the HW partition as is configured in the EXT_CSC register (it can be + * either boot or user data). + * + * Note that source address for SDIO image is stored in byte unit, like for + * any other images (except SATA). Marvell Functional Specifications for + * A38x and A39x SoCs say that source address is in sector units, but this + * is purely incorrect information. A385 BootROM really expects source address + * for SDIO images in bytes and also Marvell tools generate SDIO image with + * source address in byte units. */ #include "kwbimage.h" @@ -1374,6 +1468,8 @@ kwboot_xmodem(int tty, const void *_img, size_t size, int baudrate) * followed by the header. So align header size to xmodem block size. */ hdrsz += (KWBOOT_XM_BLKSZ - hdrsz % KWBOOT_XM_BLKSZ) % KWBOOT_XM_BLKSZ; + if (hdrsz > size) + hdrsz = size; pnum = 1; @@ -1699,6 +1795,47 @@ kwboot_img_is_secure(void *img) return 0; } +static int +kwboot_img_has_ddr_init(void *img) +{ + const struct register_set_hdr_v1 *rhdr; + const struct main_hdr_v0 *hdr0; + struct opt_hdr_v1 *ohdr; + u32 ohdrsz; + int last; + + /* + * kwbimage v0 image headers contain DDR init code either in + * extension header or in binary code header. + */ + if (kwbimage_version(img) == 0) { + hdr0 = img; + return hdr0->ext || hdr0->bin; + } + + /* + * kwbimage v1 image headers contain DDR init code either in binary + * code header or in a register set list header with SDRAM_SETUP. + */ + for_each_opt_hdr_v1 (ohdr, img) { + if (ohdr->headertype == OPT_HDR_V1_BINARY_TYPE) + return 1; + if (ohdr->headertype == OPT_HDR_V1_REGISTER_TYPE) { + rhdr = (const struct register_set_hdr_v1 *)ohdr; + ohdrsz = opt_hdr_v1_size(ohdr); + if (ohdrsz >= sizeof(*ohdr) + sizeof(rhdr->data[0].last_entry)) { + ohdrsz -= sizeof(*ohdr) + sizeof(rhdr->data[0].last_entry); + last = ohdrsz / sizeof(rhdr->data[0].entry); + if (rhdr->data[last].last_entry.delay == + REGISTER_SET_HDR_OPT_DELAY_SDRAM_SETUP) + return 1; + } + } + } + + return 0; +} + static void * kwboot_img_grow_data_right(void *img, size_t *size, size_t grow) { @@ -1854,10 +1991,26 @@ _inject_baudrate_change_code(void *img, size_t *size, int for_data, } } +static const char * +kwboot_img_type(uint8_t blockid) +{ + switch (blockid) { + case IBR_HDR_I2C_ID: return "I2C"; + case IBR_HDR_SPI_ID: return "SPI"; + case IBR_HDR_NAND_ID: return "NAND"; + case IBR_HDR_SATA_ID: return "SATA"; + case IBR_HDR_PEX_ID: return "PEX"; + case IBR_HDR_UART_ID: return "UART"; + case IBR_HDR_SDIO_ID: return "SDIO"; + default: return "unknown"; + } +} + static int kwboot_img_patch(void *img, size_t *size, int baudrate) { struct main_hdr_v1 *hdr; + struct opt_hdr_v1 *ohdr; uint32_t srcaddr; uint8_t csum; size_t hdrsz; @@ -1866,8 +2019,10 @@ kwboot_img_patch(void *img, size_t *size, int baudrate) hdr = img; - if (*size < sizeof(struct main_hdr_v1)) + if (*size < sizeof(struct main_hdr_v1)) { + fprintf(stderr, "Invalid image header size\n"); goto err; + } image_ver = kwbimage_version(img); if (image_ver != 0 && image_ver != 1) { @@ -1877,24 +2032,23 @@ kwboot_img_patch(void *img, size_t *size, int baudrate) hdrsz = kwbheader_size(hdr); - if (*size < hdrsz) + if (*size < hdrsz) { + fprintf(stderr, "Invalid image header size\n"); goto err; + } + + kwboot_printv("Detected kwbimage v%d with %s boot signature\n", image_ver, kwboot_img_type(hdr->blockid)); csum = kwboot_hdr_csum8(hdr) - hdr->checksum; - if (csum != hdr->checksum) + if (csum != hdr->checksum) { + fprintf(stderr, "Image has invalid header checksum stored in image header\n"); goto err; + } srcaddr = le32_to_cpu(hdr->srcaddr); switch (hdr->blockid) { case IBR_HDR_SATA_ID: - if (srcaddr < 1) - goto err; - - hdr->srcaddr = cpu_to_le32((srcaddr - 1) * 512); - break; - - case IBR_HDR_SDIO_ID: hdr->srcaddr = cpu_to_le32(srcaddr * 512); break; @@ -1906,18 +2060,50 @@ kwboot_img_patch(void *img, size_t *size, int baudrate) case IBR_HDR_SPI_ID: if (hdr->destaddr == cpu_to_le32(0xFFFFFFFF)) { kwboot_printv("Patching destination and execution addresses from SPI/NOR XIP area to DDR area 0x00800000\n"); - hdr->destaddr = cpu_to_le32(0x00800000); - hdr->execaddr = cpu_to_le32(0x00800000); + hdr->destaddr = cpu_to_le32(0x00800000 + le32_to_cpu(hdr->srcaddr)); + hdr->execaddr = cpu_to_le32(0x00800000 + le32_to_cpu(hdr->execaddr)); } break; } - if (hdrsz > le32_to_cpu(hdr->srcaddr) || - *size < le32_to_cpu(hdr->srcaddr) + le32_to_cpu(hdr->blocksize)) + if (hdrsz > le32_to_cpu(hdr->srcaddr)) { + fprintf(stderr, "Image has invalid data offset stored in image header\n"); + goto err; + } + + if (*size < le32_to_cpu(hdr->srcaddr) + le32_to_cpu(hdr->blocksize)) { + fprintf(stderr, "Image has invalid data size stored in image header\n"); goto err; + } + + for_each_opt_hdr_v1 (ohdr, hdr) { + if (!opt_hdr_v1_valid_size(ohdr, (const uint8_t *)hdr + hdrsz)) { + fprintf(stderr, "Invalid optional image header\n"); + goto err; + } + } + + /* + * The 32-bit data checksum is optional for UART image. If it is not + * present (checksum detected as invalid) then grow data part of the + * image for the checksum, so it can be inserted there. + */ + if (kwboot_img_csum32(img) != *kwboot_img_csum32_ptr(img)) { + if (hdr->blockid != IBR_HDR_UART_ID) { + fprintf(stderr, "Image has invalid data checksum\n"); + goto err; + } + kwboot_img_grow_data_right(img, size, sizeof(uint32_t)); + /* Update the 32-bit data checksum */ + *kwboot_img_csum32_ptr(img) = kwboot_img_csum32(img); + } - if (kwboot_img_csum32(img) != *kwboot_img_csum32_ptr(img)) + if (!kwboot_img_has_ddr_init(img) && + (le32_to_cpu(hdr->destaddr) < 0x40000000 || + le32_to_cpu(hdr->destaddr) + le32_to_cpu(hdr->blocksize) > 0x40034000)) { + fprintf(stderr, "Image does not contain DDR init code needed for UART booting\n"); goto err; + } is_secure = kwboot_img_is_secure(img); @@ -1999,6 +2185,29 @@ kwboot_img_patch(void *img, size_t *size, int baudrate) kwboot_printv("Aligning image header to Xmodem block size\n"); kwboot_img_grow_hdr(img, size, grow); + hdrsz += grow; + + /* + * kwbimage v1 contains header size field and for UART type it + * must be set to the aligned xmodem header size because BootROM + * rounds header size down to xmodem block size. + */ + if (kwbimage_version(img) == 1) { + hdr->headersz_msb = hdrsz >> 16; + hdr->headersz_lsb = cpu_to_le16(hdrsz & 0xffff); + } + } + + /* Header size and source address must be same for UART type due to A38x BootROM bug */ + if (hdrsz != le32_to_cpu(hdr->srcaddr)) { + if (is_secure) { + fprintf(stderr, "Cannot align image with secure header\n"); + goto err; + } + + kwboot_printv("Removing gap between image header and data\n"); + memmove(img + hdrsz, img + le32_to_cpu(hdr->srcaddr), le32_to_cpu(hdr->blocksize)); + hdr->srcaddr = cpu_to_le32(hdrsz); } hdr->checksum = kwboot_hdr_csum8(hdr) - csum; @@ -2182,6 +2391,7 @@ main(int argc, char **argv) KWBOOT_XM_BLKSZ + sizeof(kwboot_baud_code) + sizeof(kwboot_baud_code_data_jump) + + sizeof(uint32_t) + KWBOOT_XM_BLKSZ; if (imgpath) { diff --git a/tools/mkimage.c b/tools/mkimage.c index af7b0e0..a92d9d5 100644 --- a/tools/mkimage.c +++ b/tools/mkimage.c @@ -599,7 +599,12 @@ int main(int argc, char **argv) exit (retval); } - if ((params.type != IH_TYPE_MULTI) && (params.type != IH_TYPE_SCRIPT)) { + if (!params.skipcpy && params.type != IH_TYPE_MULTI && params.type != IH_TYPE_SCRIPT) { + if (!params.datafile) { + fprintf(stderr, "%s: Option -d with image data file was not specified\n", + params.cmdname); + exit(EXIT_FAILURE); + } dfd = open(params.datafile, O_RDONLY | O_BINARY); if (dfd < 0) { fprintf(stderr, "%s: Can't open %s: %s\n", @@ -860,7 +865,9 @@ copy_file (int ifd, const char *datafile, int pad) exit (EXIT_FAILURE); } - if (params.xflag) { + if (params.xflag && + (((params.type > IH_TYPE_INVALID) && (params.type < IH_TYPE_FLATDT)) || + (params.type == IH_TYPE_KERNEL_NOLOAD) || (params.type == IH_TYPE_FIRMWARE_IVT))) { unsigned char *p = NULL; /* * XIP: do not append the struct legacy_img_hdr at the diff --git a/tools/patman/__init__.py b/tools/patman/__init__.py index 1b98ec7..08eeffd 100644 --- a/tools/patman/__init__.py +++ b/tools/patman/__init__.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: GPL-2.0+ -__all__ = ['checkpatch', 'command', 'commit', 'control', 'cros_subprocess', - 'func_test', 'get_maintainer', 'gitutil', '__main__', 'patchstream', - 'project', 'series', 'setup', 'settings', 'terminal', - 'test_checkpatch', 'test_util', 'tools', 'tout'] +__all__ = ['checkpatch', 'commit', 'control', 'func_test', 'get_maintainer', + 'gitutil', '__main__', 'patchstream', 'project', 'series', + 'settings','setup', 'status', 'test_checkpatch', 'test_settings'] diff --git a/tools/patman/__main__.py b/tools/patman/__main__.py index 749e634..48ffbc8 100755 --- a/tools/patman/__main__.py +++ b/tools/patman/__main__.py @@ -24,10 +24,9 @@ from patman import func_test from patman import gitutil from patman import project from patman import settings -from patman import terminal -from patman import test_util -from patman import test_checkpatch -from patman import tools +from u_boot_pylib import terminal +from u_boot_pylib import test_util +from u_boot_pylib import tools epilog = '''Create patches from commits in a branch, check them and email them as specified by tags you place in the commits. Use -n to do a dry run first.''' @@ -146,11 +145,12 @@ if not args.debug: # Run our meagre tests if args.cmd == 'test': from patman import func_test + from patman import test_checkpatch result = test_util.run_test_suites( 'patman', False, False, False, None, None, None, [test_checkpatch.TestPatch, func_test.TestFunctional, - 'gitutil', 'settings', 'terminal']) + 'gitutil', 'settings']) sys.exit(0 if result.wasSuccessful() else 1) diff --git a/tools/patman/checkpatch.py b/tools/patman/checkpatch.py index d1b902d..e03cac1 100644 --- a/tools/patman/checkpatch.py +++ b/tools/patman/checkpatch.py @@ -3,13 +3,14 @@ # import collections +import concurrent.futures import os import re import sys -from patman import command from patman import gitutil -from patman import terminal +from u_boot_pylib import command +from u_boot_pylib import terminal EMACS_PREFIX = r'(?:[0-9]{4}.*\.patch:[0-9]+: )?' TYPE_NAME = r'([A-Z_]+:)?' @@ -244,26 +245,31 @@ def check_patches(verbose, args, use_tree): error_count, warning_count, check_count = 0, 0, 0 col = terminal.Color() - for fname in args: - result = check_patch(fname, verbose, use_tree=use_tree) - if not result.ok: - error_count += result.errors - warning_count += result.warnings - check_count += result.checks - print('%d errors, %d warnings, %d checks for %s:' % (result.errors, - result.warnings, result.checks, col.build(col.BLUE, fname))) - if (len(result.problems) != result.errors + result.warnings + - result.checks): - print("Internal error: some problems lost") - # Python seems to get confused by this - # pylint: disable=E1133 - for item in result.problems: - sys.stderr.write( - get_warning_msg(col, item.get('type', '<unknown>'), - item.get('file', '<unknown>'), - item.get('line', 0), item.get('msg', 'message'))) - print - #print(stdout) + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: + futures = [] + for fname in args: + f = executor.submit(check_patch, fname, verbose, use_tree=use_tree) + futures.append(f) + + for fname, f in zip(args, futures): + result = f.result() + if not result.ok: + error_count += result.errors + warning_count += result.warnings + check_count += result.checks + print('%d errors, %d warnings, %d checks for %s:' % (result.errors, + result.warnings, result.checks, col.build(col.BLUE, fname))) + if (len(result.problems) != result.errors + result.warnings + + result.checks): + print("Internal error: some problems lost") + # Python seems to get confused by this + # pylint: disable=E1133 + for item in result.problems: + sys.stderr.write( + get_warning_msg(col, item.get('type', '<unknown>'), + item.get('file', '<unknown>'), + item.get('line', 0), item.get('msg', 'message'))) + print if error_count or warning_count or check_count: str = 'checkpatch.pl found %d error(s), %d warning(s), %d checks(s)' color = col.GREEN diff --git a/tools/patman/control.py b/tools/patman/control.py index 38e98da..916ddf8 100644 --- a/tools/patman/control.py +++ b/tools/patman/control.py @@ -14,7 +14,7 @@ import sys from patman import checkpatch from patman import gitutil from patman import patchstream -from patman import terminal +from u_boot_pylib import terminal def setup(): """Do required setup before doing anything""" @@ -85,7 +85,7 @@ def check_patches(series, patch_files, run_checkpatch, verbose, use_tree): # Do a few checks on the series series.DoChecks() - # Check the patches, and run them through 'git am' just to be sure + # Check the patches if run_checkpatch: ok = checkpatch.check_patches(verbose, patch_files, use_tree) else: diff --git a/tools/patman/func_test.py b/tools/patman/func_test.py index c25a47b..42ac4ed 100644 --- a/tools/patman/func_test.py +++ b/tools/patman/func_test.py @@ -23,9 +23,9 @@ from patman import patchstream from patman.patchstream import PatchStream from patman.series import Series from patman import settings -from patman import terminal -from patman import tools -from patman.test_util import capture_sys_output +from u_boot_pylib import terminal +from u_boot_pylib import tools +from u_boot_pylib.test_util import capture_sys_output import pygit2 from patman import status @@ -240,6 +240,8 @@ class TestFunctional(unittest.TestCase): self.assertEqual('Change log missing for v3', next(lines)) self.assertEqual('Change log for unknown version v4', next(lines)) self.assertEqual("Alias 'pci' not found", next(lines)) + while next(lines) != 'Cc processing complete': + pass self.assertIn('Dry run', next(lines)) self.assertEqual('', next(lines)) self.assertIn('Send a total of %d patches' % count, next(lines)) diff --git a/tools/patman/get_maintainer.py b/tools/patman/get_maintainer.py index f7011be..8df3d12 100644 --- a/tools/patman/get_maintainer.py +++ b/tools/patman/get_maintainer.py @@ -7,8 +7,8 @@ import os import shlex import shutil -from patman import command from patman import gitutil +from u_boot_pylib import command def find_get_maintainer(script_file_name): diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py index 5e74210..6700057 100644 --- a/tools/patman/gitutil.py +++ b/tools/patman/gitutil.py @@ -5,9 +5,9 @@ import os import sys -from patman import command from patman import settings -from patman import terminal +from u_boot_pylib import command +from u_boot_pylib import terminal # True to use --no-decorate - we check this in setup() use_no_decorate = True diff --git a/tools/patman/patchstream.py b/tools/patman/patchstream.py index fb6a603..f91669a 100644 --- a/tools/patman/patchstream.py +++ b/tools/patman/patchstream.py @@ -14,10 +14,10 @@ import queue import shutil import tempfile -from patman import command from patman import commit from patman import gitutil from patman.series import Series +from u_boot_pylib import command # Tags that we detect and remove RE_REMOVE = re.compile(r'^BUG=|^TEST=|^BRANCH=|^Review URL:' diff --git a/tools/patman/patman.rst b/tools/patman/patman.rst index 6113962..038b651 100644 --- a/tools/patman/patman.rst +++ b/tools/patman/patman.rst @@ -41,6 +41,18 @@ In Linux and U-Boot this will also call get_maintainer.pl on each of your patches automatically (unless you use -m to disable this). +Installation +------------ + +You can install patman using:: + + pip install patch-manager + +The name is chosen since patman conflicts with an existing package. + +If you are using patman within the U-Boot tree, it may be easiest to add a +symlink from your local `~/.bin` directory to `/path/to/tools/patman/patman`. + How to use this tool -------------------- diff --git a/tools/patman/pyproject.toml b/tools/patman/pyproject.toml new file mode 100644 index 0000000..c5dc7c7 --- /dev/null +++ b/tools/patman/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "patch-manager" +version = "0.0.2" +authors = [ + { name="Simon Glass", email="sjg@chromium.org" }, +] +dependencies = ["u_boot_pylib"] +description = "Patman patch manager" +readme = "README.rst" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://u-boot.readthedocs.io/en/latest/develop/patman.html" +"Bug Tracker" = "https://source.denx.de/groups/u-boot/-/issues" + +[project.scripts] +patman = "patman.__main__:run_patman" + +[tool.setuptools.package-data] +patman = ["*.rst"] diff --git a/tools/patman/series.py b/tools/patman/series.py index 2eeeef7..6866e1d 100644 --- a/tools/patman/series.py +++ b/tools/patman/series.py @@ -5,14 +5,17 @@ from __future__ import print_function import collections +import concurrent.futures import itertools import os +import sys +import time from patman import get_maintainer from patman import gitutil from patman import settings -from patman import terminal -from patman import tools +from u_boot_pylib import terminal +from u_boot_pylib import tools # Series-xxx tags that we understand valid_series = ['to', 'cc', 'version', 'changes', 'prefix', 'notes', 'name', @@ -234,6 +237,49 @@ class Series(dict): str = 'Change log exists, but no version is set' print(col.build(col.RED, str)) + def GetCcForCommit(self, commit, process_tags, warn_on_error, + add_maintainers, limit, get_maintainer_script, + all_skips): + """Get the email CCs to use with a particular commit + + Uses subject tags and get_maintainers.pl script to find people to cc + on a patch + + Args: + commit (Commit): Commit to process + process_tags (bool): Process tags as if they were aliases + warn_on_error (bool): True to print a warning when an alias fails to + match, False to ignore it. + add_maintainers (bool or list of str): Either: + True/False to call the get_maintainers to CC maintainers + List of maintainers to include (for testing) + limit (int): Limit the length of the Cc list (None if no limit) + get_maintainer_script (str): The file name of the get_maintainer.pl + script (or compatible). + all_skips (set of str): Updated to include the set of bouncing email + addresses that were dropped from the output. This is essentially + a return value from this function. + + Returns: + list of str: List of email addresses to cc + """ + cc = [] + if process_tags: + cc += gitutil.build_email_list(commit.tags, + warn_on_error=warn_on_error) + cc += gitutil.build_email_list(commit.cc_list, + warn_on_error=warn_on_error) + if type(add_maintainers) == type(cc): + cc += add_maintainers + elif add_maintainers: + cc += get_maintainer.get_maintainer(get_maintainer_script, + commit.patch) + all_skips |= set(cc) & set(settings.bounces) + cc = list(set(cc) - set(settings.bounces)) + if limit is not None: + cc = cc[:limit] + return cc + def MakeCcFile(self, process_tags, cover_fname, warn_on_error, add_maintainers, limit, get_maintainer_script): """Make a cc file for us to use for per-commit Cc automation @@ -241,15 +287,15 @@ class Series(dict): Also stores in self._generated_cc to make ShowActions() faster. Args: - process_tags: Process tags as if they were aliases - cover_fname: If non-None the name of the cover letter. - warn_on_error: True to print a warning when an alias fails to match, - False to ignore it. - add_maintainers: Either: + process_tags (bool): Process tags as if they were aliases + cover_fname (str): If non-None the name of the cover letter. + warn_on_error (bool): True to print a warning when an alias fails to + match, False to ignore it. + add_maintainers (bool or list of str): Either: True/False to call the get_maintainers to CC maintainers List of maintainers to include (for testing) - limit: Limit the length of the Cc list (None if no limit) - get_maintainer_script: The file name of the get_maintainer.pl + limit (int): Limit the length of the Cc list (None if no limit) + get_maintainer_script (str): The file name of the get_maintainer.pl script (or compatible). Return: Filename of temp file created @@ -259,28 +305,42 @@ class Series(dict): fname = '/tmp/patman.%d' % os.getpid() fd = open(fname, 'w', encoding='utf-8') all_ccs = [] + all_skips = set() + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: + for i, commit in enumerate(self.commits): + commit.seq = i + commit.future = executor.submit( + self.GetCcForCommit, commit, process_tags, warn_on_error, + add_maintainers, limit, get_maintainer_script, all_skips) + + # Show progress any commits that are taking forever + lastlen = 0 + while True: + left = [commit for commit in self.commits + if not commit.future.done()] + if not left: + break + names = ', '.join(f'{c.seq + 1}:{c.subject}' + for c in left[:2]) + out = f'\r{len(left)} remaining: {names}'[:79] + spaces = ' ' * (lastlen - len(out)) + if lastlen: # Don't print anything the first time + print(out, spaces, end='') + sys.stdout.flush() + lastlen = len(out) + time.sleep(.25) + print(f'\rdone{" " * lastlen}\r', end='') + print('Cc processing complete') + for commit in self.commits: - cc = [] - if process_tags: - cc += gitutil.build_email_list(commit.tags, - warn_on_error=warn_on_error) - cc += gitutil.build_email_list(commit.cc_list, - warn_on_error=warn_on_error) - if type(add_maintainers) == type(cc): - cc += add_maintainers - elif add_maintainers: - - cc += get_maintainer.get_maintainer(get_maintainer_script, - commit.patch) - for x in set(cc) & set(settings.bounces): - print(col.build(col.YELLOW, 'Skipping "%s"' % x)) - cc = list(set(cc) - set(settings.bounces)) - if limit is not None: - cc = cc[:limit] + cc = commit.future.result() all_ccs += cc print(commit.patch, '\0'.join(sorted(set(cc))), file=fd) self._generated_cc[commit.patch] = cc + for x in sorted(all_skips): + print(col.build(col.YELLOW, f'Skipping "{x}"')) + if cover_fname: cover_cc = gitutil.build_email_list(self.get('cover_cc', '')) cover_cc = list(set(cover_cc + all_ccs)) diff --git a/tools/patman/status.py b/tools/patman/status.py index 47ed6d6..5fb436e 100644 --- a/tools/patman/status.py +++ b/tools/patman/status.py @@ -18,8 +18,8 @@ import requests from patman import patchstream from patman.patchstream import PatchStream -from patman import terminal -from patman import tout +from u_boot_pylib import terminal +from u_boot_pylib import tout # Patches which are part of a multi-patch series are shown with a prefix like # [prefix, version, sequence], for example '[RFC, v2, 3/5]'. All but the last diff --git a/tools/patman/test_checkpatch.py b/tools/patman/test_checkpatch.py index 4c2ab6e..a8bb364 100644 --- a/tools/patman/test_checkpatch.py +++ b/tools/patman/test_checkpatch.py @@ -452,6 +452,12 @@ index 0000000..2234c87 self.check_strl("cat"); self.check_strl("cpy"); + def test_schema(self): + """Check for uses of strn(cat|cpy)""" + pm = PatchMaker() + pm.add_line('arch/sandbox/dts/sandbox.dtsi', '\tu-boot,dm-pre-proper;') + self.check_single_message(pm, 'PRE_SCHEMA', 'error') + if __name__ == "__main__": unittest.main() gitutil.RunTests() diff --git a/tools/patman/test_settings.py b/tools/patman/test_settings.py index c768a2f..06b7cbc 100644 --- a/tools/patman/test_settings.py +++ b/tools/patman/test_settings.py @@ -10,7 +10,7 @@ import sys import tempfile from patman import settings -from patman import tools +from u_boot_pylib import tools @contextlib.contextmanager diff --git a/tools/proftool.c b/tools/proftool.c index 0893604..101bcb6 100644 --- a/tools/proftool.c +++ b/tools/proftool.c @@ -1713,18 +1713,11 @@ static int make_flame_tree(enum out_format_t out_format, struct flame_state state; struct flame_node *tree; struct trace_call *call; - int missing_count = 0; - int i, depth; + int i; /* maintain a stack of start times, etc. for 'calling' functions */ state.stack_ptr = 0; - /* - * The first thing in the trace may not be the top-level function, so - * set the initial depth so that no function goes below depth 0 - */ - depth = -calc_min_depth(); - tree = create_node("tree"); if (!tree) return -1; @@ -1736,16 +1729,10 @@ static int make_flame_tree(enum out_format_t out_format, ulong timestamp = call->flags & FUNCF_TIMESTAMP_MASK; struct func_info *func; - if (entry) - depth++; - else - depth--; - func = find_func_by_offset(call->func); if (!func) { warn("Cannot find function at %lx\n", text_offset + call->func); - missing_count++; continue; } diff --git a/tools/relocate-rela.c b/tools/relocate-rela.c index 2d2a2ed..fe8cd6b 100644 --- a/tools/relocate-rela.c +++ b/tools/relocate-rela.c @@ -45,6 +45,7 @@ #endif static int ei_class; +static int ei_data; static uint64_t rela_start, rela_end, text_base, dyn_start; @@ -61,6 +62,22 @@ static void debug(const char *fmt, ...) } } +static uint16_t elf16_to_cpu(uint16_t data) +{ + if (ei_data == ELFDATA2LSB) + return le16_to_cpu(data); + + return be16_to_cpu(data); +} + +static uint32_t elf32_to_cpu(uint32_t data) +{ + if (ei_data == ELFDATA2LSB) + return le32_to_cpu(data); + + return be32_to_cpu(data); +} + static bool supported_rela(Elf64_Rela *rela) { uint64_t mask = 0xffffffffULL; /* would be different on 32-bit */ @@ -234,7 +251,7 @@ static int decode_elf32(FILE *felf, char **argv) return 25; } - machine = le16_to_cpu(header.e_machine); + machine = elf16_to_cpu(header.e_machine); debug("Machine %d\n", machine); if (machine != EM_MICROBLAZE) { @@ -242,10 +259,10 @@ static int decode_elf32(FILE *felf, char **argv) return 30; } - text_base = le32_to_cpu(header.e_entry); - section_header_base = le32_to_cpu(header.e_shoff); - section_header_size = le16_to_cpu(header.e_shentsize) * - le16_to_cpu(header.e_shnum); + text_base = elf32_to_cpu(header.e_entry); + section_header_base = elf32_to_cpu(header.e_shoff); + section_header_size = elf16_to_cpu(header.e_shentsize) * + elf16_to_cpu(header.e_shnum); sh_table = malloc(section_header_size); if (!sh_table) { @@ -273,8 +290,8 @@ static int decode_elf32(FILE *felf, char **argv) return 27; } - sh_index = le16_to_cpu(header.e_shstrndx); - sh_size = le32_to_cpu(sh_table[sh_index].sh_size); + sh_index = elf16_to_cpu(header.e_shstrndx); + sh_size = elf32_to_cpu(sh_table[sh_index].sh_size); debug("e_shstrndx %x, sh_size %lx\n", sh_index, sh_size); sh_str = malloc(sh_size); @@ -289,8 +306,8 @@ static int decode_elf32(FILE *felf, char **argv) * Specifies the byte offset from the beginning of the file * to the first byte in the section. */ - sh_offset = le32_to_cpu(sh_table[sh_index].sh_offset); - sh_num = le16_to_cpu(header.e_shnum); + sh_offset = elf32_to_cpu(sh_table[sh_index].sh_offset); + sh_num = elf16_to_cpu(header.e_shnum); ret = fseek(felf, sh_offset, SEEK_SET); if (ret) { @@ -312,13 +329,13 @@ static int decode_elf32(FILE *felf, char **argv) } for (i = 0; i < sh_num; i++) { - char *sh_name = sh_str + le32_to_cpu(sh_table[i].sh_name); + char *sh_name = sh_str + elf32_to_cpu(sh_table[i].sh_name); debug("%s\n", sh_name); - sh_addr = le64_to_cpu(sh_table[i].sh_addr); - sh_offset = le64_to_cpu(sh_table[i].sh_offset); - sh_size = le64_to_cpu(sh_table[i].sh_size); + sh_addr = elf32_to_cpu(sh_table[i].sh_addr); + sh_offset = elf32_to_cpu(sh_table[i].sh_offset); + sh_size = elf32_to_cpu(sh_table[i].sh_size); if (!strcmp(".rela.dyn", sh_name)) { debug("Found section\t\".rela_dyn\"\n"); @@ -384,6 +401,9 @@ static int decode_elf(char **argv) ei_class = e_ident[4]; debug("EI_CLASS(1=32bit, 2=64bit) %d\n", ei_class); + ei_data = e_ident[5]; + debug("EI_DATA(1=little endian, 2=big endian) %d\n", ei_data); + if (ei_class == 2) return decode_elf64(felf, argv); @@ -520,9 +540,9 @@ static int rela_elf32(char **argv, FILE *f) PRIu32 " r_addend:\t%" PRIx32 "\n", rela.r_offset, rela.r_info, rela.r_addend); - swrela.r_offset = le32_to_cpu(rela.r_offset); - swrela.r_info = le32_to_cpu(rela.r_info); - swrela.r_addend = le32_to_cpu(rela.r_addend); + swrela.r_offset = elf32_to_cpu(rela.r_offset); + swrela.r_info = elf32_to_cpu(rela.r_info); + swrela.r_addend = elf32_to_cpu(rela.r_addend); debug("SWRela:\toffset:\t%" PRIx32 " r_info:\t%" PRIu32 " r_addend:\t%" PRIx32 "\n", diff --git a/tools/rmboard.py b/tools/rmboard.py index ae25632..0c56b14 100755 --- a/tools/rmboard.py +++ b/tools/rmboard.py @@ -28,7 +28,7 @@ import os import re import sys -from patman import command +from u_boot_pylib import command def rm_kconfig_include(path): """Remove a path from Kconfig files diff --git a/tools/u_boot_pylib/LICENSE b/tools/u_boot_pylib/LICENSE new file mode 100644 index 0000000..d159169 --- /dev/null +++ b/tools/u_boot_pylib/LICENSE @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/tools/u_boot_pylib/README.rst b/tools/u_boot_pylib/README.rst new file mode 100644 index 0000000..93858f5 --- /dev/null +++ b/tools/u_boot_pylib/README.rst @@ -0,0 +1,15 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +# U-Boot Python Library +===================== + +This is a Python library used by various U-Boot tools, including patman, +buildman and binman. + +The module can be installed with pip:: + + pip install u_boot_pylib + +or via setup.py:: + + ./setup.py install [--user] diff --git a/tools/u_boot_pylib/__init__.py b/tools/u_boot_pylib/__init__.py new file mode 100644 index 0000000..63c88e8 --- /dev/null +++ b/tools/u_boot_pylib/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0+ + +__all__ = ['command', 'cros_subprocess','terminal', 'test_util', 'tools', + 'tout'] diff --git a/tools/u_boot_pylib/__main__.py b/tools/u_boot_pylib/__main__.py new file mode 100755 index 0000000..8f98d7b --- /dev/null +++ b/tools/u_boot_pylib/__main__.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0+ +# +# Copyright 2023 Google LLC +# + +import os +import sys + +if __name__ == "__main__": + # Allow 'from u_boot_pylib import xxx to work' + our_path = os.path.dirname(os.path.realpath(__file__)) + sys.path.append(os.path.join(our_path, '..')) + + # Run tests + from u_boot_pylib import terminal + from u_boot_pylib import test_util + + result = test_util.run_test_suites( + 'u_boot_pylib', False, False, False, None, None, None, + ['terminal']) + + sys.exit(0 if result.wasSuccessful() else 1) diff --git a/tools/patman/command.py b/tools/u_boot_pylib/command.py index 92c453b..9bbfc5b 100644 --- a/tools/patman/command.py +++ b/tools/u_boot_pylib/command.py @@ -4,7 +4,7 @@ import os -from patman import cros_subprocess +from u_boot_pylib import cros_subprocess """Shell command ease-ups for Python.""" diff --git a/tools/patman/cros_subprocess.py b/tools/u_boot_pylib/cros_subprocess.py index cd614f3..cd614f3 100644 --- a/tools/patman/cros_subprocess.py +++ b/tools/u_boot_pylib/cros_subprocess.py diff --git a/tools/u_boot_pylib/pyproject.toml b/tools/u_boot_pylib/pyproject.toml new file mode 100644 index 0000000..3f33caf --- /dev/null +++ b/tools/u_boot_pylib/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "u_boot_pylib" +version = "0.0.2" +authors = [ + { name="Simon Glass", email="sjg@chromium.org" }, +] +description = "U-Boot python library" +readme = "README.md" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://u-boot.readthedocs.io" +"Bug Tracker" = "https://source.denx.de/groups/u-boot/-/issues" diff --git a/tools/patman/terminal.py b/tools/u_boot_pylib/terminal.py index 40d79f8..40d79f8 100644 --- a/tools/patman/terminal.py +++ b/tools/u_boot_pylib/terminal.py diff --git a/tools/patman/test_util.py b/tools/u_boot_pylib/test_util.py index 0f6d1aa..e7564e1 100644 --- a/tools/patman/test_util.py +++ b/tools/u_boot_pylib/test_util.py @@ -11,15 +11,14 @@ import os import sys import unittest -from patman import command +from u_boot_pylib import command from io import StringIO -buffer_outputs = True use_concurrent = True try: - from concurrencytest.concurrencytest import ConcurrentTestSuite - from concurrencytest.concurrencytest import fork_for_tests + from concurrencytest import ConcurrentTestSuite + from concurrencytest import fork_for_tests except: use_concurrent = False @@ -120,7 +119,6 @@ class FullTextTestResult(unittest.TextTestResult): 0: Print nothing 1: Print a dot per test 2: Print test names - 3: Print test names, and buffered outputs for failing tests """ def __init__(self, stream, descriptions, verbosity): self.verbosity = verbosity @@ -140,39 +138,12 @@ class FullTextTestResult(unittest.TextTestResult): self.printErrorList('XFAIL', self.expectedFailures) self.printErrorList('XPASS', unexpected_successes) - def addError(self, test, err): - """Called when an error has occurred.""" - super().addError(test, err) - self._mirrorOutput &= self.verbosity >= 3 - - def addFailure(self, test, err): - """Called when a test has failed.""" - super().addFailure(test, err) - self._mirrorOutput &= self.verbosity >= 3 - - def addSubTest(self, test, subtest, err): - """Called at the end of a subtest.""" - super().addSubTest(test, subtest, err) - self._mirrorOutput &= self.verbosity >= 3 - - def addSuccess(self, test): - """Called when a test has completed successfully""" - super().addSuccess(test) - # Don't print stdout/stderr for successful tests - self._mirrorOutput = False - def addSkip(self, test, reason): """Called when a test is skipped.""" # Add empty line to keep spacing consistent with other results if not reason.endswith('\n'): reason += '\n' super().addSkip(test, reason) - self._mirrorOutput &= self.verbosity >= 3 - - def addExpectedFailure(self, test, err): - """Called when an expected failure/error occurred.""" - super().addExpectedFailure(test, err) - self._mirrorOutput &= self.verbosity >= 3 def run_test_suites(toolname, debug, verbosity, test_preserve_dirs, processes, @@ -208,14 +179,12 @@ def run_test_suites(toolname, debug, verbosity, test_preserve_dirs, processes, runner = unittest.TextTestRunner( stream=sys.stdout, verbosity=(1 if verbosity is None else verbosity), - buffer=False if test_name else buffer_outputs, resultclass=FullTextTestResult, ) if use_concurrent and processes != 1: suite = ConcurrentTestSuite(suite, - fork_for_tests(processes or multiprocessing.cpu_count(), - buffer=False if test_name else buffer_outputs)) + fork_for_tests(processes or multiprocessing.cpu_count())) for module in class_and_module_list: if isinstance(module, str) and (not test_name or test_name == module): diff --git a/tools/patman/tools.py b/tools/u_boot_pylib/tools.py index 2ac814d..187725b 100644 --- a/tools/patman/tools.py +++ b/tools/u_boot_pylib/tools.py @@ -11,8 +11,8 @@ import sys import tempfile import urllib.request -from patman import command -from patman import tout +from u_boot_pylib import command +from u_boot_pylib import tout # Output directly (generally this is temporary) outdir = None diff --git a/tools/patman/tout.py b/tools/u_boot_pylib/tout.py index ff0fd92..6bd2806 100644 --- a/tools/patman/tout.py +++ b/tools/u_boot_pylib/tout.py @@ -6,7 +6,7 @@ import sys -from patman import terminal +from u_boot_pylib import terminal # Output verbosity levels that we support ERROR, WARNING, NOTICE, INFO, DETAIL, DEBUG = range(6) diff --git a/tools/u_boot_pylib/u_boot_pylib b/tools/u_boot_pylib/u_boot_pylib new file mode 120000 index 0000000..5a427d1 --- /dev/null +++ b/tools/u_boot_pylib/u_boot_pylib @@ -0,0 +1 @@ +__main__.py
\ No newline at end of file |