diff options
Diffstat (limited to 'scripts')
88 files changed, 6458 insertions, 3222 deletions
diff --git a/scripts/analyse-locks-simpletrace.py b/scripts/analyse-locks-simpletrace.py index d650dd7..bd04cd4 100755 --- a/scripts/analyse-locks-simpletrace.py +++ b/scripts/analyse-locks-simpletrace.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # Analyse lock events and compute statistics # diff --git a/scripts/analyze-inclusions b/scripts/analyze-inclusions index b6280f2..d2c5666 100644 --- a/scripts/analyze-inclusions +++ b/scripts/analyze-inclusions @@ -53,7 +53,7 @@ echo $(grep_include -F 'trace/generated-tracers.h') files include generated-trac echo $(grep_include -F 'qapi/error.h') files include qapi/error.h echo $(grep_include -F 'qom/object.h') files include qom/object.h echo $(grep_include -F 'block/aio.h') files include block/aio.h -echo $(grep_include -F 'exec/memory.h') files include exec/memory.h +echo $(grep_include -F 'system/memory.h') files include system/memory.h echo $(grep_include -F 'fpu/softfloat.h') files include fpu/softfloat.h echo $(grep_include -F 'qemu/bswap.h') files include qemu/bswap.h echo diff --git a/scripts/archive-source.sh b/scripts/archive-source.sh index e461c15..8f97b19 100755 --- a/scripts/archive-source.sh +++ b/scripts/archive-source.sh @@ -26,11 +26,28 @@ sub_file="${sub_tdir}/submodule.tar" # independent of what the developer currently has initialized # in their checkout, because the build environment is completely # different to the host OS. -subprojects="keycodemapdb libvfio-user berkeley-softfloat-3 - berkeley-testfloat-3 arbitrary-int-1-rs bilge-0.2-rs - bilge-impl-0.2-rs either-1-rs itertools-0.11-rs libc-0.2-rs proc-macro2-1-rs - proc-macro-error-1-rs proc-macro-error-attr-1-rs quote-1-rs - syn-2-rs unicode-ident-1-rs" +subprojects=( + anyhow-1-rs + arbitrary-int-1-rs + attrs-0.2-rs + berkeley-softfloat-3 + berkeley-testfloat-3 + bilge-0.2-rs + bilge-impl-0.2-rs + either-1-rs + foreign-0.3-rs + glib-sys-0.21-rs + itertools-0.11-rs + keycodemapdb + libc-0.2-rs + libvfio-user + proc-macro-error-1-rs + proc-macro-error-attr-1-rs + proc-macro2-1-rs + quote-1-rs + syn-2-rs + unicode-ident-1-rs +) sub_deinit="" function cleanup() { @@ -76,9 +93,10 @@ function subproject_dir() { git archive --format tar "$(tree_ish)" > "$tar_file" test $? -ne 0 && error "failed to archive qemu" -for sp in $subprojects; do - meson subprojects download $sp - test $? -ne 0 && error "failed to download subproject $sp" +meson subprojects download ${subprojects[@]} >/dev/null +test $? -ne 0 && error "failed to download subprojects $subprojects" + +for sp in "${subprojects[@]}"; do tar --append --file "$tar_file" --exclude=.git subprojects/"$(subproject_dir $sp)" test $? -ne 0 && error "failed to append subproject $sp to $tar_file" done diff --git a/scripts/arm_processor_error.py b/scripts/arm_processor_error.py new file mode 100644 index 0000000..73d069f --- /dev/null +++ b/scripts/arm_processor_error.py @@ -0,0 +1,476 @@ +#!/usr/bin/env python3 +# +# pylint: disable=C0301,C0114,R0903,R0912,R0913,R0914,R0915,W0511 +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Copyright (C) 2024-2025 Mauro Carvalho Chehab <mchehab+huawei@kernel.org> + +# TODO: current implementation has dummy defaults. +# +# For a better implementation, a QMP addition/call is needed to +# retrieve some data for ARM Processor Error injection: +# +# - ARM registers: power_state, mpidr. + +""" +Generate an ARM processor error CPER, compatible with +UEFI 2.9A Errata. + +Injecting such errors can be done using: + + $ ./scripts/ghes_inject.py arm + Error injected. + +Produces a simple CPER register, as detected on a Linux guest: + +[Hardware Error]: Hardware error from APEI Generic Hardware Error Source: 1 +[Hardware Error]: event severity: recoverable +[Hardware Error]: Error 0, type: recoverable +[Hardware Error]: section_type: ARM processor error +[Hardware Error]: MIDR: 0x0000000000000000 +[Hardware Error]: running state: 0x0 +[Hardware Error]: Power State Coordination Interface state: 0 +[Hardware Error]: Error info structure 0: +[Hardware Error]: num errors: 2 +[Hardware Error]: error_type: 0x02: cache error +[Hardware Error]: error_info: 0x000000000091000f +[Hardware Error]: transaction type: Data Access +[Hardware Error]: cache error, operation type: Data write +[Hardware Error]: cache level: 2 +[Hardware Error]: processor context not corrupted +[Firmware Warn]: GHES: Unhandled processor error type 0x02: cache error + +The ARM Processor Error message can be customized via command line +parameters. For instance: + + $ ./scripts/ghes_inject.py arm --mpidr 0x444 --running --affinity 1 \ + --error-info 12345678 --vendor 0x13,123,4,5,1 --ctx-array 0,1,2,3,4,5 \ + -t cache tlb bus micro-arch tlb,micro-arch + Error injected. + +Injects this error, as detected on a Linux guest: + +[Hardware Error]: Hardware error from APEI Generic Hardware Error Source: 1 +[Hardware Error]: event severity: recoverable +[Hardware Error]: Error 0, type: recoverable +[Hardware Error]: section_type: ARM processor error +[Hardware Error]: MIDR: 0x0000000000000000 +[Hardware Error]: Multiprocessor Affinity Register (MPIDR): 0x0000000000000000 +[Hardware Error]: error affinity level: 0 +[Hardware Error]: running state: 0x1 +[Hardware Error]: Power State Coordination Interface state: 0 +[Hardware Error]: Error info structure 0: +[Hardware Error]: num errors: 2 +[Hardware Error]: error_type: 0x02: cache error +[Hardware Error]: error_info: 0x0000000000bc614e +[Hardware Error]: cache level: 2 +[Hardware Error]: processor context not corrupted +[Hardware Error]: Error info structure 1: +[Hardware Error]: num errors: 2 +[Hardware Error]: error_type: 0x04: TLB error +[Hardware Error]: error_info: 0x000000000054007f +[Hardware Error]: transaction type: Instruction +[Hardware Error]: TLB error, operation type: Instruction fetch +[Hardware Error]: TLB level: 1 +[Hardware Error]: processor context not corrupted +[Hardware Error]: the error has not been corrected +[Hardware Error]: PC is imprecise +[Hardware Error]: Error info structure 2: +[Hardware Error]: num errors: 2 +[Hardware Error]: error_type: 0x08: bus error +[Hardware Error]: error_info: 0x00000080d6460fff +[Hardware Error]: transaction type: Generic +[Hardware Error]: bus error, operation type: Generic read (type of instruction or data request cannot be determined) +[Hardware Error]: affinity level at which the bus error occurred: 1 +[Hardware Error]: processor context corrupted +[Hardware Error]: the error has been corrected +[Hardware Error]: PC is imprecise +[Hardware Error]: Program execution can be restarted reliably at the PC associated with the error. +[Hardware Error]: participation type: Local processor observed +[Hardware Error]: request timed out +[Hardware Error]: address space: External Memory Access +[Hardware Error]: memory access attributes:0x20 +[Hardware Error]: access mode: secure +[Hardware Error]: Error info structure 3: +[Hardware Error]: num errors: 2 +[Hardware Error]: error_type: 0x10: micro-architectural error +[Hardware Error]: error_info: 0x0000000078da03ff +[Hardware Error]: Error info structure 4: +[Hardware Error]: num errors: 2 +[Hardware Error]: error_type: 0x14: TLB error|micro-architectural error +[Hardware Error]: Context info structure 0: +[Hardware Error]: register context type: AArch64 EL1 context registers +[Hardware Error]: 00000000: 00000000 00000000 +[Hardware Error]: Vendor specific error info has 5 bytes: +[Hardware Error]: 00000000: 13 7b 04 05 01 .{... +[Firmware Warn]: GHES: Unhandled processor error type 0x02: cache error +[Firmware Warn]: GHES: Unhandled processor error type 0x04: TLB error +[Firmware Warn]: GHES: Unhandled processor error type 0x08: bus error +[Firmware Warn]: GHES: Unhandled processor error type 0x10: micro-architectural error +[Firmware Warn]: GHES: Unhandled processor error type 0x14: TLB error|micro-architectural error +""" + +import argparse +import re + +from qmp_helper import qmp, util, cper_guid + + +class ArmProcessorEinj: + """ + Implements ARM Processor Error injection via GHES + """ + + DESC = """ + Generates an ARM processor error CPER, compatible with + UEFI 2.9A Errata. + """ + + ACPI_GHES_ARM_CPER_LENGTH = 40 + ACPI_GHES_ARM_CPER_PEI_LENGTH = 32 + + # Context types + CONTEXT_AARCH32_EL1 = 1 + CONTEXT_AARCH64_EL1 = 5 + CONTEXT_MISC_REG = 8 + + def __init__(self, subparsers): + """Initialize the error injection class and add subparser""" + + # Valid choice values + self.arm_valid_bits = { + "mpidr": util.bit(0), + "affinity": util.bit(1), + "running": util.bit(2), + "vendor": util.bit(3), + } + + self.pei_flags = { + "first": util.bit(0), + "last": util.bit(1), + "propagated": util.bit(2), + "overflow": util.bit(3), + } + + self.pei_error_types = { + "cache": util.bit(1), + "tlb": util.bit(2), + "bus": util.bit(3), + "micro-arch": util.bit(4), + } + + self.pei_valid_bits = { + "multiple-error": util.bit(0), + "flags": util.bit(1), + "error-info": util.bit(2), + "virt-addr": util.bit(3), + "phy-addr": util.bit(4), + } + + self.data = bytearray() + + parser = subparsers.add_parser("arm", description=self.DESC) + + arm_valid_bits = ",".join(self.arm_valid_bits.keys()) + flags = ",".join(self.pei_flags.keys()) + error_types = ",".join(self.pei_error_types.keys()) + pei_valid_bits = ",".join(self.pei_valid_bits.keys()) + + # UEFI N.16 ARM Validation bits + g_arm = parser.add_argument_group("ARM processor") + g_arm.add_argument("--arm", "--arm-valid", + help=f"ARM valid bits: {arm_valid_bits}") + g_arm.add_argument("-a", "--affinity", "--level", "--affinity-level", + type=lambda x: int(x, 0), + help="Affinity level (when multiple levels apply)") + g_arm.add_argument("-l", "--mpidr", type=lambda x: int(x, 0), + help="Multiprocessor Affinity Register") + g_arm.add_argument("-i", "--midr", type=lambda x: int(x, 0), + help="Main ID Register") + g_arm.add_argument("-r", "--running", + action=argparse.BooleanOptionalAction, + default=None, + help="Indicates if the processor is running or not") + g_arm.add_argument("--psci", "--psci-state", + type=lambda x: int(x, 0), + help="Power State Coordination Interface - PSCI state") + + # TODO: Add vendor-specific support + + # UEFI N.17 bitmaps (type and flags) + g_pei = parser.add_argument_group("ARM Processor Error Info (PEI)") + g_pei.add_argument("-t", "--type", nargs="+", + help=f"one or more error types: {error_types}") + g_pei.add_argument("-f", "--flags", nargs="*", + help=f"zero or more error flags: {flags}") + g_pei.add_argument("-V", "--pei-valid", "--error-valid", nargs="*", + help=f"zero or more PEI valid bits: {pei_valid_bits}") + + # UEFI N.17 Integer values + g_pei.add_argument("-m", "--multiple-error", nargs="+", + help="Number of errors: 0: Single error, 1: Multiple errors, 2-65535: Error count if known") + g_pei.add_argument("-e", "--error-info", nargs="+", + help="Error information (UEFI 2.10 tables N.18 to N.20)") + g_pei.add_argument("-p", "--physical-address", nargs="+", + help="Physical address") + g_pei.add_argument("-v", "--virtual-address", nargs="+", + help="Virtual address") + + # UEFI N.21 Context + g_ctx = parser.add_argument_group("Processor Context") + g_ctx.add_argument("--ctx-type", "--context-type", nargs="*", + help="Type of the context (0=ARM32 GPR, 5=ARM64 EL1, other values supported)") + g_ctx.add_argument("--ctx-size", "--context-size", nargs="*", + help="Minimal size of the context") + g_ctx.add_argument("--ctx-array", "--context-array", nargs="*", + help="Comma-separated arrays for each context") + + # Vendor-specific data + g_vendor = parser.add_argument_group("Vendor-specific data") + g_vendor.add_argument("--vendor", "--vendor-specific", nargs="+", + help="Vendor-specific byte arrays of data") + + # Add arguments for Generic Error Data + qmp.argparse(parser) + + parser.set_defaults(func=self.send_cper) + + def send_cper(self, args): + """Parse subcommand arguments and send a CPER via QMP""" + + qmp_cmd = qmp(args.host, args.port, args.debug) + + # Handle Generic Error Data arguments if any + qmp_cmd.set_args(args) + + is_cpu_type = re.compile(r"^([\w+]+\-)?arm\-cpu$") + cpus = qmp_cmd.search_qom("/machine/unattached/device", + "type", is_cpu_type) + + cper = {} + pei = {} + ctx = {} + vendor = {} + + arg = vars(args) + + # Handle global parameters + if args.arm: + arm_valid_init = False + cper["valid"] = util.get_choice(name="valid", + value=args.arm, + choices=self.arm_valid_bits, + suffixes=["-error", "-err"]) + else: + cper["valid"] = 0 + arm_valid_init = True + + if "running" in arg: + if args.running: + cper["running-state"] = util.bit(0) + else: + cper["running-state"] = 0 + else: + cper["running-state"] = 0 + + if arm_valid_init: + if args.affinity: + cper["valid"] |= self.arm_valid_bits["affinity"] + + if args.mpidr: + cper["valid"] |= self.arm_valid_bits["mpidr"] + + if "running-state" in cper: + cper["valid"] |= self.arm_valid_bits["running"] + + if args.psci: + cper["valid"] |= self.arm_valid_bits["running"] + + # Handle PEI + if not args.type: + args.type = ["cache-error"] + + util.get_mult_choices( + pei, + name="valid", + values=args.pei_valid, + choices=self.pei_valid_bits, + suffixes=["-valid", "--addr"], + ) + util.get_mult_choices( + pei, + name="type", + values=args.type, + choices=self.pei_error_types, + suffixes=["-error", "-err"], + ) + util.get_mult_choices( + pei, + name="flags", + values=args.flags, + choices=self.pei_flags, + suffixes=["-error", "-cap"], + ) + util.get_mult_int(pei, "error-info", args.error_info) + util.get_mult_int(pei, "multiple-error", args.multiple_error) + util.get_mult_int(pei, "phy-addr", args.physical_address) + util.get_mult_int(pei, "virt-addr", args.virtual_address) + + # Handle context + util.get_mult_int(ctx, "type", args.ctx_type, allow_zero=True) + util.get_mult_int(ctx, "minimal-size", args.ctx_size, allow_zero=True) + util.get_mult_array(ctx, "register", args.ctx_array, allow_zero=True) + + util.get_mult_array(vendor, "bytes", args.vendor, max_val=255) + + # Store PEI + pei_data = bytearray() + default_flags = self.pei_flags["first"] + default_flags |= self.pei_flags["last"] + + error_info_num = 0 + + for i, p in pei.items(): # pylint: disable=W0612 + error_info_num += 1 + + # UEFI 2.10 doesn't define how to encode error information + # when multiple types are raised. So, provide a default only + # if a single type is there + if "error-info" not in p: + if p["type"] == util.bit(1): + p["error-info"] = 0x0091000F + if p["type"] == util.bit(2): + p["error-info"] = 0x0054007F + if p["type"] == util.bit(3): + p["error-info"] = 0x80D6460FFF + if p["type"] == util.bit(4): + p["error-info"] = 0x78DA03FF + + if "valid" not in p: + p["valid"] = 0 + if "multiple-error" in p: + p["valid"] |= self.pei_valid_bits["multiple-error"] + + if "flags" in p: + p["valid"] |= self.pei_valid_bits["flags"] + + if "error-info" in p: + p["valid"] |= self.pei_valid_bits["error-info"] + + if "phy-addr" in p: + p["valid"] |= self.pei_valid_bits["phy-addr"] + + if "virt-addr" in p: + p["valid"] |= self.pei_valid_bits["virt-addr"] + + # Version + util.data_add(pei_data, 0, 1) + + util.data_add(pei_data, + self.ACPI_GHES_ARM_CPER_PEI_LENGTH, 1) + + util.data_add(pei_data, p["valid"], 2) + util.data_add(pei_data, p["type"], 1) + util.data_add(pei_data, p.get("multiple-error", 1), 2) + util.data_add(pei_data, p.get("flags", default_flags), 1) + util.data_add(pei_data, p.get("error-info", 0), 8) + util.data_add(pei_data, p.get("virt-addr", 0xDEADBEEF), 8) + util.data_add(pei_data, p.get("phy-addr", 0xABBA0BAD), 8) + + # Store Context + ctx_data = bytearray() + context_info_num = 0 + + if ctx: + ret = qmp_cmd.send_cmd("query-target", may_open=True) + + default_ctx = self.CONTEXT_MISC_REG + + if "arch" in ret: + if ret["arch"] == "aarch64": + default_ctx = self.CONTEXT_AARCH64_EL1 + elif ret["arch"] == "arm": + default_ctx = self.CONTEXT_AARCH32_EL1 + + for k in sorted(ctx.keys()): + context_info_num += 1 + + if "type" not in ctx[k]: + ctx[k]["type"] = default_ctx + + if "register" not in ctx[k]: + ctx[k]["register"] = [] + + reg_size = len(ctx[k]["register"]) + size = 0 + + if "minimal-size" in ctx: + size = ctx[k]["minimal-size"] + + size = max(size, reg_size) + + size = (size + 1) % 0xFFFE + + # Version + util.data_add(ctx_data, 0, 2) + + util.data_add(ctx_data, ctx[k]["type"], 2) + + util.data_add(ctx_data, 8 * size, 4) + + for r in ctx[k]["register"]: + util.data_add(ctx_data, r, 8) + + for i in range(reg_size, size): # pylint: disable=W0612 + util.data_add(ctx_data, 0, 8) + + # Vendor-specific bytes are not grouped + vendor_data = bytearray() + if vendor: + for k in sorted(vendor.keys()): + for b in vendor[k]["bytes"]: + util.data_add(vendor_data, b, 1) + + # Encode ARM Processor Error + data = bytearray() + + util.data_add(data, cper["valid"], 4) + + util.data_add(data, error_info_num, 2) + util.data_add(data, context_info_num, 2) + + # Calculate the length of the CPER data + cper_length = self.ACPI_GHES_ARM_CPER_LENGTH + cper_length += len(pei_data) + cper_length += len(vendor_data) + cper_length += len(ctx_data) + util.data_add(data, cper_length, 4) + + util.data_add(data, arg.get("affinity-level", 0), 1) + + # Reserved + util.data_add(data, 0, 3) + + if "midr-el1" not in arg: + if cpus: + cmd_arg = { + 'path': cpus[0], + 'property': "midr" + } + ret = qmp_cmd.send_cmd("qom-get", cmd_arg, may_open=True) + if isinstance(ret, int): + arg["midr-el1"] = ret + + util.data_add(data, arg.get("mpidr-el1", 0), 8) + util.data_add(data, arg.get("midr-el1", 0), 8) + util.data_add(data, cper["running-state"], 4) + util.data_add(data, arg.get("psci-state", 0), 4) + + # Add PEI + data.extend(pei_data) + data.extend(ctx_data) + data.extend(vendor_data) + + self.data = data + + qmp_cmd.send_cper(cper_guid.CPER_PROC_ARM, self.data) diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 6ae9d7f..3a95574 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -365,6 +365,18 @@ our @typeList = ( qr{guintptr}, ); +# Match text found in common license boilerplate comments: +# for new files the SPDX-License-Identifier line is sufficient. +our @LICENSE_BOILERPLATE = ( + "licensed under the terms of the GNU GPL", + "under the terms of the GNU General Public License", + "under the terms of the GNU Lesser General Public", + "Permission is hereby granted, free of charge", + "GNU GPL, version 2 or later", + "See the COPYING file" +); +our $LICENSE_BOILERPLATE_RE = join("|", @LICENSE_BOILERPLATE); + # Load common spelling mistakes and build regular expression list. my $misspellings; my %spelling_fix; @@ -1330,29 +1342,6 @@ sub WARN { } } -# According to tests/qtest/bios-tables-test.c: do not -# change expected file in the same commit with adding test -sub checkfilename { - my ($name, $acpi_testexpected, $acpi_nontestexpected) = @_; - - # Note: shell script that rebuilds the expected files is in the same - # directory as files themselves. - # Note: allowed diff list can be changed both when changing expected - # files and when changing tests. - if ($name =~ m#^tests/data/acpi/# and not $name =~ m#^\.sh$#) { - $$acpi_testexpected = $name; - } elsif ($name !~ m#^tests/qtest/bios-tables-test-allowed-diff.h$#) { - $$acpi_nontestexpected = $name; - } - if (defined $$acpi_testexpected and defined $$acpi_nontestexpected) { - ERROR("Do not add expected files together with tests, " . - "follow instructions in " . - "tests/qtest/bios-tables-test.c: both " . - $$acpi_testexpected . " and " . - $$acpi_nontestexpected . " found\n"); - } -} - sub checkspdx { my ($file, $expr) = @_; @@ -1379,6 +1368,9 @@ sub checkspdx { $expr =~ s/^\s*//g; $expr =~ s/\s*$//g; + # Cull C comment end + $expr =~ s/\*\/.*//; + my @bits = split / +/, $expr; my $prefer = "GPL-2.0-or-later"; @@ -1417,6 +1409,118 @@ sub checkspdx { } } +# All three of the methods below take a 'file info' record +# which is a hash ref containing +# +# 'isgit': 1 if an enhanced git diff or 0 for a plain diff +# 'githeader': 1 if still parsing git patch header, 0 otherwise +# 'linestart': line number of start of file diff +# 'lineend': line number of end of file diff +# 'filenew': the new filename +# 'fileold': the old filename (same as 'new filename' except +# for renames in git diffs) +# 'action': one of 'modified' (always) or 'new' or 'deleted' or +# 'renamed' (git diffs only) +# 'mode': file mode for new/deleted files (git diffs only) +# 'similarity': file similarity when renamed (git diffs only) +# 'facts': hash ref for storing any metadata related to checks +# + +# Called at the end of each patch, with the list of +# real filenames that were seen in the patch +sub process_file_list { + my @fileinfos = @_; + + # According to tests/qtest/bios-tables-test.c: do not + # change expected file in the same commit with adding test + my @acpi_testexpected; + my @acpi_nontestexpected; + + foreach my $fileinfo (@fileinfos) { + # Note: shell script that rebuilds the expected files is in + # the same directory as files themselves. + # Note: allowed diff list can be changed both when changing + # expected files and when changing tests. + if ($fileinfo->{filenew} =~ m#^tests/data/acpi/# && + $fileinfo->{filenew} !~ m#^\.sh$#) { + push @acpi_testexpected, $fileinfo->{filenew}; + } elsif ($fileinfo->{filenew} !~ + m#^tests/qtest/bios-tables-test-allowed-diff.h$#) { + push @acpi_nontestexpected, $fileinfo->{filenew}; + } + } + if (int(@acpi_testexpected) > 0 and int(@acpi_nontestexpected) > 0) { + ERROR("Do not add expected files together with tests, " . + "follow instructions in " . + "tests/qtest/bios-tables-test.c. Files\n\n " . + join("\n ", @acpi_testexpected) . + "\n\nand\n\n " . + join("\n ", @acpi_nontestexpected) . + "\n\nfound in the same patch\n"); + } + + my $sawmaintainers = 0; + my @maybemaintainers; + foreach my $fileinfo (@fileinfos) { + if ($fileinfo->{action} ne "modified" && + $fileinfo->{filenew} !~ m#^tests/data/acpi/#) { + push @maybemaintainers, $fileinfo->{filenew}; + } + if ($fileinfo->{filenew} eq "MAINTAINERS") { + $sawmaintainers = 1; + } + } + + # If we don't see a MAINTAINERS update, prod the user to check + if (int(@maybemaintainers) > 0 && !$sawmaintainers) { + WARN("added, moved or deleted file(s):\n\n " . + join("\n ", @maybemaintainers) . + "\n\nDoes MAINTAINERS need updating?\n"); + } +} + +# Called at the start of processing a diff hunk for a file +sub process_start_of_file { + my $fileinfo = shift; + + # Check for incorrect file permissions + if ($fileinfo->{action} eq "new" && ($fileinfo->{mode} & 0111)) { + my $permhere = $fileinfo->{linestart} . "FILE: " . + $fileinfo->{filenew} . "\n"; + if ($fileinfo->{filenew} =~ + /(\bMakefile.*|\.(c|cc|cpp|h|mak|s|S))$/) { + ERROR("do not set execute permissions for source " . + "files\n" . $permhere); + } + } +} + +# Called at the end of processing a diff hunk for a file +sub process_end_of_file { + my $fileinfo = shift; + + if ($fileinfo->{action} eq "new" && + !exists $fileinfo->{facts}->{sawspdx}) { + if ($fileinfo->{filenew} =~ + /(\.(c|h|py|pl|sh|json|inc|rs)|Makefile.*)$/) { + # source code files MUST have SPDX license declared + ERROR("New file '" . $fileinfo->{filenew} . + "' requires 'SPDX-License-Identifier'"); + } else { + # Other files MAY have SPDX license if appropriate + WARN("Does new file '" . $fileinfo->{filenew} . + "' need 'SPDX-License-Identifier'?"); + } + } + if ($fileinfo->{action} eq "new" && + exists $fileinfo->{facts}->{sawboilerplate}) { + ERROR("New file '" . $fileinfo->{filenew} . "' must " . + "not have license boilerplate header text, only " . + "the SPDX-License-Identifier, unless this file was " . + "copied from existing code already having such text."); + } +} + sub process { my $filename = shift; @@ -1437,13 +1541,10 @@ sub process { my $in_header_lines = $file ? 0 : 1; my $in_commit_log = 0; #Scanning lines before patch - my $reported_maintainer_file = 0; my $reported_mixing_imported_file = 0; my $in_imported_file = 0; my $in_no_imported_file = 0; my $non_utf8_charset = 0; - my $expect_spdx = 0; - my $expect_spdx_file; our @report = (); our $cnt_lines = 0; @@ -1455,7 +1556,10 @@ sub process { my $realfile = ''; my $realline = 0; my $realcnt = 0; + my $fileinfo; + my @fileinfolist; my $here = ''; + my $oldhere = ''; my $in_comment = 0; my $comment_edge = 0; my $first_line = 0; @@ -1468,9 +1572,6 @@ sub process { my %suppress_whiletrailers; my %suppress_export; - my $acpi_testexpected; - my $acpi_nontestexpected; - # Pre-scan the patch sanitizing the lines. sanitise_line_reset(); @@ -1593,24 +1694,78 @@ sub process { $prefix = "$filename:$realline: " if ($emacs && $file); $prefix = "$filename:$linenr: " if ($emacs && !$file); + $oldhere = $here; $here = "#$linenr: " if (!$file); $here = "#$realline: " if ($file); # extract the filename as it passes - if ($line =~ /^diff --git.*?(\S+)$/) { - $realfile = $1; - $realfile =~ s@^([^/]*)/@@ if (!$file); - checkfilename($realfile, \$acpi_testexpected, \$acpi_nontestexpected); + if ($line =~ /^diff --git\s+(\S+)\s+(\S+)$/) { + my $fileold = $1; + my $filenew = $2; + + if (defined $fileinfo) { + $fileinfo->{lineend} = $oldhere; + process_end_of_file($fileinfo) + } + $fileold =~ s@^([^/]*)/@@ if (!$file); + $filenew =~ s@^([^/]*)/@@ if (!$file); + $realfile = $filenew; + + $fileinfo = { + "isgit" => 1, + "githeader" => 1, + "linestart" => $here, + "lineend" => 0, + "fileold" => $fileold, + "filenew" => $filenew, + "action" => "modified", + "mode" => 0, + "similarity" => 0, + "facts" => {}, + }; + push @fileinfolist, $fileinfo; + } elsif (defined $fileinfo && $fileinfo->{githeader} && + $line =~ /^(new|deleted) (?:file )?mode\s+([0-7]+)$/) { + $fileinfo->{action} = $1; + $fileinfo->{mode} = oct($2); + } elsif (defined $fileinfo && $fileinfo->{githeader} && + $line =~ /^similarity index (\d+)%/) { + $fileinfo->{similarity} = int($1); + } elsif (defined $fileinfo && $fileinfo->{githeader} && + $line =~ /^rename (from|to) [\w\/\.\-]+\s*$/) { + $fileinfo->{action} = "renamed"; + # For a no-change rename, we'll never have any "+++..." + # lines, so trigger actions now + if ($1 eq "to" && $fileinfo->{similarity} == 100) { + process_start_of_file($fileinfo); + } } elsif ($line =~ /^\+\+\+\s+(\S+)/) { $realfile = $1; - $realfile =~ s@^([^/]*)/@@ if (!$file); - checkfilename($realfile, \$acpi_testexpected, \$acpi_nontestexpected); - - $p1_prefix = $1; - if (!$file && $tree && $p1_prefix ne '' && - -e "$root/$p1_prefix") { - WARN("patch prefix '$p1_prefix' exists, appears to be a -p0 patch\n"); + $realfile =~ s@^[^/]*/@@ if (!$file); + + if (defined $fileinfo && !$fileinfo->{isgit}) { + $fileinfo->{lineend} = $oldhere; + process_end_of_file($fileinfo); + } + + if (!defined $fileinfo || !$fileinfo->{isgit}) { + $fileinfo = { + "isgit" => 0, + "githeader" => 0, + "linestart" => $here, + "lineend" => 0, + "fileold" => $realfile, + "filenew" => $realfile, + "action" => "modified", + "mode" => 0, + "similarity" => 0, + "facts" => {}, + }; + push @fileinfolist, $fileinfo; + } else { + $fileinfo->{githeader} = 0; } + process_start_of_file($fileinfo); next; } @@ -1623,14 +1778,6 @@ sub process { $cnt_lines++ if ($realcnt != 0); -# Check for incorrect file permissions - if ($line =~ /^new (file )?mode.*[7531]\d{0,2}$/) { - my $permhere = $here . "FILE: $realfile\n"; - if ($realfile =~ /(\bMakefile(?:\.objs)?|\.c|\.cc|\.cpp|\.h|\.mak|\.[sS])$/) { - ERROR("do not set execute permissions for source files\n" . $permhere); - } - } - # Only allow Python 3 interpreter if ($realline == 1 && $line =~ /^\+#!\ *\/usr\/bin\/(?:env )?python$/) { @@ -1662,68 +1809,28 @@ sub process { } } -# Check if MAINTAINERS is being updated. If so, there's probably no need to -# emit the "does MAINTAINERS need updating?" message on file add/move/delete - if ($line =~ /^\s*MAINTAINERS\s*\|/) { - $reported_maintainer_file = 1; - } - -# Check for added, moved or deleted files - if (!$reported_maintainer_file && !$in_commit_log && - ($line =~ /^(?:new|deleted) file mode\s*\d+\s*$/ || - $line =~ /^rename (?:from|to) [\w\/\.\-]+\s*$/ || - ($line =~ /\{\s*([\w\/\.\-]*)\s*\=\>\s*([\w\/\.\-]*)\s*\}/ && - (defined($1) || defined($2)))) && - !(($realfile ne '') && - defined($acpi_testexpected) && - ($realfile eq $acpi_testexpected))) { - $reported_maintainer_file = 1; - WARN("added, moved or deleted file(s), does MAINTAINERS need updating?\n" . $herecurr); - } - -# All new files should have a SPDX-License-Identifier tag - if ($line =~ /^new file mode\s*\d+\s*$/) { - if ($expect_spdx) { - if ($expect_spdx_file =~ - /\.(c|h|py|pl|sh|json|inc|Makefile)$/) { - # source code files MUST have SPDX license declared - ERROR("New file '$expect_spdx_file' requires " . - "'SPDX-License-Identifer'"); - } else { - # Other files MAY have SPDX license if appropriate - WARN("Does new file '$expect_spdx_file' need " . - "'SPDX-License-Identifer'?"); - } - } - $expect_spdx = 1; - $expect_spdx_file = undef; - } elsif ($expect_spdx) { - $expect_spdx_file = $realfile unless - defined $expect_spdx_file; - - # SPDX tags may occurr in comments which were - # stripped from '$line', so use '$rawline' - if ($rawline =~ /SPDX-License-Identifier/) { - $expect_spdx = 0; - $expect_spdx_file = undef; - } +# Check SPDX-License-Identifier references a permitted license + if (($rawline =~ m,SPDX-License-Identifier: (.*?)(\*/)?\s*$,) && + $rawline !~ /^-/) { + $fileinfo->{facts}->{sawspdx} = 1; + &checkspdx($realfile, $1); } -# Check SPDX-License-Identifier references a permitted license - if ($rawline =~ m,SPDX-License-Identifier: (.*?)(\*/)?\s*$,) { - &checkspdx($realfile, $1); + if ($rawline =~ /$LICENSE_BOILERPLATE_RE/) { + $fileinfo->{facts}->{sawboilerplate} = 1; } if ($rawline =~ m,(SPDX-[a-zA-Z0-9-_]+):,) { - my $tag = $1; - my @permitted = qw( - SPDX-License-Identifier - ); - - unless (grep { /^$tag$/ } @permitted) { - ERROR("Tag $tag not permitted in QEMU code, valid " . - "choices are: " . join(", ", @permitted)); - } + my $tag = $1; + my @permitted = qw( + SPDX-License-Identifier + ); + + unless (grep { /^$tag$/ } @permitted) { + ERROR("Tag $tag not permitted in QEMU code, " . + "valid choices are: " . + join(", ", @permitted)); + } } # Check for wrappage within a valid hunk of the file @@ -2304,7 +2411,7 @@ sub process { # missing space after union, struct or enum definition if ($line =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident)?(?:\s+$Ident)?[=\{]/) { - ERROR("missing space after $1 definition\n" . $herecurr); + ERROR("missing space after $1 definition\n" . $herecurr); } # check for spacing round square brackets; allowed: @@ -2599,7 +2706,7 @@ sub process { if ($line =~ /^.\s*(Q(?:S?LIST|SIMPLEQ|TAILQ)_HEAD)\s*\(\s*[^,]/ && $line !~ /^.typedef/) { - ERROR("named $1 should be typedefed separately\n" . $herecurr); + ERROR("named $1 should be typedefed separately\n" . $herecurr); } # Need a space before open parenthesis after if, while etc @@ -3085,14 +3192,18 @@ sub process { if ($line =~ /\bsignal\s*\(/ && !($line =~ /SIG_(?:IGN|DFL)/)) { ERROR("use sigaction to establish signal handlers; signal is not portable\n" . $herecurr); } -# recommend qemu_bh_new_guarded instead of qemu_bh_new - if ($realfile =~ /.*\/hw\/.*/ && $line =~ /\bqemu_bh_new\s*\(/) { - ERROR("use qemu_bh_new_guarded() instead of qemu_bh_new() to avoid reentrancy problems\n" . $herecurr); +# recommend aio_bh_new_guarded instead of legacy qemu_bh_new / qemu_bh_new_guarded + if ($realfile =~ /.*\/hw\/.*/ && $line =~ /\bqemu_bh_new(_guarded)?\s*\(/) { + ERROR("use aio_bh_new_guarded() instead of qemu_bh_new*() to avoid reentrancy problems\n" . $herecurr); } # recommend aio_bh_new_guarded instead of aio_bh_new if ($realfile =~ /.*\/hw\/.*/ && $line =~ /\baio_bh_new\s*\(/) { ERROR("use aio_bh_new_guarded() instead of aio_bh_new() to avoid reentrancy problems\n" . $herecurr); } +# check for DEVICE_NATIVE_ENDIAN, use explicit endianness instead + if ($line =~ /\bDEVICE_NATIVE_ENDIAN\b/) { + ERROR("DEVICE_NATIVE_ENDIAN is not allowed, use DEVICE_LITTLE_ENDIAN or DEVICE_BIG_ENDIAN instead\n" . $herecurr); + } # check for module_init(), use category-specific init macros explicitly please if ($line =~ /^module_init\s*\(/) { ERROR("please use block_init(), type_init() etc. instead of module_init()\n" . $herecurr); @@ -3148,48 +3259,50 @@ sub process { # Qemu error function tests - # Find newlines in error messages - my $qemu_error_funcs = qr{error_setg| - error_setg_errno| - error_setg_win32| - error_setg_file_open| - error_set| - error_prepend| - warn_reportf_err| - error_reportf_err| - error_vreport| - warn_vreport| - info_vreport| - error_report| - warn_report| - info_report| - g_test_message}x; - - if ($rawline =~ /\b(?:$qemu_error_funcs)\s*\(.*\".*\\n/) { - ERROR("Error messages should not contain newlines\n" . $herecurr); - } + # Find newlines in error messages + my $qemu_error_funcs = qr{error_setg| + error_setg_errno| + error_setg_win32| + error_setg_file_open| + error_set| + error_prepend| + warn_reportf_err| + error_reportf_err| + error_vreport| + warn_vreport| + info_vreport| + error_report| + warn_report| + info_report| + g_test_message}x; + + if ($rawline =~ /\b(?:$qemu_error_funcs)\s*\(.*\".*\\n/) { + ERROR("Error messages should not contain newlines\n" . $herecurr); + } - # Continue checking for error messages that contains newlines. This - # check handles cases where string literals are spread over multiple lines. - # Example: - # error_report("Error msg line #1" - # "Error msg line #2\n"); - my $quoted_newline_regex = qr{\+\s*\".*\\n.*\"}; - my $continued_str_literal = qr{\+\s*\".*\"}; + # Continue checking for error messages that contains newlines. + # This check handles cases where string literals are spread + # over multiple lines. + # Example: + # error_report("Error msg line #1" + # "Error msg line #2\n"); + my $quoted_newline_regex = qr{\+\s*\".*\\n.*\"}; + my $continued_str_literal = qr{\+\s*\".*\"}; - if ($rawline =~ /$quoted_newline_regex/) { - # Backtrack to first line that does not contain only a quoted literal - # and assume that it is the start of the statement. - my $i = $linenr - 2; + if ($rawline =~ /$quoted_newline_regex/) { + # Backtrack to first line that does not contain only + # a quoted literal and assume that it is the start + # of the statement. + my $i = $linenr - 2; - while (($i >= 0) & $rawlines[$i] =~ /$continued_str_literal/) { - $i--; - } + while (($i >= 0) & $rawlines[$i] =~ /$continued_str_literal/) { + $i--; + } - if ($rawlines[$i] =~ /\b(?:$qemu_error_funcs)\s*\(/) { - ERROR("Error messages should not contain newlines\n" . $herecurr); + if ($rawlines[$i] =~ /\b(?:$qemu_error_funcs)\s*\(/) { + ERROR("Error messages should not contain newlines\n" . $herecurr); + } } - } # check for non-portable libc calls that have portable alternatives in QEMU if ($line =~ /\bffs\(/) { @@ -3243,6 +3356,11 @@ sub process { } } + if (defined $fileinfo) { + process_end_of_file($fileinfo); + } + process_file_list(@fileinfolist); + if ($is_patch && $chk_signoff && $signoff == 0) { ERROR("Missing Signed-off-by: line(s)\n"); } diff --git a/scripts/ci/gitlab-failure-analysis b/scripts/ci/gitlab-failure-analysis new file mode 100755 index 0000000..906725b --- /dev/null +++ b/scripts/ci/gitlab-failure-analysis @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# +# A script to analyse failures in the gitlab pipelines. It requires an +# API key from gitlab with the following permissions: +# - api +# - read_repository +# - read_user +# + +import argparse +import gitlab +import os + +# +# Arguments +# +class NoneForEmptyStringAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + if value == '': + setattr(namespace, self.dest, None) + else: + setattr(namespace, self.dest, value) + + +parser = argparse.ArgumentParser(description="Analyse failed GitLab CI runs.") + +parser.add_argument("--gitlab", + default="https://gitlab.com", + help="GitLab instance URL (default: https://gitlab.com).") +parser.add_argument("--id", default=11167699, + type=int, + help="GitLab project id (default: 11167699 for qemu-project/qemu)") +parser.add_argument("--token", + default=os.getenv("GITLAB_TOKEN"), + help="Your personal access token with 'api' scope.") +parser.add_argument("--branch", + type=str, + default="staging", + action=NoneForEmptyStringAction, + help="The name of the branch (default: 'staging')") +parser.add_argument("--status", + type=str, + action=NoneForEmptyStringAction, + default="failed", + help="Filter by branch status (default: 'failed')") +parser.add_argument("--count", type=int, + default=3, + help="The number of failed runs to fetch.") +parser.add_argument("--skip-jobs", + default=False, + action='store_true', + help="Skip dumping the job info") +parser.add_argument("--pipeline", type=int, + nargs="+", + default=None, + help="Explicit pipeline ID(s) to fetch.") + + +if __name__ == "__main__": + args = parser.parse_args() + + gl = gitlab.Gitlab(url=args.gitlab, private_token=args.token) + project = gl.projects.get(args.id) + + + pipelines_to_process = [] + + # Use explicit pipeline IDs if provided, otherwise fetch a list + if args.pipeline: + args.count = len(args.pipeline) + for p_id in args.pipeline: + pipelines_to_process.append(project.pipelines.get(p_id)) + else: + # Use an iterator to fetch the pipelines + pipe_iter = project.pipelines.list(iterator=True, + status=args.status, + ref=args.branch) + # Check each failed pipeline + pipelines_to_process = [next(pipe_iter) for _ in range(args.count)] + + # Check each pipeline + for p in pipelines_to_process: + + jobs = p.jobs.list(get_all=True) + failed_jobs = [j for j in jobs if j.status == "failed"] + skipped_jobs = [j for j in jobs if j.status == "skipped"] + manual_jobs = [j for j in jobs if j.status == "manual"] + + trs = p.test_report_summary.get() + total = trs.total["count"] + skipped = trs.total["skipped"] + failed = trs.total["failed"] + + print(f"{p.status} pipeline {p.id}, total jobs {len(jobs)}, " + f"skipped {len(skipped_jobs)}, " + f"failed {len(failed_jobs)}, ", + f"{total} tests, " + f"{skipped} skipped tests, " + f"{failed} failed tests") + + if not args.skip_jobs: + for j in failed_jobs: + print(f" Failed job {j.id}, {j.name}, {j.web_url}") + + # It seems we can only extract failing tests from the full + # test report, maybe there is some way to filter it. + + if failed > 0: + ftr = p.test_report.get() + failed_suites = [s for s in ftr.test_suites if + s["failed_count"] > 0] + for fs in failed_suites: + name = fs["name"] + tests = fs["test_cases"] + failed_tests = [t for t in tests if t["status"] == 'failed'] + for t in failed_tests: + print(f" Failed test {t["classname"]}, {name}, {t["name"]}") diff --git a/scripts/ci/setup/build-environment.yml b/scripts/ci/setup/build-environment.yml new file mode 100644 index 0000000..528150d --- /dev/null +++ b/scripts/ci/setup/build-environment.yml @@ -0,0 +1,56 @@ +# Copyright (c) 2021 Red Hat, Inc. +# +# Author: +# Cleber Rosa <crosa@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 or +# later. See the COPYING file in the top-level directory. +# +# This is an ansible playbook file. Run it to set up systems with the +# environment needed to build QEMU. +--- +- name: Installation of basic packages to build QEMU + hosts: all + tasks: + - name: Check for suitable ansible version + delegate_to: localhost + assert: + that: + - '((ansible_version.major == 2) and (ansible_version.minor >= 8)) or (ansible_version.major >= 3)' + msg: "Unsuitable ansible version, please use version 2.8.0 or later" + + - name: Update apt cache / upgrade packages via apt + apt: + update_cache: yes + upgrade: yes + when: + - ansible_facts['distribution'] in ['Ubuntu', 'Debian'] + + # the package lists are updated by "make lcitool-refresh" + - name: Define package list file path for Ubuntu + set_fact: + package_file: "ubuntu/ubuntu-2404-{{ ansible_facts['architecture'] }}.yaml" + when: + - ansible_facts['distribution'] == 'Ubuntu' + - ansible_facts['distribution_version'] == '24.04' + + - name: Define package list file path for Debian + set_fact: + package_file: "debian/debian-{{ ansible_facts['distribution_major_version'] }}-{{ ansible_facts['architecture'] }}.yaml" + when: + - ansible_facts['distribution'] == 'Debian' + + - name: Include package lists based on OS and architecture + include_vars: + file: "{{ package_file }}" + when: + - package_file is exists + + - name: Install packages for QEMU on Ubuntu/Debian + package: + name: "{{ packages }}" + when: + - package_file is exists + - packages is defined + + diff --git a/scripts/ci/setup/debian/debian-13-ppc64le.yaml b/scripts/ci/setup/debian/debian-13-ppc64le.yaml new file mode 100644 index 0000000..25d96ce --- /dev/null +++ b/scripts/ci/setup/debian/debian-13-ppc64le.yaml @@ -0,0 +1,135 @@ +# THIS FILE WAS AUTO-GENERATED +# +# $ lcitool variables --host-arch ppc64le debian-13 qemu +# +# https://gitlab.com/libvirt/libvirt-ci + +packages: + - bash + - bc + - bindgen + - bison + - bsdextrautils + - bzip2 + - ca-certificates + - ccache + - clang + - coreutils + - dbus + - debianutils + - diffutils + - exuberant-ctags + - findutils + - flex + - gcc + - gcovr + - gettext + - git + - hostname + - libaio-dev + - libasan8 + - libasound2-dev + - libattr1-dev + - libbpf-dev + - libbrlapi-dev + - libbz2-dev + - libc6-dev + - libcacard-dev + - libcap-ng-dev + - libcapstone-dev + - libcbor-dev + - libclang-rt-dev + - libcmocka-dev + - libcurl4-gnutls-dev + - libdaxctl-dev + - libdrm-dev + - libepoxy-dev + - libfdt-dev + - libffi-dev + - libfuse3-dev + - libgbm-dev + - libgcrypt20-dev + - libglib2.0-dev + - libglusterfs-dev + - libgnutls28-dev + - libgtk-3-dev + - libgtk-vnc-2.0-dev + - libibverbs-dev + - libiscsi-dev + - libjemalloc-dev + - libjpeg62-turbo-dev + - libjson-c-dev + - liblttng-ust-dev + - liblzo2-dev + - libncursesw5-dev + - libnfs-dev + - libnuma-dev + - libpam0g-dev + - libpcre2-dev + - libpipewire-0.3-dev + - libpixman-1-dev + - libpng-dev + - libpulse-dev + - librbd-dev + - librdmacm-dev + - libsasl2-dev + - libsdl2-dev + - libsdl2-image-dev + - libseccomp-dev + - libselinux1-dev + - libslirp-dev + - libsnappy-dev + - libsndio-dev + - libspice-protocol-dev + - libspice-server-dev + - libssh-dev + - libstd-rust-dev + - libsystemd-dev + - libtasn1-6-dev + - libubsan1 + - libudev-dev + - liburing-dev + - libusb-1.0-0-dev + - libusbredirhost-dev + - libvdeplug-dev + - libvirglrenderer-dev + - libvte-2.91-dev + - libxdp-dev + - libzstd-dev + - llvm + - locales + - make + - mtools + - multipath-tools + - ncat + - nettle-dev + - ninja-build + - openssh-client + - pkgconf + - python3 + - python3-numpy + - python3-opencv + - python3-pillow + - python3-pip + - python3-setuptools + - python3-sphinx + - python3-sphinx-rtd-theme + - python3-tomli + - python3-venv + - python3-wheel + - python3-yaml + - rpm2cpio + - rustc + - sed + - socat + - sparse + - swtpm + - systemtap-sdt-dev + - tar + - tesseract-ocr + - tesseract-ocr-eng + - vulkan-tools + - xorriso + - zlib1g-dev + - zstd + diff --git a/scripts/ci/setup/gitlab-runner.yml b/scripts/ci/setup/gitlab-runner.yml index 57e7fae..76aeaf4 100644 --- a/scripts/ci/setup/gitlab-runner.yml +++ b/scripts/ci/setup/gitlab-runner.yml @@ -16,7 +16,7 @@ tasks: - debug: msg: 'Checking for a valid GitLab registration token' - failed_when: "gitlab_runner_registration_token == 'PLEASE_PROVIDE_A_VALID_TOKEN'" + failed_when: "gitlab_runner_authentication_token == 'PLEASE_PROVIDE_A_VALID_TOKEN'" - name: Create a group for the gitlab-runner service group: @@ -56,12 +56,12 @@ url: "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" mode: 0755 when: - - ansible_facts['distribution'] == 'Ubuntu' + - ansible_facts['distribution'] in ['Ubuntu', 'Debian'] - name: Run gitlab-runner repo setup script (DEB) shell: "/root/script.deb.sh" when: - - ansible_facts['distribution'] == 'Ubuntu' + - ansible_facts['distribution'] in ['Ubuntu', 'Debian'] - name: Install gitlab-runner (DEB) ansible.builtin.apt: @@ -69,7 +69,7 @@ update_cache: yes state: present when: - - ansible_facts['distribution'] == 'Ubuntu' + - ansible_facts['distribution'] in ['Ubuntu', 'Debian'] # RPM setup - name: Get gitlab-runner repo setup script (RPM) @@ -95,15 +95,7 @@ # Register Runners - name: Register the gitlab-runner - command: "/usr/bin/gitlab-runner register --non-interactive --url {{ gitlab_runner_server_url }} --registration-token {{ gitlab_runner_registration_token }} --executor shell --tag-list {{ ansible_facts[\"architecture\"] }},{{ ansible_facts[\"distribution\"]|lower }}_{{ ansible_facts[\"distribution_version\"] }} --description '{{ ansible_facts[\"distribution\"] }} {{ ansible_facts[\"distribution_version\"] }} {{ ansible_facts[\"architecture\"] }} ({{ ansible_facts[\"os_family\"] }})'" - - # The secondary runner will still run under the single gitlab-runner service - - name: Register secondary gitlab-runner - command: "/usr/bin/gitlab-runner register --non-interactive --url {{ gitlab_runner_server_url }} --registration-token {{ gitlab_runner_registration_token }} --executor shell --tag-list aarch32,{{ ansible_facts[\"distribution\"]|lower }}_{{ ansible_facts[\"distribution_version\"] }} --description '{{ ansible_facts[\"distribution\"] }} {{ ansible_facts[\"distribution_version\"] }} {{ ansible_facts[\"architecture\"] }} ({{ ansible_facts[\"os_family\"] }})'" - when: - - ansible_facts['distribution'] == 'Ubuntu' - - ansible_facts['architecture'] == 'aarch64' - - ansible_facts['distribution_version'] == '22.04' + command: "/usr/bin/gitlab-runner register --non-interactive --url {{ gitlab_runner_server_url }} --token {{ gitlab_runner_authentication_token }} --executor shell" - name: Install the gitlab-runner service using its own functionality command: "/usr/bin/gitlab-runner install --user gitlab-runner --working-directory /home/gitlab-runner" diff --git a/scripts/ci/setup/ubuntu/build-environment.yml b/scripts/ci/setup/ubuntu/build-environment.yml deleted file mode 100644 index 56b5160..0000000 --- a/scripts/ci/setup/ubuntu/build-environment.yml +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) 2021 Red Hat, Inc. -# -# Author: -# Cleber Rosa <crosa@redhat.com> -# -# This work is licensed under the terms of the GNU GPL, version 2 or -# later. See the COPYING file in the top-level directory. -# -# This is an ansible playbook file. Run it to set up systems with the -# environment needed to build QEMU. ---- -- name: Installation of basic packages to build QEMU - hosts: all - tasks: - - name: Check for suitable ansible version - delegate_to: localhost - assert: - that: - - '((ansible_version.major == 2) and (ansible_version.minor >= 8)) or (ansible_version.major >= 3)' - msg: "Unsuitable ansible version, please use version 2.8.0 or later" - - - name: Add armhf foreign architecture to aarch64 hosts - command: dpkg --add-architecture armhf - when: - - ansible_facts['distribution'] == 'Ubuntu' - - ansible_facts['architecture'] == 'aarch64' - - - name: Update apt cache / upgrade packages via apt - apt: - update_cache: yes - upgrade: yes - when: - - ansible_facts['distribution'] == 'Ubuntu' - - # the package lists are updated by "make lcitool-refresh" - - name: Include package lists based on OS and architecture - include_vars: - file: "ubuntu-2204-{{ ansible_facts['architecture'] }}.yaml" - when: - - ansible_facts['distribution'] == 'Ubuntu' - - ansible_facts['distribution_version'] == '22.04' - - - name: Install packages for QEMU on Ubuntu 22.04 - package: - name: "{{ packages }}" - when: - - ansible_facts['distribution'] == 'Ubuntu' - - ansible_facts['distribution_version'] == '22.04' - - - name: Install armhf cross-compile packages to build QEMU on AArch64 Ubuntu 22.04 - package: - name: - - binutils-arm-linux-gnueabihf - - gcc-arm-linux-gnueabihf - - libblkid-dev:armhf - - libc6-dev:armhf - - libffi-dev:armhf - - libglib2.0-dev:armhf - - libmount-dev:armhf - - libpcre2-dev:armhf - - libpixman-1-dev:armhf - - zlib1g-dev:armhf - when: - - ansible_facts['distribution'] == 'Ubuntu' - - ansible_facts['distribution_version'] == '22.04' - - ansible_facts['architecture'] == 'aarch64' - diff --git a/scripts/ci/setup/ubuntu/ubuntu-2204-armhf-cross.yml b/scripts/ci/setup/ubuntu/ubuntu-2204-armhf-cross.yml deleted file mode 100644 index 0cc34cd..0000000 --- a/scripts/ci/setup/ubuntu/ubuntu-2204-armhf-cross.yml +++ /dev/null @@ -1,127 +0,0 @@ -# THIS FILE WAS AUTO-GENERATED -# -# $ lcitool variables --cross-arch armv7l ubuntu-2204 qemu -# -# https://gitlab.com/libvirt/libvirt-ci - -packages: - - bash - - bc - - bison - - bsdextrautils - - bzip2 - - ca-certificates - - ccache - - dbus - - debianutils - - diffutils - - exuberant-ctags - - findutils - - flex - - gcc - - gcovr - - gettext - - git - - hostname - - libglib2.0-dev - - libpcre2-dev - - libsndio-dev - - libspice-protocol-dev - - llvm - - locales - - make - - meson - - mtools - - ncat - - ninja-build - - openssh-client - - pkgconf - - python3 - - python3-numpy - - python3-opencv - - python3-pillow - - python3-pip - - python3-sphinx - - python3-sphinx-rtd-theme - - python3-tomli - - python3-venv - - python3-yaml - - rpm2cpio - - sed - - socat - - sparse - - swtpm - - tar - - tesseract-ocr - - tesseract-ocr-eng - - xorriso - - zstd - - gcc-arm-linux-gnueabihf - - libaio-dev:armhf - - libasan6:armhf - - libasound2-dev:armhf - - libattr1-dev:armhf - - libbpf-dev:armhf - - libbrlapi-dev:armhf - - libbz2-dev:armhf - - libc6-dev:armhf - - libcacard-dev:armhf - - libcap-ng-dev:armhf - - libcapstone-dev:armhf - - libcmocka-dev:armhf - - libcurl4-gnutls-dev:armhf - - libdaxctl-dev:armhf - - libdrm-dev:armhf - - libepoxy-dev:armhf - - libfdt-dev:armhf - - libffi-dev:armhf - - libfuse3-dev:armhf - - libgbm-dev:armhf - - libgcrypt20-dev:armhf - - libglib2.0-dev:armhf - - libglusterfs-dev:armhf - - libgnutls28-dev:armhf - - libgtk-3-dev:armhf - - libibumad-dev:armhf - - libibverbs-dev:armhf - - libiscsi-dev:armhf - - libjemalloc-dev:armhf - - libjpeg-turbo8-dev:armhf - - libjson-c-dev:armhf - - liblttng-ust-dev:armhf - - liblzo2-dev:armhf - - libncursesw5-dev:armhf - - libnfs-dev:armhf - - libnuma-dev:armhf - - libpam0g-dev:armhf - - libpipewire-0.3-dev:armhf - - libpixman-1-dev:armhf - - libpng-dev:armhf - - libpulse-dev:armhf - - librbd-dev:armhf - - librdmacm-dev:armhf - - libsasl2-dev:armhf - - libsdl2-dev:armhf - - libsdl2-image-dev:armhf - - libseccomp-dev:armhf - - libselinux1-dev:armhf - - libslirp-dev:armhf - - libsnappy-dev:armhf - - libspice-server-dev:armhf - - libssh-dev:armhf - - libsystemd-dev:armhf - - libtasn1-6-dev:armhf - - libubsan1:armhf - - libudev-dev:armhf - - liburing-dev:armhf - - libusb-1.0-0-dev:armhf - - libusbredirhost-dev:armhf - - libvdeplug-dev:armhf - - libvirglrenderer-dev:armhf - - libvte-2.91-dev:armhf - - libxen-dev:armhf - - libzstd-dev:armhf - - nettle-dev:armhf - - systemtap-sdt-dev:armhf - - zlib1g-dev:armhf - diff --git a/scripts/ci/setup/ubuntu/ubuntu-2204-aarch64.yaml b/scripts/ci/setup/ubuntu/ubuntu-2404-aarch64.yaml index dbcd2e0..6a72eab 100644 --- a/scripts/ci/setup/ubuntu/ubuntu-2204-aarch64.yaml +++ b/scripts/ci/setup/ubuntu/ubuntu-2404-aarch64.yaml @@ -1,18 +1,20 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool variables --host-arch aarch64 ubuntu-2204 qemu +# $ lcitool variables --host-arch aarch64 ubuntu-2404 qemu # # https://gitlab.com/libvirt/libvirt-ci packages: - bash - bc + - bindgen - bison - bsdextrautils - bzip2 - ca-certificates - ccache - clang + - coreutils - dbus - debianutils - diffutils @@ -25,7 +27,7 @@ packages: - git - hostname - libaio-dev - - libasan6 + - libasan8 - libasound2-dev - libattr1-dev - libbpf-dev @@ -36,6 +38,7 @@ packages: - libcap-ng-dev - libcapstone-dev - libcbor-dev + - libclang-rt-dev - libcmocka-dev - libcurl4-gnutls-dev - libdaxctl-dev @@ -80,6 +83,7 @@ packages: - libspice-protocol-dev - libspice-server-dev - libssh-dev + - libstd-rust-dev - libsystemd-dev - libtasn1-6-dev - libubsan1 @@ -90,12 +94,12 @@ packages: - libvdeplug-dev - libvirglrenderer-dev - libvte-2.91-dev + - libxdp-dev - libxen-dev - libzstd-dev - llvm - locales - make - - meson - mtools - multipath-tools - ncat @@ -108,13 +112,15 @@ packages: - python3-opencv - python3-pillow - python3-pip + - python3-setuptools - python3-sphinx - python3-sphinx-rtd-theme - python3-tomli - python3-venv + - python3-wheel - python3-yaml - rpm2cpio - - rustc + - rustc-1.83 - sed - socat - sparse diff --git a/scripts/ci/setup/ubuntu/ubuntu-2204-s390x.yaml b/scripts/ci/setup/ubuntu/ubuntu-2404-s390x.yaml index 4b8ee3d..6001da1 100644 --- a/scripts/ci/setup/ubuntu/ubuntu-2204-s390x.yaml +++ b/scripts/ci/setup/ubuntu/ubuntu-2404-s390x.yaml @@ -1,18 +1,20 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool variables --host-arch s390x ubuntu-2204 qemu +# $ lcitool variables --host-arch s390x ubuntu-2404 qemu # # https://gitlab.com/libvirt/libvirt-ci packages: - bash - bc + - bindgen - bison - bsdextrautils - bzip2 - ca-certificates - ccache - clang + - coreutils - dbus - debianutils - diffutils @@ -25,7 +27,7 @@ packages: - git - hostname - libaio-dev - - libasan6 + - libasan8 - libasound2-dev - libattr1-dev - libbpf-dev @@ -36,6 +38,7 @@ packages: - libcap-ng-dev - libcapstone-dev - libcbor-dev + - libclang-rt-dev - libcmocka-dev - libcurl4-gnutls-dev - libdaxctl-dev @@ -79,6 +82,7 @@ packages: - libsndio-dev - libspice-protocol-dev - libssh-dev + - libstd-rust-dev - libsystemd-dev - libtasn1-6-dev - libubsan1 @@ -89,11 +93,11 @@ packages: - libvdeplug-dev - libvirglrenderer-dev - libvte-2.91-dev + - libxdp-dev - libzstd-dev - llvm - locales - make - - meson - mtools - multipath-tools - ncat @@ -106,13 +110,15 @@ packages: - python3-opencv - python3-pillow - python3-pip + - python3-setuptools - python3-sphinx - python3-sphinx-rtd-theme - python3-tomli - python3-venv + - python3-wheel - python3-yaml - rpm2cpio - - rustc + - rustc-1.83 - sed - socat - sparse diff --git a/scripts/ci/setup/vars.yml.template b/scripts/ci/setup/vars.yml.template index 4b355fb..e9ddc05 100644 --- a/scripts/ci/setup/vars.yml.template +++ b/scripts/ci/setup/vars.yml.template @@ -6,5 +6,6 @@ ansible_to_gitlab_arch: x86_64: amd64 aarch64: arm64 s390x: s390x -# A unique token made available by GitLab to your project for registering runners -gitlab_runner_registration_token: PLEASE_PROVIDE_A_VALID_TOKEN +# A unique token made obtained from GitLab for each runner +# see: https://gitlab.com/PROJECT/REPO/-/runners/new +gitlab_runner_authentication_token: PLEASE_PROVIDE_A_VALID_TOKEN diff --git a/scripts/clean_functional_cache.py b/scripts/clean_functional_cache.py new file mode 100755 index 0000000..c3370ff --- /dev/null +++ b/scripts/clean_functional_cache.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# +# SPDX-License-Identifier: GPL-2.0-or-later +# +"""Delete stale assets from the download cache of the functional tests""" + +import os +import stat +import sys +import time +from pathlib import Path + + +cache_dir_env = os.getenv('QEMU_TEST_CACHE_DIR') +if cache_dir_env: + cache_dir = Path(cache_dir_env, "download") +else: + cache_dir = Path(Path("~").expanduser(), ".cache", "qemu", "download") + +if not cache_dir.exists(): + print(f"Cache dir {cache_dir} does not exist!", file=sys.stderr) + sys.exit(1) + +os.chdir(cache_dir) + +for file in cache_dir.iterdir(): + # Only consider the files that use a sha256 as filename: + if len(file.name) != 64: + continue + + try: + timestamp = int(file.with_suffix(".stamp").read_text()) + except FileNotFoundError: + # Assume it's an old file that was already in the cache before we + # added the code for evicting stale assets. Use the release date + # of QEMU v10.1 as a default timestamp. + timestamp = time.mktime((2025, 8, 26, 0, 0, 0, 0, 0, 0)) + + age = time.time() - timestamp + + # Delete files older than half of a year (183 days * 24h * 60m * 60s) + if age > 15811200: + print(f"Removing {cache_dir}/{file.name}.") + file.chmod(stat.S_IWRITE) + file.unlink() diff --git a/scripts/coccinelle/exec_rw_const.cocci b/scripts/coccinelle/exec_rw_const.cocci index 1a20296..4c02c94 100644 --- a/scripts/coccinelle/exec_rw_const.cocci +++ b/scripts/coccinelle/exec_rw_const.cocci @@ -21,13 +21,6 @@ expression E1, E2, E3, E4, E5; + address_space_rw(E1, E2, E3, E4, E5, true) | -- cpu_physical_memory_rw(E1, E2, E3, 0) -+ cpu_physical_memory_rw(E1, E2, E3, false) -| -- cpu_physical_memory_rw(E1, E2, E3, 1) -+ cpu_physical_memory_rw(E1, E2, E3, true) -| - - cpu_physical_memory_map(E1, E2, 0) + cpu_physical_memory_map(E1, E2, false) | @@ -62,18 +55,6 @@ symbol true, false; + address_space_write(E1, E2, E3, E4, E5) ) -// Avoid uses of cpu_physical_memory_rw() with a constant is_write argument. -@@ -expression E1, E2, E3; -@@ -( -- cpu_physical_memory_rw(E1, E2, E3, false) -+ cpu_physical_memory_read(E1, E2, E3) -| -- cpu_physical_memory_rw(E1, E2, E3, true) -+ cpu_physical_memory_write(E1, E2, E3) -) - // Remove useless cast @@ expression E1, E2, E3, E4, E5, E6; @@ -93,9 +74,6 @@ type T; + address_space_write_rom(E1, E2, E3, E4, E5) | -- cpu_physical_memory_rw(E1, (T *)(E2), E3, E4) -+ cpu_physical_memory_rw(E1, E2, E3, E4) -| - cpu_physical_memory_read(E1, (T *)(E2), E3) + cpu_physical_memory_read(E1, E2, E3) | diff --git a/scripts/codeconverter/codeconverter/qom_type_info.py b/scripts/codeconverter/codeconverter/qom_type_info.py index f92c3a4..22a2556 100644 --- a/scripts/codeconverter/codeconverter/qom_type_info.py +++ b/scripts/codeconverter/codeconverter/qom_type_info.py @@ -798,7 +798,8 @@ class RedundantTypeSizes(TypeInfoVar): # # # if 'class_init' not in fields: -# yield self.prepend(('static void %s_class_init(ObjectClass *oc, void *data)\n' +# yield self.prepend(('static void %s_class_init(ObjectClass *oc,\n' +# 'const void *data)\n' # '{\n' # '}\n\n') % (ids.lowercase)) # yield self.append_field('class_init', ids.lowercase+'_class_init') diff --git a/scripts/codeconverter/codeconverter/test_regexps.py b/scripts/codeconverter/codeconverter/test_regexps.py index 7211392..b00e9ef 100644 --- a/scripts/codeconverter/codeconverter/test_regexps.py +++ b/scripts/codeconverter/codeconverter/test_regexps.py @@ -70,15 +70,15 @@ static const TypeInfo char_file_type_info = { .name = armsse_variants[i].name, .parent = TYPE_ARMSSE, .class_init = armsse_class_init, - .class_data = (void *)&armsse_variants[i], + .class_data = &armsse_variants[i], };''', re.MULTILINE) print(RE_ARRAY_ITEM) assert fullmatch(RE_ARRAY_ITEM, '{ TYPE_HOTPLUG_HANDLER },') assert fullmatch(RE_ARRAY_ITEM, '{ TYPE_ACPI_DEVICE_IF },') assert fullmatch(RE_ARRAY_ITEM, '{ }') - assert fullmatch(RE_ARRAY_CAST, '(InterfaceInfo[])') - assert fullmatch(RE_ARRAY, '''(InterfaceInfo[]) { + assert fullmatch(RE_ARRAY_CAST, '(const InterfaceInfo[])') + assert fullmatch(RE_ARRAY, '''(const InterfaceInfo[]) { { TYPE_HOTPLUG_HANDLER }, { TYPE_ACPI_DEVICE_IF }, { } @@ -98,7 +98,7 @@ static const TypeInfo char_file_type_info = { .parent = TYPE_DEVICE, .instance_size = sizeof(CRBState), .class_init = tpm_crb_class_init, - .interfaces = (InterfaceInfo[]) { + .interfaces = (const InterfaceInfo[]) { { TYPE_TPM_IF }, { } } @@ -134,7 +134,7 @@ static const TypeInfo char_file_type_info = { .instance_size = sizeof(AcpiGedState), .instance_init = acpi_ged_initfn, .class_init = acpi_ged_class_init, - .interfaces = (InterfaceInfo[]) { + .interfaces = (const InterfaceInfo[]) { { TYPE_HOTPLUG_HANDLER }, { TYPE_ACPI_DEVICE_IF }, { } @@ -164,7 +164,7 @@ static const TypeInfo char_file_type_info = { .parent = TYPE_DEVICE, .instance_size = sizeof(CRBState), .class_init = tpm_crb_class_init, - .interfaces = (InterfaceInfo[]) { + .interfaces = (const InterfaceInfo[]) { { TYPE_TPM_IF }, { } } @@ -264,8 +264,8 @@ def test_initial_includes(): #define SILENT_ES1370 #include "qemu/osdep.h" -#include "hw/audio/soundhw.h" -#include "audio/audio.h" +#include "hw/audio/model.h" +#include "qemu/audio.h" #include "hw/pci/pci.h" #include "migration/vmstate.h" #include "qemu/module.h" diff --git a/scripts/coverity-scan/COMPONENTS.md b/scripts/coverity-scan/COMPONENTS.md index 7299590..95805b5 100644 --- a/scripts/coverity-scan/COMPONENTS.md +++ b/scripts/coverity-scan/COMPONENTS.md @@ -147,6 +147,9 @@ tcg system ~ .*/qemu(/system/.*|/accel/.*) +plugins + ~ .*/qemu(/contrib|/tests/tcg)?/plugins/.* + (headers) ~ .*/qemu(/include/.*) diff --git a/scripts/decodetree.py b/scripts/decodetree.py index e8b72da..f992472 100644 --- a/scripts/decodetree.py +++ b/scripts/decodetree.py @@ -1016,9 +1016,12 @@ def infer_format(arg, fieldmask, flds, width): else: var_flds[n] = c + if not arg: + arg = infer_argument_set(flds) + # Look for an existing format with the same argument set and fields for fmt in formats.values(): - if arg and fmt.base != arg: + if fmt.base != arg: continue if fieldmask != fmt.fieldmask: continue @@ -1029,8 +1032,6 @@ def infer_format(arg, fieldmask, flds, width): return (fmt, const_flds) name = decode_function + '_Fmt_' + str(len(formats)) - if not arg: - arg = infer_argument_set(flds) fmt = Format(name, 0, arg, 0, 0, 0, fieldmask, var_flds, width) formats[name] = fmt diff --git a/scripts/device-crash-test b/scripts/device-crash-test index da8b56e..c1576e8 100755 --- a/scripts/device-crash-test +++ b/scripts/device-crash-test @@ -16,8 +16,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program; if not, see <https://www.gnu.org/licenses/>. """ Run QEMU with all combinations of -machine and -device types, @@ -528,7 +527,7 @@ def main(): # Async QMP, when in use, is chatty about connection failures. # This script knowingly generates a ton of connection errors. # Silence this logger. - logging.getLogger('qemu.qmp.qmp_client').setLevel(logging.CRITICAL) + logging.getLogger('qemu.qmp.protocol').setLevel(logging.CRITICAL) fatal_failures = [] wl_stats = {} diff --git a/scripts/get-wraps-from-cargo-registry.py b/scripts/get-wraps-from-cargo-registry.py new file mode 100755 index 0000000..31eed5c --- /dev/null +++ b/scripts/get-wraps-from-cargo-registry.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 + +# SPDX-License-Identifier: GPL-2.0-or-later + +""" +get-wraps-from-cargo-registry.py - Update Meson subprojects from a global registry +""" + +# Copyright (C) 2025 Red Hat, Inc. +# +# Author: Paolo Bonzini <pbonzini@redhat.com> + +import argparse +import configparser +import filecmp +import glob +import os +import subprocess +import sys + + +def get_name_and_semver(namever: str) -> tuple[str, str]: + """Split a subproject name into its name and semantic version parts""" + parts = namever.rsplit("-", 1) + if len(parts) != 2: + return namever, "" + + return parts[0], parts[1] + + +class UpdateSubprojects: + cargo_registry: str + top_srcdir: str + dry_run: bool + changes: int = 0 + + def find_installed_crate(self, namever: str) -> str | None: + """Find installed crate matching name and semver prefix""" + name, semver = get_name_and_semver(namever) + + # exact version match + path = os.path.join(self.cargo_registry, f"{name}-{semver}") + if os.path.exists(path): + return f"{name}-{semver}" + + # semver match + matches = sorted(glob.glob(f"{path}.*")) + return os.path.basename(matches[0]) if matches else None + + def compare_build_rs(self, orig_dir: str, registry_namever: str) -> None: + """Warn if the build.rs in the original directory differs from the registry version.""" + orig_build_rs = os.path.join(orig_dir, "build.rs") + new_build_rs = os.path.join(self.cargo_registry, registry_namever, "build.rs") + + msg = None + if os.path.isfile(orig_build_rs) != os.path.isfile(new_build_rs): + if os.path.isfile(orig_build_rs): + msg = f"build.rs removed in {registry_namever}" + if os.path.isfile(new_build_rs): + msg = f"build.rs added in {registry_namever}" + + elif os.path.isfile(orig_build_rs) and not filecmp.cmp(orig_build_rs, new_build_rs): + msg = f"build.rs changed from {orig_dir} to {registry_namever}" + + if msg: + print(f"⚠️ Warning: {msg}") + print(" This may affect the build process - please review the differences.") + + def update_subproject(self, wrap_file: str, registry_namever: str) -> None: + """Modify [wrap-file] section to point to self.cargo_registry.""" + assert wrap_file.endswith("-rs.wrap") + wrap_name = wrap_file[:-5] + + env = os.environ.copy() + env["MESON_PACKAGE_CACHE_DIR"] = self.cargo_registry + + config = configparser.ConfigParser() + config.read(wrap_file) + if "wrap-file" not in config: + return + + # do not download the wrap, always use the local copy + orig_dir = config["wrap-file"]["directory"] + if os.path.exists(orig_dir) and orig_dir != registry_namever: + self.compare_build_rs(orig_dir, registry_namever) + + if self.dry_run: + if orig_dir == registry_namever: + print(f"Will install {orig_dir} from registry.") + else: + print(f"Will replace {orig_dir} with {registry_namever}.") + self.changes += 1 + return + + config["wrap-file"]["directory"] = registry_namever + for key in list(config["wrap-file"].keys()): + if key.startswith("source"): + del config["wrap-file"][key] + + # replace existing directory with installed version + if os.path.exists(orig_dir): + subprocess.run( + ["meson", "subprojects", "purge", "--confirm", wrap_name], + cwd=self.top_srcdir, + env=env, + check=True, + ) + + with open(wrap_file, "w") as f: + config.write(f) + + if orig_dir == registry_namever: + print(f"Installing {orig_dir} from registry.") + else: + print(f"Replacing {orig_dir} with {registry_namever}.") + patch_dir = config["wrap-file"]["patch_directory"] + patch_dir = os.path.join("packagefiles", patch_dir) + _, ver = registry_namever.rsplit("-", 1) + subprocess.run( + ["meson", "rewrite", "kwargs", "set", "project", "/", "version", ver], + cwd=patch_dir, + env=env, + check=True, + ) + + subprocess.run( + ["meson", "subprojects", "download", wrap_name], + cwd=self.top_srcdir, + env=env, + check=True, + ) + self.changes += 1 + + @staticmethod + def parse_cmdline() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Replace Meson subprojects with packages in a Cargo registry" + ) + parser.add_argument( + "--cargo-registry", + default=os.environ.get("CARGO_REGISTRY"), + help="Path to Cargo registry (default: CARGO_REGISTRY env var)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + default=False, + help="Do not actually replace anything", + ) + + args = parser.parse_args() + if not args.cargo_registry: + print("error: CARGO_REGISTRY environment variable not set and --cargo-registry not provided") + sys.exit(1) + + return args + + def __init__(self, args: argparse.Namespace): + self.cargo_registry = args.cargo_registry + self.dry_run = args.dry_run + self.top_srcdir = os.getcwd() + + def main(self) -> None: + if not os.path.exists("subprojects"): + print("'subprojects' directory not found, nothing to do.") + return + + os.chdir("subprojects") + for wrap_file in sorted(glob.glob("*-rs.wrap")): + namever = wrap_file[:-8] # Remove '-rs.wrap' + + registry_namever = self.find_installed_crate(namever) + if not registry_namever: + print(f"No installed crate found for {wrap_file}") + continue + + self.update_subproject(wrap_file, registry_namever) + + if self.changes: + if self.dry_run: + print("Rerun without --dry-run to apply changes.") + else: + print(f"✨ {self.changes} subproject(s) updated!") + else: + print("No changes.") + + +if __name__ == "__main__": + args = UpdateSubprojects.parse_cmdline() + UpdateSubprojects(args).main() diff --git a/scripts/ghes_inject.py b/scripts/ghes_inject.py new file mode 100755 index 0000000..9a23520 --- /dev/null +++ b/scripts/ghes_inject.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Copyright (C) 2024-2025 Mauro Carvalho Chehab <mchehab+huawei@kernel.org> + +""" +Handle ACPI GHESv2 error injection logic QEMU QMP interface. +""" + +import argparse +import sys + +from arm_processor_error import ArmProcessorEinj + +EINJ_DESC = """ +Handle ACPI GHESv2 error injection logic QEMU QMP interface. + +It allows using UEFI BIOS EINJ features to generate GHES records. + +It helps testing CPER and GHES drivers at the guest OS and how +userspace applications at the guest handle them. +""" + +def main(): + """Main program""" + + # Main parser - handle generic args like QEMU QMP TCP socket options + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, + usage="%(prog)s [options]", + description=EINJ_DESC) + + g_options = parser.add_argument_group("QEMU QMP socket options") + g_options.add_argument("-H", "--host", default="localhost", type=str, + help="host name") + g_options.add_argument("-P", "--port", default=4445, type=int, + help="TCP port number") + g_options.add_argument('-d', '--debug', action='store_true') + + subparsers = parser.add_subparsers() + + ArmProcessorEinj(subparsers) + + args = parser.parse_args() + if "func" in args: + args.func(args) + else: + sys.exit(f"Please specify a valid command for {sys.argv[0]}") + +if __name__ == "__main__": + main() diff --git a/scripts/kernel-doc b/scripts/kernel-doc deleted file mode 100755 index fec83f5..0000000 --- a/scripts/kernel-doc +++ /dev/null @@ -1,2442 +0,0 @@ -#!/usr/bin/env perl -# SPDX-License-Identifier: GPL-2.0-only - -use warnings; -use strict; - -## Copyright (c) 1998 Michael Zucchi, All Rights Reserved ## -## Copyright (C) 2000, 1 Tim Waugh <twaugh@redhat.com> ## -## Copyright (C) 2001 Simon Huggins ## -## Copyright (C) 2005-2012 Randy Dunlap ## -## Copyright (C) 2012 Dan Luedtke ## -## ## -## #define enhancements by Armin Kuster <akuster@mvista.com> ## -## Copyright (c) 2000 MontaVista Software, Inc. ## -## ## -## This software falls under the GNU General Public License. ## -## Please read the COPYING file for more information ## - -# 18/01/2001 - Cleanups -# Functions prototyped as foo(void) same as foo() -# Stop eval'ing where we don't need to. -# -- huggie@earth.li - -# 27/06/2001 - Allowed whitespace after initial "/**" and -# allowed comments before function declarations. -# -- Christian Kreibich <ck@whoop.org> - -# Still to do: -# - add perldoc documentation -# - Look more closely at some of the scarier bits :) - -# 26/05/2001 - Support for separate source and object trees. -# Return error code. -# Keith Owens <kaos@ocs.com.au> - -# 23/09/2001 - Added support for typedefs, structs, enums and unions -# Support for Context section; can be terminated using empty line -# Small fixes (like spaces vs. \s in regex) -# -- Tim Jansen <tim@tjansen.de> - -# 25/07/2012 - Added support for HTML5 -# -- Dan Luedtke <mail@danrl.de> - -sub usage { - my $message = <<"EOF"; -Usage: $0 [OPTION ...] FILE ... - -Read C language source or header FILEs, extract embedded documentation comments, -and print formatted documentation to standard output. - -The documentation comments are identified by "/**" opening comment mark. See -Documentation/doc-guide/kernel-doc.rst for the documentation comment syntax. - -Output format selection (mutually exclusive): - -man Output troff manual page format. This is the default. - -rst Output reStructuredText format. - -none Do not output documentation, only warnings. - -Output format selection modifier (affects only ReST output): - - -sphinx-version Use the ReST C domain dialect compatible with an - specific Sphinx Version. - If not specified, kernel-doc will auto-detect using - the sphinx-build version found on PATH. - -Output selection (mutually exclusive): - -export Only output documentation for symbols that have been - exported using EXPORT_SYMBOL() or EXPORT_SYMBOL_GPL() - in any input FILE or -export-file FILE. - -internal Only output documentation for symbols that have NOT been - exported using EXPORT_SYMBOL() or EXPORT_SYMBOL_GPL() - in any input FILE or -export-file FILE. - -function NAME Only output documentation for the given function(s) - or DOC: section title(s). All other functions and DOC: - sections are ignored. May be specified multiple times. - -nosymbol NAME Exclude the specified symbols from the output - documentation. May be specified multiple times. - -Output selection modifiers: - -no-doc-sections Do not output DOC: sections. - -enable-lineno Enable output of #define LINENO lines. Only works with - reStructuredText format. - -export-file FILE Specify an additional FILE in which to look for - EXPORT_SYMBOL() and EXPORT_SYMBOL_GPL(). To be used with - -export or -internal. May be specified multiple times. - -Other parameters: - -v Verbose output, more warnings and other information. - -h Print this help. - -Werror Treat warnings as errors. - -EOF - print $message; - exit 1; -} - -# -# format of comments. -# In the following table, (...)? signifies optional structure. -# (...)* signifies 0 or more structure elements -# /** -# * function_name(:)? (- short description)? -# (* @parameterx: (description of parameter x)?)* -# (* a blank line)? -# * (Description:)? (Description of function)? -# * (section header: (section description)? )* -# (*)?*/ -# -# So .. the trivial example would be: -# -# /** -# * my_function -# */ -# -# If the Description: header tag is omitted, then there must be a blank line -# after the last parameter specification. -# e.g. -# /** -# * my_function - does my stuff -# * @my_arg: its mine damnit -# * -# * Does my stuff explained. -# */ -# -# or, could also use: -# /** -# * my_function - does my stuff -# * @my_arg: its mine damnit -# * Description: Does my stuff explained. -# */ -# etc. -# -# Besides functions you can also write documentation for structs, unions, -# enums and typedefs. Instead of the function name you must write the name -# of the declaration; the struct/union/enum/typedef must always precede -# the name. Nesting of declarations is not supported. -# Use the argument mechanism to document members or constants. -# e.g. -# /** -# * struct my_struct - short description -# * @a: first member -# * @b: second member -# * -# * Longer description -# */ -# struct my_struct { -# int a; -# int b; -# /* private: */ -# int c; -# }; -# -# All descriptions can be multiline, except the short function description. -# -# For really longs structs, you can also describe arguments inside the -# body of the struct. -# eg. -# /** -# * struct my_struct - short description -# * @a: first member -# * @b: second member -# * -# * Longer description -# */ -# struct my_struct { -# int a; -# int b; -# /** -# * @c: This is longer description of C -# * -# * You can use paragraphs to describe arguments -# * using this method. -# */ -# int c; -# }; -# -# This should be use only for struct/enum members. -# -# You can also add additional sections. When documenting kernel functions you -# should document the "Context:" of the function, e.g. whether the functions -# can be called form interrupts. Unlike other sections you can end it with an -# empty line. -# A non-void function should have a "Return:" section describing the return -# value(s). -# Example-sections should contain the string EXAMPLE so that they are marked -# appropriately in DocBook. -# -# Example: -# /** -# * user_function - function that can only be called in user context -# * @a: some argument -# * Context: !in_interrupt() -# * -# * Some description -# * Example: -# * user_function(22); -# */ -# ... -# -# -# All descriptive text is further processed, scanning for the following special -# patterns, which are highlighted appropriately. -# -# 'funcname()' - function -# '$ENVVAR' - environmental variable -# '&struct_name' - name of a structure (up to two words including 'struct') -# '&struct_name.member' - name of a structure member -# '@parameter' - name of a parameter -# '%CONST' - name of a constant. -# '``LITERAL``' - literal string without any spaces on it. - -## init lots of data - -my $errors = 0; -my $warnings = 0; -my $anon_struct_union = 0; - -# match expressions used to find embedded type information -my $type_constant = '\b``([^\`]+)``\b'; -my $type_constant2 = '\%([-_\w]+)'; -my $type_func = '(\w+)\(\)'; -my $type_param = '\@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)'; -my $type_param_ref = '([\!]?)\@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)'; -my $type_fp_param = '\@(\w+)\(\)'; # Special RST handling for func ptr params -my $type_fp_param2 = '\@(\w+->\S+)\(\)'; # Special RST handling for structs with func ptr params -my $type_env = '(\$\w+)'; -my $type_enum = '#(enum\s*([_\w]+))'; -my $type_struct = '#(struct\s*([_\w]+))'; -my $type_typedef = '#(([A-Z][_\w]*))'; -my $type_union = '#(union\s*([_\w]+))'; -my $type_member = '#([_\w]+)(\.|->)([_\w]+)'; -my $type_fallback = '(?!)'; # this never matches -my $type_member_func = $type_member . '\(\)'; - -# Output conversion substitutions. -# One for each output format - -# these are pretty rough -my @highlights_man = ( - [$type_constant, "\$1"], - [$type_constant2, "\$1"], - [$type_func, "\\\\fB\$1\\\\fP"], - [$type_enum, "\\\\fI\$1\\\\fP"], - [$type_struct, "\\\\fI\$1\\\\fP"], - [$type_typedef, "\\\\fI\$1\\\\fP"], - [$type_union, "\\\\fI\$1\\\\fP"], - [$type_param, "\\\\fI\$1\\\\fP"], - [$type_param_ref, "\\\\fI\$1\$2\\\\fP"], - [$type_member, "\\\\fI\$1\$2\$3\\\\fP"], - [$type_fallback, "\\\\fI\$1\\\\fP"] - ); -my $blankline_man = ""; - -# rst-mode -my @highlights_rst = ( - [$type_constant, "``\$1``"], - [$type_constant2, "``\$1``"], - # Note: need to escape () to avoid func matching later - [$type_member_func, "\\:c\\:type\\:`\$1\$2\$3\\\\(\\\\) <\$1>`"], - [$type_member, "\\:c\\:type\\:`\$1\$2\$3 <\$1>`"], - [$type_fp_param, "**\$1\\\\(\\\\)**"], - [$type_fp_param2, "**\$1\\\\(\\\\)**"], - [$type_func, "\$1()"], - [$type_enum, "\\:c\\:type\\:`\$1 <\$2>`"], - [$type_struct, "\\:c\\:type\\:`\$1 <\$2>`"], - [$type_typedef, "\\:c\\:type\\:`\$1 <\$2>`"], - [$type_union, "\\:c\\:type\\:`\$1 <\$2>`"], - # in rst this can refer to any type - [$type_fallback, "\\:c\\:type\\:`\$1`"], - [$type_param_ref, "**\$1\$2**"] - ); -my $blankline_rst = "\n"; - -# read arguments -if ($#ARGV == -1) { - usage(); -} - -my $kernelversion; -my ($sphinx_major, $sphinx_minor, $sphinx_patch); - -my $dohighlight = ""; - -my $verbose = 0; -my $Werror = 0; -my $output_mode = "rst"; -my $output_preformatted = 0; -my $no_doc_sections = 0; -my $enable_lineno = 0; -my @highlights = @highlights_rst; -my $blankline = $blankline_rst; -my $modulename = "Kernel API"; - -use constant { - OUTPUT_ALL => 0, # output all symbols and doc sections - OUTPUT_INCLUDE => 1, # output only specified symbols - OUTPUT_EXPORTED => 2, # output exported symbols - OUTPUT_INTERNAL => 3, # output non-exported symbols -}; -my $output_selection = OUTPUT_ALL; -my $show_not_found = 0; # No longer used - -my @export_file_list; - -my @build_time; -if (defined($ENV{'KBUILD_BUILD_TIMESTAMP'}) && - (my $seconds = `date -d"${ENV{'KBUILD_BUILD_TIMESTAMP'}}" +%s`) ne '') { - @build_time = gmtime($seconds); -} else { - @build_time = localtime; -} - -my $man_date = ('January', 'February', 'March', 'April', 'May', 'June', - 'July', 'August', 'September', 'October', - 'November', 'December')[$build_time[4]] . - " " . ($build_time[5]+1900); - -# Essentially these are globals. -# They probably want to be tidied up, made more localised or something. -# CAVEAT EMPTOR! Some of the others I localised may not want to be, which -# could cause "use of undefined value" or other bugs. -my ($function, %function_table, %parametertypes, $declaration_purpose); -my %nosymbol_table = (); -my $declaration_start_line; -my ($type, $declaration_name, $return_type); -my ($newsection, $newcontents, $prototype, $brcount, %source_map); - -if (defined($ENV{'KBUILD_VERBOSE'})) { - $verbose = "$ENV{'KBUILD_VERBOSE'}"; -} - -if (defined($ENV{'KDOC_WERROR'})) { - $Werror = "$ENV{'KDOC_WERROR'}"; -} - -if (defined($ENV{'KCFLAGS'})) { - my $kcflags = "$ENV{'KCFLAGS'}"; - - if ($kcflags =~ /Werror/) { - $Werror = 1; - } -} - -# Generated docbook code is inserted in a template at a point where -# docbook v3.1 requires a non-zero sequence of RefEntry's; see: -# https://www.oasis-open.org/docbook/documentation/reference/html/refentry.html -# We keep track of number of generated entries and generate a dummy -# if needs be to ensure the expanded template can be postprocessed -# into html. -my $section_counter = 0; - -my $lineprefix=""; - -# Parser states -use constant { - STATE_NORMAL => 0, # normal code - STATE_NAME => 1, # looking for function name - STATE_BODY_MAYBE => 2, # body - or maybe more description - STATE_BODY => 3, # the body of the comment - STATE_BODY_WITH_BLANK_LINE => 4, # the body, which has a blank line - STATE_PROTO => 5, # scanning prototype - STATE_DOCBLOCK => 6, # documentation block - STATE_INLINE => 7, # gathering doc outside main block -}; -my $state; -my $in_doc_sect; -my $leading_space; - -# Inline documentation state -use constant { - STATE_INLINE_NA => 0, # not applicable ($state != STATE_INLINE) - STATE_INLINE_NAME => 1, # looking for member name (@foo:) - STATE_INLINE_TEXT => 2, # looking for member documentation - STATE_INLINE_END => 3, # done - STATE_INLINE_ERROR => 4, # error - Comment without header was found. - # Spit a warning as it's not - # proper kernel-doc and ignore the rest. -}; -my $inline_doc_state; - -#declaration types: can be -# 'function', 'struct', 'union', 'enum', 'typedef' -my $decl_type; - -my $doc_start = '^/\*\*\s*$'; # Allow whitespace at end of comment start. -my $doc_end = '\*/'; -my $doc_com = '\s*\*\s*'; -my $doc_com_body = '\s*\* ?'; -my $doc_decl = $doc_com . '(\w+)'; -# @params and a strictly limited set of supported section names -my $doc_sect = $doc_com . - '\s*(\@[.\w]+|\@\.\.\.|description|context|returns?|notes?|examples?)\s*:(.*)'; -my $doc_content = $doc_com_body . '(.*)'; -my $doc_block = $doc_com . 'DOC:\s*(.*)?'; -my $doc_inline_start = '^\s*/\*\*\s*$'; -my $doc_inline_sect = '\s*\*\s*(@\s*[\w][\w\.]*\s*):(.*)'; -my $doc_inline_end = '^\s*\*/\s*$'; -my $doc_inline_oneline = '^\s*/\*\*\s*(@[\w\s]+):\s*(.*)\s*\*/\s*$'; -my $export_symbol = '^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*;'; - -my %parameterdescs; -my %parameterdesc_start_lines; -my @parameterlist; -my %sections; -my @sectionlist; -my %section_start_lines; -my $sectcheck; -my $struct_actual; - -my $contents = ""; -my $new_start_line = 0; - -# the canonical section names. see also $doc_sect above. -my $section_default = "Description"; # default section -my $section_intro = "Introduction"; -my $section = $section_default; -my $section_context = "Context"; -my $section_return = "Return"; - -my $undescribed = "-- undescribed --"; - -reset_state(); - -while ($ARGV[0] =~ m/^--?(.*)/) { - my $cmd = $1; - shift @ARGV; - if ($cmd eq "man") { - $output_mode = "man"; - @highlights = @highlights_man; - $blankline = $blankline_man; - } elsif ($cmd eq "rst") { - $output_mode = "rst"; - @highlights = @highlights_rst; - $blankline = $blankline_rst; - } elsif ($cmd eq "none") { - $output_mode = "none"; - } elsif ($cmd eq "module") { # not needed for XML, inherits from calling document - $modulename = shift @ARGV; - } elsif ($cmd eq "function") { # to only output specific functions - $output_selection = OUTPUT_INCLUDE; - $function = shift @ARGV; - $function_table{$function} = 1; - } elsif ($cmd eq "nosymbol") { # Exclude specific symbols - my $symbol = shift @ARGV; - $nosymbol_table{$symbol} = 1; - } elsif ($cmd eq "export") { # only exported symbols - $output_selection = OUTPUT_EXPORTED; - %function_table = (); - } elsif ($cmd eq "internal") { # only non-exported symbols - $output_selection = OUTPUT_INTERNAL; - %function_table = (); - } elsif ($cmd eq "export-file") { - my $file = shift @ARGV; - push(@export_file_list, $file); - } elsif ($cmd eq "v") { - $verbose = 1; - } elsif ($cmd eq "Werror") { - $Werror = 1; - } elsif (($cmd eq "h") || ($cmd eq "help")) { - usage(); - } elsif ($cmd eq 'no-doc-sections') { - $no_doc_sections = 1; - } elsif ($cmd eq 'enable-lineno') { - $enable_lineno = 1; - } elsif ($cmd eq 'show-not-found') { - $show_not_found = 1; # A no-op but don't fail - } elsif ($cmd eq "sphinx-version") { - my $ver_string = shift @ARGV; - if ($ver_string =~ m/^(\d+)(\.\d+)?(\.\d+)?/) { - $sphinx_major = $1; - if (defined($2)) { - $sphinx_minor = substr($2,1); - } else { - $sphinx_minor = 0; - } - if (defined($3)) { - $sphinx_patch = substr($3,1) - } else { - $sphinx_patch = 0; - } - } else { - die "Sphinx version should either major.minor or major.minor.patch format\n"; - } - } else { - # Unknown argument - usage(); - } -} - -# continue execution near EOF; - -# The C domain dialect changed on Sphinx 3. So, we need to check the -# version in order to produce the right tags. -sub findprog($) -{ - foreach(split(/:/, $ENV{PATH})) { - return "$_/$_[0]" if(-x "$_/$_[0]"); - } -} - -sub get_sphinx_version() -{ - my $ver; - - my $cmd = "sphinx-build"; - if (!findprog($cmd)) { - my $cmd = "sphinx-build3"; - if (!findprog($cmd)) { - $sphinx_major = 1; - $sphinx_minor = 2; - $sphinx_patch = 0; - printf STDERR "Warning: Sphinx version not found. Using default (Sphinx version %d.%d.%d)\n", - $sphinx_major, $sphinx_minor, $sphinx_patch; - return; - } - } - - open IN, "$cmd --version 2>&1 |"; - while (<IN>) { - if (m/^\s*sphinx-build\s+([\d]+)\.([\d\.]+)(\+\/[\da-f]+)?$/) { - $sphinx_major = $1; - $sphinx_minor = $2; - $sphinx_patch = $3; - last; - } - # Sphinx 1.2.x uses a different format - if (m/^\s*Sphinx.*\s+([\d]+)\.([\d\.]+)$/) { - $sphinx_major = $1; - $sphinx_minor = $2; - $sphinx_patch = $3; - last; - } - } - close IN; -} - -# get kernel version from env -sub get_kernel_version() { - my $version = 'unknown kernel version'; - - if (defined($ENV{'KERNELVERSION'})) { - $version = $ENV{'KERNELVERSION'}; - } - return $version; -} - -# -sub print_lineno { - my $lineno = shift; - if ($enable_lineno && defined($lineno)) { - print "#define LINENO " . $lineno . "\n"; - } -} -## -# dumps section contents to arrays/hashes intended for that purpose. -# -sub dump_section { - my $file = shift; - my $name = shift; - my $contents = join "\n", @_; - - if ($name =~ m/$type_param/) { - $name = $1; - $parameterdescs{$name} = $contents; - $sectcheck = $sectcheck . $name . " "; - $parameterdesc_start_lines{$name} = $new_start_line; - $new_start_line = 0; - } elsif ($name eq "@\.\.\.") { - $name = "..."; - $parameterdescs{$name} = $contents; - $sectcheck = $sectcheck . $name . " "; - $parameterdesc_start_lines{$name} = $new_start_line; - $new_start_line = 0; - } else { - if (defined($sections{$name}) && ($sections{$name} ne "")) { - # Only warn on user specified duplicate section names. - if ($name ne $section_default) { - print STDERR "${file}:$.: warning: duplicate section name '$name'\n"; - ++$warnings; - } - $sections{$name} .= $contents; - } else { - $sections{$name} = $contents; - push @sectionlist, $name; - $section_start_lines{$name} = $new_start_line; - $new_start_line = 0; - } - } -} - -## -# dump DOC: section after checking that it should go out -# -sub dump_doc_section { - my $file = shift; - my $name = shift; - my $contents = join "\n", @_; - - if ($no_doc_sections) { - return; - } - - return if (defined($nosymbol_table{$name})); - - if (($output_selection == OUTPUT_ALL) || - (($output_selection == OUTPUT_INCLUDE) && - defined($function_table{$name}))) - { - dump_section($file, $name, $contents); - output_blockhead({'sectionlist' => \@sectionlist, - 'sections' => \%sections, - 'module' => $modulename, - 'content-only' => ($output_selection != OUTPUT_ALL), }); - } -} - -## -# output function -# -# parameterdescs, a hash. -# function => "function name" -# parameterlist => @list of parameters -# parameterdescs => %parameter descriptions -# sectionlist => @list of sections -# sections => %section descriptions -# - -sub output_highlight { - my $contents = join "\n",@_; - my $line; - -# DEBUG -# if (!defined $contents) { -# use Carp; -# confess "output_highlight got called with no args?\n"; -# } - -# print STDERR "contents b4:$contents\n"; - eval $dohighlight; - die $@ if $@; -# print STDERR "contents af:$contents\n"; - - foreach $line (split "\n", $contents) { - if (! $output_preformatted) { - $line =~ s/^\s*//; - } - if ($line eq ""){ - if (! $output_preformatted) { - print $lineprefix, $blankline; - } - } else { - if ($output_mode eq "man" && substr($line, 0, 1) eq ".") { - print "\\&$line"; - } else { - print $lineprefix, $line; - } - } - print "\n"; - } -} - -## -# output function in man -sub output_function_man(%) { - my %args = %{$_[0]}; - my ($parameter, $section); - my $count; - - print ".TH \"$args{'function'}\" 9 \"$args{'function'}\" \"$man_date\" \"Kernel Hacker's Manual\" LINUX\n"; - - print ".SH NAME\n"; - print $args{'function'} . " \\- " . $args{'purpose'} . "\n"; - - print ".SH SYNOPSIS\n"; - if ($args{'functiontype'} ne "") { - print ".B \"" . $args{'functiontype'} . "\" " . $args{'function'} . "\n"; - } else { - print ".B \"" . $args{'function'} . "\n"; - } - $count = 0; - my $parenth = "("; - my $post = ","; - foreach my $parameter (@{$args{'parameterlist'}}) { - if ($count == $#{$args{'parameterlist'}}) { - $post = ");"; - } - $type = $args{'parametertypes'}{$parameter}; - if ($type =~ m/([^\(]*\(\*)\s*\)\s*\(([^\)]*)\)/) { - # pointer-to-function - print ".BI \"" . $parenth . $1 . "\" " . " \") (" . $2 . ")" . $post . "\"\n"; - } else { - $type =~ s/([^\*])$/$1 /; - print ".BI \"" . $parenth . $type . "\" " . " \"" . $post . "\"\n"; - } - $count++; - $parenth = ""; - } - - print ".SH ARGUMENTS\n"; - foreach $parameter (@{$args{'parameterlist'}}) { - my $parameter_name = $parameter; - $parameter_name =~ s/\[.*//; - - print ".IP \"" . $parameter . "\" 12\n"; - output_highlight($args{'parameterdescs'}{$parameter_name}); - } - foreach $section (@{$args{'sectionlist'}}) { - print ".SH \"", uc $section, "\"\n"; - output_highlight($args{'sections'}{$section}); - } -} - -## -# output enum in man -sub output_enum_man(%) { - my %args = %{$_[0]}; - my ($parameter, $section); - my $count; - - print ".TH \"$args{'module'}\" 9 \"enum $args{'enum'}\" \"$man_date\" \"API Manual\" LINUX\n"; - - print ".SH NAME\n"; - print "enum " . $args{'enum'} . " \\- " . $args{'purpose'} . "\n"; - - print ".SH SYNOPSIS\n"; - print "enum " . $args{'enum'} . " {\n"; - $count = 0; - foreach my $parameter (@{$args{'parameterlist'}}) { - print ".br\n.BI \" $parameter\"\n"; - if ($count == $#{$args{'parameterlist'}}) { - print "\n};\n"; - last; - } - else { - print ", \n.br\n"; - } - $count++; - } - - print ".SH Constants\n"; - foreach $parameter (@{$args{'parameterlist'}}) { - my $parameter_name = $parameter; - $parameter_name =~ s/\[.*//; - - print ".IP \"" . $parameter . "\" 12\n"; - output_highlight($args{'parameterdescs'}{$parameter_name}); - } - foreach $section (@{$args{'sectionlist'}}) { - print ".SH \"$section\"\n"; - output_highlight($args{'sections'}{$section}); - } -} - -## -# output struct in man -sub output_struct_man(%) { - my %args = %{$_[0]}; - my ($parameter, $section); - - print ".TH \"$args{'module'}\" 9 \"" . $args{'type'} . " " . $args{'struct'} . "\" \"$man_date\" \"API Manual\" LINUX\n"; - - print ".SH NAME\n"; - print $args{'type'} . " " . $args{'struct'} . " \\- " . $args{'purpose'} . "\n"; - - my $declaration = $args{'definition'}; - $declaration =~ s/\t/ /g; - $declaration =~ s/\n/"\n.br\n.BI \"/g; - print ".SH SYNOPSIS\n"; - print $args{'type'} . " " . $args{'struct'} . " {\n.br\n"; - print ".BI \"$declaration\n};\n.br\n\n"; - - print ".SH Members\n"; - foreach $parameter (@{$args{'parameterlist'}}) { - ($parameter =~ /^#/) && next; - - my $parameter_name = $parameter; - $parameter_name =~ s/\[.*//; - - ($args{'parameterdescs'}{$parameter_name} ne $undescribed) || next; - print ".IP \"" . $parameter . "\" 12\n"; - output_highlight($args{'parameterdescs'}{$parameter_name}); - } - foreach $section (@{$args{'sectionlist'}}) { - print ".SH \"$section\"\n"; - output_highlight($args{'sections'}{$section}); - } -} - -## -# output typedef in man -sub output_typedef_man(%) { - my %args = %{$_[0]}; - my ($parameter, $section); - - print ".TH \"$args{'module'}\" 9 \"$args{'typedef'}\" \"$man_date\" \"API Manual\" LINUX\n"; - - print ".SH NAME\n"; - print "typedef " . $args{'typedef'} . " \\- " . $args{'purpose'} . "\n"; - - foreach $section (@{$args{'sectionlist'}}) { - print ".SH \"$section\"\n"; - output_highlight($args{'sections'}{$section}); - } -} - -sub output_blockhead_man(%) { - my %args = %{$_[0]}; - my ($parameter, $section); - my $count; - - print ".TH \"$args{'module'}\" 9 \"$args{'module'}\" \"$man_date\" \"API Manual\" LINUX\n"; - - foreach $section (@{$args{'sectionlist'}}) { - print ".SH \"$section\"\n"; - output_highlight($args{'sections'}{$section}); - } -} - -## -# output in restructured text -# - -# -# This could use some work; it's used to output the DOC: sections, and -# starts by putting out the name of the doc section itself, but that tends -# to duplicate a header already in the template file. -# -sub output_blockhead_rst(%) { - my %args = %{$_[0]}; - my ($parameter, $section); - - foreach $section (@{$args{'sectionlist'}}) { - next if (defined($nosymbol_table{$section})); - - if ($output_selection != OUTPUT_INCLUDE) { - print "**$section**\n\n"; - } - print_lineno($section_start_lines{$section}); - output_highlight_rst($args{'sections'}{$section}); - print "\n"; - } -} - -# -# Apply the RST highlights to a sub-block of text. -# -sub highlight_block($) { - # The dohighlight kludge requires the text be called $contents - my $contents = shift; - eval $dohighlight; - die $@ if $@; - return $contents; -} - -# -# Regexes used only here. -# -my $sphinx_literal = '^[^.].*::$'; -my $sphinx_cblock = '^\.\.\ +code-block::'; - -sub output_highlight_rst { - my $input = join "\n",@_; - my $output = ""; - my $line; - my $in_literal = 0; - my $litprefix; - my $block = ""; - - foreach $line (split "\n",$input) { - # - # If we're in a literal block, see if we should drop out - # of it. Otherwise pass the line straight through unmunged. - # - if ($in_literal) { - if (! ($line =~ /^\s*$/)) { - # - # If this is the first non-blank line in a literal - # block we need to figure out what the proper indent is. - # - if ($litprefix eq "") { - $line =~ /^(\s*)/; - $litprefix = '^' . $1; - $output .= $line . "\n"; - } elsif (! ($line =~ /$litprefix/)) { - $in_literal = 0; - } else { - $output .= $line . "\n"; - } - } else { - $output .= $line . "\n"; - } - } - # - # Not in a literal block (or just dropped out) - # - if (! $in_literal) { - $block .= $line . "\n"; - if (($line =~ /$sphinx_literal/) || ($line =~ /$sphinx_cblock/)) { - $in_literal = 1; - $litprefix = ""; - $output .= highlight_block($block); - $block = "" - } - } - } - - if ($block) { - $output .= highlight_block($block); - } - foreach $line (split "\n", $output) { - print $lineprefix . $line . "\n"; - } -} - -sub output_function_rst(%) { - my %args = %{$_[0]}; - my ($parameter, $section); - my $oldprefix = $lineprefix; - my $start = ""; - my $is_macro = 0; - - if ($sphinx_major < 3) { - if ($args{'typedef'}) { - print ".. c:type:: ". $args{'function'} . "\n\n"; - print_lineno($declaration_start_line); - print " **Typedef**: "; - $lineprefix = ""; - output_highlight_rst($args{'purpose'}); - $start = "\n\n**Syntax**\n\n ``"; - $is_macro = 1; - } else { - print ".. c:function:: "; - } - } else { - if ($args{'typedef'} || $args{'functiontype'} eq "") { - $is_macro = 1; - print ".. c:macro:: ". $args{'function'} . "\n\n"; - } else { - print ".. c:function:: "; - } - - if ($args{'typedef'}) { - print_lineno($declaration_start_line); - print " **Typedef**: "; - $lineprefix = ""; - output_highlight_rst($args{'purpose'}); - $start = "\n\n**Syntax**\n\n ``"; - } else { - print "``" if ($is_macro); - } - } - if ($args{'functiontype'} ne "") { - $start .= $args{'functiontype'} . " " . $args{'function'} . " ("; - } else { - $start .= $args{'function'} . " ("; - } - print $start; - - my $count = 0; - foreach my $parameter (@{$args{'parameterlist'}}) { - if ($count ne 0) { - print ", "; - } - $count++; - $type = $args{'parametertypes'}{$parameter}; - - if ($type =~ m/([^\(]*\(\*)\s*\)\s*\(([^\)]*)\)/) { - # pointer-to-function - print $1 . $parameter . ") (" . $2 . ")"; - } else { - print $type; - } - } - if ($is_macro) { - print ")``\n\n"; - } else { - print ")\n\n"; - } - if (!$args{'typedef'}) { - print_lineno($declaration_start_line); - $lineprefix = " "; - output_highlight_rst($args{'purpose'}); - print "\n"; - } - - print "**Parameters**\n\n"; - $lineprefix = " "; - foreach $parameter (@{$args{'parameterlist'}}) { - my $parameter_name = $parameter; - $parameter_name =~ s/\[.*//; - $type = $args{'parametertypes'}{$parameter}; - - if ($type ne "") { - print "``$type``\n"; - } else { - print "``$parameter``\n"; - } - - print_lineno($parameterdesc_start_lines{$parameter_name}); - - if (defined($args{'parameterdescs'}{$parameter_name}) && - $args{'parameterdescs'}{$parameter_name} ne $undescribed) { - output_highlight_rst($args{'parameterdescs'}{$parameter_name}); - } else { - print " *undescribed*\n"; - } - print "\n"; - } - - $lineprefix = $oldprefix; - output_section_rst(@_); -} - -sub output_section_rst(%) { - my %args = %{$_[0]}; - my $section; - my $oldprefix = $lineprefix; - $lineprefix = ""; - - foreach $section (@{$args{'sectionlist'}}) { - print "**$section**\n\n"; - print_lineno($section_start_lines{$section}); - output_highlight_rst($args{'sections'}{$section}); - print "\n"; - } - print "\n"; - $lineprefix = $oldprefix; -} - -sub output_enum_rst(%) { - my %args = %{$_[0]}; - my ($parameter); - my $oldprefix = $lineprefix; - my $count; - - if ($sphinx_major < 3) { - my $name = "enum " . $args{'enum'}; - print "\n\n.. c:type:: " . $name . "\n\n"; - } else { - my $name = $args{'enum'}; - print "\n\n.. c:enum:: " . $name . "\n\n"; - } - print_lineno($declaration_start_line); - $lineprefix = " "; - output_highlight_rst($args{'purpose'}); - print "\n"; - - print "**Constants**\n\n"; - $lineprefix = " "; - foreach $parameter (@{$args{'parameterlist'}}) { - print "``$parameter``\n"; - if ($args{'parameterdescs'}{$parameter} ne $undescribed) { - output_highlight_rst($args{'parameterdescs'}{$parameter}); - } else { - print " *undescribed*\n"; - } - print "\n"; - } - - $lineprefix = $oldprefix; - output_section_rst(@_); -} - -sub output_typedef_rst(%) { - my %args = %{$_[0]}; - my ($parameter); - my $oldprefix = $lineprefix; - my $name; - - if ($sphinx_major < 3) { - $name = "typedef " . $args{'typedef'}; - } else { - $name = $args{'typedef'}; - } - print "\n\n.. c:type:: " . $name . "\n\n"; - print_lineno($declaration_start_line); - $lineprefix = " "; - output_highlight_rst($args{'purpose'}); - print "\n"; - - $lineprefix = $oldprefix; - output_section_rst(@_); -} - -sub output_struct_rst(%) { - my %args = %{$_[0]}; - my ($parameter); - my $oldprefix = $lineprefix; - - if ($sphinx_major < 3) { - my $name = $args{'type'} . " " . $args{'struct'}; - print "\n\n.. c:type:: " . $name . "\n\n"; - } else { - my $name = $args{'struct'}; - if ($args{'type'} eq 'union') { - print "\n\n.. c:union:: " . $name . "\n\n"; - } else { - print "\n\n.. c:struct:: " . $name . "\n\n"; - } - } - print_lineno($declaration_start_line); - $lineprefix = " "; - output_highlight_rst($args{'purpose'}); - print "\n"; - - print "**Definition**\n\n"; - print "::\n\n"; - my $declaration = $args{'definition'}; - $declaration =~ s/\t/ /g; - print " " . $args{'type'} . " " . $args{'struct'} . " {\n$declaration };\n\n"; - - print "**Members**\n\n"; - $lineprefix = " "; - foreach $parameter (@{$args{'parameterlist'}}) { - ($parameter =~ /^#/) && next; - - my $parameter_name = $parameter; - $parameter_name =~ s/\[.*//; - - ($args{'parameterdescs'}{$parameter_name} ne $undescribed) || next; - $type = $args{'parametertypes'}{$parameter}; - print_lineno($parameterdesc_start_lines{$parameter_name}); - print "``" . $parameter . "``\n"; - output_highlight_rst($args{'parameterdescs'}{$parameter_name}); - print "\n"; - } - print "\n"; - - $lineprefix = $oldprefix; - output_section_rst(@_); -} - -## none mode output functions - -sub output_function_none(%) { -} - -sub output_enum_none(%) { -} - -sub output_typedef_none(%) { -} - -sub output_struct_none(%) { -} - -sub output_blockhead_none(%) { -} - -## -# generic output function for all types (function, struct/union, typedef, enum); -# calls the generated, variable output_ function name based on -# functype and output_mode -sub output_declaration { - no strict 'refs'; - my $name = shift; - my $functype = shift; - my $func = "output_${functype}_$output_mode"; - - return if (defined($nosymbol_table{$name})); - - if (($output_selection == OUTPUT_ALL) || - (($output_selection == OUTPUT_INCLUDE || - $output_selection == OUTPUT_EXPORTED) && - defined($function_table{$name})) || - ($output_selection == OUTPUT_INTERNAL && - !($functype eq "function" && defined($function_table{$name})))) - { - &$func(@_); - $section_counter++; - } -} - -## -# generic output function - calls the right one based on current output mode. -sub output_blockhead { - no strict 'refs'; - my $func = "output_blockhead_" . $output_mode; - &$func(@_); - $section_counter++; -} - -## -# takes a declaration (struct, union, enum, typedef) and -# invokes the right handler. NOT called for functions. -sub dump_declaration($$) { - no strict 'refs'; - my ($prototype, $file) = @_; - my $func = "dump_" . $decl_type; - &$func(@_); -} - -sub dump_union($$) { - dump_struct(@_); -} - -sub dump_struct($$) { - my $x = shift; - my $file = shift; - - if ($x =~ /(struct|union)\s+(\w+)\s*\{(.*)\}(\s*(__packed|__aligned|____cacheline_aligned_in_smp|____cacheline_aligned|__attribute__\s*\(\([a-z0-9,_\s\(\)]*\)\)))*/) { - my $decl_type = $1; - $declaration_name = $2; - my $members = $3; - - # ignore members marked private: - $members =~ s/\/\*\s*private:.*?\/\*\s*public:.*?\*\///gosi; - $members =~ s/\/\*\s*private:.*//gosi; - # strip comments: - $members =~ s/\/\*.*?\*\///gos; - # strip attributes - $members =~ s/\s*__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)/ /gi; - $members =~ s/\s*__aligned\s*\([^;]*\)/ /gos; - $members =~ s/\s*__packed\s*/ /gos; - $members =~ s/\s*CRYPTO_MINALIGN_ATTR/ /gos; - $members =~ s/\s*____cacheline_aligned_in_smp/ /gos; - $members =~ s/\s*____cacheline_aligned/ /gos; - - # replace DECLARE_BITMAP - $members =~ s/__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)/DECLARE_BITMAP($1, __ETHTOOL_LINK_MODE_MASK_NBITS)/gos; - $members =~ s/DECLARE_BITMAP\s*\(([^,)]+),\s*([^,)]+)\)/unsigned long $1\[BITS_TO_LONGS($2)\]/gos; - # replace DECLARE_HASHTABLE - $members =~ s/DECLARE_HASHTABLE\s*\(([^,)]+),\s*([^,)]+)\)/unsigned long $1\[1 << (($2) - 1)\]/gos; - # replace DECLARE_KFIFO - $members =~ s/DECLARE_KFIFO\s*\(([^,)]+),\s*([^,)]+),\s*([^,)]+)\)/$2 \*$1/gos; - # replace DECLARE_KFIFO_PTR - $members =~ s/DECLARE_KFIFO_PTR\s*\(([^,)]+),\s*([^,)]+)\)/$2 \*$1/gos; - - my $declaration = $members; - - # Split nested struct/union elements as newer ones - while ($members =~ m/(struct|union)([^\{\};]+)\{([^\{\}]*)\}([^\{\}\;]*)\;/) { - my $newmember; - my $maintype = $1; - my $ids = $4; - my $content = $3; - foreach my $id(split /,/, $ids) { - $newmember .= "$maintype $id; "; - - $id =~ s/[:\[].*//; - $id =~ s/^\s*\**(\S+)\s*/$1/; - foreach my $arg (split /;/, $content) { - next if ($arg =~ m/^\s*$/); - if ($arg =~ m/^([^\(]+\(\*?\s*)([\w\.]*)(\s*\).*)/) { - # pointer-to-function - my $type = $1; - my $name = $2; - my $extra = $3; - next if (!$name); - if ($id =~ m/^\s*$/) { - # anonymous struct/union - $newmember .= "$type$name$extra; "; - } else { - $newmember .= "$type$id.$name$extra; "; - } - } else { - my $type; - my $names; - $arg =~ s/^\s+//; - $arg =~ s/\s+$//; - # Handle bitmaps - $arg =~ s/:\s*\d+\s*//g; - # Handle arrays - $arg =~ s/\[.*\]//g; - # The type may have multiple words, - # and multiple IDs can be defined, like: - # const struct foo, *bar, foobar - # So, we remove spaces when parsing the - # names, in order to match just names - # and commas for the names - $arg =~ s/\s*,\s*/,/g; - if ($arg =~ m/(.*)\s+([\S+,]+)/) { - $type = $1; - $names = $2; - } else { - $newmember .= "$arg; "; - next; - } - foreach my $name (split /,/, $names) { - $name =~ s/^\s*\**(\S+)\s*/$1/; - next if (($name =~ m/^\s*$/)); - if ($id =~ m/^\s*$/) { - # anonymous struct/union - $newmember .= "$type $name; "; - } else { - $newmember .= "$type $id.$name; "; - } - } - } - } - } - $members =~ s/(struct|union)([^\{\};]+)\{([^\{\}]*)\}([^\{\}\;]*)\;/$newmember/; - } - - # Ignore other nested elements, like enums - $members =~ s/(\{[^\{\}]*\})//g; - - create_parameterlist($members, ';', $file, $declaration_name); - check_sections($file, $declaration_name, $decl_type, $sectcheck, $struct_actual); - - # Adjust declaration for better display - $declaration =~ s/([\{;])/$1\n/g; - $declaration =~ s/\}\s+;/};/g; - # Better handle inlined enums - do {} while ($declaration =~ s/(enum\s+\{[^\}]+),([^\n])/$1,\n$2/); - - my @def_args = split /\n/, $declaration; - my $level = 1; - $declaration = ""; - foreach my $clause (@def_args) { - $clause =~ s/^\s+//; - $clause =~ s/\s+$//; - $clause =~ s/\s+/ /; - next if (!$clause); - $level-- if ($clause =~ m/(\})/ && $level > 1); - if (!($clause =~ m/^\s*#/)) { - $declaration .= "\t" x $level; - } - $declaration .= "\t" . $clause . "\n"; - $level++ if ($clause =~ m/(\{)/ && !($clause =~m/\}/)); - } - output_declaration($declaration_name, - 'struct', - {'struct' => $declaration_name, - 'module' => $modulename, - 'definition' => $declaration, - 'parameterlist' => \@parameterlist, - 'parameterdescs' => \%parameterdescs, - 'parametertypes' => \%parametertypes, - 'sectionlist' => \@sectionlist, - 'sections' => \%sections, - 'purpose' => $declaration_purpose, - 'type' => $decl_type - }); - } - else { - print STDERR "${file}:$.: error: Cannot parse struct or union!\n"; - ++$errors; - } -} - - -sub show_warnings($$) { - my $functype = shift; - my $name = shift; - - return 0 if (defined($nosymbol_table{$name})); - - return 1 if ($output_selection == OUTPUT_ALL); - - if ($output_selection == OUTPUT_EXPORTED) { - if (defined($function_table{$name})) { - return 1; - } else { - return 0; - } - } - if ($output_selection == OUTPUT_INTERNAL) { - if (!($functype eq "function" && defined($function_table{$name}))) { - return 1; - } else { - return 0; - } - } - if ($output_selection == OUTPUT_INCLUDE) { - if (defined($function_table{$name})) { - return 1; - } else { - return 0; - } - } - die("Please add the new output type at show_warnings()"); -} - -sub dump_enum($$) { - my $x = shift; - my $file = shift; - my $members; - - - $x =~ s@/\*.*?\*/@@gos; # strip comments. - # strip #define macros inside enums - $x =~ s@#\s*((define|ifdef)\s+|endif)[^;]*;@@gos; - - if ($x =~ /typedef\s+enum\s*\{(.*)\}\s*(\w*)\s*;/) { - $declaration_name = $2; - $members = $1; - } elsif ($x =~ /enum\s+(\w*)\s*\{(.*)\}/) { - $declaration_name = $1; - $members = $2; - } - - if ($declaration_name) { - my %_members; - - $members =~ s/\s+$//; - - foreach my $arg (split ',', $members) { - $arg =~ s/^\s*(\w+).*/$1/; - push @parameterlist, $arg; - if (!$parameterdescs{$arg}) { - $parameterdescs{$arg} = $undescribed; - if (show_warnings("enum", $declaration_name)) { - print STDERR "${file}:$.: warning: Enum value '$arg' not described in enum '$declaration_name'\n"; - } - } - $_members{$arg} = 1; - } - - while (my ($k, $v) = each %parameterdescs) { - if (!exists($_members{$k})) { - if (show_warnings("enum", $declaration_name)) { - print STDERR "${file}:$.: warning: Excess enum value '$k' description in '$declaration_name'\n"; - } - } - } - - output_declaration($declaration_name, - 'enum', - {'enum' => $declaration_name, - 'module' => $modulename, - 'parameterlist' => \@parameterlist, - 'parameterdescs' => \%parameterdescs, - 'sectionlist' => \@sectionlist, - 'sections' => \%sections, - 'purpose' => $declaration_purpose - }); - } else { - print STDERR "${file}:$.: error: Cannot parse enum!\n"; - ++$errors; - } -} - -my $typedef_type = qr { ((?:\s+[\w\*]+){1,8})\s* }x; -my $typedef_ident = qr { \*?\s*(\w\S+)\s* }x; -my $typedef_args = qr { \s*\((.*)\); }x; - -my $typedef1 = qr { typedef$typedef_type\($typedef_ident\)$typedef_args }x; -my $typedef2 = qr { typedef$typedef_type$typedef_ident$typedef_args }x; - -sub dump_typedef($$) { - my $x = shift; - my $file = shift; - - $x =~ s@/\*.*?\*/@@gos; # strip comments. - - # Parse function typedef prototypes - if ($x =~ $typedef1 || $x =~ $typedef2) { - $return_type = $1; - $declaration_name = $2; - my $args = $3; - $return_type =~ s/^\s+//; - - create_parameterlist($args, ',', $file, $declaration_name); - - output_declaration($declaration_name, - 'function', - {'function' => $declaration_name, - 'typedef' => 1, - 'module' => $modulename, - 'functiontype' => $return_type, - 'parameterlist' => \@parameterlist, - 'parameterdescs' => \%parameterdescs, - 'parametertypes' => \%parametertypes, - 'sectionlist' => \@sectionlist, - 'sections' => \%sections, - 'purpose' => $declaration_purpose - }); - return; - } - - while (($x =~ /\(*.\)\s*;$/) || ($x =~ /\[*.\]\s*;$/)) { - $x =~ s/\(*.\)\s*;$/;/; - $x =~ s/\[*.\]\s*;$/;/; - } - - if ($x =~ /typedef.*\s+(\w+)\s*;/) { - $declaration_name = $1; - - output_declaration($declaration_name, - 'typedef', - {'typedef' => $declaration_name, - 'module' => $modulename, - 'sectionlist' => \@sectionlist, - 'sections' => \%sections, - 'purpose' => $declaration_purpose - }); - } - else { - print STDERR "${file}:$.: error: Cannot parse typedef!\n"; - ++$errors; - } -} - -sub save_struct_actual($) { - my $actual = shift; - - # strip all spaces from the actual param so that it looks like one string item - $actual =~ s/\s*//g; - $struct_actual = $struct_actual . $actual . " "; -} - -sub create_parameterlist($$$$) { - my $args = shift; - my $splitter = shift; - my $file = shift; - my $declaration_name = shift; - my $type; - my $param; - - # temporarily replace commas inside function pointer definition - while ($args =~ /(\([^\),]+),/) { - $args =~ s/(\([^\),]+),/$1#/g; - } - - foreach my $arg (split($splitter, $args)) { - # strip comments - $arg =~ s/\/\*.*\*\///; - # strip leading/trailing spaces - $arg =~ s/^\s*//; - $arg =~ s/\s*$//; - $arg =~ s/\s+/ /; - - if ($arg =~ /^#/) { - # Treat preprocessor directive as a typeless variable just to fill - # corresponding data structures "correctly". Catch it later in - # output_* subs. - push_parameter($arg, "", "", $file); - } elsif ($arg =~ m/\(.+\)\s*\(/) { - # pointer-to-function - $arg =~ tr/#/,/; - $arg =~ m/[^\(]+\(\*?\s*([\w\.]*)\s*\)/; - $param = $1; - $type = $arg; - $type =~ s/([^\(]+\(\*?)\s*$param/$1/; - save_struct_actual($param); - push_parameter($param, $type, $arg, $file, $declaration_name); - } elsif ($arg) { - $arg =~ s/\s*:\s*/:/g; - $arg =~ s/\s*\[/\[/g; - - my @args = split('\s*,\s*', $arg); - if ($args[0] =~ m/\*/) { - $args[0] =~ s/(\*+)\s*/ $1/; - } - - my @first_arg; - if ($args[0] =~ /^(.*\s+)(.*?\[.*\].*)$/) { - shift @args; - push(@first_arg, split('\s+', $1)); - push(@first_arg, $2); - } else { - @first_arg = split('\s+', shift @args); - } - - unshift(@args, pop @first_arg); - $type = join " ", @first_arg; - - foreach $param (@args) { - if ($param =~ m/^(\*+)\s*(.*)/) { - save_struct_actual($2); - - push_parameter($2, "$type $1", $arg, $file, $declaration_name); - } - elsif ($param =~ m/(.*?):(\d+)/) { - if ($type ne "") { # skip unnamed bit-fields - save_struct_actual($1); - push_parameter($1, "$type:$2", $arg, $file, $declaration_name) - } - } - else { - save_struct_actual($param); - push_parameter($param, $type, $arg, $file, $declaration_name); - } - } - } - } -} - -sub push_parameter($$$$$) { - my $param = shift; - my $type = shift; - my $org_arg = shift; - my $file = shift; - my $declaration_name = shift; - - if (($anon_struct_union == 1) && ($type eq "") && - ($param eq "}")) { - return; # ignore the ending }; from anon. struct/union - } - - $anon_struct_union = 0; - $param =~ s/[\[\)].*//; - - if ($type eq "" && $param =~ /\.\.\.$/) - { - if (!$param =~ /\w\.\.\.$/) { - # handles unnamed variable parameters - $param = "..."; - } - elsif ($param =~ /\w\.\.\.$/) { - # for named variable parameters of the form `x...`, remove the dots - $param =~ s/\.\.\.$//; - } - if (!defined $parameterdescs{$param} || $parameterdescs{$param} eq "") { - $parameterdescs{$param} = "variable arguments"; - } - } - elsif ($type eq "" && ($param eq "" or $param eq "void")) - { - $param="void"; - $parameterdescs{void} = "no arguments"; - } - elsif ($type eq "" && ($param eq "struct" or $param eq "union")) - # handle unnamed (anonymous) union or struct: - { - $type = $param; - $param = "{unnamed_" . $param . "}"; - $parameterdescs{$param} = "anonymous\n"; - $anon_struct_union = 1; - } - - # warn if parameter has no description - # (but ignore ones starting with # as these are not parameters - # but inline preprocessor statements); - # Note: It will also ignore void params and unnamed structs/unions - if (!defined $parameterdescs{$param} && $param !~ /^#/) { - $parameterdescs{$param} = $undescribed; - - if (show_warnings($type, $declaration_name) && $param !~ /\./) { - print STDERR - "${file}:$.: warning: Function parameter or member '$param' not described in '$declaration_name'\n"; - ++$warnings; - } - } - - # strip spaces from $param so that it is one continuous string - # on @parameterlist; - # this fixes a problem where check_sections() cannot find - # a parameter like "addr[6 + 2]" because it actually appears - # as "addr[6", "+", "2]" on the parameter list; - # but it's better to maintain the param string unchanged for output, - # so just weaken the string compare in check_sections() to ignore - # "[blah" in a parameter string; - ###$param =~ s/\s*//g; - push @parameterlist, $param; - $org_arg =~ s/\s\s+/ /g; - $parametertypes{$param} = $org_arg; -} - -sub check_sections($$$$$) { - my ($file, $decl_name, $decl_type, $sectcheck, $prmscheck) = @_; - my @sects = split ' ', $sectcheck; - my @prms = split ' ', $prmscheck; - my $err; - my ($px, $sx); - my $prm_clean; # strip trailing "[array size]" and/or beginning "*" - - foreach $sx (0 .. $#sects) { - $err = 1; - foreach $px (0 .. $#prms) { - $prm_clean = $prms[$px]; - $prm_clean =~ s/\[.*\]//; - $prm_clean =~ s/__attribute__\s*\(\([a-z,_\*\s\(\)]*\)\)//i; - # ignore array size in a parameter string; - # however, the original param string may contain - # spaces, e.g.: addr[6 + 2] - # and this appears in @prms as "addr[6" since the - # parameter list is split at spaces; - # hence just ignore "[..." for the sections check; - $prm_clean =~ s/\[.*//; - - ##$prm_clean =~ s/^\**//; - if ($prm_clean eq $sects[$sx]) { - $err = 0; - last; - } - } - if ($err) { - if ($decl_type eq "function") { - print STDERR "${file}:$.: warning: " . - "Excess function parameter " . - "'$sects[$sx]' " . - "description in '$decl_name'\n"; - ++$warnings; - } - } - } -} - -## -# Checks the section describing the return value of a function. -sub check_return_section { - my $file = shift; - my $declaration_name = shift; - my $return_type = shift; - - # Ignore an empty return type (It's a macro) - # Ignore functions with a "void" return type. (But don't ignore "void *") - if (($return_type eq "") || ($return_type =~ /void\s*\w*\s*$/)) { - return; - } - - if (!defined($sections{$section_return}) || - $sections{$section_return} eq "") { - print STDERR "${file}:$.: warning: " . - "No description found for return value of " . - "'$declaration_name'\n"; - ++$warnings; - } -} - -## -# takes a function prototype and the name of the current file being -# processed and spits out all the details stored in the global -# arrays/hashes. -sub dump_function($$) { - my $prototype = shift; - my $file = shift; - my $noret = 0; - - print_lineno($new_start_line); - - $prototype =~ s/^static +//; - $prototype =~ s/^extern +//; - $prototype =~ s/^asmlinkage +//; - $prototype =~ s/^inline +//; - $prototype =~ s/^__inline__ +//; - $prototype =~ s/^__inline +//; - $prototype =~ s/^__always_inline +//; - $prototype =~ s/^noinline +//; - $prototype =~ s/__init +//; - $prototype =~ s/__init_or_module +//; - $prototype =~ s/__meminit +//; - $prototype =~ s/__must_check +//; - $prototype =~ s/__weak +//; - $prototype =~ s/__sched +//; - $prototype =~ s/__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +//; - my $define = $prototype =~ s/^#\s*define\s+//; #ak added - $prototype =~ s/__attribute__\s*\(\( - (?: - [\w\s]++ # attribute name - (?:\([^)]*+\))? # attribute arguments - \s*+,? # optional comma at the end - )+ - \)\)\s+//x; - - # Strip QEMU specific compiler annotations - $prototype =~ s/QEMU_[A-Z_]+ +//; - - # Yes, this truly is vile. We are looking for: - # 1. Return type (may be nothing if we're looking at a macro) - # 2. Function name - # 3. Function parameters. - # - # All the while we have to watch out for function pointer parameters - # (which IIRC is what the two sections are for), C types (these - # regexps don't even start to express all the possibilities), and - # so on. - # - # If you mess with these regexps, it's a good idea to check that - # the following functions' documentation still comes out right: - # - parport_register_device (function pointer parameters) - # - atomic_set (macro) - # - pci_match_device, __copy_to_user (long return type) - - if ($define && $prototype =~ m/^()([a-zA-Z0-9_~:]+)\s+/) { - # This is an object-like macro, it has no return type and no parameter - # list. - # Function-like macros are not allowed to have spaces between - # declaration_name and opening parenthesis (notice the \s+). - $return_type = $1; - $declaration_name = $2; - $noret = 1; - } elsif ($prototype =~ m/^()([a-zA-Z0-9_~:]+)\s*\(([^\(]*)\)/ || - $prototype =~ m/^(\w+)\s+([a-zA-Z0-9_~:]+)\s*\(([^\(]*)\)/ || - $prototype =~ m/^(\w+\s*\*+)\s*([a-zA-Z0-9_~:]+)\s*\(([^\(]*)\)/ || - $prototype =~ m/^(\w+\s+\w+)\s+([a-zA-Z0-9_~:]+)\s*\(([^\(]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s*\*+)\s*([a-zA-Z0-9_~:]+)\s*\(([^\(]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s+\w+)\s+([a-zA-Z0-9_~:]+)\s*\(([^\(]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s+\w+\s*\*+)\s*([a-zA-Z0-9_~:]+)\s*\(([^\(]*)\)/ || - $prototype =~ m/^()([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+)\s+([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+\s*\*+)\s*([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+\s+\w+)\s+([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s*\*+)\s*([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s+\w+)\s+([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s+\w+\s*\*+)\s*([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s+\w+\s+\w+)\s+([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s+\w+\s+\w+\s*\*+)\s*([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s*\*+\s*\w+\s*\*+\s*)\s*([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/) { - $return_type = $1; - $declaration_name = $2; - my $args = $3; - - create_parameterlist($args, ',', $file, $declaration_name); - } else { - print STDERR "${file}:$.: warning: cannot understand function prototype: '$prototype'\n"; - return; - } - - my $prms = join " ", @parameterlist; - check_sections($file, $declaration_name, "function", $sectcheck, $prms); - - # This check emits a lot of warnings at the moment, because many - # functions don't have a 'Return' doc section. So until the number - # of warnings goes sufficiently down, the check is only performed in - # verbose mode. - # TODO: always perform the check. - if ($verbose && !$noret) { - check_return_section($file, $declaration_name, $return_type); - } - - # The function parser can be called with a typedef parameter. - # Handle it. - if ($return_type =~ /typedef/) { - output_declaration($declaration_name, - 'function', - {'function' => $declaration_name, - 'typedef' => 1, - 'module' => $modulename, - 'functiontype' => $return_type, - 'parameterlist' => \@parameterlist, - 'parameterdescs' => \%parameterdescs, - 'parametertypes' => \%parametertypes, - 'sectionlist' => \@sectionlist, - 'sections' => \%sections, - 'purpose' => $declaration_purpose - }); - } else { - output_declaration($declaration_name, - 'function', - {'function' => $declaration_name, - 'module' => $modulename, - 'functiontype' => $return_type, - 'parameterlist' => \@parameterlist, - 'parameterdescs' => \%parameterdescs, - 'parametertypes' => \%parametertypes, - 'sectionlist' => \@sectionlist, - 'sections' => \%sections, - 'purpose' => $declaration_purpose - }); - } -} - -sub reset_state { - $function = ""; - %parameterdescs = (); - %parametertypes = (); - @parameterlist = (); - %sections = (); - @sectionlist = (); - $sectcheck = ""; - $struct_actual = ""; - $prototype = ""; - - $state = STATE_NORMAL; - $inline_doc_state = STATE_INLINE_NA; -} - -sub tracepoint_munge($) { - my $file = shift; - my $tracepointname = 0; - my $tracepointargs = 0; - - if ($prototype =~ m/TRACE_EVENT\((.*?),/) { - $tracepointname = $1; - } - if ($prototype =~ m/DEFINE_SINGLE_EVENT\((.*?),/) { - $tracepointname = $1; - } - if ($prototype =~ m/DEFINE_EVENT\((.*?),(.*?),/) { - $tracepointname = $2; - } - $tracepointname =~ s/^\s+//; #strip leading whitespace - if ($prototype =~ m/TP_PROTO\((.*?)\)/) { - $tracepointargs = $1; - } - if (($tracepointname eq 0) || ($tracepointargs eq 0)) { - print STDERR "${file}:$.: warning: Unrecognized tracepoint format: \n". - "$prototype\n"; - } else { - $prototype = "static inline void trace_$tracepointname($tracepointargs)"; - } -} - -sub syscall_munge() { - my $void = 0; - - $prototype =~ s@[\r\n]+@ @gos; # strip newlines/CR's -## if ($prototype =~ m/SYSCALL_DEFINE0\s*\(\s*(a-zA-Z0-9_)*\s*\)/) { - if ($prototype =~ m/SYSCALL_DEFINE0/) { - $void = 1; -## $prototype = "long sys_$1(void)"; - } - - $prototype =~ s/SYSCALL_DEFINE.*\(/long sys_/; # fix return type & func name - if ($prototype =~ m/long (sys_.*?),/) { - $prototype =~ s/,/\(/; - } elsif ($void) { - $prototype =~ s/\)/\(void\)/; - } - - # now delete all of the odd-number commas in $prototype - # so that arg types & arg names don't have a comma between them - my $count = 0; - my $len = length($prototype); - if ($void) { - $len = 0; # skip the for-loop - } - for (my $ix = 0; $ix < $len; $ix++) { - if (substr($prototype, $ix, 1) eq ',') { - $count++; - if ($count % 2 == 1) { - substr($prototype, $ix, 1) = ' '; - } - } - } -} - -sub process_proto_function($$) { - my $x = shift; - my $file = shift; - - $x =~ s@\/\/.*$@@gos; # strip C99-style comments to end of line - - if ($x =~ m#\s*/\*\s+MACDOC\s*#io || ($x =~ /^#/ && $x !~ /^#\s*define/)) { - # do nothing - } - elsif ($x =~ /([^\{]*)/) { - $prototype .= $1; - } - - if (($x =~ /\{/) || ($x =~ /\#\s*define/) || ($x =~ /;/)) { - $prototype =~ s@/\*.*?\*/@@gos; # strip comments. - $prototype =~ s@[\r\n]+@ @gos; # strip newlines/cr's. - $prototype =~ s@^\s+@@gos; # strip leading spaces - - # Handle prototypes for function pointers like: - # int (*pcs_config)(struct foo) - $prototype =~ s@^(\S+\s+)\(\s*\*(\S+)\)@$1$2@gos; - - if ($prototype =~ /SYSCALL_DEFINE/) { - syscall_munge(); - } - if ($prototype =~ /TRACE_EVENT/ || $prototype =~ /DEFINE_EVENT/ || - $prototype =~ /DEFINE_SINGLE_EVENT/) - { - tracepoint_munge($file); - } - dump_function($prototype, $file); - reset_state(); - } -} - -sub process_proto_type($$) { - my $x = shift; - my $file = shift; - - $x =~ s@[\r\n]+@ @gos; # strip newlines/cr's. - $x =~ s@^\s+@@gos; # strip leading spaces - $x =~ s@\s+$@@gos; # strip trailing spaces - $x =~ s@\/\/.*$@@gos; # strip C99-style comments to end of line - - if ($x =~ /^#/) { - # To distinguish preprocessor directive from regular declaration later. - $x .= ";"; - } - - while (1) { - if ( $x =~ /([^\{\};]*)([\{\};])(.*)/ ) { - if( length $prototype ) { - $prototype .= " " - } - $prototype .= $1 . $2; - ($2 eq '{') && $brcount++; - ($2 eq '}') && $brcount--; - if (($2 eq ';') && ($brcount == 0)) { - dump_declaration($prototype, $file); - reset_state(); - last; - } - $x = $3; - } else { - $prototype .= $x; - last; - } - } -} - - -sub map_filename($) { - my $file; - my ($orig_file) = @_; - - if (defined($ENV{'SRCTREE'})) { - $file = "$ENV{'SRCTREE'}" . "/" . $orig_file; - } else { - $file = $orig_file; - } - - if (defined($source_map{$file})) { - $file = $source_map{$file}; - } - - return $file; -} - -sub process_export_file($) { - my ($orig_file) = @_; - my $file = map_filename($orig_file); - - if (!open(IN,"<$file")) { - print STDERR "Error: Cannot open file $file\n"; - ++$errors; - return; - } - - while (<IN>) { - if (/$export_symbol/) { - next if (defined($nosymbol_table{$2})); - $function_table{$2} = 1; - } - } - - close(IN); -} - -# -# Parsers for the various processing states. -# -# STATE_NORMAL: looking for the /** to begin everything. -# -sub process_normal() { - if (/$doc_start/o) { - $state = STATE_NAME; # next line is always the function name - $in_doc_sect = 0; - $declaration_start_line = $. + 1; - } -} - -# -# STATE_NAME: Looking for the "name - description" line -# -sub process_name($$) { - my $file = shift; - my $identifier; - my $descr; - - if (/$doc_block/o) { - $state = STATE_DOCBLOCK; - $contents = ""; - $new_start_line = $.; - - if ( $1 eq "" ) { - $section = $section_intro; - } else { - $section = $1; - } - } - elsif (/$doc_decl/o) { - $identifier = $1; - if (/\s*([\w\s]+?)(\s*-|:)/) { - $identifier = $1; - } - - $state = STATE_BODY; - # if there's no @param blocks need to set up default section - # here - $contents = ""; - $section = $section_default; - $new_start_line = $. + 1; - if (/[-:](.*)/) { - # strip leading/trailing/multiple spaces - $descr= $1; - $descr =~ s/^\s*//; - $descr =~ s/\s*$//; - $descr =~ s/\s+/ /g; - $declaration_purpose = $descr; - $state = STATE_BODY_MAYBE; - } else { - $declaration_purpose = ""; - } - - if (($declaration_purpose eq "") && $verbose) { - print STDERR "${file}:$.: warning: missing initial short description on line:\n"; - print STDERR $_; - ++$warnings; - } - - if ($identifier =~ m/^struct\b/) { - $decl_type = 'struct'; - } elsif ($identifier =~ m/^union\b/) { - $decl_type = 'union'; - } elsif ($identifier =~ m/^enum\b/) { - $decl_type = 'enum'; - } elsif ($identifier =~ m/^typedef\b/) { - $decl_type = 'typedef'; - } else { - $decl_type = 'function'; - } - - if ($verbose) { - print STDERR "${file}:$.: info: Scanning doc for $identifier\n"; - } - } else { - print STDERR "${file}:$.: warning: Cannot understand $_ on line $.", - " - I thought it was a doc line\n"; - ++$warnings; - $state = STATE_NORMAL; - } -} - - -# -# STATE_BODY and STATE_BODY_MAYBE: the bulk of a kerneldoc comment. -# -sub process_body($$) { - my $file = shift; - - # Until all named variable macro parameters are - # documented using the bare name (`x`) rather than with - # dots (`x...`), strip the dots: - if ($section =~ /\w\.\.\.$/) { - $section =~ s/\.\.\.$//; - - if ($verbose) { - print STDERR "${file}:$.: warning: Variable macro arguments should be documented without dots\n"; - ++$warnings; - } - } - - if ($state == STATE_BODY_WITH_BLANK_LINE && /^\s*\*\s?\S/) { - dump_section($file, $section, $contents); - $section = $section_default; - $new_start_line = $.; - $contents = ""; - } - - if (/$doc_sect/i) { # case insensitive for supported section names - $newsection = $1; - $newcontents = $2; - - # map the supported section names to the canonical names - if ($newsection =~ m/^description$/i) { - $newsection = $section_default; - } elsif ($newsection =~ m/^context$/i) { - $newsection = $section_context; - } elsif ($newsection =~ m/^returns?$/i) { - $newsection = $section_return; - } elsif ($newsection =~ m/^\@return$/) { - # special: @return is a section, not a param description - $newsection = $section_return; - } - - if (($contents ne "") && ($contents ne "\n")) { - if (!$in_doc_sect && $verbose) { - print STDERR "${file}:$.: warning: contents before sections\n"; - ++$warnings; - } - dump_section($file, $section, $contents); - $section = $section_default; - } - - $in_doc_sect = 1; - $state = STATE_BODY; - $contents = $newcontents; - $new_start_line = $.; - while (substr($contents, 0, 1) eq " ") { - $contents = substr($contents, 1); - } - if ($contents ne "") { - $contents .= "\n"; - } - $section = $newsection; - $leading_space = undef; - } elsif (/$doc_end/) { - if (($contents ne "") && ($contents ne "\n")) { - dump_section($file, $section, $contents); - $section = $section_default; - $contents = ""; - } - # look for doc_com + <text> + doc_end: - if ($_ =~ m'\s*\*\s*[a-zA-Z_0-9:\.]+\*/') { - print STDERR "${file}:$.: warning: suspicious ending line: $_"; - ++$warnings; - } - - $prototype = ""; - $state = STATE_PROTO; - $brcount = 0; - $new_start_line = $. + 1; - } elsif (/$doc_content/) { - if ($1 eq "") { - if ($section eq $section_context) { - dump_section($file, $section, $contents); - $section = $section_default; - $contents = ""; - $new_start_line = $.; - $state = STATE_BODY; - } else { - if ($section ne $section_default) { - $state = STATE_BODY_WITH_BLANK_LINE; - } else { - $state = STATE_BODY; - } - $contents .= "\n"; - } - } elsif ($state == STATE_BODY_MAYBE) { - # Continued declaration purpose - chomp($declaration_purpose); - $declaration_purpose .= " " . $1; - $declaration_purpose =~ s/\s+/ /g; - } else { - my $cont = $1; - if ($section =~ m/^@/ || $section eq $section_context) { - if (!defined $leading_space) { - if ($cont =~ m/^(\s+)/) { - $leading_space = $1; - } else { - $leading_space = ""; - } - } - $cont =~ s/^$leading_space//; - } - $contents .= $cont . "\n"; - } - } else { - # i dont know - bad line? ignore. - print STDERR "${file}:$.: warning: bad line: $_"; - ++$warnings; - } -} - - -# -# STATE_PROTO: reading a function/whatever prototype. -# -sub process_proto($$) { - my $file = shift; - - if (/$doc_inline_oneline/) { - $section = $1; - $contents = $2; - if ($contents ne "") { - $contents .= "\n"; - dump_section($file, $section, $contents); - $section = $section_default; - $contents = ""; - } - } elsif (/$doc_inline_start/) { - $state = STATE_INLINE; - $inline_doc_state = STATE_INLINE_NAME; - } elsif ($decl_type eq 'function') { - process_proto_function($_, $file); - } else { - process_proto_type($_, $file); - } -} - -# -# STATE_DOCBLOCK: within a DOC: block. -# -sub process_docblock($$) { - my $file = shift; - - if (/$doc_end/) { - dump_doc_section($file, $section, $contents); - $section = $section_default; - $contents = ""; - $function = ""; - %parameterdescs = (); - %parametertypes = (); - @parameterlist = (); - %sections = (); - @sectionlist = (); - $prototype = ""; - $state = STATE_NORMAL; - } elsif (/$doc_content/) { - if ( $1 eq "" ) { - $contents .= $blankline; - } else { - $contents .= $1 . "\n"; - } - } -} - -# -# STATE_INLINE: docbook comments within a prototype. -# -sub process_inline($$) { - my $file = shift; - - # First line (state 1) needs to be a @parameter - if ($inline_doc_state == STATE_INLINE_NAME && /$doc_inline_sect/o) { - $section = $1; - $contents = $2; - $new_start_line = $.; - if ($contents ne "") { - while (substr($contents, 0, 1) eq " ") { - $contents = substr($contents, 1); - } - $contents .= "\n"; - } - $inline_doc_state = STATE_INLINE_TEXT; - # Documentation block end */ - } elsif (/$doc_inline_end/) { - if (($contents ne "") && ($contents ne "\n")) { - dump_section($file, $section, $contents); - $section = $section_default; - $contents = ""; - } - $state = STATE_PROTO; - $inline_doc_state = STATE_INLINE_NA; - # Regular text - } elsif (/$doc_content/) { - if ($inline_doc_state == STATE_INLINE_TEXT) { - $contents .= $1 . "\n"; - # nuke leading blank lines - if ($contents =~ /^\s*$/) { - $contents = ""; - } - } elsif ($inline_doc_state == STATE_INLINE_NAME) { - $inline_doc_state = STATE_INLINE_ERROR; - print STDERR "${file}:$.: warning: "; - print STDERR "Incorrect use of kernel-doc format: $_"; - ++$warnings; - } - } -} - - -sub process_file($) { - my $file; - my $initial_section_counter = $section_counter; - my ($orig_file) = @_; - - $file = map_filename($orig_file); - - if (!open(IN_FILE,"<$file")) { - print STDERR "Error: Cannot open file $file\n"; - ++$errors; - return; - } - - $. = 1; - - $section_counter = 0; - while (<IN_FILE>) { - while (s/\\\s*$//) { - $_ .= <IN_FILE>; - } - # Replace tabs by spaces - while ($_ =~ s/\t+/' ' x (length($&) * 8 - length($`) % 8)/e) {}; - # Hand this line to the appropriate state handler - if ($state == STATE_NORMAL) { - process_normal(); - } elsif ($state == STATE_NAME) { - process_name($file, $_); - } elsif ($state == STATE_BODY || $state == STATE_BODY_MAYBE || - $state == STATE_BODY_WITH_BLANK_LINE) { - process_body($file, $_); - } elsif ($state == STATE_INLINE) { # scanning for inline parameters - process_inline($file, $_); - } elsif ($state == STATE_PROTO) { - process_proto($file, $_); - } elsif ($state == STATE_DOCBLOCK) { - process_docblock($file, $_); - } - } - - # Make sure we got something interesting. - if ($initial_section_counter == $section_counter && $ - output_mode ne "none") { - if ($output_selection == OUTPUT_INCLUDE) { - print STDERR "${file}:1: warning: '$_' not found\n" - for keys %function_table; - } - else { - print STDERR "${file}:1: warning: no structured comments found\n"; - } - } - close IN_FILE; -} - - -if ($output_mode eq "rst") { - get_sphinx_version() if (!$sphinx_major); -} - -$kernelversion = get_kernel_version(); - -# generate a sequence of code that will splice in highlighting information -# using the s// operator. -for (my $k = 0; $k < @highlights; $k++) { - my $pattern = $highlights[$k][0]; - my $result = $highlights[$k][1]; -# print STDERR "scanning pattern:$pattern, highlight:($result)\n"; - $dohighlight .= "\$contents =~ s:$pattern:$result:gs;\n"; -} - -# Read the file that maps relative names to absolute names for -# separate source and object directories and for shadow trees. -if (open(SOURCE_MAP, "<.tmp_filelist.txt")) { - my ($relname, $absname); - while(<SOURCE_MAP>) { - chop(); - ($relname, $absname) = (split())[0..1]; - $relname =~ s:^/+::; - $source_map{$relname} = $absname; - } - close(SOURCE_MAP); -} - -if ($output_selection == OUTPUT_EXPORTED || - $output_selection == OUTPUT_INTERNAL) { - - push(@export_file_list, @ARGV); - - foreach (@export_file_list) { - chomp; - process_export_file($_); - } -} - -foreach (@ARGV) { - chomp; - process_file($_); -} -if ($verbose && $errors) { - print STDERR "$errors errors\n"; -} -if ($verbose && $warnings) { - print STDERR "$warnings warnings\n"; -} - -if ($Werror && $warnings) { - print STDERR "$warnings warnings as Errors\n"; - exit($warnings); -} else { - exit($output_mode eq "none" ? 0 : $errors) -} diff --git a/scripts/kernel-doc.py b/scripts/kernel-doc.py new file mode 100755 index 0000000..fc3d46e --- /dev/null +++ b/scripts/kernel-doc.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>. +# +# pylint: disable=C0103,R0915 +# +# Converted from the kernel-doc script originally written in Perl +# under GPLv2, copyrighted since 1998 by the following authors: +# +# Aditya Srivastava <yashsri421@gmail.com> +# Akira Yokosawa <akiyks@gmail.com> +# Alexander A. Klimov <grandmaster@al2klimov.de> +# Alexander Lobakin <aleksander.lobakin@intel.com> +# André Almeida <andrealmeid@igalia.com> +# Andy Shevchenko <andriy.shevchenko@linux.intel.com> +# Anna-Maria Behnsen <anna-maria@linutronix.de> +# Armin Kuster <akuster@mvista.com> +# Bart Van Assche <bart.vanassche@sandisk.com> +# Ben Hutchings <ben@decadent.org.uk> +# Borislav Petkov <bbpetkov@yahoo.de> +# Chen-Yu Tsai <wenst@chromium.org> +# Coco Li <lixiaoyan@google.com> +# Conchúr Navid <conchur@web.de> +# Daniel Santos <daniel.santos@pobox.com> +# Danilo Cesar Lemes de Paula <danilo.cesar@collabora.co.uk> +# Dan Luedtke <mail@danrl.de> +# Donald Hunter <donald.hunter@gmail.com> +# Gabriel Krisman Bertazi <krisman@collabora.co.uk> +# Greg Kroah-Hartman <gregkh@linuxfoundation.org> +# Harvey Harrison <harvey.harrison@gmail.com> +# Horia Geanta <horia.geanta@freescale.com> +# Ilya Dryomov <idryomov@gmail.com> +# Jakub Kicinski <kuba@kernel.org> +# Jani Nikula <jani.nikula@intel.com> +# Jason Baron <jbaron@redhat.com> +# Jason Gunthorpe <jgg@nvidia.com> +# Jérémy Bobbio <lunar@debian.org> +# Johannes Berg <johannes.berg@intel.com> +# Johannes Weiner <hannes@cmpxchg.org> +# Jonathan Cameron <Jonathan.Cameron@huawei.com> +# Jonathan Corbet <corbet@lwn.net> +# Jonathan Neuschäfer <j.neuschaefer@gmx.net> +# Kamil Rytarowski <n54@gmx.com> +# Kees Cook <kees@kernel.org> +# Laurent Pinchart <laurent.pinchart@ideasonboard.com> +# Levin, Alexander (Sasha Levin) <alexander.levin@verizon.com> +# Linus Torvalds <torvalds@linux-foundation.org> +# Lucas De Marchi <lucas.demarchi@profusion.mobi> +# Mark Rutland <mark.rutland@arm.com> +# Markus Heiser <markus.heiser@darmarit.de> +# Martin Waitz <tali@admingilde.org> +# Masahiro Yamada <masahiroy@kernel.org> +# Matthew Wilcox <willy@infradead.org> +# Mauro Carvalho Chehab <mchehab+huawei@kernel.org> +# Michal Wajdeczko <michal.wajdeczko@intel.com> +# Michael Zucchi +# Mike Rapoport <rppt@linux.ibm.com> +# Niklas Söderlund <niklas.soderlund@corigine.com> +# Nishanth Menon <nm@ti.com> +# Paolo Bonzini <pbonzini@redhat.com> +# Pavan Kumar Linga <pavan.kumar.linga@intel.com> +# Pavel Pisa <pisa@cmp.felk.cvut.cz> +# Peter Maydell <peter.maydell@linaro.org> +# Pierre-Louis Bossart <pierre-louis.bossart@linux.intel.com> +# Randy Dunlap <rdunlap@infradead.org> +# Richard Kennedy <richard@rsk.demon.co.uk> +# Rich Walker <rw@shadow.org.uk> +# Rolf Eike Beer <eike-kernel@sf-tec.de> +# Sakari Ailus <sakari.ailus@linux.intel.com> +# Silvio Fricke <silvio.fricke@gmail.com> +# Simon Huggins +# Tim Waugh <twaugh@redhat.com> +# Tomasz Warniełło <tomasz.warniello@gmail.com> +# Utkarsh Tripathi <utripathi2002@gmail.com> +# valdis.kletnieks@vt.edu <valdis.kletnieks@vt.edu> +# Vegard Nossum <vegard.nossum@oracle.com> +# Will Deacon <will.deacon@arm.com> +# Yacine Belkadi <yacine.belkadi.1@gmail.com> +# Yujie Liu <yujie.liu@intel.com> + +""" +kernel_doc +========== + +Print formatted kernel documentation to stdout + +Read C language source or header FILEs, extract embedded +documentation comments, and print formatted documentation +to standard output. + +The documentation comments are identified by the "/**" +opening comment mark. + +See Documentation/doc-guide/kernel-doc.rst for the +documentation comment syntax. +""" + +import argparse +import logging +import os +import sys + +# Import Python modules + +LIB_DIR = "lib/kdoc" +SRC_DIR = os.path.dirname(os.path.realpath(__file__)) + +sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) + +from kdoc_files import KernelFiles # pylint: disable=C0413 +from kdoc_output import RestFormat, ManFormat # pylint: disable=C0413 + +DESC = """ +Read C language source or header FILEs, extract embedded documentation comments, +and print formatted documentation to standard output. + +The documentation comments are identified by the "/**" opening comment mark. + +See Documentation/doc-guide/kernel-doc.rst for the documentation comment syntax. +""" + +EXPORT_FILE_DESC = """ +Specify an additional FILE in which to look for EXPORT_SYMBOL information. + +May be used multiple times. +""" + +EXPORT_DESC = """ +Only output documentation for the symbols that have been +exported using EXPORT_SYMBOL() and related macros in any input +FILE or -export-file FILE. +""" + +INTERNAL_DESC = """ +Only output documentation for the symbols that have NOT been +exported using EXPORT_SYMBOL() and related macros in any input +FILE or -export-file FILE. +""" + +FUNCTION_DESC = """ +Only output documentation for the given function or DOC: section +title. All other functions and DOC: sections are ignored. + +May be used multiple times. +""" + +NOSYMBOL_DESC = """ +Exclude the specified symbol from the output documentation. + +May be used multiple times. +""" + +FILES_DESC = """ +Header and C source files to be parsed. +""" + +WARN_CONTENTS_BEFORE_SECTIONS_DESC = """ +Warns if there are contents before sections (deprecated). + +This option is kept just for backward-compatibility, but it does nothing, +neither here nor at the original Perl script. +""" + + +class MsgFormatter(logging.Formatter): + """Helper class to format warnings on a similar way to kernel-doc.pl""" + + def format(self, record): + record.levelname = record.levelname.capitalize() + return logging.Formatter.format(self, record) + +def main(): + """Main program""" + + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, + description=DESC) + + # Normal arguments + + parser.add_argument("-v", "-verbose", "--verbose", action="store_true", + help="Verbose output, more warnings and other information.") + + parser.add_argument("-d", "-debug", "--debug", action="store_true", + help="Enable debug messages") + + parser.add_argument("-M", "-modulename", "--modulename", + default="Kernel API", + help="Allow setting a module name at the output.") + + parser.add_argument("-l", "-enable-lineno", "--enable_lineno", + action="store_true", + help="Enable line number output (only in ReST mode)") + + # Arguments to control the warning behavior + + parser.add_argument("-Wreturn", "--wreturn", action="store_true", + help="Warns about the lack of a return markup on functions.") + + parser.add_argument("-Wshort-desc", "-Wshort-description", "--wshort-desc", + action="store_true", + help="Warns if initial short description is missing") + + parser.add_argument("-Wcontents-before-sections", + "--wcontents-before-sections", action="store_true", + help=WARN_CONTENTS_BEFORE_SECTIONS_DESC) + + parser.add_argument("-Wall", "--wall", action="store_true", + help="Enable all types of warnings") + + parser.add_argument("-Werror", "--werror", action="store_true", + help="Treat warnings as errors.") + + parser.add_argument("-export-file", "--export-file", action='append', + help=EXPORT_FILE_DESC) + + # Output format mutually-exclusive group + + out_group = parser.add_argument_group("Output format selection (mutually exclusive)") + + out_fmt = out_group.add_mutually_exclusive_group() + + out_fmt.add_argument("-m", "-man", "--man", action="store_true", + help="Output troff manual page format.") + out_fmt.add_argument("-r", "-rst", "--rst", action="store_true", + help="Output reStructuredText format (default).") + out_fmt.add_argument("-N", "-none", "--none", action="store_true", + help="Do not output documentation, only warnings.") + + # Output selection mutually-exclusive group + + sel_group = parser.add_argument_group("Output selection (mutually exclusive)") + sel_mut = sel_group.add_mutually_exclusive_group() + + sel_mut.add_argument("-e", "-export", "--export", action='store_true', + help=EXPORT_DESC) + + sel_mut.add_argument("-i", "-internal", "--internal", action='store_true', + help=INTERNAL_DESC) + + sel_mut.add_argument("-s", "-function", "--symbol", action='append', + help=FUNCTION_DESC) + + # Those are valid for all 3 types of filter + parser.add_argument("-n", "-nosymbol", "--nosymbol", action='append', + help=NOSYMBOL_DESC) + + parser.add_argument("-D", "-no-doc-sections", "--no-doc-sections", + action='store_true', help="Don't outputt DOC sections") + + parser.add_argument("files", metavar="FILE", + nargs="+", help=FILES_DESC) + + args = parser.parse_args() + + if args.wall: + args.wreturn = True + args.wshort_desc = True + args.wcontents_before_sections = True + + logger = logging.getLogger() + + if not args.debug: + logger.setLevel(logging.INFO) + else: + logger.setLevel(logging.DEBUG) + + formatter = MsgFormatter('%(levelname)s: %(message)s') + + handler = logging.StreamHandler() + handler.setFormatter(formatter) + + logger.addHandler(handler) + + python_ver = sys.version_info[:2] + if python_ver < (3,6): + logger.warning("Python 3.6 or later is required by kernel-doc") + + # Return 0 here to avoid breaking compilation + sys.exit(0) + + if python_ver < (3,7): + logger.warning("Python 3.7 or later is required for correct results") + + if args.man: + out_style = ManFormat(modulename=args.modulename) + elif args.none: + out_style = None + else: + out_style = RestFormat() + + kfiles = KernelFiles(verbose=args.verbose, + out_style=out_style, werror=args.werror, + wreturn=args.wreturn, wshort_desc=args.wshort_desc, + wcontents_before_sections=args.wcontents_before_sections) + + kfiles.parse(args.files, export_file=args.export_file) + + for t in kfiles.msg(enable_lineno=args.enable_lineno, export=args.export, + internal=args.internal, symbol=args.symbol, + nosymbol=args.nosymbol, export_file=args.export_file, + no_doc_sections=args.no_doc_sections): + msg = t[1] + if msg: + print(msg) + + error_count = kfiles.errors + if not error_count: + sys.exit(0) + + if args.werror: + print(f"{error_count} warnings as errors") + sys.exit(error_count) + + if args.verbose: + print(f"{error_count} errors") + + if args.none: + sys.exit(0) + + sys.exit(error_count) + + +# Call main method +if __name__ == "__main__": + main() diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py new file mode 100644 index 0000000..9e09b45 --- /dev/null +++ b/scripts/lib/kdoc/kdoc_files.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>. +# +# pylint: disable=R0903,R0913,R0914,R0917 + +""" +Parse lernel-doc tags on multiple kernel source files. +""" + +import argparse +import logging +import os +import re + +from kdoc_parser import KernelDoc +from kdoc_output import OutputFormat + + +class GlobSourceFiles: + """ + Parse C source code file names and directories via an Interactor. + """ + + def __init__(self, srctree=None, valid_extensions=None): + """ + Initialize valid extensions with a tuple. + + If not defined, assume default C extensions (.c and .h) + + It would be possible to use python's glob function, but it is + very slow, and it is not interactive. So, it would wait to read all + directories before actually do something. + + So, let's use our own implementation. + """ + + if not valid_extensions: + self.extensions = (".c", ".h") + else: + self.extensions = valid_extensions + + self.srctree = srctree + + def _parse_dir(self, dirname): + """Internal function to parse files recursively""" + + with os.scandir(dirname) as obj: + for entry in obj: + name = os.path.join(dirname, entry.name) + + if entry.is_dir(): + yield from self._parse_dir(name) + + if not entry.is_file(): + continue + + basename = os.path.basename(name) + + if not basename.endswith(self.extensions): + continue + + yield name + + def parse_files(self, file_list, file_not_found_cb): + """ + Define an interator to parse all source files from file_list, + handling directories if any + """ + + if not file_list: + return + + for fname in file_list: + if self.srctree: + f = os.path.join(self.srctree, fname) + else: + f = fname + + if os.path.isdir(f): + yield from self._parse_dir(f) + elif os.path.isfile(f): + yield f + elif file_not_found_cb: + file_not_found_cb(fname) + + +class KernelFiles(): + """ + Parse kernel-doc tags on multiple kernel source files. + + There are two type of parsers defined here: + - self.parse_file(): parses both kernel-doc markups and + EXPORT_SYMBOL* macros; + - self.process_export_file(): parses only EXPORT_SYMBOL* macros. + """ + + def warning(self, msg): + """Ancillary routine to output a warning and increment error count""" + + self.config.log.warning(msg) + self.errors += 1 + + def error(self, msg): + """Ancillary routine to output an error and increment error count""" + + self.config.log.error(msg) + self.errors += 1 + + def parse_file(self, fname): + """ + Parse a single Kernel source. + """ + + # Prevent parsing the same file twice if results are cached + if fname in self.files: + return + + doc = KernelDoc(self.config, fname) + export_table, entries = doc.parse_kdoc() + + self.export_table[fname] = export_table + + self.files.add(fname) + self.export_files.add(fname) # parse_kdoc() already check exports + + self.results[fname] = entries + + def process_export_file(self, fname): + """ + Parses EXPORT_SYMBOL* macros from a single Kernel source file. + """ + + # Prevent parsing the same file twice if results are cached + if fname in self.export_files: + return + + doc = KernelDoc(self.config, fname) + export_table = doc.parse_export() + + if not export_table: + self.error(f"Error: Cannot check EXPORT_SYMBOL* on {fname}") + export_table = set() + + self.export_table[fname] = export_table + self.export_files.add(fname) + + def file_not_found_cb(self, fname): + """ + Callback to warn if a file was not found. + """ + + self.error(f"Cannot find file {fname}") + + def __init__(self, verbose=False, out_style=None, + werror=False, wreturn=False, wshort_desc=False, + wcontents_before_sections=False, + logger=None): + """ + Initialize startup variables and parse all files + """ + + if not verbose: + verbose = bool(os.environ.get("KBUILD_VERBOSE", 0)) + + if out_style is None: + out_style = OutputFormat() + + if not werror: + kcflags = os.environ.get("KCFLAGS", None) + if kcflags: + match = re.search(r"(\s|^)-Werror(\s|$)/", kcflags) + if match: + werror = True + + # reading this variable is for backwards compat just in case + # someone was calling it with the variable from outside the + # kernel's build system + kdoc_werror = os.environ.get("KDOC_WERROR", None) + if kdoc_werror: + werror = kdoc_werror + + # Some variables are global to the parser logic as a whole as they are + # used to send control configuration to KernelDoc class. As such, + # those variables are read-only inside the KernelDoc. + self.config = argparse.Namespace + + self.config.verbose = verbose + self.config.werror = werror + self.config.wreturn = wreturn + self.config.wshort_desc = wshort_desc + self.config.wcontents_before_sections = wcontents_before_sections + + if not logger: + self.config.log = logging.getLogger("kernel-doc") + else: + self.config.log = logger + + self.config.warning = self.warning + + self.config.src_tree = os.environ.get("SRCTREE", None) + + # Initialize variables that are internal to KernelFiles + + self.out_style = out_style + + self.errors = 0 + self.results = {} + + self.files = set() + self.export_files = set() + self.export_table = {} + + def parse(self, file_list, export_file=None): + """ + Parse all files + """ + + glob = GlobSourceFiles(srctree=self.config.src_tree) + + for fname in glob.parse_files(file_list, self.file_not_found_cb): + self.parse_file(fname) + + for fname in glob.parse_files(export_file, self.file_not_found_cb): + self.process_export_file(fname) + + def out_msg(self, fname, name, arg): + """ + Return output messages from a file name using the output style + filtering. + + If output type was not handled by the syler, return None. + """ + + # NOTE: we can add rules here to filter out unwanted parts, + # although OutputFormat.msg already does that. + + return self.out_style.msg(fname, name, arg) + + def msg(self, enable_lineno=False, export=False, internal=False, + symbol=None, nosymbol=None, no_doc_sections=False, + filenames=None, export_file=None): + """ + Interacts over the kernel-doc results and output messages, + returning kernel-doc markups on each interaction + """ + + self.out_style.set_config(self.config) + + if not filenames: + filenames = sorted(self.results.keys()) + + glob = GlobSourceFiles(srctree=self.config.src_tree) + + for fname in filenames: + function_table = set() + + if internal or export: + if not export_file: + export_file = [fname] + + for f in glob.parse_files(export_file, self.file_not_found_cb): + function_table |= self.export_table[f] + + if symbol: + for s in symbol: + function_table.add(s) + + self.out_style.set_filter(export, internal, symbol, nosymbol, + function_table, enable_lineno, + no_doc_sections) + + msg = "" + if fname not in self.results: + self.config.log.warning("No kernel-doc for file %s", fname) + continue + + for arg in self.results[fname]: + m = self.out_msg(fname, arg.name, arg) + + if m is None: + ln = arg.get("ln", 0) + dtype = arg.get('type', "") + + self.config.log.warning("%s:%d Can't handle %s", + fname, ln, dtype) + else: + msg += m + + if msg: + yield fname, msg diff --git a/scripts/lib/kdoc/kdoc_item.py b/scripts/lib/kdoc/kdoc_item.py new file mode 100644 index 0000000..b3b2257 --- /dev/null +++ b/scripts/lib/kdoc/kdoc_item.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# A class that will, eventually, encapsulate all of the parsed data that we +# then pass into the output modules. +# + +class KdocItem: + def __init__(self, name, type, start_line, **other_stuff): + self.name = name + self.type = type + self.declaration_start_line = start_line + self.sections = {} + self.sections_start_lines = {} + self.parameterlist = [] + self.parameterdesc_start_lines = [] + self.parameterdescs = {} + self.parametertypes = {} + # + # Just save everything else into our own dict so that the output + # side can grab it directly as before. As we move things into more + # structured data, this will, hopefully, fade away. + # + self.other_stuff = other_stuff + + def get(self, key, default = None): + return self.other_stuff.get(key, default) + + def __getitem__(self, key): + return self.get(key) + + # + # Tracking of section and parameter information. + # + def set_sections(self, sections, start_lines): + self.sections = sections + self.section_start_lines = start_lines + + def set_params(self, names, descs, types, starts): + self.parameterlist = names + self.parameterdescs = descs + self.parametertypes = types + self.parameterdesc_start_lines = starts diff --git a/scripts/lib/kdoc/kdoc_output.py b/scripts/lib/kdoc/kdoc_output.py new file mode 100644 index 0000000..39fa872 --- /dev/null +++ b/scripts/lib/kdoc/kdoc_output.py @@ -0,0 +1,749 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>. +# +# pylint: disable=C0301,R0902,R0911,R0912,R0913,R0914,R0915,R0917 + +""" +Implement output filters to print kernel-doc documentation. + +The implementation uses a virtual base class (OutputFormat) which +contains a dispatches to virtual methods, and some code to filter +out output messages. + +The actual implementation is done on one separate class per each type +of output. Currently, there are output classes for ReST and man/troff. +""" + +import os +import re +from datetime import datetime + +from kdoc_parser import KernelDoc, type_param +from kdoc_re import KernRe + + +function_pointer = KernRe(r"([^\(]*\(\*)\s*\)\s*\(([^\)]*)\)", cache=False) + +# match expressions used to find embedded type information +type_constant = KernRe(r"\b``([^\`]+)``\b", cache=False) +type_constant2 = KernRe(r"\%([-_*\w]+)", cache=False) +type_func = KernRe(r"(\w+)\(\)", cache=False) +type_param_ref = KernRe(r"([\!~\*]?)\@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)", cache=False) + +# Special RST handling for func ptr params +type_fp_param = KernRe(r"\@(\w+)\(\)", cache=False) + +# Special RST handling for structs with func ptr params +type_fp_param2 = KernRe(r"\@(\w+->\S+)\(\)", cache=False) + +type_env = KernRe(r"(\$\w+)", cache=False) +type_enum = KernRe(r"#(enum\s*([_\w]+))", cache=False) +type_struct = KernRe(r"#(struct\s*([_\w]+))", cache=False) +type_typedef = KernRe(r"#(([A-Z][_\w]*))", cache=False) +type_union = KernRe(r"#(union\s*([_\w]+))", cache=False) +type_member = KernRe(r"#([_\w]+)(\.|->)([_\w]+)", cache=False) +type_fallback = KernRe(r"((?!))", cache=False) # this never matches +type_member_func = type_member + KernRe(r"\(\)", cache=False) + + +class OutputFormat: + """ + Base class for OutputFormat. If used as-is, it means that only + warnings will be displayed. + """ + + # output mode. + OUTPUT_ALL = 0 # output all symbols and doc sections + OUTPUT_INCLUDE = 1 # output only specified symbols + OUTPUT_EXPORTED = 2 # output exported symbols + OUTPUT_INTERNAL = 3 # output non-exported symbols + + # Virtual member to be overriden at the inherited classes + highlights = [] + + def __init__(self): + """Declare internal vars and set mode to OUTPUT_ALL""" + + self.out_mode = self.OUTPUT_ALL + self.enable_lineno = None + self.nosymbol = {} + self.symbol = None + self.function_table = None + self.config = None + self.no_doc_sections = False + + self.data = "" + + def set_config(self, config): + """ + Setup global config variables used by both parser and output. + """ + + self.config = config + + def set_filter(self, export, internal, symbol, nosymbol, function_table, + enable_lineno, no_doc_sections): + """ + Initialize filter variables according with the requested mode. + + Only one choice is valid between export, internal and symbol. + + The nosymbol filter can be used on all modes. + """ + + self.enable_lineno = enable_lineno + self.no_doc_sections = no_doc_sections + self.function_table = function_table + + if symbol: + self.out_mode = self.OUTPUT_INCLUDE + elif export: + self.out_mode = self.OUTPUT_EXPORTED + elif internal: + self.out_mode = self.OUTPUT_INTERNAL + else: + self.out_mode = self.OUTPUT_ALL + + if nosymbol: + self.nosymbol = set(nosymbol) + + + def highlight_block(self, block): + """ + Apply the RST highlights to a sub-block of text. + """ + + for r, sub in self.highlights: + block = r.sub(sub, block) + + return block + + def out_warnings(self, args): + """ + Output warnings for identifiers that will be displayed. + """ + + for log_msg in args.warnings: + self.config.warning(log_msg) + + def check_doc(self, name, args): + """Check if DOC should be output""" + + if self.no_doc_sections: + return False + + if name in self.nosymbol: + return False + + if self.out_mode == self.OUTPUT_ALL: + self.out_warnings(args) + return True + + if self.out_mode == self.OUTPUT_INCLUDE: + if name in self.function_table: + self.out_warnings(args) + return True + + return False + + def check_declaration(self, dtype, name, args): + """ + Checks if a declaration should be output or not based on the + filtering criteria. + """ + + if name in self.nosymbol: + return False + + if self.out_mode == self.OUTPUT_ALL: + self.out_warnings(args) + return True + + if self.out_mode in [self.OUTPUT_INCLUDE, self.OUTPUT_EXPORTED]: + if name in self.function_table: + return True + + if self.out_mode == self.OUTPUT_INTERNAL: + if dtype != "function": + self.out_warnings(args) + return True + + if name not in self.function_table: + self.out_warnings(args) + return True + + return False + + def msg(self, fname, name, args): + """ + Handles a single entry from kernel-doc parser + """ + + self.data = "" + + dtype = args.type + + if dtype == "doc": + self.out_doc(fname, name, args) + return self.data + + if not self.check_declaration(dtype, name, args): + return self.data + + if dtype == "function": + self.out_function(fname, name, args) + return self.data + + if dtype == "enum": + self.out_enum(fname, name, args) + return self.data + + if dtype == "typedef": + self.out_typedef(fname, name, args) + return self.data + + if dtype in ["struct", "union"]: + self.out_struct(fname, name, args) + return self.data + + # Warn if some type requires an output logic + self.config.log.warning("doesn't now how to output '%s' block", + dtype) + + return None + + # Virtual methods to be overridden by inherited classes + # At the base class, those do nothing. + def out_doc(self, fname, name, args): + """Outputs a DOC block""" + + def out_function(self, fname, name, args): + """Outputs a function""" + + def out_enum(self, fname, name, args): + """Outputs an enum""" + + def out_typedef(self, fname, name, args): + """Outputs a typedef""" + + def out_struct(self, fname, name, args): + """Outputs a struct""" + + +class RestFormat(OutputFormat): + """Consts and functions used by ReST output""" + + highlights = [ + (type_constant, r"``\1``"), + (type_constant2, r"``\1``"), + + # Note: need to escape () to avoid func matching later + (type_member_func, r":c:type:`\1\2\3\\(\\) <\1>`"), + (type_member, r":c:type:`\1\2\3 <\1>`"), + (type_fp_param, r"**\1\\(\\)**"), + (type_fp_param2, r"**\1\\(\\)**"), + (type_func, r"\1()"), + (type_enum, r":c:type:`\1 <\2>`"), + (type_struct, r":c:type:`\1 <\2>`"), + (type_typedef, r":c:type:`\1 <\2>`"), + (type_union, r":c:type:`\1 <\2>`"), + + # in rst this can refer to any type + (type_fallback, r":c:type:`\1`"), + (type_param_ref, r"**\1\2**") + ] + blankline = "\n" + + sphinx_literal = KernRe(r'^[^.].*::$', cache=False) + sphinx_cblock = KernRe(r'^\.\.\ +code-block::', cache=False) + + def __init__(self): + """ + Creates class variables. + + Not really mandatory, but it is a good coding style and makes + pylint happy. + """ + + super().__init__() + self.lineprefix = "" + + def print_lineno(self, ln): + """Outputs a line number""" + + if self.enable_lineno and ln is not None: + ln += 1 + self.data += f".. LINENO {ln}\n" + + def output_highlight(self, args): + """ + Outputs a C symbol that may require being converted to ReST using + the self.highlights variable + """ + + input_text = args + output = "" + in_literal = False + litprefix = "" + block = "" + + for line in input_text.strip("\n").split("\n"): + + # If we're in a literal block, see if we should drop out of it. + # Otherwise, pass the line straight through unmunged. + if in_literal: + if line.strip(): # If the line is not blank + # If this is the first non-blank line in a literal block, + # figure out the proper indent. + if not litprefix: + r = KernRe(r'^(\s*)') + if r.match(line): + litprefix = '^' + r.group(1) + else: + litprefix = "" + + output += line + "\n" + elif not KernRe(litprefix).match(line): + in_literal = False + else: + output += line + "\n" + else: + output += line + "\n" + + # Not in a literal block (or just dropped out) + if not in_literal: + block += line + "\n" + if self.sphinx_literal.match(line) or self.sphinx_cblock.match(line): + in_literal = True + litprefix = "" + output += self.highlight_block(block) + block = "" + + # Handle any remaining block + if block: + output += self.highlight_block(block) + + # Print the output with the line prefix + for line in output.strip("\n").split("\n"): + self.data += self.lineprefix + line + "\n" + + def out_section(self, args, out_docblock=False): + """ + Outputs a block section. + + This could use some work; it's used to output the DOC: sections, and + starts by putting out the name of the doc section itself, but that + tends to duplicate a header already in the template file. + """ + for section, text in args.sections.items(): + # Skip sections that are in the nosymbol_table + if section in self.nosymbol: + continue + + if out_docblock: + if not self.out_mode == self.OUTPUT_INCLUDE: + self.data += f".. _{section}:\n\n" + self.data += f'{self.lineprefix}**{section}**\n\n' + else: + self.data += f'{self.lineprefix}**{section}**\n\n' + + self.print_lineno(args.section_start_lines.get(section, 0)) + self.output_highlight(text) + self.data += "\n" + self.data += "\n" + + def out_doc(self, fname, name, args): + if not self.check_doc(name, args): + return + self.out_section(args, out_docblock=True) + + def out_function(self, fname, name, args): + + oldprefix = self.lineprefix + signature = "" + + func_macro = args.get('func_macro', False) + if func_macro: + signature = name + else: + if args.get('functiontype'): + signature = args['functiontype'] + " " + signature += name + " (" + + ln = args.declaration_start_line + count = 0 + for parameter in args.parameterlist: + if count != 0: + signature += ", " + count += 1 + dtype = args.parametertypes.get(parameter, "") + + if function_pointer.search(dtype): + signature += function_pointer.group(1) + parameter + function_pointer.group(3) + else: + signature += dtype + + if not func_macro: + signature += ")" + + self.print_lineno(ln) + if args.get('typedef') or not args.get('functiontype'): + self.data += f".. c:macro:: {name}\n\n" + + if args.get('typedef'): + self.data += " **Typedef**: " + self.lineprefix = "" + self.output_highlight(args.get('purpose', "")) + self.data += "\n\n**Syntax**\n\n" + self.data += f" ``{signature}``\n\n" + else: + self.data += f"``{signature}``\n\n" + else: + self.data += f".. c:function:: {signature}\n\n" + + if not args.get('typedef'): + self.print_lineno(ln) + self.lineprefix = " " + self.output_highlight(args.get('purpose', "")) + self.data += "\n" + + # Put descriptive text into a container (HTML <div>) to help set + # function prototypes apart + self.lineprefix = " " + + if args.parameterlist: + self.data += ".. container:: kernelindent\n\n" + self.data += f"{self.lineprefix}**Parameters**\n\n" + + for parameter in args.parameterlist: + parameter_name = KernRe(r'\[.*').sub('', parameter) + dtype = args.parametertypes.get(parameter, "") + + if dtype: + self.data += f"{self.lineprefix}``{dtype}``\n" + else: + self.data += f"{self.lineprefix}``{parameter}``\n" + + self.print_lineno(args.parameterdesc_start_lines.get(parameter_name, 0)) + + self.lineprefix = " " + if parameter_name in args.parameterdescs and \ + args.parameterdescs[parameter_name] != KernelDoc.undescribed: + + self.output_highlight(args.parameterdescs[parameter_name]) + self.data += "\n" + else: + self.data += f"{self.lineprefix}*undescribed*\n\n" + self.lineprefix = " " + + self.out_section(args) + self.lineprefix = oldprefix + + def out_enum(self, fname, name, args): + + oldprefix = self.lineprefix + ln = args.declaration_start_line + + self.data += f"\n\n.. c:enum:: {name}\n\n" + + self.print_lineno(ln) + self.lineprefix = " " + self.output_highlight(args.get('purpose', '')) + self.data += "\n" + + self.data += ".. container:: kernelindent\n\n" + outer = self.lineprefix + " " + self.lineprefix = outer + " " + self.data += f"{outer}**Constants**\n\n" + + for parameter in args.parameterlist: + self.data += f"{outer}``{parameter}``\n" + + if args.parameterdescs.get(parameter, '') != KernelDoc.undescribed: + self.output_highlight(args.parameterdescs[parameter]) + else: + self.data += f"{self.lineprefix}*undescribed*\n\n" + self.data += "\n" + + self.lineprefix = oldprefix + self.out_section(args) + + def out_typedef(self, fname, name, args): + + oldprefix = self.lineprefix + ln = args.declaration_start_line + + self.data += f"\n\n.. c:type:: {name}\n\n" + + self.print_lineno(ln) + self.lineprefix = " " + + self.output_highlight(args.get('purpose', '')) + + self.data += "\n" + + self.lineprefix = oldprefix + self.out_section(args) + + def out_struct(self, fname, name, args): + + purpose = args.get('purpose', "") + declaration = args.get('definition', "") + dtype = args.type + ln = args.declaration_start_line + + self.data += f"\n\n.. c:{dtype}:: {name}\n\n" + + self.print_lineno(ln) + + oldprefix = self.lineprefix + self.lineprefix += " " + + self.output_highlight(purpose) + self.data += "\n" + + self.data += ".. container:: kernelindent\n\n" + self.data += f"{self.lineprefix}**Definition**::\n\n" + + self.lineprefix = self.lineprefix + " " + + declaration = declaration.replace("\t", self.lineprefix) + + self.data += f"{self.lineprefix}{dtype} {name}" + ' {' + "\n" + self.data += f"{declaration}{self.lineprefix}" + "};\n\n" + + self.lineprefix = " " + self.data += f"{self.lineprefix}**Members**\n\n" + for parameter in args.parameterlist: + if not parameter or parameter.startswith("#"): + continue + + parameter_name = parameter.split("[", maxsplit=1)[0] + + if args.parameterdescs.get(parameter_name) == KernelDoc.undescribed: + continue + + self.print_lineno(args.parameterdesc_start_lines.get(parameter_name, 0)) + + self.data += f"{self.lineprefix}``{parameter}``\n" + + self.lineprefix = " " + self.output_highlight(args.parameterdescs[parameter_name]) + self.lineprefix = " " + + self.data += "\n" + + self.data += "\n" + + self.lineprefix = oldprefix + self.out_section(args) + + +class ManFormat(OutputFormat): + """Consts and functions used by man pages output""" + + highlights = ( + (type_constant, r"\1"), + (type_constant2, r"\1"), + (type_func, r"\\fB\1\\fP"), + (type_enum, r"\\fI\1\\fP"), + (type_struct, r"\\fI\1\\fP"), + (type_typedef, r"\\fI\1\\fP"), + (type_union, r"\\fI\1\\fP"), + (type_param, r"\\fI\1\\fP"), + (type_param_ref, r"\\fI\1\2\\fP"), + (type_member, r"\\fI\1\2\3\\fP"), + (type_fallback, r"\\fI\1\\fP") + ) + blankline = "" + + date_formats = [ + "%a %b %d %H:%M:%S %Z %Y", + "%a %b %d %H:%M:%S %Y", + "%Y-%m-%d", + "%b %d %Y", + "%B %d %Y", + "%m %d %Y", + ] + + def __init__(self, modulename): + """ + Creates class variables. + + Not really mandatory, but it is a good coding style and makes + pylint happy. + """ + + super().__init__() + self.modulename = modulename + + dt = None + tstamp = os.environ.get("KBUILD_BUILD_TIMESTAMP") + if tstamp: + for fmt in self.date_formats: + try: + dt = datetime.strptime(tstamp, fmt) + break + except ValueError: + pass + + if not dt: + dt = datetime.now() + + self.man_date = dt.strftime("%B %Y") + + def output_highlight(self, block): + """ + Outputs a C symbol that may require being highlighted with + self.highlights variable using troff syntax + """ + + contents = self.highlight_block(block) + + if isinstance(contents, list): + contents = "\n".join(contents) + + for line in contents.strip("\n").split("\n"): + line = KernRe(r"^\s*").sub("", line) + if not line: + continue + + if line[0] == ".": + self.data += "\\&" + line + "\n" + else: + self.data += line + "\n" + + def out_doc(self, fname, name, args): + if not self.check_doc(name, args): + return + + self.data += f'.TH "{self.modulename}" 9 "{self.modulename}" "{self.man_date}" "API Manual" LINUX' + "\n" + + for section, text in args.sections.items(): + self.data += f'.SH "{section}"' + "\n" + self.output_highlight(text) + + def out_function(self, fname, name, args): + """output function in man""" + + self.data += f'.TH "{name}" 9 "{name}" "{self.man_date}" "Kernel Hacker\'s Manual" LINUX' + "\n" + + self.data += ".SH NAME\n" + self.data += f"{name} \\- {args['purpose']}\n" + + self.data += ".SH SYNOPSIS\n" + if args.get('functiontype', ''): + self.data += f'.B "{args["functiontype"]}" {name}' + "\n" + else: + self.data += f'.B "{name}' + "\n" + + count = 0 + parenth = "(" + post = "," + + for parameter in args.parameterlist: + if count == len(args.parameterlist) - 1: + post = ");" + + dtype = args.parametertypes.get(parameter, "") + if function_pointer.match(dtype): + # Pointer-to-function + self.data += f'".BI "{parenth}{function_pointer.group(1)}" " ") ({function_pointer.group(2)}){post}"' + "\n" + else: + dtype = KernRe(r'([^\*])$').sub(r'\1 ', dtype) + + self.data += f'.BI "{parenth}{dtype}" "{post}"' + "\n" + count += 1 + parenth = "" + + if args.parameterlist: + self.data += ".SH ARGUMENTS\n" + + for parameter in args.parameterlist: + parameter_name = re.sub(r'\[.*', '', parameter) + + self.data += f'.IP "{parameter}" 12' + "\n" + self.output_highlight(args.parameterdescs.get(parameter_name, "")) + + for section, text in args.sections.items(): + self.data += f'.SH "{section.upper()}"' + "\n" + self.output_highlight(text) + + def out_enum(self, fname, name, args): + self.data += f'.TH "{self.modulename}" 9 "enum {name}" "{self.man_date}" "API Manual" LINUX' + "\n" + + self.data += ".SH NAME\n" + self.data += f"enum {name} \\- {args['purpose']}\n" + + self.data += ".SH SYNOPSIS\n" + self.data += f"enum {name}" + " {\n" + + count = 0 + for parameter in args.parameterlist: + self.data += f'.br\n.BI " {parameter}"' + "\n" + if count == len(args.parameterlist) - 1: + self.data += "\n};\n" + else: + self.data += ", \n.br\n" + + count += 1 + + self.data += ".SH Constants\n" + + for parameter in args.parameterlist: + parameter_name = KernRe(r'\[.*').sub('', parameter) + self.data += f'.IP "{parameter}" 12' + "\n" + self.output_highlight(args.parameterdescs.get(parameter_name, "")) + + for section, text in args.sections.items(): + self.data += f'.SH "{section}"' + "\n" + self.output_highlight(text) + + def out_typedef(self, fname, name, args): + module = self.modulename + purpose = args.get('purpose') + + self.data += f'.TH "{module}" 9 "{name}" "{self.man_date}" "API Manual" LINUX' + "\n" + + self.data += ".SH NAME\n" + self.data += f"typedef {name} \\- {purpose}\n" + + for section, text in args.sections.items(): + self.data += f'.SH "{section}"' + "\n" + self.output_highlight(text) + + def out_struct(self, fname, name, args): + module = self.modulename + purpose = args.get('purpose') + definition = args.get('definition') + + self.data += f'.TH "{module}" 9 "{args.type} {name}" "{self.man_date}" "API Manual" LINUX' + "\n" + + self.data += ".SH NAME\n" + self.data += f"{args.type} {name} \\- {purpose}\n" + + # Replace tabs with two spaces and handle newlines + declaration = definition.replace("\t", " ") + declaration = KernRe(r"\n").sub('"\n.br\n.BI "', declaration) + + self.data += ".SH SYNOPSIS\n" + self.data += f"{args.type} {name} " + "{" + "\n.br\n" + self.data += f'.BI "{declaration}\n' + "};\n.br\n\n" + + self.data += ".SH Members\n" + for parameter in args.parameterlist: + if parameter.startswith("#"): + continue + + parameter_name = re.sub(r"\[.*", "", parameter) + + if args.parameterdescs.get(parameter_name) == KernelDoc.undescribed: + continue + + self.data += f'.IP "{parameter}" 12' + "\n" + self.output_highlight(args.parameterdescs.get(parameter_name)) + + for section, text in args.sections.items(): + self.data += f'.SH "{section}"' + "\n" + self.output_highlight(text) diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py new file mode 100644 index 0000000..32b4356 --- /dev/null +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -0,0 +1,1670 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>. +# +# pylint: disable=C0301,C0302,R0904,R0912,R0913,R0914,R0915,R0917,R1702 + +""" +kdoc_parser +=========== + +Read a C language source or header FILE and extract embedded +documentation comments +""" + +import sys +import re +from pprint import pformat + +from kdoc_re import NestedMatch, KernRe +from kdoc_item import KdocItem + +# +# Regular expressions used to parse kernel-doc markups at KernelDoc class. +# +# Let's declare them in lowercase outside any class to make easier to +# convert from the python script. +# +# As those are evaluated at the beginning, no need to cache them +# + +# Allow whitespace at end of comment start. +doc_start = KernRe(r'^/\*\*\s*$', cache=False) + +doc_end = KernRe(r'\*/', cache=False) +doc_com = KernRe(r'\s*\*\s*', cache=False) +doc_com_body = KernRe(r'\s*\* ?', cache=False) +doc_decl = doc_com + KernRe(r'(\w+)', cache=False) + +# @params and a strictly limited set of supported section names +# Specifically: +# Match @word: +# @...: +# @{section-name}: +# while trying to not match literal block starts like "example::" +# +known_section_names = 'description|context|returns?|notes?|examples?' +known_sections = KernRe(known_section_names, flags = re.I) +doc_sect = doc_com + \ + KernRe(r'\s*(\@[.\w]+|\@\.\.\.|' + known_section_names + r')\s*:([^:].*)?$', + flags=re.I, cache=False) + +doc_content = doc_com_body + KernRe(r'(.*)', cache=False) +doc_inline_start = KernRe(r'^\s*/\*\*\s*$', cache=False) +doc_inline_sect = KernRe(r'\s*\*\s*(@\s*[\w][\w\.]*\s*):(.*)', cache=False) +doc_inline_end = KernRe(r'^\s*\*/\s*$', cache=False) +doc_inline_oneline = KernRe(r'^\s*/\*\*\s*(@[\w\s]+):\s*(.*)\s*\*/\s*$', cache=False) +attribute = KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", + flags=re.I | re.S, cache=False) + +export_symbol = KernRe(r'^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*', cache=False) +export_symbol_ns = KernRe(r'^\s*EXPORT_SYMBOL_NS(_GPL)?\s*\(\s*(\w+)\s*,\s*"\S+"\)\s*', cache=False) + +type_param = KernRe(r"\@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)", cache=False) + +# +# Tests for the beginning of a kerneldoc block in its various forms. +# +doc_block = doc_com + KernRe(r'DOC:\s*(.*)?', cache=False) +doc_begin_data = KernRe(r"^\s*\*?\s*(struct|union|enum|typedef)\b\s*(\w*)", cache = False) +doc_begin_func = KernRe(str(doc_com) + # initial " * ' + r"(?:\w+\s*\*\s*)?" + # type (not captured) + r'(?:define\s+)?' + # possible "define" (not captured) + r'(\w+)\s*(?:\(\w*\))?\s*' + # name and optional "(...)" + r'(?:[-:].*)?$', # description (not captured) + cache = False) + +# +# A little helper to get rid of excess white space +# +multi_space = KernRe(r'\s\s+') +def trim_whitespace(s): + return multi_space.sub(' ', s.strip()) + +class state: + """ + State machine enums + """ + + # Parser states + NORMAL = 0 # normal code + NAME = 1 # looking for function name + DECLARATION = 2 # We have seen a declaration which might not be done + BODY = 3 # the body of the comment + SPECIAL_SECTION = 4 # doc section ending with a blank line + PROTO = 5 # scanning prototype + DOCBLOCK = 6 # documentation block + INLINE_NAME = 7 # gathering doc outside main block + INLINE_TEXT = 8 # reading the body of inline docs + + name = [ + "NORMAL", + "NAME", + "DECLARATION", + "BODY", + "SPECIAL_SECTION", + "PROTO", + "DOCBLOCK", + "INLINE_NAME", + "INLINE_TEXT", + ] + + +SECTION_DEFAULT = "Description" # default section + +class KernelEntry: + + def __init__(self, config, ln): + self.config = config + + self._contents = [] + self.prototype = "" + + self.warnings = [] + + self.parameterlist = [] + self.parameterdescs = {} + self.parametertypes = {} + self.parameterdesc_start_lines = {} + + self.section_start_lines = {} + self.sections = {} + + self.anon_struct_union = False + + self.leading_space = None + + # State flags + self.brcount = 0 + self.declaration_start_line = ln + 1 + + # + # Management of section contents + # + def add_text(self, text): + self._contents.append(text) + + def contents(self): + return '\n'.join(self._contents) + '\n' + + # TODO: rename to emit_message after removal of kernel-doc.pl + def emit_msg(self, log_msg, warning=True): + """Emit a message""" + + if not warning: + self.config.log.info(log_msg) + return + + # Delegate warning output to output logic, as this way it + # will report warnings/info only for symbols that are output + + self.warnings.append(log_msg) + return + + # + # Begin a new section. + # + def begin_section(self, line_no, title = SECTION_DEFAULT, dump = False): + if dump: + self.dump_section(start_new = True) + self.section = title + self.new_start_line = line_no + + def dump_section(self, start_new=True): + """ + Dumps section contents to arrays/hashes intended for that purpose. + """ + # + # If we have accumulated no contents in the default ("description") + # section, don't bother. + # + if self.section == SECTION_DEFAULT and not self._contents: + return + name = self.section + contents = self.contents() + + if type_param.match(name): + name = type_param.group(1) + + self.parameterdescs[name] = contents + self.parameterdesc_start_lines[name] = self.new_start_line + + self.new_start_line = 0 + + else: + if name in self.sections and self.sections[name] != "": + # Only warn on user-specified duplicate section names + if name != SECTION_DEFAULT: + self.emit_msg(self.new_start_line, + f"duplicate section name '{name}'\n") + # Treat as a new paragraph - add a blank line + self.sections[name] += '\n' + contents + else: + self.sections[name] = contents + self.section_start_lines[name] = self.new_start_line + self.new_start_line = 0 + +# self.config.log.debug("Section: %s : %s", name, pformat(vars(self))) + + if start_new: + self.section = SECTION_DEFAULT + self._contents = [] + + +class KernelDoc: + """ + Read a C language source or header FILE and extract embedded + documentation comments. + """ + + # Section names + + section_context = "Context" + section_return = "Return" + + undescribed = "-- undescribed --" + + def __init__(self, config, fname): + """Initialize internal variables""" + + self.fname = fname + self.config = config + + # Initial state for the state machines + self.state = state.NORMAL + + # Store entry currently being processed + self.entry = None + + # Place all potential outputs into an array + self.entries = [] + + # + # We need Python 3.7 for its "dicts remember the insertion + # order" guarantee + # + if sys.version_info.major == 3 and sys.version_info.minor < 7: + self.emit_msg(0, + 'Python 3.7 or later is required for correct results') + + def emit_msg(self, ln, msg, warning=True): + """Emit a message""" + + log_msg = f"{self.fname}:{ln} {msg}" + + if self.entry: + self.entry.emit_msg(log_msg, warning) + return + + if warning: + self.config.log.warning(log_msg) + else: + self.config.log.info(log_msg) + + def dump_section(self, start_new=True): + """ + Dumps section contents to arrays/hashes intended for that purpose. + """ + + if self.entry: + self.entry.dump_section(start_new) + + # TODO: rename it to store_declaration after removal of kernel-doc.pl + def output_declaration(self, dtype, name, **args): + """ + Stores the entry into an entry array. + + The actual output and output filters will be handled elsewhere + """ + + item = KdocItem(name, dtype, self.entry.declaration_start_line, **args) + item.warnings = self.entry.warnings + + # Drop empty sections + # TODO: improve empty sections logic to emit warnings + sections = self.entry.sections + for section in ["Description", "Return"]: + if section in sections and not sections[section].rstrip(): + del sections[section] + item.set_sections(sections, self.entry.section_start_lines) + item.set_params(self.entry.parameterlist, self.entry.parameterdescs, + self.entry.parametertypes, + self.entry.parameterdesc_start_lines) + self.entries.append(item) + + self.config.log.debug("Output: %s:%s = %s", dtype, name, pformat(args)) + + def reset_state(self, ln): + """ + Ancillary routine to create a new entry. It initializes all + variables used by the state machine. + """ + + self.entry = KernelEntry(self.config, ln) + + # State flags + self.state = state.NORMAL + + def push_parameter(self, ln, decl_type, param, dtype, + org_arg, declaration_name): + """ + Store parameters and their descriptions at self.entry. + """ + + if self.entry.anon_struct_union and dtype == "" and param == "}": + return # Ignore the ending }; from anonymous struct/union + + self.entry.anon_struct_union = False + + param = KernRe(r'[\[\)].*').sub('', param, count=1) + + if dtype == "" and param.endswith("..."): + if KernRe(r'\w\.\.\.$').search(param): + # For named variable parameters of the form `x...`, + # remove the dots + param = param[:-3] + else: + # Handles unnamed variable parameters + param = "..." + + if param not in self.entry.parameterdescs or \ + not self.entry.parameterdescs[param]: + + self.entry.parameterdescs[param] = "variable arguments" + + elif dtype == "" and (not param or param == "void"): + param = "void" + self.entry.parameterdescs[param] = "no arguments" + + elif dtype == "" and param in ["struct", "union"]: + # Handle unnamed (anonymous) union or struct + dtype = param + param = "{unnamed_" + param + "}" + self.entry.parameterdescs[param] = "anonymous\n" + self.entry.anon_struct_union = True + + # Handle cache group enforcing variables: they do not need + # to be described in header files + elif "__cacheline_group" in param: + # Ignore __cacheline_group_begin and __cacheline_group_end + return + + # Warn if parameter has no description + # (but ignore ones starting with # as these are not parameters + # but inline preprocessor statements) + if param not in self.entry.parameterdescs and not param.startswith("#"): + self.entry.parameterdescs[param] = self.undescribed + + if "." not in param: + if decl_type == 'function': + dname = f"{decl_type} parameter" + else: + dname = f"{decl_type} member" + + self.emit_msg(ln, + f"{dname} '{param}' not described in '{declaration_name}'") + + # Strip spaces from param so that it is one continuous string on + # parameterlist. This fixes a problem where check_sections() + # cannot find a parameter like "addr[6 + 2]" because it actually + # appears as "addr[6", "+", "2]" on the parameter list. + # However, it's better to maintain the param string unchanged for + # output, so just weaken the string compare in check_sections() + # to ignore "[blah" in a parameter string. + + self.entry.parameterlist.append(param) + org_arg = KernRe(r'\s\s+').sub(' ', org_arg) + self.entry.parametertypes[param] = org_arg + + + def create_parameter_list(self, ln, decl_type, args, + splitter, declaration_name): + """ + Creates a list of parameters, storing them at self.entry. + """ + + # temporarily replace all commas inside function pointer definition + arg_expr = KernRe(r'(\([^\),]+),') + while arg_expr.search(args): + args = arg_expr.sub(r"\1#", args) + + for arg in args.split(splitter): + # Strip comments + arg = KernRe(r'\/\*.*\*\/').sub('', arg) + + # Ignore argument attributes + arg = KernRe(r'\sPOS0?\s').sub(' ', arg) + + # Strip leading/trailing spaces + arg = arg.strip() + arg = KernRe(r'\s+').sub(' ', arg, count=1) + + if arg.startswith('#'): + # Treat preprocessor directive as a typeless variable just to fill + # corresponding data structures "correctly". Catch it later in + # output_* subs. + + # Treat preprocessor directive as a typeless variable + self.push_parameter(ln, decl_type, arg, "", + "", declaration_name) + + elif KernRe(r'\(.+\)\s*\(').search(arg): + # Pointer-to-function + + arg = arg.replace('#', ',') + + r = KernRe(r'[^\(]+\(\*?\s*([\w\[\]\.]*)\s*\)') + if r.match(arg): + param = r.group(1) + else: + self.emit_msg(ln, f"Invalid param: {arg}") + param = arg + + dtype = KernRe(r'([^\(]+\(\*?)\s*' + re.escape(param)).sub(r'\1', arg) + self.push_parameter(ln, decl_type, param, dtype, + arg, declaration_name) + + elif KernRe(r'\(.+\)\s*\[').search(arg): + # Array-of-pointers + + arg = arg.replace('#', ',') + r = KernRe(r'[^\(]+\(\s*\*\s*([\w\[\]\.]*?)\s*(\s*\[\s*[\w]+\s*\]\s*)*\)') + if r.match(arg): + param = r.group(1) + else: + self.emit_msg(ln, f"Invalid param: {arg}") + param = arg + + dtype = KernRe(r'([^\(]+\(\*?)\s*' + re.escape(param)).sub(r'\1', arg) + + self.push_parameter(ln, decl_type, param, dtype, + arg, declaration_name) + + elif arg: + arg = KernRe(r'\s*:\s*').sub(":", arg) + arg = KernRe(r'\s*\[').sub('[', arg) + + args = KernRe(r'\s*,\s*').split(arg) + if args[0] and '*' in args[0]: + args[0] = re.sub(r'(\*+)\s*', r' \1', args[0]) + + first_arg = [] + r = KernRe(r'^(.*\s+)(.*?\[.*\].*)$') + if args[0] and r.match(args[0]): + args.pop(0) + first_arg.extend(r.group(1)) + first_arg.append(r.group(2)) + else: + first_arg = KernRe(r'\s+').split(args.pop(0)) + + args.insert(0, first_arg.pop()) + dtype = ' '.join(first_arg) + + for param in args: + if KernRe(r'^(\*+)\s*(.*)').match(param): + r = KernRe(r'^(\*+)\s*(.*)') + if not r.match(param): + self.emit_msg(ln, f"Invalid param: {param}") + continue + + param = r.group(1) + + self.push_parameter(ln, decl_type, r.group(2), + f"{dtype} {r.group(1)}", + arg, declaration_name) + + elif KernRe(r'(.*?):(\w+)').search(param): + r = KernRe(r'(.*?):(\w+)') + if not r.match(param): + self.emit_msg(ln, f"Invalid param: {param}") + continue + + if dtype != "": # Skip unnamed bit-fields + self.push_parameter(ln, decl_type, r.group(1), + f"{dtype}:{r.group(2)}", + arg, declaration_name) + else: + self.push_parameter(ln, decl_type, param, dtype, + arg, declaration_name) + + def check_sections(self, ln, decl_name, decl_type): + """ + Check for errors inside sections, emitting warnings if not found + parameters are described. + """ + for section in self.entry.sections: + if section not in self.entry.parameterlist and \ + not known_sections.search(section): + if decl_type == 'function': + dname = f"{decl_type} parameter" + else: + dname = f"{decl_type} member" + self.emit_msg(ln, + f"Excess {dname} '{section}' description in '{decl_name}'") + + def check_return_section(self, ln, declaration_name, return_type): + """ + If the function doesn't return void, warns about the lack of a + return description. + """ + + if not self.config.wreturn: + return + + # Ignore an empty return type (It's a macro) + # Ignore functions with a "void" return type (but not "void *") + if not return_type or KernRe(r'void\s*\w*\s*$').search(return_type): + return + + if not self.entry.sections.get("Return", None): + self.emit_msg(ln, + f"No description found for return value of '{declaration_name}'") + + def dump_struct(self, ln, proto): + """ + Store an entry for an struct or union + """ + + type_pattern = r'(struct|union)' + + qualifiers = [ + "__attribute__", + "__packed", + "__aligned", + "____cacheline_aligned_in_smp", + "____cacheline_aligned", + ] + + definition_body = r'\{(.*)\}\s*' + "(?:" + '|'.join(qualifiers) + ")?" + struct_members = KernRe(type_pattern + r'([^\{\};]+)(\{)([^\{\}]*)(\})([^\{\}\;]*)(\;)') + + # Extract struct/union definition + members = None + declaration_name = None + decl_type = None + + r = KernRe(type_pattern + r'\s+(\w+)\s*' + definition_body) + if r.search(proto): + decl_type = r.group(1) + declaration_name = r.group(2) + members = r.group(3) + else: + r = KernRe(r'typedef\s+' + type_pattern + r'\s*' + definition_body + r'\s*(\w+)\s*;') + + if r.search(proto): + decl_type = r.group(1) + declaration_name = r.group(3) + members = r.group(2) + + if not members: + self.emit_msg(ln, f"{proto} error: Cannot parse struct or union!") + return + + if self.entry.identifier != declaration_name: + self.emit_msg(ln, + f"expecting prototype for {decl_type} {self.entry.identifier}. Prototype was for {decl_type} {declaration_name} instead\n") + return + + args_pattern = r'([^,)]+)' + + sub_prefixes = [ + (KernRe(r'\/\*\s*private:.*?\/\*\s*public:.*?\*\/', re.S | re.I), ''), + (KernRe(r'\/\*\s*private:.*', re.S | re.I), ''), + + # Strip comments + (KernRe(r'\/\*.*?\*\/', re.S), ''), + + # Strip attributes + (attribute, ' '), + (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), + (KernRe(r'\s*__counted_by\s*\([^;]*\)', re.S), ' '), + (KernRe(r'\s*__counted_by_(le|be)\s*\([^;]*\)', re.S), ' '), + (KernRe(r'\s*__packed\s*', re.S), ' '), + (KernRe(r'\s*CRYPTO_MINALIGN_ATTR', re.S), ' '), + (KernRe(r'\s*____cacheline_aligned_in_smp', re.S), ' '), + (KernRe(r'\s*____cacheline_aligned', re.S), ' '), + + # Unwrap struct_group macros based on this definition: + # __struct_group(TAG, NAME, ATTRS, MEMBERS...) + # which has variants like: struct_group(NAME, MEMBERS...) + # Only MEMBERS arguments require documentation. + # + # Parsing them happens on two steps: + # + # 1. drop struct group arguments that aren't at MEMBERS, + # storing them as STRUCT_GROUP(MEMBERS) + # + # 2. remove STRUCT_GROUP() ancillary macro. + # + # The original logic used to remove STRUCT_GROUP() using an + # advanced regex: + # + # \bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*; + # + # with two patterns that are incompatible with + # Python re module, as it has: + # + # - a recursive pattern: (?1) + # - an atomic grouping: (?>...) + # + # I tried a simpler version: but it didn't work either: + # \bSTRUCT_GROUP\(([^\)]+)\)[^;]*; + # + # As it doesn't properly match the end parenthesis on some cases. + # + # So, a better solution was crafted: there's now a NestedMatch + # class that ensures that delimiters after a search are properly + # matched. So, the implementation to drop STRUCT_GROUP() will be + # handled in separate. + + (KernRe(r'\bstruct_group\s*\(([^,]*,)', re.S), r'STRUCT_GROUP('), + (KernRe(r'\bstruct_group_attr\s*\(([^,]*,){2}', re.S), r'STRUCT_GROUP('), + (KernRe(r'\bstruct_group_tagged\s*\(([^,]*),([^,]*),', re.S), r'struct \1 \2; STRUCT_GROUP('), + (KernRe(r'\b__struct_group\s*\(([^,]*,){3}', re.S), r'STRUCT_GROUP('), + + # Replace macros + # + # TODO: use NestedMatch for FOO($1, $2, ...) matches + # + # it is better to also move those to the NestedMatch logic, + # to ensure that parenthesis will be properly matched. + + (KernRe(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), + (KernRe(r'DECLARE_PHY_INTERFACE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'), + (KernRe(r'DECLARE_BITMAP\s*\(' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'unsigned long \1[BITS_TO_LONGS(\2)]'), + (KernRe(r'DECLARE_HASHTABLE\s*\(' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'unsigned long \1[1 << ((\2) - 1)]'), + (KernRe(r'DECLARE_KFIFO\s*\(' + args_pattern + r',\s*' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'\2 *\1'), + (KernRe(r'DECLARE_KFIFO_PTR\s*\(' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'\2 *\1'), + (KernRe(r'(?:__)?DECLARE_FLEX_ARRAY\s*\(' + args_pattern + r',\s*' + args_pattern + r'\)', re.S), r'\1 \2[]'), + (KernRe(r'DEFINE_DMA_UNMAP_ADDR\s*\(' + args_pattern + r'\)', re.S), r'dma_addr_t \1'), + (KernRe(r'DEFINE_DMA_UNMAP_LEN\s*\(' + args_pattern + r'\)', re.S), r'__u32 \1'), + (KernRe(r'VIRTIO_DECLARE_FEATURES\s*\(' + args_pattern + r'\)', re.S), r'u64 \1; u64 \1_array[VIRTIO_FEATURES_DWORDS]'), + ] + + # Regexes here are guaranteed to have the end limiter matching + # the start delimiter. Yet, right now, only one replace group + # is allowed. + + sub_nested_prefixes = [ + (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), + ] + + for search, sub in sub_prefixes: + members = search.sub(sub, members) + + nested = NestedMatch() + + for search, sub in sub_nested_prefixes: + members = nested.sub(search, sub, members) + + # Keeps the original declaration as-is + declaration = members + + # Split nested struct/union elements + # + # This loop was simpler at the original kernel-doc perl version, as + # while ($members =~ m/$struct_members/) { ... } + # reads 'members' string on each interaction. + # + # Python behavior is different: it parses 'members' only once, + # creating a list of tuples from the first interaction. + # + # On other words, this won't get nested structs. + # + # So, we need to have an extra loop on Python to override such + # re limitation. + + while True: + tuples = struct_members.findall(members) + if not tuples: + break + + for t in tuples: + newmember = "" + maintype = t[0] + s_ids = t[5] + content = t[3] + + oldmember = "".join(t) + + for s_id in s_ids.split(','): + s_id = s_id.strip() + + newmember += f"{maintype} {s_id}; " + s_id = KernRe(r'[:\[].*').sub('', s_id) + s_id = KernRe(r'^\s*\**(\S+)\s*').sub(r'\1', s_id) + + for arg in content.split(';'): + arg = arg.strip() + + if not arg: + continue + + r = KernRe(r'^([^\(]+\(\*?\s*)([\w\.]*)(\s*\).*)') + if r.match(arg): + # Pointer-to-function + dtype = r.group(1) + name = r.group(2) + extra = r.group(3) + + if not name: + continue + + if not s_id: + # Anonymous struct/union + newmember += f"{dtype}{name}{extra}; " + else: + newmember += f"{dtype}{s_id}.{name}{extra}; " + + else: + arg = arg.strip() + # Handle bitmaps + arg = KernRe(r':\s*\d+\s*').sub('', arg) + + # Handle arrays + arg = KernRe(r'\[.*\]').sub('', arg) + + # Handle multiple IDs + arg = KernRe(r'\s*,\s*').sub(',', arg) + + r = KernRe(r'(.*)\s+([\S+,]+)') + + if r.search(arg): + dtype = r.group(1) + names = r.group(2) + else: + newmember += f"{arg}; " + continue + + for name in names.split(','): + name = KernRe(r'^\s*\**(\S+)\s*').sub(r'\1', name).strip() + + if not name: + continue + + if not s_id: + # Anonymous struct/union + newmember += f"{dtype} {name}; " + else: + newmember += f"{dtype} {s_id}.{name}; " + + members = members.replace(oldmember, newmember) + + # Ignore other nested elements, like enums + members = re.sub(r'(\{[^\{\}]*\})', '', members) + + self.create_parameter_list(ln, decl_type, members, ';', + declaration_name) + self.check_sections(ln, declaration_name, decl_type) + + # Adjust declaration for better display + declaration = KernRe(r'([\{;])').sub(r'\1\n', declaration) + declaration = KernRe(r'\}\s+;').sub('};', declaration) + + # Better handle inlined enums + while True: + r = KernRe(r'(enum\s+\{[^\}]+),([^\n])') + if not r.search(declaration): + break + + declaration = r.sub(r'\1,\n\2', declaration) + + def_args = declaration.split('\n') + level = 1 + declaration = "" + for clause in def_args: + + clause = clause.strip() + clause = KernRe(r'\s+').sub(' ', clause, count=1) + + if not clause: + continue + + if '}' in clause and level > 1: + level -= 1 + + if not KernRe(r'^\s*#').match(clause): + declaration += "\t" * level + + declaration += "\t" + clause + "\n" + if "{" in clause and "}" not in clause: + level += 1 + + self.output_declaration(decl_type, declaration_name, + definition=declaration, + purpose=self.entry.declaration_purpose) + + def dump_enum(self, ln, proto): + """ + Stores an enum inside self.entries array. + """ + + # Ignore members marked private + proto = KernRe(r'\/\*\s*private:.*?\/\*\s*public:.*?\*\/', flags=re.S).sub('', proto) + proto = KernRe(r'\/\*\s*private:.*}', flags=re.S).sub('}', proto) + + # Strip comments + proto = KernRe(r'\/\*.*?\*\/', flags=re.S).sub('', proto) + + # Strip #define macros inside enums + proto = KernRe(r'#\s*((define|ifdef|if)\s+|endif)[^;]*;', flags=re.S).sub('', proto) + + # + # Parse out the name and members of the enum. Typedef form first. + # + r = KernRe(r'typedef\s+enum\s*\{(.*)\}\s*(\w*)\s*;') + if r.search(proto): + declaration_name = r.group(2) + members = r.group(1).rstrip() + # + # Failing that, look for a straight enum + # + else: + r = KernRe(r'enum\s+(\w*)\s*\{(.*)\}') + if r.match(proto): + declaration_name = r.group(1) + members = r.group(2).rstrip() + # + # OK, this isn't going to work. + # + else: + self.emit_msg(ln, f"{proto}: error: Cannot parse enum!") + return + # + # Make sure we found what we were expecting. + # + if self.entry.identifier != declaration_name: + if self.entry.identifier == "": + self.emit_msg(ln, + f"{proto}: wrong kernel-doc identifier on prototype") + else: + self.emit_msg(ln, + f"expecting prototype for enum {self.entry.identifier}. " + f"Prototype was for enum {declaration_name} instead") + return + + if not declaration_name: + declaration_name = "(anonymous)" + # + # Parse out the name of each enum member, and verify that we + # have a description for it. + # + member_set = set() + members = KernRe(r'\([^;)]*\)').sub('', members) + for arg in members.split(','): + if not arg: + continue + arg = KernRe(r'^\s*(\w+).*').sub(r'\1', arg) + self.entry.parameterlist.append(arg) + if arg not in self.entry.parameterdescs: + self.entry.parameterdescs[arg] = self.undescribed + self.emit_msg(ln, + f"Enum value '{arg}' not described in enum '{declaration_name}'") + member_set.add(arg) + # + # Ensure that every described member actually exists in the enum. + # + for k in self.entry.parameterdescs: + if k not in member_set: + self.emit_msg(ln, + f"Excess enum value '%{k}' description in '{declaration_name}'") + + self.output_declaration('enum', declaration_name, + purpose=self.entry.declaration_purpose) + + def dump_declaration(self, ln, prototype): + """ + Stores a data declaration inside self.entries array. + """ + + if self.entry.decl_type == "enum": + self.dump_enum(ln, prototype) + elif self.entry.decl_type == "typedef": + self.dump_typedef(ln, prototype) + elif self.entry.decl_type in ["union", "struct"]: + self.dump_struct(ln, prototype) + else: + # This would be a bug + self.emit_message(ln, f'Unknown declaration type: {self.entry.decl_type}') + + def dump_function(self, ln, prototype): + """ + Stores a function of function macro inside self.entries array. + """ + + func_macro = False + return_type = '' + decl_type = 'function' + + # Prefixes that would be removed + sub_prefixes = [ + (r"^static +", "", 0), + (r"^extern +", "", 0), + (r"^asmlinkage +", "", 0), + (r"^inline +", "", 0), + (r"^__inline__ +", "", 0), + (r"^__inline +", "", 0), + (r"^__always_inline +", "", 0), + (r"^noinline +", "", 0), + (r"^__FORTIFY_INLINE +", "", 0), + (r"QEMU_[A-Z_]+ +", "", 0), + (r"__init +", "", 0), + (r"__init_or_module +", "", 0), + (r"__deprecated +", "", 0), + (r"__flatten +", "", 0), + (r"__meminit +", "", 0), + (r"__must_check +", "", 0), + (r"__weak +", "", 0), + (r"__sched +", "", 0), + (r"_noprof", "", 0), + (r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +", "", 0), + (r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +", "", 0), + (r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +", "", 0), + (r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)", r"\1, \2", 0), + (r"__attribute_const__ +", "", 0), + + # It seems that Python support for re.X is broken: + # At least for me (Python 3.13), this didn't work +# (r""" +# __attribute__\s*\(\( +# (?: +# [\w\s]+ # attribute name +# (?:\([^)]*\))? # attribute arguments +# \s*,? # optional comma at the end +# )+ +# \)\)\s+ +# """, "", re.X), + + # So, remove whitespaces and comments from it + (r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+", "", 0), + ] + + for search, sub, flags in sub_prefixes: + prototype = KernRe(search, flags).sub(sub, prototype) + + # Macros are a special case, as they change the prototype format + new_proto = KernRe(r"^#\s*define\s+").sub("", prototype) + if new_proto != prototype: + is_define_proto = True + prototype = new_proto + else: + is_define_proto = False + + # Yes, this truly is vile. We are looking for: + # 1. Return type (may be nothing if we're looking at a macro) + # 2. Function name + # 3. Function parameters. + # + # All the while we have to watch out for function pointer parameters + # (which IIRC is what the two sections are for), C types (these + # regexps don't even start to express all the possibilities), and + # so on. + # + # If you mess with these regexps, it's a good idea to check that + # the following functions' documentation still comes out right: + # - parport_register_device (function pointer parameters) + # - atomic_set (macro) + # - pci_match_device, __copy_to_user (long return type) + + name = r'[a-zA-Z0-9_~:]+' + prototype_end1 = r'[^\(]*' + prototype_end2 = r'[^\{]*' + prototype_end = fr'\(({prototype_end1}|{prototype_end2})\)' + + # Besides compiling, Perl qr{[\w\s]+} works as a non-capturing group. + # So, this needs to be mapped in Python with (?:...)? or (?:...)+ + + type1 = r'(?:[\w\s]+)?' + type2 = r'(?:[\w\s]+\*+)+' + + found = False + + if is_define_proto: + r = KernRe(r'^()(' + name + r')\s+') + + if r.search(prototype): + return_type = '' + declaration_name = r.group(2) + func_macro = True + + found = True + + if not found: + patterns = [ + rf'^()({name})\s*{prototype_end}', + rf'^({type1})\s+({name})\s*{prototype_end}', + rf'^({type2})\s*({name})\s*{prototype_end}', + ] + + for p in patterns: + r = KernRe(p) + + if r.match(prototype): + + return_type = r.group(1) + declaration_name = r.group(2) + args = r.group(3) + + self.create_parameter_list(ln, decl_type, args, ',', + declaration_name) + + found = True + break + if not found: + self.emit_msg(ln, + f"cannot understand function prototype: '{prototype}'") + return + + if self.entry.identifier != declaration_name: + self.emit_msg(ln, + f"expecting prototype for {self.entry.identifier}(). Prototype was for {declaration_name}() instead") + return + + self.check_sections(ln, declaration_name, "function") + + self.check_return_section(ln, declaration_name, return_type) + + if 'typedef' in return_type: + self.output_declaration(decl_type, declaration_name, + typedef=True, + functiontype=return_type, + purpose=self.entry.declaration_purpose, + func_macro=func_macro) + else: + self.output_declaration(decl_type, declaration_name, + typedef=False, + functiontype=return_type, + purpose=self.entry.declaration_purpose, + func_macro=func_macro) + + def dump_typedef(self, ln, proto): + """ + Stores a typedef inside self.entries array. + """ + + typedef_type = r'((?:\s+[\w\*]+\b){0,7}\s+(?:\w+\b|\*+))\s*' + typedef_ident = r'\*?\s*(\w\S+)\s*' + typedef_args = r'\s*\((.*)\);' + + typedef1 = KernRe(r'typedef' + typedef_type + r'\(' + typedef_ident + r'\)' + typedef_args) + typedef2 = KernRe(r'typedef' + typedef_type + typedef_ident + typedef_args) + + # Strip comments + proto = KernRe(r'/\*.*?\*/', flags=re.S).sub('', proto) + + # Parse function typedef prototypes + for r in [typedef1, typedef2]: + if not r.match(proto): + continue + + return_type = r.group(1).strip() + declaration_name = r.group(2) + args = r.group(3) + + if self.entry.identifier != declaration_name: + self.emit_msg(ln, + f"expecting prototype for typedef {self.entry.identifier}. Prototype was for typedef {declaration_name} instead\n") + return + + decl_type = 'function' + self.create_parameter_list(ln, decl_type, args, ',', declaration_name) + + self.output_declaration(decl_type, declaration_name, + typedef=True, + functiontype=return_type, + purpose=self.entry.declaration_purpose) + return + + # Handle nested parentheses or brackets + r = KernRe(r'(\(*.\)\s*|\[*.\]\s*);$') + while r.search(proto): + proto = r.sub('', proto) + + # Parse simple typedefs + r = KernRe(r'typedef.*\s+(\w+)\s*;') + if r.match(proto): + declaration_name = r.group(1) + + if self.entry.identifier != declaration_name: + self.emit_msg(ln, + f"expecting prototype for typedef {self.entry.identifier}. Prototype was for typedef {declaration_name} instead\n") + return + + self.output_declaration('typedef', declaration_name, + purpose=self.entry.declaration_purpose) + return + + self.emit_msg(ln, "error: Cannot parse typedef!") + + @staticmethod + def process_export(function_set, line): + """ + process EXPORT_SYMBOL* tags + + This method doesn't use any variable from the class, so declare it + with a staticmethod decorator. + """ + + # We support documenting some exported symbols with different + # names. A horrible hack. + suffixes = [ '_noprof' ] + + # Note: it accepts only one EXPORT_SYMBOL* per line, as having + # multiple export lines would violate Kernel coding style. + + if export_symbol.search(line): + symbol = export_symbol.group(2) + elif export_symbol_ns.search(line): + symbol = export_symbol_ns.group(2) + else: + return False + # + # Found an export, trim out any special suffixes + # + for suffix in suffixes: + # Be backward compatible with Python < 3.9 + if symbol.endswith(suffix): + symbol = symbol[:-len(suffix)] + function_set.add(symbol) + return True + + def process_normal(self, ln, line): + """ + STATE_NORMAL: looking for the /** to begin everything. + """ + + if not doc_start.match(line): + return + + # start a new entry + self.reset_state(ln) + + # next line is always the function name + self.state = state.NAME + + def process_name(self, ln, line): + """ + STATE_NAME: Looking for the "name - description" line + """ + # + # Check for a DOC: block and handle them specially. + # + if doc_block.search(line): + + if not doc_block.group(1): + self.entry.begin_section(ln, "Introduction") + else: + self.entry.begin_section(ln, doc_block.group(1)) + + self.entry.identifier = self.entry.section + self.state = state.DOCBLOCK + # + # Otherwise we're looking for a normal kerneldoc declaration line. + # + elif doc_decl.search(line): + self.entry.identifier = doc_decl.group(1) + + # Test for data declaration + if doc_begin_data.search(line): + self.entry.decl_type = doc_begin_data.group(1) + self.entry.identifier = doc_begin_data.group(2) + # + # Look for a function description + # + elif doc_begin_func.search(line): + self.entry.identifier = doc_begin_func.group(1) + self.entry.decl_type = "function" + # + # We struck out. + # + else: + self.emit_msg(ln, + f"This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst\n{line}") + self.state = state.NORMAL + return + # + # OK, set up for a new kerneldoc entry. + # + self.state = state.BODY + self.entry.identifier = self.entry.identifier.strip(" ") + # if there's no @param blocks need to set up default section here + self.entry.begin_section(ln + 1) + # + # Find the description portion, which *should* be there but + # isn't always. + # (We should be able to capture this from the previous parsing - someday) + # + r = KernRe("[-:](.*)") + if r.search(line): + self.entry.declaration_purpose = trim_whitespace(r.group(1)) + self.state = state.DECLARATION + else: + self.entry.declaration_purpose = "" + + if not self.entry.declaration_purpose and self.config.wshort_desc: + self.emit_msg(ln, + f"missing initial short description on line:\n{line}") + + if not self.entry.identifier and self.entry.decl_type != "enum": + self.emit_msg(ln, + f"wrong kernel-doc identifier on line:\n{line}") + self.state = state.NORMAL + + if self.config.verbose: + self.emit_msg(ln, + f"Scanning doc for {self.entry.decl_type} {self.entry.identifier}", + warning=False) + # + # Failed to find an identifier. Emit a warning + # + else: + self.emit_msg(ln, f"Cannot find identifier on line:\n{line}") + + # + # Helper function to determine if a new section is being started. + # + def is_new_section(self, ln, line): + if doc_sect.search(line): + self.state = state.BODY + # + # Pick out the name of our new section, tweaking it if need be. + # + newsection = doc_sect.group(1) + if newsection.lower() == 'description': + newsection = 'Description' + elif newsection.lower() == 'context': + newsection = 'Context' + self.state = state.SPECIAL_SECTION + elif newsection.lower() in ["@return", "@returns", + "return", "returns"]: + newsection = "Return" + self.state = state.SPECIAL_SECTION + elif newsection[0] == '@': + self.state = state.SPECIAL_SECTION + # + # Initialize the contents, and get the new section going. + # + newcontents = doc_sect.group(2) + if not newcontents: + newcontents = "" + self.dump_section() + self.entry.begin_section(ln, newsection) + self.entry.leading_space = None + + self.entry.add_text(newcontents.lstrip()) + return True + return False + + # + # Helper function to detect (and effect) the end of a kerneldoc comment. + # + def is_comment_end(self, ln, line): + if doc_end.search(line): + self.dump_section() + + # Look for doc_com + <text> + doc_end: + r = KernRe(r'\s*\*\s*[a-zA-Z_0-9:\.]+\*/') + if r.match(line): + self.emit_msg(ln, f"suspicious ending line: {line}") + + self.entry.prototype = "" + self.entry.new_start_line = ln + 1 + + self.state = state.PROTO + return True + return False + + + def process_decl(self, ln, line): + """ + STATE_DECLARATION: We've seen the beginning of a declaration + """ + if self.is_new_section(ln, line) or self.is_comment_end(ln, line): + return + # + # Look for anything with the " * " line beginning. + # + if doc_content.search(line): + cont = doc_content.group(1) + # + # A blank line means that we have moved out of the declaration + # part of the comment (without any "special section" parameter + # descriptions). + # + if cont == "": + self.state = state.BODY + # + # Otherwise we have more of the declaration section to soak up. + # + else: + self.entry.declaration_purpose = \ + trim_whitespace(self.entry.declaration_purpose + ' ' + cont) + else: + # Unknown line, ignore + self.emit_msg(ln, f"bad line: {line}") + + + def process_special(self, ln, line): + """ + STATE_SPECIAL_SECTION: a section ending with a blank line + """ + # + # If we have hit a blank line (only the " * " marker), then this + # section is done. + # + if KernRe(r"\s*\*\s*$").match(line): + self.entry.begin_section(ln, dump = True) + self.state = state.BODY + return + # + # Not a blank line, look for the other ways to end the section. + # + if self.is_new_section(ln, line) or self.is_comment_end(ln, line): + return + # + # OK, we should have a continuation of the text for this section. + # + if doc_content.search(line): + cont = doc_content.group(1) + # + # If the lines of text after the first in a special section have + # leading white space, we need to trim it out or Sphinx will get + # confused. For the second line (the None case), see what we + # find there and remember it. + # + if self.entry.leading_space is None: + r = KernRe(r'^(\s+)') + if r.match(cont): + self.entry.leading_space = len(r.group(1)) + else: + self.entry.leading_space = 0 + # + # Otherwise, before trimming any leading chars, be *sure* + # that they are white space. We should maybe warn if this + # isn't the case. + # + for i in range(0, self.entry.leading_space): + if cont[i] != " ": + self.entry.leading_space = i + break + # + # Add the trimmed result to the section and we're done. + # + self.entry.add_text(cont[self.entry.leading_space:]) + else: + # Unknown line, ignore + self.emit_msg(ln, f"bad line: {line}") + + def process_body(self, ln, line): + """ + STATE_BODY: the bulk of a kerneldoc comment. + """ + if self.is_new_section(ln, line) or self.is_comment_end(ln, line): + return + + if doc_content.search(line): + cont = doc_content.group(1) + self.entry.add_text(cont) + else: + # Unknown line, ignore + self.emit_msg(ln, f"bad line: {line}") + + def process_inline_name(self, ln, line): + """STATE_INLINE_NAME: beginning of docbook comments within a prototype.""" + + if doc_inline_sect.search(line): + self.entry.begin_section(ln, doc_inline_sect.group(1)) + self.entry.add_text(doc_inline_sect.group(2).lstrip()) + self.state = state.INLINE_TEXT + elif doc_inline_end.search(line): + self.dump_section() + self.state = state.PROTO + elif doc_content.search(line): + self.emit_msg(ln, f"Incorrect use of kernel-doc format: {line}") + self.state = state.PROTO + # else ... ?? + + def process_inline_text(self, ln, line): + """STATE_INLINE_TEXT: docbook comments within a prototype.""" + + if doc_inline_end.search(line): + self.dump_section() + self.state = state.PROTO + elif doc_content.search(line): + self.entry.add_text(doc_content.group(1)) + # else ... ?? + + def syscall_munge(self, ln, proto): # pylint: disable=W0613 + """ + Handle syscall definitions + """ + + is_void = False + + # Strip newlines/CR's + proto = re.sub(r'[\r\n]+', ' ', proto) + + # Check if it's a SYSCALL_DEFINE0 + if 'SYSCALL_DEFINE0' in proto: + is_void = True + + # Replace SYSCALL_DEFINE with correct return type & function name + proto = KernRe(r'SYSCALL_DEFINE.*\(').sub('long sys_', proto) + + r = KernRe(r'long\s+(sys_.*?),') + if r.search(proto): + proto = KernRe(',').sub('(', proto, count=1) + elif is_void: + proto = KernRe(r'\)').sub('(void)', proto, count=1) + + # Now delete all of the odd-numbered commas in the proto + # so that argument types & names don't have a comma between them + count = 0 + length = len(proto) + + if is_void: + length = 0 # skip the loop if is_void + + for ix in range(length): + if proto[ix] == ',': + count += 1 + if count % 2 == 1: + proto = proto[:ix] + ' ' + proto[ix + 1:] + + return proto + + def tracepoint_munge(self, ln, proto): + """ + Handle tracepoint definitions + """ + + tracepointname = None + tracepointargs = None + + # Match tracepoint name based on different patterns + r = KernRe(r'TRACE_EVENT\((.*?),') + if r.search(proto): + tracepointname = r.group(1) + + r = KernRe(r'DEFINE_SINGLE_EVENT\((.*?),') + if r.search(proto): + tracepointname = r.group(1) + + r = KernRe(r'DEFINE_EVENT\((.*?),(.*?),') + if r.search(proto): + tracepointname = r.group(2) + + if tracepointname: + tracepointname = tracepointname.lstrip() + + r = KernRe(r'TP_PROTO\((.*?)\)') + if r.search(proto): + tracepointargs = r.group(1) + + if not tracepointname or not tracepointargs: + self.emit_msg(ln, + f"Unrecognized tracepoint format:\n{proto}\n") + else: + proto = f"static inline void trace_{tracepointname}({tracepointargs})" + self.entry.identifier = f"trace_{self.entry.identifier}" + + return proto + + def process_proto_function(self, ln, line): + """Ancillary routine to process a function prototype""" + + # strip C99-style comments to end of line + line = KernRe(r"\/\/.*$", re.S).sub('', line) + # + # Soak up the line's worth of prototype text, stopping at { or ; if present. + # + if KernRe(r'\s*#\s*define').match(line): + self.entry.prototype = line + elif not line.startswith('#'): # skip other preprocessor stuff + r = KernRe(r'([^\{]*)') + if r.match(line): + self.entry.prototype += r.group(1) + " " + # + # If we now have the whole prototype, clean it up and declare victory. + # + if '{' in line or ';' in line or KernRe(r'\s*#\s*define').match(line): + # strip comments and surrounding spaces + self.entry.prototype = KernRe(r'/\*.*\*/').sub('', self.entry.prototype).strip() + # + # Handle self.entry.prototypes for function pointers like: + # int (*pcs_config)(struct foo) + # by turning it into + # int pcs_config(struct foo) + # + r = KernRe(r'^(\S+\s+)\(\s*\*(\S+)\)') + self.entry.prototype = r.sub(r'\1\2', self.entry.prototype) + # + # Handle special declaration syntaxes + # + if 'SYSCALL_DEFINE' in self.entry.prototype: + self.entry.prototype = self.syscall_munge(ln, + self.entry.prototype) + else: + r = KernRe(r'TRACE_EVENT|DEFINE_EVENT|DEFINE_SINGLE_EVENT') + if r.search(self.entry.prototype): + self.entry.prototype = self.tracepoint_munge(ln, + self.entry.prototype) + # + # ... and we're done + # + self.dump_function(ln, self.entry.prototype) + self.reset_state(ln) + + def process_proto_type(self, ln, line): + """Ancillary routine to process a type""" + + # Strip C99-style comments and surrounding whitespace + line = KernRe(r"//.*$", re.S).sub('', line).strip() + if not line: + return # nothing to see here + + # To distinguish preprocessor directive from regular declaration later. + if line.startswith('#'): + line += ";" + # + # Split the declaration on any of { } or ;, and accumulate pieces + # until we hit a semicolon while not inside {brackets} + # + r = KernRe(r'(.*?)([{};])') + for chunk in r.split(line): + if chunk: # Ignore empty matches + self.entry.prototype += chunk + # + # This cries out for a match statement ... someday after we can + # drop Python 3.9 ... + # + if chunk == '{': + self.entry.brcount += 1 + elif chunk == '}': + self.entry.brcount -= 1 + elif chunk == ';' and self.entry.brcount <= 0: + self.dump_declaration(ln, self.entry.prototype) + self.reset_state(ln) + return + # + # We hit the end of the line while still in the declaration; put + # in a space to represent the newline. + # + self.entry.prototype += ' ' + + def process_proto(self, ln, line): + """STATE_PROTO: reading a function/whatever prototype.""" + + if doc_inline_oneline.search(line): + self.entry.begin_section(ln, doc_inline_oneline.group(1)) + self.entry.add_text(doc_inline_oneline.group(2)) + self.dump_section() + + elif doc_inline_start.search(line): + self.state = state.INLINE_NAME + + elif self.entry.decl_type == 'function': + self.process_proto_function(ln, line) + + else: + self.process_proto_type(ln, line) + + def process_docblock(self, ln, line): + """STATE_DOCBLOCK: within a DOC: block.""" + + if doc_end.search(line): + self.dump_section() + self.output_declaration("doc", self.entry.identifier) + self.reset_state(ln) + + elif doc_content.search(line): + self.entry.add_text(doc_content.group(1)) + + def parse_export(self): + """ + Parses EXPORT_SYMBOL* macros from a single Kernel source file. + """ + + export_table = set() + + try: + with open(self.fname, "r", encoding="utf8", + errors="backslashreplace") as fp: + + for line in fp: + self.process_export(export_table, line) + + except IOError: + return None + + return export_table + + # + # The state/action table telling us which function to invoke in + # each state. + # + state_actions = { + state.NORMAL: process_normal, + state.NAME: process_name, + state.BODY: process_body, + state.DECLARATION: process_decl, + state.SPECIAL_SECTION: process_special, + state.INLINE_NAME: process_inline_name, + state.INLINE_TEXT: process_inline_text, + state.PROTO: process_proto, + state.DOCBLOCK: process_docblock, + } + + def parse_kdoc(self): + """ + Open and process each line of a C source file. + The parsing is controlled via a state machine, and the line is passed + to a different process function depending on the state. The process + function may update the state as needed. + + Besides parsing kernel-doc tags, it also parses export symbols. + """ + + prev = "" + prev_ln = None + export_table = set() + + try: + with open(self.fname, "r", encoding="utf8", + errors="backslashreplace") as fp: + for ln, line in enumerate(fp): + + line = line.expandtabs().strip("\n") + + # Group continuation lines on prototypes + if self.state == state.PROTO: + if line.endswith("\\"): + prev += line.rstrip("\\") + if not prev_ln: + prev_ln = ln + continue + + if prev: + ln = prev_ln + line = prev + line + prev = "" + prev_ln = None + + self.config.log.debug("%d %s: %s", + ln, state.name[self.state], + line) + + # This is an optimization over the original script. + # There, when export_file was used for the same file, + # it was read twice. Here, we use the already-existing + # loop to parse exported symbols as well. + # + if (self.state != state.NORMAL) or \ + not self.process_export(export_table, line): + # Hand this line to the appropriate state handler + self.state_actions[self.state](self, ln, line) + + except OSError: + self.config.log.error(f"Error: Cannot open file {self.fname}") + + return export_table, self.entries diff --git a/scripts/lib/kdoc/kdoc_re.py b/scripts/lib/kdoc/kdoc_re.py new file mode 100644 index 0000000..612223e --- /dev/null +++ b/scripts/lib/kdoc/kdoc_re.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>. + +""" +Regular expression ancillary classes. + +Those help caching regular expressions and do matching for kernel-doc. +""" + +import re + +# Local cache for regular expressions +re_cache = {} + + +class KernRe: + """ + Helper class to simplify regex declaration and usage, + + It calls re.compile for a given pattern. It also allows adding + regular expressions and define sub at class init time. + + Regular expressions can be cached via an argument, helping to speedup + searches. + """ + + def _add_regex(self, string, flags): + """ + Adds a new regex or re-use it from the cache. + """ + self.regex = re_cache.get(string, None) + if not self.regex: + self.regex = re.compile(string, flags=flags) + if self.cache: + re_cache[string] = self.regex + + def __init__(self, string, cache=True, flags=0): + """ + Compile a regular expression and initialize internal vars. + """ + + self.cache = cache + self.last_match = None + + self._add_regex(string, flags) + + def __str__(self): + """ + Return the regular expression pattern. + """ + return self.regex.pattern + + def __add__(self, other): + """ + Allows adding two regular expressions into one. + """ + + return KernRe(str(self) + str(other), cache=self.cache or other.cache, + flags=self.regex.flags | other.regex.flags) + + def match(self, string): + """ + Handles a re.match storing its results + """ + + self.last_match = self.regex.match(string) + return self.last_match + + def search(self, string): + """ + Handles a re.search storing its results + """ + + self.last_match = self.regex.search(string) + return self.last_match + + def findall(self, string): + """ + Alias to re.findall + """ + + return self.regex.findall(string) + + def split(self, string): + """ + Alias to re.split + """ + + return self.regex.split(string) + + def sub(self, sub, string, count=0): + """ + Alias to re.sub + """ + + return self.regex.sub(sub, string, count=count) + + def group(self, num): + """ + Returns the group results of the last match + """ + + return self.last_match.group(num) + + +class NestedMatch: + """ + Finding nested delimiters is hard with regular expressions. It is + even harder on Python with its normal re module, as there are several + advanced regular expressions that are missing. + + This is the case of this pattern: + + '\\bSTRUCT_GROUP(\\(((?:(?>[^)(]+)|(?1))*)\\))[^;]*;' + + which is used to properly match open/close parenthesis of the + string search STRUCT_GROUP(), + + Add a class that counts pairs of delimiters, using it to match and + replace nested expressions. + + The original approach was suggested by: + https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex + + Although I re-implemented it to make it more generic and match 3 types + of delimiters. The logic checks if delimiters are paired. If not, it + will ignore the search string. + """ + + # TODO: make NestedMatch handle multiple match groups + # + # Right now, regular expressions to match it are defined only up to + # the start delimiter, e.g.: + # + # \bSTRUCT_GROUP\( + # + # is similar to: STRUCT_GROUP\((.*)\) + # except that the content inside the match group is delimiter's aligned. + # + # The content inside parenthesis are converted into a single replace + # group (e.g. r`\1'). + # + # It would be nice to change such definition to support multiple + # match groups, allowing a regex equivalent to. + # + # FOO\((.*), (.*), (.*)\) + # + # it is probably easier to define it not as a regular expression, but + # with some lexical definition like: + # + # FOO(arg1, arg2, arg3) + + DELIMITER_PAIRS = { + '{': '}', + '(': ')', + '[': ']', + } + + RE_DELIM = re.compile(r'[\{\}\[\]\(\)]') + + def _search(self, regex, line): + """ + Finds paired blocks for a regex that ends with a delimiter. + + The suggestion of using finditer to match pairs came from: + https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex + but I ended using a different implementation to align all three types + of delimiters and seek for an initial regular expression. + + The algorithm seeks for open/close paired delimiters and place them + into a stack, yielding a start/stop position of each match when the + stack is zeroed. + + The algorithm shoud work fine for properly paired lines, but will + silently ignore end delimiters that preceeds an start delimiter. + This should be OK for kernel-doc parser, as unaligned delimiters + would cause compilation errors. So, we don't need to rise exceptions + to cover such issues. + """ + + stack = [] + + for match_re in regex.finditer(line): + start = match_re.start() + offset = match_re.end() + + d = line[offset - 1] + if d not in self.DELIMITER_PAIRS: + continue + + end = self.DELIMITER_PAIRS[d] + stack.append(end) + + for match in self.RE_DELIM.finditer(line[offset:]): + pos = match.start() + offset + + d = line[pos] + + if d in self.DELIMITER_PAIRS: + end = self.DELIMITER_PAIRS[d] + + stack.append(end) + continue + + # Does the end delimiter match what it is expected? + if stack and d == stack[-1]: + stack.pop() + + if not stack: + yield start, offset, pos + 1 + break + + def search(self, regex, line): + """ + This is similar to re.search: + + It matches a regex that it is followed by a delimiter, + returning occurrences only if all delimiters are paired. + """ + + for t in self._search(regex, line): + + yield line[t[0]:t[2]] + + def sub(self, regex, sub, line, count=0): + """ + This is similar to re.sub: + + It matches a regex that it is followed by a delimiter, + replacing occurrences only if all delimiters are paired. + + if r'\1' is used, it works just like re: it places there the + matched paired data with the delimiter stripped. + + If count is different than zero, it will replace at most count + items. + """ + out = "" + + cur_pos = 0 + n = 0 + + for start, end, pos in self._search(regex, line): + out += line[cur_pos:start] + + # Value, ignoring start/end delimiters + value = line[end:pos - 1] + + # replaces \1 at the sub string, if \1 is used there + new_sub = sub + new_sub = new_sub.replace(r'\1', value) + + out += new_sub + + # Drop end ';' if any + if line[pos] == ';': + pos += 1 + + cur_pos = pos + n += 1 + + if count and count >= n: + break + + # Append the remaining string + l = len(line) + out += line[cur_pos:l] + + return out diff --git a/scripts/make-release b/scripts/make-release index 8c3594a..bc1b43c 100755 --- a/scripts/make-release +++ b/scripts/make-release @@ -40,8 +40,9 @@ fi # Only include wraps that are invoked with subproject() SUBPROJECTS="libvfio-user keycodemapdb berkeley-softfloat-3 - berkeley-testfloat-3 arbitrary-int-1-rs bilge-0.2-rs - bilge-impl-0.2-rs either-1-rs itertools-0.11-rs libc-0.2-rs proc-macro2-1-rs + berkeley-testfloat-3 anyhow-1-rs arbitrary-int-1-rs attrs-0.2-rs bilge-0.2-rs + bilge-impl-0.2-rs either-1-rs foreign-0.3-rs itertools-0.11-rs + libc-0.2-rs proc-macro2-1-rs proc-macro-error-1-rs proc-macro-error-attr-1-rs quote-1-rs syn-2-rs unicode-ident-1-rs" @@ -61,17 +62,15 @@ meson subprojects download $SUBPROJECTS (cd roms/skiboot && ./make_version.sh > .version) # Fetch edk2 submodule's submodules, since it won't have access to them via # the tarball later. -# -# A more uniform way to handle this sort of situation would be nice, but we -# don't necessarily have much control over how a submodule handles its -# submodule dependencies, so we continue to handle these on a case-by-case -# basis for now. -(cd roms/edk2 && \ - git submodule update --init --depth 1 -- \ - ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3 \ - BaseTools/Source/C/BrotliCompress/brotli \ - CryptoPkg/Library/OpensslLib/openssl \ - MdeModulePkg/Library/BrotliCustomDecompressLib/brotli) + +# As recommended by the EDK2 readme, we don't use --recursive here. +# EDK2 won't use any code or feature from a submodule of a submodule, +# so we don't need to add them to the tarball. +# Although we don't necessarily need all of the submodules that EDK2 +# has, we clone them all, to avoid running into problems where EDK2 +# adds a new submodule or changes its use of an existing one and +# the sources we ship in the tarball then fail to build. +(cd roms/edk2 && git submodule update --init --depth 1) popd exclude=(--exclude=.git) diff --git a/scripts/meson-buildoptions.py b/scripts/meson-buildoptions.py index a3e2247..e636d25 100644 --- a/scripts/meson-buildoptions.py +++ b/scripts/meson-buildoptions.py @@ -111,13 +111,13 @@ def help_line(left, opt, indent, long): right = f'{opt["description"]}' if long: value = get_help(opt) - if value != "auto" and value != "": + if value not in {"", "auto"}: right += f" [{value}]" if "choices" in opt and long: choices = "/".join(sorted(opt["choices"])) right += f" (choices: {choices})" - for x in wrap(" " + left, right, indent): - sh_print(x) + for line in wrap(" " + left, right, indent): + sh_print(line) # Return whether the option (a dictionary) can be used with @@ -144,18 +144,18 @@ def require_arg(opt): return not ({"enabled", "disabled"}.intersection(opt["choices"])) -def filter_options(json): - if ":" in json["name"]: +def filter_options(opt): + if ":" in opt["name"]: return False - if json["section"] == "user": - return json["name"] not in SKIP_OPTIONS + if opt["section"] == "user": + return opt["name"] not in SKIP_OPTIONS else: - return json["name"] in BUILTIN_OPTIONS + return opt["name"] in BUILTIN_OPTIONS -def load_options(json): - json = [x for x in json if filter_options(x)] - return sorted(json, key=lambda x: x["name"]) +def load_options(opts): + opts = [opt for opt in opts if filter_options(opt)] + return sorted(opts, key=lambda opt: opt["name"]) def cli_option(opt): @@ -223,7 +223,7 @@ def print_parse(options): key = cli_option(opt) name = opt["name"] if require_arg(opt): - if opt["type"] == "array" and not "choices" in opt: + if opt["type"] == "array" and "choices" not in opt: print(f' --{key}=*) quote_sh "-D{name}=$(meson_option_build_array $2)" ;;') else: print(f' --{key}=*) quote_sh "-D{name}=$2" ;;') @@ -241,14 +241,19 @@ def print_parse(options): print(" esac") print("}") -json_data = sys.stdin.read() -try: - options = load_options(json.loads(json_data)) -except: - print("Failure in scripts/meson-buildoptions.py parsing stdin as json", - file=sys.stderr) - print(json_data, file=sys.stderr) - sys.exit(1) -print("# This file is generated by meson-buildoptions.py, do not edit!") -print_help(options) -print_parse(options) + +def main(): + json_data = sys.stdin.read() + try: + options = load_options(json.loads(json_data)) + except: + print("Failure in scripts/meson-buildoptions.py parsing stdin as json", + file=sys.stderr) + print(json_data, file=sys.stderr) + sys.exit(1) + print("# This file is generated by meson-buildoptions.py, do not edit!") + print_help(options) + print_parse(options) + + +sys.exit(main()) diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh index 3e8e008..3d0d132 100644 --- a/scripts/meson-buildoptions.sh +++ b/scripts/meson-buildoptions.sh @@ -58,6 +58,7 @@ meson_options_help() { printf "%s\n" ' --enable-ubsan enable undefined behaviour sanitizer' printf "%s\n" ' --firmwarepath=VALUES search PATH for firmware files [share/qemu-' printf "%s\n" ' firmware]' + printf "%s\n" ' --gdb=VALUE Path to GDB' printf "%s\n" ' --iasl=VALUE Path to ACPI disassembler' printf "%s\n" ' --includedir=VALUE Header file directory [include]' printf "%s\n" ' --interp-prefix=VALUE where to find shared libraries etc., use %M for' @@ -80,7 +81,7 @@ meson_options_help() { printf "%s\n" ' --tls-priority=VALUE Default TLS protocol/cipher priority string' printf "%s\n" ' [NORMAL]' printf "%s\n" ' --with-coroutine=CHOICE coroutine backend to use (choices:' - printf "%s\n" ' auto/sigaltstack/ucontext/windows)' + printf "%s\n" ' auto/sigaltstack/ucontext/wasm/windows)' printf "%s\n" ' --with-pkgversion=VALUE use specified string as sub-version of the' printf "%s\n" ' package' printf "%s\n" ' --with-suffix=VALUE Suffix for QEMU data/modules/config directories' @@ -97,8 +98,6 @@ meson_options_help() { printf "%s\n" ' alsa ALSA sound support' printf "%s\n" ' attr attr/xattr support' printf "%s\n" ' auth-pam PAM access control' - printf "%s\n" ' avx2 AVX2 optimizations' - printf "%s\n" ' avx512bw AVX512BW optimizations' printf "%s\n" ' blkio libblkio block device driver' printf "%s\n" ' bochs bochs image format support' printf "%s\n" ' bpf eBPF support' @@ -132,6 +131,7 @@ meson_options_help() { printf "%s\n" ' hv-balloon hv-balloon driver (requires Glib 2.68+ GTree API)' printf "%s\n" ' hvf HVF acceleration support' printf "%s\n" ' iconv Font glyph conversion support' + printf "%s\n" ' igvm Independent Guest Virtual Machine (IGVM) file support' printf "%s\n" ' jack JACK sound support' printf "%s\n" ' keyring Linux keyring support' printf "%s\n" ' kvm KVM acceleration support' @@ -155,6 +155,7 @@ meson_options_help() { printf "%s\n" ' membarrier membarrier system call (for Linux 4.14+ or Windows' printf "%s\n" ' modules modules support (non Windows)' printf "%s\n" ' mpath Multipath persistent reservation passthrough' + printf "%s\n" ' mshv MSHV acceleration support' printf "%s\n" ' multiprocess Out of process device emulation support' printf "%s\n" ' netmap netmap network backend support' printf "%s\n" ' nettle nettle cryptography support' @@ -164,6 +165,7 @@ meson_options_help() { printf "%s\n" ' oss OSS sound support' printf "%s\n" ' pa PulseAudio sound support' printf "%s\n" ' parallels parallels image format support' + printf "%s\n" ' passt passt network backend support' printf "%s\n" ' pipewire PipeWire sound support' printf "%s\n" ' pixman pixman support' printf "%s\n" ' plugins TCG plugins via shared library loading' @@ -198,6 +200,7 @@ meson_options_help() { printf "%s\n" ' u2f U2F emulation support' printf "%s\n" ' uadk UADK Library support' printf "%s\n" ' usb-redir libusbredir support' + printf "%s\n" ' valgrind valgrind debug support for coroutine stacks' printf "%s\n" ' vde vde network backend support' printf "%s\n" ' vdi vdi image format support' printf "%s\n" ' vduse-blk-export' @@ -243,10 +246,6 @@ _meson_option_parse() { --audio-drv-list=*) quote_sh "-Daudio_drv_list=$2" ;; --enable-auth-pam) printf "%s" -Dauth_pam=enabled ;; --disable-auth-pam) printf "%s" -Dauth_pam=disabled ;; - --enable-avx2) printf "%s" -Davx2=enabled ;; - --disable-avx2) printf "%s" -Davx2=disabled ;; - --enable-avx512bw) printf "%s" -Davx512bw=enabled ;; - --disable-avx512bw) printf "%s" -Davx512bw=disabled ;; --enable-gcov) printf "%s" -Db_coverage=true ;; --disable-gcov) printf "%s" -Db_coverage=false ;; --enable-lto) printf "%s" -Db_lto=true ;; @@ -326,6 +325,7 @@ _meson_option_parse() { --disable-fuzzing) printf "%s" -Dfuzzing=false ;; --enable-gcrypt) printf "%s" -Dgcrypt=enabled ;; --disable-gcrypt) printf "%s" -Dgcrypt=disabled ;; + --gdb=*) quote_sh "-Dgdb=$2" ;; --enable-gettext) printf "%s" -Dgettext=enabled ;; --disable-gettext) printf "%s" -Dgettext=disabled ;; --enable-gio) printf "%s" -Dgio=enabled ;; @@ -351,6 +351,8 @@ _meson_option_parse() { --iasl=*) quote_sh "-Diasl=$2" ;; --enable-iconv) printf "%s" -Diconv=enabled ;; --disable-iconv) printf "%s" -Diconv=disabled ;; + --enable-igvm) printf "%s" -Digvm=enabled ;; + --disable-igvm) printf "%s" -Digvm=disabled ;; --includedir=*) quote_sh "-Dincludedir=$2" ;; --enable-install-blobs) printf "%s" -Dinstall_blobs=true ;; --disable-install-blobs) printf "%s" -Dinstall_blobs=false ;; @@ -409,6 +411,8 @@ _meson_option_parse() { --disable-modules) printf "%s" -Dmodules=disabled ;; --enable-mpath) printf "%s" -Dmpath=enabled ;; --disable-mpath) printf "%s" -Dmpath=disabled ;; + --enable-mshv) printf "%s" -Dmshv=enabled ;; + --disable-mshv) printf "%s" -Dmshv=disabled ;; --enable-multiprocess) printf "%s" -Dmultiprocess=enabled ;; --disable-multiprocess) printf "%s" -Dmultiprocess=disabled ;; --enable-netmap) printf "%s" -Dnetmap=enabled ;; @@ -427,6 +431,8 @@ _meson_option_parse() { --disable-pa) printf "%s" -Dpa=disabled ;; --enable-parallels) printf "%s" -Dparallels=enabled ;; --disable-parallels) printf "%s" -Dparallels=disabled ;; + --enable-passt) printf "%s" -Dpasst=enabled ;; + --disable-passt) printf "%s" -Dpasst=disabled ;; --enable-pipewire) printf "%s" -Dpipewire=enabled ;; --disable-pipewire) printf "%s" -Dpipewire=disabled ;; --enable-pixman) printf "%s" -Dpixman=enabled ;; @@ -526,6 +532,8 @@ _meson_option_parse() { --disable-ubsan) printf "%s" -Dubsan=false ;; --enable-usb-redir) printf "%s" -Dusb_redir=enabled ;; --disable-usb-redir) printf "%s" -Dusb_redir=disabled ;; + --enable-valgrind) printf "%s" -Dvalgrind=enabled ;; + --disable-valgrind) printf "%s" -Dvalgrind=disabled ;; --enable-vde) printf "%s" -Dvde=enabled ;; --disable-vde) printf "%s" -Dvde=disabled ;; --enable-vdi) printf "%s" -Dvdi=enabled ;; diff --git a/scripts/minikconf.py b/scripts/minikconf.py index 6f7f43b..4de5aee 100644 --- a/scripts/minikconf.py +++ b/scripts/minikconf.py @@ -340,7 +340,7 @@ class KconfigParser: @classmethod def parse(self, fp, mode=None): - data = KconfigData(mode or KconfigParser.defconfig) + data = KconfigData(mode or defconfig) parser = KconfigParser(data) parser.parse_file(fp) return data @@ -363,7 +363,9 @@ class KconfigParser: def do_assignment(self, var, val): if not var.startswith("CONFIG_"): - raise Error('assigned variable should start with CONFIG_') + raise KconfigParserError( + self, "assigned variable should start with CONFIG_" + ) var = self.data.do_var(var[7:]) self.data.do_assignment(var, val) diff --git a/scripts/mkemmc.sh b/scripts/mkemmc.sh new file mode 100755 index 0000000..4293882 --- /dev/null +++ b/scripts/mkemmc.sh @@ -0,0 +1,216 @@ +#!/bin/sh -e +# SPDX-License-Identifier: GPL-2.0-only +# +# Create eMMC block device image from boot, RPMB and user data images +# +# Copyright (c) Siemens, 2025 +# +# Authors: +# Jan Kiszka <jan.kiszka@siemens.com> +# + +usage() { + echo "$0 [OPTIONS] USER_IMG[:SIZE] OUTPUT_IMG" + echo "" + echo "SIZE must be a power of 2 up to 2G and multiples of 512 byte from there on." + echo "If no SIZE is specified, the size of USER_ING will be used (rounded up)." + echo "" + echo "Supported options:" + echo " -b BOOT1_IMG[:SIZE] Add boot partitions. SIZE must be multiples of 128K. If" + echo " no SIZE is specified, the size of BOOT1_IMG will be" + echo " used (rounded up). BOOT1_IMG will be stored in boot" + echo " partition 1, and a boot partition 2 of the same size" + echo " will be created as empty (all zeros) unless -B is" + echo " specified as well." + echo " -B BOOT2_IMG Fill boot partition 2 with BOOT2_IMG. Must be combined" + echo " with -b which is also defining the partition size." + echo " -r RPMB_IMG[:SIZE] Add RPMB partition. SIZE must be multiples of 128K. If" + echo " no SIZE is specified, the size of RPMB_IMG will be" + echo " used (rounded up)." + echo " -h, --help This help" + echo "" + echo "All SIZE parameters support the units K, M, G. If SIZE is smaller than the" + echo "associated image, it will be truncated in the output image." + exit "$1" +} + +process_size() { + name=$1 + image_file=$2 + alignment=$3 + image_arg=$4 + if [ "${image_arg#*:}" = "$image_arg" ]; then + if ! size=$(wc -c < "$image_file" 2>/dev/null); then + echo "Missing $name image '$image_file'." >&2 + exit 1 + fi + if [ "$alignment" = 128 ]; then + size=$(( (size + 128 * 1024 - 1) & ~(128 * 1024 - 1) )) + elif [ $size -gt $((2 * 1024 * 1024 * 1024)) ]; then + size=$(( (size + 511) & ~511 )) + elif [ $(( size & (size - 1) )) -gt 0 ]; then + n=0 + while [ "$size" -gt 0 ]; do + size=$((size >> 1)) + n=$((n + 1)) + done + size=$((1 << n)) + fi + else + value="${image_arg#*:}" + if [ "${value%K}" != "$value" ]; then + size=${value%K} + multiplier=1024 + elif [ "${value%M}" != "$value" ]; then + size=${value%M} + multiplier=$((1024 * 1024)) + elif [ "${value%G}" != "$value" ]; then + size=${value%G} + multiplier=$((1024 * 1024 * 1024)) + else + size=$value + multiplier=1 + fi + # check if "$size" is a valid integer by doing a self-comparison + if [ "$size" -eq "$size" ] 2>/dev/null; then + size=$((size * multiplier)) + else + echo "Invalid value '$value' specified for $image_file image size." >&2 + exit 1 + fi + if [ "$alignment" = 128 ]; then + if [ $(( size & (128 * 1024 - 1) )) -ne 0 ]; then + echo "The $name image size must be multiples of 128K." >&2 + exit 1 + fi + elif [ $size -gt $((2 * 1024 * 1024 * 1024)) ]; then + if [ $(( size & 511)) -ne 0 ]; then + echo "The $name image size must be multiples of 512 (if >2G)." >&2 + exit 1 + fi + elif [ $(( size & (size - 1) )) -gt 0 ]; then + echo "The $name image size must be power of 2 (up to 2G)." >&2 + exit 1 + fi + fi + echo $size +} + +check_truncation() { + image_file=$1 + output_size=$2 + if [ "$image_file" = "/dev/zero" ]; then + return + fi + if ! actual_size=$(wc -c < "$image_file" 2>/dev/null); then + echo "Missing image '$image_file'." >&2 + exit 1 + fi + if [ "$actual_size" -gt "$output_size" ]; then + echo "Warning: image '$image_file' will be truncated on output." + fi +} + +userimg= +outimg= +bootimg1= +bootimg2=/dev/zero +bootsz=0 +rpmbimg= +rpmbsz=0 + +while [ $# -gt 0 ]; do + case "$1" in + -b) + shift + [ $# -ge 1 ] || usage 1 + bootimg1=${1%%:*} + bootsz=$(process_size boot "$bootimg1" 128 "$1") + shift + ;; + -B) + shift + [ $# -ge 1 ] || usage 1 + bootimg2=$1 + shift + ;; + -r) + shift + [ $# -ge 1 ] || usage 1 + rpmbimg=${1%%:*} + rpmbsz=$(process_size RPMB "$rpmbimg" 128 "$1") + shift + ;; + -h|--help) + usage 0 + ;; + *) + if [ -z "$userimg" ]; then + userimg=${1%%:*} + usersz=$(process_size user "$userimg" U "$1") + elif [ -z "$outimg" ]; then + outimg=$1 + else + usage 1 + fi + shift + ;; + esac +done + +[ -n "$outimg" ] || usage 1 + +if [ "$bootsz" -gt $((32640 * 1024)) ]; then + echo "Boot image size is larger than 32640K." >&2 + exit 1 +fi +if [ "$rpmbsz" -gt $((16384 * 1024)) ]; then + echo "RPMB image size is larger than 16384K." >&2 + exit 1 +fi + +echo "Creating eMMC image" + +truncate -s 0 "$outimg" +pos=0 + +if [ "$bootsz" -gt 0 ]; then + echo " Boot partition 1 and 2: $((bootsz / 1024))K each" + blocks=$(( bootsz / (128 * 1024) )) + check_truncation "$bootimg1" "$bootsz" + dd if="$bootimg1" of="$outimg" conv=sparse bs=128K count=$blocks \ + status=none + check_truncation "$bootimg2" "$bootsz" + dd if="$bootimg2" of="$outimg" conv=sparse bs=128K count=$blocks \ + seek=$blocks status=none + pos=$((2 * bootsz)) +fi + +if [ "$rpmbsz" -gt 0 ]; then + echo " RPMB partition: $((rpmbsz / 1024))K" + blocks=$(( rpmbsz / (128 * 1024) )) + check_truncation "$rpmbimg" "$rpmbsz" + dd if="$rpmbimg" of="$outimg" conv=sparse bs=128K count=$blocks \ + seek=$(( pos / (128 * 1024) )) status=none + pos=$((pos + rpmbsz)) +fi + +if [ "$usersz" -lt 1024 ]; then + echo " User data: $usersz bytes" +elif [ "$usersz" -lt $((1024 * 1024)) ]; then + echo " User data: $(( (usersz + 1023) / 1024 ))K ($usersz)" +elif [ "$usersz" -lt $((1024 * 1024 * 1024)) ]; then + echo " User data: $(( (usersz + 1048575) / 1048576))M ($usersz)" +else + echo " User data: $(( (usersz + 1073741823) / 1073741824))G ($usersz)" +fi +check_truncation "$userimg" "$usersz" +dd if="$userimg" of="$outimg" conv=sparse bs=128K seek=$(( pos / (128 * 1024) )) \ + count=$(( (usersz + 128 * 1024 - 1) / (128 * 1024) )) status=none +pos=$((pos + usersz)) +truncate -s $pos "$outimg" + +echo "" +echo "Instantiate by appending to the qemu command line:" +echo " -drive file=$outimg,if=none,format=raw,id=emmc-img" +echo " -device emmc,boot-partition-size=$bootsz,rpmb-partition-size=$rpmbsz,drive=emmc-img" diff --git a/scripts/modinfo-collect.py b/scripts/modinfo-collect.py index 4e7584d..db78b16 100644 --- a/scripts/modinfo-collect.py +++ b/scripts/modinfo-collect.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- import os import sys @@ -7,15 +6,6 @@ import json import shlex import subprocess -def find_command(src, target, compile_commands): - for command in compile_commands: - if command['file'] != src: - continue - if target != '' and command['command'].find(target) == -1: - continue - return command['command'] - return 'false' - def process_command(src, command): skip = False out = [] @@ -43,14 +33,22 @@ def main(args): print("MODINFO_DEBUG target %s" % target) arch = target[:-8] # cut '-softmmu' print("MODINFO_START arch \"%s\" MODINFO_END" % arch) + with open('compile_commands.json') as f: - compile_commands = json.load(f) - for src in args: + compile_commands_json = json.load(f) + compile_commands = { x['output']: x for x in compile_commands_json } + + for obj in args: + entry = compile_commands.get(obj, None) + if not entry: + sys.stderr.write(f'modinfo: Could not find object file {obj}') + sys.exit(1) + src = entry['file'] if not src.endswith('.c'): print("MODINFO_DEBUG skip %s" % src) continue + command = entry['command'] print("MODINFO_DEBUG src %s" % src) - command = find_command(src, target, compile_commands) cmdline = process_command(src, command) print("MODINFO_DEBUG cmd", cmdline) result = subprocess.run(cmdline, stdout = subprocess.PIPE, diff --git a/scripts/modinfo-generate.py b/scripts/modinfo-generate.py index b1538fc..aaf2354 100644 --- a/scripts/modinfo-generate.py +++ b/scripts/modinfo-generate.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- import os import sys diff --git a/scripts/mtest2make.py b/scripts/mtest2make.py index 2ef375f..4b252de 100644 --- a/scripts/mtest2make.py +++ b/scripts/mtest2make.py @@ -8,24 +8,23 @@ from collections import defaultdict import itertools import json import os -import shlex import sys class Suite(object): def __init__(self): self.deps = set() - self.speeds = ['quick'] + self.speeds = set() def names(self, base): - return [base if speed == 'quick' else f'{base}-{speed}' for speed in self.speeds] + return [f'{base}-{speed}' for speed in self.speeds] -print(''' +print(r''' SPEED = quick -.speed.quick = $(foreach s,$(sort $(filter-out %-slow %-thorough, $1)), --suite $s) -.speed.slow = $(foreach s,$(sort $(filter-out %-thorough, $1)), --suite $s) -.speed.thorough = $(foreach s,$(sort $1), --suite $s) +.speed.quick = $(sort $(filter-out %-slow %-thorough, $1)) +.speed.slow = $(sort $(filter-out %-thorough, $1)) +.speed.thorough = $(sort $1) TIMEOUT_MULTIPLIER ?= 1 .mtestargs = --no-rebuild -t $(TIMEOUT_MULTIPLIER) @@ -34,8 +33,10 @@ ifneq ($(SPEED), quick) endif .mtestargs += $(subst -j,--num-processes , $(filter-out -j, $(lastword -j1 $(filter -j%, $(MAKEFLAGS))))) -.check.mtestargs = $(MTESTARGS) $(.mtestargs) $(if $(V),--verbose,--print-errorlogs) -.bench.mtestargs = $(MTESTARGS) $(.mtestargs) --benchmark --verbose''') +.check.mtestargs = $(MTESTARGS) $(.mtestargs) $(if $(V),--verbose,--print-errorlogs) \ + $(foreach s, $(sort $(.check.mtest-suites)), --suite $s) +.bench.mtestargs = $(MTESTARGS) $(.mtestargs) --benchmark --verbose \ + $(foreach s, $(sort $(.bench.mtest-suites)), --suite $s)''') introspect = json.load(sys.stdin) @@ -57,13 +58,13 @@ def process_tests(test, targets, suites): s = s.split(':')[1] if s == 'slow' or s == 'thorough': continue + suites[s].deps.update(deps) if s.endswith('-slow'): s = s[:-5] - suites[s].speeds.append('slow') + suites[s].speeds.add('slow') if s.endswith('-thorough'): s = s[:-9] - suites[s].speeds.append('thorough') - suites[s].deps.update(deps) + suites[s].speeds.add('thorough') def emit_prolog(suites, prefix): all_targets = ' '.join((f'{prefix}-{k}' for k in suites.keys())) @@ -72,29 +73,26 @@ def emit_prolog(suites, prefix): print(f'all-{prefix}-targets = {all_targets}') print(f'all-{prefix}-xml = {all_xml}') print(f'.PHONY: {prefix} do-meson-{prefix} {prefix}-report.junit.xml $(all-{prefix}-targets) $(all-{prefix}-xml)') - print(f'ifeq ($(filter {prefix}, $(MAKECMDGOALS)),)') - print(f'.{prefix}.mtestargs += $(call .speed.$(SPEED), $(.{prefix}.mtest-suites))') - print(f'endif') + print(f'ninja-cmd-goals += $(foreach s, $(.{prefix}.mtest-suites), $(.{prefix}-$s.deps))') print(f'{prefix}-build: run-ninja') print(f'{prefix} $(all-{prefix}-targets): do-meson-{prefix}') print(f'do-meson-{prefix}: run-ninja; $(if $(MAKE.n),,+)$(MESON) test $(.{prefix}.mtestargs)') print(f'{prefix}-report.junit.xml $(all-{prefix}-xml): {prefix}-report%.junit.xml: run-ninja') print(f'\t$(MAKE) {prefix}$* MTESTARGS="$(MTESTARGS) --logbase {prefix}-report$*" && ln -f meson-logs/$@ .') -def emit_suite_deps(name, suite, prefix): +def emit_suite(name, suite, prefix): deps = ' '.join(suite.deps) - targets = [f'{prefix}-{name}', f'{prefix}-report-{name}.junit.xml', f'{prefix}', f'{prefix}-report.junit.xml', - f'{prefix}-build'] print() print(f'.{prefix}-{name}.deps = {deps}') - for t in targets: - print(f'.ninja-goals.{t} += $(.{prefix}-{name}.deps)') + print(f'.ninja-goals.check-build += $(.{prefix}-{name}.deps)') -def emit_suite(name, suite, prefix): - emit_suite_deps(name, suite, prefix) - targets = f'{prefix}-{name} {prefix}-report-{name}.junit.xml {prefix} {prefix}-report.junit.xml' + names = ' '.join(suite.names(name)) + targets = f'{prefix}-{name} {prefix}-report-{name}.junit.xml' + if not name.endswith('-slow') and not name.endswith('-thorough'): + targets += f' {prefix} {prefix}-report.junit.xml' print(f'ifneq ($(filter {targets}, $(MAKECMDGOALS)),)') - print(f'.{prefix}.mtest-suites += ' + ' '.join(suite.names(name))) + # for the "base" suite possibly add FOO-slow and FOO-thorough + print(f".{prefix}.mtest-suites += {name} $(call .speed.$(SPEED), {names})") print(f'endif') targets = {t['id']: [os.path.relpath(f) for f in t['filename']] diff --git a/scripts/nsis.py b/scripts/nsis.py index af4e064..33069f4 100644 --- a/scripts/nsis.py +++ b/scripts/nsis.py @@ -23,7 +23,7 @@ def find_deps(exe_or_dll, search_path, analyzed_deps): output = subprocess.check_output(["objdump", "-p", exe_or_dll], text=True) output = output.split("\n") for line in output: - if not line.startswith("\tDLL Name: "): + if not line.lstrip().startswith("DLL Name: "): continue dep = line.split("DLL Name: ")[1].strip() @@ -114,7 +114,7 @@ def main(): "-DSRCDIR=" + args.srcdir, "-DBINDIR=" + destdir + prefix, ] - if args.cpu == "x86_64": + if args.cpu == "aarch64" or args.cpu == "x86_64": makensis += ["-DW64"] makensis += ["-DDLLDIR=" + dlldir] diff --git a/scripts/oss-fuzz/minimize_qtest_trace.py b/scripts/oss-fuzz/minimize_qtest_trace.py index d1f3990..414a6d9 100755 --- a/scripts/oss-fuzz/minimize_qtest_trace.py +++ b/scripts/oss-fuzz/minimize_qtest_trace.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ This takes a crashing qtest trace and tries to remove superfluous operations diff --git a/scripts/oss-fuzz/output_reproducer.py b/scripts/oss-fuzz/output_reproducer.py index e8ef76b..0df96cf 100755 --- a/scripts/oss-fuzz/output_reproducer.py +++ b/scripts/oss-fuzz/output_reproducer.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Convert plain qtest traces to C or Bash reproducers diff --git a/scripts/oss-fuzz/reorder_fuzzer_qtest_trace.py b/scripts/oss-fuzz/reorder_fuzzer_qtest_trace.py index b154a25..8af0d5d 100755 --- a/scripts/oss-fuzz/reorder_fuzzer_qtest_trace.py +++ b/scripts/oss-fuzz/reorder_fuzzer_qtest_trace.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Use this to convert qtest log info from a generic fuzzer input into a qtest diff --git a/scripts/probe-gdb-support.py b/scripts/probe-gdb-support.py index 6bcadce..43c7030 100644 --- a/scripts/probe-gdb-support.py +++ b/scripts/probe-gdb-support.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# coding: utf-8 # # Probe gdb for supported architectures. # diff --git a/scripts/qapi/.flake8 b/scripts/qapi/.flake8 deleted file mode 100644 index a873ff6..0000000 --- a/scripts/qapi/.flake8 +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -# Prefer pylint's bare-except checks to flake8's -extend-ignore = E722 diff --git a/scripts/qapi/.isort.cfg b/scripts/qapi/.isort.cfg deleted file mode 100644 index 643caa1..0000000 --- a/scripts/qapi/.isort.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[settings] -force_grid_wrap=4 -force_sort_within_sections=True -include_trailing_comma=True -line_length=72 -lines_after_imports=2 -multi_line_output=3 diff --git a/scripts/qapi/backend.py b/scripts/qapi/backend.py index 14e60aa6..49ae6ec 100644 --- a/scripts/qapi/backend.py +++ b/scripts/qapi/backend.py @@ -13,6 +13,7 @@ from .visit import gen_visit class QAPIBackend(ABC): + # pylint: disable=too-few-public-methods @abstractmethod def generate(self, @@ -36,6 +37,7 @@ class QAPIBackend(ABC): class QAPICBackend(QAPIBackend): + # pylint: disable=too-few-public-methods def generate(self, schema: QAPISchema, diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py index 7914227..9dede74 100644 --- a/scripts/qapi/commands.py +++ b/scripts/qapi/commands.py @@ -13,16 +13,10 @@ This work is licensed under the terms of the GNU GPL, version 2. See the COPYING file in the top-level directory. """ -from typing import ( - Dict, - List, - Optional, - Set, -) +from typing import List, Optional from .common import c_name, mcgen from .gen import ( - QAPIGenC, QAPISchemaModularCVisitor, build_params, gen_features, @@ -112,11 +106,7 @@ def gen_call(name: str, ''') if ret_type: - ret += mcgen(''' - - qmp_marshal_output_%(c_name)s(retval, ret, errp); -''', - c_name=ret_type.c_name()) + ret += gen_marshal_output(ret_type) if gen_tracing: if ret_type: @@ -142,22 +132,16 @@ def gen_call(name: str, def gen_marshal_output(ret_type: QAPISchemaType) -> str: return mcgen(''' -static void qmp_marshal_output_%(c_name)s(%(c_type)s ret_in, - QObject **ret_out, Error **errp) -{ - Visitor *v; - - v = qobject_output_visitor_new_qmp(ret_out); - if (visit_type_%(c_name)s(v, "unused", &ret_in, errp)) { - visit_complete(v, ret_out); + ov = qobject_output_visitor_new_qmp(ret); + if (visit_type_%(c_name)s(ov, "unused", &retval, errp)) { + visit_complete(ov, ret); } - visit_free(v); - v = qapi_dealloc_visitor_new(); - visit_type_%(c_name)s(v, "unused", &ret_in, NULL); - visit_free(v); -} + visit_free(ov); + ov = qapi_dealloc_visitor_new(); + visit_type_%(c_name)s(ov, "unused", &retval, NULL); + visit_free(ov); ''', - c_type=ret_type.c_type(), c_name=ret_type.c_name()) + c_name=ret_type.c_name()) def build_marshal_proto(name: str, @@ -209,6 +193,7 @@ def gen_marshal(name: str, if ret_type: ret += mcgen(''' %(c_type)s retval; + Visitor *ov; ''', c_type=ret_type.c_type()) @@ -308,11 +293,9 @@ class QAPISchemaGenCommandVisitor(QAPISchemaModularCVisitor): prefix, 'qapi-commands', ' * Schema-defined QAPI/QMP commands', None, __doc__, gen_tracing=gen_tracing) - self._visited_ret_types: Dict[QAPIGenC, Set[QAPISchemaType]] = {} self._gen_tracing = gen_tracing def _begin_user_module(self, name: str) -> None: - self._visited_ret_types[self._genc] = set() commands = self._module_basename('qapi-commands', name) types = self._module_basename('qapi-types', name) visit = self._module_basename('qapi-visit', name) @@ -386,16 +369,6 @@ void %(c_prefix)sqmp_init_marshal(QmpCommandList *cmds) coroutine: bool) -> None: if not gen: return - # FIXME: If T is a user-defined type, the user is responsible - # for making this work, i.e. to make T's condition the - # conjunction of the T-returning commands' conditions. If T - # is a built-in type, this isn't possible: the - # qmp_marshal_output_T() will be generated unconditionally. - if ret_type and ret_type not in self._visited_ret_types[self._genc]: - self._visited_ret_types[self._genc].add(ret_type) - with ifcontext(ret_type.ifcond, - self._genh, self._genc): - self._genc.add(gen_marshal_output(ret_type)) with ifcontext(ifcond, self._genh, self._genc): self._genh.add(gen_command_decl(name, arg_type, boxed, ret_type, coroutine)) diff --git a/scripts/qapi/error.py b/scripts/qapi/error.py index e35e4dd..f73bc55 100644 --- a/scripts/qapi/error.py +++ b/scripts/qapi/error.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Copyright (c) 2017-2019 Red Hat Inc. # # Authors: diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index cae0a08..f40b247 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Copyright IBM, Corp. 2011 # Copyright (c) 2013-2021 Red Hat Inc. # diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py index d3c56d45..0c9b8db 100644 --- a/scripts/qapi/gen.py +++ b/scripts/qapi/gen.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # QAPI code generation # # Copyright (c) 2015-2019 Red Hat Inc. diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py index 89ee5d5..7e28de2 100644 --- a/scripts/qapi/introspect.py +++ b/scripts/qapi/introspect.py @@ -59,7 +59,7 @@ from .source import QAPISourceInfo # # Sadly, mypy does not support recursive types; so the _Stub alias is used to # mark the imprecision in the type model where we'd otherwise use JSONValue. -_Stub = Any +_Stub = Any # pylint: disable=invalid-name _Scalar = Union[str, bool, None] _NonScalar = Union[Dict[str, _Stub], List[_Stub]] _Value = Union[_Scalar, _NonScalar] diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini deleted file mode 100644 index 8109470..0000000 --- a/scripts/qapi/mypy.ini +++ /dev/null @@ -1,4 +0,0 @@ -[mypy] -strict = True -disallow_untyped_calls = False -python_version = 3.8 diff --git a/scripts/qapi/parser.py b/scripts/qapi/parser.py index 949d9e8..c3cf339 100644 --- a/scripts/qapi/parser.py +++ b/scripts/qapi/parser.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # QAPI schema parser # # Copyright IBM, Corp. 2011 @@ -110,6 +108,11 @@ class QAPISchemaParser: self.exprs: List[QAPIExpression] = [] self.docs: List[QAPIDoc] = [] + # State for tracking qmp-example blocks and simple + # :: literal blocks. + self._literal_mode = False + self._literal_mode_indent = 0 + # Showtime! self._parse() @@ -425,12 +428,55 @@ class QAPISchemaParser: if self.val != '##': raise QAPIParseError( self, "junk after '##' at end of documentation comment") + self._literal_mode = False return None if self.val == '#': return '' if self.val[1] != ' ': raise QAPIParseError(self, "missing space after #") - return self.val[2:].rstrip() + + line = self.val[2:].rstrip() + + if re.match(r'(\.\. +qmp-example)? *::$', line): + self._literal_mode = True + self._literal_mode_indent = 0 + elif self._literal_mode and line: + indent = must_match(r'\s*', line).end() + if self._literal_mode_indent == 0: + self._literal_mode_indent = indent + elif indent < self._literal_mode_indent: + # ReST directives stop at decreasing indentation + self._literal_mode = False + + if not self._literal_mode: + self._validate_doc_line_format(line) + + return line + + def _validate_doc_line_format(self, line: str) -> None: + """ + Validate documentation format rules for a single line: + 1. Lines should not exceed 70 characters + 2. Sentences should be separated by two spaces + """ + full_line_length = len(line) + 2 # "# " = 2 characters + if full_line_length > 70: + # Skip URL lines - they can't be broken + if re.match(r' *(https?|ftp)://[^ ]*$', line): + pass + else: + raise QAPIParseError( + self, "documentation line longer than 70 characters") + + single_space_pattern = r'(\be\.g\.|^ *\d\.|([.!?])) [A-Z0-9(]' + for m in list(re.finditer(single_space_pattern, line)): + if not m.group(2): + continue + # HACK so the error message points to the offending spot + self.pos = self.line_pos + 2 + m.start(2) + 1 + raise QAPIParseError( + self, "Use two spaces between sentences\n" + "If this not the end of a sentence, please report a bug.") @staticmethod def _match_at_name_colon(string: str) -> Optional[Match[str]]: @@ -584,10 +630,6 @@ class QAPISchemaParser: doc.append_line(text) line = self.get_doc_indented(doc) no_more_args = True - elif line.startswith('='): - raise QAPIParseError( - self, - "unexpected '=' markup in definition documentation") else: # plain paragraph doc.ensure_untagged_section(self.info) @@ -597,22 +639,15 @@ class QAPISchemaParser: # Free-form documentation doc = QAPIDoc(info) doc.ensure_untagged_section(self.info) - first = True while line is not None: if match := self._match_at_name_colon(line): raise QAPIParseError( self, "'@%s:' not allowed in free-form documentation" % match.group(1)) - if line.startswith('='): - if not first: - raise QAPIParseError( - self, - "'=' heading must come first in a comment block") doc.append_line(line) self.accept(False) line = self.get_doc_line() - first = False self.accept() doc.end() @@ -815,6 +850,43 @@ class QAPIDoc: % feature.name) self.features[feature.name].connect(feature) + def ensure_returns(self, info: QAPISourceInfo) -> None: + + def _insert_near_kind( + kind: QAPIDoc.Kind, + new_sect: QAPIDoc.Section, + after: bool = False, + ) -> bool: + for idx, sect in enumerate(reversed(self.all_sections)): + if sect.kind == kind: + pos = len(self.all_sections) - idx - 1 + if after: + pos += 1 + self.all_sections.insert(pos, new_sect) + return True + return False + + if any(s.kind == QAPIDoc.Kind.RETURNS for s in self.all_sections): + return + + # Stub "Returns" section for undocumented returns value + stub = QAPIDoc.Section(info, QAPIDoc.Kind.RETURNS) + + if any(_insert_near_kind(kind, stub, after) for kind, after in ( + # 1. If arguments, right after those. + (QAPIDoc.Kind.MEMBER, True), + # 2. Elif errors, right *before* those. + (QAPIDoc.Kind.ERRORS, False), + # 3. Elif features, right *before* those. + (QAPIDoc.Kind.FEATURE, False), + )): + return + + # Otherwise, it should go right after the intro. The intro + # is always the first section and is always present (even + # when empty), so we can insert directly at index=1 blindly. + self.all_sections.insert(1, stub) + def check_expr(self, expr: QAPIExpression) -> None: if 'command' in expr: if self.returns and 'returns' not in expr: diff --git a/scripts/qapi/pylintrc b/scripts/qapi/pylintrc index d24eece..e16283a 100644 --- a/scripts/qapi/pylintrc +++ b/scripts/qapi/pylintrc @@ -19,6 +19,7 @@ disable=consider-using-f-string, too-many-instance-attributes, too-many-positional-arguments, too-many-statements, + unknown-option-value, useless-option-value, [REPORTS] diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py index cbe3b5a..8d88b40 100644 --- a/scripts/qapi/schema.py +++ b/scripts/qapi/schema.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # QAPI schema internal representation # # Copyright (c) 2015-2019 Red Hat Inc. @@ -1062,6 +1060,9 @@ class QAPISchemaCommand(QAPISchemaDefinition): if self.arg_type and self.arg_type.is_implicit(): self.arg_type.connect_doc(doc) + if self.ret_type and self.info: + doc.ensure_returns(self.info) + def visit(self, visitor: QAPISchemaVisitor) -> None: super().visit(visitor) visitor.visit_command( diff --git a/scripts/qemu-guest-agent/fsfreeze-hook b/scripts/qemu-guest-agent/fsfreeze-hook index c1feb6f..5b915af 100755 --- a/scripts/qemu-guest-agent/fsfreeze-hook +++ b/scripts/qemu-guest-agent/fsfreeze-hook @@ -1,11 +1,12 @@ #!/bin/sh -# This script is executed when a guest agent receives fsfreeze-freeze and -# fsfreeze-thaw command, if it is specified in --fsfreeze-hook (-F) -# option of qemu-ga or placed in default path (/etc/qemu/fsfreeze-hook). -# When the agent receives fsfreeze-freeze request, this script is issued with -# "freeze" argument before the filesystem is frozen. And for fsfreeze-thaw -# request, it is issued with "thaw" argument after filesystem is thawed. +# This script is executed when the guest agent receives fsfreeze-freeze and +# fsfreeze-thaw commands, provided that the --fsfreeze-hook (-F) option of +# qemu-ga is specified and the script is placed in /etc/qemu/fsfreeze-hook or in +# the path specified together with -F. When the agent receives fsfreeze-freeze +# requests, this script is called with "freeze" as its argument before the +# filesystem is frozen. And for fsfreeze-thaw requests, it is called with "thaw" +# as its argument after the filesystem is thawed. LOGFILE=/var/log/qga-fsfreeze-hook.log FSFREEZE_D=$(dirname -- "$0")/fsfreeze-hook.d diff --git a/scripts/qemu-plugin-symbols.py b/scripts/qemu-plugin-symbols.py index e285ebb..6964497 100755 --- a/scripts/qemu-plugin-symbols.py +++ b/scripts/qemu-plugin-symbols.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # Extract QEMU Plugin API symbols from a header file # diff --git a/scripts/qemugdb/tcg.py b/scripts/qemugdb/tcg.py index 16c03c0..22529c7 100644 --- a/scripts/qemugdb/tcg.py +++ b/scripts/qemugdb/tcg.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # GDB debugging support, TCG status # # Copyright 2016 Linaro Ltd diff --git a/scripts/qemugdb/timers.py b/scripts/qemugdb/timers.py index 46537b2..5714f92 100644 --- a/scripts/qemugdb/timers.py +++ b/scripts/qemugdb/timers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # GDB debugging support # # Copyright 2017 Linaro Ltd diff --git a/scripts/qmp_helper.py b/scripts/qmp_helper.py new file mode 100755 index 0000000..c1e7e0f --- /dev/null +++ b/scripts/qmp_helper.py @@ -0,0 +1,703 @@ +#!/usr/bin/env python3 +# +# pylint: disable=C0103,E0213,E1135,E1136,E1137,R0902,R0903,R0912,R0913,R0917 +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Copyright (C) 2024-2025 Mauro Carvalho Chehab <mchehab+huawei@kernel.org> + +""" +Helper classes to be used by ghes_inject command classes. +""" + +import json +import sys + +from datetime import datetime +from os import path as os_path + +try: + qemu_dir = os_path.abspath(os_path.dirname(os_path.dirname(__file__))) + sys.path.append(os_path.join(qemu_dir, 'python')) + + from qemu.qmp.legacy import QEMUMonitorProtocol + +except ModuleNotFoundError as exc: + print(f"Module '{exc.name}' not found.") + print("Try export PYTHONPATH=top-qemu-dir/python or run from top-qemu-dir") + sys.exit(1) + +from base64 import b64encode + +class util: + """ + Ancillary functions to deal with bitmaps, parse arguments, + generate GUID and encode data on a bytearray buffer. + """ + + # + # Helper routines to handle multiple choice arguments + # + def get_choice(name, value, choices, suffixes=None, bitmask=True): + """Produce a list from multiple choice argument""" + + new_values = 0 + + if not value: + return new_values + + for val in value.split(","): + val = val.lower() + + if suffixes: + for suffix in suffixes: + val = val.removesuffix(suffix) + + if val not in choices.keys(): + if suffixes: + for suffix in suffixes: + if val + suffix in choices.keys(): + val += suffix + break + + if val not in choices.keys(): + sys.exit(f"Error on '{name}': choice '{val}' is invalid.") + + val = choices[val] + + if bitmask: + new_values |= val + else: + if new_values: + sys.exit(f"Error on '{name}': only one value is accepted.") + + new_values = val + + return new_values + + def get_array(name, values, max_val=None): + """Add numbered hashes from integer lists into an array""" + + array = [] + + for value in values: + for val in value.split(","): + try: + val = int(val, 0) + except ValueError: + sys.exit(f"Error on '{name}': {val} is not an integer") + + if val < 0: + sys.exit(f"Error on '{name}': {val} is not unsigned") + + if max_val and val > max_val: + sys.exit(f"Error on '{name}': {val} is too little") + + array.append(val) + + return array + + def get_mult_array(mult, name, values, allow_zero=False, max_val=None): + """Add numbered hashes from integer lists""" + + if not allow_zero: + if not values: + return + else: + if values is None: + return + + if not values: + i = 0 + if i not in mult: + mult[i] = {} + + mult[i][name] = [] + return + + i = 0 + for value in values: + for val in value.split(","): + try: + val = int(val, 0) + except ValueError: + sys.exit(f"Error on '{name}': {val} is not an integer") + + if val < 0: + sys.exit(f"Error on '{name}': {val} is not unsigned") + + if max_val and val > max_val: + sys.exit(f"Error on '{name}': {val} is too little") + + if i not in mult: + mult[i] = {} + + if name not in mult[i]: + mult[i][name] = [] + + mult[i][name].append(val) + + i += 1 + + + def get_mult_choices(mult, name, values, choices, + suffixes=None, allow_zero=False): + """Add numbered hashes from multiple choice arguments""" + + if not allow_zero: + if not values: + return + else: + if values is None: + return + + i = 0 + for val in values: + new_values = util.get_choice(name, val, choices, suffixes) + + if i not in mult: + mult[i] = {} + + mult[i][name] = new_values + i += 1 + + + def get_mult_int(mult, name, values, allow_zero=False): + """Add numbered hashes from integer arguments""" + if not allow_zero: + if not values: + return + else: + if values is None: + return + + i = 0 + for val in values: + try: + val = int(val, 0) + except ValueError: + sys.exit(f"Error on '{name}': {val} is not an integer") + + if val < 0: + sys.exit(f"Error on '{name}': {val} is not unsigned") + + if i not in mult: + mult[i] = {} + + mult[i][name] = val + i += 1 + + + # + # Data encode helper functions + # + def bit(b): + """Simple macro to define a bit on a bitmask""" + return 1 << b + + + def data_add(data, value, num_bytes): + """Adds bytes from value inside a bitarray""" + + data.extend(value.to_bytes(num_bytes, byteorder="little")) # pylint: disable=E1101 + + def dump_bytearray(name, data): + """Does an hexdump of a byte array, grouping in bytes""" + + print(f"{name} ({len(data)} bytes):") + + for ln_start in range(0, len(data), 16): + ln_end = min(ln_start + 16, len(data)) + print(f" {ln_start:08x} ", end="") + for i in range(ln_start, ln_end): + print(f"{data[i]:02x} ", end="") + for i in range(ln_end, ln_start + 16): + print(" ", end="") + print(" ", end="") + for i in range(ln_start, ln_end): + if data[i] >= 32 and data[i] < 127: + print(chr(data[i]), end="") + else: + print(".", end="") + + print() + print() + + def time(string): + """Handle BCD timestamps used on Generic Error Data Block""" + + time = None + + # Formats to be used when parsing time stamps + formats = [ + "%Y-%m-%d %H:%M:%S", + ] + + if string == "now": + time = datetime.now() + + if time is None: + for fmt in formats: + try: + time = datetime.strptime(string, fmt) + break + except ValueError: + pass + + if time is None: + raise ValueError("Invalid time format") + + return time + +class guid: + """ + Simple class to handle GUID fields. + """ + + def __init__(self, time_low, time_mid, time_high, nodes): + """Initialize a GUID value""" + + assert len(nodes) == 8 + + self.time_low = time_low + self.time_mid = time_mid + self.time_high = time_high + self.nodes = nodes + + @classmethod + def UUID(cls, guid_str): + """Initialize a GUID using a string on its standard format""" + + if len(guid_str) != 36: + print("Size not 36") + raise ValueError('Invalid GUID size') + + # It is easier to parse without separators. So, drop them + guid_str = guid_str.replace('-', '') + + if len(guid_str) != 32: + print("Size not 32", guid_str, len(guid_str)) + raise ValueError('Invalid GUID hex size') + + time_low = 0 + time_mid = 0 + time_high = 0 + nodes = [] + + for i in reversed(range(16, 32, 2)): + h = guid_str[i:i + 2] + value = int(h, 16) + nodes.insert(0, value) + + time_high = int(guid_str[12:16], 16) + time_mid = int(guid_str[8:12], 16) + time_low = int(guid_str[0:8], 16) + + return cls(time_low, time_mid, time_high, nodes) + + def __str__(self): + """Output a GUID value on its default string representation""" + + clock = self.nodes[0] << 8 | self.nodes[1] + + node = 0 + for i in range(2, len(self.nodes)): + node = node << 8 | self.nodes[i] + + s = f"{self.time_low:08x}-{self.time_mid:04x}-" + s += f"{self.time_high:04x}-{clock:04x}-{node:012x}" + return s + + def to_bytes(self): + """Output a GUID value in bytes""" + + data = bytearray() + + util.data_add(data, self.time_low, 4) + util.data_add(data, self.time_mid, 2) + util.data_add(data, self.time_high, 2) + data.extend(bytearray(self.nodes)) + + return data + +class qmp: + """ + Opens a connection and send/receive QMP commands. + """ + + def send_cmd(self, command, args=None, may_open=False, return_error=True): + """Send a command to QMP, optinally opening a connection""" + + if may_open: + self._connect() + elif not self.connected: + return False + + msg = { 'execute': command } + if args: + msg['arguments'] = args + + try: + obj = self.qmp_monitor.cmd_obj(msg) + # Can we use some other exception class here? + except Exception as e: # pylint: disable=W0718 + print(f"Command: {command}") + print(f"Failed to inject error: {e}.") + return None + + if "return" in obj: + if isinstance(obj.get("return"), dict): + if obj["return"]: + return obj["return"] + return "OK" + + return obj["return"] + + if isinstance(obj.get("error"), dict): + error = obj["error"] + if return_error: + print(f"Command: {msg}") + print(f'{error["class"]}: {error["desc"]}') + else: + print(json.dumps(obj)) + + return None + + def _close(self): + """Shutdown and close the socket, if opened""" + if not self.connected: + return + + self.qmp_monitor.close() + self.connected = False + + def _connect(self): + """Connect to a QMP TCP/IP port, if not connected yet""" + + if self.connected: + return True + + try: + self.qmp_monitor.connect(negotiate=True) + except ConnectionError: + sys.exit(f"Can't connect to QMP host {self.host}:{self.port}") + + self.connected = True + + return True + + BLOCK_STATUS_BITS = { + "uncorrectable": util.bit(0), + "correctable": util.bit(1), + "multi-uncorrectable": util.bit(2), + "multi-correctable": util.bit(3), + } + + ERROR_SEVERITY = { + "recoverable": 0, + "fatal": 1, + "corrected": 2, + "none": 3, + } + + VALIDATION_BITS = { + "fru-id": util.bit(0), + "fru-text": util.bit(1), + "timestamp": util.bit(2), + } + + GEDB_FLAGS_BITS = { + "recovered": util.bit(0), + "prev-error": util.bit(1), + "simulated": util.bit(2), + } + + GENERIC_DATA_SIZE = 72 + + def argparse(parser): + """Prepare a parser group to query generic error data""" + + block_status_bits = ",".join(qmp.BLOCK_STATUS_BITS.keys()) + error_severity_enum = ",".join(qmp.ERROR_SEVERITY.keys()) + validation_bits = ",".join(qmp.VALIDATION_BITS.keys()) + gedb_flags_bits = ",".join(qmp.GEDB_FLAGS_BITS.keys()) + + g_gen = parser.add_argument_group("Generic Error Data") # pylint: disable=E1101 + g_gen.add_argument("--block-status", + help=f"block status bits: {block_status_bits}") + g_gen.add_argument("--raw-data", nargs="+", + help="Raw data inside the Error Status Block") + g_gen.add_argument("--error-severity", "--severity", + help=f"error severity: {error_severity_enum}") + g_gen.add_argument("--gen-err-valid-bits", + "--generic-error-validation-bits", + help=f"validation bits: {validation_bits}") + g_gen.add_argument("--fru-id", type=guid.UUID, + help="GUID representing a physical device") + g_gen.add_argument("--fru-text", + help="ASCII string identifying the FRU hardware") + g_gen.add_argument("--timestamp", type=util.time, + help="Time when the error info was collected") + g_gen.add_argument("--precise", "--precise-timestamp", + action='store_true', + help="Marks the timestamp as precise if --timestamp is used") + g_gen.add_argument("--gedb-flags", + help=f"General Error Data Block flags: {gedb_flags_bits}") + + def set_args(self, args): + """Set the arguments optionally defined via self.argparse()""" + + if args.block_status: + self.block_status = util.get_choice(name="block-status", + value=args.block_status, + choices=self.BLOCK_STATUS_BITS, + bitmask=False) + if args.raw_data: + self.raw_data = util.get_array("raw-data", args.raw_data, + max_val=255) + print(self.raw_data) + + if args.error_severity: + self.error_severity = util.get_choice(name="error-severity", + value=args.error_severity, + choices=self.ERROR_SEVERITY, + bitmask=False) + + if args.fru_id: + self.fru_id = args.fru_id.to_bytes() + if not args.gen_err_valid_bits: + self.validation_bits |= self.VALIDATION_BITS["fru-id"] + + if args.fru_text: + text = bytearray(args.fru_text.encode('ascii')) + if len(text) > 20: + sys.exit("FRU text is too big to fit") + + self.fru_text = text + if not args.gen_err_valid_bits: + self.validation_bits |= self.VALIDATION_BITS["fru-text"] + + if args.timestamp: + time = args.timestamp + century = int(time.year / 100) + + bcd = bytearray() + util.data_add(bcd, (time.second // 10) << 4 | (time.second % 10), 1) + util.data_add(bcd, (time.minute // 10) << 4 | (time.minute % 10), 1) + util.data_add(bcd, (time.hour // 10) << 4 | (time.hour % 10), 1) + + if args.precise: + util.data_add(bcd, 1, 1) + else: + util.data_add(bcd, 0, 1) + + util.data_add(bcd, (time.day // 10) << 4 | (time.day % 10), 1) + util.data_add(bcd, (time.month // 10) << 4 | (time.month % 10), 1) + util.data_add(bcd, + ((time.year % 100) // 10) << 4 | (time.year % 10), 1) + util.data_add(bcd, ((century % 100) // 10) << 4 | (century % 10), 1) + + self.timestamp = bcd + if not args.gen_err_valid_bits: + self.validation_bits |= self.VALIDATION_BITS["timestamp"] + + if args.gen_err_valid_bits: + self.validation_bits = util.get_choice(name="validation", + value=args.gen_err_valid_bits, + choices=self.VALIDATION_BITS) + + def __init__(self, host, port, debug=False): + """Initialize variables used by the QMP send logic""" + + self.connected = False + self.host = host + self.port = port + self.debug = debug + + # ACPI 6.1: 18.3.2.7.1 Generic Error Data: Generic Error Status Block + self.block_status = self.BLOCK_STATUS_BITS["uncorrectable"] + self.raw_data = [] + self.error_severity = self.ERROR_SEVERITY["recoverable"] + + # ACPI 6.1: 18.3.2.7.1 Generic Error Data: Generic Error Data Entry + self.validation_bits = 0 + self.flags = 0 + self.fru_id = bytearray(16) + self.fru_text = bytearray(20) + self.timestamp = bytearray(8) + + self.qmp_monitor = QEMUMonitorProtocol(address=(self.host, self.port)) + + # + # Socket QMP send command + # + def send_cper_raw(self, cper_data): + """Send a raw CPER data to QEMU though QMP TCP socket""" + + data = b64encode(bytes(cper_data)).decode('ascii') + + cmd_arg = { + 'cper': data + } + + self._connect() + + if self.send_cmd("inject-ghes-v2-error", cmd_arg): + print("Error injected.") + + def send_cper(self, notif_type, payload): + """Send commands to QEMU though QMP TCP socket""" + + # Fill CPER record header + + # NOTE: bits 4 to 13 of block status contain the number of + # data entries in the data section. This is currently unsupported. + + cper_length = len(payload) + data_length = cper_length + len(self.raw_data) + self.GENERIC_DATA_SIZE + + # Generic Error Data Entry + gede = bytearray() + + gede.extend(notif_type.to_bytes()) + util.data_add(gede, self.error_severity, 4) + util.data_add(gede, 0x300, 2) + util.data_add(gede, self.validation_bits, 1) + util.data_add(gede, self.flags, 1) + util.data_add(gede, cper_length, 4) + gede.extend(self.fru_id) + gede.extend(self.fru_text) + gede.extend(self.timestamp) + + # Generic Error Status Block + gebs = bytearray() + + if self.raw_data: + raw_data_offset = len(gebs) + else: + raw_data_offset = 0 + + util.data_add(gebs, self.block_status, 4) + util.data_add(gebs, raw_data_offset, 4) + util.data_add(gebs, len(self.raw_data), 4) + util.data_add(gebs, data_length, 4) + util.data_add(gebs, self.error_severity, 4) + + cper_data = bytearray() + cper_data.extend(gebs) + cper_data.extend(gede) + cper_data.extend(bytearray(self.raw_data)) + cper_data.extend(bytearray(payload)) + + if self.debug: + print(f"GUID: {notif_type}") + + util.dump_bytearray("Generic Error Status Block", gebs) + util.dump_bytearray("Generic Error Data Entry", gede) + + if self.raw_data: + util.dump_bytearray("Raw data", bytearray(self.raw_data)) + + util.dump_bytearray("Payload", payload) + + self.send_cper_raw(cper_data) + + + def search_qom(self, path, prop, regex): + """ + Return a list of devices that match path array like: + + /machine/unattached/device + /machine/peripheral-anon/device + ... + """ + + found = [] + + i = 0 + while 1: + dev = f"{path}[{i}]" + args = { + 'path': dev, + 'property': prop + } + ret = self.send_cmd("qom-get", args, may_open=True, + return_error=False) + if not ret: + break + + if isinstance(ret, str): + if regex.search(ret): + found.append(dev) + + i += 1 + if i > 10000: + print("Too many objects returned by qom-get!") + break + + return found + +class cper_guid: + """ + Contains CPER GUID, as per: + https://uefi.org/specs/UEFI/2.10/Apx_N_Common_Platform_Error_Record.html + """ + + CPER_PROC_GENERIC = guid(0x9876CCAD, 0x47B4, 0x4bdb, + [0xB6, 0x5E, 0x16, 0xF1, + 0x93, 0xC4, 0xF3, 0xDB]) + + CPER_PROC_X86 = guid(0xDC3EA0B0, 0xA144, 0x4797, + [0xB9, 0x5B, 0x53, 0xFA, + 0x24, 0x2B, 0x6E, 0x1D]) + + CPER_PROC_ITANIUM = guid(0xe429faf1, 0x3cb7, 0x11d4, + [0xbc, 0xa7, 0x00, 0x80, + 0xc7, 0x3c, 0x88, 0x81]) + + CPER_PROC_ARM = guid(0xE19E3D16, 0xBC11, 0x11E4, + [0x9C, 0xAA, 0xC2, 0x05, + 0x1D, 0x5D, 0x46, 0xB0]) + + CPER_PLATFORM_MEM = guid(0xA5BC1114, 0x6F64, 0x4EDE, + [0xB8, 0x63, 0x3E, 0x83, + 0xED, 0x7C, 0x83, 0xB1]) + + CPER_PLATFORM_MEM2 = guid(0x61EC04FC, 0x48E6, 0xD813, + [0x25, 0xC9, 0x8D, 0xAA, + 0x44, 0x75, 0x0B, 0x12]) + + CPER_PCIE = guid(0xD995E954, 0xBBC1, 0x430F, + [0xAD, 0x91, 0xB4, 0x4D, + 0xCB, 0x3C, 0x6F, 0x35]) + + CPER_PCI_BUS = guid(0xC5753963, 0x3B84, 0x4095, + [0xBF, 0x78, 0xED, 0xDA, + 0xD3, 0xF9, 0xC9, 0xDD]) + + CPER_PCI_DEV = guid(0xEB5E4685, 0xCA66, 0x4769, + [0xB6, 0xA2, 0x26, 0x06, + 0x8B, 0x00, 0x13, 0x26]) + + CPER_FW_ERROR = guid(0x81212A96, 0x09ED, 0x4996, + [0x94, 0x71, 0x8D, 0x72, + 0x9C, 0x8E, 0x69, 0xED]) + + CPER_DMA_GENERIC = guid(0x5B51FEF7, 0xC79D, 0x4434, + [0x8F, 0x1B, 0xAA, 0x62, + 0xDE, 0x3E, 0x2C, 0x64]) + + CPER_DMA_VT = guid(0x71761D37, 0x32B2, 0x45cd, + [0xA7, 0xD0, 0xB0, 0xFE, + 0xDD, 0x93, 0xE8, 0xCF]) + + CPER_DMA_IOMMU = guid(0x036F84E1, 0x7F37, 0x428c, + [0xA7, 0x9E, 0x57, 0x5F, + 0xDF, 0xAA, 0x84, 0xEC]) + + CPER_CCIX_PER = guid(0x91335EF6, 0xEBFB, 0x4478, + [0xA6, 0xA6, 0x88, 0xB7, + 0x28, 0xCF, 0x75, 0xD7]) + + CPER_CXL_PROT_ERR = guid(0x80B9EFB4, 0x52B5, 0x4DE3, + [0xA7, 0x77, 0x68, 0x78, + 0x4B, 0x77, 0x10, 0x48]) diff --git a/scripts/rdma-migration-helper.sh b/scripts/rdma-migration-helper.sh new file mode 100755 index 0000000..d784d15 --- /dev/null +++ b/scripts/rdma-migration-helper.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +# Copied from blktests +get_ipv4_addr() +{ + ip -4 -o addr show dev "$1" | + sed -n 's/.*[[:blank:]]inet[[:blank:]]*\([^[:blank:]/]*\).*/\1/p' | + head -1 | tr -d '\n' +} + +get_ipv6_addr() { + ipv6=$(ip -6 -o addr show dev "$1" | + sed -n 's/.*[[:blank:]]inet6[[:blank:]]*\([^[:blank:]/]*\).*/\1/p' | + head -1 | tr -d '\n') + + [ $? -eq 0 ] || return + + if [[ "$ipv6" =~ ^fe80: ]]; then + echo -n "[$ipv6%$1]" + else + echo -n "[$ipv6]" + fi +} + +# existing rdma interfaces +rdma_interfaces() +{ + rdma link show | sed -nE 's/^link .* netdev ([^ ]+).*$/\1 /p' | + grep -Ev '^(lo|tun|tap)' +} + +# existing valid ipv4 interfaces +ipv4_interfaces() +{ + ip -o addr show | awk '/inet / {print $2}' | grep -Ev '^(lo|tun|tap)' +} + +ipv6_interfaces() +{ + ip -o addr show | awk '/inet6 / {print $2}' | grep -Ev '^(lo|tun|tap)' +} + +rdma_rxe_detect() +{ + family=$1 + for r in $(rdma_interfaces) + do + "$family"_interfaces | grep -qw $r && get_"$family"_addr $r && return + done + + return 1 +} + +rdma_rxe_setup() +{ + family=$1 + for i in $("$family"_interfaces) + do + if rdma_interfaces | grep -qw $i; then + echo "$family: Reuse the existing rdma/rxe ${i}_rxe" \ + "for $i with $(get_"$family"_addr $i)" + return + fi + + rdma link add "${i}_rxe" type rxe netdev "$i" && { + echo "$family: Setup new rdma/rxe ${i}_rxe" \ + "for $i with $(get_"$family"_addr $i)" + return + } + done + + echo "$family: Failed to setup any new rdma/rxe link" >&2 + return 1 +} + +rdma_rxe_clean() +{ + modprobe -r rdma_rxe +} + +IP_FAMILY=${IP_FAMILY:-ipv4} +if [ "$IP_FAMILY" != "ipv6" ] && [ "$IP_FAMILY" != "ipv4" ]; then + echo "Unknown ip family '$IP_FAMILY', only ipv4 or ipv6 is supported." >&2 + exit 1 +fi + +operation=${1:-detect} + +command -v rdma >/dev/null || { + echo "Command 'rdma' is not available, please install it first." >&2 + exit 1 +} + +if [ "$operation" == "setup" ] || [ "$operation" == "clean" ]; then + [ "$UID" == 0 ] || { + echo "Root privilege is required to setup/clean a rdma/rxe link" >&2 + exit 1 + } + if [ "$operation" == "setup" ]; then + rdma_rxe_setup ipv4 + rdma_rxe_setup ipv6 + else + rdma_rxe_clean + fi +elif [ "$operation" == "detect" ]; then + rdma_rxe_detect "$IP_FAMILY" +else + echo "Usage: $0 [setup | detect | clean]" +fi diff --git a/scripts/replay-dump.py b/scripts/replay-dump.py index 4ce7ff5..0976365 100755 --- a/scripts/replay-dump.py +++ b/scripts/replay-dump.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # Dump the contents of a recorded execution stream # diff --git a/scripts/rust-to-clang-target-test.sh b/scripts/rust-to-clang-target-test.sh new file mode 100755 index 0000000..ff6f8fc --- /dev/null +++ b/scripts/rust-to-clang-target-test.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env sh +# +# Copyright (C) 2025 Red Hat, Inc. +# +# Based on rust_to_clang_target() tests from rust-bindgen. +# +# SPDX-License-Identifier: GPL-2.0-or-later + +scripts_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$scripts_dir/rust-to-clang-target.sh" + +test_case() { + input="$1" + expected="$2" + result=$(rust_to_clang_target "$input") + + if [ "$result" = "$expected" ]; then + echo " OK: '$input' -> '$result'" + else + echo " FAILED: '$input'" + echo " Expected: '$expected'" + echo " Got: '$result'" + exit 1 + fi +} + +echo "Running tests..." + +test_case "aarch64-apple-ios" "arm64-apple-ios" +test_case "riscv64gc-unknown-linux-gnu" "riscv64-unknown-linux-gnu" +test_case "riscv64imac-unknown-none-elf" "riscv64-unknown-none-elf" +test_case "riscv32imc-unknown-none-elf" "riscv32-unknown-none-elf" +test_case "riscv32imac-unknown-none-elf" "riscv32-unknown-none-elf" +test_case "riscv32imafc-unknown-none-elf" "riscv32-unknown-none-elf" +test_case "riscv32i-unknown-none-elf" "riscv32-unknown-none-elf" +test_case "riscv32imc-esp-espidf" "riscv32-esp-elf" +test_case "xtensa-esp32-espidf" "xtensa-esp32-elf" +test_case "aarch64-apple-ios-sim" "arm64-apple-ios-simulator" +test_case "aarch64-apple-tvos-sim" "arm64-apple-tvos-simulator" +test_case "aarch64-apple-watchos-sim" "arm64-apple-watchos-simulator" + +echo "" +echo "All tests passed!" diff --git a/scripts/rust-to-clang-target.sh b/scripts/rust-to-clang-target.sh new file mode 100644 index 0000000..72db7e1 --- /dev/null +++ b/scripts/rust-to-clang-target.sh @@ -0,0 +1,60 @@ +# Copyright (C) 2025 Red Hat, Inc. +# +# Based on rust_to_clang_target() from rust-bindgen. +# +# SPDX-License-Identifier: GPL-2.0-or-later + +rust_to_clang_target() { + rust_target="$1" + + # Split the string by hyphens + triple_parts="" + old_IFS="$IFS" + IFS='-' + for part in $rust_target; do + triple_parts="$triple_parts $part" + done + IFS="$old_IFS" + set -- $triple_parts + + # RISC-V + case "$1" in + riscv32*) + set -- "riscv32" "${2}" "${3}" "${4}" + ;; + riscv64*) + set -- "riscv64" "${2}" "${3}" "${4}" + ;; + esac + + # Apple + if [ "$2" = "apple" ]; then + if [ "$1" = "aarch64" ]; then + set -- "arm64" "${2}" "${3}" "${4}" + fi + if [ "$4" = "sim" ]; then + set -- "${1}" "${2}" "${3}" "simulator" + fi + fi + + # ESP-IDF + if [ "$3" = "espidf" ]; then + set -- "${1}" "${2}" "elf" "${4}" + fi + + # Reassemble the string + new_triple="" + first=1 + for part in "$@"; do + if [ -n "$part" ]; then + if [ "$first" -eq 1 ]; then + new_triple="$part" + first=0 + else + new_triple="$new_triple-$part" + fi + fi + done + + echo "$new_triple" +} diff --git a/scripts/rust/rust_root_crate.sh b/scripts/rust/rust_root_crate.sh index 975bddf..f05b8d0 100755 --- a/scripts/rust/rust_root_crate.sh +++ b/scripts/rust/rust_root_crate.sh @@ -4,7 +4,7 @@ set -eu cat <<EOF // @generated -// This file is autogenerated by scripts/rust_root_crate.sh +// This file is autogenerated by scripts/rust/rust_root_crate.sh EOF diff --git a/scripts/rust/rustc_args.py b/scripts/rust/rustc_args.py index 2633157..63b0748 100644 --- a/scripts/rust/rustc_args.py +++ b/scripts/rust/rustc_args.py @@ -104,10 +104,7 @@ def generate_lint_flags(cargo_toml: CargoTOML, strict_lints: bool) -> Iterable[s else: raise Exception(f"invalid level {level} for {prefix}{lint}") - # This may change if QEMU ever invokes clippy-driver or rustdoc by - # hand. For now, check the syntax but do not add non-rustc lints to - # the command line. - if k == "rust" and not (strict_lints and lint in STRICT_LINTS): + if not (strict_lints and lint in STRICT_LINTS): lint_list.append(LintFlag(flags=[flag, prefix + lint], priority=priority)) if strict_lints: diff --git a/scripts/tracetool.py b/scripts/tracetool.py index 5de9ce9..0fdc9cb 100755 --- a/scripts/tracetool.py +++ b/scripts/tracetool.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Command-line wrapper for the tracetool machinery. diff --git a/scripts/tracetool/__init__.py b/scripts/tracetool/__init__.py index bc03238..74062d2 100644 --- a/scripts/tracetool/__init__.py +++ b/scripts/tracetool/__init__.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ Machinery for generating tracing-related intermediate files. @@ -12,12 +12,13 @@ __maintainer__ = "Stefan Hajnoczi" __email__ = "stefanha@redhat.com" +import os import re import sys -import weakref +from pathlib import PurePath -import tracetool.format import tracetool.backend +import tracetool.format def error_write(*lines): @@ -29,6 +30,49 @@ def error(*lines): error_write(*lines) sys.exit(1) +FMT_TOKEN = re.compile(r'''(?: + " ( (?: [^"\\] | \\[\\"abfnrt] | # a string literal + \\x[0-9a-fA-F][0-9a-fA-F]) *? ) " + | ( PRI [duixX] (?:8|16|32|64|PTR|MAX) ) # a PRIxxx macro + | \s+ # spaces (ignored) + )''', re.X) + +PRI_SIZE_MAP = { + '8': 'hh', + '16': 'h', + '32': '', + '64': 'll', + 'PTR': 't', + 'MAX': 'j', +} + +def expand_format_string(c_fmt, prefix=""): + def pri_macro_to_fmt(pri_macro): + assert pri_macro.startswith("PRI") + fmt_type = pri_macro[3] # 'd', 'i', 'u', or 'x' + fmt_size = pri_macro[4:] # '8', '16', '32', '64', 'PTR', 'MAX' + + size = PRI_SIZE_MAP.get(fmt_size, None) + if size is None: + raise Exception(f"unknown macro {pri_macro}") + return size + fmt_type + + result = prefix + pos = 0 + while pos < len(c_fmt): + m = FMT_TOKEN.match(c_fmt, pos) + if not m: + print("No match at position", pos, ":", repr(c_fmt[pos:]), file=sys.stderr) + raise Exception("syntax error in trace file") + if m[1]: + substr = m[1] + elif m[2]: + substr = pri_macro_to_fmt(m[2]) + else: + substr = "" + result += substr + pos = m.end() + return result out_lineno = 1 out_filename = '<none>' @@ -36,7 +80,7 @@ out_fobj = sys.stdout def out_open(filename): global out_filename, out_fobj - out_filename = filename + out_filename = posix_relpath(filename) out_fobj = open(filename, 'wt') def out(*lines, **kwargs): @@ -88,6 +132,49 @@ ALLOWED_TYPES = [ "ptrdiff_t", ] +C_TYPE_KEYWORDS = {"char", "int", "void", "short", "long", "signed", "unsigned"} + +C_TO_RUST_TYPE_MAP = { + "int": "std::ffi::c_int", + "long": "std::ffi::c_long", + "long long": "std::ffi::c_longlong", + "short": "std::ffi::c_short", + "char": "std::ffi::c_char", + "bool": "bool", + "unsigned": "std::ffi::c_uint", + # multiple keywords, keep them sorted + "long unsigned": "std::ffi::c_long", + "long long unsigned": "std::ffi::c_ulonglong", + "short unsigned": "std::ffi::c_ushort", + "char unsigned": "u8", + "int8_t": "i8", + "uint8_t": "u8", + "int16_t": "i16", + "uint16_t": "u16", + "int32_t": "i32", + "uint32_t": "u32", + "int64_t": "i64", + "uint64_t": "u64", + "void": "()", + "size_t": "usize", + "ssize_t": "isize", + "uintptr_t": "usize", + "ptrdiff_t": "isize", +} + +# Rust requires manual casting of <32-bit types when passing them to +# variable-argument functions. +RUST_VARARGS_SMALL_TYPES = { + "std::ffi::c_short", + "std::ffi::c_ushort", + "std::ffi::c_char", + "i8", + "u8", + "i16", + "u16", + "bool", +} + def validate_type(name): bits = name.split(" ") for bit in bits: @@ -103,6 +190,38 @@ def validate_type(name): "other complex pointer types should be " "declared as 'void *'" % name) +def c_type_to_rust(name): + ptr = False + const = False + name = name.rstrip() + if name[-1] == '*': + name = name[:-1].rstrip() + ptr = True + if name[-1] == '*': + # pointers to pointers are the same as void* + name = "void" + + bits = name.split() + if "const" in bits: + const = True + bits.remove("const") + if bits[0] in C_TYPE_KEYWORDS: + if "signed" in bits: + bits.remove("signed") + if len(bits) > 1 and "int" in bits: + bits.remove("int") + bits.sort() + name = ' '.join(bits) + else: + if len(bits) > 1: + raise ValueError("Invalid type '%s'." % name) + name = bits[0] + + ty = C_TO_RUST_TYPE_MAP[name.strip()] + if ptr: + ty = f'*{"const" if const else "mut"} {ty}' + return ty + class Arguments: """Event arguments description.""" @@ -120,10 +239,6 @@ class Arguments: else: self._args.append(arg) - def copy(self): - """Create a new copy.""" - return Arguments(list(self._args)) - @staticmethod def build(arg_str): """Build and Arguments instance from an argument string. @@ -168,10 +283,16 @@ class Arguments: def __str__(self): """String suitable for declaring function arguments.""" + def onearg(t, n): + if t[-1] == '*': + return "".join([t, n]) + else: + return " ".join([t, n]) + if len(self._args) == 0: return "void" else: - return ", ".join([ " ".join([t, n]) for t,n in self._args ]) + return ", ".join([ onearg(t, n) for t,n in self._args ]) def __repr__(self): """Evaluable string representation for this object.""" @@ -189,6 +310,43 @@ class Arguments: """List of argument names casted to their type.""" return ["(%s)%s" % (type_, name) for type_, name in self._args] + def rust_decl_extern(self): + """Return a Rust argument list for an extern "C" function""" + return ", ".join((f"_{name}: {c_type_to_rust(type_)}" + for type_, name in self._args)) + + def rust_decl(self): + """Return a Rust argument list for a tracepoint function""" + def decl_type(type_): + if type_ == "const char *": + return "&std::ffi::CStr" + return c_type_to_rust(type_) + + return ", ".join((f"_{name}: {decl_type(type_)}" + for type_, name in self._args)) + + def rust_call_extern(self): + """Return a Rust argument list for a call to an extern "C" function""" + def rust_cast(name, type_): + if type_ == "const char *": + return f"_{name}.as_ptr()" + return f"_{name}" + + return ", ".join((rust_cast(name, type_) for type_, name in self._args)) + + def rust_call_varargs(self): + """Return a Rust argument list for a call to a C varargs function""" + def rust_cast(name, type_): + if type_ == "const char *": + return f"_{name}.as_ptr()" + + type_ = c_type_to_rust(type_) + if type_ in RUST_VARARGS_SMALL_TYPES: + return f"_{name} as std::ffi::c_int" + return f"_{name} /* as {type_} */" + + return ", ".join((rust_cast(name, type_) for type_, name in self._args)) + class Event(object): """Event description. @@ -214,13 +372,12 @@ class Event(object): r"(?P<name>\w+)" r"\((?P<args>[^)]*)\)" r"\s*" - r"(?:(?:(?P<fmt_trans>\".+),)?\s*(?P<fmt>\".+))?" + r"(?P<fmt>\".+)?" r"\s*") - _VALID_PROPS = set(["disable", "vcpu"]) + _VALID_PROPS = set(["disable"]) - def __init__(self, name, props, fmt, args, lineno, filename, orig=None, - event_trans=None, event_exec=None): + def __init__(self, name, props, fmt, args, lineno, filename): """ Parameters ---------- @@ -228,20 +385,14 @@ class Event(object): Event name. props : list of str Property names. - fmt : str, list of str - Event printing format string(s). + fmt : str + Event printing format string. args : Arguments Event arguments. lineno : int The line number in the input file. filename : str The path to the input file. - orig : Event or None - Original Event before transformation/generation. - event_trans : Event or None - Generated translation-time event ("tcg" property). - event_exec : Event or None - Generated execution-time event ("tcg" property). """ self.name = name @@ -250,29 +401,16 @@ class Event(object): self.args = args self.lineno = int(lineno) self.filename = str(filename) - self.event_trans = event_trans - self.event_exec = event_exec if len(args) > 10: raise ValueError("Event '%s' has more than maximum permitted " "argument count" % name) - if orig is None: - self.original = weakref.ref(self) - else: - self.original = orig - unknown_props = set(self.properties) - self._VALID_PROPS if len(unknown_props) > 0: raise ValueError("Unknown properties: %s" % ", ".join(unknown_props)) - assert isinstance(self.fmt, str) or len(self.fmt) == 2 - def copy(self): - """Create a new copy.""" - return Event(self.name, list(self.properties), self.fmt, - self.args.copy(), self.lineno, self.filename, - self, self.event_trans, self.event_exec) @staticmethod def build(line_str, lineno, filename): @@ -294,8 +432,7 @@ class Event(object): name = groups["name"] props = groups["props"].split() fmt = groups["fmt"] - fmt_trans = groups["fmt_trans"] - if fmt.find("%m") != -1 or fmt_trans.find("%m") != -1: + if fmt.find("%m") != -1: raise ValueError("Event format '%m' is forbidden, pass the error " "as an explicit trace argument") if fmt.endswith(r'\n"'): @@ -304,33 +441,25 @@ class Event(object): if '\\n' in fmt: raise ValueError("Event format must not use new line character") - if len(fmt_trans) > 0: - fmt = [fmt_trans, fmt] args = Arguments.build(groups["args"]) - return Event(name, props, fmt, args, lineno, filename) + return Event(name, props, fmt, args, lineno, posix_relpath(filename)) def __repr__(self): """Evaluable string representation for this object.""" - if isinstance(self.fmt, str): - fmt = self.fmt - else: - fmt = "%s, %s" % (self.fmt[0], self.fmt[1]) return "Event('%s %s(%s) %s')" % (" ".join(self.properties), self.name, self.args, - fmt) + self.fmt) # Star matching on PRI is dangerous as one might have multiple # arguments with that format, hence the non-greedy version of it. _FMT = re.compile(r"(%[\d\.]*\w+|%.*?PRI\S+)") def formats(self): """List conversion specifiers in the argument print format string.""" - assert not isinstance(self.fmt, list) return self._FMT.findall(self.fmt) QEMU_TRACE = "trace_%(name)s" - QEMU_TRACE_NOCHECK = "_nocheck__" + QEMU_TRACE QEMU_TRACE_TCG = QEMU_TRACE + "_tcg" QEMU_DSTATE = "_TRACE_%(NAME)s_DSTATE" QEMU_BACKEND_DSTATE = "TRACE_%(NAME)s_BACKEND_DSTATE" @@ -447,3 +576,10 @@ def generate(events, group, format, backends, tracetool.backend.dtrace.PROBEPREFIX = probe_prefix tracetool.format.generate(events, format, backend, group) + +def posix_relpath(path, start=None): + try: + path = os.path.relpath(path, start) + except ValueError: + pass + return PurePath(path).as_posix() diff --git a/scripts/tracetool/backend/__init__.py b/scripts/tracetool/backend/__init__.py index 7bfcc86..9109a78 100644 --- a/scripts/tracetool/backend/__init__.py +++ b/scripts/tracetool/backend/__init__.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ Backend management. @@ -19,11 +19,15 @@ All backends must generate their contents through the 'tracetool.out' routine. Backend attributes ------------------ -========= ==================================================================== -Attribute Description -========= ==================================================================== -PUBLIC If exists and is set to 'True', the backend is considered "public". -========= ==================================================================== +=========================== ==================================================== +Attribute Description +=========================== ==================================================== +PUBLIC If exists and is set to 'True', the backend is + considered "public". +CHECK_TRACE_EVENT_GET_STATE If exists and is set to 'True', the backend-specific + code inside the tracepoint is emitted within an + ``if trace_event_get_state()`` conditional. +=========================== ==================================================== Backend functions @@ -94,29 +98,40 @@ def exists(name): if name == "nop": return True name = name.replace("-", "_") - return tracetool.try_import("tracetool.backend." + name)[1] + return tracetool.try_import("tracetool.backend." + name)[0] class Wrapper: def __init__(self, backends, format): self._backends = [backend.replace("-", "_") for backend in backends] self._format = format.replace("-", "_") + self.check_trace_event_get_state = False for backend in self._backends: assert exists(backend) assert tracetool.format.exists(self._format) + for backend in self.backend_modules(): + check_trace_event_get_state = getattr(backend, "CHECK_TRACE_EVENT_GET_STATE", False) + self.check_trace_event_get_state = self.check_trace_event_get_state or check_trace_event_get_state - def _run_function(self, name, *args, **kwargs): + def backend_modules(self): for backend in self._backends: - func = tracetool.try_import("tracetool.backend." + backend, - name % self._format, None)[1] - if func is not None: - func(*args, **kwargs) + module = tracetool.try_import("tracetool.backend." + backend)[1] + if module is not None: + yield module + + def _run_function(self, name, *args, check_trace_event_get_state=None, **kwargs): + for backend in self.backend_modules(): + func = getattr(backend, name % self._format, None) + if func is not None and \ + (check_trace_event_get_state is None or + check_trace_event_get_state == getattr(backend, 'CHECK_TRACE_EVENT_GET_STATE', False)): + func(*args, **kwargs) def generate_begin(self, events, group): self._run_function("generate_%s_begin", events, group) - def generate(self, event, group): - self._run_function("generate_%s", event, group) + def generate(self, event, group, check_trace_event_get_state=None): + self._run_function("generate_%s", event, group, check_trace_event_get_state=check_trace_event_get_state) def generate_backend_dstate(self, event, group): self._run_function("generate_%s_backend_dstate", event, group) diff --git a/scripts/tracetool/backend/dtrace.py b/scripts/tracetool/backend/dtrace.py index e17edc9..b4af403 100644 --- a/scripts/tracetool/backend/dtrace.py +++ b/scripts/tracetool/backend/dtrace.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ DTrace/SystemTAP backend. diff --git a/scripts/tracetool/backend/ftrace.py b/scripts/tracetool/backend/ftrace.py index baed2ae..e03698a 100644 --- a/scripts/tracetool/backend/ftrace.py +++ b/scripts/tracetool/backend/ftrace.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ Ftrace built-in backend. @@ -12,12 +12,11 @@ __maintainer__ = "Stefan Hajnoczi" __email__ = "stefanha@redhat.com" -import os.path - -from tracetool import out +from tracetool import out, expand_format_string PUBLIC = True +CHECK_TRACE_EVENT_GET_STATE = True def generate_h_begin(events, group): @@ -30,24 +29,13 @@ def generate_h(event, group): if len(event.args) > 0: argnames = ", " + argnames - out(' {', - ' char ftrace_buf[MAX_TRACE_STRLEN];', - ' int unused __attribute__ ((unused));', - ' int trlen;', - ' if (trace_event_get_state(%(event_id)s)) {', - '#line %(event_lineno)d "%(event_filename)s"', - ' trlen = snprintf(ftrace_buf, MAX_TRACE_STRLEN,', - ' "%(name)s " %(fmt)s "\\n" %(argnames)s);', + out('#line %(event_lineno)d "%(event_filename)s"', + ' ftrace_write("%(name)s " %(fmt)s "\\n" %(argnames)s);', '#line %(out_next_lineno)d "%(out_filename)s"', - ' trlen = MIN(trlen, MAX_TRACE_STRLEN - 1);', - ' unused = write(trace_marker_fd, ftrace_buf, trlen);', - ' }', - ' }', name=event.name, args=event.args, - event_id="TRACE_" + event.name.upper(), event_lineno=event.lineno, - event_filename=os.path.relpath(event.filename), + event_filename=event.filename, fmt=event.fmt.rstrip("\n"), argnames=argnames) @@ -55,3 +43,9 @@ def generate_h(event, group): def generate_h_backend_dstate(event, group): out(' trace_event_get_state_dynamic_by_id(%(event_id)s) || \\', event_id="TRACE_" + event.name.upper()) + +def generate_rs(event, group): + out(' let format_string = c"%(fmt)s";', + ' unsafe {bindings::ftrace_write(format_string.as_ptr() as *const c_char, %(args)s);}', + fmt=expand_format_string(event.fmt), + args=event.args.rust_call_varargs()) diff --git a/scripts/tracetool/backend/log.py b/scripts/tracetool/backend/log.py index de27b7e..9e3e504 100644 --- a/scripts/tracetool/backend/log.py +++ b/scripts/tracetool/backend/log.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ Stderr built-in backend. @@ -12,17 +12,15 @@ __maintainer__ = "Stefan Hajnoczi" __email__ = "stefanha@redhat.com" -import os.path - -from tracetool import out +from tracetool import out, expand_format_string PUBLIC = True +CHECK_TRACE_EVENT_GET_STATE = True def generate_h_begin(events, group): out('#include "qemu/log-for-trace.h"', - '#include "qemu/error-report.h"', '') @@ -31,31 +29,13 @@ def generate_h(event, group): if len(event.args) > 0: argnames = ", " + argnames - if "vcpu" in event.properties: - # already checked on the generic format code - cond = "true" - else: - cond = "trace_event_get_state(%s)" % ("TRACE_" + event.name.upper()) - - out(' if (%(cond)s && qemu_loglevel_mask(LOG_TRACE)) {', - ' if (message_with_timestamp) {', - ' struct timeval _now;', - ' gettimeofday(&_now, NULL);', - '#line %(event_lineno)d "%(event_filename)s"', - ' qemu_log("%%d@%%zu.%%06zu:%(name)s " %(fmt)s "\\n",', - ' qemu_get_thread_id(),', - ' (size_t)_now.tv_sec, (size_t)_now.tv_usec', - ' %(argnames)s);', - '#line %(out_next_lineno)d "%(out_filename)s"', - ' } else {', + out(' if (qemu_loglevel_mask(LOG_TRACE)) {', '#line %(event_lineno)d "%(event_filename)s"', ' qemu_log("%(name)s " %(fmt)s "\\n"%(argnames)s);', '#line %(out_next_lineno)d "%(out_filename)s"', ' }', - ' }', - cond=cond, event_lineno=event.lineno, - event_filename=os.path.relpath(event.filename), + event_filename=event.filename, name=event.name, fmt=event.fmt.rstrip("\n"), argnames=argnames) @@ -64,3 +44,11 @@ def generate_h(event, group): def generate_h_backend_dstate(event, group): out(' trace_event_get_state_dynamic_by_id(%(event_id)s) || \\', event_id="TRACE_" + event.name.upper()) + +def generate_rs(event, group): + out(' let format_string = c"%(fmt)s\\n";', + ' if (unsafe { bindings::qemu_loglevel } & bindings::LOG_TRACE) != 0 {', + ' unsafe { bindings::qemu_log(format_string.as_ptr() as *const c_char, %(args)s);}', + ' }', + fmt=expand_format_string(event.fmt, event.name + " "), + args=event.args.rust_call_varargs()) diff --git a/scripts/tracetool/backend/simple.py b/scripts/tracetool/backend/simple.py index a74d61f..b131e4f 100644 --- a/scripts/tracetool/backend/simple.py +++ b/scripts/tracetool/backend/simple.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ Simple built-in backend. @@ -16,6 +16,7 @@ from tracetool import out PUBLIC = True +CHECK_TRACE_EVENT_GET_STATE = True def is_string(arg): @@ -36,7 +37,7 @@ def generate_h_begin(events, group): def generate_h(event, group): - out(' _simple_%(api)s(%(args)s);', + out(' _simple_%(api)s(%(args)s);', api=event.api(), args=", ".join(event.args.names())) @@ -72,22 +73,10 @@ def generate_c(event, group): if len(event.args) == 0: sizestr = '0' - event_id = 'TRACE_' + event.name.upper() - if "vcpu" in event.properties: - # already checked on the generic format code - cond = "true" - else: - cond = "trace_event_get_state(%s)" % event_id - out('', - ' if (!%(cond)s) {', - ' return;', - ' }', - '', ' if (trace_record_start(&rec, %(event_obj)s.id, %(size_str)s)) {', ' return; /* Trace Buffer Full, Event Dropped ! */', ' }', - cond=cond, event_obj=event.api(event.QEMU_EVENT), size_str=sizestr) @@ -109,3 +98,10 @@ def generate_c(event, group): out(' trace_record_finish(&rec);', '}', '') + +def generate_rs(event, group): + out(' extern "C" { fn _simple_%(api)s(%(rust_args)s); }', + ' unsafe { _simple_%(api)s(%(args)s); }', + api=event.api(), + rust_args=event.args.rust_decl_extern(), + args=event.args.rust_call_extern()) diff --git a/scripts/tracetool/backend/syslog.py b/scripts/tracetool/backend/syslog.py index 012970f..12b8265 100644 --- a/scripts/tracetool/backend/syslog.py +++ b/scripts/tracetool/backend/syslog.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ Syslog built-in backend. @@ -12,12 +12,11 @@ __maintainer__ = "Stefan Hajnoczi" __email__ = "stefanha@redhat.com" -import os.path - -from tracetool import out +from tracetool import out, expand_format_string PUBLIC = True +CHECK_TRACE_EVENT_GET_STATE = True def generate_h_begin(events, group): @@ -30,24 +29,20 @@ def generate_h(event, group): if len(event.args) > 0: argnames = ", " + argnames - if "vcpu" in event.properties: - # already checked on the generic format code - cond = "true" - else: - cond = "trace_event_get_state(%s)" % ("TRACE_" + event.name.upper()) - - out(' if (%(cond)s) {', - '#line %(event_lineno)d "%(event_filename)s"', + out('#line %(event_lineno)d "%(event_filename)s"', ' syslog(LOG_INFO, "%(name)s " %(fmt)s %(argnames)s);', '#line %(out_next_lineno)d "%(out_filename)s"', - ' }', - cond=cond, event_lineno=event.lineno, - event_filename=os.path.relpath(event.filename), + event_filename=event.filename, name=event.name, fmt=event.fmt.rstrip("\n"), argnames=argnames) +def generate_rs(event, group): + out(' let format_string = c"%(fmt)s";', + ' unsafe {::trace::syslog(::trace::LOG_INFO, format_string.as_ptr() as *const c_char, %(args)s);}', + fmt=expand_format_string(event.fmt), + args=event.args.rust_call_varargs()) def generate_h_backend_dstate(event, group): out(' trace_event_get_state_dynamic_by_id(%(event_id)s) || \\', diff --git a/scripts/tracetool/backend/ust.py b/scripts/tracetool/backend/ust.py index c857516..3aa9bb1 100644 --- a/scripts/tracetool/backend/ust.py +++ b/scripts/tracetool/backend/ust.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ LTTng User Space Tracing backend. diff --git a/scripts/tracetool/format/__init__.py b/scripts/tracetool/format/__init__.py index 2dc46f3..7b9d1b5 100644 --- a/scripts/tracetool/format/__init__.py +++ b/scripts/tracetool/format/__init__.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ Format management. @@ -70,7 +70,7 @@ def exists(name): if len(name) == 0: return False name = name.replace("-", "_") - return tracetool.try_import("tracetool.format." + name)[1] + return tracetool.try_import("tracetool.format." + name)[0] def generate(events, format, backend, group): diff --git a/scripts/tracetool/format/c.py b/scripts/tracetool/format/c.py index 69edf0d..50e0331 100644 --- a/scripts/tracetool/format/c.py +++ b/scripts/tracetool/format/c.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ trace/generated-tracers.c @@ -22,6 +22,7 @@ def generate(events, backend, group): header = "trace-" + group + ".h" out('/* This file is autogenerated by tracetool, do not edit. */', + '/* SPDX-License-Identifier: GPL-2.0-or-later */', '', '#include "qemu/osdep.h"', '#include "qemu/module.h"', @@ -36,7 +37,7 @@ def generate(events, backend, group): ' .id = 0,', ' .name = \"%(name)s\",', ' .sstate = %(sstate)s,', - ' .dstate = &%(dstate)s ', + ' .dstate = &%(dstate)s', '};', event = e.api(e.QEMU_EVENT), name = e.name, diff --git a/scripts/tracetool/format/d.py b/scripts/tracetool/format/d.py index ebfb714..e9e33df 100644 --- a/scripts/tracetool/format/d.py +++ b/scripts/tracetool/format/d.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ trace/generated-tracers.dtrace (DTrace only). @@ -39,7 +39,8 @@ def generate(events, backend, group): if not events and platform != "darwin": return - out('/* This file is autogenerated by tracetool, do not edit. */' + out('/* This file is autogenerated by tracetool, do not edit. */', + '/* SPDX-License-Identifier: GPL-2.0-or-later */', '', 'provider qemu {') diff --git a/scripts/tracetool/format/h.py b/scripts/tracetool/format/h.py index ea126b0..dd58713 100644 --- a/scripts/tracetool/format/h.py +++ b/scripts/tracetool/format/h.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ trace/generated-tracers.h @@ -19,6 +19,7 @@ def generate(events, backend, group): header = "trace/control.h" out('/* This file is autogenerated by tracetool, do not edit. */', + '/* SPDX-License-Identifier: GPL-2.0-or-later */', '', '#ifndef TRACE_%s_GENERATED_TRACERS_H' % group.upper(), '#define TRACE_%s_GENERATED_TRACERS_H' % group.upper(), @@ -39,11 +40,6 @@ def generate(events, backend, group): enabled = 0 else: enabled = 1 - if "tcg-exec" in e.properties: - # a single define for the two "sub-events" - out('#define TRACE_%(name)s_ENABLED %(enabled)d', - name=e.original.name.upper(), - enabled=enabled) out('#define TRACE_%s_ENABLED %d' % (e.name.upper(), enabled)) backend.generate_begin(events, group) @@ -59,33 +55,24 @@ def generate(events, backend, group): out(' false)') - # tracer without checks out('', 'static inline void %(api)s(%(args)s)', '{', - api=e.api(e.QEMU_TRACE_NOCHECK), + api=e.api(), args=e.args) if "disable" not in e.properties: - backend.generate(e, group) - + backend.generate(e, group, check_trace_event_get_state=False) + + if backend.check_trace_event_get_state: + event_id = 'TRACE_' + e.name.upper() + cond = "trace_event_get_state(%s)" % event_id + out(' if (%(cond)s) {', + cond=cond) + backend.generate(e, group, check_trace_event_get_state=True) + out(' }') out('}') - cond = "true" - - out('', - 'static inline void %(api)s(%(args)s)', - '{', - ' if (%(cond)s) {', - ' %(api_nocheck)s(%(names)s);', - ' }', - '}', - api=e.api(), - api_nocheck=e.api(e.QEMU_TRACE_NOCHECK), - args=e.args, - names=", ".join(e.args.names()), - cond=cond) - backend.generate_end(events, group) out('#endif /* TRACE_%s_GENERATED_TRACERS_H */' % group.upper()) diff --git a/scripts/tracetool/format/log_stap.py b/scripts/tracetool/format/log_stap.py index b49afab..259303a 100644 --- a/scripts/tracetool/format/log_stap.py +++ b/scripts/tracetool/format/log_stap.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ Generate .stp file that printfs log messages (DTrace with SystemTAP only). @@ -18,8 +18,6 @@ from tracetool.backend.dtrace import binary, probeprefix from tracetool.backend.simple import is_string from tracetool.format.stap import stap_escape -def global_var_name(name): - return probeprefix().replace(".", "_") + "_" + name STATE_SKIP = 0 STATE_LITERAL = 1 @@ -88,6 +86,7 @@ def c_fmt_to_stap(fmt): def generate(events, backend, group): out('/* This file is autogenerated by tracetool, do not edit. */', + '/* SPDX-License-Identifier: GPL-2.0-or-later */', '') for event_id, e in enumerate(events): diff --git a/scripts/tracetool/format/rs.py b/scripts/tracetool/format/rs.py new file mode 100644 index 0000000..32ac4e5 --- /dev/null +++ b/scripts/tracetool/format/rs.py @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: GPL-2.0-or-later + +""" +trace-DIR.rs +""" + +__author__ = "Tanish Desai <tanishdesai37@gmail.com>" +__copyright__ = "Copyright 2025, Tanish Desai <tanishdesai37@gmail.com>" +__license__ = "GPL version 2 or (at your option) any later version" + +__maintainer__ = "Stefan Hajnoczi" +__email__ = "stefanha@redhat.com" + + +from tracetool import out + + +def generate(events, backend, group): + out('// SPDX-License-Identifier: GPL-2.0-or-later', + '// This file is @generated by tracetool, do not edit.', + '', + '#[allow(unused_imports)]', + 'use std::ffi::c_char;', + '#[allow(unused_imports)]', + 'use util::bindings;', + '', + '#[inline(always)]', + 'fn trace_event_state_is_enabled(dstate: u16) -> bool {', + ' (unsafe { trace_events_enabled_count }) != 0 && dstate != 0', + '}', + '', + 'extern "C" {', + ' static mut trace_events_enabled_count: u32;', + '}',) + + out('extern "C" {') + + for e in events: + out(' static mut %s: u16;' % e.api(e.QEMU_DSTATE)) + out('}') + + backend.generate_begin(events, group) + + for e in events: + out('', + '#[inline(always)]', + '#[allow(dead_code)]', + 'pub fn %(api)s(%(args)s)', + '{', + api=e.api(e.QEMU_TRACE), + args=e.args.rust_decl()) + + if "disable" not in e.properties: + backend.generate(e, group, check_trace_event_get_state=False) + if backend.check_trace_event_get_state: + event_id = 'TRACE_' + e.name.upper() + out(' if trace_event_state_is_enabled(unsafe { _%(event_id)s_DSTATE}) {', + event_id = event_id, + api=e.api()) + backend.generate(e, group, check_trace_event_get_state=True) + out(' }') + out('}') + + backend.generate_end(events, group) diff --git a/scripts/tracetool/format/simpletrace_stap.py b/scripts/tracetool/format/simpletrace_stap.py index 4f4633b..c7bde97 100644 --- a/scripts/tracetool/format/simpletrace_stap.py +++ b/scripts/tracetool/format/simpletrace_stap.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ Generate .stp file that outputs simpletrace binary traces (DTrace with SystemTAP only). @@ -17,11 +17,10 @@ from tracetool.backend.dtrace import probeprefix from tracetool.backend.simple import is_string from tracetool.format.stap import stap_escape -def global_var_name(name): - return probeprefix().replace(".", "_") + "_" + name def generate(events, backend, group): out('/* This file is autogenerated by tracetool, do not edit. */', + '/* SPDX-License-Identifier: GPL-2.0-or-later */', '') for event_id, e in enumerate(events): diff --git a/scripts/tracetool/format/stap.py b/scripts/tracetool/format/stap.py index a218b04..285c920 100644 --- a/scripts/tracetool/format/stap.py +++ b/scripts/tracetool/format/stap.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ Generate .stp file (DTrace with SystemTAP only). @@ -38,6 +38,7 @@ def generate(events, backend, group): if "disable" not in e.properties] out('/* This file is autogenerated by tracetool, do not edit. */', + '/* SPDX-License-Identifier: GPL-2.0-or-later */', '') for e in events: diff --git a/scripts/tracetool/format/ust_events_c.py b/scripts/tracetool/format/ust_events_c.py index deced95..074226b 100644 --- a/scripts/tracetool/format/ust_events_c.py +++ b/scripts/tracetool/format/ust_events_c.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ trace/generated-ust.c @@ -20,6 +20,7 @@ def generate(events, backend, group): if "disabled" not in e.properties] out('/* This file is autogenerated by tracetool, do not edit. */', + '/* SPDX-License-Identifier: GPL-2.0-or-later */', '', '#include "qemu/osdep.h"', '', diff --git a/scripts/tracetool/format/ust_events_h.py b/scripts/tracetool/format/ust_events_h.py index b99fe68..cee7970 100644 --- a/scripts/tracetool/format/ust_events_h.py +++ b/scripts/tracetool/format/ust_events_h.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# SPDX-License-Identifier: GPL-2.0-or-later """ trace/generated-ust-provider.h @@ -25,6 +25,7 @@ def generate(events, backend, group): include = "trace-ust.h" out('/* This file is autogenerated by tracetool, do not edit. */', + '/* SPDX-License-Identifier: GPL-2.0-or-later */', '', '#undef TRACEPOINT_PROVIDER', '#define TRACEPOINT_PROVIDER qemu', diff --git a/scripts/update-linux-headers.sh b/scripts/update-linux-headers.sh index 8913e4f..844d9cb 100755 --- a/scripts/update-linux-headers.sh +++ b/scripts/update-linux-headers.sh @@ -90,6 +90,7 @@ cp_portable() { -e 's/<linux\/\([^>]*\)>/"standard-headers\/linux\/\1"/' \ -e "$arch_cmd" \ -e 's/__bitwise//' \ + -e 's/__counted_by(\w*)//' \ -e 's/__attribute__((packed))/QEMU_PACKED/' \ -e 's/__inline__/inline/' \ -e 's/__BITS_PER_LONG/HOST_LONG_BITS/' \ @@ -156,11 +157,6 @@ EOF cp "$hdrdir/include/asm/unistd_32.h" "$output/linux-headers/asm-s390/" cp "$hdrdir/include/asm/unistd_64.h" "$output/linux-headers/asm-s390/" fi - if [ $arch = arm ]; then - cp "$hdrdir/include/asm/unistd-eabi.h" "$output/linux-headers/asm-arm/" - cp "$hdrdir/include/asm/unistd-oabi.h" "$output/linux-headers/asm-arm/" - cp "$hdrdir/include/asm/unistd-common.h" "$output/linux-headers/asm-arm/" - fi if [ $arch = arm64 ]; then cp "$hdrdir/include/asm/sve_context.h" "$output/linux-headers/asm-arm64/" cp "$hdrdir/include/asm/unistd_64.h" "$output/linux-headers/asm-arm64/" @@ -177,7 +173,7 @@ EOF # Remove everything except the macros from bootparam.h avoiding the # unnecessary import of several video/ist/etc headers - sed -e '/__ASSEMBLY__/,/__ASSEMBLY__/d' \ + sed -e '/__ASSEMBLER__/,/__ASSEMBLER__/d' \ "$hdrdir/include/asm/bootparam.h" > "$hdrdir/bootparam.h" cp_portable "$hdrdir/bootparam.h" \ "$output/include/standard-headers/asm-$arch" @@ -200,7 +196,7 @@ rm -rf "$output/linux-headers/linux" mkdir -p "$output/linux-headers/linux" for header in const.h stddef.h kvm.h vfio.h vfio_ccw.h vfio_zdev.h vhost.h \ psci.h psp-sev.h userfaultfd.h memfd.h mman.h nvme_ioctl.h \ - vduse.h iommufd.h bits.h; do + vduse.h iommufd.h bits.h mshv.h; do cp "$hdrdir/include/linux/$header" "$output/linux-headers/linux" done diff --git a/scripts/userfaultfd-wrlat.py b/scripts/userfaultfd-wrlat.py index 0684be4..a61a9ab 100755 --- a/scripts/userfaultfd-wrlat.py +++ b/scripts/userfaultfd-wrlat.py @@ -17,7 +17,6 @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from __future__ import print_function from bcc import BPF from ctypes import c_ushort, c_int, c_ulonglong from time import sleep diff --git a/scripts/vmstate-static-checker.py b/scripts/vmstate-static-checker.py index 9c0e6b8..89b100e 100755 --- a/scripts/vmstate-static-checker.py +++ b/scripts/vmstate-static-checker.py @@ -21,6 +21,7 @@ import argparse import json +import pathlib import sys # Count the number of errors found @@ -42,6 +43,7 @@ def check_fields_match(name, s_field, d_field): # Some fields changed names between qemu versions. This list # is used to allow such changes in each section / description. changed_names = { + 'acpi-ghes': ['ghes_addr_le', 'hw_error_le'], 'apic': ['timer', 'timer_expiry'], 'e1000': ['dev', 'parent_obj'], 'ehci': ['dev', 'pcidev'], @@ -90,6 +92,7 @@ def check_fields_match(name, s_field, d_field): 'mem_win_size', 'mig_mem_win_size', 'io_win_addr', 'mig_io_win_addr', 'io_win_size', 'mig_io_win_size'], + 'hpet': ['num_timers', 'num_timers_save'], } if not name in changed_names: @@ -380,10 +383,10 @@ def main(): help_text = "Parse JSON-formatted vmstate dumps from QEMU in files SRC and DEST. Checks whether migration from SRC to DEST QEMU versions would break based on the VMSTATE information contained within the JSON outputs. The JSON output is created from a QEMU invocation with the -dump-vmstate parameter and a filename argument to it. Other parameters to QEMU do not matter, except the -M (machine type) parameter." parser = argparse.ArgumentParser(description=help_text) - parser.add_argument('-s', '--src', type=argparse.FileType('r'), + parser.add_argument('-s', '--src', type=pathlib.Path, required=True, help='json dump from src qemu') - parser.add_argument('-d', '--dest', type=argparse.FileType('r'), + parser.add_argument('-d', '--dest', type=pathlib.Path, required=True, help='json dump from dest qemu') parser.add_argument('--reverse', required=False, default=False, @@ -391,10 +394,10 @@ def main(): help='reverse the direction') args = parser.parse_args() - src_data = json.load(args.src) - dest_data = json.load(args.dest) - args.src.close() - args.dest.close() + with open(args.src, 'r', encoding='utf-8') as src_fh: + src_data = json.load(src_fh) + with open(args.dest, 'r', encoding='utf-8') as dst_fh: + dest_data = json.load(dst_fh) if args.reverse: temp = src_data |