diff options
Diffstat (limited to 'scripts')
64 files changed, 2255 insertions, 656 deletions
diff --git a/scripts/analyze-inclusions b/scripts/analyze-inclusions index b6280f2..d2c5666 100644 --- a/scripts/analyze-inclusions +++ b/scripts/analyze-inclusions @@ -53,7 +53,7 @@ echo $(grep_include -F 'trace/generated-tracers.h') files include generated-trac echo $(grep_include -F 'qapi/error.h') files include qapi/error.h echo $(grep_include -F 'qom/object.h') files include qom/object.h echo $(grep_include -F 'block/aio.h') files include block/aio.h -echo $(grep_include -F 'exec/memory.h') files include exec/memory.h +echo $(grep_include -F 'system/memory.h') files include system/memory.h echo $(grep_include -F 'fpu/softfloat.h') files include fpu/softfloat.h echo $(grep_include -F 'qemu/bswap.h') files include qemu/bswap.h echo diff --git a/scripts/analyze-migration.py b/scripts/analyze-migration.py index 8a254a5..67631ac 100755 --- a/scripts/analyze-migration.py +++ b/scripts/analyze-migration.py @@ -65,6 +65,9 @@ class MigrationFile(object): def tell(self): return self.file.tell() + def seek(self, a, b): + return self.file.seek(a, b) + # The VMSD description is at the end of the file, after EOF. Look for # the last NULL byte, then for the beginning brace of JSON. def read_migration_debug_json(self): @@ -272,11 +275,24 @@ class S390StorageAttributes(object): self.section_key = section_key def read(self): + pos = 0 while True: addr_flags = self.file.read64() flags = addr_flags & 0xfff - if (flags & (self.STATTR_FLAG_DONE | self.STATTR_FLAG_EOS)): + + if flags & self.STATTR_FLAG_DONE: + pos = self.file.tell() + continue + elif flags & self.STATTR_FLAG_EOS: return + else: + # No EOS came after DONE, that's OK, but rewind the + # stream because this is not our data. + if pos: + self.file.seek(pos, os.SEEK_SET) + return + raise Exception("Unknown flags %x", flags) + if (flags & self.STATTR_FLAG_ERROR): raise Exception("Error in migration stream") count = self.file.read64() @@ -401,6 +417,28 @@ class VMSDFieldIntLE(VMSDFieldInt): super(VMSDFieldIntLE, self).__init__(desc, file) self.dtype = '<i%d' % self.size +class VMSDFieldNull(VMSDFieldGeneric): + NULL_PTR_MARKER = b'0' + + def __init__(self, desc, file): + super(VMSDFieldNull, self).__init__(desc, file) + + def __repr__(self): + # A NULL pointer is encoded in the stream as a '0' to + # disambiguate from a mere 0x0 value and avoid consumers + # trying to follow the NULL pointer. Displaying '0', 0x30 or + # 0x0 when analyzing the JSON debug stream could become + # confusing, so use an explicit term instead. + return "nullptr" + + def __str__(self): + return self.__repr__() + + def read(self): + super(VMSDFieldNull, self).read() + assert(self.data == self.NULL_PTR_MARKER) + return self.data + class VMSDFieldBool(VMSDFieldGeneric): def __init__(self, desc, file): super(VMSDFieldBool, self).__init__(desc, file) @@ -429,6 +467,9 @@ class VMSDFieldStruct(VMSDFieldGeneric): super(VMSDFieldStruct, self).__init__(desc, file) self.data = collections.OrderedDict() + if 'fields' not in self.desc['struct']: + raise Exception("No fields in struct. VMSD:\n%s" % self.desc) + # When we see compressed array elements, unfold them here new_fields = [] for field in self.desc['struct']['fields']: @@ -461,15 +502,25 @@ class VMSDFieldStruct(VMSDFieldGeneric): field['data'] = reader(field, self.file) field['data'].read() - if 'index' in field: - if field['name'] not in self.data: - self.data[field['name']] = [] - a = self.data[field['name']] - if len(a) != int(field['index']): - raise Exception("internal index of data field unmatched (%d/%d)" % (len(a), int(field['index']))) - a.append(field['data']) + fname = field['name'] + fdata = field['data'] + + # The field could be: + # i) a single data entry, e.g. uint64 + # ii) an array, indicated by it containing the 'index' key + # + # However, the overall data after parsing the whole + # stream, could be a mix of arrays and single data fields, + # all sharing the same field name due to how QEMU breaks + # up arrays with NULL pointers into multiple compressed + # array segments. + if fname not in self.data: + self.data[fname] = fdata + elif type(self.data[fname]) == list: + self.data[fname].append(fdata) else: - self.data[field['name']] = field['data'] + tmp = self.data[fname] + self.data[fname] = [tmp, fdata] if 'subsections' in self.desc['struct']: for subsection in self.desc['struct']['subsections']: @@ -477,6 +528,10 @@ class VMSDFieldStruct(VMSDFieldGeneric): raise Exception("Subsection %s not found at offset %x" % ( subsection['vmsd_name'], self.file.tell())) name = self.file.readstr() version_id = self.file.read32() + + if not subsection: + raise Exception("Empty description for subsection: %s" % name) + self.data[name] = VMSDSection(self.file, version_id, subsection, (name, 0)) self.data[name].read() @@ -535,6 +590,7 @@ vmsd_field_readers = { "bitmap" : VMSDFieldGeneric, "struct" : VMSDFieldStruct, "capability": VMSDFieldCap, + "nullptr": VMSDFieldNull, "unknown" : VMSDFieldGeneric, } @@ -564,7 +620,9 @@ class MigrationDump(object): QEMU_VM_SUBSECTION = 0x05 QEMU_VM_VMDESCRIPTION = 0x06 QEMU_VM_CONFIGURATION = 0x07 + QEMU_VM_COMMAND = 0x08 QEMU_VM_SECTION_FOOTER= 0x7e + QEMU_MIG_CMD_SWITCHOVER_START = 0x0b def __init__(self, filename): self.section_classes = { @@ -574,10 +632,13 @@ class MigrationDump(object): } self.filename = filename self.vmsd_desc = None + self.vmsd_json = "" - def read(self, desc_only = False, dump_memory = False, write_memory = False): + def read(self, desc_only = False, dump_memory = False, + write_memory = False): # Read in the whole file file = MigrationFile(self.filename) + self.vmsd_json = file.read_migration_debug_json() # File magic data = file.read32() @@ -626,6 +687,15 @@ class MigrationDump(object): elif section_type == self.QEMU_VM_SECTION_PART or section_type == self.QEMU_VM_SECTION_END: section_id = file.read32() self.sections[section_id].read() + elif section_type == self.QEMU_VM_COMMAND: + command_type = file.read16() + command_data_len = file.read16() + if command_type != self.QEMU_MIG_CMD_SWITCHOVER_START: + raise Exception("Unknown QEMU_VM_COMMAND: %x" % + (command_type)) + if command_data_len != 0: + raise Exception("Invalid SWITCHOVER_START length: %x" % + (command_data_len)) elif section_type == self.QEMU_VM_SECTION_FOOTER: read_section_id = file.read32() if read_section_id != section_id: @@ -635,9 +705,11 @@ class MigrationDump(object): file.close() def load_vmsd_json(self, file): - vmsd_json = file.read_migration_debug_json() - self.vmsd_desc = json.loads(vmsd_json, object_pairs_hook=collections.OrderedDict) + self.vmsd_desc = json.loads(self.vmsd_json, + object_pairs_hook=collections.OrderedDict) for device in self.vmsd_desc['devices']: + if 'fields' not in device: + raise Exception("vmstate for device %s has no fields" % device['name']) key = (device['name'], device['instance_id']) value = ( VMSDSection, device ) self.section_classes[key] = value @@ -666,31 +738,34 @@ args = parser.parse_args() jsonenc = JSONEncoder(indent=4, separators=(',', ': ')) -if args.extract: - dump = MigrationDump(args.file) +if not any([args.extract, args.dump == "state", args.dump == "desc"]): + raise Exception("Please specify either -x, -d state or -d desc") - dump.read(desc_only = True) - print("desc.json") - f = open("desc.json", "w") - f.truncate() - f.write(jsonenc.encode(dump.vmsd_desc)) - f.close() - - dump.read(write_memory = True) - dict = dump.getDict() - print("state.json") - f = open("state.json", "w") - f.truncate() - f.write(jsonenc.encode(dict)) - f.close() -elif args.dump == "state": +try: dump = MigrationDump(args.file) - dump.read(dump_memory = args.memory) - dict = dump.getDict() - print(jsonenc.encode(dict)) -elif args.dump == "desc": - dump = MigrationDump(args.file) - dump.read(desc_only = True) - print(jsonenc.encode(dump.vmsd_desc)) -else: - raise Exception("Please specify either -x, -d state or -d desc") + + if args.extract: + dump.read(desc_only = True) + + print("desc.json") + f = open("desc.json", "w") + f.truncate() + f.write(jsonenc.encode(dump.vmsd_desc)) + f.close() + + dump.read(write_memory = True) + dict = dump.getDict() + print("state.json") + f = open("state.json", "w") + f.truncate() + f.write(jsonenc.encode(dict)) + f.close() + elif args.dump == "state": + dump.read(dump_memory = args.memory) + dict = dump.getDict() + print(jsonenc.encode(dict)) + elif args.dump == "desc": + dump.read(desc_only = True) + print(jsonenc.encode(dump.vmsd_desc)) +except Exception: + raise Exception("Full JSON dump:\n%s", dump.vmsd_json) diff --git a/scripts/archive-source.sh b/scripts/archive-source.sh index 65af806..035828c 100755 --- a/scripts/archive-source.sh +++ b/scripts/archive-source.sh @@ -26,7 +26,12 @@ sub_file="${sub_tdir}/submodule.tar" # independent of what the developer currently has initialized # in their checkout, because the build environment is completely # different to the host OS. -subprojects="keycodemapdb libvfio-user berkeley-softfloat-3 berkeley-testfloat-3" +subprojects="keycodemapdb libvfio-user berkeley-softfloat-3 + berkeley-testfloat-3 anyhow-1-rs arbitrary-int-1-rs bilge-0.2-rs + bilge-impl-0.2-rs either-1-rs foreign-0.3-rs itertools-0.11-rs + libc-0.2-rs proc-macro2-1-rs + proc-macro-error-1-rs proc-macro-error-attr-1-rs quote-1-rs + syn-2-rs unicode-ident-1-rs" sub_deinit="" function cleanup() { @@ -48,13 +53,34 @@ function tree_ish() { echo "$retval" } +function subproject_dir() { + if test ! -f "subprojects/$1.wrap"; then + error "scripts/archive-source.sh should only process wrap subprojects" + fi + + # Print the directory key of the wrap file, defaulting to the + # subproject name. The wrap file is in ini format and should + # have a single section only. There should be only one section + # named "[wrap-*]", which helps keeping the script simple. + local dir + dir=$(sed -n \ + -e '/^\[wrap-[a-z][a-z]*\]$/,/^\[/{' \ + -e '/^directory *= */!b' \ + -e 's///p' \ + -e 'q' \ + -e '}' \ + "subprojects/$1.wrap") + + echo "${dir:-$1}" +} + git archive --format tar "$(tree_ish)" > "$tar_file" test $? -ne 0 && error "failed to archive qemu" for sp in $subprojects; do meson subprojects download $sp test $? -ne 0 && error "failed to download subproject $sp" - tar --append --file "$tar_file" --exclude=.git subprojects/$sp + tar --append --file "$tar_file" --exclude=.git subprojects/"$(subproject_dir $sp)" test $? -ne 0 && error "failed to append subproject $sp to $tar_file" done exit 0 diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index ff373a7..833f20f 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -365,6 +365,18 @@ our @typeList = ( qr{guintptr}, ); +# Match text found in common license boilerplate comments: +# for new files the SPDX-License-Identifier line is sufficient. +our @LICENSE_BOILERPLATE = ( + "licensed under the terms of the GNU GPL", + "under the terms of the GNU General Public License", + "under the terms of the GNU Lesser General Public", + "Permission is hereby granted, free of charge", + "GNU GPL, version 2 or later", + "See the COPYING file" +); +our $LICENSE_BOILERPLATE_RE = join("|", @LICENSE_BOILERPLATE); + # Load common spelling mistakes and build regular expression list. my $misspellings; my %spelling_fix; @@ -1330,26 +1342,179 @@ sub WARN { } } -# According to tests/qtest/bios-tables-test.c: do not -# change expected file in the same commit with adding test -sub checkfilename { - my ($name, $acpi_testexpected, $acpi_nontestexpected) = @_; - - # Note: shell script that rebuilds the expected files is in the same - # directory as files themselves. - # Note: allowed diff list can be changed both when changing expected - # files and when changing tests. - if ($name =~ m#^tests/data/acpi/# and not $name =~ m#^\.sh$#) { - $$acpi_testexpected = $name; - } elsif ($name !~ m#^tests/qtest/bios-tables-test-allowed-diff.h$#) { - $$acpi_nontestexpected = $name; +sub checkspdx { + my ($file, $expr) = @_; + + # Imported Linux headers probably have SPDX tags, but if they + # don't we're not requiring contributors to fix this, as these + # files are not expected to be modified locally in QEMU. + # Also don't accidentally detect own checking code. + if ($file =~ m,include/standard-headers, || + $file =~ m,linux-headers, || + $file =~ m,checkpatch.pl,) { + return; + } + + my $origexpr = $expr; + + # Flatten sub-expressions + $expr =~ s/\(|\)/ /g; + $expr =~ s/OR|AND/ /g; + + # Merge WITH exceptions to the license + $expr =~ s/\s+WITH\s+/-WITH-/g; + + # Cull more leading/trailing whitespace + $expr =~ s/^\s*//g; + $expr =~ s/\s*$//g; + + my @bits = split / +/, $expr; + + my $prefer = "GPL-2.0-or-later"; + my @valid = qw( + GPL-2.0-only + LGPL-2.1-only + LGPL-2.1-or-later + BSD-2-Clause + BSD-3-Clause + MIT + ); + + my $nonpreferred = 0; + my @unknown = (); + foreach my $bit (@bits) { + if ($bit eq $prefer) { + next; + } + if (grep /^$bit$/, @valid) { + $nonpreferred = 1; + } else { + push @unknown, $bit; + } + } + if (@unknown) { + ERROR("Saw unacceptable licenses '" . join(',', @unknown) . + "', valid choices for QEMU are:\n" . join("\n", $prefer, @valid)); + } + + if ($nonpreferred) { + WARN("Saw acceptable license '$origexpr' but note '$prefer' is " . + "preferred for new files unless the code is derived from a " . + "source file with an existing declared license that must be " . + "retained. Please explain the license choice in the commit " . + "message."); + } +} + +# All three of the methods below take a 'file info' record +# which is a hash ref containing +# +# 'isgit': 1 if an enhanced git diff or 0 for a plain diff +# 'githeader': 1 if still parsing git patch header, 0 otherwise +# 'linestart': line number of start of file diff +# 'lineend': line number of end of file diff +# 'filenew': the new filename +# 'fileold': the old filename (same as 'new filename' except +# for renames in git diffs) +# 'action': one of 'modified' (always) or 'new' or 'deleted' or +# 'renamed' (git diffs only) +# 'mode': file mode for new/deleted files (git diffs only) +# 'similarity': file similarity when renamed (git diffs only) +# 'facts': hash ref for storing any metadata related to checks +# + +# Called at the end of each patch, with the list of +# real filenames that were seen in the patch +sub process_file_list { + my @fileinfos = @_; + + # According to tests/qtest/bios-tables-test.c: do not + # change expected file in the same commit with adding test + my @acpi_testexpected; + my @acpi_nontestexpected; + + foreach my $fileinfo (@fileinfos) { + # Note: shell script that rebuilds the expected files is in + # the same directory as files themselves. + # Note: allowed diff list can be changed both when changing + # expected files and when changing tests. + if ($fileinfo->{filenew} =~ m#^tests/data/acpi/# && + $fileinfo->{filenew} !~ m#^\.sh$#) { + push @acpi_testexpected, $fileinfo->{filenew}; + } elsif ($fileinfo->{filenew} !~ + m#^tests/qtest/bios-tables-test-allowed-diff.h$#) { + push @acpi_nontestexpected, $fileinfo->{filenew}; + } } - if (defined $$acpi_testexpected and defined $$acpi_nontestexpected) { + if (int(@acpi_testexpected) > 0 and int(@acpi_nontestexpected) > 0) { ERROR("Do not add expected files together with tests, " . "follow instructions in " . - "tests/qtest/bios-tables-test.c: both " . - $$acpi_testexpected . " and " . - $$acpi_nontestexpected . " found\n"); + "tests/qtest/bios-tables-test.c. Files\n\n " . + join("\n ", @acpi_testexpected) . + "\n\nand\n\n " . + join("\n ", @acpi_nontestexpected) . + "\n\nfound in the same patch\n"); + } + + my $sawmaintainers = 0; + my @maybemaintainers; + foreach my $fileinfo (@fileinfos) { + if ($fileinfo->{action} ne "modified" && + $fileinfo->{filenew} !~ m#^tests/data/acpi/#) { + push @maybemaintainers, $fileinfo->{filenew}; + } + if ($fileinfo->{filenew} eq "MAINTAINERS") { + $sawmaintainers = 1; + } + } + + # If we don't see a MAINTAINERS update, prod the user to check + if (int(@maybemaintainers) > 0 && !$sawmaintainers) { + WARN("added, moved or deleted file(s):\n\n " . + join("\n ", @maybemaintainers) . + "\n\nDoes MAINTAINERS need updating?\n"); + } +} + +# Called at the start of processing a diff hunk for a file +sub process_start_of_file { + my $fileinfo = shift; + + # Check for incorrect file permissions + if ($fileinfo->{action} eq "new" && ($fileinfo->{mode} & 0111)) { + my $permhere = $fileinfo->{linestart} . "FILE: " . + $fileinfo->{filenew} . "\n"; + if ($fileinfo->{filenew} =~ + /(\bMakefile.*|\.(c|cc|cpp|h|mak|s|S))$/) { + ERROR("do not set execute permissions for source " . + "files\n" . $permhere); + } + } +} + +# Called at the end of processing a diff hunk for a file +sub process_end_of_file { + my $fileinfo = shift; + + if ($fileinfo->{action} eq "new" && + !exists $fileinfo->{facts}->{sawspdx}) { + if ($fileinfo->{filenew} =~ + /(\.(c|h|py|pl|sh|json|inc|rs)|Makefile.*)$/) { + # source code files MUST have SPDX license declared + ERROR("New file '" . $fileinfo->{filenew} . + "' requires 'SPDX-License-Identifier'"); + } else { + # Other files MAY have SPDX license if appropriate + WARN("Does new file '" . $fileinfo->{filenew} . + "' need 'SPDX-License-Identifier'?"); + } + } + if ($fileinfo->{action} eq "new" && + exists $fileinfo->{facts}->{sawboilerplate}) { + ERROR("New file '" . $fileinfo->{filenew} . "' must " . + "not have license boilerplate header text, only " . + "the SPDX-License-Identifier, unless this file was " . + "copied from existing code already having such text."); } } @@ -1373,7 +1538,9 @@ sub process { my $in_header_lines = $file ? 0 : 1; my $in_commit_log = 0; #Scanning lines before patch - my $reported_maintainer_file = 0; + my $reported_mixing_imported_file = 0; + my $in_imported_file = 0; + my $in_no_imported_file = 0; my $non_utf8_charset = 0; our @report = (); @@ -1386,7 +1553,10 @@ sub process { my $realfile = ''; my $realline = 0; my $realcnt = 0; + my $fileinfo; + my @fileinfolist; my $here = ''; + my $oldhere = ''; my $in_comment = 0; my $comment_edge = 0; my $first_line = 0; @@ -1399,9 +1569,6 @@ sub process { my %suppress_whiletrailers; my %suppress_export; - my $acpi_testexpected; - my $acpi_nontestexpected; - # Pre-scan the patch sanitizing the lines. sanitise_line_reset(); @@ -1524,18 +1691,54 @@ sub process { $prefix = "$filename:$realline: " if ($emacs && $file); $prefix = "$filename:$linenr: " if ($emacs && !$file); + $oldhere = $here; $here = "#$linenr: " if (!$file); $here = "#$realline: " if ($file); # extract the filename as it passes - if ($line =~ /^diff --git.*?(\S+)$/) { - $realfile = $1; - $realfile =~ s@^([^/]*)/@@ if (!$file); - checkfilename($realfile, \$acpi_testexpected, \$acpi_nontestexpected); + if ($line =~ /^diff --git\s+(\S+)\s+(\S+)$/) { + my $fileold = $1; + my $filenew = $2; + + if (defined $fileinfo) { + $fileinfo->{lineend} = $oldhere; + process_end_of_file($fileinfo) + } + $fileold =~ s@^([^/]*)/@@ if (!$file); + $filenew =~ s@^([^/]*)/@@ if (!$file); + $realfile = $filenew; + + $fileinfo = { + "isgit" => 1, + "githeader" => 1, + "linestart" => $here, + "lineend" => 0, + "fileold" => $fileold, + "filenew" => $filenew, + "action" => "modified", + "mode" => 0, + "similarity" => 0, + "facts" => {}, + }; + push @fileinfolist, $fileinfo; + } elsif (defined $fileinfo && $fileinfo->{githeader} && + $line =~ /^(new|deleted) (?:file )?mode\s+([0-7]+)$/) { + $fileinfo->{action} = $1; + $fileinfo->{mode} = oct($2); + } elsif (defined $fileinfo && $fileinfo->{githeader} && + $line =~ /^similarity index (\d+)%/) { + $fileinfo->{similarity} = int($1); + } elsif (defined $fileinfo && $fileinfo->{githeader} && + $line =~ /^rename (from|to) [\w\/\.\-]+\s*$/) { + $fileinfo->{action} = "renamed"; + # For a no-change rename, we'll never have any "+++..." + # lines, so trigger actions now + if ($1 eq "to" && $fileinfo->{similarity} == 100) { + process_start_of_file($fileinfo); + } } elsif ($line =~ /^\+\+\+\s+(\S+)/) { $realfile = $1; $realfile =~ s@^([^/]*)/@@ if (!$file); - checkfilename($realfile, \$acpi_testexpected, \$acpi_nontestexpected); $p1_prefix = $1; if (!$file && $tree && $p1_prefix ne '' && @@ -1543,6 +1746,30 @@ sub process { WARN("patch prefix '$p1_prefix' exists, appears to be a -p0 patch\n"); } + if (defined $fileinfo && !$fileinfo->{isgit}) { + $fileinfo->{lineend} = $oldhere; + process_end_of_file($fileinfo); + } + + if (!defined $fileinfo || !$fileinfo->{isgit}) { + $fileinfo = { + "isgit" => 0, + "githeader" => 0, + "linestart" => $here, + "lineend" => 0, + "fileold" => $realfile, + "filenew" => $realfile, + "action" => "modified", + "mode" => 0, + "similarity" => 0, + "facts" => {}, + }; + push @fileinfolist, $fileinfo; + } else { + $fileinfo->{githeader} = 0; + } + process_start_of_file($fileinfo); + next; } @@ -1554,14 +1781,6 @@ sub process { $cnt_lines++ if ($realcnt != 0); -# Check for incorrect file permissions - if ($line =~ /^new (file )?mode.*[7531]\d{0,2}$/) { - my $permhere = $here . "FILE: $realfile\n"; - if ($realfile =~ /(\bMakefile(?:\.objs)?|\.c|\.cc|\.cpp|\.h|\.mak|\.[sS])$/) { - ERROR("do not set execute permissions for source files\n" . $permhere); - } - } - # Only allow Python 3 interpreter if ($realline == 1 && $line =~ /^\+#!\ *\/usr\/bin\/(?:env )?python$/) { @@ -1593,23 +1812,27 @@ sub process { } } -# Check if MAINTAINERS is being updated. If so, there's probably no need to -# emit the "does MAINTAINERS need updating?" message on file add/move/delete - if ($line =~ /^\s*MAINTAINERS\s*\|/) { - $reported_maintainer_file = 1; +# Check SPDX-License-Identifier references a permitted license + if ($rawline =~ m,SPDX-License-Identifier: (.*?)(\*/)?\s*$,) { + $fileinfo->{facts}->{sawspdx} = 1; + &checkspdx($realfile, $1); } -# Check for added, moved or deleted files - if (!$reported_maintainer_file && !$in_commit_log && - ($line =~ /^(?:new|deleted) file mode\s*\d+\s*$/ || - $line =~ /^rename (?:from|to) [\w\/\.\-]+\s*$/ || - ($line =~ /\{\s*([\w\/\.\-]*)\s*\=\>\s*([\w\/\.\-]*)\s*\}/ && - (defined($1) || defined($2)))) && - !(($realfile ne '') && - defined($acpi_testexpected) && - ($realfile eq $acpi_testexpected))) { - $reported_maintainer_file = 1; - WARN("added, moved or deleted file(s), does MAINTAINERS need updating?\n" . $herecurr); + if ($rawline =~ /$LICENSE_BOILERPLATE_RE/) { + $fileinfo->{facts}->{sawboilerplate} = 1; + } + + if ($rawline =~ m,(SPDX-[a-zA-Z0-9-_]+):,) { + my $tag = $1; + my @permitted = qw( + SPDX-License-Identifier + ); + + unless (grep { /^$tag$/ } @permitted) { + ERROR("Tag $tag not permitted in QEMU code, " . + "valid choices are: " . + join(", ", @permitted)); + } } # Check for wrappage within a valid hunk of the file @@ -1673,6 +1896,27 @@ sub process { # ignore non-hunk lines and lines being removed next if (!$hunk_line || $line =~ /^-/); +# Check that updating imported files from Linux are not mixed with other changes + if ($realfile =~ /^(linux-headers|include\/standard-headers)\//) { + if (!$in_imported_file) { + WARN("added, moved or deleted file(s) " . + "imported from Linux, are you using " . + "scripts/update-linux-headers.sh?\n" . + $herecurr); + } + $in_imported_file = 1; + } else { + $in_no_imported_file = 1; + } + + if (!$reported_mixing_imported_file && + $in_imported_file && $in_no_imported_file) { + ERROR("headers imported from Linux should be self-" . + "contained in a patch with no other changes\n" . + $herecurr); + $reported_mixing_imported_file = 1; + } + # ignore files that are being periodically imported from Linux next if ($realfile =~ /^(linux-headers|include\/standard-headers)\//); @@ -2169,7 +2413,7 @@ sub process { # missing space after union, struct or enum definition if ($line =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident)?(?:\s+$Ident)?[=\{]/) { - ERROR("missing space after $1 definition\n" . $herecurr); + ERROR("missing space after $1 definition\n" . $herecurr); } # check for spacing round square brackets; allowed: @@ -2222,7 +2466,7 @@ sub process { } } # Check operator spacing. - if (!($line=~/\#\s*include/)) { + if (!($line=~/\#\s*(include|import)/)) { my $ops = qr{ <<=|>>=|<=|>=|==|!=| \+=|-=|\*=|\/=|%=|\^=|\|=|&=| @@ -2464,7 +2708,7 @@ sub process { if ($line =~ /^.\s*(Q(?:S?LIST|SIMPLEQ|TAILQ)_HEAD)\s*\(\s*[^,]/ && $line !~ /^.typedef/) { - ERROR("named $1 should be typedefed separately\n" . $herecurr); + ERROR("named $1 should be typedefed separately\n" . $herecurr); } # Need a space before open parenthesis after if, while etc @@ -3013,48 +3257,50 @@ sub process { # Qemu error function tests - # Find newlines in error messages - my $qemu_error_funcs = qr{error_setg| - error_setg_errno| - error_setg_win32| - error_setg_file_open| - error_set| - error_prepend| - warn_reportf_err| - error_reportf_err| - error_vreport| - warn_vreport| - info_vreport| - error_report| - warn_report| - info_report| - g_test_message}x; - - if ($rawline =~ /\b(?:$qemu_error_funcs)\s*\(.*\".*\\n/) { - ERROR("Error messages should not contain newlines\n" . $herecurr); - } + # Find newlines in error messages + my $qemu_error_funcs = qr{error_setg| + error_setg_errno| + error_setg_win32| + error_setg_file_open| + error_set| + error_prepend| + warn_reportf_err| + error_reportf_err| + error_vreport| + warn_vreport| + info_vreport| + error_report| + warn_report| + info_report| + g_test_message}x; + + if ($rawline =~ /\b(?:$qemu_error_funcs)\s*\(.*\".*\\n/) { + ERROR("Error messages should not contain newlines\n" . $herecurr); + } - # Continue checking for error messages that contains newlines. This - # check handles cases where string literals are spread over multiple lines. - # Example: - # error_report("Error msg line #1" - # "Error msg line #2\n"); - my $quoted_newline_regex = qr{\+\s*\".*\\n.*\"}; - my $continued_str_literal = qr{\+\s*\".*\"}; + # Continue checking for error messages that contains newlines. + # This check handles cases where string literals are spread + # over multiple lines. + # Example: + # error_report("Error msg line #1" + # "Error msg line #2\n"); + my $quoted_newline_regex = qr{\+\s*\".*\\n.*\"}; + my $continued_str_literal = qr{\+\s*\".*\"}; - if ($rawline =~ /$quoted_newline_regex/) { - # Backtrack to first line that does not contain only a quoted literal - # and assume that it is the start of the statement. - my $i = $linenr - 2; + if ($rawline =~ /$quoted_newline_regex/) { + # Backtrack to first line that does not contain only + # a quoted literal and assume that it is the start + # of the statement. + my $i = $linenr - 2; - while (($i >= 0) & $rawlines[$i] =~ /$continued_str_literal/) { - $i--; - } + while (($i >= 0) & $rawlines[$i] =~ /$continued_str_literal/) { + $i--; + } - if ($rawlines[$i] =~ /\b(?:$qemu_error_funcs)\s*\(/) { - ERROR("Error messages should not contain newlines\n" . $herecurr); + if ($rawlines[$i] =~ /\b(?:$qemu_error_funcs)\s*\(/) { + ERROR("Error messages should not contain newlines\n" . $herecurr); + } } - } # check for non-portable libc calls that have portable alternatives in QEMU if ($line =~ /\bffs\(/) { @@ -3078,6 +3324,10 @@ sub process { if ($line =~ /\b(g_)?assert\(0\)/) { ERROR("use g_assert_not_reached() instead of assert(0)\n" . $herecurr); } + if ($line =~ /\b(g_)?assert\(false\)/) { + ERROR("use g_assert_not_reached() instead of assert(false)\n" . + $herecurr); + } if ($line =~ /\bstrerrorname_np\(/) { ERROR("use strerror() instead of strerrorname_np()\n" . $herecurr); } @@ -3104,6 +3354,11 @@ sub process { } } + if (defined $fileinfo) { + process_end_of_file($fileinfo); + } + process_file_list(@fileinfolist); + if ($is_patch && $chk_signoff && $signoff == 0) { ERROR("Missing Signed-off-by: line(s)\n"); } diff --git a/scripts/ci/gitlab-ci-section b/scripts/ci/gitlab-ci-section new file mode 100644 index 0000000..9bbe804 --- /dev/null +++ b/scripts/ci/gitlab-ci-section @@ -0,0 +1,29 @@ +# Copyright (c) 2024 Linaro Ltd +# SPDX-License-Identifier: GPL-2.0-or-later + +# gitlab-ci-section: This is a shell script fragment which defines +# functions section_start and section_end which will emit marker lines +# that GitLab will interpret as the beginning or end of a "collapsible +# section" in a CI job log. See +# https://docs.gitlab.com/ee/ci/yaml/script.html#expand-and-collapse-job-log-sections +# +# This is intended to be sourced in the before_script section of +# a CI config; the section_start and section_end functions will +# then be available for use in the before_script and script sections. + +# Section names are [-_.A-Za-z0-9] and the section_start pairs with +# a section_end with the same section name. +# The description can be any printable text without newlines; this is +# what will appear in the log. + +# Usage: +# section_start section_name "Description of the section" +section_start () { + printf "section_start:%s:%s\r\e[0K%s\n" "$(date +%s)" "$1" "$2" +} + +# Usage: +# section_end section_name +section_end () { + printf "section_end:%s:%s\r\e[0K\n" "$(date +%s)" "$1" +} diff --git a/scripts/ci/setup/gitlab-runner.yml b/scripts/ci/setup/gitlab-runner.yml index 7bdafab..57e7fae 100644 --- a/scripts/ci/setup/gitlab-runner.yml +++ b/scripts/ci/setup/gitlab-runner.yml @@ -49,30 +49,51 @@ - debug: msg: gitlab-runner arch is {{ gitlab_runner_arch }} - - name: Download the matching gitlab-runner (DEB) + # Debian/Ubuntu setup + - name: Get gitlab-runner repo setup script (DEB) get_url: dest: "/root/" - url: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/deb/gitlab-runner_{{ gitlab_runner_arch }}.deb" + url: "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" + mode: 0755 when: - ansible_facts['distribution'] == 'Ubuntu' - - name: Download the matching gitlab-runner (RPM) + - name: Run gitlab-runner repo setup script (DEB) + shell: "/root/script.deb.sh" + when: + - ansible_facts['distribution'] == 'Ubuntu' + + - name: Install gitlab-runner (DEB) + ansible.builtin.apt: + name: gitlab-runner + update_cache: yes + state: present + when: + - ansible_facts['distribution'] == 'Ubuntu' + + # RPM setup + - name: Get gitlab-runner repo setup script (RPM) get_url: dest: "/root/" - url: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/rpm/gitlab-runner_{{ gitlab_runner_arch }}.rpm" + url: "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.rpm.sh" + mode: 0755 when: - ansible_facts['distribution'] == 'CentOS' - - name: Install gitlab-runner via package manager (DEB) - apt: deb="/root/gitlab-runner_{{ gitlab_runner_arch }}.deb" + - name: Run gitlab-runner repo setup script (RPM) + shell: "/root/script.rpm.sh" when: - - ansible_facts['distribution'] == 'Ubuntu' + - ansible_facts['distribution'] == 'CentOS' - - name: Install gitlab-runner via package manager (RPM) - yum: name="/root/gitlab-runner_{{ gitlab_runner_arch }}.rpm" + - name: Install gitlab-runner (RPM) + yum: + name: gitlab-runner + update_cache: yes + state: present when: - ansible_facts['distribution'] == 'CentOS' + # Register Runners - name: Register the gitlab-runner command: "/usr/bin/gitlab-runner register --non-interactive --url {{ gitlab_runner_server_url }} --registration-token {{ gitlab_runner_registration_token }} --executor shell --tag-list {{ ansible_facts[\"architecture\"] }},{{ ansible_facts[\"distribution\"]|lower }}_{{ ansible_facts[\"distribution_version\"] }} --description '{{ ansible_facts[\"distribution\"] }} {{ ansible_facts[\"distribution_version\"] }} {{ ansible_facts[\"architecture\"] }} ({{ ansible_facts[\"os_family\"] }})'" diff --git a/scripts/ci/setup/ubuntu/build-environment.yml b/scripts/ci/setup/ubuntu/build-environment.yml index edf1900..56b5160 100644 --- a/scripts/ci/setup/ubuntu/build-environment.yml +++ b/scripts/ci/setup/ubuntu/build-environment.yml @@ -39,7 +39,6 @@ when: - ansible_facts['distribution'] == 'Ubuntu' - ansible_facts['distribution_version'] == '22.04' - - ansible_facts['architecture'] == 'aarch64' or ansible_facts['architecture'] == 'x86_64' - name: Install packages for QEMU on Ubuntu 22.04 package: @@ -47,7 +46,6 @@ when: - ansible_facts['distribution'] == 'Ubuntu' - ansible_facts['distribution_version'] == '22.04' - - ansible_facts['architecture'] == 'aarch64' or ansible_facts['architecture'] == 'x86_64' - name: Install armhf cross-compile packages to build QEMU on AArch64 Ubuntu 22.04 package: diff --git a/scripts/ci/setup/ubuntu/ubuntu-2204-aarch64.yaml b/scripts/ci/setup/ubuntu/ubuntu-2204-aarch64.yaml index fd5489c..f11e980 100644 --- a/scripts/ci/setup/ubuntu/ubuntu-2204-aarch64.yaml +++ b/scripts/ci/setup/ubuntu/ubuntu-2204-aarch64.yaml @@ -35,6 +35,7 @@ packages: - libcacard-dev - libcap-ng-dev - libcapstone-dev + - libcbor-dev - libcmocka-dev - libcurl4-gnutls-dev - libdaxctl-dev @@ -49,6 +50,7 @@ packages: - libglusterfs-dev - libgnutls28-dev - libgtk-3-dev + - libgtk-vnc-2.0-dev - libibverbs-dev - libiscsi-dev - libjemalloc-dev @@ -112,6 +114,7 @@ packages: - python3-venv - python3-yaml - rpm2cpio + - rustc-1.77 - sed - socat - sparse @@ -120,6 +123,7 @@ packages: - tar - tesseract-ocr - tesseract-ocr-eng + - vulkan-tools - xorriso - zlib1g-dev - zstd diff --git a/scripts/ci/setup/ubuntu/ubuntu-2204-s390x.yaml b/scripts/ci/setup/ubuntu/ubuntu-2204-s390x.yaml index afa0450..6559cb2 100644 --- a/scripts/ci/setup/ubuntu/ubuntu-2204-s390x.yaml +++ b/scripts/ci/setup/ubuntu/ubuntu-2204-s390x.yaml @@ -35,6 +35,7 @@ packages: - libcacard-dev - libcap-ng-dev - libcapstone-dev + - libcbor-dev - libcmocka-dev - libcurl4-gnutls-dev - libdaxctl-dev @@ -49,6 +50,7 @@ packages: - libglusterfs-dev - libgnutls28-dev - libgtk-3-dev + - libgtk-vnc-2.0-dev - libibverbs-dev - libiscsi-dev - libjemalloc-dev @@ -110,6 +112,7 @@ packages: - python3-venv - python3-yaml - rpm2cpio + - rustc-1.77 - sed - socat - sparse @@ -118,6 +121,7 @@ packages: - tar - tesseract-ocr - tesseract-ocr-eng + - vulkan-tools - xorriso - zlib1g-dev - zstd diff --git a/scripts/clean-includes b/scripts/clean-includes index bdbf404..25dbf16 100755 --- a/scripts/clean-includes +++ b/scripts/clean-includes @@ -130,8 +130,8 @@ for f in "$@"; do *include/qemu/compiler.h | \ *include/qemu/qemu-plugin.h | \ *include/glib-compat.h | \ - *include/sysemu/os-posix.h | \ - *include/sysemu/os-win32.h | \ + *include/system/os-posix.h | \ + *include/system/os-win32.h | \ *include/standard-headers/ ) # Removing include lines from osdep.h itself would be counterproductive. echo "SKIPPING $f (special case header)" @@ -174,7 +174,7 @@ for f in "$@"; do <limits.h> <unistd.h> <time.h> <ctype.h> <errno.h> <fcntl.h> <sys/stat.h> <sys/time.h> <assert.h> <signal.h> <glib.h> <sys/stat.h> <sys/time.h> <assert.h> <signal.h> <glib.h> <sys/mman.h> - "sysemu/os-posix.h, sysemu/os-win32.h "glib-compat.h" + "system/os-posix.h, system/os-win32.h "glib-compat.h" "qemu/typedefs.h" ))' "$f" diff --git a/scripts/cocci-macro-file.h b/scripts/cocci-macro-file.h index d247a50..c64831d 100644 --- a/scripts/cocci-macro-file.h +++ b/scripts/cocci-macro-file.h @@ -23,11 +23,7 @@ #define G_GNUC_WARN_UNUSED_RESULT __attribute__((warn_unused_result)) #define G_GNUC_NULL_TERMINATED __attribute__((sentinel)) -#if defined(_WIN32) && (defined(__x86_64__) || defined(__i386__)) -# define QEMU_PACKED __attribute__((gcc_struct, packed)) -#else -# define QEMU_PACKED __attribute__((packed)) -#endif +#define QEMU_PACKED __attribute__((packed)) #define cat(x,y) x ## y #define cat2(x,y) cat(x,y) diff --git a/scripts/coccinelle/device-reset.cocci b/scripts/coccinelle/device-reset.cocci new file mode 100644 index 0000000..510042a --- /dev/null +++ b/scripts/coccinelle/device-reset.cocci @@ -0,0 +1,30 @@ +// Convert opencoded DeviceClass::reset assignments to calls to +// device_class_set_legacy_reset() +// +// Copyright Linaro Ltd 2024 +// This work is licensed under the terms of the GNU GPLv2 or later. +// +// spatch --macro-file scripts/cocci-macro-file.h \ +// --sp-file scripts/coccinelle/device-reset.cocci \ +// --keep-comments --smpl-spacing --in-place --include-headers --dir hw +// +// For simplicity we assume some things about the code we're modifying +// that happen to be true for all our targets: +// * all cpu_class_set_parent_reset() callsites have a 'DeviceClass *dc' local +// * the parent reset field in the target CPU class is 'parent_reset' +// * no reset function already has a 'dev' local + +@@ +identifier dc, resetfn; +@@ + DeviceClass *dc; + ... +- dc->reset = resetfn; ++ device_class_set_legacy_reset(dc, resetfn); +@@ +identifier dc, resetfn; +@@ + DeviceClass *dc; + ... +- dc->reset = &resetfn; ++ device_class_set_legacy_reset(dc, resetfn); diff --git a/scripts/codeconverter/codeconverter/qom_type_info.py b/scripts/codeconverter/codeconverter/qom_type_info.py index 255cb59..22a2556 100644 --- a/scripts/codeconverter/codeconverter/qom_type_info.py +++ b/scripts/codeconverter/codeconverter/qom_type_info.py @@ -798,7 +798,8 @@ class RedundantTypeSizes(TypeInfoVar): # # # if 'class_init' not in fields: -# yield self.prepend(('static void %s_class_init(ObjectClass *oc, void *data)\n' +# yield self.prepend(('static void %s_class_init(ObjectClass *oc,\n' +# 'const void *data)\n' # '{\n' # '}\n\n') % (ids.lowercase)) # yield self.append_field('class_init', ids.lowercase+'_class_init') @@ -901,26 +902,6 @@ class TypeRegisterCall(FileMatch): regexp = S(r'^[ \t]*', NAMED('func_name', 'type_register'), r'\s*\(&\s*', NAMED('name', RE_IDENTIFIER), r'\s*\);[ \t]*\n') -class MakeTypeRegisterStatic(TypeRegisterCall): - """Make type_register() call static if variable is static const""" - def gen_patches(self): - var = self.file.find_match(TypeInfoVar, self.name) - if var is None: - self.warn("can't find TypeInfo var declaration for %s", self.name) - return - if var.is_static() and var.is_const(): - yield self.group_match('func_name').make_patch('type_register_static') - -class MakeTypeRegisterNotStatic(TypeRegisterStaticCall): - """Make type_register() call static if variable is static const""" - def gen_patches(self): - var = self.file.find_match(TypeInfoVar, self.name) - if var is None: - self.warn("can't find TypeInfo var declaration for %s", self.name) - return - if not var.is_static() or not var.is_const(): - yield self.group_match('func_name').make_patch('type_register') - class TypeInfoMacro(FileMatch): """TYPE_INFO macro usage""" regexp = S(r'^[ \t]*TYPE_INFO\s*\(\s*', NAMED('name', RE_IDENTIFIER), r'\s*\)[ \t]*;?[ \t]*\n') diff --git a/scripts/codeconverter/codeconverter/test_regexps.py b/scripts/codeconverter/codeconverter/test_regexps.py index a445634..4526268 100644 --- a/scripts/codeconverter/codeconverter/test_regexps.py +++ b/scripts/codeconverter/codeconverter/test_regexps.py @@ -70,15 +70,15 @@ static const TypeInfo char_file_type_info = { .name = armsse_variants[i].name, .parent = TYPE_ARMSSE, .class_init = armsse_class_init, - .class_data = (void *)&armsse_variants[i], + .class_data = &armsse_variants[i], };''', re.MULTILINE) print(RE_ARRAY_ITEM) assert fullmatch(RE_ARRAY_ITEM, '{ TYPE_HOTPLUG_HANDLER },') assert fullmatch(RE_ARRAY_ITEM, '{ TYPE_ACPI_DEVICE_IF },') assert fullmatch(RE_ARRAY_ITEM, '{ }') - assert fullmatch(RE_ARRAY_CAST, '(InterfaceInfo[])') - assert fullmatch(RE_ARRAY, '''(InterfaceInfo[]) { + assert fullmatch(RE_ARRAY_CAST, '(const InterfaceInfo[])') + assert fullmatch(RE_ARRAY, '''(const InterfaceInfo[]) { { TYPE_HOTPLUG_HANDLER }, { TYPE_ACPI_DEVICE_IF }, { } @@ -98,7 +98,7 @@ static const TypeInfo char_file_type_info = { .parent = TYPE_DEVICE, .instance_size = sizeof(CRBState), .class_init = tpm_crb_class_init, - .interfaces = (InterfaceInfo[]) { + .interfaces = (const InterfaceInfo[]) { { TYPE_TPM_IF }, { } } @@ -134,7 +134,7 @@ static const TypeInfo char_file_type_info = { .instance_size = sizeof(AcpiGedState), .instance_init = acpi_ged_initfn, .class_init = acpi_ged_class_init, - .interfaces = (InterfaceInfo[]) { + .interfaces = (const InterfaceInfo[]) { { TYPE_HOTPLUG_HANDLER }, { TYPE_ACPI_DEVICE_IF }, { } @@ -164,7 +164,7 @@ static const TypeInfo char_file_type_info = { .parent = TYPE_DEVICE, .instance_size = sizeof(CRBState), .class_init = tpm_crb_class_init, - .interfaces = (InterfaceInfo[]) { + .interfaces = (const InterfaceInfo[]) { { TYPE_TPM_IF }, { } } @@ -269,7 +269,7 @@ def test_initial_includes(): #include "hw/pci/pci.h" #include "migration/vmstate.h" #include "qemu/module.h" -#include "sysemu/dma.h" +#include "system/dma.h" /* Missing stuff: SCTRL_P[12](END|ST)INC @@ -278,5 +278,5 @@ def test_initial_includes(): m = InitialIncludes.domatch(c) assert m print(repr(m.group(0))) - assert m.group(0).endswith('#include "sysemu/dma.h"\n') + assert m.group(0).endswith('#include "system/dma.h"\n') diff --git a/scripts/coverity-scan/COMPONENTS.md b/scripts/coverity-scan/COMPONENTS.md index 858190b..7299590 100644 --- a/scripts/coverity-scan/COMPONENTS.md +++ b/scripts/coverity-scan/COMPONENTS.md @@ -9,9 +9,6 @@ arm avr ~ .*/qemu((/include)?/hw/avr/.*|/target/avr/.*) -cris - ~ .*/qemu((/include)?/hw/cris/.*|/target/cris/.*) - hexagon-gen (component should be ignored in analysis) ~ .*/qemu(/target/hexagon/.*generated.*) @@ -79,7 +76,7 @@ chardev ~ .*/qemu((/include)?/chardev/.*) crypto - ~ .*/qemu((/include)?/crypto/.*|/hw/.*/.*crypto.*|(/include/sysemu|/backends)/cryptodev.*|/host/include/.*/host/crypto/.*) + ~ .*/qemu((/include)?/crypto/.*|/hw/.*/.*crypto.*|(/include/system|/backends)/cryptodev.*|/host/include/.*/host/crypto/.*) disas ~ .*/qemu((/include)?/disas.*) @@ -147,7 +144,7 @@ kvm tcg ~ .*/qemu(/accel/tcg|/replay|/tcg)/.* -sysemu +system ~ .*/qemu(/system/.*|/accel/.*) (headers) diff --git a/scripts/device-crash-test b/scripts/device-crash-test index da8b56e..1ecb966 100755 --- a/scripts/device-crash-test +++ b/scripts/device-crash-test @@ -16,8 +16,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program; if not, see <https://www.gnu.org/licenses/>. """ Run QEMU with all combinations of -machine and -device types, diff --git a/scripts/gensyscalls.sh b/scripts/gensyscalls.sh deleted file mode 100755 index 8495728..0000000 --- a/scripts/gensyscalls.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/sh -# -# Update syscall_nr.h files from linux headers asm-generic/unistd.h -# -# This code is licensed under the GPL version 2 or later. See -# the COPYING file in the top-level directory. -# - -linux="$1" -output="$2" - -TMP=$(mktemp -d) - -if [ "$linux" = "" ] ; then - echo "Needs path to linux source tree" 1>&2 - exit 1 -fi - -if [ "$output" = "" ] ; then - output="$PWD" -fi - -upper() -{ - echo "$1" | tr "[:lower:]" "[:upper:]" | tr "[:punct:]" "_" -} - -qemu_arch() -{ - case "$1" in - arm64) - echo "aarch64" - ;; - *) - echo "$1" - ;; - esac -} - -read_includes() -{ - arch=$1 - bits=$2 - - cpp -P -nostdinc -fdirectives-only \ - -D_UAPI_ASM_$(upper ${arch})_BITSPERLONG_H \ - -D__ASM_$(upper ${arch})_BITSPERLONG_H \ - -D__BITS_PER_LONG=${bits} \ - -I${linux}/arch/${arch}/include/uapi/ \ - -I${linux}/include/uapi \ - -I${TMP} \ - "${linux}/arch/${arch}/include/uapi/asm/unistd.h" -} - -filter_defines() -{ - grep -e "#define __NR_" -e "#define __NR3264" -} - -rename_defines() -{ - sed "s/ __NR_/ TARGET_NR_/g;s/(__NR_/(TARGET_NR_/g" -} - -evaluate_values() -{ - sed "s/#define TARGET_NR_/QEMU TARGET_NR_/" | \ - cpp -P -nostdinc | \ - sed "s/^QEMU /#define /" -} - -generate_syscall_nr() -{ - arch=$1 - bits=$2 - file="$3" - guard="$(upper LINUX_USER_$(qemu_arch $arch)_$(basename "$file"))" - - (echo "/*" - echo " * This file contains the system call numbers." - echo " * Do not modify." - echo " * This file is generated by scripts/gensyscalls.sh" - echo " */" - echo "#ifndef ${guard}" - echo "#define ${guard}" - echo - read_includes $arch $bits | filter_defines | rename_defines | \ - evaluate_values | sort -n -k 3 - echo - echo "#endif /* ${guard} */") > "$file" -} - -mkdir "$TMP/asm" -> "$TMP/asm/bitsperlong.h" - -generate_syscall_nr arm64 64 "$output/linux-user/aarch64/syscall_nr.h" -generate_syscall_nr openrisc 32 "$output/linux-user/openrisc/syscall_nr.h" - -generate_syscall_nr riscv 32 "$output/linux-user/riscv/syscall32_nr.h" -generate_syscall_nr riscv 64 "$output/linux-user/riscv/syscall64_nr.h" -generate_syscall_nr hexagon 32 "$output/linux-user/hexagon/syscall_nr.h" -generate_syscall_nr loongarch 64 "$output/linux-user/loongarch64/syscall_nr.h" -rm -fr "$TMP" diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 240923d..fec83f5 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -1,5 +1,5 @@ #!/usr/bin/env perl -# SPDX-License-Identifier: GPL-2.0 +# SPDX-License-Identifier: GPL-2.0-only use warnings; use strict; diff --git a/scripts/make-release b/scripts/make-release index 6e0433d..4509a9f 100755 --- a/scripts/make-release +++ b/scripts/make-release @@ -10,6 +10,28 @@ # This work is licensed under the terms of the GNU GPLv2 or later. # See the COPYING file in the top-level directory. +function subproject_dir() { + if test ! -f "$src/subprojects/$1.wrap"; then + echo "scripts/archive-source.sh should only process wrap subprojects" + exit 1 + fi + + # Print the directory key of the wrap file, defaulting to the + # subproject name. The wrap file is in ini format and should + # have a single section only. There should be only one section + # named "[wrap-*]", which helps keeping the script simple. + local dir + dir=$(sed -n \ + -e '/^\[wrap-[a-z][a-z]*\]$/,/^\[/{' \ + -e '/^directory *= */!b' \ + -e 's///p' \ + -e 'q' \ + -e '}' \ + "$src/subprojects/$1.wrap") + + echo "${dir:-$1}" +} + if [ $# -ne 2 ]; then echo "Usage:" echo " $0 gitrepo version" @@ -17,7 +39,12 @@ if [ $# -ne 2 ]; then fi # Only include wraps that are invoked with subproject() -SUBPROJECTS="libvfio-user keycodemapdb berkeley-softfloat-3 berkeley-testfloat-3" +SUBPROJECTS="libvfio-user keycodemapdb berkeley-softfloat-3 + berkeley-testfloat-3 anyhow-1-rs arbitrary-int-1-rs bilge-0.2-rs + bilge-impl-0.2-rs either-1-rs foreign-0.3-rs itertools-0.11-rs + libc-0.2-rs proc-macro2-1-rs + proc-macro-error-1-rs proc-macro-error-attr-1-rs quote-1-rs + syn-2-rs unicode-ident-1-rs" src="$1" version="$2" @@ -47,5 +74,13 @@ meson subprojects download $SUBPROJECTS CryptoPkg/Library/OpensslLib/openssl \ MdeModulePkg/Library/BrotliCustomDecompressLib/brotli) popd -tar --exclude=.git -cJf ${destination}.tar.xz ${destination} + +exclude=(--exclude=.git) +# include the tarballs in subprojects/packagecache but not their expansion +for sp in $SUBPROJECTS; do + if grep -xqF "[wrap-file]" $src/subprojects/$sp.wrap; then + exclude+=(--exclude=subprojects/"$(subproject_dir $sp)") + fi +done +tar "${exclude[@]}" -cJf ${destination}.tar.xz ${destination} rm -rf ${destination} diff --git a/scripts/meson-buildoptions.py b/scripts/meson-buildoptions.py index 4814a8f..a3e2247 100644 --- a/scripts/meson-buildoptions.py +++ b/scripts/meson-buildoptions.py @@ -241,8 +241,14 @@ def print_parse(options): print(" esac") print("}") - -options = load_options(json.load(sys.stdin)) +json_data = sys.stdin.read() +try: + options = load_options(json.loads(json_data)) +except: + print("Failure in scripts/meson-buildoptions.py parsing stdin as json", + file=sys.stderr) + print(json_data, file=sys.stderr) + sys.exit(1) print("# This file is generated by meson-buildoptions.py, do not edit!") print_help(options) print_parse(options) diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh index c97079a..73e0770 100644 --- a/scripts/meson-buildoptions.sh +++ b/scripts/meson-buildoptions.sh @@ -21,6 +21,7 @@ meson_options_help() { printf "%s\n" ' --disable-relocatable toggle relocatable install' printf "%s\n" ' --docdir=VALUE Base directory for documentation installation' printf "%s\n" ' (can be empty) [share/doc]' + printf "%s\n" ' --enable-asan enable address sanitizer' printf "%s\n" ' --enable-block-drv-whitelist-in-tools' printf "%s\n" ' use block whitelist also in tools instead of only' printf "%s\n" ' QEMU' @@ -46,13 +47,15 @@ meson_options_help() { printf "%s\n" ' getrandom()' printf "%s\n" ' --enable-safe-stack SafeStack Stack Smash Protection (requires' printf "%s\n" ' clang/llvm and coroutine backend ucontext)' - printf "%s\n" ' --enable-sanitizers enable default sanitizers' + printf "%s\n" ' --enable-strict-rust-lints' + printf "%s\n" ' Enable stricter set of Rust warnings' printf "%s\n" ' --enable-strip Strip targets on install' printf "%s\n" ' --enable-tcg-interpreter TCG with bytecode interpreter (slow)' printf "%s\n" ' --enable-trace-backends=CHOICES' printf "%s\n" ' Set available tracing backends [log] (choices:' printf "%s\n" ' dtrace/ftrace/log/nop/simple/syslog/ust)' printf "%s\n" ' --enable-tsan enable thread sanitizer' + printf "%s\n" ' --enable-ubsan enable undefined behaviour sanitizer' printf "%s\n" ' --firmwarepath=VALUES search PATH for firmware files [share/qemu-' printf "%s\n" ' firmware]' printf "%s\n" ' --iasl=VALUE Path to ACPI disassembler' @@ -71,12 +74,13 @@ meson_options_help() { printf "%s\n" ' "manufacturer" name for qemu-ga registry entries' printf "%s\n" ' [QEMU]' printf "%s\n" ' --qemu-ga-version=VALUE version number for qemu-ga installer' + printf "%s\n" ' --rtsig-map=VALUE default value of QEMU_RTSIG_MAP [NULL]' printf "%s\n" ' --smbd=VALUE Path to smbd for slirp networking' printf "%s\n" ' --sysconfdir=VALUE Sysconf data directory [etc]' printf "%s\n" ' --tls-priority=VALUE Default TLS protocol/cipher priority string' printf "%s\n" ' [NORMAL]' printf "%s\n" ' --with-coroutine=CHOICE coroutine backend to use (choices:' - printf "%s\n" ' auto/sigaltstack/ucontext/windows)' + printf "%s\n" ' auto/sigaltstack/ucontext/wasm/windows)' printf "%s\n" ' --with-pkgversion=VALUE use specified string as sub-version of the' printf "%s\n" ' package' printf "%s\n" ' --with-suffix=VALUE Suffix for QEMU data/modules/config directories' @@ -93,8 +97,6 @@ meson_options_help() { printf "%s\n" ' alsa ALSA sound support' printf "%s\n" ' attr attr/xattr support' printf "%s\n" ' auth-pam PAM access control' - printf "%s\n" ' avx2 AVX2 optimizations' - printf "%s\n" ' avx512bw AVX512BW optimizations' printf "%s\n" ' blkio libblkio block device driver' printf "%s\n" ' bochs bochs image format support' printf "%s\n" ' bpf eBPF support' @@ -132,6 +134,7 @@ meson_options_help() { printf "%s\n" ' keyring Linux keyring support' printf "%s\n" ' kvm KVM acceleration support' printf "%s\n" ' l2tpv3 l2tpv3 network backend support' + printf "%s\n" ' libcbor libcbor support' printf "%s\n" ' libdaxctl libdaxctl support' printf "%s\n" ' libdw debuginfo support' printf "%s\n" ' libiscsi libiscsi userspace initiator' @@ -163,6 +166,8 @@ meson_options_help() { printf "%s\n" ' pixman pixman support' printf "%s\n" ' plugins TCG plugins via shared library loading' printf "%s\n" ' png PNG support with libpng' + printf "%s\n" ' pvg macOS paravirtualized graphics support' + printf "%s\n" ' qatzip QATzip compression support' printf "%s\n" ' qcow1 qcow1 image format support' printf "%s\n" ' qed qed image format support' printf "%s\n" ' qga-vss build QGA VSS support (broken with MinGW)' @@ -170,6 +175,7 @@ meson_options_help() { printf "%s\n" ' rbd Ceph block device driver' printf "%s\n" ' rdma Enable RDMA-based migration' printf "%s\n" ' replication replication support' + printf "%s\n" ' rust Rust support' printf "%s\n" ' rutabaga-gfx rutabaga_gfx support' printf "%s\n" ' sdl SDL user interface' printf "%s\n" ' sdl-image SDL Image support for icons' @@ -190,6 +196,7 @@ meson_options_help() { printf "%s\n" ' u2f U2F emulation support' printf "%s\n" ' uadk UADK Library support' printf "%s\n" ' usb-redir libusbredir support' + printf "%s\n" ' valgrind valgrind debug support for coroutine stacks' printf "%s\n" ' vde vde network backend support' printf "%s\n" ' vdi vdi image format support' printf "%s\n" ' vduse-blk-export' @@ -206,8 +213,6 @@ meson_options_help() { printf "%s\n" ' vhost-vdpa vhost-vdpa kernel backend support' printf "%s\n" ' virglrenderer virgl rendering support' printf "%s\n" ' virtfs virtio-9p support' - printf "%s\n" ' virtfs-proxy-helper' - printf "%s\n" ' virtio-9p proxy helper support' printf "%s\n" ' vmdk vmdk image format support' printf "%s\n" ' vmnet vmnet.framework network backend support' printf "%s\n" ' vnc VNC server' @@ -230,15 +235,13 @@ _meson_option_parse() { --disable-af-xdp) printf "%s" -Daf_xdp=disabled ;; --enable-alsa) printf "%s" -Dalsa=enabled ;; --disable-alsa) printf "%s" -Dalsa=disabled ;; + --enable-asan) printf "%s" -Dasan=true ;; + --disable-asan) printf "%s" -Dasan=false ;; --enable-attr) printf "%s" -Dattr=enabled ;; --disable-attr) printf "%s" -Dattr=disabled ;; --audio-drv-list=*) quote_sh "-Daudio_drv_list=$2" ;; --enable-auth-pam) printf "%s" -Dauth_pam=enabled ;; --disable-auth-pam) printf "%s" -Dauth_pam=disabled ;; - --enable-avx2) printf "%s" -Davx2=enabled ;; - --disable-avx2) printf "%s" -Davx2=disabled ;; - --enable-avx512bw) printf "%s" -Davx512bw=enabled ;; - --disable-avx512bw) printf "%s" -Davx512bw=disabled ;; --enable-gcov) printf "%s" -Db_coverage=true ;; --disable-gcov) printf "%s" -Db_coverage=false ;; --enable-lto) printf "%s" -Db_lto=true ;; @@ -355,6 +358,8 @@ _meson_option_parse() { --disable-kvm) printf "%s" -Dkvm=disabled ;; --enable-l2tpv3) printf "%s" -Dl2tpv3=enabled ;; --disable-l2tpv3) printf "%s" -Dl2tpv3=disabled ;; + --enable-libcbor) printf "%s" -Dlibcbor=enabled ;; + --disable-libcbor) printf "%s" -Dlibcbor=disabled ;; --enable-libdaxctl) printf "%s" -Dlibdaxctl=enabled ;; --disable-libdaxctl) printf "%s" -Dlibdaxctl=disabled ;; --libdir=*) quote_sh "-Dlibdir=$2" ;; @@ -427,6 +432,10 @@ _meson_option_parse() { --enable-png) printf "%s" -Dpng=enabled ;; --disable-png) printf "%s" -Dpng=disabled ;; --prefix=*) quote_sh "-Dprefix=$2" ;; + --enable-pvg) printf "%s" -Dpvg=enabled ;; + --disable-pvg) printf "%s" -Dpvg=disabled ;; + --enable-qatzip) printf "%s" -Dqatzip=enabled ;; + --disable-qatzip) printf "%s" -Dqatzip=disabled ;; --enable-qcow1) printf "%s" -Dqcow1=enabled ;; --disable-qcow1) printf "%s" -Dqcow1=disabled ;; --enable-qed) printf "%s" -Dqed=enabled ;; @@ -452,12 +461,13 @@ _meson_option_parse() { --disable-replication) printf "%s" -Dreplication=disabled ;; --enable-rng-none) printf "%s" -Drng_none=true ;; --disable-rng-none) printf "%s" -Drng_none=false ;; + --rtsig-map=*) quote_sh "-Drtsig_map=$2" ;; + --enable-rust) printf "%s" -Drust=enabled ;; + --disable-rust) printf "%s" -Drust=disabled ;; --enable-rutabaga-gfx) printf "%s" -Drutabaga_gfx=enabled ;; --disable-rutabaga-gfx) printf "%s" -Drutabaga_gfx=disabled ;; --enable-safe-stack) printf "%s" -Dsafe_stack=true ;; --disable-safe-stack) printf "%s" -Dsafe_stack=false ;; - --enable-sanitizers) printf "%s" -Dsanitizers=true ;; - --disable-sanitizers) printf "%s" -Dsanitizers=false ;; --enable-sdl) printf "%s" -Dsdl=enabled ;; --disable-sdl) printf "%s" -Dsdl=disabled ;; --enable-sdl-image) printf "%s" -Dsdl_image=enabled ;; @@ -485,6 +495,8 @@ _meson_option_parse() { --disable-spice-protocol) printf "%s" -Dspice_protocol=disabled ;; --enable-stack-protector) printf "%s" -Dstack_protector=enabled ;; --disable-stack-protector) printf "%s" -Dstack_protector=disabled ;; + --enable-strict-rust-lints) printf "%s" -Dstrict_rust_lints=true ;; + --disable-strict-rust-lints) printf "%s" -Dstrict_rust_lints=false ;; --enable-strip) printf "%s" -Dstrip=true ;; --disable-strip) printf "%s" -Dstrip=false ;; --sysconfdir=*) quote_sh "-Dsysconfdir=$2" ;; @@ -505,8 +517,12 @@ _meson_option_parse() { --disable-u2f) printf "%s" -Du2f=disabled ;; --enable-uadk) printf "%s" -Duadk=enabled ;; --disable-uadk) printf "%s" -Duadk=disabled ;; + --enable-ubsan) printf "%s" -Dubsan=true ;; + --disable-ubsan) printf "%s" -Dubsan=false ;; --enable-usb-redir) printf "%s" -Dusb_redir=enabled ;; --disable-usb-redir) printf "%s" -Dusb_redir=disabled ;; + --enable-valgrind) printf "%s" -Dvalgrind=enabled ;; + --disable-valgrind) printf "%s" -Dvalgrind=disabled ;; --enable-vde) printf "%s" -Dvde=enabled ;; --disable-vde) printf "%s" -Dvde=disabled ;; --enable-vdi) printf "%s" -Dvdi=enabled ;; @@ -533,8 +549,6 @@ _meson_option_parse() { --disable-virglrenderer) printf "%s" -Dvirglrenderer=disabled ;; --enable-virtfs) printf "%s" -Dvirtfs=enabled ;; --disable-virtfs) printf "%s" -Dvirtfs=disabled ;; - --enable-virtfs-proxy-helper) printf "%s" -Dvirtfs_proxy_helper=enabled ;; - --disable-virtfs-proxy-helper) printf "%s" -Dvirtfs_proxy_helper=disabled ;; --enable-vmdk) printf "%s" -Dvmdk=enabled ;; --disable-vmdk) printf "%s" -Dvmdk=disabled ;; --enable-vmnet) printf "%s" -Dvmnet=enabled ;; diff --git a/scripts/minikconf.py b/scripts/minikconf.py index bcd9101..6f7f43b 100644 --- a/scripts/minikconf.py +++ b/scripts/minikconf.py @@ -112,7 +112,7 @@ class KconfigData: def set_value(self, val, clause): self.clauses_for_var.append(clause) if self.has_value() and self.value != val: - print("The following clauses were found for " + self.name) + print("The following clauses were found for " + self.name, file=sys.stderr) for i in self.clauses_for_var: print(" " + str(i), file=sys.stderr) raise KconfigDataError('contradiction between clauses when setting %s' % self) diff --git a/scripts/modinfo-collect.py b/scripts/modinfo-collect.py index 4e7584d..48bd92b 100644 --- a/scripts/modinfo-collect.py +++ b/scripts/modinfo-collect.py @@ -7,15 +7,6 @@ import json import shlex import subprocess -def find_command(src, target, compile_commands): - for command in compile_commands: - if command['file'] != src: - continue - if target != '' and command['command'].find(target) == -1: - continue - return command['command'] - return 'false' - def process_command(src, command): skip = False out = [] @@ -43,14 +34,22 @@ def main(args): print("MODINFO_DEBUG target %s" % target) arch = target[:-8] # cut '-softmmu' print("MODINFO_START arch \"%s\" MODINFO_END" % arch) + with open('compile_commands.json') as f: - compile_commands = json.load(f) - for src in args: + compile_commands_json = json.load(f) + compile_commands = { x['output']: x for x in compile_commands_json } + + for obj in args: + entry = compile_commands.get(obj, None) + if not entry: + sys.stderr.print('modinfo: Could not find object file', obj) + sys.exit(1) + src = entry['file'] if not src.endswith('.c'): print("MODINFO_DEBUG skip %s" % src) continue + command = entry['command'] print("MODINFO_DEBUG src %s" % src) - command = find_command(src, target, compile_commands) cmdline = process_command(src, command) print("MODINFO_DEBUG cmd", cmdline) result = subprocess.run(cmdline, stdout = subprocess.PIPE, diff --git a/scripts/mtest2make.py b/scripts/mtest2make.py index eb01a05..2ef375f 100644 --- a/scripts/mtest2make.py +++ b/scripts/mtest2make.py @@ -27,7 +27,7 @@ SPEED = quick .speed.slow = $(foreach s,$(sort $(filter-out %-thorough, $1)), --suite $s) .speed.thorough = $(foreach s,$(sort $1), --suite $s) -TIMEOUT_MULTIPLIER = 1 +TIMEOUT_MULTIPLIER ?= 1 .mtestargs = --no-rebuild -t $(TIMEOUT_MULTIPLIER) ifneq ($(SPEED), quick) .mtestargs += --setup $(SPEED) diff --git a/scripts/nsis.py b/scripts/nsis.py index 03ed760..8f46963 100644 --- a/scripts/nsis.py +++ b/scripts/nsis.py @@ -23,7 +23,7 @@ def find_deps(exe_or_dll, search_path, analyzed_deps): output = subprocess.check_output(["objdump", "-p", exe_or_dll], text=True) output = output.split("\n") for line in output: - if not line.startswith("\tDLL Name: "): + if not line.lstrip().startswith("DLL Name: "): continue dep = line.split("DLL Name: ")[1].strip() @@ -37,10 +37,10 @@ def find_deps(exe_or_dll, search_path, analyzed_deps): analyzed_deps.add(dep) # locate the dll dependencies recursively - rdeps = find_deps(dll, search_path, analyzed_deps) + analyzed_deps, rdeps = find_deps(dll, search_path, analyzed_deps) deps.extend(rdeps) - return deps + return analyzed_deps, deps def main(): parser = argparse.ArgumentParser(description="QEMU NSIS build helper.") @@ -92,18 +92,18 @@ def main(): dlldir = os.path.join(destdir + prefix, "dll") os.mkdir(dlldir) + analyzed_deps = set() for exe in glob.glob(os.path.join(destdir + prefix, "*.exe")): signcode(exe) # find all dll dependencies - deps = set(find_deps(exe, search_path, set())) + analyzed_deps, deps = find_deps(exe, search_path, analyzed_deps) + deps = set(deps) deps.remove(exe) # copy all dlls to the DLLDIR for dep in deps: dllfile = os.path.join(dlldir, os.path.basename(dep)) - if (os.path.exists(dllfile)): - continue print("Copying '%s' to '%s'" % (dep, dllfile)) shutil.copy(dep, dllfile) diff --git a/scripts/probe-gdb-support.py b/scripts/probe-gdb-support.py index 46d6c00..6bcadce 100644 --- a/scripts/probe-gdb-support.py +++ b/scripts/probe-gdb-support.py @@ -19,59 +19,61 @@ import argparse import re -from subprocess import check_output, STDOUT +from subprocess import check_output, STDOUT, CalledProcessError +import sys -# mappings from gdb arch to QEMU target -mappings = { - "alpha" : "alpha", +# Mappings from gdb arch to QEMU target +MAP = { + "alpha" : ["alpha"], "aarch64" : ["aarch64", "aarch64_be"], - "armv7": "arm", + "armv7": ["arm"], "armv8-a" : ["aarch64", "aarch64_be"], - "avr" : "avr", - "cris" : "cris", + "avr" : ["avr"], # no hexagon in upstream gdb - "hppa1.0" : "hppa", - "i386" : "i386", - "i386:x86-64" : "x86_64", - "Loongarch64" : "loongarch64", - "m68k" : "m68k", - "MicroBlaze" : "microblaze", + "hppa1.0" : ["hppa"], + "i386" : ["i386"], + "i386:x86-64" : ["x86_64"], + "Loongarch64" : ["loongarch64"], + "m68k" : ["m68k"], + "MicroBlaze" : ["microblaze"], "mips:isa64" : ["mips64", "mips64el"], - "or1k" : "or1k", - "powerpc:common" : "ppc", + "or1k" : ["or1k"], + "powerpc:common" : ["ppc"], "powerpc:common64" : ["ppc64", "ppc64le"], - "riscv:rv32" : "riscv32", - "riscv:rv64" : "riscv64", - "s390:64-bit" : "s390x", + "riscv:rv32" : ["riscv32"], + "riscv:rv64" : ["riscv64"], + "s390:64-bit" : ["s390x"], "sh4" : ["sh4", "sh4eb"], - "sparc": "sparc", - "sparc:v8plus": "sparc32plus", - "sparc:v9a" : "sparc64", + "sparc": ["sparc"], + "sparc:v8plus": ["sparc32plus"], + "sparc:v9a" : ["sparc64"], # no tricore in upstream gdb "xtensa" : ["xtensa", "xtensaeb"] } + def do_probe(gdb): - gdb_out = check_output([gdb, - "-ex", "set architecture", - "-ex", "quit"], stderr=STDOUT) + try: + gdb_out = check_output([gdb, + "-ex", "set architecture", + "-ex", "quit"], stderr=STDOUT, encoding="utf-8") + except (OSError) as e: + sys.exit(e) + except CalledProcessError as e: + sys.exit(f'{e}. Output:\n\n{e.output}') + + found_gdb_archs = re.search(r'Valid arguments are (.*)', gdb_out) - m = re.search(r"Valid arguments are (.*)", - gdb_out.decode("utf-8")) + targets = set() + if found_gdb_archs: + gdb_archs = found_gdb_archs.group(1).split(", ") + mapped_gdb_archs = [arch for arch in gdb_archs if arch in MAP] - valid_arches = set() + targets = {target for arch in mapped_gdb_archs for target in MAP[arch]} - if m.group(1): - for arch in m.group(1).split(", "): - if arch in mappings: - mapping = mappings[arch] - if isinstance(mapping, str): - valid_arches.add(mapping) - else: - for entry in mapping: - valid_arches.add(entry) + # QEMU targets + return targets - return valid_arches def main() -> None: parser = argparse.ArgumentParser(description='Probe GDB Architectures') diff --git a/scripts/qapi/.flake8 b/scripts/qapi/.flake8 deleted file mode 100644 index a873ff6..0000000 --- a/scripts/qapi/.flake8 +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -# Prefer pylint's bare-except checks to flake8's -extend-ignore = E722 diff --git a/scripts/qapi/.isort.cfg b/scripts/qapi/.isort.cfg deleted file mode 100644 index 643caa1..0000000 --- a/scripts/qapi/.isort.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[settings] -force_grid_wrap=4 -force_sort_within_sections=True -include_trailing_comma=True -line_length=72 -lines_after_imports=2 -multi_line_output=3 diff --git a/scripts/qapi/backend.py b/scripts/qapi/backend.py new file mode 100644 index 0000000..49ae6ec --- /dev/null +++ b/scripts/qapi/backend.py @@ -0,0 +1,65 @@ +# This work is licensed under the terms of the GNU GPL, version 2 or later. +# See the COPYING file in the top-level directory. + +from abc import ABC, abstractmethod + +from .commands import gen_commands +from .events import gen_events +from .features import gen_features +from .introspect import gen_introspect +from .schema import QAPISchema +from .types import gen_types +from .visit import gen_visit + + +class QAPIBackend(ABC): + # pylint: disable=too-few-public-methods + + @abstractmethod + def generate(self, + schema: QAPISchema, + output_dir: str, + prefix: str, + unmask: bool, + builtins: bool, + gen_tracing: bool) -> None: + """ + Generate code for the given schema into the target directory. + + :param schema: The primary QAPI schema object. + :param output_dir: The output directory to store generated code. + :param prefix: Optional C-code prefix for symbol names. + :param unmask: Expose non-ABI names through introspection? + :param builtins: Generate code for built-in types? + + :raise QAPIError: On failures. + """ + + +class QAPICBackend(QAPIBackend): + # pylint: disable=too-few-public-methods + + def generate(self, + schema: QAPISchema, + output_dir: str, + prefix: str, + unmask: bool, + builtins: bool, + gen_tracing: bool) -> None: + """ + Generate C code for the given schema into the target directory. + + :param schema_file: The primary QAPI schema file. + :param output_dir: The output directory to store generated code. + :param prefix: Optional C-code prefix for symbol names. + :param unmask: Expose non-ABI names through introspection? + :param builtins: Generate code for built-in types? + + :raise QAPIError: On failures. + """ + gen_types(schema, output_dir, prefix, builtins) + gen_features(schema, output_dir, prefix) + gen_visit(schema, output_dir, prefix, builtins) + gen_commands(schema, output_dir, prefix, gen_tracing) + gen_events(schema, output_dir, prefix) + gen_introspect(schema, output_dir, prefix, unmask) diff --git a/scripts/qapi/commands.py b/scripts/qapi/commands.py index 79951a8..7914227 100644 --- a/scripts/qapi/commands.py +++ b/scripts/qapi/commands.py @@ -25,7 +25,7 @@ from .gen import ( QAPIGenC, QAPISchemaModularCVisitor, build_params, - gen_special_features, + gen_features, ifcontext, ) from .schema import ( @@ -298,7 +298,7 @@ def gen_register_command(name: str, ''', name=name, c_name=c_name(name), opts=' | '.join(options) or 0, - feats=gen_special_features(features)) + feats=gen_features(features)) return ret @@ -320,7 +320,7 @@ class QAPISchemaGenCommandVisitor(QAPISchemaModularCVisitor): #include "qemu/osdep.h" #include "qapi/compat-policy.h" #include "qapi/visitor.h" -#include "qapi/qmp/qdict.h" +#include "qobject/qdict.h" #include "qapi/dealloc-visitor.h" #include "qapi/error.h" #include "%(visit)s.h" @@ -330,7 +330,7 @@ class QAPISchemaGenCommandVisitor(QAPISchemaModularCVisitor): if self._gen_tracing and commands != 'qapi-commands': self._genc.add(mcgen(''' -#include "qapi/qmp/qjson.h" +#include "qobject/qjson.h" #include "trace/trace-%(nm)s_trace_events.h" ''', nm=c_name(commands, protect=False))) @@ -346,7 +346,7 @@ class QAPISchemaGenCommandVisitor(QAPISchemaModularCVisitor): def visit_begin(self, schema: QAPISchema) -> None: self._add_module('./init', ' * QAPI Commands initialization') self._genh.add(mcgen(''' -#include "qapi/qmp/dispatch.h" +#include "qapi/qmp-registry.h" void %(c_prefix)sqmp_init_marshal(QmpCommandList *cmds); ''', @@ -355,6 +355,7 @@ void %(c_prefix)sqmp_init_marshal(QmpCommandList *cmds); #include "qemu/osdep.h" #include "%(prefix)sqapi-commands.h" #include "%(prefix)sqapi-init-commands.h" +#include "%(prefix)sqapi-features.h" void %(c_prefix)sqmp_init_marshal(QmpCommandList *cmds) { diff --git a/scripts/qapi/common.py b/scripts/qapi/common.py index 737b059..d7c8aa3 100644 --- a/scripts/qapi/common.py +++ b/scripts/qapi/common.py @@ -40,22 +40,28 @@ def camel_to_upper(value: str) -> str: ENUM_Name2 -> ENUM_NAME2 ENUM24_Name -> ENUM24_NAME """ - c_fun_str = c_name(value, False) - if value.isupper(): - return c_fun_str - - new_name = '' - length = len(c_fun_str) - for i in range(length): - char = c_fun_str[i] - # When char is upper case and no '_' appears before, do more checks - if char.isupper() and (i > 0) and c_fun_str[i - 1] != '_': - if i < length - 1 and c_fun_str[i + 1].islower(): - new_name += '_' - elif c_fun_str[i - 1].isdigit(): - new_name += '_' - new_name += char - return new_name.lstrip('_').upper() + ret = value[0] + upc = value[0].isupper() + + # Copy remainder of ``value`` to ``ret`` with '_' inserted + for ch in value[1:]: + if ch.isupper() == upc: + pass + elif upc: + # ``ret`` ends in upper case, next char isn't: insert '_' + # before the last upper case char unless there is one + # already, or it's at the beginning + if len(ret) > 2 and ret[-2].isalnum(): + ret = ret[:-1] + '_' + ret[-1] + else: + # ``ret`` doesn't end in upper case, next char is: insert + # '_' before it + if ret[-1].isalnum(): + ret += '_' + ret += ch + upc = ch.isupper() + + return c_name(ret.upper()).lstrip('_') def c_enum_const(type_name: str, @@ -68,9 +74,9 @@ def c_enum_const(type_name: str, :param const_name: The name of this constant. :param prefix: Optional, prefix that overrides the type_name. """ - if prefix is not None: - type_name = prefix - return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper() + if prefix is None: + prefix = camel_to_upper(type_name) + return prefix + '_' + c_name(const_name, False).upper() def c_name(name: str, protect: bool = True) -> str: diff --git a/scripts/qapi/events.py b/scripts/qapi/events.py index d1f6399..d179b0e 100644 --- a/scripts/qapi/events.py +++ b/scripts/qapi/events.py @@ -194,7 +194,7 @@ class QAPISchemaGenEventVisitor(QAPISchemaModularCVisitor): #include "%(visit)s.h" #include "qapi/compat-policy.h" #include "qapi/error.h" -#include "qapi/qmp/qdict.h" +#include "qobject/qdict.h" #include "qapi/qmp-event.h" ''', events=events, visit=visit, diff --git a/scripts/qapi/features.py b/scripts/qapi/features.py new file mode 100644 index 0000000..5756320 --- /dev/null +++ b/scripts/qapi/features.py @@ -0,0 +1,48 @@ +""" +QAPI features generator + +Copyright 2024 Red Hat + +This work is licensed under the terms of the GNU GPL, version 2. +# See the COPYING file in the top-level directory. +""" + +from typing import ValuesView + +from .common import c_enum_const, c_name +from .gen import QAPISchemaMonolithicCVisitor +from .schema import QAPISchema, QAPISchemaFeature + + +class QAPISchemaGenFeatureVisitor(QAPISchemaMonolithicCVisitor): + + def __init__(self, prefix: str): + super().__init__( + prefix, 'qapi-features', + ' * Schema-defined QAPI features', + __doc__) + + self.features: ValuesView[QAPISchemaFeature] + + def visit_begin(self, schema: QAPISchema) -> None: + self.features = schema.features() + self._genh.add("#include \"qapi/util.h\"\n\n") + + def visit_end(self) -> None: + self._genh.add("typedef enum {\n") + for f in self.features: + self._genh.add(f" {c_enum_const('qapi_feature', f.name)}") + if f.name in QAPISchemaFeature.SPECIAL_NAMES: + self._genh.add(f" = {c_enum_const('qapi', f.name)},\n") + else: + self._genh.add(",\n") + + self._genh.add("} " + c_name('QapiFeature') + ";\n") + + +def gen_features(schema: QAPISchema, + output_dir: str, + prefix: str) -> None: + vis = QAPISchemaGenFeatureVisitor(prefix) + schema.visit(vis) + vis.write(output_dir) diff --git a/scripts/qapi/gen.py b/scripts/qapi/gen.py index 6a8abe0..d3c56d45 100644 --- a/scripts/qapi/gen.py +++ b/scripts/qapi/gen.py @@ -24,6 +24,7 @@ from typing import ( ) from .common import ( + c_enum_const, c_fname, c_name, guardend, @@ -40,10 +41,10 @@ from .schema import ( from .source import QAPISourceInfo -def gen_special_features(features: Sequence[QAPISchemaFeature]) -> str: - special_features = [f"1u << QAPI_{feat.name.upper()}" - for feat in features if feat.is_special()] - return ' | '.join(special_features) or '0' +def gen_features(features: Sequence[QAPISchemaFeature]) -> str: + feats = [f"1u << {c_enum_const('qapi_feature', feat.name)}" + for feat in features] + return ' | '.join(feats) or '0' class QAPIGen: diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py index ac14b20..89ee5d5 100644 --- a/scripts/qapi/introspect.py +++ b/scripts/qapi/introspect.py @@ -11,6 +11,7 @@ This work is licensed under the terms of the GNU GPL, version 2. See the COPYING file in the top-level directory. """ +from dataclasses import dataclass from typing import ( Any, Dict, @@ -79,19 +80,16 @@ SchemaInfoCommand = Dict[str, object] _ValueT = TypeVar('_ValueT', bound=_Value) +@dataclass class Annotated(Generic[_ValueT]): """ Annotated generally contains a SchemaInfo-like type (as a dict), But it also used to wrap comments/ifconds around scalar leaf values, for the benefit of features and enums. """ - # TODO: Remove after Python 3.7 adds @dataclass: - # pylint: disable=too-few-public-methods - def __init__(self, value: _ValueT, ifcond: QAPISchemaIfCond, - comment: Optional[str] = None): - self.value = value - self.comment: Optional[str] = comment - self.ifcond = ifcond + value: _ValueT + ifcond: QAPISchemaIfCond + comment: Optional[str] = None def _tree_to_qlit(obj: JSONValue, @@ -197,7 +195,7 @@ class QAPISchemaGenIntrospectVisitor(QAPISchemaMonolithicCVisitor): # generate C name = c_name(self._prefix, protect=False) + 'qmp_schema_qlit' self._genh.add(mcgen(''' -#include "qapi/qmp/qlit.h" +#include "qobject/qlit.h" extern const QLitObject %(c_name)s; ''', diff --git a/scripts/qapi/main.py b/scripts/qapi/main.py index 316736b..0e2a6ae 100644 --- a/scripts/qapi/main.py +++ b/scripts/qapi/main.py @@ -8,17 +8,14 @@ This is the main entry point for generating C code from the QAPI schema. """ import argparse +from importlib import import_module import sys from typing import Optional -from .commands import gen_commands +from .backend import QAPIBackend, QAPICBackend from .common import must_match from .error import QAPIError -from .events import gen_events -from .introspect import gen_introspect from .schema import QAPISchema -from .types import gen_types -from .visit import gen_visit def invalid_prefix_char(prefix: str) -> Optional[str]: @@ -28,31 +25,36 @@ def invalid_prefix_char(prefix: str) -> Optional[str]: return None -def generate(schema_file: str, - output_dir: str, - prefix: str, - unmask: bool = False, - builtins: bool = False, - gen_tracing: bool = False) -> None: - """ - Generate C code for the given schema into the target directory. +def create_backend(path: str) -> QAPIBackend: + if path is None: + return QAPICBackend() - :param schema_file: The primary QAPI schema file. - :param output_dir: The output directory to store generated code. - :param prefix: Optional C-code prefix for symbol names. - :param unmask: Expose non-ABI names through introspection? - :param builtins: Generate code for built-in types? + module_path, dot, class_name = path.rpartition('.') + if not dot: + raise QAPIError("argument of -B must be of the form MODULE.CLASS") - :raise QAPIError: On failures. - """ - assert invalid_prefix_char(prefix) is None + try: + mod = import_module(module_path) + except Exception as ex: + raise QAPIError(f"unable to import '{module_path}': {ex}") from ex + + try: + klass = getattr(mod, class_name) + except AttributeError as ex: + raise QAPIError( + f"module '{module_path}' has no class '{class_name}'") from ex + + try: + backend = klass() + except Exception as ex: + raise QAPIError( + f"backend '{path}' cannot be instantiated: {ex}") from ex + + if not isinstance(backend, QAPIBackend): + raise QAPIError( + f"backend '{path}' must be an instance of QAPIBackend") - schema = QAPISchema(schema_file) - gen_types(schema, output_dir, prefix, builtins) - gen_visit(schema, output_dir, prefix, builtins) - gen_commands(schema, output_dir, prefix, gen_tracing) - gen_events(schema, output_dir, prefix) - gen_introspect(schema, output_dir, prefix, unmask) + return backend def main() -> int: @@ -75,6 +77,8 @@ def main() -> int: parser.add_argument('-u', '--unmask-non-abi-names', action='store_true', dest='unmask', help="expose non-ABI names in introspection") + parser.add_argument('-B', '--backend', default=None, + help="Python module name for code generator") # Option --suppress-tracing exists so we can avoid solving build system # problems. TODO Drop it when we no longer need it. @@ -91,12 +95,14 @@ def main() -> int: return 1 try: - generate(args.schema, - output_dir=args.output_dir, - prefix=args.prefix, - unmask=args.unmask, - builtins=args.builtins, - gen_tracing=not args.suppress_tracing) + schema = QAPISchema(args.schema) + backend = create_backend(args.backend) + backend.generate(schema, + output_dir=args.output_dir, + prefix=args.prefix, + unmask=args.unmask, + builtins=args.builtins, + gen_tracing=not args.suppress_tracing) except QAPIError as err: print(err, file=sys.stderr) return 1 diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini deleted file mode 100644 index 8109470..0000000 --- a/scripts/qapi/mypy.ini +++ /dev/null @@ -1,4 +0,0 @@ -[mypy] -strict = True -disallow_untyped_calls = False -python_version = 3.8 diff --git a/scripts/qapi/parser.py b/scripts/qapi/parser.py index adc85b5..949d9e8 100644 --- a/scripts/qapi/parser.py +++ b/scripts/qapi/parser.py @@ -14,7 +14,7 @@ # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. -from collections import OrderedDict +import enum import os import re from typing import ( @@ -154,7 +154,7 @@ class QAPISchemaParser: "value of 'include' must be a string") incl_fname = os.path.join(os.path.dirname(self._fname), include) - self._add_expr(OrderedDict({'include': incl_fname}), info) + self._add_expr({'include': incl_fname}, info) exprs_include = self._include(include, info, incl_fname, self._included) if exprs_include: @@ -355,7 +355,7 @@ class QAPISchemaParser: raise QAPIParseError(self, "stray '%s'" % match.group(0)) def get_members(self) -> Dict[str, object]: - expr: Dict[str, object] = OrderedDict() + expr: Dict[str, object] = {} if self.tok == '}': self.accept() return expr @@ -575,7 +575,10 @@ class QAPISchemaParser: ) raise QAPIParseError(self, emsg) - doc.new_tagged_section(self.info, match.group(1)) + doc.new_tagged_section( + self.info, + QAPIDoc.Kind.from_string(match.group(1)) + ) text = line[match.end():] if text: doc.append_line(text) @@ -586,7 +589,7 @@ class QAPISchemaParser: self, "unexpected '=' markup in definition documentation") else: - # tag-less paragraph + # plain paragraph doc.ensure_untagged_section(self.info) doc.append_line(line) line = self.get_doc_paragraph(doc) @@ -635,23 +638,51 @@ class QAPIDoc: Free-form documentation blocks consist only of a body section. """ + class Kind(enum.Enum): + PLAIN = 0 + MEMBER = 1 + FEATURE = 2 + RETURNS = 3 + ERRORS = 4 + SINCE = 5 + TODO = 6 + + @staticmethod + def from_string(kind: str) -> 'QAPIDoc.Kind': + return QAPIDoc.Kind[kind.upper()] + + def __str__(self) -> str: + return self.name.title() + class Section: # pylint: disable=too-few-public-methods - def __init__(self, info: QAPISourceInfo, - tag: Optional[str] = None): + def __init__( + self, + info: QAPISourceInfo, + kind: 'QAPIDoc.Kind', + ): # section source info, i.e. where it begins self.info = info - # section tag, if any ('Returns', '@name', ...) - self.tag = tag + # section kind + self.kind = kind # section text without tag self.text = '' + def __repr__(self) -> str: + return f"<QAPIDoc.Section kind={self.kind!r} text={self.text!r}>" + def append_line(self, line: str) -> None: self.text += line + '\n' class ArgSection(Section): - def __init__(self, info: QAPISourceInfo, tag: str): - super().__init__(info, tag) + def __init__( + self, + info: QAPISourceInfo, + kind: 'QAPIDoc.Kind', + name: str + ): + super().__init__(info, kind) + self.name = name self.member: Optional['QAPISchemaMember'] = None def connect(self, member: 'QAPISchemaMember') -> None: @@ -663,7 +694,9 @@ class QAPIDoc: # definition doc's symbol, None for free-form doc self.symbol: Optional[str] = symbol # the sections in textual order - self.all_sections: List[QAPIDoc.Section] = [QAPIDoc.Section(info)] + self.all_sections: List[QAPIDoc.Section] = [ + QAPIDoc.Section(info, QAPIDoc.Kind.PLAIN) + ] # the body section self.body: Optional[QAPIDoc.Section] = self.all_sections[0] # dicts mapping parameter/feature names to their description @@ -680,55 +713,71 @@ class QAPIDoc: def end(self) -> None: for section in self.all_sections: section.text = section.text.strip('\n') - if section.tag is not None and section.text == '': + if section.kind != QAPIDoc.Kind.PLAIN and section.text == '': raise QAPISemError( - section.info, "text required after '%s:'" % section.tag) + section.info, "text required after '%s:'" % section.kind) def ensure_untagged_section(self, info: QAPISourceInfo) -> None: - if self.all_sections and not self.all_sections[-1].tag: + kind = QAPIDoc.Kind.PLAIN + + if self.all_sections and self.all_sections[-1].kind == kind: # extend current section - self.all_sections[-1].text += '\n' + section = self.all_sections[-1] + if not section.text: + # Section is empty so far; update info to start *here*. + section.info = info + section.text += '\n' return + # start new section - section = self.Section(info) + section = self.Section(info, kind) self.sections.append(section) self.all_sections.append(section) - def new_tagged_section(self, info: QAPISourceInfo, tag: str) -> None: - section = self.Section(info, tag) - if tag == 'Returns': + def new_tagged_section( + self, + info: QAPISourceInfo, + kind: 'QAPIDoc.Kind', + ) -> None: + section = self.Section(info, kind) + if kind == QAPIDoc.Kind.RETURNS: if self.returns: raise QAPISemError( - info, "duplicated '%s' section" % tag) + info, "duplicated '%s' section" % kind) self.returns = section - elif tag == 'Errors': + elif kind == QAPIDoc.Kind.ERRORS: if self.errors: raise QAPISemError( - info, "duplicated '%s' section" % tag) + info, "duplicated '%s' section" % kind) self.errors = section - elif tag == 'Since': + elif kind == QAPIDoc.Kind.SINCE: if self.since: raise QAPISemError( - info, "duplicated '%s' section" % tag) + info, "duplicated '%s' section" % kind) self.since = section self.sections.append(section) self.all_sections.append(section) - def _new_description(self, info: QAPISourceInfo, name: str, - desc: Dict[str, ArgSection]) -> None: + def _new_description( + self, + info: QAPISourceInfo, + name: str, + kind: 'QAPIDoc.Kind', + desc: Dict[str, ArgSection] + ) -> None: if not name: raise QAPISemError(info, "invalid parameter name") if name in desc: raise QAPISemError(info, "'%s' parameter name duplicated" % name) - section = self.ArgSection(info, '@' + name) + section = self.ArgSection(info, kind, name) self.all_sections.append(section) desc[name] = section def new_argument(self, info: QAPISourceInfo, name: str) -> None: - self._new_description(info, name, self.args) + self._new_description(info, name, QAPIDoc.Kind.MEMBER, self.args) def new_feature(self, info: QAPISourceInfo, name: str) -> None: - self._new_description(info, name, self.features) + self._new_description(info, name, QAPIDoc.Kind.FEATURE, self.features) def append_line(self, line: str) -> None: self.all_sections[-1].append_line(line) @@ -740,8 +789,23 @@ class QAPIDoc: raise QAPISemError(member.info, "%s '%s' lacks documentation" % (member.role, member.name)) - self.args[member.name] = QAPIDoc.ArgSection( - self.info, '@' + member.name) + # Insert stub documentation section for missing member docs. + # TODO: drop when undocumented members are outlawed + + section = QAPIDoc.ArgSection( + self.info, QAPIDoc.Kind.MEMBER, member.name) + self.args[member.name] = section + + # Determine where to insert stub doc - it should go at the + # end of the members section(s), if any. Note that index 0 + # is assumed to be an untagged intro section, even if it is + # empty. + index = 1 + if len(self.all_sections) > 1: + while self.all_sections[index].kind == QAPIDoc.Kind.MEMBER: + index += 1 + self.all_sections.insert(index, section) + self.args[member.name].connect(member) def connect_feature(self, feature: 'QAPISchemaFeature') -> None: diff --git a/scripts/qapi/pylintrc b/scripts/qapi/pylintrc index c028a1f..e16283a 100644 --- a/scripts/qapi/pylintrc +++ b/scripts/qapi/pylintrc @@ -17,7 +17,9 @@ disable=consider-using-f-string, too-many-arguments, too-many-branches, too-many-instance-attributes, + too-many-positional-arguments, too-many-statements, + unknown-option-value, useless-option-value, [REPORTS] diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py index d65c35f..cbe3b5a 100644 --- a/scripts/qapi/schema.py +++ b/scripts/qapi/schema.py @@ -19,7 +19,6 @@ from __future__ import annotations from abc import ABC, abstractmethod -from collections import OrderedDict import os import re from typing import ( @@ -29,6 +28,7 @@ from typing import ( List, Optional, Union, + ValuesView, cast, ) @@ -556,7 +556,7 @@ class QAPISchemaObjectType(QAPISchemaType): super().check(schema) assert self._checked and not self._check_complete - seen = OrderedDict() + seen = {} if self._base_name: self.base = schema.resolve_type(self._base_name, self.info, "'base'") @@ -933,8 +933,11 @@ class QAPISchemaEnumMember(QAPISchemaMember): class QAPISchemaFeature(QAPISchemaMember): role = 'feature' + # Features which are standardized across all schemas + SPECIAL_NAMES = ['deprecated', 'unstable'] + def is_special(self) -> bool: - return self.name in ('deprecated', 'unstable') + return self.name in QAPISchemaFeature.SPECIAL_NAMES class QAPISchemaObjectTypeMember(QAPISchemaMember): @@ -1137,7 +1140,17 @@ class QAPISchema: self.docs = parser.docs self._entity_list: List[QAPISchemaEntity] = [] self._entity_dict: Dict[str, QAPISchemaDefinition] = {} - self._module_dict: Dict[str, QAPISchemaModule] = OrderedDict() + self._module_dict: Dict[str, QAPISchemaModule] = {} + # NB, values in the dict will identify the first encountered + # usage of a named feature only + self._feature_dict: Dict[str, QAPISchemaFeature] = {} + + # All schemas get the names defined in the QapiSpecialFeature enum. + # Rely on dict iteration order matching insertion order so that + # the special names are emitted first when generating code. + for f in QAPISchemaFeature.SPECIAL_NAMES: + self._feature_dict[f] = QAPISchemaFeature(f, None) + self._schema_dir = os.path.dirname(fname) self._make_module(QAPISchemaModule.BUILTIN_MODULE_NAME) self._make_module(fname) @@ -1147,6 +1160,9 @@ class QAPISchema: self._def_exprs(exprs) self.check() + def features(self) -> ValuesView[QAPISchemaFeature]: + return self._feature_dict.values() + def _def_entity(self, ent: QAPISchemaEntity) -> None: self._entity_list.append(ent) @@ -1249,7 +1265,7 @@ class QAPISchema: [{'name': n} for n in qtypes], None) self._def_definition(QAPISchemaEnumType( - 'QType', None, None, None, None, qtype_values, 'QTYPE')) + 'QType', None, None, None, None, qtype_values, None)) def _make_features( self, @@ -1258,6 +1274,12 @@ class QAPISchema: ) -> List[QAPISchemaFeature]: if features is None: return [] + + for f in features: + feat = QAPISchemaFeature(f['name'], info) + if feat.name not in self._feature_dict: + self._feature_dict[feat.name] = feat + return [QAPISchemaFeature(f['name'], info, QAPISchemaIfCond(f.get('if'))) for f in features] @@ -1431,7 +1453,7 @@ class QAPISchema: ifcond = QAPISchemaIfCond(expr.get('if')) info = expr.info features = self._make_features(expr.get('features'), info) - if isinstance(data, OrderedDict): + if isinstance(data, dict): data = self._make_implicit_object_type( name, info, ifcond, 'arg', self._make_members(data, info)) @@ -1450,7 +1472,7 @@ class QAPISchema: ifcond = QAPISchemaIfCond(expr.get('if')) info = expr.info features = self._make_features(expr.get('features'), info) - if isinstance(data, OrderedDict): + if isinstance(data, dict): data = self._make_implicit_object_type( name, info, ifcond, 'arg', self._make_members(data, info)) @@ -1485,6 +1507,12 @@ class QAPISchema: for doc in self.docs: doc.check() + features = list(self._feature_dict.values()) + if len(features) > 64: + raise QAPISemError( + features[64].info, + "Maximum of 64 schema features is permitted") + def visit(self, visitor: QAPISchemaVisitor) -> None: visitor.visit_begin(self) for mod in self._module_dict.values(): diff --git a/scripts/qapi/source.py b/scripts/qapi/source.py index 7b379fd..ffdc3f4 100644 --- a/scripts/qapi/source.py +++ b/scripts/qapi/source.py @@ -47,9 +47,9 @@ class QAPISourceInfo: self.defn_meta = meta self.defn_name = name - def next_line(self: T) -> T: + def next_line(self: T, n: int = 1) -> T: info = copy.copy(self) - info.line += 1 + info.line += n return info def loc(self) -> str: diff --git a/scripts/qapi/types.py b/scripts/qapi/types.py index 0dd0b00..2bf7533 100644 --- a/scripts/qapi/types.py +++ b/scripts/qapi/types.py @@ -16,11 +16,7 @@ This work is licensed under the terms of the GNU GPL, version 2. from typing import List, Optional from .common import c_enum_const, c_name, mcgen -from .gen import ( - QAPISchemaModularCVisitor, - gen_special_features, - ifcontext, -) +from .gen import QAPISchemaModularCVisitor, gen_features, ifcontext from .schema import ( QAPISchema, QAPISchemaAlternatives, @@ -61,17 +57,17 @@ const QEnumLookup %(c_name)s_lookup = { index=index, name=memb.name) ret += memb.ifcond.gen_endif() - special_features = gen_special_features(memb.features) - if special_features != '0': + features = gen_features(memb.features) + if features != '0': feats += mcgen(''' - [%(index)s] = %(special_features)s, + [%(index)s] = %(features)s, ''', - index=index, special_features=special_features) + index=index, features=features) if feats: ret += mcgen(''' }, - .special_features = (const unsigned char[%(max_index)s]) { + .features = (const uint64_t[%(max_index)s]) { ''', max_index=max_index) ret += feats @@ -308,11 +304,14 @@ class QAPISchemaGenTypeVisitor(QAPISchemaModularCVisitor): #include "qapi/dealloc-visitor.h" #include "%(types)s.h" #include "%(visit)s.h" +#include "%(prefix)sqapi-features.h" ''', - types=types, visit=visit)) + types=types, visit=visit, + prefix=self._prefix)) self._genh.preamble_add(mcgen(''' #include "qapi/qapi-builtin-types.h" -''')) +''', + prefix=self._prefix)) def visit_begin(self, schema: QAPISchema) -> None: # gen_object() is recursive, ensure it doesn't visit the empty type diff --git a/scripts/qapi/visit.py b/scripts/qapi/visit.py index 12f92e4..36e2409 100644 --- a/scripts/qapi/visit.py +++ b/scripts/qapi/visit.py @@ -21,11 +21,7 @@ from .common import ( indent, mcgen, ) -from .gen import ( - QAPISchemaModularCVisitor, - gen_special_features, - ifcontext, -) +from .gen import QAPISchemaModularCVisitor, gen_features, ifcontext from .schema import ( QAPISchema, QAPISchemaAlternatives, @@ -103,15 +99,15 @@ bool visit_type_%(c_name)s_members(Visitor *v, %(c_name)s *obj, Error **errp) ''', name=memb.name, has=has) indent.increase() - special_features = gen_special_features(memb.features) - if special_features != '0': + features = gen_features(memb.features) + if features != '0': ret += mcgen(''' - if (visit_policy_reject(v, "%(name)s", %(special_features)s, errp)) { + if (visit_policy_reject(v, "%(name)s", %(features)s, errp)) { return false; } - if (!visit_policy_skip(v, "%(name)s", %(special_features)s)) { + if (!visit_policy_skip(v, "%(name)s", %(features)s)) { ''', - name=memb.name, special_features=special_features) + name=memb.name, features=features) indent.increase() ret += mcgen(''' if (!visit_type_%(c_type)s(v, "%(name)s", &obj->%(c_name)s, errp)) { @@ -120,7 +116,7 @@ bool visit_type_%(c_name)s_members(Visitor *v, %(c_name)s *obj, Error **errp) ''', c_type=memb.type.c_name(), name=memb.name, c_name=c_name(memb.name)) - if special_features != '0': + if features != '0': indent.decrease() ret += mcgen(''' } @@ -360,8 +356,9 @@ class QAPISchemaGenVisitVisitor(QAPISchemaModularCVisitor): #include "qemu/osdep.h" #include "qapi/error.h" #include "%(visit)s.h" +#include "%(prefix)sqapi-features.h" ''', - visit=visit)) + visit=visit, prefix=self._prefix)) self._genh.preamble_add(mcgen(''' #include "qapi/qapi-builtin-visit.h" #include "%(types)s.h" diff --git a/scripts/qcow2-to-stdout.py b/scripts/qcow2-to-stdout.py new file mode 100755 index 0000000..06b7c13 --- /dev/null +++ b/scripts/qcow2-to-stdout.py @@ -0,0 +1,449 @@ +#!/usr/bin/env python3 + +# This tool reads a disk image in any format and converts it to qcow2, +# writing the result directly to stdout. +# +# Copyright (C) 2024 Igalia, S.L. +# +# Authors: Alberto Garcia <berto@igalia.com> +# Madeeha Javed <javed@igalia.com> +# +# SPDX-License-Identifier: GPL-2.0-or-later +# +# qcow2 files produced by this script are always arranged like this: +# +# - qcow2 header +# - refcount table +# - refcount blocks +# - L1 table +# - L2 tables +# - Data clusters +# +# A note about variable names: in qcow2 there is one refcount table +# and one (active) L1 table, although each can occupy several +# clusters. For the sake of simplicity the code sometimes talks about +# refcount tables and L1 tables when referring to those clusters. + +import argparse +import errno +import math +import os +import signal +import struct +import subprocess +import sys +import tempfile +import time +from contextlib import contextmanager + +QCOW2_DEFAULT_CLUSTER_SIZE = 65536 +QCOW2_DEFAULT_REFCOUNT_BITS = 16 +QCOW2_FEATURE_NAME_TABLE = 0x6803F857 +QCOW2_DATA_FILE_NAME_STRING = 0x44415441 +QCOW2_V3_HEADER_LENGTH = 112 # Header length in QEMU 9.0. Must be a multiple of 8 +QCOW2_INCOMPAT_DATA_FILE_BIT = 2 +QCOW2_AUTOCLEAR_DATA_FILE_RAW_BIT = 1 +QCOW_OFLAG_COPIED = 1 << 63 +QEMU_STORAGE_DAEMON = "qemu-storage-daemon" + + +def bitmap_set(bitmap, idx): + bitmap[idx // 8] |= 1 << (idx % 8) + + +def bitmap_is_set(bitmap, idx): + return (bitmap[idx // 8] & (1 << (idx % 8))) != 0 + + +def bitmap_iterator(bitmap, length): + for idx in range(length): + if bitmap_is_set(bitmap, idx): + yield idx + + +def align_up(num, d): + return d * math.ceil(num / d) + + +# Holes in the input file contain only zeroes so we can skip them and +# save time. This function returns the indexes of the clusters that +# are known to contain data. Those are the ones that we need to read. +def clusters_with_data(fd, cluster_size): + data_to = 0 + while True: + try: + data_from = os.lseek(fd, data_to, os.SEEK_DATA) + data_to = align_up(os.lseek(fd, data_from, os.SEEK_HOLE), cluster_size) + for idx in range(data_from // cluster_size, data_to // cluster_size): + yield idx + except OSError as err: + if err.errno == errno.ENXIO: # End of file reached + break + raise err + + +# write_qcow2_content() expects a raw input file. If we have a different +# format we can use qemu-storage-daemon to make it appear as raw. +@contextmanager +def get_input_as_raw_file(input_file, input_format): + if input_format == "raw": + yield input_file + return + try: + temp_dir = tempfile.mkdtemp() + pid_file = os.path.join(temp_dir, "pid") + raw_file = os.path.join(temp_dir, "raw") + open(raw_file, "wb").close() + ret = subprocess.run( + [ + QEMU_STORAGE_DAEMON, + "--daemonize", + "--pidfile", pid_file, + "--blockdev", f"driver=file,node-name=file0,driver=file,filename={input_file},read-only=on", + "--blockdev", f"driver={input_format},node-name=disk0,file=file0,read-only=on", + "--export", f"type=fuse,id=export0,node-name=disk0,mountpoint={raw_file},writable=off", + ], + capture_output=True, + ) + if ret.returncode != 0: + sys.exit("[Error] Could not start the qemu-storage-daemon:\n" + + ret.stderr.decode().rstrip('\n')) + yield raw_file + finally: + # Kill the storage daemon on exit + # and remove all temporary files + if os.path.exists(pid_file): + with open(pid_file, "r") as f: + pid = int(f.readline()) + os.kill(pid, signal.SIGTERM) + while os.path.exists(pid_file): + time.sleep(0.1) + os.unlink(raw_file) + os.rmdir(temp_dir) + + +def write_features(cluster, offset, data_file_name): + if data_file_name is not None: + encoded_name = data_file_name.encode("utf-8") + padded_name_len = align_up(len(encoded_name), 8) + struct.pack_into(f">II{padded_name_len}s", cluster, offset, + QCOW2_DATA_FILE_NAME_STRING, + len(encoded_name), + encoded_name) + offset += 8 + padded_name_len + + qcow2_features = [ + # Incompatible + (0, 0, "dirty bit"), + (0, 1, "corrupt bit"), + (0, 2, "external data file"), + (0, 3, "compression type"), + (0, 4, "extended L2 entries"), + # Compatible + (1, 0, "lazy refcounts"), + # Autoclear + (2, 0, "bitmaps"), + (2, 1, "raw external data"), + ] + struct.pack_into(">I", cluster, offset, QCOW2_FEATURE_NAME_TABLE) + struct.pack_into(">I", cluster, offset + 4, len(qcow2_features) * 48) + offset += 8 + for feature_type, feature_bit, feature_name in qcow2_features: + struct.pack_into(">BB46s", cluster, offset, + feature_type, feature_bit, feature_name.encode("ascii")) + offset += 48 + + +def write_qcow2_content(input_file, cluster_size, refcount_bits, data_file_name, data_file_raw): + # Some basic values + l1_entries_per_table = cluster_size // 8 + l2_entries_per_table = cluster_size // 8 + refcounts_per_table = cluster_size // 8 + refcounts_per_block = cluster_size * 8 // refcount_bits + + # Virtual disk size, number of data clusters and L1 entries + disk_size = align_up(os.path.getsize(input_file), 512) + total_data_clusters = math.ceil(disk_size / cluster_size) + l1_entries = math.ceil(total_data_clusters / l2_entries_per_table) + allocated_l1_tables = math.ceil(l1_entries / l1_entries_per_table) + + # Max L1 table size is 32 MB (QCOW_MAX_L1_SIZE in block/qcow2.h) + if (l1_entries * 8) > (32 * 1024 * 1024): + sys.exit("[Error] The image size is too large. Try using a larger cluster size.") + + # Two bitmaps indicating which L1 and L2 entries are set + l1_bitmap = bytearray(allocated_l1_tables * l1_entries_per_table // 8) + l2_bitmap = bytearray(l1_entries * l2_entries_per_table // 8) + allocated_l2_tables = 0 + allocated_data_clusters = 0 + + if data_file_raw: + # If data_file_raw is set then all clusters are allocated and + # we don't need to read the input file at all. + allocated_l2_tables = l1_entries + for idx in range(l1_entries): + bitmap_set(l1_bitmap, idx) + for idx in range(total_data_clusters): + bitmap_set(l2_bitmap, idx) + else: + # Open the input file for reading + fd = os.open(input_file, os.O_RDONLY) + zero_cluster = bytes(cluster_size) + # Read all the clusters that contain data + for idx in clusters_with_data(fd, cluster_size): + cluster = os.pread(fd, cluster_size, cluster_size * idx) + # If the last cluster is smaller than cluster_size pad it with zeroes + if len(cluster) < cluster_size: + cluster += bytes(cluster_size - len(cluster)) + # If a cluster has non-zero data then it must be allocated + # in the output file and its L2 entry must be set + if cluster != zero_cluster: + bitmap_set(l2_bitmap, idx) + allocated_data_clusters += 1 + # Allocated data clusters also need their corresponding L1 entry and L2 table + l1_idx = math.floor(idx / l2_entries_per_table) + if not bitmap_is_set(l1_bitmap, l1_idx): + bitmap_set(l1_bitmap, l1_idx) + allocated_l2_tables += 1 + + # Total amount of allocated clusters excluding the refcount blocks and table + total_allocated_clusters = 1 + allocated_l1_tables + allocated_l2_tables + if data_file_name is None: + total_allocated_clusters += allocated_data_clusters + + # Clusters allocated for the refcount blocks and table + allocated_refcount_blocks = math.ceil(total_allocated_clusters / refcounts_per_block) + allocated_refcount_tables = math.ceil(allocated_refcount_blocks / refcounts_per_table) + + # Now we have a problem because allocated_refcount_blocks and allocated_refcount_tables... + # (a) increase total_allocated_clusters, and + # (b) need to be recalculated when total_allocated_clusters is increased + # So we need to repeat the calculation as long as the numbers change + while True: + new_total_allocated_clusters = total_allocated_clusters + allocated_refcount_tables + allocated_refcount_blocks + new_allocated_refcount_blocks = math.ceil(new_total_allocated_clusters / refcounts_per_block) + if new_allocated_refcount_blocks > allocated_refcount_blocks: + allocated_refcount_blocks = new_allocated_refcount_blocks + allocated_refcount_tables = math.ceil(allocated_refcount_blocks / refcounts_per_table) + else: + break + + # Now that we have the final numbers we can update total_allocated_clusters + total_allocated_clusters += allocated_refcount_tables + allocated_refcount_blocks + + # At this point we have the exact number of clusters that the output + # image is going to use so we can calculate all the offsets. + current_cluster_idx = 1 + + refcount_table_offset = current_cluster_idx * cluster_size + current_cluster_idx += allocated_refcount_tables + + refcount_block_offset = current_cluster_idx * cluster_size + current_cluster_idx += allocated_refcount_blocks + + l1_table_offset = current_cluster_idx * cluster_size + current_cluster_idx += allocated_l1_tables + + l2_table_offset = current_cluster_idx * cluster_size + current_cluster_idx += allocated_l2_tables + + data_clusters_offset = current_cluster_idx * cluster_size + + # Calculate some values used in the qcow2 header + if allocated_l1_tables == 0: + l1_table_offset = 0 + + hdr_cluster_bits = int(math.log2(cluster_size)) + hdr_refcount_bits = int(math.log2(refcount_bits)) + hdr_length = QCOW2_V3_HEADER_LENGTH + hdr_incompat_features = 0 + if data_file_name is not None: + hdr_incompat_features |= 1 << QCOW2_INCOMPAT_DATA_FILE_BIT + hdr_autoclear_features = 0 + if data_file_raw: + hdr_autoclear_features |= 1 << QCOW2_AUTOCLEAR_DATA_FILE_RAW_BIT + + ### Write qcow2 header + cluster = bytearray(cluster_size) + struct.pack_into(">4sIQIIQIIQQIIQQQQII", cluster, 0, + b"QFI\xfb", # QCOW magic string + 3, # version + 0, # backing file offset + 0, # backing file sizes + hdr_cluster_bits, + disk_size, + 0, # encryption method + l1_entries, + l1_table_offset, + refcount_table_offset, + allocated_refcount_tables, + 0, # number of snapshots + 0, # snapshot table offset + hdr_incompat_features, + 0, # compatible features + hdr_autoclear_features, + hdr_refcount_bits, + hdr_length, + ) + + write_features(cluster, hdr_length, data_file_name) + + sys.stdout.buffer.write(cluster) + + ### Write refcount table + cur_offset = refcount_block_offset + remaining_refcount_table_entries = allocated_refcount_blocks # Each entry is a pointer to a refcount block + while remaining_refcount_table_entries > 0: + cluster = bytearray(cluster_size) + to_write = min(remaining_refcount_table_entries, refcounts_per_table) + remaining_refcount_table_entries -= to_write + for idx in range(to_write): + struct.pack_into(">Q", cluster, idx * 8, cur_offset) + cur_offset += cluster_size + sys.stdout.buffer.write(cluster) + + ### Write refcount blocks + remaining_refcount_block_entries = total_allocated_clusters # One entry for each allocated cluster + for tbl in range(allocated_refcount_blocks): + cluster = bytearray(cluster_size) + to_write = min(remaining_refcount_block_entries, refcounts_per_block) + remaining_refcount_block_entries -= to_write + # All refcount entries contain the number 1. The only difference + # is their bit width, defined when the image is created. + for idx in range(to_write): + if refcount_bits == 64: + struct.pack_into(">Q", cluster, idx * 8, 1) + elif refcount_bits == 32: + struct.pack_into(">L", cluster, idx * 4, 1) + elif refcount_bits == 16: + struct.pack_into(">H", cluster, idx * 2, 1) + elif refcount_bits == 8: + cluster[idx] = 1 + elif refcount_bits == 4: + cluster[idx // 2] |= 1 << ((idx % 2) * 4) + elif refcount_bits == 2: + cluster[idx // 4] |= 1 << ((idx % 4) * 2) + elif refcount_bits == 1: + cluster[idx // 8] |= 1 << (idx % 8) + sys.stdout.buffer.write(cluster) + + ### Write L1 table + cur_offset = l2_table_offset + for tbl in range(allocated_l1_tables): + cluster = bytearray(cluster_size) + for idx in range(l1_entries_per_table): + l1_idx = tbl * l1_entries_per_table + idx + if bitmap_is_set(l1_bitmap, l1_idx): + struct.pack_into(">Q", cluster, idx * 8, cur_offset | QCOW_OFLAG_COPIED) + cur_offset += cluster_size + sys.stdout.buffer.write(cluster) + + ### Write L2 tables + cur_offset = data_clusters_offset + for tbl in range(l1_entries): + # Skip the empty L2 tables. We can identify them because + # there is no L1 entry pointing at them. + if bitmap_is_set(l1_bitmap, tbl): + cluster = bytearray(cluster_size) + for idx in range(l2_entries_per_table): + l2_idx = tbl * l2_entries_per_table + idx + if bitmap_is_set(l2_bitmap, l2_idx): + if data_file_name is None: + struct.pack_into(">Q", cluster, idx * 8, cur_offset | QCOW_OFLAG_COPIED) + cur_offset += cluster_size + else: + struct.pack_into(">Q", cluster, idx * 8, (l2_idx * cluster_size) | QCOW_OFLAG_COPIED) + sys.stdout.buffer.write(cluster) + + ### Write data clusters + if data_file_name is None: + for idx in bitmap_iterator(l2_bitmap, total_data_clusters): + cluster = os.pread(fd, cluster_size, cluster_size * idx) + # If the last cluster is smaller than cluster_size pad it with zeroes + if len(cluster) < cluster_size: + cluster += bytes(cluster_size - len(cluster)) + sys.stdout.buffer.write(cluster) + + if not data_file_raw: + os.close(fd) + + +def main(): + # Command-line arguments + parser = argparse.ArgumentParser( + description="This program converts a QEMU disk image to qcow2 " + "and writes it to the standard output" + ) + parser.add_argument("input_file", help="name of the input file") + parser.add_argument( + "-f", + dest="input_format", + metavar="input_format", + help="format of the input file (default: raw)", + default="raw", + ) + parser.add_argument( + "-c", + dest="cluster_size", + metavar="cluster_size", + help=f"qcow2 cluster size (default: {QCOW2_DEFAULT_CLUSTER_SIZE})", + default=QCOW2_DEFAULT_CLUSTER_SIZE, + type=int, + choices=[1 << x for x in range(9, 22)], + ) + parser.add_argument( + "-r", + dest="refcount_bits", + metavar="refcount_bits", + help=f"width of the reference count entries (default: {QCOW2_DEFAULT_REFCOUNT_BITS})", + default=QCOW2_DEFAULT_REFCOUNT_BITS, + type=int, + choices=[1 << x for x in range(7)], + ) + parser.add_argument( + "-d", + dest="data_file", + help="create an image with input_file as an external data file", + action="store_true", + ) + parser.add_argument( + "-R", + dest="data_file_raw", + help="enable data_file_raw on the generated image (implies -d)", + action="store_true", + ) + args = parser.parse_args() + + if args.data_file_raw: + args.data_file = True + + if not os.path.isfile(args.input_file): + sys.exit(f"[Error] {args.input_file} does not exist or is not a regular file.") + + if args.data_file and args.input_format != "raw": + sys.exit("[Error] External data files can only be used with raw input images") + + # A 512 byte header is too small for the data file name extension + if args.data_file and args.cluster_size == 512: + sys.exit("[Error] External data files require a larger cluster size") + + if sys.stdout.isatty(): + sys.exit("[Error] Refusing to write to a tty. Try redirecting stdout.") + + if args.data_file: + data_file_name = args.input_file + else: + data_file_name = None + + with get_input_as_raw_file(args.input_file, args.input_format) as raw_file: + write_qcow2_content( + raw_file, + args.cluster_size, + args.refcount_bits, + data_file_name, + args.data_file_raw, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/qemu-binfmt-conf.sh b/scripts/qemu-binfmt-conf.sh index 6ef9f11..5fd462b 100755 --- a/scripts/qemu-binfmt-conf.sh +++ b/scripts/qemu-binfmt-conf.sh @@ -144,35 +144,35 @@ loongarch64_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x loongarch64_mask='\xff\xff\xff\xff\xff\xff\xff\xfc\x00\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff' loongarch64_family=loongarch -qemu_get_family() { - cpu=${HOST_ARCH:-$(uname -m)} +# Converts the name of a host CPU architecture to the corresponding QEMU +# target. +# +# FIXME: This can probably be simplified a lot by dropping most entries. +# Remember that the script is only used on Linux, so we only need to +# handle the strings Linux uses to report the host CPU architecture. +qemu_normalize() { + cpu="$1" case "$cpu" in - amd64|i386|i486|i586|i686|i86pc|BePC|x86_64) + i[3-6]86) echo "i386" ;; - mips*) - echo "mips" + amd64) + echo "x86_64" ;; - "Power Macintosh"|ppc64|powerpc|ppc) + powerpc) echo "ppc" ;; - ppc64el|ppc64le) - echo "ppcle" + ppc64el) + echo "ppc64le" ;; - arm|armel|armhf|arm64|armv[4-9]*l|aarch64) + armel|armhf|armv[4-9]*l) echo "arm" ;; - armeb|armv[4-9]*b|aarch64_be) + armv[4-9]*b) echo "armeb" ;; - sparc*) - echo "sparc" - ;; - riscv*) - echo "riscv" - ;; - loongarch*) - echo "loongarch" + arm64) + echo "aarch64" ;; *) echo "$cpu" @@ -205,6 +205,9 @@ Usage: qemu-binfmt-conf.sh [--qemu-path PATH][--debian][--systemd CPU] --persistent: if yes, the interpreter is loaded when binfmt is configured and remains in memory. All future uses are cloned from the open file. + --ignore-family: if yes, it is assumed that the host CPU (e.g. riscv64) + can't natively run programs targeting a CPU that is + part of the same family (e.g. riscv32). --preserve-argv0 preserve argv[0] To import templates with update-binfmts, use : @@ -309,7 +312,13 @@ EOF qemu_set_binfmts() { # probe cpu type - host_family=$(qemu_get_family) + host_cpu=$(qemu_normalize ${HOST_ARCH:-$(uname -m)}) + host_family=$(eval echo \$${host_cpu}_family) + + if [ "$host_family" = "" ] ; then + echo "INTERNAL ERROR: unknown host cpu $host_cpu" 1>&2 + exit 1 + fi # register the interpreter for each cpu except for the native one @@ -318,20 +327,28 @@ qemu_set_binfmts() { mask=$(eval echo \$${cpu}_mask) family=$(eval echo \$${cpu}_family) + target="$cpu" + if [ "$cpu" = "i486" ] ; then + target="i386" + fi + + qemu="$QEMU_PATH/qemu-$target$QEMU_SUFFIX" + if [ "$magic" = "" ] || [ "$mask" = "" ] || [ "$family" = "" ] ; then echo "INTERNAL ERROR: unknown cpu $cpu" 1>&2 continue fi - qemu="$QEMU_PATH/qemu-$cpu" - if [ "$cpu" = "i486" ] ; then - qemu="$QEMU_PATH/qemu-i386" + if [ "$host_family" = "$family" ] ; then + # When --ignore-family is used, we have to generate rules even + # for targets that are in the same family as the host CPU. The + # only exception is of course when the CPU types exactly match + if [ "$target" = "$host_cpu" ] || [ "$IGNORE_FAMILY" = "no" ] ; then + continue + fi fi - qemu="$qemu$QEMU_SUFFIX" - if [ "$host_family" != "$family" ] ; then - $BINFMT_SET - fi + $BINFMT_SET done } @@ -346,10 +363,11 @@ CREDENTIAL=no PERSISTENT=no PRESERVE_ARG0=no QEMU_SUFFIX="" +IGNORE_FAMILY=no _longopts="debian,systemd:,qemu-path:,qemu-suffix:,exportdir:,help,credential:,\ -persistent:,preserve-argv0:" -options=$(getopt -o ds:Q:S:e:hc:p:g:F: -l ${_longopts} -- "$@") +persistent:,preserve-argv0:,ignore-family:" +options=$(getopt -o ds:Q:S:e:hc:p:g:F:i: -l ${_longopts} -- "$@") eval set -- "$options" while true ; do @@ -409,6 +427,10 @@ while true ; do shift PRESERVE_ARG0="$1" ;; + -i|--ignore-family) + shift + IGNORE_FAMILY="$1" + ;; *) break ;; diff --git a/scripts/qemu-gdb.py b/scripts/qemu-gdb.py index 4d2a9f6..cfae94a 100644 --- a/scripts/qemu-gdb.py +++ b/scripts/qemu-gdb.py @@ -45,3 +45,5 @@ coroutine.CoroutineBt() # Default to silently passing through SIGUSR1, because QEMU sends it # to itself a lot. gdb.execute('handle SIGUSR1 pass noprint nostop') +# Always print full stack for python errors, easier to debug and report issues +gdb.execute('set python print-stack full') diff --git a/scripts/qemu-guest-agent/fsfreeze-hook b/scripts/qemu-guest-agent/fsfreeze-hook index 13aafd4..c1feb6f 100755 --- a/scripts/qemu-guest-agent/fsfreeze-hook +++ b/scripts/qemu-guest-agent/fsfreeze-hook @@ -19,15 +19,43 @@ is_ignored_file() { return 1 } +USE_SYSLOG=0 +# if log file is not writable, fallback to syslog +[ ! -w "$LOGFILE" ] && USE_SYSLOG=1 +# try to update log file and fallback to syslog if it fails +touch "$LOGFILE" &>/dev/null || USE_SYSLOG=1 + +# Ensure the log file is writable, fallback to syslog if not +log_message() { + local message="$1" + if [ "$USE_SYSLOG" -eq 0 ]; then + printf "%s: %s\n" "$(date)" "$message" >>"$LOGFILE" + else + logger -t qemu-ga-freeze-hook "$message" + fi +} + # Iterate executables in directory "fsfreeze-hook.d" with the specified args [ ! -d "$FSFREEZE_D" ] && exit 0 + for file in "$FSFREEZE_D"/* ; do is_ignored_file "$file" && continue [ -x "$file" ] || continue - printf "$(date): execute $file $@\n" >>$LOGFILE - "$file" "$@" >>$LOGFILE 2>&1 - STATUS=$? - printf "$(date): $file finished with status=$STATUS\n" >>$LOGFILE + + log_message "Executing $file $@" + if [ "$USE_SYSLOG" -eq 0 ]; then + "$file" "$@" >>"$LOGFILE" 2>&1 + STATUS=$? + else + "$file" "$@" 2>&1 | logger -t qemu-ga-freeze-hook + STATUS=${PIPESTATUS[0]} + fi + + if [ $STATUS -ne 0 ]; then + log_message "Error: $file finished with status=$STATUS" + else + log_message "$file finished successfully" + fi done exit 0 diff --git a/scripts/qemu-plugin-symbols.py b/scripts/qemu-plugin-symbols.py new file mode 100755 index 0000000..e285ebb --- /dev/null +++ b/scripts/qemu-plugin-symbols.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Extract QEMU Plugin API symbols from a header file +# +# Copyright 2024 Linaro Ltd +# +# Author: Pierrick Bouvier <pierrick.bouvier@linaro.org> +# +# This work is licensed under the terms of the GNU GPL, version 2 or later. +# See the COPYING file in the top-level directory. +# +# SPDX-License-Identifier: GPL-2.0-or-later + +import argparse +import re + +def extract_symbols(plugin_header): + with open(plugin_header) as file: + content = file.read() + # Remove QEMU_PLUGIN_API macro definition. + content = content.replace('#define QEMU_PLUGIN_API', '') + expected = content.count('QEMU_PLUGIN_API') + # Find last word between QEMU_PLUGIN_API and (, matching on several lines. + # We use *? non-greedy quantifier. + syms = re.findall(r'QEMU_PLUGIN_API.*?(\w+)\s*\(', content, re.DOTALL) + syms.sort() + # Ensure we found as many symbols as API markers. + assert len(syms) == expected + return syms + +def main() -> None: + parser = argparse.ArgumentParser(description='Extract QEMU plugin symbols') + parser.add_argument('plugin_header', help='Path to QEMU plugin header.') + args = parser.parse_args() + + syms = extract_symbols(args.plugin_header) + + print('{') + for s in syms: + print(" {};".format(s)) + print('};') + +if __name__ == '__main__': + main() diff --git a/scripts/qemu-trace-stap b/scripts/qemu-trace-stap index eb6e951..e983460 100755 --- a/scripts/qemu-trace-stap +++ b/scripts/qemu-trace-stap @@ -56,6 +56,7 @@ def tapset_dir(binary): def cmd_run(args): + stap = which("stap") prefix = probe_prefix(args.binary) tapsets = tapset_dir(args.binary) @@ -76,7 +77,7 @@ def cmd_run(args): # We request an 8MB buffer, since the stap default 1MB buffer # can be easily overflowed by frequently firing QEMU traces - stapargs = ["stap", "-s", "8", "-I", tapsets ] + stapargs = [stap, "-s", "8", "-I", tapsets ] if args.pid is not None: stapargs.extend(["-x", args.pid]) stapargs.extend(["-e", script]) @@ -84,6 +85,7 @@ def cmd_run(args): def cmd_list(args): + stap = which("stap") tapsets = tapset_dir(args.binary) if args.verbose: @@ -96,7 +98,7 @@ def cmd_list(args): if verbose: print("Listing probes with name '%s'" % script) - proc = subprocess.Popen(["stap", "-I", tapsets, "-l", script], + proc = subprocess.Popen([stap, "-I", tapsets, "-l", script], stdout=subprocess.PIPE, universal_newlines=True) out, err = proc.communicate() diff --git a/scripts/qemugdb/coroutine.py b/scripts/qemugdb/coroutine.py index 7db46d4..e98fc48 100644 --- a/scripts/qemugdb/coroutine.py +++ b/scripts/qemugdb/coroutine.py @@ -13,28 +13,9 @@ import gdb VOID_PTR = gdb.lookup_type('void').pointer() -def get_fs_base(): - '''Fetch %fs base value using arch_prctl(ARCH_GET_FS). This is - pthread_self().''' - # %rsp - 120 is scratch space according to the SystemV ABI - old = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)') - gdb.execute('call (int)arch_prctl(0x1003, $rsp - 120)', False, True) - fs_base = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)') - gdb.execute('set *(uint64_t*)($rsp - 120) = %s' % old, False, True) - return fs_base - def pthread_self(): - '''Fetch pthread_self() from the glibc start_thread function.''' - f = gdb.newest_frame() - while f.name() != 'start_thread': - f = f.older() - if f is None: - return get_fs_base() - - try: - return f.read_var("arg") - except ValueError: - return get_fs_base() + '''Fetch the base address of TLS.''' + return gdb.parse_and_eval("$fs_base") def get_glibc_pointer_guard(): '''Fetch glibc pointer guard value''' @@ -65,9 +46,60 @@ def get_jmpbuf_regs(jmpbuf): 'r15': jmpbuf[JB_R15], 'rip': glibc_ptr_demangle(jmpbuf[JB_PC], pointer_guard) } -def bt_jmpbuf(jmpbuf): - '''Backtrace a jmpbuf''' - regs = get_jmpbuf_regs(jmpbuf) +def symbol_lookup(addr): + # Example: "__clone3 + 44 in section .text of /lib64/libc.so.6" + result = gdb.execute(f"info symbol {hex(addr)}", to_string=True).strip() + try: + if "+" in result: + (func, result) = result.split(" + ") + (offset, result) = result.split(" in ") + else: + offset = "0" + (func, result) = result.split(" in ") + func_str = f"{func}<+{offset}> ()" + except: + return f"??? ({result})" + + # Example: Line 321 of "../util/coroutine-ucontext.c" starts at address + # 0x55cf3894d993 <qemu_coroutine_switch+99> and ends at 0x55cf3894d9ab + # <qemu_coroutine_switch+123>. + result = gdb.execute(f"info line *{hex(addr)}", to_string=True).strip() + if not result.startswith("Line "): + return func_str + result = result[5:] + + try: + result = result.split(" starts ")[0] + (line, path) = result.split(" of ") + path = path.replace("\"", "") + except: + return func_str + + return f"{func_str} at {path}:{line}" + +def dump_backtrace(regs): + ''' + Backtrace dump with raw registers, mimic GDB command 'bt'. + ''' + # Here only rbp and rip that matter.. + rbp = regs['rbp'] + rip = regs['rip'] + i = 0 + + while rbp: + # For all return addresses on stack, we want to look up symbol/line + # on the CALL command, because the return address is the next + # instruction instead of the CALL. Here -1 would work for any + # sized CALL instruction. + print(f"#{i} {hex(rip)} in {symbol_lookup(rip if i == 0 else rip-1)}") + rip = gdb.parse_and_eval(f"*(uint64_t *)(uint64_t)({hex(rbp)} + 8)") + rbp = gdb.parse_and_eval(f"*(uint64_t *)(uint64_t)({hex(rbp)})") + i += 1 + +def dump_backtrace_live(regs): + ''' + Backtrace dump with gdb's 'bt' command, only usable in a live session. + ''' old = dict() # remember current stack frame and select the topmost @@ -88,6 +120,17 @@ def bt_jmpbuf(jmpbuf): selected_frame.select() +def bt_jmpbuf(jmpbuf): + '''Backtrace a jmpbuf''' + regs = get_jmpbuf_regs(jmpbuf) + try: + # This reuses gdb's "bt" command, which can be slightly prettier + # but only works with live sessions. + dump_backtrace_live(regs) + except: + # If above doesn't work, fallback to poor man's unwind + dump_backtrace(regs) + def co_cast(co): return co.cast(gdb.lookup_type('CoroutineUContext').pointer()) @@ -120,10 +163,15 @@ class CoroutineBt(gdb.Command): gdb.execute("bt") - if gdb.parse_and_eval("qemu_in_coroutine()") == False: - return + try: + # This only works with a live session + co_ptr = gdb.parse_and_eval("qemu_coroutine_self()") + except: + # Fallback to use hard-coded ucontext vars if it's coredump + co_ptr = gdb.parse_and_eval("co_tls_current") - co_ptr = gdb.parse_and_eval("qemu_coroutine_self()") + if co_ptr == False: + return while True: co = co_cast(co_ptr) diff --git a/scripts/qom-cast-macro-clean-cocci-gen.py b/scripts/qom-cast-macro-clean-cocci-gen.py index 2fa8438..5aa51d0 100644 --- a/scripts/qom-cast-macro-clean-cocci-gen.py +++ b/scripts/qom-cast-macro-clean-cocci-gen.py @@ -13,8 +13,11 @@ # --in-place \ # --dir . # -# SPDX-FileContributor: Philippe Mathieu-Daudé <philmd@linaro.org> -# SPDX-FileCopyrightText: 2023 Linaro Ltd. +# Copyright (c) 2023 Linaro Ltd. +# +# Authors: +# Philippe Mathieu-Daudé +# # SPDX-License-Identifier: GPL-2.0-or-later import re diff --git a/scripts/rdma-migration-helper.sh b/scripts/rdma-migration-helper.sh new file mode 100755 index 0000000..d784d15 --- /dev/null +++ b/scripts/rdma-migration-helper.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +# Copied from blktests +get_ipv4_addr() +{ + ip -4 -o addr show dev "$1" | + sed -n 's/.*[[:blank:]]inet[[:blank:]]*\([^[:blank:]/]*\).*/\1/p' | + head -1 | tr -d '\n' +} + +get_ipv6_addr() { + ipv6=$(ip -6 -o addr show dev "$1" | + sed -n 's/.*[[:blank:]]inet6[[:blank:]]*\([^[:blank:]/]*\).*/\1/p' | + head -1 | tr -d '\n') + + [ $? -eq 0 ] || return + + if [[ "$ipv6" =~ ^fe80: ]]; then + echo -n "[$ipv6%$1]" + else + echo -n "[$ipv6]" + fi +} + +# existing rdma interfaces +rdma_interfaces() +{ + rdma link show | sed -nE 's/^link .* netdev ([^ ]+).*$/\1 /p' | + grep -Ev '^(lo|tun|tap)' +} + +# existing valid ipv4 interfaces +ipv4_interfaces() +{ + ip -o addr show | awk '/inet / {print $2}' | grep -Ev '^(lo|tun|tap)' +} + +ipv6_interfaces() +{ + ip -o addr show | awk '/inet6 / {print $2}' | grep -Ev '^(lo|tun|tap)' +} + +rdma_rxe_detect() +{ + family=$1 + for r in $(rdma_interfaces) + do + "$family"_interfaces | grep -qw $r && get_"$family"_addr $r && return + done + + return 1 +} + +rdma_rxe_setup() +{ + family=$1 + for i in $("$family"_interfaces) + do + if rdma_interfaces | grep -qw $i; then + echo "$family: Reuse the existing rdma/rxe ${i}_rxe" \ + "for $i with $(get_"$family"_addr $i)" + return + fi + + rdma link add "${i}_rxe" type rxe netdev "$i" && { + echo "$family: Setup new rdma/rxe ${i}_rxe" \ + "for $i with $(get_"$family"_addr $i)" + return + } + done + + echo "$family: Failed to setup any new rdma/rxe link" >&2 + return 1 +} + +rdma_rxe_clean() +{ + modprobe -r rdma_rxe +} + +IP_FAMILY=${IP_FAMILY:-ipv4} +if [ "$IP_FAMILY" != "ipv6" ] && [ "$IP_FAMILY" != "ipv4" ]; then + echo "Unknown ip family '$IP_FAMILY', only ipv4 or ipv6 is supported." >&2 + exit 1 +fi + +operation=${1:-detect} + +command -v rdma >/dev/null || { + echo "Command 'rdma' is not available, please install it first." >&2 + exit 1 +} + +if [ "$operation" == "setup" ] || [ "$operation" == "clean" ]; then + [ "$UID" == 0 ] || { + echo "Root privilege is required to setup/clean a rdma/rxe link" >&2 + exit 1 + } + if [ "$operation" == "setup" ]; then + rdma_rxe_setup ipv4 + rdma_rxe_setup ipv6 + else + rdma_rxe_clean + fi +elif [ "$operation" == "detect" ]; then + rdma_rxe_detect "$IP_FAMILY" +else + echo "Usage: $0 [setup | detect | clean]" +fi diff --git a/scripts/replay-dump.py b/scripts/replay-dump.py index d668193..4ce7ff5 100755 --- a/scripts/replay-dump.py +++ b/scripts/replay-dump.py @@ -20,6 +20,8 @@ import argparse import struct +import os +import sys from collections import namedtuple from os import path @@ -99,7 +101,7 @@ def call_decode(table, index, dumpfile): print("Could not decode index: %d" % (index)) print("Entry is: %s" % (decoder)) print("Decode Table is:\n%s" % (table)) - return False + raise(Exception("unknown event")) else: return decoder.fn(decoder.eid, decoder.name, dumpfile) @@ -120,7 +122,7 @@ def print_event(eid, name, string=None, event_count=None): def decode_unimp(eid, name, _unused_dumpfile): "Unimplemented decoder, will trigger exit" print("%s not handled - will now stop" % (name)) - return False + raise(Exception("unhandled event")) def decode_plain(eid, name, _unused_dumpfile): "Plain events without additional data" @@ -134,6 +136,30 @@ def swallow_async_qword(eid, name, dumpfile): print(" %s(%d) @ %d" % (name, eid, step_id)) return True +def swallow_bytes(eid, name, dumpfile, nr): + """Swallow nr bytes of data without looking at it""" + dumpfile.seek(nr, os.SEEK_CUR) + +total_insns = 0 + +def decode_instruction(eid, name, dumpfile): + global total_insns + ins_diff = read_dword(dumpfile) + total_insns += ins_diff + print_event(eid, name, "+ %d -> %d" % (ins_diff, total_insns)) + return True + +def decode_interrupt(eid, name, dumpfile): + print_event(eid, name) + return True + +def decode_exception(eid, name, dumpfile): + print_event(eid, name) + return True + +# v12 does away with the additional event byte and encodes it in the main type +# Between v8 and v9, REPLAY_ASYNC_BH_ONESHOT was added, but we don't decode +# those versions so leave it out. async_decode_table = [ Decoder(0, "REPLAY_ASYNC_EVENT_BH", swallow_async_qword), Decoder(1, "REPLAY_ASYNC_INPUT", decode_unimp), Decoder(2, "REPLAY_ASYNC_INPUT_SYNC", decode_unimp), @@ -142,8 +168,8 @@ async_decode_table = [ Decoder(0, "REPLAY_ASYNC_EVENT_BH", swallow_async_qword), Decoder(5, "REPLAY_ASYNC_EVENT_NET", decode_unimp), ] # See replay_read_events/replay_read_event -def decode_async(eid, name, dumpfile): - """Decode an ASYNC event""" +def decode_async_old(eid, name, dumpfile): + """Decode an ASYNC event (pre-v8)""" print_event(eid, name) @@ -157,13 +183,37 @@ def decode_async(eid, name, dumpfile): return call_decode(async_decode_table, async_event_kind, dumpfile) -total_insns = 0 +def decode_async_bh(eid, name, dumpfile): + op_id = read_qword(dumpfile) + print_event(eid, name) + return True -def decode_instruction(eid, name, dumpfile): - global total_insns - ins_diff = read_dword(dumpfile) - total_insns += ins_diff - print_event(eid, name, "+ %d -> %d" % (ins_diff, total_insns)) +def decode_async_bh_oneshot(eid, name, dumpfile): + op_id = read_qword(dumpfile) + print_event(eid, name) + return True + +def decode_async_char_read(eid, name, dumpfile): + char_id = read_byte(dumpfile) + size = read_dword(dumpfile) + print_event(eid, name, "device:%x chars:%s" % (char_id, dumpfile.read(size))) + return True + +def decode_async_block(eid, name, dumpfile): + op_id = read_qword(dumpfile) + print_event(eid, name) + return True + +def decode_async_net(eid, name, dumpfile): + net_id = read_byte(dumpfile) + flags = read_dword(dumpfile) + size = read_dword(dumpfile) + swallow_bytes(eid, name, dumpfile, size) + print_event(eid, name, "net:%x flags:%x bytes:%d" % (net_id, flags, size)) + return True + +def decode_shutdown(eid, name, dumpfile): + print_event(eid, name) return True def decode_char_write(eid, name, dumpfile): @@ -177,7 +227,22 @@ def decode_audio_out(eid, name, dumpfile): print_event(eid, name, "%d" % (audio_data)) return True -def decode_checkpoint(eid, name, dumpfile): +def decode_random(eid, name, dumpfile): + ret = read_dword(dumpfile) + size = read_dword(dumpfile) + swallow_bytes(eid, name, dumpfile, size) + if (ret): + print_event(eid, name, "%d bytes (getrandom failed)" % (size)) + else: + print_event(eid, name, "%d bytes" % (size)) + return True + +def decode_clock(eid, name, dumpfile): + clock_data = read_qword(dumpfile) + print_event(eid, name, "0x%x" % (clock_data)) + return True + +def __decode_checkpoint(eid, name, dumpfile, old): """Decode a checkpoint. Checkpoints contain a series of async events with their own specific data. @@ -189,38 +254,33 @@ def decode_checkpoint(eid, name, dumpfile): # if the next event is EVENT_ASYNC there are a bunch of # async events to read, otherwise we are done - if next_event != 3: - print_event(eid, name, "no additional data", event_number) - else: + if (old and next_event == 3) or (not old and next_event >= 3 and next_event <= 9): print_event(eid, name, "more data follows", event_number) + else: + print_event(eid, name, "no additional data", event_number) replay_state.reuse_event(next_event) return True +def decode_checkpoint_old(eid, name, dumpfile): + return __decode_checkpoint(eid, name, dumpfile, False) + +def decode_checkpoint(eid, name, dumpfile): + return __decode_checkpoint(eid, name, dumpfile, True) + def decode_checkpoint_init(eid, name, dumpfile): print_event(eid, name) return True -def decode_interrupt(eid, name, dumpfile): +def decode_end(eid, name, dumpfile): print_event(eid, name) - return True - -def decode_clock(eid, name, dumpfile): - clock_data = read_qword(dumpfile) - print_event(eid, name, "0x%x" % (clock_data)) - return True - -def decode_random(eid, name, dumpfile): - ret = read_dword(dumpfile) - data = read_array(dumpfile) - print_event(eid, "%d bytes of random data" % len(data)) - return True + return False # pre-MTTCG merge v5_event_table = [Decoder(0, "EVENT_INSTRUCTION", decode_instruction), Decoder(1, "EVENT_INTERRUPT", decode_interrupt), Decoder(2, "EVENT_EXCEPTION", decode_plain), - Decoder(3, "EVENT_ASYNC", decode_async), + Decoder(3, "EVENT_ASYNC", decode_async_old), Decoder(4, "EVENT_SHUTDOWN", decode_unimp), Decoder(5, "EVENT_CHAR_WRITE", decode_char_write), Decoder(6, "EVENT_CHAR_READ_ALL", decode_unimp), @@ -242,7 +302,7 @@ v5_event_table = [Decoder(0, "EVENT_INSTRUCTION", decode_instruction), v6_event_table = [Decoder(0, "EVENT_INSTRUCTION", decode_instruction), Decoder(1, "EVENT_INTERRUPT", decode_interrupt), Decoder(2, "EVENT_EXCEPTION", decode_plain), - Decoder(3, "EVENT_ASYNC", decode_async), + Decoder(3, "EVENT_ASYNC", decode_async_old), Decoder(4, "EVENT_SHUTDOWN", decode_unimp), Decoder(5, "EVENT_CHAR_WRITE", decode_char_write), Decoder(6, "EVENT_CHAR_READ_ALL", decode_unimp), @@ -266,7 +326,7 @@ v6_event_table = [Decoder(0, "EVENT_INSTRUCTION", decode_instruction), v7_event_table = [Decoder(0, "EVENT_INSTRUCTION", decode_instruction), Decoder(1, "EVENT_INTERRUPT", decode_interrupt), Decoder(2, "EVENT_EXCEPTION", decode_unimp), - Decoder(3, "EVENT_ASYNC", decode_async), + Decoder(3, "EVENT_ASYNC", decode_async_old), Decoder(4, "EVENT_SHUTDOWN", decode_unimp), Decoder(5, "EVENT_SHUTDOWN_HOST_ERR", decode_unimp), Decoder(6, "EVENT_SHUTDOWN_HOST_QMP", decode_unimp), @@ -296,32 +356,31 @@ v7_event_table = [Decoder(0, "EVENT_INSTRUCTION", decode_instruction), v12_event_table = [Decoder(0, "EVENT_INSTRUCTION", decode_instruction), Decoder(1, "EVENT_INTERRUPT", decode_interrupt), - Decoder(2, "EVENT_EXCEPTION", decode_plain), - Decoder(3, "EVENT_ASYNC", decode_async), - Decoder(4, "EVENT_ASYNC", decode_async), - Decoder(5, "EVENT_ASYNC", decode_async), - Decoder(6, "EVENT_ASYNC", decode_async), - Decoder(6, "EVENT_ASYNC", decode_async), - Decoder(8, "EVENT_ASYNC", decode_async), - Decoder(9, "EVENT_ASYNC", decode_async), - Decoder(10, "EVENT_ASYNC", decode_async), - Decoder(11, "EVENT_SHUTDOWN", decode_unimp), - Decoder(12, "EVENT_SHUTDOWN_HOST_ERR", decode_unimp), - Decoder(13, "EVENT_SHUTDOWN_HOST_QMP_QUIT", decode_unimp), - Decoder(14, "EVENT_SHUTDOWN_HOST_QMP_RESET", decode_unimp), - Decoder(14, "EVENT_SHUTDOWN_HOST_SIGNAL", decode_unimp), - Decoder(15, "EVENT_SHUTDOWN_HOST_UI", decode_unimp), - Decoder(16, "EVENT_SHUTDOWN_GUEST_SHUTDOWN", decode_unimp), - Decoder(17, "EVENT_SHUTDOWN_GUEST_RESET", decode_unimp), - Decoder(18, "EVENT_SHUTDOWN_GUEST_PANIC", decode_unimp), - Decoder(19, "EVENT_SHUTDOWN_GUEST_SUBSYSTEM_RESET", decode_unimp), - Decoder(20, "EVENT_SHUTDOWN_GUEST_SNAPSHOT_LOAD", decode_unimp), - Decoder(21, "EVENT_SHUTDOWN___MAX", decode_unimp), + Decoder(2, "EVENT_EXCEPTION", decode_exception), + Decoder(3, "EVENT_ASYNC_BH", decode_async_bh), + Decoder(4, "EVENT_ASYNC_BH_ONESHOT", decode_async_bh_oneshot), + Decoder(5, "EVENT_ASYNC_INPUT", decode_unimp), + Decoder(6, "EVENT_ASYNC_INPUT_SYNC", decode_unimp), + Decoder(7, "EVENT_ASYNC_CHAR_READ", decode_async_char_read), + Decoder(8, "EVENT_ASYNC_BLOCK", decode_async_block), + Decoder(9, "EVENT_ASYNC_NET", decode_async_net), + Decoder(10, "EVENT_SHUTDOWN", decode_shutdown), + Decoder(11, "EVENT_SHUTDOWN_HOST_ERR", decode_shutdown), + Decoder(12, "EVENT_SHUTDOWN_HOST_QMP_QUIT", decode_shutdown), + Decoder(13, "EVENT_SHUTDOWN_HOST_QMP_RESET", decode_shutdown), + Decoder(14, "EVENT_SHUTDOWN_HOST_SIGNAL", decode_shutdown), + Decoder(15, "EVENT_SHUTDOWN_HOST_UI", decode_shutdown), + Decoder(16, "EVENT_SHUTDOWN_GUEST_SHUTDOWN", decode_shutdown), + Decoder(17, "EVENT_SHUTDOWN_GUEST_RESET", decode_shutdown), + Decoder(18, "EVENT_SHUTDOWN_GUEST_PANIC", decode_shutdown), + Decoder(19, "EVENT_SHUTDOWN_SUBSYS_RESET", decode_shutdown), + Decoder(20, "EVENT_SHUTDOWN_SNAPSHOT_LOAD", decode_shutdown), + Decoder(21, "EVENT_SHUTDOWN___MAX", decode_shutdown), Decoder(22, "EVENT_CHAR_WRITE", decode_char_write), Decoder(23, "EVENT_CHAR_READ_ALL", decode_unimp), Decoder(24, "EVENT_CHAR_READ_ALL_ERROR", decode_unimp), - Decoder(25, "EVENT_AUDIO_IN", decode_unimp), - Decoder(26, "EVENT_AUDIO_OUT", decode_audio_out), + Decoder(25, "EVENT_AUDIO_OUT", decode_audio_out), + Decoder(26, "EVENT_AUDIO_IN", decode_unimp), Decoder(27, "EVENT_RANDOM", decode_random), Decoder(28, "EVENT_CLOCK_HOST", decode_clock), Decoder(29, "EVENT_CLOCK_VIRTUAL_RT", decode_clock), @@ -334,6 +393,7 @@ v12_event_table = [Decoder(0, "EVENT_INSTRUCTION", decode_instruction), Decoder(36, "EVENT_CP_CLOCK_VIRTUAL_RT", decode_checkpoint), Decoder(37, "EVENT_CP_INIT", decode_checkpoint_init), Decoder(38, "EVENT_CP_RESET", decode_checkpoint), + Decoder(39, "EVENT_END", decode_end), ] def parse_arguments(): @@ -375,6 +435,7 @@ def decode_file(filename): dumpfile) except Exception as inst: print(f"error {inst}") + sys.exit(1) finally: print(f"Reached {dumpfile.tell()} of {dumpsize} bytes") diff --git a/scripts/rust/rust_root_crate.sh b/scripts/rust/rust_root_crate.sh new file mode 100755 index 0000000..975bddf --- /dev/null +++ b/scripts/rust/rust_root_crate.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +set -eu + +cat <<EOF +// @generated +// This file is autogenerated by scripts/rust_root_crate.sh + +EOF + +for crate in $*; do + echo "extern crate $crate;" +done diff --git a/scripts/rust/rustc_args.py b/scripts/rust/rustc_args.py new file mode 100644 index 0000000..63b0748 --- /dev/null +++ b/scripts/rust/rustc_args.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 + +"""Generate rustc arguments for meson rust builds. + +This program generates --cfg compile flags for the configuration headers passed +as arguments. + +Copyright (c) 2024 Linaro Ltd. + +Authors: + Manos Pitsidianakis <manos.pitsidianakis@linaro.org> + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see <http://www.gnu.org/licenses/>. +""" + +import argparse +from dataclasses import dataclass +import logging +from pathlib import Path +from typing import Any, Iterable, List, Mapping, Optional, Set + +try: + import tomllib +except ImportError: + import tomli as tomllib + +STRICT_LINTS = {"unknown_lints", "warnings"} + + +class CargoTOML: + tomldata: Mapping[Any, Any] + workspace_data: Mapping[Any, Any] + check_cfg: Set[str] + + def __init__(self, path: Optional[str], workspace: Optional[str]): + if path is not None: + with open(path, 'rb') as f: + self.tomldata = tomllib.load(f) + else: + self.tomldata = {"lints": {"workspace": True}} + + if workspace is not None: + with open(workspace, 'rb') as f: + self.workspace_data = tomllib.load(f) + if "workspace" not in self.workspace_data: + self.workspace_data["workspace"] = {} + + self.check_cfg = set(self.find_check_cfg()) + + def find_check_cfg(self) -> Iterable[str]: + toml_lints = self.lints + rust_lints = toml_lints.get("rust", {}) + cfg_lint = rust_lints.get("unexpected_cfgs", {}) + return cfg_lint.get("check-cfg", []) + + @property + def lints(self) -> Mapping[Any, Any]: + return self.get_table("lints", True) + + def get_table(self, key: str, can_be_workspace: bool = False) -> Mapping[Any, Any]: + table = self.tomldata.get(key, {}) + if can_be_workspace and table.get("workspace", False) is True: + table = self.workspace_data["workspace"].get(key, {}) + + return table + + +@dataclass +class LintFlag: + flags: List[str] + priority: int + + +def generate_lint_flags(cargo_toml: CargoTOML, strict_lints: bool) -> Iterable[str]: + """Converts Cargo.toml lints to rustc -A/-D/-F/-W flags.""" + + toml_lints = cargo_toml.lints + + lint_list = [] + for k, v in toml_lints.items(): + prefix = "" if k == "rust" else k + "::" + for lint, data in v.items(): + level = data if isinstance(data, str) else data["level"] + priority = 0 if isinstance(data, str) else data.get("priority", 0) + if level == "deny": + flag = "-D" + elif level == "allow": + flag = "-A" + elif level == "warn": + flag = "-W" + elif level == "forbid": + flag = "-F" + else: + raise Exception(f"invalid level {level} for {prefix}{lint}") + + if not (strict_lints and lint in STRICT_LINTS): + lint_list.append(LintFlag(flags=[flag, prefix + lint], priority=priority)) + + if strict_lints: + for lint in STRICT_LINTS: + lint_list.append(LintFlag(flags=["-D", lint], priority=1000000)) + + lint_list.sort(key=lambda x: x.priority) + for lint in lint_list: + yield from lint.flags + + +def generate_cfg_flags(header: str, cargo_toml: CargoTOML) -> Iterable[str]: + """Converts defines from config[..].h headers to rustc --cfg flags.""" + + with open(header, encoding="utf-8") as cfg: + config = [l.split()[1:] for l in cfg if l.startswith("#define")] + + cfg_list = [] + for cfg in config: + name = cfg[0] + if f'cfg({name})' not in cargo_toml.check_cfg: + continue + if len(cfg) >= 2 and cfg[1] != "1": + continue + cfg_list.append("--cfg") + cfg_list.append(name) + return cfg_list + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("-v", "--verbose", action="store_true") + parser.add_argument( + "--config-headers", + metavar="CONFIG_HEADER", + action="append", + dest="config_headers", + help="paths to any configuration C headers (*.h files), if any", + required=False, + default=[], + ) + parser.add_argument( + metavar="TOML_FILE", + action="store", + dest="cargo_toml", + help="path to Cargo.toml file", + nargs='?', + ) + parser.add_argument( + "--workspace", + metavar="DIR", + action="store", + dest="workspace", + help="path to root of the workspace", + required=False, + default=None, + ) + parser.add_argument( + "--features", + action="store_true", + dest="features", + help="generate --check-cfg arguments for features", + required=False, + default=None, + ) + parser.add_argument( + "--lints", + action="store_true", + dest="lints", + help="generate arguments from [lints] table", + required=False, + default=None, + ) + parser.add_argument( + "--rustc-version", + metavar="VERSION", + dest="rustc_version", + action="store", + help="version of rustc", + required=False, + default="1.0.0", + ) + parser.add_argument( + "--strict-lints", + action="store_true", + dest="strict_lints", + help="apply stricter checks (for nightly Rust)", + default=False, + ) + args = parser.parse_args() + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + logging.debug("args: %s", args) + + rustc_version = tuple((int(x) for x in args.rustc_version.split('.')[0:2])) + if args.workspace: + workspace_cargo_toml = Path(args.workspace, "Cargo.toml").resolve() + cargo_toml = CargoTOML(args.cargo_toml, str(workspace_cargo_toml)) + else: + cargo_toml = CargoTOML(args.cargo_toml, None) + + if args.lints: + for tok in generate_lint_flags(cargo_toml, args.strict_lints): + print(tok) + + if rustc_version >= (1, 80): + if args.lints: + print("--check-cfg") + print("cfg(test)") + for cfg in sorted(cargo_toml.check_cfg): + print("--check-cfg") + print(cfg) + if args.features: + for feature in cargo_toml.get_table("features"): + if feature != "default": + print("--check-cfg") + print(f'cfg(feature,values("{feature}"))') + + for header in args.config_headers: + for tok in generate_cfg_flags(header, cargo_toml): + print(tok) + + +if __name__ == "__main__": + main() diff --git a/scripts/symlink-install-tree.py b/scripts/symlink-install-tree.py index 8ed97e3..b725638 100644 --- a/scripts/symlink-install-tree.py +++ b/scripts/symlink-install-tree.py @@ -4,6 +4,7 @@ from pathlib import PurePath import errno import json import os +import shlex import subprocess import sys @@ -14,7 +15,7 @@ def destdir_join(d1: str, d2: str) -> str: return str(PurePath(d1, *PurePath(d2).parts[1:])) introspect = os.environ.get('MESONINTROSPECT') -out = subprocess.run([*introspect.split(' '), '--installed'], +out = subprocess.run([*shlex.split(introspect), '--installed'], stdout=subprocess.PIPE, check=True).stdout for source, dest in json.loads(out).items(): bundle_dest = destdir_join('qemu-bundle', dest) diff --git a/scripts/tracetool/__init__.py b/scripts/tracetool/__init__.py index bc03238..6dfcbf7 100644 --- a/scripts/tracetool/__init__.py +++ b/scripts/tracetool/__init__.py @@ -12,12 +12,14 @@ __maintainer__ = "Stefan Hajnoczi" __email__ = "stefanha@redhat.com" +import os import re import sys import weakref +from pathlib import PurePath -import tracetool.format import tracetool.backend +import tracetool.format def error_write(*lines): @@ -36,7 +38,7 @@ out_fobj = sys.stdout def out_open(filename): global out_filename, out_fobj - out_filename = filename + out_filename = posix_relpath(filename) out_fobj = open(filename, 'wt') def out(*lines, **kwargs): @@ -308,7 +310,7 @@ class Event(object): fmt = [fmt_trans, fmt] args = Arguments.build(groups["args"]) - return Event(name, props, fmt, args, lineno, filename) + return Event(name, props, fmt, args, lineno, posix_relpath(filename)) def __repr__(self): """Evaluable string representation for this object.""" @@ -447,3 +449,10 @@ def generate(events, group, format, backends, tracetool.backend.dtrace.PROBEPREFIX = probe_prefix tracetool.format.generate(events, format, backend, group) + +def posix_relpath(path, start=None): + try: + path = os.path.relpath(path, start) + except ValueError: + pass + return PurePath(path).as_posix() diff --git a/scripts/tracetool/backend/ftrace.py b/scripts/tracetool/backend/ftrace.py index baed2ae..5fa30cc 100644 --- a/scripts/tracetool/backend/ftrace.py +++ b/scripts/tracetool/backend/ftrace.py @@ -12,8 +12,6 @@ __maintainer__ = "Stefan Hajnoczi" __email__ = "stefanha@redhat.com" -import os.path - from tracetool import out @@ -47,7 +45,7 @@ def generate_h(event, group): args=event.args, event_id="TRACE_" + event.name.upper(), event_lineno=event.lineno, - event_filename=os.path.relpath(event.filename), + event_filename=event.filename, fmt=event.fmt.rstrip("\n"), argnames=argnames) diff --git a/scripts/tracetool/backend/log.py b/scripts/tracetool/backend/log.py index de27b7e..17ba1cd 100644 --- a/scripts/tracetool/backend/log.py +++ b/scripts/tracetool/backend/log.py @@ -12,8 +12,6 @@ __maintainer__ = "Stefan Hajnoczi" __email__ = "stefanha@redhat.com" -import os.path - from tracetool import out @@ -55,7 +53,7 @@ def generate_h(event, group): ' }', cond=cond, event_lineno=event.lineno, - event_filename=os.path.relpath(event.filename), + event_filename=event.filename, name=event.name, fmt=event.fmt.rstrip("\n"), argnames=argnames) diff --git a/scripts/tracetool/backend/simple.py b/scripts/tracetool/backend/simple.py index a74d61f..2688d4b 100644 --- a/scripts/tracetool/backend/simple.py +++ b/scripts/tracetool/backend/simple.py @@ -36,8 +36,17 @@ def generate_h_begin(events, group): def generate_h(event, group): - out(' _simple_%(api)s(%(args)s);', + event_id = 'TRACE_' + event.name.upper() + if "vcpu" in event.properties: + # already checked on the generic format code + cond = "true" + else: + cond = "trace_event_get_state(%s)" % event_id + out(' if (%(cond)s) {', + ' _simple_%(api)s(%(args)s);', + ' }', api=event.api(), + cond=cond, args=", ".join(event.args.names())) @@ -72,22 +81,10 @@ def generate_c(event, group): if len(event.args) == 0: sizestr = '0' - event_id = 'TRACE_' + event.name.upper() - if "vcpu" in event.properties: - # already checked on the generic format code - cond = "true" - else: - cond = "trace_event_get_state(%s)" % event_id - out('', - ' if (!%(cond)s) {', - ' return;', - ' }', - '', ' if (trace_record_start(&rec, %(event_obj)s.id, %(size_str)s)) {', ' return; /* Trace Buffer Full, Event Dropped ! */', ' }', - cond=cond, event_obj=event.api(event.QEMU_EVENT), size_str=sizestr) diff --git a/scripts/tracetool/backend/syslog.py b/scripts/tracetool/backend/syslog.py index 012970f..5a3a00f 100644 --- a/scripts/tracetool/backend/syslog.py +++ b/scripts/tracetool/backend/syslog.py @@ -12,8 +12,6 @@ __maintainer__ = "Stefan Hajnoczi" __email__ = "stefanha@redhat.com" -import os.path - from tracetool import out @@ -43,7 +41,7 @@ def generate_h(event, group): ' }', cond=cond, event_lineno=event.lineno, - event_filename=os.path.relpath(event.filename), + event_filename=event.filename, name=event.name, fmt=event.fmt.rstrip("\n"), argnames=argnames) diff --git a/scripts/update-linux-headers.sh b/scripts/update-linux-headers.sh index c34ac64..b43b8ef 100755 --- a/scripts/update-linux-headers.sh +++ b/scripts/update-linux-headers.sh @@ -163,6 +163,7 @@ EOF fi if [ $arch = arm64 ]; then cp "$hdrdir/include/asm/sve_context.h" "$output/linux-headers/asm-arm64/" + cp "$hdrdir/include/asm/unistd_64.h" "$output/linux-headers/asm-arm64/" fi if [ $arch = x86 ]; then cp "$hdrdir/include/asm/unistd_32.h" "$output/linux-headers/asm-x86/" @@ -176,7 +177,7 @@ EOF # Remove everything except the macros from bootparam.h avoiding the # unnecessary import of several video/ist/etc headers - sed -e '/__ASSEMBLY__/,/__ASSEMBLY__/d' \ + sed -e '/__ASSEMBLER__/,/__ASSEMBLER__/d' \ "$hdrdir/include/asm/bootparam.h" > "$hdrdir/bootparam.h" cp_portable "$hdrdir/bootparam.h" \ "$output/include/standard-headers/asm-$arch" @@ -185,6 +186,12 @@ EOF fi if [ $arch = riscv ]; then cp "$hdrdir/include/asm/ptrace.h" "$output/linux-headers/asm-riscv/" + cp "$hdrdir/include/asm/unistd_32.h" "$output/linux-headers/asm-riscv/" + cp "$hdrdir/include/asm/unistd_64.h" "$output/linux-headers/asm-riscv/" + fi + if [ $arch = loongarch ]; then + cp "$hdrdir/include/asm/kvm_para.h" "$output/linux-headers/asm-loongarch/" + cp "$hdrdir/include/asm/unistd_64.h" "$output/linux-headers/asm-loongarch/" fi done arch= @@ -251,6 +258,7 @@ for i in "$hdrdir"/include/linux/*virtio*.h \ "$hdrdir/include/linux/kernel.h" \ "$hdrdir/include/linux/kvm_para.h" \ "$hdrdir/include/linux/vhost_types.h" \ + "$hdrdir/include/linux/vmclock-abi.h" \ "$hdrdir/include/linux/sysinfo.h"; do cp_portable "$i" "$output/include/standard-headers/linux" done diff --git a/scripts/update-syscalltbl.sh b/scripts/update-syscalltbl.sh index 2d23e56..f0927c5 100755 --- a/scripts/update-syscalltbl.sh +++ b/scripts/update-syscalltbl.sh @@ -1,13 +1,18 @@ TBL_LIST="\ arch/alpha/kernel/syscalls/syscall.tbl,linux-user/alpha/syscall.tbl \ arch/arm/tools/syscall.tbl,linux-user/arm/syscall.tbl \ +scripts/syscall.tbl,linux-user/aarch64/syscall_64.tbl \ +scripts/syscall.tbl,linux-user/hexagon/syscall.tbl \ +scripts/syscall.tbl,linux-user/loongarch64/syscall.tbl \ arch/m68k/kernel/syscalls/syscall.tbl,linux-user/m68k/syscall.tbl \ arch/microblaze/kernel/syscalls/syscall.tbl,linux-user/microblaze/syscall.tbl \ arch/mips/kernel/syscalls/syscall_n32.tbl,linux-user/mips64/syscall_n32.tbl \ arch/mips/kernel/syscalls/syscall_n64.tbl,linux-user/mips64/syscall_n64.tbl \ arch/mips/kernel/syscalls/syscall_o32.tbl,linux-user/mips/syscall_o32.tbl \ +scripts/syscall.tbl,linux-user/openrisc/syscall.tbl \ arch/parisc/kernel/syscalls/syscall.tbl,linux-user/hppa/syscall.tbl \ arch/powerpc/kernel/syscalls/syscall.tbl,linux-user/ppc/syscall.tbl \ +scripts/syscall.tbl,linux-user/riscv/syscall.tbl \ arch/s390/kernel/syscalls/syscall.tbl,linux-user/s390x/syscall.tbl \ arch/sh/kernel/syscalls/syscall.tbl,linux-user/sh4/syscall.tbl \ arch/sparc/kernel/syscalls/syscall.tbl,linux-user/sparc64/syscall.tbl \ diff --git a/scripts/vmstate-static-checker.py b/scripts/vmstate-static-checker.py index 9c0e6b8..2335e25 100755 --- a/scripts/vmstate-static-checker.py +++ b/scripts/vmstate-static-checker.py @@ -42,6 +42,7 @@ def check_fields_match(name, s_field, d_field): # Some fields changed names between qemu versions. This list # is used to allow such changes in each section / description. changed_names = { + 'acpi-ghes': ['ghes_addr_le', 'hw_error_le'], 'apic': ['timer', 'timer_expiry'], 'e1000': ['dev', 'parent_obj'], 'ehci': ['dev', 'pcidev'], @@ -90,6 +91,7 @@ def check_fields_match(name, s_field, d_field): 'mem_win_size', 'mig_mem_win_size', 'io_win_addr', 'mig_io_win_addr', 'io_win_size', 'mig_io_win_size'], + 'hpet': ['num_timers', 'num_timers_save'], } if not name in changed_names: |