Fix comparison of unsigned long int to int in record_linux_system_call.

The if statement in case gdb_sys_ioctl in function
record_linux_system_call in file gdb/linux-record.c is as follows:

   if (tmpulongest == tdep->ioctl_FIOCLEX
      || tmpulongest == tdep->ioctl_FIONCLEX
    ....
      || tmpulongest == tdep->ioctl_TCSETSW
     ...
   }

The PowerPC ioctl value for ioctl_TCSETW is 0x802c7415.  The variable
ioctl_TCSETW is defined in gdb/linux-record.h as an int.  The TCSETW value
has the MSB set to one so it is a negative integer.  The comparison of the
unsigned long value tmpulongest to a negative integer value for
ioctl_TCSETSW fails.

This patch changes the declarations for the ioctl_* values in struct
linux_record_tdep to unsigned long to fix the comparisons between
tmpulongest and the tdep->ioctl_* values.

An additional test gdb.reverse/test_ioctl_TCSETSW.exp is added to verify
the gdb record_linux_system_call() if statement for the ioctl TCSETSW
succeeds.

This patch has been tested on Power 10 and Intel with no test failures.

1 files changed, 45 insertions, 0 deletions

diff --git a/gdb/testsuite/gdb.reverse/test_ioctl_TCSETSW.exp b/gdb/testsuite/gdb.reverse/test_ioctl_TCSETSW.exp
new file mode 100644
index 0000000..86a62eb
--- /dev/null
+++ b/gdb/testsuite/gdb.reverse/test_ioctl_TCSETSW.exp
@@ -0,0 +1,45 @@
+# Copyright 2008-2022 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Test ioctl TCSETSW record for PowerPC.
+#
+
+standard_testfile .c
+
+if { [prepare_for_testing "failed to prepare" ${testfile} ${srcfile}] } {
+    return -1
+}
+
+if ![runto_main] then {
+    untested "could not run to main"
+    continue
+}
+
+# Recording of ioctls calls requires record full
+gdb_test_no_output "record full"
+
+set stop [gdb_get_line_number "TCSETSW call"]
+gdb_test "break $stop" ".*Breakpoint .*" "stop at TCSETSW"
+gdb_test "continue"  ".*Breakpoint .*" "at TCSETSW call"
+
+set test "handle TCSETSW"
+gdb_test_multiple "step"  $test {
+    -re "Process record and replay target doesn't support ioctl request 0x.*$gdb_prompt $" {
+	fail $test
+    }
+    -re ".*result = 0.*$gdb_prompt $" {
+	pass $test
+    }
+}