C++: handle glibc's ptrace(enum __ptrace_request, ...)

Building in C++ mode issues ~40 warnings like this:

 ../../src/gdb/linux-nat.c: In function ‘int linux_handle_extended_wait(lwp_info*, int, int)’:
 ../../src/gdb/linux-nat.c:2016:51: warning: invalid conversion from ‘int’ to ‘__ptrace_request’ [-fpermissive]
	ptrace (PTRACE_GETEVENTMSG, pid, 0, &new_pid);

The issue is that in glibc, ptrace's first parameter is an enum.
That's not a problem if we pick the PTRACE_XXX requests from
sys/ptrace.h, as those will be values of the corresponding enum.
However, we have fallback definitions for PTRACE_XXX symbols when the
system headers miss them (such as PTRACE_GETEVENTMSG above), and those
are plain integer constants.  E.g., nat/linux-ptrace.h:

 #define PTRACE_GETEVENTMSG	0x4201

One idea would be to fix this by defining those fallbacks like:

 -#define PTRACE_GETEVENTMSG	0x4201
 +#define PTRACE_GETEVENTMSG	((enum __ptrace_request) 0x4201)

However, while glibc's ptrace uses enum __ptrace_request for first
parameter:

  extern long int ptrace (enum __ptrace_request __request, ...) __THROW;

other libc's, like e.g., Android's bionic do not -- in that case, the
first parameter is int:

  long ptrace(int request, pid_t pid, void * addr, void * data);

So the fix I came up is to make configure/ptrace.m4 also detect the
type of the ptrace's first parameter and defin PTRACE_TYPE_ARG1, as
already does the for parameters 3-4, and then simply wrap ptrace with
a macro that casts the first argument to the detected type.  (I'm
leaving adding a nicer wrapper for when we drop building in C).

While this adds the wrapper, GNU/Linux files won't use it until the
next patch, which makes all native GNU/Linux files include
gdb_ptrace.h.

gdb/ChangeLog:
2015-07-24  Pedro Alves  <palves@redhat.com>

	* ptrace.m4 (ptrace tests): Test in C++ mode.  Try with 'enum
	__ptrace_request as first parameter type instead of int.
	(PTRACE_TYPE_ARG1): Define.
	* nat/gdb_ptrace.h [!PTRACE_TYPE_ARG5] (ptrace): Define as wrapper
	that casts first argument to PTRACE_TYPE_ARG1.
	* config.in: Regenerate.
	* configure: Regenerate.

gdb/gdbserver/ChangeLog:
2015-07-24  Pedro Alves  <palves@redhat.com>

	* config.in: Regenerate.
	* configure: Regenerate.

8 files changed, 215 insertions, 21 deletions

diff --git a/gdb/ChangeLog b/gdb/ChangeLog
index 0272d67..57be707 100644
--- a/gdb/ChangeLog
+++ b/gdb/ChangeLog
@@ -1,5 +1,15 @@
 2015-07-24  Pedro Alves  <palves@redhat.com>
 
+	* ptrace.m4 (ptrace tests): Test in C++ mode.  Try with 'enum
+	__ptrace_request as first parameter type instead of int.
+	(PTRACE_TYPE_ARG1): Define.
+	* nat/gdb_ptrace.h [!PTRACE_TYPE_ARG5] (ptrace): Define as wrapper
+	that casts first argument to PTRACE_TYPE_ARG1.
+	* config.in: Regenerate.
+	* configure: Regenerate.
+
+2015-07-24  Pedro Alves  <palves@redhat.com>
+
 	* gdb_ptrace.h: Move ...
 	* nat/gdb_ptrace.h: ... here.
 	* inf-ptrace.c: Adjust.
diff --git a/gdb/config.in b/gdb/config.in
index 9ef53b3..d188088 100644
--- a/gdb/config.in
+++ b/gdb/config.in
@@ -653,6 +653,9 @@
    assorted other type changes. */
 #undef PROC_SERVICE_IS_OLD
 
+/* Define to the type of arg 1 for ptrace. */
+#undef PTRACE_TYPE_ARG1
+
 /* Define to the type of arg 3 for ptrace. */
 #undef PTRACE_TYPE_ARG3
 
diff --git a/gdb/configure b/gdb/configure
index e8979f0..e1b778b 100755
--- a/gdb/configure
+++ b/gdb/configure
@@ -2398,6 +2398,51 @@ $as_echo "$ac_res" >&6; }
   eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;}
 
 } # ac_fn_c_check_member
+
+# ac_fn_cxx_check_decl LINENO SYMBOL VAR
+# --------------------------------------
+# Tests whether SYMBOL is declared, setting cache variable VAR accordingly.
+ac_fn_cxx_check_decl ()
+{
+  as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+  as_decl_name=`echo $2|sed 's/ *(.*//'`
+  as_decl_use=`echo $2|sed -e 's/(/((/' -e 's/)/) 0&/' -e 's/,/) 0& (/g'`
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $as_decl_name is declared" >&5
+$as_echo_n "checking whether $as_decl_name is declared... " >&6; }
+if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then :
+  $as_echo_n "(cached) " >&6
+else
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+$4
+int
+main ()
+{
+#ifndef $as_decl_name
+#ifdef __cplusplus
+  (void) $as_decl_use;
+#else
+  (void) $as_decl_name;
+#endif
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_cxx_try_compile "$LINENO"; then :
+  eval "$3=yes"
+else
+  eval "$3=no"
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+eval ac_res=\$$3
+	       { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+$as_echo "$ac_res" >&6; }
+  eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;}
+
+} # ac_fn_cxx_check_decl
 cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
@@ -11926,8 +11971,36 @@ $as_echo "#define HAVE_SIGSETJMP 1" >>confdefs.h
 fi
 
 
-# Check the return and argument types of ptrace.  No canned test for
-# this, so roll our own.
+# Check the return and argument types of ptrace.
+
+
+
+for ac_header in sys/ptrace.h ptrace.h
+do :
+  as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
+ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default"
+eval as_val=\$$as_ac_Header
+   if test "x$as_val" = x""yes; then :
+  cat >>confdefs.h <<_ACEOF
+#define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+
+# Needs to be tested in C++ mode, to detect whether we need to cast
+# the first argument to enum __ptrace_request.
+if test "$enable_build_with_cxx" = "yes"; then
+    ac_ext=cpp
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+
+fi
+
 gdb_ptrace_headers='
 #include <sys/types.h>
 #if HAVE_SYS_PTRACE_H
@@ -11938,7 +12011,7 @@ gdb_ptrace_headers='
 #endif
 '
 # There is no point in checking if we don't have a prototype.
-ac_fn_c_check_decl "$LINENO" "ptrace" "ac_cv_have_decl_ptrace" "$gdb_ptrace_headers
+ac_fn_cxx_check_decl "$LINENO" "ptrace" "ac_cv_have_decl_ptrace" "$gdb_ptrace_headers
 "
 if test "x$ac_cv_have_decl_ptrace" = x""yes; then :
   ac_have_decl=1
@@ -11976,7 +12049,7 @@ extern long ptrace (enum __ptrace_request, ...);
   return 0;
 }
 _ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
+if ac_fn_cxx_try_compile "$LINENO"; then :
   gdb_cv_func_ptrace_ret='long'
 else
   cat confdefs.h - <<_ACEOF >conftest.$ac_ext
@@ -11990,7 +12063,7 @@ extern int ptrace ();
   return 0;
 }
 _ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
+if ac_fn_cxx_try_compile "$LINENO"; then :
   gdb_cv_func_ptrace_ret='int'
 else
   gdb_cv_func_ptrace_ret='long'
@@ -12024,8 +12097,8 @@ extern long ptrace (enum __ptrace_request, ...);
   return 0;
 }
 _ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
-  gdb_cv_func_ptrace_args='int,int,long,long'
+if ac_fn_cxx_try_compile "$LINENO"; then :
+  gdb_cv_func_ptrace_args='enum __ptrace_request,int,long,long'
 else
 
 for gdb_arg1 in 'int' 'long'; do
@@ -12046,7 +12119,7 @@ extern $gdb_cv_func_ptrace_ret
   return 0;
 }
 _ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
+if ac_fn_cxx_try_compile "$LINENO"; then :
   gdb_cv_func_ptrace_args="$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4";
     break 4;
 fi
@@ -12066,7 +12139,7 @@ extern $gdb_cv_func_ptrace_ret
   return 0;
 }
 _ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
+if ac_fn_cxx_try_compile "$LINENO"; then :
 
 gdb_cv_func_ptrace_args="$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4,$gdb_arg5";
     break 5;
@@ -12091,6 +12164,11 @@ IFS=$ac_save_IFS
 shift
 
 cat >>confdefs.h <<_ACEOF
+#define PTRACE_TYPE_ARG1 $1
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
 #define PTRACE_TYPE_ARG3 $3
 _ACEOF
 
@@ -12107,6 +12185,16 @@ _ACEOF
 
 fi
 
+if test "$enable_build_with_cxx" = "yes"; then
+    ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+fi
+
+
 if test "$cross_compiling" = no; then
   { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether setpgrp takes no argument" >&5
 $as_echo_n "checking whether setpgrp takes no argument... " >&6; }
@@ -13925,8 +14013,7 @@ $as_echo_n "checking compiler warning flags... " >&6; }
 	    CFLAGS="$CFLAGS $w"
 	    saved_CXXFLAGS="$CXXFLAGS"
 	    CXXFLAGS="$CXXFLAGS $w"
-
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+	    cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 /* end confdefs.h.  */
 
 int
diff --git a/gdb/gdbserver/ChangeLog b/gdb/gdbserver/ChangeLog
index d1f10cd..664f674 100644
--- a/gdb/gdbserver/ChangeLog
+++ b/gdb/gdbserver/ChangeLog
@@ -1,5 +1,10 @@
 2015-07-24  Pedro Alves  <palves@redhat.com>
 
+	* config.in: Regenerate.
+	* configure: Regenerate.
+
+2015-07-24  Pedro Alves  <palves@redhat.com>
+
 	* acinclude.m4: Include ../ptrace.m4.
 	* configure.ac: Call GDB_AC_PTRACE.
 	* config.in, configure: Regenerate.
diff --git a/gdb/gdbserver/config.in b/gdb/gdbserver/config.in
index 6f2d0a1..e2566a9 100644
--- a/gdb/gdbserver/config.in
+++ b/gdb/gdbserver/config.in
@@ -310,6 +310,9 @@
 /* Additional package description */
 #undef PKGVERSION
 
+/* Define to the type of arg 1 for ptrace. */
+#undef PTRACE_TYPE_ARG1
+
 /* Define to the type of arg 3 for ptrace. */
 #undef PTRACE_TYPE_ARG3
 
diff --git a/gdb/gdbserver/configure b/gdb/gdbserver/configure
index fc6b966..409f42b 100755
--- a/gdb/gdbserver/configure
+++ b/gdb/gdbserver/configure
@@ -1878,6 +1878,51 @@ $as_echo "$ac_res" >&6; }
 
 } # ac_fn_c_check_decl
 
+# ac_fn_cxx_check_decl LINENO SYMBOL VAR
+# --------------------------------------
+# Tests whether SYMBOL is declared, setting cache variable VAR accordingly.
+ac_fn_cxx_check_decl ()
+{
+  as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+  as_decl_name=`echo $2|sed 's/ *(.*//'`
+  as_decl_use=`echo $2|sed -e 's/(/((/' -e 's/)/) 0&/' -e 's/,/) 0& (/g'`
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $as_decl_name is declared" >&5
+$as_echo_n "checking whether $as_decl_name is declared... " >&6; }
+if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then :
+  $as_echo_n "(cached) " >&6
+else
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+$4
+int
+main ()
+{
+#ifndef $as_decl_name
+#ifdef __cplusplus
+  (void) $as_decl_use;
+#else
+  (void) $as_decl_name;
+#endif
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_cxx_try_compile "$LINENO"; then :
+  eval "$3=yes"
+else
+  eval "$3=no"
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+eval ac_res=\$$3
+	       { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+$as_echo "$ac_res" >&6; }
+  eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;}
+
+} # ac_fn_cxx_check_decl
+
 # ac_fn_c_check_type LINENO TYPE VAR INCLUDES
 # -------------------------------------------
 # Tests whether TYPE exists after having included INCLUDES, setting cache
@@ -5751,6 +5796,7 @@ fi
 # Check the return and argument types of ptrace.
 
 
+
 for ac_header in sys/ptrace.h ptrace.h
 do :
   as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
@@ -5766,6 +5812,17 @@ fi
 done
 
 
+# Needs to be tested in C++ mode, to detect whether we need to cast
+# the first argument to enum __ptrace_request.
+if test "$enable_build_with_cxx" = "yes"; then
+    ac_ext=cpp
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+
+fi
+
 gdb_ptrace_headers='
 #include <sys/types.h>
 #if HAVE_SYS_PTRACE_H
@@ -5776,7 +5833,7 @@ gdb_ptrace_headers='
 #endif
 '
 # There is no point in checking if we don't have a prototype.
-ac_fn_c_check_decl "$LINENO" "ptrace" "ac_cv_have_decl_ptrace" "$gdb_ptrace_headers
+ac_fn_cxx_check_decl "$LINENO" "ptrace" "ac_cv_have_decl_ptrace" "$gdb_ptrace_headers
 "
 if test "x$ac_cv_have_decl_ptrace" = x""yes; then :
   ac_have_decl=1
@@ -5814,7 +5871,7 @@ extern long ptrace (enum __ptrace_request, ...);
   return 0;
 }
 _ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
+if ac_fn_cxx_try_compile "$LINENO"; then :
   gdb_cv_func_ptrace_ret='long'
 else
   cat confdefs.h - <<_ACEOF >conftest.$ac_ext
@@ -5828,7 +5885,7 @@ extern int ptrace ();
   return 0;
 }
 _ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
+if ac_fn_cxx_try_compile "$LINENO"; then :
   gdb_cv_func_ptrace_ret='int'
 else
   gdb_cv_func_ptrace_ret='long'
@@ -5862,8 +5919,8 @@ extern long ptrace (enum __ptrace_request, ...);
   return 0;
 }
 _ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
-  gdb_cv_func_ptrace_args='int,int,long,long'
+if ac_fn_cxx_try_compile "$LINENO"; then :
+  gdb_cv_func_ptrace_args='enum __ptrace_request,int,long,long'
 else
 
 for gdb_arg1 in 'int' 'long'; do
@@ -5884,7 +5941,7 @@ extern $gdb_cv_func_ptrace_ret
   return 0;
 }
 _ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
+if ac_fn_cxx_try_compile "$LINENO"; then :
   gdb_cv_func_ptrace_args="$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4";
     break 4;
 fi
@@ -5904,7 +5961,7 @@ extern $gdb_cv_func_ptrace_ret
   return 0;
 }
 _ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
+if ac_fn_cxx_try_compile "$LINENO"; then :
 
 gdb_cv_func_ptrace_args="$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4,$gdb_arg5";
     break 5;
@@ -5929,6 +5986,11 @@ IFS=$ac_save_IFS
 shift
 
 cat >>confdefs.h <<_ACEOF
+#define PTRACE_TYPE_ARG1 $1
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
 #define PTRACE_TYPE_ARG3 $3
 _ACEOF
 
@@ -5945,6 +6007,15 @@ _ACEOF
 
 fi
 
+if test "$enable_build_with_cxx" = "yes"; then
+    ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+fi
+
 
 # Check for UST
 ustlibs=""
@@ -6087,8 +6158,7 @@ $as_echo_n "checking compiler warning flags... " >&6; }
 	    CFLAGS="$CFLAGS $w"
 	    saved_CXXFLAGS="$CXXFLAGS"
 	    CXXFLAGS="$CXXFLAGS $w"
-
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+	    cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 /* end confdefs.h.  */
 
 int
diff --git a/gdb/nat/gdb_ptrace.h b/gdb/nat/gdb_ptrace.h
index a969681..0a98239 100644
--- a/gdb/nat/gdb_ptrace.h
+++ b/gdb/nat/gdb_ptrace.h
@@ -144,6 +144,10 @@ extern PTRACE_TYPE_RET ptrace();
 #  define ptrace(request, pid, addr, data) \
           ptrace (request, pid, addr, data, 0)
 # endif
+#else
+/* Wrapper that avoids adding a pointless cast to all callers.  */
+# define ptrace(request, pid, addr, data) \
+         ptrace ((PTRACE_TYPE_ARG1) request, pid, addr, data)
 #endif
 
 #endif /* gdb_ptrace.h */
diff --git a/gdb/ptrace.m4 b/gdb/ptrace.m4
index e5fe7fa..f01c692 100644
--- a/gdb/ptrace.m4
+++ b/gdb/ptrace.m4
@@ -22,6 +22,12 @@ AC_DEFUN([GDB_AC_PTRACE],
 
 AC_CHECK_HEADERS([sys/ptrace.h ptrace.h])
 
+# Needs to be tested in C++ mode, to detect whether we need to cast
+# the first argument to enum __ptrace_request.
+if test "$enable_build_with_cxx" = "yes"; then
+    AC_LANG_PUSH([C++])
+fi
+
 gdb_ptrace_headers='
 #include <sys/types.h>
 #if HAVE_SYS_PTRACE_H
@@ -52,7 +58,7 @@ AC_DEFINE_UNQUOTED(PTRACE_TYPE_RET, $gdb_cv_func_ptrace_ret,
 AC_CACHE_CHECK([types of arguments for ptrace], gdb_cv_func_ptrace_args, [
   AC_TRY_COMPILE($gdb_ptrace_headers,
     [extern long ptrace (enum __ptrace_request, ...);],
-    [gdb_cv_func_ptrace_args='int,int,long,long'],[
+    [gdb_cv_func_ptrace_args='enum __ptrace_request,int,long,long'],[
 for gdb_arg1 in 'int' 'long'; do
  for gdb_arg2 in 'pid_t' 'int' 'long'; do
   for gdb_arg3 in 'int *' 'caddr_t' 'int' 'long' 'void *'; do
@@ -81,6 +87,8 @@ ac_save_IFS=$IFS; IFS=','
 set dummy `echo "$gdb_cv_func_ptrace_args" | sed 's/\*/\*/g'`
 IFS=$ac_save_IFS
 shift
+AC_DEFINE_UNQUOTED(PTRACE_TYPE_ARG1, $[1],
+  [Define to the type of arg 1 for ptrace.])
 AC_DEFINE_UNQUOTED(PTRACE_TYPE_ARG3, $[3],
   [Define to the type of arg 3 for ptrace.])
 AC_DEFINE_UNQUOTED(PTRACE_TYPE_ARG4, $[4],
@@ -89,4 +97,8 @@ if test -n "$[5]"; then
   AC_DEFINE_UNQUOTED(PTRACE_TYPE_ARG5, $[5],
     [Define to the type of arg 5 for ptrace.])
 fi
+
+if test "$enable_build_with_cxx" = "yes"; then
+    AC_LANG_POP([C++])
+fi
 ])