x32: Properly pass long to syscall [BZ #25810]

X32 has 32-bit long and pointer with 64-bit off_t.  Since x32 psABI
requires that pointers passed in registers must be zero-extended to
64bit, x32 can share many syscall interfaces with LP64.  When a LP64
syscall with long and unsigned long arguments is used for x32, these
arguments must be properly extended to 64-bit.  Otherwise if the upper
32 bits of the register have undefined value, such a syscall will be
rejected by kernel.

Enforce zero-extension for pointers and array system call arguments.
For integer types, extend to int64_t (the full register) using a
regular cast, resulting in zero or sign extension based on the
signedness of the original type.

For

       void *mmap(void *addr, size_t length, int prot, int flags,
                  int fd, off_t offset);

we now generate

   0:	41 f7 c1 ff 0f 00 00 	test   $0xfff,%r9d
   7:	75 1f                	jne    28 <__mmap64+0x28>
   9:	48 63 d2             	movslq %edx,%rdx
   c:	89 f6                	mov    %esi,%esi
   e:	4d 63 c0             	movslq %r8d,%r8
  11:	4c 63 d1             	movslq %ecx,%r10
  14:	b8 09 00 00 40       	mov    $0x40000009,%eax
  19:	0f 05                	syscall

That is

1. addr is unchanged.
2. length is zero-extend to 64 bits.
3. prot is sign-extend to 64 bits.
4. flags is sign-extend to 64 bits.
5. fd is sign-extend to 64 bits.
6. offset is unchanged.

For int arguments, since kernel uses only the lower 32 bits and ignores
the upper 32 bits in 64-bit registers, these work correctly.

Tested on x86-64 and x32.  There are no code changes on x86-64.

2 files changed, 25 insertions, 6 deletions

diff --git a/sysdeps/unix/sysv/linux/x86_64/sysdep.h b/sysdeps/unix/sysv/linux/x86_64/sysdep.h
index 24d8b8e..85f8f94 100644
--- a/sysdeps/unix/sysv/linux/x86_64/sysdep.h
+++ b/sysdeps/unix/sysv/linux/x86_64/sysdep.h
@@ -177,12 +177,15 @@
 /* Registers clobbered by syscall.  */
 # define REGISTERS_CLOBBERED_BY_SYSCALL "cc", "r11", "cx"
 
-/* Create a variable 'name' based on type 'X' to avoid explicit types.
-   This is mainly used set use 64-bits arguments in x32.   */
-#define TYPEFY(X, name) __typeof__ ((X) - (X)) name
-/* Explicit cast the argument to avoid integer from pointer warning on
-   x32.  */
-#define ARGIFY(X) ((__typeof__ ((X) - (X))) (X))
+/* NB: This also works when X is an array.  For an array X,  type of
+   (X) - (X) is ptrdiff_t, which is signed, since size of ptrdiff_t
+   == size of pointer, cast is a NOP.   */
+#define TYPEFY1(X) __typeof__ ((X) - (X))
+/* Explicit cast the argument.  */
+#define ARGIFY(X) ((TYPEFY1 (X)) (X))
+/* Create a variable 'name' based on type of variable 'X' to avoid
+   explicit types.  */
+#define TYPEFY(X, name) __typeof__ (ARGIFY (X)) name
 
 #undef INTERNAL_SYSCALL
 #define INTERNAL_SYSCALL(name, nr, args...)				\
diff --git a/sysdeps/unix/sysv/linux/x86_64/x32/sysdep.h b/sysdeps/unix/sysv/linux/x86_64/x32/sysdep.h
index 5bf9eed..a37d520 100644
--- a/sysdeps/unix/sysv/linux/x86_64/x32/sysdep.h
+++ b/sysdeps/unix/sysv/linux/x86_64/x32/sysdep.h
@@ -26,4 +26,20 @@
 #undef LO_HI_LONG
 #define LO_HI_LONG(val) (val)
 
+#ifndef __ASSEMBLER__
+# undef ARGIFY
+/* Enforce zero-extension for pointers and array system call arguments.
+   For integer types, extend to int64_t (the full register) using a
+   regular cast, resulting in zero or sign extension based on the
+   signedness of the original type.  */
+# define ARGIFY(X) \
+ ({									\
+    _Pragma ("GCC diagnostic push");					\
+    _Pragma ("GCC diagnostic ignored \"-Wpointer-to-int-cast\"");	\
+    (__builtin_classify_type (X) == 5					\
+     ? (uintptr_t) (X) : (int64_t) (X));				\
+    _Pragma ("GCC diagnostic pop");					\
+  })
+#endif	/* __ASSEMBLER__ */
+
 #endif /* linux/x86_64/x32/sysdep.h */