linux: Inline __syscall_internal_cancel and __syscall_cancelazanella/cancel-wrappers-inline

It improves some interception tools such as valgrind, however on
multithread the __syscall_cancel_arch is called.

The result libc.so has a slight larger code size:

ABI             master        patched         diff        increase
aarch64        1658673        1669121        10448           0.63%
x86_64         1976656        1985744         9088           0.46%
i686           2233622        2251130        17508           0.78%
powerpc64le    2382448        2396768        14320           0.60%

It mimics internally how cancellable entrypoints were implemented
before 89b53077d2a58f00e7debdfe58afabe953dac60d, where cancellation
handlign were done inline in the syscall wraper.

9 files changed, 105 insertions, 113 deletions

diff --git a/elf/Makefile b/elf/Makefile
index 3d60000..47c235c 100644
--- a/elf/Makefile
+++ b/elf/Makefile
@@ -1457,7 +1457,9 @@ $(objpfx)dl-allobjs.os: $(all-rtld-routines:%=$(objpfx)%.os)
 # when compiled for libc.
 rtld-stubbed-symbols = \
   __libc_assert_fail \
-  __syscall_cancel \
+  __syscall_cancel_arch \
+  __libc_single_threaded_internal \
+  __syscall_do_cancel \
   calloc \
   free \
   malloc \
diff --git a/nptl/cancellation.c b/nptl/cancellation.c
index 156e63d..7aa6ea7 100644
--- a/nptl/cancellation.c
+++ b/nptl/cancellation.c
@@ -19,66 +19,6 @@
 #include <stdlib.h>
 #include "pthreadP.h"
 
-/* Called by the INTERNAL_SYSCALL_CANCEL macro, check for cancellation and
-   returns the syscall value or its negative error code.  */
-long int
-__internal_syscall_cancel (__syscall_arg_t a1, __syscall_arg_t a2,
-			   __syscall_arg_t a3, __syscall_arg_t a4,
-			   __syscall_arg_t a5, __syscall_arg_t a6,
-			   __SYSCALL_CANCEL7_ARG_DEF
-			   __syscall_arg_t nr)
-{
-  long int result;
-  struct pthread *pd = THREAD_SELF;
-
-  /* If cancellation is not enabled, call the syscall directly and also
-     for thread terminatation to avoid call __syscall_do_cancel while
-     executing cleanup handlers.  */
-  int ch = atomic_load_relaxed (&pd->cancelhandling);
-  if (SINGLE_THREAD_P || !cancel_enabled (ch) || cancel_exiting (ch))
-    {
-      result = INTERNAL_SYSCALL_NCS_CALL (nr, a1, a2, a3, a4, a5, a6
-					  __SYSCALL_CANCEL7_ARCH_ARG7);
-      if (INTERNAL_SYSCALL_ERROR_P (result))
-	return -INTERNAL_SYSCALL_ERRNO (result);
-      return result;
-    }
-
-  /* Call the arch-specific entry points that contains the globals markers
-     to be checked by SIGCANCEL handler.  */
-  result = __syscall_cancel_arch (&pd->cancelhandling, nr, a1, a2, a3, a4, a5,
-			          a6 __SYSCALL_CANCEL7_ARCH_ARG7);
-
-  /* If the cancellable syscall was interrupted by SIGCANCEL and it has no
-     side-effect, cancel the thread if cancellation is enabled.  */
-  ch = atomic_load_relaxed (&pd->cancelhandling);
-  /* The behaviour here assumes that EINTR is returned only if there are no
-     visible side effects.  POSIX Issue 7 has not yet provided any stronger
-     language for close, and in theory the close syscall could return EINTR
-     and leave the file descriptor open (conforming and leaks).  It expects
-     that no such kernel is used with glibc.  */
-  if (result == -EINTR && cancel_enabled_and_canceled (ch))
-    __syscall_do_cancel ();
-
-  return result;
-}
-
-/* Called by the SYSCALL_CANCEL macro, check for cancellation and return the
-   syscall expected success value (usually 0) or, in case of failure, -1 and
-   sets errno to syscall return value.  */
-long int
-__syscall_cancel (__syscall_arg_t a1, __syscall_arg_t a2,
-		  __syscall_arg_t a3, __syscall_arg_t a4,
-		  __syscall_arg_t a5, __syscall_arg_t a6,
-		  __SYSCALL_CANCEL7_ARG_DEF __syscall_arg_t nr)
-{
-  int r = __internal_syscall_cancel (a1, a2, a3, a4, a5, a6,
-				     __SYSCALL_CANCEL7_ARG nr);
-  return __glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (r))
-	 ? SYSCALL_ERROR_LABEL (INTERNAL_SYSCALL_ERRNO (r))
-	 : r;
-}
-
 /* Called by __syscall_cancel_arch or function above start the thread
    cancellation.  */
 _Noreturn void
diff --git a/nptl/futex-internal.c b/nptl/futex-internal.c
index b81ca07..6d8ef73 100644
--- a/nptl/futex-internal.c
+++ b/nptl/futex-internal.c
@@ -17,7 +17,7 @@
    <https://www.gnu.org/licenses/>.  */
 
 #include <errno.h>
-#include <sysdep.h>
+#include <sysdep-cancel.h>
 #include <time.h>
 #include <futex-internal.h>
 #include <kernel-features.h>
diff --git a/nptl/sem_waitcommon.c b/nptl/sem_waitcommon.c
index 95450ee..0d59e81 100644
--- a/nptl/sem_waitcommon.c
+++ b/nptl/sem_waitcommon.c
@@ -18,7 +18,7 @@
 
 #include <kernel-features.h>
 #include <errno.h>
-#include <sysdep.h>
+#include <sysdep-cancel.h>
 #include <futex-internal.h>
 #include <internaltypes.h>
 #include <semaphore.h>
diff --git a/sysdeps/unix/sysdep.h b/sysdeps/unix/sysdep.h
index 2cc9872..8c1642f 100644
--- a/sysdeps/unix/sysdep.h
+++ b/sysdeps/unix/sysdep.h
@@ -154,42 +154,31 @@
 # define __SYSCALL_CANCEL7_ARG7
 # define __SYSCALL_CANCEL7_ARCH_ARG7
 #endif
-long int __internal_syscall_cancel (__syscall_arg_t a1, __syscall_arg_t a2,
-				    __syscall_arg_t a3, __syscall_arg_t a4,
-				    __syscall_arg_t a5, __syscall_arg_t a6,
-				    __SYSCALL_CANCEL7_ARG_DEF
-				    __syscall_arg_t nr) attribute_hidden;
-
-long int __syscall_cancel (__syscall_arg_t arg1, __syscall_arg_t arg2,
-			   __syscall_arg_t arg3, __syscall_arg_t arg4,
-			   __syscall_arg_t arg5, __syscall_arg_t arg6,
-			   __SYSCALL_CANCEL7_ARG_DEF
-			   __syscall_arg_t nr) attribute_hidden;
 
 #define __SYSCALL_CANCEL0(name)						\
-  __syscall_cancel (0, 0, 0, 0, 0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
+  syscall_cancel (0, 0, 0, 0, 0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __SYSCALL_CANCEL1(name, a1)					\
-  __syscall_cancel (__SSC (a1), 0, 0, 0, 0, 0,				\
-		    __SYSCALL_CANCEL7_ARG __NR_##name)
+  syscall_cancel (__SSC (a1), 0, 0, 0, 0, 0,				\
+		  __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __SYSCALL_CANCEL2(name, a1, a2) \
-  __syscall_cancel (__SSC (a1), __SSC (a2), 0, 0, 0, 0,			\
-		    __SYSCALL_CANCEL7_ARG __NR_##name)
+  syscall_cancel (__SSC (a1), __SSC (a2), 0, 0, 0, 0,			\
+		  __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __SYSCALL_CANCEL3(name, a1, a2, a3) \
-  __syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), 0, 0, 0,	\
-		    __SYSCALL_CANCEL7_ARG __NR_##name)
+  syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), 0, 0, 0,	\
+		  __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __SYSCALL_CANCEL4(name, a1, a2, a3, a4) \
-  __syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),			\
-		    __SSC(a4), 0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
+  syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),			\
+		  __SSC(a4), 0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __SYSCALL_CANCEL5(name, a1, a2, a3, a4, a5) \
-  __syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC(a4),	\
-		    __SSC (a5), 0, __SYSCALL_CANCEL7_ARG __NR_##name)
+  syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC(a4),	\
+		  __SSC (a5), 0, __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __SYSCALL_CANCEL6(name, a1, a2, a3, a4, a5, a6) \
-  __syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC (a4),	\
-		    __SSC (a5), __SSC (a6), __SYSCALL_CANCEL7_ARG	\
-		    __NR_##name)
+  syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC (a4),	\
+		  __SSC (a5), __SSC (a6), __SYSCALL_CANCEL7_ARG	\
+		  __NR_##name)
 #define __SYSCALL_CANCEL7(name, a1, a2, a3, a4, a5, a6, a7)		\
-  __syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC (a4),	\
-		    __SSC (a5), __SSC (a6), __SSC (a7), __NR_##name)
+  syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC (a4),	\
+		  __SSC (a5), __SSC (a6), __SSC (a7), __NR_##name)
 
 #define __SYSCALL_CANCEL_NARGS_X(a,b,c,d,e,f,g,h,n,...) n
 #define __SYSCALL_CANCEL_NARGS(...) \
@@ -206,33 +195,33 @@ long int __syscall_cancel (__syscall_arg_t arg1, __syscall_arg_t arg2,
   __SYSCALL_CANCEL_DISP (__SYSCALL_CANCEL, __VA_ARGS__)
 
 #define __INTERNAL_SYSCALL_CANCEL0(name)				\
-  __internal_syscall_cancel (0, 0, 0, 0, 0, 0, __SYSCALL_CANCEL7_ARG	\
+  internal_syscall_cancel (0, 0, 0, 0, 0, 0, __SYSCALL_CANCEL7_ARG	\
 			     __NR_##name)
 #define __INTERNAL_SYSCALL_CANCEL1(name, a1)				\
-  __internal_syscall_cancel (__SSC (a1), 0, 0, 0, 0, 0,			\
-			     __SYSCALL_CANCEL7_ARG __NR_##name)
+  internal_syscall_cancel (__SSC (a1), 0, 0, 0, 0, 0,			\
+			   __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __INTERNAL_SYSCALL_CANCEL2(name, a1, a2)			\
-  __internal_syscall_cancel (__SSC (a1), __SSC (a2), 0, 0, 0, 0,	\
-			     __SYSCALL_CANCEL7_ARG __NR_##name)
+  internal_syscall_cancel (__SSC (a1), __SSC (a2), 0, 0, 0, 0,	\
+			   __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __INTERNAL_SYSCALL_CANCEL3(name, a1, a2, a3)			\
-  __internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), 0,	\
-			     0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
+  internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), 0,	\
+			   0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __INTERNAL_SYSCALL_CANCEL4(name, a1, a2, a3, a4)		\
-  __internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),	\
-			     __SSC(a4), 0, 0,				\
-			     __SYSCALL_CANCEL7_ARG __NR_##name)
+  internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),	\
+			   __SSC(a4), 0, 0,				\
+			   __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __INTERNAL_SYSCALL_CANCEL5(name, a1, a2, a3, a4, a5)		\
-  __internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),	\
-			     __SSC(a4), __SSC (a5), 0,			\
-			     __SYSCALL_CANCEL7_ARG __NR_##name)
+  internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),	\
+			   __SSC(a4), __SSC (a5), 0,			\
+			   __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __INTERNAL_SYSCALL_CANCEL6(name, a1, a2, a3, a4, a5, a6)	\
-  __internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),	\
-			     __SSC (a4), __SSC (a5), __SSC (a6),	\
-			     __SYSCALL_CANCEL7_ARG __NR_##name)
+  internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),	\
+			   __SSC (a4), __SSC (a5), __SSC (a6),	\
+			   __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __INTERNAL_SYSCALL_CANCEL7(name, a1, a2, a3, a4, a5, a6, a7) \
-  __internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),     \
-			     __SSC (a4), __SSC (a5), __SSC (a6),     \
-			     __SSC (a7), __NR_##name)
+  internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),     \
+			   __SSC (a4), __SSC (a5), __SSC (a6),     \
+			   __SSC (a7), __NR_##name)
 
 /* Issue a cancellable syscall defined by syscall number NAME plus any other
    argument required.  If an error occurs its value is returned as an negative
diff --git a/sysdeps/unix/sysv/linux/epoll_pwait2.c b/sysdeps/unix/sysv/linux/epoll_pwait2.c
index bbb5f04..a58a359 100644
--- a/sysdeps/unix/sysv/linux/epoll_pwait2.c
+++ b/sysdeps/unix/sysv/linux/epoll_pwait2.c
@@ -17,7 +17,7 @@
    <https://www.gnu.org/licenses/>.  */
 
 #include <sys/epoll.h>
-#include <sysdep.h>
+#include <sysdep-cancel.h>
 
 int
 __epoll_pwait2_time64 (int fd, struct epoll_event *ev, int maxev,
diff --git a/sysdeps/unix/sysv/linux/recvmmsg.c b/sysdeps/unix/sysv/linux/recvmmsg.c
index 3b00bb2..7ce2de6 100644
--- a/sysdeps/unix/sysv/linux/recvmmsg.c
+++ b/sysdeps/unix/sysv/linux/recvmmsg.c
@@ -16,7 +16,7 @@
    <https://www.gnu.org/licenses/>.  */
 
 #include <sys/socket.h>
-#include <sysdep.h>
+#include <sysdep-cancel.h>
 #include <socketcall.h>
 
 static int
diff --git a/sysdeps/unix/sysv/linux/sigtimedwait.c b/sysdeps/unix/sysv/linux/sigtimedwait.c
index ad76628..5781c63 100644
--- a/sysdeps/unix/sysv/linux/sigtimedwait.c
+++ b/sysdeps/unix/sysv/linux/sigtimedwait.c
@@ -16,7 +16,7 @@
    <https://www.gnu.org/licenses/>.  */
 
 #include <signal.h>
-#include <sysdep.h>
+#include <sysdep-cancel.h>
 
 int
 __sigtimedwait64 (const sigset_t *set, siginfo_t *info,
diff --git a/sysdeps/unix/sysv/linux/sysdep-cancel.h b/sysdeps/unix/sysv/linux/sysdep-cancel.h
index 96c9a90..6e3f045 100644
--- a/sysdeps/unix/sysv/linux/sysdep-cancel.h
+++ b/sysdeps/unix/sysv/linux/sysdep-cancel.h
@@ -1,4 +1,4 @@
-/* Single-thread optimization definitions.  Linux version.
+/* Cancellable syscall definitions  Linux version.
    Copyright (C) 2017-2025 Free Software Foundation, Inc.
 
    This file is part of the GNU C Library.
@@ -21,5 +21,66 @@
 #define _SYSDEP_CANCEL_H
 
 #include <sysdep.h>
+#include "pthreadP.h"
+
+/* Called by the INTERNAL_SYSCALL_CANCEL macro, check for cancellation and
+   returns the syscall value or its negative error code.  */
+static __always_inline long int
+internal_syscall_cancel (__syscall_arg_t a1, __syscall_arg_t a2,
+			 __syscall_arg_t a3, __syscall_arg_t a4,
+			 __syscall_arg_t a5, __syscall_arg_t a6,
+			 __SYSCALL_CANCEL7_ARG_DEF
+			 __syscall_arg_t nr)
+{
+  long int result;
+  struct pthread *pd = THREAD_SELF;
+
+  /* If cancellation is not enabled, call the syscall directly and also
+     for thread terminatation to avoid call __syscall_do_cancel while
+     executing cleanup handlers.  */
+  int ch = atomic_load_relaxed (&pd->cancelhandling);
+  if (SINGLE_THREAD_P || !cancel_enabled (ch) || cancel_exiting (ch))
+    {
+      result = INTERNAL_SYSCALL_NCS_CALL (nr, a1, a2, a3, a4, a5, a6
+					  __SYSCALL_CANCEL7_ARCH_ARG7);
+      if (INTERNAL_SYSCALL_ERROR_P (result))
+	return -INTERNAL_SYSCALL_ERRNO (result);
+      return result;
+    }
+
+  /* Call the arch-specific entry points that contains the globals markers
+     to be checked by SIGCANCEL handler.  */
+  result = __syscall_cancel_arch (&pd->cancelhandling, nr, a1, a2, a3, a4, a5,
+			          a6 __SYSCALL_CANCEL7_ARCH_ARG7);
+
+  /* If the cancellable syscall was interrupted by SIGCANCEL and it has no
+     side-effect, cancel the thread if cancellation is enabled.  */
+  ch = atomic_load_relaxed (&pd->cancelhandling);
+  /* The behaviour here assumes that EINTR is returned only if there are no
+     visible side effects.  POSIX Issue 7 has not yet provided any stronger
+     language for close, and in theory the close syscall could return EINTR
+     and leave the file descriptor open (conforming and leaks).  It expects
+     that no such kernel is used with glibc.  */
+  if (result == -EINTR && cancel_enabled_and_canceled (ch))
+    __syscall_do_cancel ();
+
+  return result;
+}
+
+/* Called by the SYSCALL_CANCEL macro, check for cancellation and return the
+   syscall expected success value (usually 0) or, in case of failure, -1 and
+   sets errno to syscall return value.  */
+static __always_inline long int
+syscall_cancel (__syscall_arg_t a1, __syscall_arg_t a2,
+		__syscall_arg_t a3, __syscall_arg_t a4,
+		__syscall_arg_t a5, __syscall_arg_t a6,
+		__SYSCALL_CANCEL7_ARG_DEF __syscall_arg_t nr)
+{
+  int r = internal_syscall_cancel (a1, a2, a3, a4, a5, a6,
+				   __SYSCALL_CANCEL7_ARG nr);
+  return __glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (r))
+	 ? SYSCALL_ERROR_LABEL (INTERNAL_SYSCALL_ERRNO (r))
+	 : r;
+}
 
 #endif