path: root/sysdeps/aarch64/morello/memcpy.S

/* Copyright (C) 2022 Free Software Foundation, Inc.

   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library.  If not, see
   <https://www.gnu.org/licenses/>.  */

#include <sysdep.h>

/* Assumptions:
 *
 * ARMv8-a, AArch64, Morello, Advanced SIMD, unaligned accesses.
 *
 */

#ifndef MEMMOVE
# define MEMMOVE memmove
#endif
#ifndef MEMCPY
# define MEMCPY memcpy
#endif

#define xdstin	x0
#define xsrc	x1
#define count	x2
#define xsrcend	x4
#define A_l	x6
#define A_lw	w6
#define A_h	x7
#define B_l	x8
#define B_lw	w8
#define B_h	x9
#define C_lw	w10
#define auoff   x14
#define cap_count   x15
#define tmp1	x16
#define tmp2	x17

#if defined(__CHERI_PURE_CAPABILITY__)
#define dstin	c0
#define src	c1
#define dst	c3
#define srcend	c4
#define dstend	c5
#define tmp1_ptr c16
#else
#define dstin	x0
#define src	x1
#define dst	x3
#define srcend	x4
#define dstend	x5
#define tmp1_ptr x16
#endif

#define A_q	q0
#define B_q	q1
#define C_q	q2
#define D_q	q3
#define E_q	q4
#define F_q	q5
#define G_q	q6
#define H_q	q7

#define A_cap   c6
#define B_cap   c7
#define C_cap   c8
#define D_cap   c9
#define E_cap   c10
#define F_cap   c11
#define G_cap   c12
#define H_cap   c13


/* This algorithm has not been benchmarked. It's derived
   from the base aarch64 one with small changes to account
   for copying tags.

   1. We're copying less than 16 bytes, so no capabilities.
      Use the traditional code path for these.
   2. src mod 16 != dst mode 16. We're not copying capabilities,
      so again use the traditional memcpy.
   3. We're copying more than 8 capabilities plus the head and tail.
    a. No overlap, use forward copy
    b. Overlap, use backward copy
   4. We're copying 0..8 capabilities
    a. No capabilities to copy. This means we are copying 16..30 bytes.
       Use the existing code path to do this from the original algorithm.
    b. Copying 1..2 capabilities plus the head and tail
       Use a branchless sequence.
    c. Copying 3..4 capabilities plus the head and tail
       Use a branchless sequence.
    d. Copying 5..8 capabilities plus the head and tail
       Use a branchless sequence.
 */

ENTRY (MEMCPY)
	PTR_ARG (0)
	PTR_ARG (1)
	SIZE_ARG (2)
	add	srcend, src, count
	add	dstend, dstin, count

	/* Copies of less than 16 bytes don't use capabilities. */
	cmp	count, 16
	b.lo	L(copy16)

	/* If src mod 16 != dst mod 16 we're not transfering tags. */
	and	tmp1, xsrc, 15
	and	tmp2, xdstin, 15
	cmp	tmp1, tmp2
	b.ne	L(memcpy_nocap)

	/* Get the number of capabilities that we need to store. */
	neg	tmp2, tmp1
	add	tmp2, tmp2, 16
	and	auoff, tmp2, 15

	sub	cap_count, count, auoff
	lsr	cap_count, cap_count, 4

	cmp	cap_count, 8
	b.hi	L(copy_long_cap)
	cmp	cap_count, 2
	b.hi	L(copy32_128_cap)

	/* Copy 0..2 capabilities using a branchless sequence. */
	cbz	cap_count, L(copy32)
	ldr	A_q, [src]
	ldr	B_q, [srcend, -16]
	add	src, src, auoff /* align up src to 16 bytes */
#if defined(__CHERI_PURE_CAPABILITY__)
	alignd	srcend, srcend, 4
#else
	bic	srcend, srcend, 15
#endif
	ldr	A_cap, [src]
	ldr	B_cap, [srcend, -16]
	str	A_q, [dstin]
	str	B_q, [dstend, -16]
	add	tmp1_ptr, dstin, auoff /* align up dstin to 16 bytes */
#if defined (__CHERI_PURE_CAPABILITY__)
	alignd	dstend, dstend, 4
#else
	bic	dstend, dstend, 15
#endif
	str	A_cap, [tmp1_ptr]
	str	B_cap, [dstend, -16]
	ret

	.p2align 4
L(copy32_128_cap):
	cmp	cap_count, 4
	b.hi	L(copy128_cap)
	/* Copy 3..4 capabilties using a branchless sequence. */
	ldr	A_q, [src]
	ldr	B_q, [srcend, -16]
	add	src, src, auoff /* align up src to 16 bytes */
#if defined (__CHERI_PURE_CAPABILITY__)
	alignd	srcend, srcend, 4
#else
	bic	srcend, srcend, 15
#endif
	ldp	A_cap, B_cap, [src]
	ldp	C_cap, D_cap, [srcend, -32]
	str	A_q, [dstin]
	str	B_q, [dstend, -16]
	add	tmp1_ptr, dstin, auoff /* align up dstin to 16 bytes */
#if defined (__CHERI_PURE_CAPABILITY__)
	alignd  dstend, dstend, 4
#else
	bic	dstend, dstend, 15
#endif
	stp	A_cap, B_cap, [tmp1_ptr]
	stp	C_cap, D_cap, [dstend, -32]
	ret

	.p2align 4
L(copy128_cap):
	/* Copy 5..8 capabilities using a branchless sequence. */
	ldr	A_q, [src]
	ldr	B_q, [srcend, -16]
	add	src, src, auoff /* align up src to 16 bytes */
#if defined (__CHERI_PURE_CAPABILITY__)
	alignd	srcend, srcend, 4
#else
	bic	srcend, srcend, 15
#endif
	ldp	A_cap, B_cap, [src]
	ldp	C_cap, D_cap, [src, 32]
	ldp	E_cap, F_cap, [srcend, -32]
	ldp	G_cap, H_cap, [srcend, -64]
	str	A_q, [dstin]
	str	B_q, [dstend, -16]
	add	tmp1_ptr, dstin, auoff /* align up src to 16 bytes */
#if defined (__CHERI_PURE_CAPABILITY__)
	alignd	dstend, dstend, 4
#else
	bic	dstend, dstend, 15
#endif
	stp	A_cap, B_cap, [tmp1_ptr]
	stp	C_cap, D_cap, [tmp1_ptr, 32]
	stp	E_cap, F_cap, [dstend, -32]
	stp	G_cap, H_cap, [dstend, -64]
	ret

L(copy_long_cap):
	/* Use backwards copy if there is an overlap.  */
	sub	tmp1, xdstin, xsrc
	cmp	tmp1, count
	b.lo	L(copy_long_backwards_cap)

	/* Copy 16 bytes and then align src to 16-byte alignment.  */
	ldr	D_q, [src]
	ldr	E_cap, [src, auoff]
	and	tmp1, xsrc, 15
#if defined(__CHERI_PURE_CAPABILITY__)
	alignd	src, src, 4
	neg	tmp2, tmp1
	add	dst, dstin, tmp2
#else
	bic	src, src, 15
	sub	dst, dstin, tmp1
#endif
	add	count, count, tmp1	/* Count is now 16 too large.  */
	ldp	A_cap, B_cap, [src, 16]
	str	D_q, [dstin]
	str	E_cap, [dstin, auoff]
	ldp	C_cap, D_cap, [src, 48]
	subs	count, count, 128 + 16	/* Test and readjust count.  */
	b.ls	L(copy64_from_end_cap)
L(loop64_cap):
	stp	A_cap, B_cap, [dst, 16]
	ldp	A_cap, B_cap, [src, 80]
	stp	C_cap, D_cap, [dst, 48]
	ldp	C_cap, D_cap, [src, 112]
	add	src, src, 64
	add	dst, dst, 64
	subs	count, count, 64
	b.hi	L(loop64_cap)

	/* Write the last iteration and copy the last 16-byte aligned 64 byte block
	   from the end and the tail.  */
L(copy64_from_end_cap):
	ldr	A_q, [srcend, -16]
#if defined(__CHERI_PURE_CAPABILITY__)
	alignd	srcend, srcend, 4
	alignd	tmp1_ptr, dstend, 4
#else
	bic	srcend, srcend, 15
	bic	tmp1_ptr, dstend, 15
#endif
	ldp	E_cap, F_cap, [srcend, -64]
	stp	A_cap, B_cap, [dst, 16]
	ldp	A_cap, B_cap, [srcend, -32]
	stp	C_cap, D_cap, [dst, 48]
	stp	E_cap, F_cap, [tmp1_ptr, -64]
	str	A_q, [dstend, -16]
	stp	A_cap, B_cap, [tmp1_ptr, -32]
	ret

L(copy_long_backwards_cap):
	cbz	tmp1, L(copy0)
	ldr	D_q, [srcend, -16]
	and	tmp1, xsrcend, 15
#if defined(__CHERI_PURE_CAPABILITY__)
	alignd	srcend, srcend, 4
	neg	tmp2, tmp1
	add	count, count, tmp2
#else
	bic	srcend, srcend, 15
	sub	count, count, tmp1
#endif
	ldp	A_cap, B_cap, [srcend, -32]
	str	D_q, [dstend, -16]
	ldp	C_cap, D_cap, [srcend, -64]
#if defined(__CHERI_PURE_CAPABILITY__)
	add	dstend, dstend, tmp2  /* tmp1 was negated above to tmp2. */
#else
	sub	dstend, dstend, tmp1
#endif
	subs	count, count, 128
	b.ls	L(copy64_from_start)

L(loop64_backwards_cap):
	str	B_cap, [dstend, -16]
	str	A_cap, [dstend, -32]
	ldp	A_cap, B_cap, [srcend, -96]
	str	D_cap, [dstend, -48]
	str	C_cap, [dstend, -64]!
	ldp	C_cap, D_cap, [srcend, -128]
	sub	srcend, srcend, 64
	subs	count, count, 64
	b.hi	L(loop64_backwards_cap)

	/* Write the last iteration and copy 64 bytes from the start.  */
L(copy64_from_start_cap):
	ldr	A_q, [src]
	add	src, src, auoff /* align up src to 16 bytes */
	add	tmp1_ptr, dstin, auoff /* align up dstin to 16 bytes */
	ldp	E_cap, F_cap, [src, 32]
	stp	A_cap, B_cap, [dstend, -32]
	ldp	A_cap, B_cap, [src]
	stp	C_cap, D_cap, [dstend, -64]
	stp	E_cap, F_cap, [tmp1_ptr, 32]
	str	A_q, [dstin]
	stp	A_cap, B_cap, [tmp1_ptr]
	ret

L(memcpy_nocap):
	cmp	count, 128
	b.hi	L(copy_long)
	cmp	count, 32
	b.hi	L(copy32_128)

L(copy32):
	/* Small copies: 16..32 bytes.  */
        /* Note the 0..15 case is already handled at the start of memcpy */
	ldr	A_q, [src]
	ldr	B_q, [srcend, -16]
	str	A_q, [dstin]
	str	B_q, [dstend, -16]
	ret

	/* Copy 8-15 bytes.  */
L(copy16):
	tbz	count, 3, L(copy8)
	ldr	A_l, [src]
	ldr	A_h, [srcend, -8]
	str	A_l, [dstin]
	str	A_h, [dstend, -8]
	ret

	.p2align 3
	/* Copy 4-7 bytes.  */
L(copy8):
	tbz	count, 2, L(copy4)
	ldr	A_lw, [src]
	ldr	B_lw, [srcend, -4]
	str	A_lw, [dstin]
	str	B_lw, [dstend, -4]
	ret

	/* Copy 0..3 bytes using a branchless sequence.  */
L(copy4):
	cbz	count, L(copy0)
	lsr	tmp1, count, 1
	ldrb	A_lw, [src]
	ldrb	C_lw, [srcend, -1]
	ldrb	B_lw, [src, tmp1]
	strb	A_lw, [dstin]
	strb	B_lw, [dstin, tmp1]
	strb	C_lw, [dstend, -1]
L(copy0):
	ret

	.p2align 4
	/* Medium copies: 33..128 bytes.  */
L(copy32_128):
	ldp	A_q, B_q, [src]
	ldp	C_q, D_q, [srcend, -32]
	cmp	count, 64
	b.hi	L(copy128)
	stp	A_q, B_q, [dstin]
	stp	C_q, D_q, [dstend, -32]
	ret

	.p2align 4
	/* Copy 65..128 bytes.  */
L(copy128):
	ldp	E_q, F_q, [src, 32]
	cmp	count, 96
	b.ls	L(copy96)
	ldp	G_q, H_q, [srcend, -64]
	stp	G_q, H_q, [dstend, -64]
L(copy96):
	stp	A_q, B_q, [dstin]
	stp	E_q, F_q, [dstin, 32]
	stp	C_q, D_q, [dstend, -32]
	ret

	/* Copy more than 128 bytes.  */
L(copy_long):
	/* Use backwards copy if there is an overlap.  */
	sub	tmp1, xdstin, xsrc
	cmp	tmp1, count
	b.lo	L(copy_long_backwards)

	/* Copy 16 bytes and then align src to 16-byte alignment.  */
	ldr	D_q, [src]
	and	tmp1, xsrc, 15
#if defined(__CHERI_PURE_CAPABILITY__)
	alignd	src, src, 4
	neg	tmp2, tmp1
	add	dst, dstin, tmp2
#else
	bic	src, src, 15
	sub	dst, dstin, tmp1
#endif
	add	count, count, tmp1	/* Count is now 16 too large.  */
	ldp	A_q, B_q, [src, 16]
	str	D_q, [dstin]
	ldp	C_q, D_q, [src, 48]
	subs	count, count, 128 + 16	/* Test and readjust count.  */
	b.ls	L(copy64_from_end)
L(loop64):
	stp	A_q, B_q, [dst, 16]
	ldp	A_q, B_q, [src, 80]
	stp	C_q, D_q, [dst, 48]
	ldp	C_q, D_q, [src, 112]
	add	src, src, 64
	add	dst, dst, 64
	subs	count, count, 64
	b.hi	L(loop64)

	/* Write the last iteration and copy 64 bytes from the end.  */
L(copy64_from_end):
	ldp	E_q, F_q, [srcend, -64]
	stp	A_q, B_q, [dst, 16]
	ldp	A_q, B_q, [srcend, -32]
	stp	C_q, D_q, [dst, 48]
	stp	E_q, F_q, [dstend, -64]
	stp	A_q, B_q, [dstend, -32]
	ret

	/* Large backwards copy for overlapping copies.
	   Copy 16 bytes and then align srcend to 16-byte alignment.  */
L(copy_long_backwards):
	cbz	tmp1, L(copy0)
	ldr	D_q, [srcend, -16]
	and	tmp1, xsrcend, 15
#if defined(__CHERI_PURE_CAPABILITY__)
	alignd	srcend, srcend, 4
	neg	tmp2, tmp1
	add	count, count, tmp2
#else
	bic	srcend, srcend, 15
	sub	count, count, tmp1
#endif
	ldp	A_q, B_q, [srcend, -32]
	str	D_q, [dstend, -16]
	ldp	C_q, D_q, [srcend, -64]
#if defined(__CHERI_PURE_CAPABILITY__)
	add	dstend, dstend, tmp2  /* tmp1 was already negated above. */
#else
	sub	dstend, dstend, tmp1
#endif
	subs	count, count, 128
	b.ls	L(copy64_from_start)

L(loop64_backwards):
	str	B_q, [dstend, -16]
	str	A_q, [dstend, -32]
	ldp	A_q, B_q, [srcend, -96]
	str	D_q, [dstend, -48]
	str	C_q, [dstend, -64]!
	ldp	C_q, D_q, [srcend, -128]
	sub	srcend, srcend, 64
	subs	count, count, 64
	b.hi	L(loop64_backwards)

	/* Write the last iteration and copy 64 bytes from the start.  */
L(copy64_from_start):
	ldp	E_q, F_q, [src, 32]
	stp	A_q, B_q, [dstend, -32]
	ldp	A_q, B_q, [src]
	stp	C_q, D_q, [dstend, -64]
	stp	E_q, F_q, [dstin, 32]
	stp	A_q, B_q, [dstin]
	ret

END (MEMCPY)

weak_alias (MEMCPY, MEMMOVE)
libc_hidden_builtin_def (MEMCPY)
libc_hidden_builtin_def (MEMMOVE)