diff options
Diffstat (limited to 'sysdeps/aarch64/memcpy.S')
| -rw-r--r-- | sysdeps/aarch64/memcpy.S | 82 |
1 files changed, 47 insertions, 35 deletions
diff --git a/sysdeps/aarch64/memcpy.S b/sysdeps/aarch64/memcpy.S index 7e1163e6a0..e7b48b8a0b 100644 --- a/sysdeps/aarch64/memcpy.S +++ b/sysdeps/aarch64/memcpy.S @@ -41,17 +41,19 @@ #define C_h x11 #define D_l x12 #define D_h x13 -#define E_l src -#define E_h count -#define F_l srcend -#define F_h dst +#define E_l x14 +#define E_h x15 +#define F_l x16 +#define F_h x17 #define G_l count #define G_h dst +#define H_l src +#define H_h srcend #define tmp1 x14 -/* Copies are split into 3 main cases: small copies of up to 16 bytes, - medium copies of 17..96 bytes which are fully unrolled. Large copies - of more than 96 bytes align the destination and use an unrolled loop +/* Copies are split into 3 main cases: small copies of up to 32 bytes, + medium copies of 33..128 bytes which are fully unrolled. Large copies + of more than 128 bytes align the destination and use an unrolled loop processing 64 bytes per iteration. In order to share code with memmove, small and medium copies read all data before writing, allowing any kind of overlap. So small, medium @@ -73,7 +75,7 @@ ENTRY_ALIGN (MEMMOVE, 6) DELOUSE (2) sub tmp1, dstin, src - cmp count, 96 + cmp count, 128 ccmp tmp1, count, 2, hi b.lo L(move_long) @@ -89,31 +91,39 @@ ENTRY (MEMCPY) prfm PLDL1KEEP, [src] add srcend, src, count add dstend, dstin, count - cmp count, 16 - b.ls L(copy16) - cmp count, 96 + cmp count, 32 + b.ls L(copy32) + cmp count, 128 b.hi L(copy_long) - /* Medium copies: 17..96 bytes. */ - sub tmp1, count, 1 + /* Medium copies: 33..128 bytes. */ ldp A_l, A_h, [src] - tbnz tmp1, 6, L(copy96) - ldp D_l, D_h, [srcend, -16] - tbz tmp1, 5, 1f ldp B_l, B_h, [src, 16] ldp C_l, C_h, [srcend, -32] + ldp D_l, D_h, [srcend, -16] + cmp count, 64 + b.hi L(copy128) + stp A_l, A_h, [dstin] stp B_l, B_h, [dstin, 16] stp C_l, C_h, [dstend, -32] -1: - stp A_l, A_h, [dstin] stp D_l, D_h, [dstend, -16] ret .p2align 4 - /* Small copies: 0..16 bytes. */ -L(copy16): - cmp count, 8 + /* Small copies: 0..32 bytes. */ +L(copy32): + /* 16-32 bytes. */ + cmp count, 16 b.lo 1f + ldp A_l, A_h, [src] + ldp B_l, B_h, [srcend, -16] + stp A_l, A_h, [dstin] + stp B_l, B_h, [dstend, -16] + ret + .p2align 4 +1: + /* 8-15 bytes. */ + tbz count, 3, 1f ldr A_l, [src] ldr A_h, [srcend, -8] str A_l, [dstin] @@ -121,6 +131,7 @@ L(copy16): ret .p2align 4 1: + /* 4-7 bytes. */ tbz count, 2, 1f ldr A_lw, [src] ldr A_hw, [srcend, -4] @@ -142,24 +153,25 @@ L(copy16): 2: ret .p2align 4 - /* Copy 64..96 bytes. Copy 64 bytes from the start and - 32 bytes from the end. */ -L(copy96): - ldp B_l, B_h, [src, 16] - ldp C_l, C_h, [src, 32] - ldp D_l, D_h, [src, 48] - ldp E_l, E_h, [srcend, -32] - ldp F_l, F_h, [srcend, -16] + /* Copy 65..128 bytes. Copy 64 bytes from the start and + 64 bytes from the end. */ +L(copy128): + ldp E_l, E_h, [src, 32] + ldp F_l, F_h, [src, 48] + ldp G_l, G_h, [srcend, -64] + ldp H_l, H_h, [srcend, -48] stp A_l, A_h, [dstin] stp B_l, B_h, [dstin, 16] - stp C_l, C_h, [dstin, 32] - stp D_l, D_h, [dstin, 48] - stp E_l, E_h, [dstend, -32] - stp F_l, F_h, [dstend, -16] + stp E_l, E_h, [dstin, 32] + stp F_l, F_h, [dstin, 48] + stp G_l, G_h, [dstend, -64] + stp H_l, H_h, [dstend, -48] + stp C_l, C_h, [dstend, -32] + stp D_l, D_h, [dstend, -16] ret /* Align DST to 16 byte alignment so that we don't cross cache line - boundaries on both loads and stores. There are at least 96 bytes + boundaries on both loads and stores. There are at least 128 bytes to copy, so copy 16 bytes unaligned and then align. The loop copies 64 bytes per iteration and prefetches one iteration ahead. */ @@ -215,7 +227,7 @@ L(move_long): add dstend, dstin, count /* Align dstend to 16 byte alignment so that we don't cross cache line - boundaries on both loads and stores. There are at least 96 bytes + boundaries on both loads and stores. There are at least 128 bytes to copy, so copy 16 bytes unaligned and then align. The loop copies 64 bytes per iteration and prefetches one iteration ahead. */ |