diff options
| -rw-r--r-- | sysdeps/powerpc/powerpc32/sysdep.h | 15 | ||||
| -rw-r--r-- | sysdeps/powerpc/powerpc64/sysdep.h | 120 | ||||
| -rw-r--r-- | sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S | 9 | ||||
| -rw-r--r-- | sysdeps/unix/sysv/linux/powerpc/powerpc64/vfork.S | 6 | ||||
| -rw-r--r-- | sysdeps/unix/sysv/linux/powerpc/syscall.S | 11 | ||||
| -rw-r--r-- | sysdeps/unix/sysv/linux/powerpc/sysdep.h | 93 |
6 files changed, 214 insertions, 40 deletions
diff --git a/sysdeps/powerpc/powerpc32/sysdep.h b/sysdeps/powerpc/powerpc32/sysdep.h index 829eec266a..a3fc1f0ca7 100644 --- a/sysdeps/powerpc/powerpc32/sysdep.h +++ b/sysdeps/powerpc/powerpc32/sysdep.h @@ -92,7 +92,10 @@ GOT_LABEL: ; \ #define DO_CALL(syscall) \ li 0,syscall; \ - sc + DO_CALL_SC + +#define DO_CALL_SC \ + sc #undef JUMPTARGET #ifdef PIC @@ -106,14 +109,20 @@ GOT_LABEL: ; \ # define HIDDEN_JUMPTARGET(name) __GI_##name##@local #endif +#define TAIL_CALL_SYSCALL_ERROR \ + b __syscall_error@local + #define PSEUDO(name, syscall_name, args) \ .section ".text"; \ ENTRY (name) \ DO_CALL (SYS_ify (syscall_name)); +#define RET_SC \ + bnslr+; + #define PSEUDO_RET \ - bnslr+; \ - b __syscall_error@local + RET_SC; \ + TAIL_CALL_SYSCALL_ERROR #define ret PSEUDO_RET #undef PSEUDO_END diff --git a/sysdeps/powerpc/powerpc64/sysdep.h b/sysdeps/powerpc/powerpc64/sysdep.h index d557098898..f6a0619743 100644 --- a/sysdeps/powerpc/powerpc64/sysdep.h +++ b/sysdeps/powerpc/powerpc64/sysdep.h @@ -17,6 +17,7 @@ <https://www.gnu.org/licenses/>. */ #include <sysdeps/powerpc/sysdep.h> +#include <tls.h> #ifdef __ASSEMBLER__ @@ -263,10 +264,80 @@ LT_LABELSUFFIX(name,_name_end): ; \ TRACEBACK_MASK(name,mask); \ END_2(name) -#define DO_CALL(syscall) \ - li 0,syscall; \ +/* We will allocate a new frame to save LR and the non-volatile register used to + read the TCB when checking for scv support on syscall code. We actually just + need the minimum frame size plus room for 1 reg (8 bytes). But the ABI + mandates stack frames should be aligned at 16 Bytes, so we end up allocating + a bit more space then what will actually be used. */ +#define SCV_FRAME_SIZE (FRAME_MIN_SIZE+16) +#define SCV_FRAME_NVOLREG_SAVE FRAME_MIN_SIZE + +/* Allocate frame and save register */ +#define NVOLREG_SAVE \ + stdu r1,-SCV_FRAME_SIZE(r1); \ + std r31,SCV_FRAME_NVOLREG_SAVE(r1); \ + cfi_adjust_cfa_offset(SCV_FRAME_SIZE); + +/* Restore register and destroy frame */ +#define NVOLREG_RESTORE \ + ld r31,SCV_FRAME_NVOLREG_SAVE(r1); \ + addi r1,r1,SCV_FRAME_SIZE; \ + cfi_adjust_cfa_offset(-SCV_FRAME_SIZE); + +/* Check PPC_FEATURE2_SCV bit from hwcap2 in the TCB. If it is not set, scv is + not available, then go to JUMPFALSE (label given by the macro's caller). We + save the value we read from the TCB in a non-volatile register so we can + reuse it later when exiting from the syscall in PSEUDO_RET. Note that for + the static case we need an extra check to guarantee the thread pointer has + already been initialized, otherwise we may try to access an invalid address + if a syscall is called before the TLS has been setup. */ + .macro CHECK_SCV_SUPPORT REG JUMPFALSE + +#ifndef SHARED + /* Check if thread pointer has already been setup. */ + cmpdi r13,0 + beq \JUMPFALSE +#endif + + /* Read PPC_FEATURE2_SCV from TCB and store it in REG */ + ld \REG,TCB_HWCAP(PT_THREAD_POINTER) + andis. \REG,\REG,PPC_FEATURE2_SCV>>16 + + beq \JUMPFALSE + .endm + +#if IS_IN(rtld) +# define DO_CALL(syscall) \ + li r0,syscall; \ + DO_CALL_SC +#else +/* Before doing the syscall, check if we can use scv. scv is supported by P9 + and later with Linux v5.9 and later. If so, use it. Otherwise, fallback to + sc. We use a non-volatile register to save hwcap2 from the TCB, so we need + to save its content beforehand. */ +# define DO_CALL(syscall) \ + li r0,syscall; \ + NVOLREG_SAVE; \ + CHECK_SCV_SUPPORT r31 0f; \ + DO_CALL_SCV; \ + b 1f; \ +0: DO_CALL_SC; \ +1: +#endif /* IS_IN(rtld) */ + +/* DO_CALL_SC and DO_CALL_SCV expect the syscall number to be in r0. */ +#define DO_CALL_SC \ sc +#define DO_CALL_SCV \ + mflr r9; \ + std r9,FRAME_LR_SAVE(r1); \ + cfi_offset(lr,FRAME_LR_SAVE); \ + scv 0; \ + ld r9,FRAME_LR_SAVE(r1); \ + mtlr r9; \ + cfi_restore(lr); + /* ppc64 is always PIC */ #undef JUMPTARGET #define JUMPTARGET(name) FUNC_LABEL(name) @@ -304,9 +375,32 @@ LT_LABELSUFFIX(name,_name_end): ; \ .endif #endif -#define PSEUDO_RET \ - bnslr+; \ +#if IS_IN(rtld) +# define PSEUDO_RET \ + RET_SC; \ TAIL_CALL_SYSCALL_ERROR +#else +/* This should only be called after a DO_CALL. In such cases, r31 contains the + value of PPC_FEATURE2_SCV read from hwcap2 by CHECK_SCV_SUPPORT. If it is + set, we know we have entered the kernel using scv, so handle the return code + accordingly. */ +# define PSEUDO_RET \ + cmpdi cr5,r31,0; \ + NVOLREG_RESTORE; \ + beq cr5,0f; \ + RET_SCV; \ + b 1f; \ +0: RET_SC; \ +1: TAIL_CALL_SYSCALL_ERROR +#endif + +#define RET_SCV \ + cmpdi r3,0; \ + bgelr+; \ + neg r3,r3; + +#define RET_SC \ + bnslr+; #define ret PSEUDO_RET @@ -319,8 +413,15 @@ LT_LABELSUFFIX(name,_name_end): ; \ ENTRY (name); \ DO_CALL (SYS_ify (syscall_name)) -#define PSEUDO_RET_NOERRNO \ +#if IS_IN(rtld) +# define PSEUDO_RET_NOERRNO \ blr +#else +/* This should only be called after a DO_CALL. */ +# define PSEUDO_RET_NOERRNO \ + NVOLREG_RESTORE; \ + blr +#endif /* IS_IN(rtld) */ #define ret_NOERRNO PSEUDO_RET_NOERRNO @@ -333,8 +434,15 @@ LT_LABELSUFFIX(name,_name_end): ; \ ENTRY (name); \ DO_CALL (SYS_ify (syscall_name)) -#define PSEUDO_RET_ERRVAL \ +#if IS_IN(rtld) +# define PSEUDO_RET_ERRVAL \ + blr +#else +/* This should only be called after a DO_CALL. */ +# define PSEUDO_RET_ERRVAL \ + NVOLREG_RESTORE; \ blr +#endif /* IS_IN(rtld) */ #define ret_ERRVAL PSEUDO_RET_ERRVAL diff --git a/sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S b/sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S index b30641c805..fc496fa671 100644 --- a/sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S +++ b/sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S @@ -68,7 +68,8 @@ ENTRY (__clone) cfi_endproc /* Do the call. */ - DO_CALL(SYS_ify(clone)) + li r0,SYS_ify(clone) + DO_CALL_SC /* Check for child process. */ cmpdi cr1,r3,0 @@ -82,7 +83,8 @@ ENTRY (__clone) bctrl ld r2,FRAME_TOC_SAVE(r1) - DO_CALL(SYS_ify(exit)) + li r0,(SYS_ify(exit)) + DO_CALL_SC /* We won't ever get here but provide a nop so that the linker will insert a toc adjusting stub if necessary. */ nop @@ -104,7 +106,8 @@ L(parent): cfi_restore(r30) cfi_restore(r31) - PSEUDO_RET + RET_SC + TAIL_CALL_SYSCALL_ERROR END (__clone) diff --git a/sysdeps/unix/sysv/linux/powerpc/powerpc64/vfork.S b/sysdeps/unix/sysv/linux/powerpc/powerpc64/vfork.S index 17199fb14a..a71f69e929 100644 --- a/sysdeps/unix/sysv/linux/powerpc/powerpc64/vfork.S +++ b/sysdeps/unix/sysv/linux/powerpc/powerpc64/vfork.S @@ -28,9 +28,11 @@ ENTRY (__vfork) CALL_MCOUNT 0 - DO_CALL (SYS_ify (vfork)) + li r0,SYS_ify (vfork) + DO_CALL_SC - PSEUDO_RET + RET_SC + TAIL_CALL_SYSCALL_ERROR PSEUDO_END (__vfork) libc_hidden_def (__vfork) diff --git a/sysdeps/unix/sysv/linux/powerpc/syscall.S b/sysdeps/unix/sysv/linux/powerpc/syscall.S index 48dade4642..9fc4ddd3cb 100644 --- a/sysdeps/unix/sysv/linux/powerpc/syscall.S +++ b/sysdeps/unix/sysv/linux/powerpc/syscall.S @@ -25,6 +25,13 @@ ENTRY (syscall) mr r6,r7 mr r7,r8 mr r8,r9 - sc - PSEUDO_RET +#if !IS_IN(rtld) && (defined(__PPC64__) || defined(__powerpc64__)) + CHECK_SCV_SUPPORT r9 0f + DO_CALL_SCV + RET_SCV + b 1f +#endif +0: DO_CALL_SC + RET_SC +1: TAIL_CALL_SYSCALL_ERROR PSEUDO_END (syscall) diff --git a/sysdeps/unix/sysv/linux/powerpc/sysdep.h b/sysdeps/unix/sysv/linux/powerpc/sysdep.h index b2bca598b9..7f69804edc 100644 --- a/sysdeps/unix/sysv/linux/powerpc/sysdep.h +++ b/sysdeps/unix/sysv/linux/powerpc/sysdep.h @@ -64,39 +64,84 @@ #define INTERNAL_VSYSCALL_CALL(funcptr, nr, args...) \ INTERNAL_VSYSCALL_CALL_TYPE(funcptr, long int, nr, args) +#define DECLARE_REGS \ + register long int r0 __asm__ ("r0"); \ + register long int r3 __asm__ ("r3"); \ + register long int r4 __asm__ ("r4"); \ + register long int r5 __asm__ ("r5"); \ + register long int r6 __asm__ ("r6"); \ + register long int r7 __asm__ ("r7"); \ + register long int r8 __asm__ ("r8"); -#undef INTERNAL_SYSCALL -#define INTERNAL_SYSCALL_NCS(name, nr, args...) \ - ({ \ - register long int r0 __asm__ ("r0"); \ - register long int r3 __asm__ ("r3"); \ - register long int r4 __asm__ ("r4"); \ - register long int r5 __asm__ ("r5"); \ - register long int r6 __asm__ ("r6"); \ - register long int r7 __asm__ ("r7"); \ - register long int r8 __asm__ ("r8"); \ - LOADARGS_##nr (name, ##args); \ - __asm__ __volatile__ \ - ("sc\n\t" \ - "mfcr %0\n\t" \ - "0:" \ - : "=&r" (r0), \ - "=&r" (r3), "=&r" (r4), "=&r" (r5), \ - "=&r" (r6), "=&r" (r7), "=&r" (r8) \ - : ASM_INPUT_##nr \ - : "r9", "r10", "r11", "r12", \ - "cr0", "ctr", "memory"); \ - r0 & (1 << 28) ? -r3 : r3; \ +#define SYSCALL_SCV(nr) \ + ({ \ + __asm__ __volatile__ \ + ("scv 0\n\t" \ + "0:" \ + : "=&r" (r0), \ + "=&r" (r3), "=&r" (r4), "=&r" (r5), \ + "=&r" (r6), "=&r" (r7), "=&r" (r8) \ + : ASM_INPUT_##nr \ + : "r9", "r10", "r11", "r12", \ + "lr", "ctr", "memory"); \ + r3; \ }) -#define INTERNAL_SYSCALL(name, nr, args...) \ - INTERNAL_SYSCALL_NCS (__NR_##name, nr, args) + +#define SYSCALL_SC(nr) \ + ({ \ + __asm__ __volatile__ \ + ("sc\n\t" \ + "mfcr %0\n\t" \ + "0:" \ + : "=&r" (r0), \ + "=&r" (r3), "=&r" (r4), "=&r" (r5), \ + "=&r" (r6), "=&r" (r7), "=&r" (r8) \ + : ASM_INPUT_##nr \ + : "r9", "r10", "r11", "r12", \ + "cr0", "ctr", "memory"); \ + r0 & (1 << 28) ? -r3 : r3; \ + }) + +/* This will only be non-empty for 64-bit systems, see below. */ +#define TRY_SYSCALL_SCV(nr) #if defined(__PPC64__) || defined(__powerpc64__) # define SYSCALL_ARG_SIZE 8 + +/* For the static case, unlike the dynamic loader, there is no compile-time way + to check if we are inside startup code. So we need to check if the thread + pointer has already been setup before trying to access the TLS. */ +# ifndef SHARED +# define CHECK_THREAD_POINTER (__thread_register != 0) +# else +# define CHECK_THREAD_POINTER (1) +# endif + +/* When inside the dynamic loader, the thread pointer may not have been + initialized yet, so don't check for scv support in that case. */ +# if !IS_IN(rtld) +# undef TRY_SYSCALL_SCV +# define TRY_SYSCALL_SCV(nr) \ + CHECK_THREAD_POINTER && THREAD_GET_HWCAP() & PPC_FEATURE2_SCV ? \ + SYSCALL_SCV(nr) : +# endif + #else # define SYSCALL_ARG_SIZE 4 #endif +# define INTERNAL_SYSCALL_NCS(name, nr, args...) \ + ({ \ + DECLARE_REGS; \ + LOADARGS_##nr (name, ##args); \ + TRY_SYSCALL_SCV(nr) \ + SYSCALL_SC(nr); \ + }) + +#undef INTERNAL_SYSCALL +#define INTERNAL_SYSCALL(name, nr, args...) \ + INTERNAL_SYSCALL_NCS (__NR_##name, nr, args) + #define LOADARGS_0(name, dummy) \ r0 = name #define LOADARGS_1(name, __arg1) \ |