diff options
Diffstat (limited to 'sysdeps')
184 files changed, 4889 insertions, 2094 deletions
diff --git a/sysdeps/aarch64/fpu/log10f_advsimd.c b/sysdeps/aarch64/fpu/log10f_advsimd.c index 9347422a77..82228b599a 100644 --- a/sysdeps/aarch64/fpu/log10f_advsimd.c +++ b/sysdeps/aarch64/fpu/log10f_advsimd.c @@ -22,11 +22,11 @@ static const struct data { - uint32x4_t min_norm; + uint32x4_t off, offset_lower_bound; uint16x8_t special_bound; + uint32x4_t mantissa_mask; float32x4_t poly[8]; float32x4_t inv_ln10, ln2; - uint32x4_t off, mantissa_mask; } data = { /* Use order 9 for log10(1+x), i.e. order 8 for log10(1+x)/x, with x in [-1/3, 1/3] (offset=2/3). Max. relative error: 0x1.068ee468p-25. */ @@ -35,18 +35,22 @@ static const struct data V4 (-0x1.0fc92cp-4f), V4 (0x1.f5f76ap-5f) }, .ln2 = V4 (0x1.62e43p-1f), .inv_ln10 = V4 (0x1.bcb7b2p-2f), - .min_norm = V4 (0x00800000), - .special_bound = V8 (0x7f00), /* asuint32(inf) - min_norm. */ + /* Lower bound is the smallest positive normal float 0x00800000. For + optimised register use subnormals are detected after offset has been + subtracted, so lower bound is 0x0080000 - offset (which wraps around). */ + .offset_lower_bound = V4 (0x00800000 - 0x3f2aaaab), + .special_bound = V8 (0x7f00), /* top16(asuint32(inf) - 0x00800000). */ .off = V4 (0x3f2aaaab), /* 0.666667. */ .mantissa_mask = V4 (0x007fffff), }; static float32x4_t VPCS_ATTR NOINLINE -special_case (float32x4_t x, float32x4_t y, float32x4_t p, float32x4_t r2, - uint16x4_t cmp) +special_case (float32x4_t y, uint32x4_t u_off, float32x4_t p, float32x4_t r2, + uint16x4_t cmp, const struct data *d) { /* Fall back to scalar code. */ - return v_call_f32 (log10f, x, vfmaq_f32 (y, p, r2), vmovl_u16 (cmp)); + return v_call_f32 (log10f, vreinterpretq_f32_u32 (vaddq_u32 (u_off, d->off)), + vfmaq_f32 (y, p, r2), vmovl_u16 (cmp)); } /* Fast implementation of AdvSIMD log10f, @@ -58,15 +62,21 @@ special_case (float32x4_t x, float32x4_t y, float32x4_t p, float32x4_t r2, float32x4_t VPCS_ATTR NOINLINE V_NAME_F1 (log10) (float32x4_t x) { const struct data *d = ptr_barrier (&data); - uint32x4_t u = vreinterpretq_u32_f32 (x); - uint16x4_t special = vcge_u16 (vsubhn_u32 (u, d->min_norm), - vget_low_u16 (d->special_bound)); + + /* To avoid having to mov x out of the way, keep u after offset has been + applied, and recover x by adding the offset back in the special-case + handler. */ + uint32x4_t u_off = vreinterpretq_u32_f32 (x); /* x = 2^n * (1+r), where 2/3 < 1+r < 4/3. */ - u = vsubq_u32 (u, d->off); + u_off = vsubq_u32 (u_off, d->off); float32x4_t n = vcvtq_f32_s32 ( - vshrq_n_s32 (vreinterpretq_s32_u32 (u), 23)); /* signextend. */ - u = vaddq_u32 (vandq_u32 (u, d->mantissa_mask), d->off); + vshrq_n_s32 (vreinterpretq_s32_u32 (u_off), 23)); /* signextend. */ + + uint16x4_t special = vcge_u16 (vsubhn_u32 (u_off, d->offset_lower_bound), + vget_low_u16 (d->special_bound)); + + uint32x4_t u = vaddq_u32 (vandq_u32 (u_off, d->mantissa_mask), d->off); float32x4_t r = vsubq_f32 (vreinterpretq_f32_u32 (u), v_f32 (1.0f)); /* y = log10(1+r) + n * log10(2). */ @@ -77,7 +87,7 @@ float32x4_t VPCS_ATTR NOINLINE V_NAME_F1 (log10) (float32x4_t x) y = vmulq_f32 (y, d->inv_ln10); if (__glibc_unlikely (v_any_u16h (special))) - return special_case (x, y, poly, r2, special); + return special_case (y, u_off, poly, r2, special, d); return vfmaq_f32 (y, poly, r2); } libmvec_hidden_def (V_NAME_F1 (log10)) diff --git a/sysdeps/aarch64/fpu/log2f_advsimd.c b/sysdeps/aarch64/fpu/log2f_advsimd.c index db21836749..84effe4fe9 100644 --- a/sysdeps/aarch64/fpu/log2f_advsimd.c +++ b/sysdeps/aarch64/fpu/log2f_advsimd.c @@ -22,9 +22,9 @@ static const struct data { - uint32x4_t min_norm; + uint32x4_t off, offset_lower_bound; uint16x8_t special_bound; - uint32x4_t off, mantissa_mask; + uint32x4_t mantissa_mask; float32x4_t poly[9]; } data = { /* Coefficients generated using Remez algorithm approximate @@ -34,18 +34,22 @@ static const struct data V4 (-0x1.715458p-1f), V4 (0x1.ec701cp-2f), V4 (-0x1.7171a4p-2f), V4 (0x1.27a0b8p-2f), V4 (-0x1.e5143ep-3f), V4 (0x1.9d8ecap-3f), V4 (-0x1.c675bp-3f), V4 (0x1.9e495p-3f) }, - .min_norm = V4 (0x00800000), - .special_bound = V8 (0x7f00), /* asuint32(inf) - min_norm. */ + /* Lower bound is the smallest positive normal float 0x00800000. For + optimised register use subnormals are detected after offset has been + subtracted, so lower bound is 0x0080000 - offset (which wraps around). */ + .offset_lower_bound = V4 (0x00800000 - 0x3f2aaaab), + .special_bound = V8 (0x7f00), /* top16(asuint32(inf) - 0x00800000). */ .off = V4 (0x3f2aaaab), /* 0.666667. */ .mantissa_mask = V4 (0x007fffff), }; static float32x4_t VPCS_ATTR NOINLINE -special_case (float32x4_t x, float32x4_t n, float32x4_t p, float32x4_t r, - uint16x4_t cmp) +special_case (float32x4_t n, uint32x4_t u_off, float32x4_t p, float32x4_t r, + uint16x4_t cmp, const struct data *d) { /* Fall back to scalar code. */ - return v_call_f32 (log2f, x, vfmaq_f32 (n, p, r), vmovl_u16 (cmp)); + return v_call_f32 (log2f, vreinterpretq_f32_u32 (vaddq_u32 (u_off, d->off)), + vfmaq_f32 (n, p, r), vmovl_u16 (cmp)); } /* Fast implementation for single precision AdvSIMD log2, @@ -56,15 +60,21 @@ special_case (float32x4_t x, float32x4_t n, float32x4_t p, float32x4_t r, float32x4_t VPCS_ATTR NOINLINE V_NAME_F1 (log2) (float32x4_t x) { const struct data *d = ptr_barrier (&data); - uint32x4_t u = vreinterpretq_u32_f32 (x); - uint16x4_t special = vcge_u16 (vsubhn_u32 (u, d->min_norm), - vget_low_u16 (d->special_bound)); + + /* To avoid having to mov x out of the way, keep u after offset has been + applied, and recover x by adding the offset back in the special-case + handler. */ + uint32x4_t u_off = vreinterpretq_u32_f32 (x); /* x = 2^n * (1+r), where 2/3 < 1+r < 4/3. */ - u = vsubq_u32 (u, d->off); + u_off = vsubq_u32 (u_off, d->off); float32x4_t n = vcvtq_f32_s32 ( - vshrq_n_s32 (vreinterpretq_s32_u32 (u), 23)); /* signextend. */ - u = vaddq_u32 (vandq_u32 (u, d->mantissa_mask), d->off); + vshrq_n_s32 (vreinterpretq_s32_u32 (u_off), 23)); /* signextend. */ + + uint16x4_t special = vcge_u16 (vsubhn_u32 (u_off, d->offset_lower_bound), + vget_low_u16 (d->special_bound)); + + uint32x4_t u = vaddq_u32 (vandq_u32 (u_off, d->mantissa_mask), d->off); float32x4_t r = vsubq_f32 (vreinterpretq_f32_u32 (u), v_f32 (1.0f)); /* y = log2(1+r) + n. */ @@ -72,7 +82,7 @@ float32x4_t VPCS_ATTR NOINLINE V_NAME_F1 (log2) (float32x4_t x) float32x4_t p = v_pw_horner_8_f32 (r, r2, d->poly); if (__glibc_unlikely (v_any_u16h (special))) - return special_case (x, n, p, r, special); + return special_case (n, u_off, p, r, special, d); return vfmaq_f32 (n, p, r); } libmvec_hidden_def (V_NAME_F1 (log2)) diff --git a/sysdeps/aarch64/fpu/logf_advsimd.c b/sysdeps/aarch64/fpu/logf_advsimd.c index 3c0d0fcdc7..c20dbfd6c0 100644 --- a/sysdeps/aarch64/fpu/logf_advsimd.c +++ b/sysdeps/aarch64/fpu/logf_advsimd.c @@ -21,20 +21,22 @@ static const struct data { - uint32x4_t min_norm; + uint32x4_t off, offset_lower_bound; uint16x8_t special_bound; + uint32x4_t mantissa_mask; float32x4_t poly[7]; - float32x4_t ln2, tiny_bound; - uint32x4_t off, mantissa_mask; + float32x4_t ln2; } data = { /* 3.34 ulp error. */ .poly = { V4 (-0x1.3e737cp-3f), V4 (0x1.5a9aa2p-3f), V4 (-0x1.4f9934p-3f), V4 (0x1.961348p-3f), V4 (-0x1.00187cp-2f), V4 (0x1.555d7cp-2f), V4 (-0x1.ffffc8p-2f) }, .ln2 = V4 (0x1.62e43p-1f), - .tiny_bound = V4 (0x1p-126), - .min_norm = V4 (0x00800000), - .special_bound = V8 (0x7f00), /* asuint32(inf) - min_norm. */ + /* Lower bound is the smallest positive normal float 0x00800000. For + optimised register use subnormals are detected after offset has been + subtracted, so lower bound is 0x0080000 - offset (which wraps around). */ + .offset_lower_bound = V4 (0x00800000 - 0x3f2aaaab), + .special_bound = V8 (0x7f00), /* top16(asuint32(inf) - 0x00800000). */ .off = V4 (0x3f2aaaab), /* 0.666667. */ .mantissa_mask = V4 (0x007fffff) }; @@ -42,32 +44,37 @@ static const struct data #define P(i) d->poly[7 - i] static float32x4_t VPCS_ATTR NOINLINE -special_case (float32x4_t x, float32x4_t y, float32x4_t r2, float32x4_t p, - uint16x4_t cmp) +special_case (float32x4_t p, uint32x4_t u_off, float32x4_t y, float32x4_t r2, + uint16x4_t cmp, const struct data *d) { /* Fall back to scalar code. */ - return v_call_f32 (logf, x, vfmaq_f32 (p, y, r2), vmovl_u16 (cmp)); + return v_call_f32 (logf, vreinterpretq_f32_u32 (vaddq_u32 (u_off, d->off)), + vfmaq_f32 (p, y, r2), vmovl_u16 (cmp)); } float32x4_t VPCS_ATTR NOINLINE V_NAME_F1 (log) (float32x4_t x) { const struct data *d = ptr_barrier (&data); float32x4_t n, p, q, r, r2, y; - uint32x4_t u; + uint32x4_t u, u_off; uint16x4_t cmp; - u = vreinterpretq_u32_f32 (x); - cmp = vcge_u16 (vsubhn_u32 (u, d->min_norm), - vget_low_u16 (d->special_bound)); + /* To avoid having to mov x out of the way, keep u after offset has been + applied, and recover x by adding the offset back in the special-case + handler. */ + u_off = vreinterpretq_u32_f32 (x); /* x = 2^n * (1+r), where 2/3 < 1+r < 4/3. */ - u = vsubq_u32 (u, d->off); + u_off = vsubq_u32 (u_off, d->off); n = vcvtq_f32_s32 ( - vshrq_n_s32 (vreinterpretq_s32_u32 (u), 23)); /* signextend. */ - u = vandq_u32 (u, d->mantissa_mask); + vshrq_n_s32 (vreinterpretq_s32_u32 (u_off), 23)); /* signextend. */ + u = vandq_u32 (u_off, d->mantissa_mask); u = vaddq_u32 (u, d->off); r = vsubq_f32 (vreinterpretq_f32_u32 (u), v_f32 (1.0f)); + cmp = vcge_u16 (vsubhn_u32 (u_off, d->offset_lower_bound), + vget_low_u16 (d->special_bound)); + /* y = log(1+r) + n*ln2. */ r2 = vmulq_f32 (r, r); /* n*ln2 + r + r2*(P1 + r*P2 + r2*(P3 + r*P4 + r2*(P5 + r*P6 + r2*P7))). */ @@ -80,7 +87,7 @@ float32x4_t VPCS_ATTR NOINLINE V_NAME_F1 (log) (float32x4_t x) p = vfmaq_f32 (r, d->ln2, n); if (__glibc_unlikely (v_any_u16h (cmp))) - return special_case (x, y, r2, p, cmp); + return special_case (p, u_off, y, r2, cmp, d); return vfmaq_f32 (p, y, r2); } libmvec_hidden_def (V_NAME_F1 (log)) diff --git a/sysdeps/aarch64/libm-test-ulps b/sysdeps/aarch64/libm-test-ulps index 6c96304611..846fb2c29e 100644 --- a/sysdeps/aarch64/libm-test-ulps +++ b/sysdeps/aarch64/libm-test-ulps @@ -1082,24 +1082,24 @@ float: 1 ldouble: 3 Function: "exp10m1": -double: 2 -float: 1 -ldouble: 1 +double: 4 +float: 2 +ldouble: 3 Function: "exp10m1_downward": -double: 1 -float: 1 -ldouble: 3 +double: 3 +float: 3 +ldouble: 6 Function: "exp10m1_towardzero": -double: 1 -float: 1 -ldouble: 3 +double: 2 +float: 3 +ldouble: 6 Function: "exp10m1_upward": -double: 3 -float: 1 -ldouble: 3 +double: 5 +float: 3 +ldouble: 6 Function: "exp2": double: 1 @@ -1130,24 +1130,24 @@ float: 1 ldouble: 2 Function: "exp2m1": -double: 1 -float: 1 -ldouble: 1 +double: 2 +float: 2 +ldouble: 2 Function: "exp2m1_downward": -double: 1 -float: 1 -ldouble: 2 +double: 3 +float: 3 +ldouble: 3 Function: "exp2m1_towardzero": -double: 2 -float: 1 -ldouble: 2 +double: 3 +float: 2 +ldouble: 4 Function: "exp2m1_upward": -double: 1 -float: 1 -ldouble: 2 +double: 3 +float: 3 +ldouble: 5 Function: "exp_advsimd": double: 1 @@ -1356,24 +1356,24 @@ float: 2 ldouble: 1 Function: "log10p1": -double: 1 -float: 1 +double: 2 +float: 2 ldouble: 3 Function: "log10p1_downward": double: 2 -float: 1 -ldouble: 2 +float: 3 +ldouble: 4 Function: "log10p1_towardzero": -double: 2 +double: 3 float: 2 -ldouble: 2 +ldouble: 3 Function: "log10p1_upward": double: 2 -float: 1 -ldouble: 3 +float: 3 +ldouble: 4 Function: "log1p": double: 1 @@ -1432,8 +1432,8 @@ float: 3 ldouble: 1 Function: "log2p1": -double: 1 -float: 1 +double: 2 +float: 2 ldouble: 3 Function: "log2p1_downward": @@ -1447,9 +1447,9 @@ float: 2 ldouble: 2 Function: "log2p1_upward": -double: 1 +double: 2 float: 2 -ldouble: 2 +ldouble: 3 Function: "log_advsimd": double: 1 diff --git a/sysdeps/aarch64/memset.S b/sysdeps/aarch64/memset.S index 7ef77ee8c9..b76dde1557 100644 --- a/sysdeps/aarch64/memset.S +++ b/sysdeps/aarch64/memset.S @@ -1,4 +1,5 @@ -/* Copyright (C) 2012-2024 Free Software Foundation, Inc. +/* Generic optimized memset using SIMD. + Copyright (C) 2012-2024 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -17,7 +18,6 @@ <https://www.gnu.org/licenses/>. */ #include <sysdep.h> -#include "memset-reg.h" #ifndef MEMSET # define MEMSET memset @@ -25,130 +25,131 @@ /* Assumptions: * - * ARMv8-a, AArch64, unaligned accesses + * ARMv8-a, AArch64, Advanced SIMD, unaligned accesses. * */ -ENTRY (MEMSET) +#define dstin x0 +#define valw w1 +#define count x2 +#define dst x3 +#define dstend x4 +#define zva_val x5 +#define off x3 +#define dstend2 x5 +ENTRY (MEMSET) PTR_ARG (0) SIZE_ARG (2) dup v0.16B, valw + cmp count, 16 + b.lo L(set_small) + add dstend, dstin, count + cmp count, 64 + b.hs L(set_128) - cmp count, 96 - b.hi L(set_long) - cmp count, 16 - b.hs L(set_medium) - mov val, v0.D[0] + /* Set 16..63 bytes. */ + mov off, 16 + and off, off, count, lsr 1 + sub dstend2, dstend, off + str q0, [dstin] + str q0, [dstin, off] + str q0, [dstend2, -16] + str q0, [dstend, -16] + ret + .p2align 4 /* Set 0..15 bytes. */ - tbz count, 3, 1f - str val, [dstin] - str val, [dstend, -8] - ret - nop -1: tbz count, 2, 2f - str valw, [dstin] - str valw, [dstend, -4] +L(set_small): + add dstend, dstin, count + cmp count, 4 + b.lo 2f + lsr off, count, 3 + sub dstend2, dstend, off, lsl 2 + str s0, [dstin] + str s0, [dstin, off, lsl 2] + str s0, [dstend2, -4] + str s0, [dstend, -4] ret + + /* Set 0..3 bytes. */ 2: cbz count, 3f + lsr off, count, 1 strb valw, [dstin] - tbz count, 1, 3f - strh valw, [dstend, -2] + strb valw, [dstin, off] + strb valw, [dstend, -1] 3: ret - /* Set 17..96 bytes. */ -L(set_medium): - str q0, [dstin] - tbnz count, 6, L(set96) - str q0, [dstend, -16] - tbz count, 5, 1f - str q0, [dstin, 16] - str q0, [dstend, -32] -1: ret - .p2align 4 - /* Set 64..96 bytes. Write 64 bytes from the start and - 32 bytes from the end. */ -L(set96): - str q0, [dstin, 16] +L(set_128): + bic dst, dstin, 15 + cmp count, 128 + b.hi L(set_long) + stp q0, q0, [dstin] stp q0, q0, [dstin, 32] + stp q0, q0, [dstend, -64] stp q0, q0, [dstend, -32] ret - .p2align 3 - nop + .p2align 4 L(set_long): - and valw, valw, 255 - bic dst, dstin, 15 str q0, [dstin] - cmp count, 256 - ccmp valw, 0, 0, cs - b.eq L(try_zva) -L(no_zva): - sub count, dstend, dst /* Count is 16 too large. */ - sub dst, dst, 16 /* Dst is biased by -32. */ - sub count, count, 64 + 16 /* Adjust count and bias for loop. */ -1: stp q0, q0, [dst, 32] - stp q0, q0, [dst, 64]! -L(tail64): - subs count, count, 64 - b.hi 1b -2: stp q0, q0, [dstend, -64] + str q0, [dst, 16] + tst valw, 255 + b.ne L(no_zva) +#ifndef ZVA64_ONLY + mrs zva_val, dczid_el0 + and zva_val, zva_val, 31 + cmp zva_val, 4 /* ZVA size is 64 bytes. */ + b.ne L(zva_128) +#endif + stp q0, q0, [dst, 32] + bic dst, dstin, 63 + sub count, dstend, dst /* Count is now 64 too large. */ + sub count, count, 64 + 64 /* Adjust count and bias for loop. */ + + /* Write last bytes before ZVA loop. */ + stp q0, q0, [dstend, -64] stp q0, q0, [dstend, -32] + + .p2align 4 +L(zva64_loop): + add dst, dst, 64 + dc zva, dst + subs count, count, 64 + b.hi L(zva64_loop) ret -L(try_zva): -#ifndef ZVA64_ONLY .p2align 3 - mrs tmp1, dczid_el0 - tbnz tmp1w, 4, L(no_zva) - and tmp1w, tmp1w, 15 - cmp tmp1w, 4 /* ZVA size is 64 bytes. */ - b.ne L(zva_128) - nop -#endif - /* Write the first and last 64 byte aligned block using stp rather - than using DC ZVA. This is faster on some cores. - */ - .p2align 4 -L(zva_64): - str q0, [dst, 16] +L(no_zva): + sub count, dstend, dst /* Count is 32 too large. */ + sub count, count, 64 + 32 /* Adjust count and bias for loop. */ +L(no_zva_loop): stp q0, q0, [dst, 32] - bic dst, dst, 63 stp q0, q0, [dst, 64] - stp q0, q0, [dst, 96] - sub count, dstend, dst /* Count is now 128 too large. */ - sub count, count, 128+64+64 /* Adjust count and bias for loop. */ - add dst, dst, 128 -1: dc zva, dst add dst, dst, 64 subs count, count, 64 - b.hi 1b - stp q0, q0, [dst, 0] - stp q0, q0, [dst, 32] + b.hi L(no_zva_loop) stp q0, q0, [dstend, -64] stp q0, q0, [dstend, -32] ret #ifndef ZVA64_ONLY - .p2align 3 + .p2align 4 L(zva_128): - cmp tmp1w, 5 /* ZVA size is 128 bytes. */ - b.ne L(zva_other) + cmp zva_val, 5 /* ZVA size is 128 bytes. */ + b.ne L(no_zva) - str q0, [dst, 16] stp q0, q0, [dst, 32] stp q0, q0, [dst, 64] stp q0, q0, [dst, 96] bic dst, dst, 127 sub count, dstend, dst /* Count is now 128 too large. */ - sub count, count, 128+128 /* Adjust count and bias for loop. */ - add dst, dst, 128 -1: dc zva, dst - add dst, dst, 128 + sub count, count, 128 + 128 /* Adjust count and bias for loop. */ +1: add dst, dst, 128 + dc zva, dst subs count, count, 128 b.hi 1b stp q0, q0, [dstend, -128] @@ -156,35 +157,6 @@ L(zva_128): stp q0, q0, [dstend, -64] stp q0, q0, [dstend, -32] ret - -L(zva_other): - mov tmp2w, 4 - lsl zva_lenw, tmp2w, tmp1w - add tmp1, zva_len, 64 /* Max alignment bytes written. */ - cmp count, tmp1 - blo L(no_zva) - - sub tmp2, zva_len, 1 - add tmp1, dst, zva_len - add dst, dst, 16 - subs count, tmp1, dst /* Actual alignment bytes to write. */ - bic tmp1, tmp1, tmp2 /* Aligned dc zva start address. */ - beq 2f -1: stp q0, q0, [dst], 64 - stp q0, q0, [dst, -32] - subs count, count, 64 - b.hi 1b -2: mov dst, tmp1 - sub count, dstend, tmp1 /* Remaining bytes to write. */ - subs count, count, zva_len - b.lo 4f -3: dc zva, dst - add dst, dst, zva_len - subs count, count, zva_len - b.hs 3b -4: add count, count, zva_len - sub dst, dst, 32 /* Bias dst for tail loop. */ - b L(tail64) #endif END (MEMSET) diff --git a/sysdeps/aarch64/multiarch/memset_a64fx.S b/sysdeps/aarch64/multiarch/memset_a64fx.S index 2e6d882fc9..f665b5a891 100644 --- a/sysdeps/aarch64/multiarch/memset_a64fx.S +++ b/sysdeps/aarch64/multiarch/memset_a64fx.S @@ -18,7 +18,6 @@ <https://www.gnu.org/licenses/>. */ #include <sysdep.h> -#include <sysdeps/aarch64/memset-reg.h> /* Assumptions: * @@ -36,6 +35,14 @@ .arch armv8.2-a+sve +#define dstin x0 +#define valw w1 +#define count x2 +#define dst x3 +#define dstend x4 +#define tmp1 x5 +#define tmp2 x6 + .macro st1b_unroll first=0, last=7 st1b z0.b, p0, [dst, \first, mul vl] .if \last-\first diff --git a/sysdeps/aarch64/multiarch/memset_emag.S b/sysdeps/aarch64/multiarch/memset_emag.S index 6d714ed0e1..cf1b25f2ed 100644 --- a/sysdeps/aarch64/multiarch/memset_emag.S +++ b/sysdeps/aarch64/multiarch/memset_emag.S @@ -18,7 +18,6 @@ <https://www.gnu.org/licenses/>. */ #include <sysdep.h> -#include "memset-reg.h" /* Assumptions: * @@ -26,6 +25,13 @@ * */ +#define dstin x0 +#define val x1 +#define valw w1 +#define count x2 +#define dst x3 +#define dstend x4 + ENTRY (__memset_emag) PTR_ARG (0) diff --git a/sysdeps/aarch64/multiarch/memset_kunpeng.S b/sysdeps/aarch64/multiarch/memset_kunpeng.S index 7b21550137..f815c20b03 100644 --- a/sysdeps/aarch64/multiarch/memset_kunpeng.S +++ b/sysdeps/aarch64/multiarch/memset_kunpeng.S @@ -18,7 +18,6 @@ <https://www.gnu.org/licenses/>. */ #include <sysdep.h> -#include <sysdeps/aarch64/memset-reg.h> /* Assumptions: * @@ -26,6 +25,12 @@ * */ +#define dstin x0 +#define valw w1 +#define count x2 +#define dst x3 +#define dstend x4 + ENTRY (__memset_kunpeng) PTR_ARG (0) diff --git a/sysdeps/aarch64/multiarch/memset_oryon1.S b/sysdeps/aarch64/multiarch/memset_oryon1.S index b43a43b54e..6fa28a9bd0 100644 --- a/sysdeps/aarch64/multiarch/memset_oryon1.S +++ b/sysdeps/aarch64/multiarch/memset_oryon1.S @@ -19,12 +19,18 @@ <https://www.gnu.org/licenses/>. */ #include <sysdep.h> -#include "memset-reg.h" /* Assumptions: ARMv8-a, AArch64, unaligned accesses */ +#define dstin x0 +#define val x1 +#define valw w1 +#define count x2 +#define dst x3 +#define dstend x4 + ENTRY (__memset_oryon1) PTR_ARG (0) diff --git a/sysdeps/aarch64/strlen.S b/sysdeps/aarch64/strlen.S index ab2a576cdb..352fb40d3a 100644 --- a/sysdeps/aarch64/strlen.S +++ b/sysdeps/aarch64/strlen.S @@ -1,4 +1,5 @@ -/* Copyright (C) 2012-2024 Free Software Foundation, Inc. +/* Generic optimized strlen using SIMD. + Copyright (C) 2012-2024 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -56,36 +57,50 @@ ENTRY (STRLEN) shrn vend.8b, vhas_nul.8h, 4 /* 128->64 */ fmov synd, dend lsr synd, synd, shift - cbz synd, L(loop) + cbz synd, L(next16) rbit synd, synd clz result, synd lsr result, result, 2 ret +L(next16): + ldr data, [src, 16] + cmeq vhas_nul.16b, vdata.16b, 0 + shrn vend.8b, vhas_nul.8h, 4 /* 128->64 */ + fmov synd, dend + cbz synd, L(loop) + add src, src, 16 +#ifndef __AARCH64EB__ + rbit synd, synd +#endif + sub result, src, srcin + clz tmp, synd + add result, result, tmp, lsr 2 + ret + .p2align 5 L(loop): - ldr data, [src, 16] + ldr data, [src, 32]! cmeq vhas_nul.16b, vdata.16b, 0 - umaxp vend.16b, vhas_nul.16b, vhas_nul.16b + addhn vend.8b, vhas_nul.8h, vhas_nul.8h fmov synd, dend cbnz synd, L(loop_end) - ldr data, [src, 32]! + ldr data, [src, 16] cmeq vhas_nul.16b, vdata.16b, 0 - umaxp vend.16b, vhas_nul.16b, vhas_nul.16b + addhn vend.8b, vhas_nul.8h, vhas_nul.8h fmov synd, dend cbz synd, L(loop) - sub src, src, 16 + add src, src, 16 L(loop_end): - shrn vend.8b, vhas_nul.8h, 4 /* 128->64 */ - sub result, src, srcin - fmov synd, dend + sub result, shift, src, lsl 2 /* (srcin - src) << 2. */ #ifndef __AARCH64EB__ rbit synd, synd + sub result, result, 3 #endif - add result, result, 16 clz tmp, synd - add result, result, tmp, lsr 2 + sub result, tmp, result + lsr result, result, 2 ret END (STRLEN) diff --git a/sysdeps/arc/fpu/libm-test-ulps b/sysdeps/arc/fpu/libm-test-ulps index 41c8ef16d7..7e2c2dff13 100644 --- a/sysdeps/arc/fpu/libm-test-ulps +++ b/sysdeps/arc/fpu/libm-test-ulps @@ -787,6 +787,22 @@ Function: "exp10_upward": double: 4 float: 1 +Function: "exp10m1": +double: 4 +float: 2 + +Function: "exp10m1_downward": +double: 3 +float: 3 + +Function: "exp10m1_towardzero": +double: 2 +float: 3 + +Function: "exp10m1_upward": +double: 5 +float: 3 + Function: "exp2": double: 1 float: 1 @@ -801,6 +817,22 @@ Function: "exp2_upward": double: 2 float: 1 +Function: "exp2m1": +double: 2 +float: 2 + +Function: "exp2m1_downward": +double: 3 +float: 3 + +Function: "exp2m1_towardzero": +double: 3 +float: 2 + +Function: "exp2m1_upward": +double: 3 +float: 3 + Function: "exp_downward": double: 1 float: 1 @@ -943,6 +975,22 @@ Function: "log10_upward": double: 3 float: 4 +Function: "log10p1": +double: 2 +float: 2 + +Function: "log10p1_downward": +double: 2 +float: 3 + +Function: "log10p1_towardzero": +double: 3 +float: 2 + +Function: "log10p1_upward": +double: 2 +float: 3 + Function: "log1p": double: 1 float: 1 @@ -971,6 +1019,22 @@ Function: "log2_upward": double: 1 float: 1 +Function: "log2p1": +double: 2 +float: 2 + +Function: "log2p1_downward": +double: 2 +float: 2 + +Function: "log2p1_towardzero": +double: 2 +float: 2 + +Function: "log2p1_upward": +double: 2 +float: 2 + Function: "log_towardzero": double: 1 float: 1 diff --git a/sysdeps/arc/nofpu/libm-test-ulps b/sysdeps/arc/nofpu/libm-test-ulps index d3f45957d4..d9d6c76c3e 100644 --- a/sysdeps/arc/nofpu/libm-test-ulps +++ b/sysdeps/arc/nofpu/libm-test-ulps @@ -192,9 +192,17 @@ Function: "exp10": double: 2 float: 1 +Function: "exp10m1": +double: 3 +float: 2 + Function: "exp2": double: 1 +Function: "exp2m1": +double: 2 +float: 2 + Function: "expm1": double: 1 float: 1 @@ -205,6 +213,7 @@ float: 7 Function: "hypot": double: 1 +float: 1 Function: "j0": double: 2 @@ -226,6 +235,10 @@ Function: "log10": double: 2 float: 2 +Function: "log10p1": +double: 1 +float: 2 + Function: "log1p": double: 1 float: 1 @@ -234,6 +247,10 @@ Function: "log2": double: 2 float: 1 +Function: "log2p1": +double: 1 +float: 2 + Function: "logp1": double: 1 float: 1 diff --git a/sysdeps/arm/libm-test-ulps b/sysdeps/arm/libm-test-ulps index cad14a1ef1..100e9d1956 100644 --- a/sysdeps/arm/libm-test-ulps +++ b/sysdeps/arm/libm-test-ulps @@ -778,20 +778,20 @@ double: 2 float: 1 Function: "exp10m1": -double: 2 -float: 1 +double: 3 +float: 2 Function: "exp10m1_downward": -double: 1 -float: 1 +double: 3 +float: 3 Function: "exp10m1_towardzero": -double: 1 -float: 1 +double: 2 +float: 3 Function: "exp10m1_upward": -double: 3 -float: 1 +double: 4 +float: 4 Function: "exp2": double: 1 @@ -810,20 +810,20 @@ double: 1 float: 1 Function: "exp2m1": -double: 1 -float: 1 +double: 2 +float: 2 Function: "exp2m1_downward": -double: 2 -float: 1 +double: 3 +float: 3 Function: "exp2m1_towardzero": -double: 2 -float: 1 +double: 3 +float: 2 Function: "exp2m1_upward": -double: 1 -float: 1 +double: 3 +float: 3 Function: "exp_downward": double: 1 @@ -967,19 +967,19 @@ float: 2 Function: "log10p1": double: 1 -float: 1 +float: 2 Function: "log10p1_downward": double: 2 -float: 1 +float: 3 Function: "log10p1_towardzero": -double: 2 +double: 3 float: 2 Function: "log10p1_upward": double: 2 -float: 1 +float: 3 Function: "log1p": double: 1 @@ -1015,7 +1015,7 @@ float: 3 Function: "log2p1": double: 1 -float: 1 +float: 2 Function: "log2p1_downward": double: 2 @@ -1026,7 +1026,7 @@ double: 2 float: 2 Function: "log2p1_upward": -double: 1 +double: 2 float: 2 Function: "log_downward": diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h index 656e8a3fa0..259ce2e7d6 100644 --- a/sysdeps/generic/ldsodefs.h +++ b/sysdeps/generic/ldsodefs.h @@ -1200,10 +1200,8 @@ extern void _dl_get_tls_static_info (size_t *sizep, size_t *alignp); extern void _dl_allocate_static_tls (struct link_map *map) attribute_hidden; -/* These are internal entry points to the two halves of _dl_allocate_tls, - only used within rtld.c itself at startup time. */ extern void *_dl_allocate_tls_storage (void) attribute_hidden; -extern void *_dl_allocate_tls_init (void *, bool); +extern void *_dl_allocate_tls_init (void *result, bool main_thread); rtld_hidden_proto (_dl_allocate_tls_init) /* True if the TCB has been set up. */ @@ -1241,13 +1239,7 @@ extern void *_dl_open (const char *name, int mode, const void *caller, extern int _dl_scope_free (void *) attribute_hidden; -/* Add module to slot information data. If DO_ADD is false, only the - required memory is allocated. Must be called with GL - (dl_load_tls_lock) acquired. If the function has already been called - for the link map L with !do_add, then this function will not raise - an exception, otherwise it is possible that it encounters a memory - allocation failure. */ -extern void _dl_add_to_slotinfo (struct link_map *l, bool do_add) +extern bool _dl_add_to_slotinfo (struct link_map *l, bool do_add) attribute_hidden; /* Update slot information data for at least the generation of the diff --git a/sysdeps/aarch64/memset-reg.h b/sysdeps/generic/mremap-failure.h index 6c7f60b37e..bc0d476368 100644 --- a/sysdeps/aarch64/memset-reg.h +++ b/sysdeps/generic/mremap-failure.h @@ -1,5 +1,5 @@ -/* Register aliases for memset to be used across implementations. - Copyright (C) 2017-2024 Free Software Foundation, Inc. +/* mremap failure handling. Generic version. + Copyright (C) 2024 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -16,15 +16,10 @@ License along with the GNU C Library; if not, see <https://www.gnu.org/licenses/>. */ -#define dstin x0 -#define val x1 -#define valw w1 -#define count x2 -#define dst x3 -#define dstend x4 -#define tmp1 x5 -#define tmp1w w5 -#define tmp2 x6 -#define tmp2w w6 -#define zva_len x7 -#define zva_lenw w7 +/* Return exit value on mremap failure with errno ERR. */ + +static int +mremap_failure_exit (int err) +{ + return EXIT_FAILURE; +} diff --git a/sysdeps/generic/syscall_types.h b/sysdeps/generic/syscall_types.h new file mode 100644 index 0000000000..2ddeaa2b5f --- /dev/null +++ b/sysdeps/generic/syscall_types.h @@ -0,0 +1,25 @@ +/* Types and macros used for syscall issuing. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#ifndef _SYSCALL_TYPES_H +#define _SYSCALL_TYPES_H + +typedef long int __syscall_arg_t; +#define __SSC(__x) ((__syscall_arg_t) (__x)) + +#endif diff --git a/sysdeps/gnu/Makefile b/sysdeps/gnu/Makefile index c89eb8ed1c..d23ded40a0 100644 --- a/sysdeps/gnu/Makefile +++ b/sysdeps/gnu/Makefile @@ -16,37 +16,67 @@ # <https://www.gnu.org/licenses/>. ifeq ($(subdir),login) -sysdep_routines += setutxent getutxent endutxent getutxid getutxline \ - pututxline utmpxname updwtmpx getutmpx getutmp +sysdep_routines += \ + endutxent \ + getutmp \ + getutmpx \ + getutxent \ + getutxid \ + getutxline \ + pututxline \ + setutxent \ + updwtmpx \ + utmpxname \ + # sysdep_routines -sysdep_headers += utmpx.h bits/utmpx.h +sysdep_headers += \ + bits/utmpx.h \ + utmpx.h \ + # sysdep_headers endif ifeq ($(subdir),inet) -sysdep_headers += netinet/udp.h netinet/ip_icmp.h +sysdep_headers += \ + netinet/ip_icmp.h \ + netinet/udp.h \ + # sysdep_headers endif ifeq ($(subdir),misc) -sysdep_headers += sys/mtio.h +sysdep_headers += \ + sys/mtio.h \ + # sysdep_headers endif ifeq ($(subdir),csu) -routines += unwind-resume -shared-only-routines += unwind-resume +routines += \ + unwind-resume \ + # routines +shared-only-routines += \ + unwind-resume \ + # shared-only-routines CFLAGS-unwind-resume.c += -fexceptions -fasynchronous-unwind-tables endif + ifeq ($(subdir),rt) -librt-sysdep_routines += rt-unwind-resume -librt-shared-only-routines += rt-unwind-resume +librt-sysdep_routines += \ + rt-unwind-resume \ + # librt-sysdep_routines +librt-shared-only-routines += \ + rt-unwind-resume \ + # librt-shared-only-routines CFLAGS-rt-unwind-resume.c += -fexceptions -fasynchronous-unwind-tables endif + ifeq ($(subdir),sysvipc) -sysdep_headers += bits/ipc-perm.h \ - bits/types/struct_msqid_ds.h \ - bits/types/struct_shmid_ds.h +sysdep_headers += \ + bits/ipc-perm.h \ + bits/types/struct_msqid_ds.h \ + bits/types/struct_shmid_ds.h + # sysdep_headers endif diff --git a/sysdeps/hppa/dl-fptr.h b/sysdeps/hppa/dl-fptr.h index 7c5eb0bd30..b7fd9cf71a 100644 --- a/sysdeps/hppa/dl-fptr.h +++ b/sysdeps/hppa/dl-fptr.h @@ -19,7 +19,28 @@ #ifndef dl_hppa_fptr_h #define dl_hppa_fptr_h 1 -#include <sysdeps/generic/dl-fptr.h> +/* An FDESC is a function descriptor. */ + +struct fdesc + { + ElfW(Addr) ip; /* code entry point */ + ElfW(Addr) gp; /* global pointer */ + }; + +struct fdesc_table + { + struct fdesc_table *next; + unsigned int len; /* # of entries in fdesc table */ + volatile unsigned int first_unused; /* index of first available entry */ + struct fdesc fdesc[0]; + }; + +struct link_map; + +extern ElfW(Addr) _dl_boot_fptr_table []; + +extern ElfW(Addr) _dl_make_fptr (struct link_map *, const ElfW(Sym) *, + ElfW(Addr)); /* Initialize function pointer code. Call before relocation processing. */ extern void _dl_fptr_init (void); diff --git a/sysdeps/hppa/fpu/libm-test-ulps b/sysdeps/hppa/fpu/libm-test-ulps index d91a85bfc8..4f0709c817 100644 --- a/sysdeps/hppa/fpu/libm-test-ulps +++ b/sysdeps/hppa/fpu/libm-test-ulps @@ -797,20 +797,20 @@ double: 2 float: 1 Function: "exp10m1": -double: 2 -float: 1 +double: 4 +float: 2 Function: "exp10m1_downward": -double: 1 -float: 1 +double: 3 +float: 3 Function: "exp10m1_towardzero": -double: 1 -float: 1 +double: 2 +float: 3 Function: "exp10m1_upward": -double: 3 -float: 1 +double: 5 +float: 4 Function: "exp2": double: 1 @@ -829,20 +829,20 @@ double: 1 float: 1 Function: "exp2m1": -double: 1 -float: 1 +double: 2 +float: 2 Function: "exp2m1_downward": -double: 2 -float: 1 +double: 3 +float: 3 Function: "exp2m1_towardzero": -double: 2 -float: 1 +double: 3 +float: 2 Function: "exp2m1_upward": -double: 1 -float: 1 +double: 3 +float: 3 Function: "exp_downward": double: 1 @@ -903,12 +903,12 @@ Function: "hypot_upward": double: 1 Function: "j0": -double: 2 +double: 4 float: 9 ldouble: 2 Function: "j0_downward": -double: 5 +double: 6 float: 9 Function: "j0_towardzero": @@ -925,11 +925,11 @@ float: 9 ldouble: 1 Function: "j1_downward": -double: 5 +double: 9 float: 8 Function: "j1_towardzero": -double: 4 +double: 5 float: 8 Function: "j1_upward": @@ -971,6 +971,7 @@ double: 5 float: 6 Function: "log": +double: 1 float: 1 Function: "log10": @@ -991,20 +992,20 @@ double: 2 float: 2 Function: "log10p1": -double: 1 -float: 1 +double: 2 +float: 2 Function: "log10p1_downward": double: 2 -float: 1 +float: 3 Function: "log10p1_towardzero": -double: 2 +double: 3 float: 2 Function: "log10p1_upward": double: 2 -float: 1 +float: 3 Function: "log1p": double: 1 @@ -1039,8 +1040,8 @@ double: 3 float: 3 Function: "log2p1": -double: 1 -float: 1 +double: 2 +float: 2 Function: "log2p1_downward": double: 2 @@ -1051,7 +1052,7 @@ double: 2 float: 2 Function: "log2p1_upward": -double: 1 +double: 2 float: 2 Function: "log_downward": diff --git a/sysdeps/htl/pt-attr-getdetachstate.c b/sysdeps/htl/pt-attr-getdetachstate.c index 8b077e38f3..da27398260 100644 --- a/sysdeps/htl/pt-attr-getdetachstate.c +++ b/sysdeps/htl/pt-attr-getdetachstate.c @@ -17,6 +17,7 @@ <https://www.gnu.org/licenses/>. */ #include <pthread.h> +#include <shlib-compat.h> #include <pt-internal.h> int @@ -26,4 +27,8 @@ __pthread_attr_getdetachstate (const pthread_attr_t *attr, int *detachstate) return 0; } -weak_alias (__pthread_attr_getdetachstate, pthread_attr_getdetachstate); +versioned_symbol (libc, __pthread_attr_getdetachstate, pthread_attr_getdetachstate, GLIBC_2_21); + +#if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_12, GLIBC_2_21) +compat_symbol (libc, __pthread_attr_getdetachstate, pthread_attr_getdetachstate, GLIBC_2_12); +#endif diff --git a/sysdeps/htl/pt-attr-getinheritsched.c b/sysdeps/htl/pt-attr-getinheritsched.c index 078face3eb..ba02e202c3 100644 --- a/sysdeps/htl/pt-attr-getinheritsched.c +++ b/sysdeps/htl/pt-attr-getinheritsched.c @@ -17,6 +17,7 @@ <https://www.gnu.org/licenses/>. */ #include <pthread.h> +#include <shlib-compat.h> #include <pt-internal.h> int @@ -26,4 +27,8 @@ __pthread_attr_getinheritsched (const pthread_attr_t *attr, int *inheritsched) return 0; } -weak_alias (__pthread_attr_getinheritsched, pthread_attr_getinheritsched); +versioned_symbol (libc, __pthread_attr_getinheritsched, pthread_attr_getinheritsched, GLIBC_2_21); + +#if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_12, GLIBC_2_21) +compat_symbol (libc, __pthread_attr_getinheritsched, pthread_attr_getinheritsched, GLIBC_2_12); +#endif diff --git a/sysdeps/htl/pt-attr-getschedparam.c b/sysdeps/htl/pt-attr-getschedparam.c index 3fa001871e..35eeedc442 100644 --- a/sysdeps/htl/pt-attr-getschedparam.c +++ b/sysdeps/htl/pt-attr-getschedparam.c @@ -19,6 +19,7 @@ #include <pthread.h> #include <sched.h> #include <string.h> +#include <shlib-compat.h> #include <pt-internal.h> @@ -30,4 +31,8 @@ __pthread_attr_getschedparam (const pthread_attr_t *attr, return 0; } -weak_alias (__pthread_attr_getschedparam, pthread_attr_getschedparam); +versioned_symbol (libc, __pthread_attr_getschedparam, pthread_attr_getschedparam, GLIBC_2_21); + +#if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_12, GLIBC_2_21) +compat_symbol (libc, __pthread_attr_getschedparam, pthread_attr_getschedparam, GLIBC_2_12); +#endif diff --git a/sysdeps/htl/pt-attr-getschedpolicy.c b/sysdeps/htl/pt-attr-getschedpolicy.c index 8b4cc7ce48..5deaf01c5e 100644 --- a/sysdeps/htl/pt-attr-getschedpolicy.c +++ b/sysdeps/htl/pt-attr-getschedpolicy.c @@ -17,6 +17,7 @@ <https://www.gnu.org/licenses/>. */ #include <pthread.h> +#include <shlib-compat.h> #include <pt-internal.h> int @@ -26,4 +27,8 @@ __pthread_attr_getschedpolicy (const pthread_attr_t *attr, int *policy) return 0; } -weak_alias (__pthread_attr_getschedpolicy, pthread_attr_getschedpolicy); +versioned_symbol (libc, __pthread_attr_getschedpolicy, pthread_attr_getschedpolicy, GLIBC_2_21); + +#if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_12, GLIBC_2_21) +compat_symbol (libc, __pthread_attr_getschedpolicy, pthread_attr_getschedpolicy, GLIBC_2_12); +#endif diff --git a/sysdeps/htl/pt-attr-setdetachstate.c b/sysdeps/htl/pt-attr-setdetachstate.c index b40557f816..26b6f91dc6 100644 --- a/sysdeps/htl/pt-attr-setdetachstate.c +++ b/sysdeps/htl/pt-attr-setdetachstate.c @@ -17,6 +17,7 @@ <https://www.gnu.org/licenses/>. */ #include <pthread.h> +#include <shlib-compat.h> #include <pt-internal.h> int @@ -35,4 +36,8 @@ __pthread_attr_setdetachstate (pthread_attr_t *attr, int detachstate) return 0; } -weak_alias (__pthread_attr_setdetachstate, pthread_attr_setdetachstate); +versioned_symbol (libc, __pthread_attr_setdetachstate, pthread_attr_setdetachstate, GLIBC_2_21); + +#if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_12, GLIBC_2_21) +compat_symbol (libc, __pthread_attr_setdetachstate, pthread_attr_setdetachstate, GLIBC_2_12); +#endif diff --git a/sysdeps/htl/pt-attr-setinheritsched.c b/sysdeps/htl/pt-attr-setinheritsched.c index 533b805b12..4ad0bb1dec 100644 --- a/sysdeps/htl/pt-attr-setinheritsched.c +++ b/sysdeps/htl/pt-attr-setinheritsched.c @@ -17,6 +17,7 @@ <https://www.gnu.org/licenses/>. */ #include <pthread.h> +#include <shlib-compat.h> #include <pt-internal.h> int @@ -35,4 +36,8 @@ __pthread_attr_setinheritsched (pthread_attr_t *attr, int inheritsched) return 0; } -weak_alias (__pthread_attr_setinheritsched, pthread_attr_setinheritsched); +versioned_symbol (libc, __pthread_attr_setinheritsched, pthread_attr_setinheritsched, GLIBC_2_21); + +#if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_12, GLIBC_2_21) +compat_symbol (libc, __pthread_attr_setinheritsched, pthread_attr_setinheritsched, GLIBC_2_12); +#endif diff --git a/sysdeps/htl/pt-attr-setschedpolicy.c b/sysdeps/htl/pt-attr-setschedpolicy.c index d8086e8316..9648c774d0 100644 --- a/sysdeps/htl/pt-attr-setschedpolicy.c +++ b/sysdeps/htl/pt-attr-setschedpolicy.c @@ -17,6 +17,7 @@ <https://www.gnu.org/licenses/>. */ #include <pthread.h> +#include <shlib-compat.h> #include <pt-internal.h> int @@ -39,4 +40,8 @@ __pthread_attr_setschedpolicy (pthread_attr_t *attr, int policy) return 0; } -weak_alias (__pthread_attr_setschedpolicy, pthread_attr_setschedpolicy); +versioned_symbol (libc, __pthread_attr_setschedpolicy, pthread_attr_setschedpolicy, GLIBC_2_21); + +#if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_12, GLIBC_2_21) +compat_symbol (libc, __pthread_attr_setschedpolicy, pthread_attr_setschedpolicy, GLIBC_2_12); +#endif diff --git a/sysdeps/htl/pt-equal.c b/sysdeps/htl/pt-equal.c index 6411110608..7727cca370 100644 --- a/sysdeps/htl/pt-equal.c +++ b/sysdeps/htl/pt-equal.c @@ -17,6 +17,7 @@ <https://www.gnu.org/licenses/>. */ #include <pthread.h> +#include <shlib-compat.h> #include <pt-internal.h> /* Return true if __T1 and __T2 both name the same thread. Otherwise, @@ -27,4 +28,8 @@ __pthread_equal (pthread_t __t1, pthread_t __t2) return __t1 == __t2; } -weak_alias (__pthread_equal, pthread_equal); +versioned_symbol (libc, __pthread_equal, pthread_equal, GLIBC_2_21); + +#if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_12, GLIBC_2_21) +compat_symbol (libc, __pthread_equal, pthread_equal, GLIBC_2_12); +#endif diff --git a/sysdeps/htl/pt-getschedparam.c b/sysdeps/htl/pt-getschedparam.c index 8ec3d54bd4..0c6a2d0ab3 100644 --- a/sysdeps/htl/pt-getschedparam.c +++ b/sysdeps/htl/pt-getschedparam.c @@ -17,6 +17,7 @@ <https://www.gnu.org/licenses/>. */ #include <pthread.h> +#include <shlib-compat.h> #include <pt-internal.h> int @@ -28,4 +29,8 @@ __pthread_getschedparam (pthread_t thread, int *policy, return 0; } -weak_alias (__pthread_getschedparam, pthread_getschedparam); +versioned_symbol (libc, __pthread_getschedparam, pthread_getschedparam, GLIBC_2_21); + +#if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_12, GLIBC_2_21) +compat_symbol (libc, __pthread_getschedparam, pthread_getschedparam, GLIBC_2_12); +#endif diff --git a/sysdeps/htl/pt-setschedparam.c b/sysdeps/htl/pt-setschedparam.c index aea4e7b0e4..5594c9f390 100644 --- a/sysdeps/htl/pt-setschedparam.c +++ b/sysdeps/htl/pt-setschedparam.c @@ -17,6 +17,7 @@ <https://www.gnu.org/licenses/>. */ #include <pthread.h> +#include <shlib-compat.h> #include <pt-internal.h> int @@ -26,5 +27,9 @@ __pthread_setschedparam (pthread_t thread, int policy, return ENOSYS; } -weak_alias (__pthread_setschedparam, pthread_setschedparam); +versioned_symbol (libc, __pthread_setschedparam, pthread_setschedparam, GLIBC_2_21); stub_warning (pthread_setschedparam) + +#if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_12, GLIBC_2_21) +compat_symbol (libc, __pthread_setschedparam, pthread_setschedparam, GLIBC_2_12); +#endif diff --git a/sysdeps/i386/fpu/libm-test-ulps b/sysdeps/i386/fpu/libm-test-ulps index 14c6d12f37..9e46244752 100644 --- a/sysdeps/i386/fpu/libm-test-ulps +++ b/sysdeps/i386/fpu/libm-test-ulps @@ -1154,28 +1154,28 @@ float128: 3 ldouble: 2 Function: "exp10m1": -double: 1 +double: 3 float: 1 -float128: 1 -ldouble: 1 +float128: 3 +ldouble: 4 Function: "exp10m1_downward": -double: 1 -float: 1 -float128: 3 -ldouble: 2 +double: 3 +float: 3 +float128: 6 +ldouble: 6 Function: "exp10m1_towardzero": -double: 1 -float: 1 -float128: 3 -ldouble: 2 +double: 2 +float: 2 +float128: 6 +ldouble: 5 Function: "exp10m1_upward": -double: 3 -float: 1 -float128: 3 -ldouble: 2 +double: 5 +float: 3 +float128: 6 +ldouble: 6 Function: "exp2": double: 1 @@ -1197,28 +1197,28 @@ float128: 2 ldouble: 1 Function: "exp2m1": -double: 1 -float: 1 -float128: 1 -ldouble: 1 - -Function: "exp2m1_downward": double: 2 float: 1 float128: 2 -ldouble: 2 +ldouble: 3 + +Function: "exp2m1_downward": +double: 3 +float: 3 +float128: 3 +ldouble: 6 Function: "exp2m1_towardzero": -double: 2 -float: 1 -float128: 2 -ldouble: 2 +double: 3 +float: 2 +float128: 4 +ldouble: 5 Function: "exp2m1_upward": -double: 1 -float: 1 -float128: 2 -ldouble: 3 +double: 3 +float: 3 +float128: 5 +ldouble: 6 Function: "exp_downward": double: 1 @@ -1427,25 +1427,25 @@ Function: "log10p1": double: 1 float: 2 float128: 3 -ldouble: 2 +ldouble: 4 Function: "log10p1_downward": double: 2 -float: 1 -float128: 2 -ldouble: 4 +float: 3 +float128: 4 +ldouble: 8 Function: "log10p1_towardzero": double: 2 float: 1 -float128: 2 -ldouble: 4 +float128: 3 +ldouble: 8 Function: "log10p1_upward": double: 2 -float: 2 -float128: 3 -ldouble: 3 +float: 3 +float128: 4 +ldouble: 6 Function: "log1p": double: 1 @@ -1497,7 +1497,7 @@ Function: "log2p1": double: 1 float: 1 float128: 3 -ldouble: 2 +ldouble: 4 Function: "log2p1_downward": double: 2 @@ -1512,9 +1512,9 @@ float128: 2 ldouble: 3 Function: "log2p1_upward": -double: 1 +double: 2 float: 2 -float128: 2 +float128: 3 ldouble: 5 Function: "log_downward": diff --git a/sysdeps/i386/i686/fpu/multiarch/libm-test-ulps b/sysdeps/i386/i686/fpu/multiarch/libm-test-ulps index fff853e93f..5c41bf36dc 100644 --- a/sysdeps/i386/i686/fpu/multiarch/libm-test-ulps +++ b/sysdeps/i386/i686/fpu/multiarch/libm-test-ulps @@ -1157,28 +1157,28 @@ float128: 3 ldouble: 2 Function: "exp10m1": -double: 1 +double: 3 float: 1 -float128: 1 -ldouble: 1 +float128: 3 +ldouble: 4 Function: "exp10m1_downward": -double: 1 -float: 1 -float128: 3 -ldouble: 2 +double: 3 +float: 3 +float128: 6 +ldouble: 6 Function: "exp10m1_towardzero": -double: 1 -float: 1 -float128: 3 -ldouble: 2 +double: 2 +float: 2 +float128: 6 +ldouble: 5 Function: "exp10m1_upward": -double: 3 -float: 1 -float128: 3 -ldouble: 2 +double: 5 +float: 3 +float128: 6 +ldouble: 6 Function: "exp2": double: 1 @@ -1200,28 +1200,28 @@ float128: 2 ldouble: 1 Function: "exp2m1": -double: 1 -float: 1 -float128: 1 -ldouble: 1 - -Function: "exp2m1_downward": double: 2 float: 1 float128: 2 ldouble: 3 +Function: "exp2m1_downward": +double: 3 +float: 3 +float128: 3 +ldouble: 6 + Function: "exp2m1_towardzero": -double: 2 -float: 1 -float128: 2 -ldouble: 3 +double: 3 +float: 2 +float128: 4 +ldouble: 6 Function: "exp2m1_upward": -double: 1 -float: 1 -float128: 2 -ldouble: 3 +double: 3 +float: 3 +float128: 5 +ldouble: 6 Function: "exp_downward": double: 1 @@ -1430,25 +1430,25 @@ Function: "log10p1": double: 1 float: 2 float128: 3 -ldouble: 2 +ldouble: 4 Function: "log10p1_downward": double: 2 float: 1 -float128: 2 -ldouble: 4 +float128: 4 +ldouble: 8 Function: "log10p1_towardzero": double: 2 float: 1 -float128: 2 -ldouble: 4 +float128: 3 +ldouble: 8 Function: "log10p1_upward": double: 2 float: 2 -float128: 3 -ldouble: 3 +float128: 4 +ldouble: 6 Function: "log1p": double: 1 @@ -1500,7 +1500,7 @@ Function: "log2p1": double: 1 float: 1 float128: 3 -ldouble: 2 +ldouble: 4 Function: "log2p1_downward": double: 2 @@ -1517,7 +1517,7 @@ ldouble: 4 Function: "log2p1_upward": double: 1 float: 2 -float128: 2 +float128: 3 ldouble: 5 Function: "log_downward": diff --git a/sysdeps/loongarch/dl-machine.h b/sysdeps/loongarch/dl-machine.h index ab6f1da7c0..a15d8e0ab6 100644 --- a/sysdeps/loongarch/dl-machine.h +++ b/sysdeps/loongarch/dl-machine.h @@ -223,6 +223,13 @@ elf_machine_rela (struct link_map *map, struct r_scope_elem *scope[], { td->arg = _dl_make_tlsdesc_dynamic (sym_map, sym->st_value + reloc->r_addend); +# ifndef __loongarch_soft_float + if (RTLD_SUPPORT_LASX) + td->entry = _dl_tlsdesc_dynamic_lasx; + else if (RTLD_SUPPORT_LSX) + td->entry = _dl_tlsdesc_dynamic_lsx; + else +# endif td->entry = _dl_tlsdesc_dynamic; } else diff --git a/sysdeps/loongarch/dl-tlsdesc-dynamic.h b/sysdeps/loongarch/dl-tlsdesc-dynamic.h new file mode 100644 index 0000000000..3daf991001 --- /dev/null +++ b/sysdeps/loongarch/dl-tlsdesc-dynamic.h @@ -0,0 +1,222 @@ +/* Thread-local storage handling in the ELF dynamic linker. + LoongArch version. + Copyright (C) 2024 Free Software Foundation, Inc. + + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#define FRAME_SIZE (-((-14 * SZREG) & ALMASK)) + + /* Handler for dynamic TLS symbols. + Prototype: + _dl_tlsdesc_dynamic (tlsdesc *) ; + + The second word of the descriptor points to a + tlsdesc_dynamic_arg structure. + + Returns the offset between the thread pointer and the + object referenced by the argument. + + ptrdiff_t + _dl_tlsdesc_dynamic (struct tlsdesc *tdp) + { + struct tlsdesc_dynamic_arg *td = tdp->arg; + dtv_t *dtv = *(dtv_t **)((char *)__thread_pointer - SIZE_OF_TCB); + if (__glibc_likely (td->gen_count <= dtv[0].counter + && (dtv[td->tlsinfo.ti_module].pointer.val + != TLS_DTV_UNALLOCATED), + 1)) + return dtv[td->tlsinfo.ti_module].pointer.val + + td->tlsinfo.ti_offset + - __thread_pointer; + + return ___tls_get_addr (&td->tlsinfo) - __thread_pointer; + } */ + .hidden _dl_tlsdesc_dynamic + .global _dl_tlsdesc_dynamic + .type _dl_tlsdesc_dynamic,%function + cfi_startproc + .align 2 +_dl_tlsdesc_dynamic: + /* Save just enough registers to support fast path, if we fall + into slow path we will save additional registers. */ + ADDI sp, sp, -32 + cfi_adjust_cfa_offset (32) + REG_S t0, sp, 0 + REG_S t1, sp, 8 + REG_S t2, sp, 16 + cfi_rel_offset (12, 0) + cfi_rel_offset (13, 8) + cfi_rel_offset (14, 16) + +/* Runtime Storage Layout of Thread-Local Storage + TP point to the start of TLS block. + + dtv +Low address TCB ----------------> dtv0(counter) + TP --> static_block0 <----- dtv1 + static_block1 <----- dtv2 + static_block2 <----- dtv3 + dynamic_block0 <----- dtv4 +Hign address dynamic_block1 <----- dtv5 */ + + REG_L t0, tp, -SIZE_OF_TCB /* t0 = dtv */ + REG_L a0, a0, TLSDESC_ARG /* a0(td) = tdp->arg */ + REG_L t1, a0, TLSDESC_GEN_COUNT /* t1 = td->gen_count */ + REG_L t2, t0, DTV_COUNTER /* t2 = dtv[0].counter */ + /* If dtv[0].counter < td->gen_count, goto slow path. */ + bltu t2, t1, .Lslow + + REG_L t1, a0, TLSDESC_MODID /* t1 = td->tlsinfo.ti_module */ + /* t1 = t1 * sizeof(dtv_t) = t1 * (2 * sizeof(void*)) */ + slli.d t1, t1, 4 + add.d t1, t1, t0 /* t1 = dtv[td->tlsinfo.ti_module] */ + REG_L t1, t1, 0 /* t1 = dtv[td->tlsinfo.ti_module].pointer.val */ + li.d t2, TLS_DTV_UNALLOCATED + /* If dtv[td->tlsinfo.ti_module].pointer.val is TLS_DTV_UNALLOCATED, + goto slow path. */ + beq t1, t2, .Lslow + + cfi_remember_state + REG_L t2, a0, TLSDESC_MODOFF /* t2 = td->tlsinfo.ti_offset */ + /* dtv[td->tlsinfo.ti_module].pointer.val + td->tlsinfo.ti_offset */ + add.d a0, t1, t2 +.Lret: + sub.d a0, a0, tp + REG_L t0, sp, 0 + REG_L t1, sp, 8 + REG_L t2, sp, 16 + ADDI sp, sp, 32 + cfi_adjust_cfa_offset (-32) + RET + +.Lslow: + /* This is the slow path. We need to call __tls_get_addr() which + means we need to save and restore all the register that the + callee will trash. */ + + /* Save the remaining registers that we must treat as caller save. */ + cfi_restore_state + ADDI sp, sp, -FRAME_SIZE + cfi_adjust_cfa_offset (FRAME_SIZE) + REG_S ra, sp, 0 * SZREG + REG_S a1, sp, 1 * SZREG + REG_S a2, sp, 2 * SZREG + REG_S a3, sp, 3 * SZREG + REG_S a4, sp, 4 * SZREG + REG_S a5, sp, 5 * SZREG + REG_S a6, sp, 6 * SZREG + REG_S a7, sp, 7 * SZREG + REG_S t3, sp, 8 * SZREG + REG_S t4, sp, 9 * SZREG + REG_S t5, sp, 10 * SZREG + REG_S t6, sp, 11 * SZREG + REG_S t7, sp, 12 * SZREG + REG_S t8, sp, 13 * SZREG + cfi_rel_offset (1, 0 * SZREG) + cfi_rel_offset (5, 1 * SZREG) + cfi_rel_offset (6, 2 * SZREG) + cfi_rel_offset (7, 3 * SZREG) + cfi_rel_offset (8, 4 * SZREG) + cfi_rel_offset (9, 5 * SZREG) + cfi_rel_offset (10, 6 * SZREG) + cfi_rel_offset (11, 7 * SZREG) + cfi_rel_offset (15, 8 * SZREG) + cfi_rel_offset (16, 9 * SZREG) + cfi_rel_offset (17, 10 * SZREG) + cfi_rel_offset (18, 11 * SZREG) + cfi_rel_offset (19, 12 * SZREG) + cfi_rel_offset (20, 13 * SZREG) + +#ifndef __loongarch_soft_float + + /* Save fcsr0 register. + Only one physical fcsr0 register, fcsr1-fcsr3 are aliases + of some fields in fcsr0. */ + movfcsr2gr t0, fcsr0 + st.w t0, sp, FRAME_SIZE + 24 /* Use the spare slot above t2. */ + +#ifdef USE_LASX + #define V_REG_S xvst + #define V_REG_L xvld + #define V_SPACE (-((-32 * SZXREG) & ALMASK)) /* Space for LASX registers. */ + #define V_REG(n) $xr##n + #define V_REGS 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16, \ + 17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 + #define V_REGSZ SZXREG +#elif defined USE_LSX + #define V_REG_S vst + #define V_REG_L vld + #define V_SPACE (-((-32 * SZVREG) & ALMASK)) /* Space for LSX registers. */ + #define V_REG(n) $vr##n + #define V_REGS 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16, \ + 17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 + #define V_REGSZ SZVREG +#else + #define V_REG_S fst.d + #define V_REG_L fld.d + #define V_SPACE (-((-24 * SZFREG) & ALMASK)) /* Space for FLOAT registers. */ + #define V_REG(n) $f##n + #define V_REGS 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23 + #define V_REGSZ SZFREG +#endif + + ADDI sp, sp, -V_SPACE + cfi_adjust_cfa_offset (V_SPACE) + .irp i,V_REGS + V_REG_S V_REG(\i), sp, \i * V_REGSZ + .endr + +#endif /* #ifndef __loongarch_soft_float */ + + bl HIDDEN_JUMPTARGET(__tls_get_addr) + ADDI a0, a0, -TLS_DTV_OFFSET + +#ifndef __loongarch_soft_float + + .irp i,V_REGS + V_REG_L V_REG(\i), sp, \i * V_REGSZ + .endr + ADDI sp, sp, V_SPACE + cfi_adjust_cfa_offset (-V_SPACE) + + /* Restore fcsr0 register. */ + ld.w t0, sp, FRAME_SIZE + 24 + movgr2fcsr fcsr0, t0 + +#endif /* #ifndef __loongarch_soft_float */ + + REG_L ra, sp, 0 * SZREG + REG_L a1, sp, 1 * SZREG + REG_L a2, sp, 2 * SZREG + REG_L a3, sp, 3 * SZREG + REG_L a4, sp, 4 * SZREG + REG_L a5, sp, 5 * SZREG + REG_L a6, sp, 6 * SZREG + REG_L a7, sp, 7 * SZREG + REG_L t3, sp, 8 * SZREG + REG_L t4, sp, 9 * SZREG + REG_L t5, sp, 10 * SZREG + REG_L t6, sp, 11 * SZREG + REG_L t7, sp, 12 * SZREG + REG_L t8, sp, 13 * SZREG + ADDI sp, sp, FRAME_SIZE + cfi_adjust_cfa_offset (-FRAME_SIZE) + + b .Lret + cfi_endproc + .size _dl_tlsdesc_dynamic, .-_dl_tlsdesc_dynamic + .hidden HIDDEN_JUMPTARGET(__tls_get_addr) diff --git a/sysdeps/loongarch/dl-tlsdesc.S b/sysdeps/loongarch/dl-tlsdesc.S index a6627cc754..be76c07cce 100644 --- a/sysdeps/loongarch/dl-tlsdesc.S +++ b/sysdeps/loongarch/dl-tlsdesc.S @@ -59,376 +59,46 @@ _dl_tlsdesc_undefweak: cfi_endproc .size _dl_tlsdesc_undefweak, .-_dl_tlsdesc_undefweak - #ifdef SHARED -#define FRAME_SIZE (-((-14 * SZREG) & ALMASK)) -#define FRAME_SIZE_LSX (-((-32 * SZVREG) & ALMASK)) -#define FRAME_SIZE_LASX (-((-32 * SZXREG) & ALMASK)) -#define FRAME_SIZE_FLOAT (-((-24 * SZFREG) & ALMASK)) - - /* Handler for dynamic TLS symbols. - Prototype: - _dl_tlsdesc_dynamic (tlsdesc *) ; - - The second word of the descriptor points to a - tlsdesc_dynamic_arg structure. - - Returns the offset between the thread pointer and the - object referenced by the argument. - - ptrdiff_t - _dl_tlsdesc_dynamic (struct tlsdesc *tdp) - { - struct tlsdesc_dynamic_arg *td = tdp->arg; - dtv_t *dtv = *(dtv_t **)((char *)__thread_pointer - SIZE_OF_TCB); - if (__glibc_likely (td->gen_count <= dtv[0].counter - && (dtv[td->tlsinfo.ti_module].pointer.val - != TLS_DTV_UNALLOCATED), - 1)) - return dtv[td->tlsinfo.ti_module].pointer.val - + td->tlsinfo.ti_offset - - __thread_pointer; - - return ___tls_get_addr (&td->tlsinfo) - __thread_pointer; - } */ - .hidden _dl_tlsdesc_dynamic - .global _dl_tlsdesc_dynamic - .type _dl_tlsdesc_dynamic,%function - cfi_startproc - .align 2 -_dl_tlsdesc_dynamic: - /* Save just enough registers to support fast path, if we fall - into slow path we will save additional registers. */ - ADDI sp, sp, -32 - REG_S t0, sp, 0 - REG_S t1, sp, 8 - REG_S t2, sp, 16 - -/* Runtime Storage Layout of Thread-Local Storage - TP point to the start of TLS block. - - dtv -Low address TCB ----------------> dtv0(counter) - TP --> static_block0 <----- dtv1 - static_block1 <----- dtv2 - static_block2 <----- dtv3 - dynamic_block0 <----- dtv4 -Hign address dynamic_block1 <----- dtv5 */ - - REG_L t0, tp, -SIZE_OF_TCB /* t0 = dtv */ - REG_L a0, a0, TLSDESC_ARG /* a0(td) = tdp->arg */ - REG_L t1, a0, TLSDESC_GEN_COUNT /* t1 = td->gen_count */ - REG_L t2, t0, DTV_COUNTER /* t2 = dtv[0].counter */ - /* If dtv[0].counter < td->gen_count, goto slow path. */ - bltu t2, t1, .Lslow - - REG_L t1, a0, TLSDESC_MODID /* t1 = td->tlsinfo.ti_module */ - /* t1 = t1 * sizeof(dtv_t) = t1 * (2 * sizeof(void*)) */ - slli.d t1, t1, 4 - add.d t1, t1, t0 /* t1 = dtv[td->tlsinfo.ti_module] */ - REG_L t1, t1, 0 /* t1 = dtv[td->tlsinfo.ti_module].pointer.val */ - li.d t2, TLS_DTV_UNALLOCATED - /* If dtv[td->tlsinfo.ti_module].pointer.val is TLS_DTV_UNALLOCATED, - goto slow path. */ - beq t1, t2, .Lslow - - REG_L t2, a0, TLSDESC_MODOFF /* t2 = td->tlsinfo.ti_offset */ - /* dtv[td->tlsinfo.ti_module].pointer.val + td->tlsinfo.ti_offset */ - add.d a0, t1, t2 -.Lret: - sub.d a0, a0, tp - REG_L t0, sp, 0 - REG_L t1, sp, 8 - REG_L t2, sp, 16 - ADDI sp, sp, 32 - RET - -.Lslow: - /* This is the slow path. We need to call __tls_get_addr() which - means we need to save and restore all the register that the - callee will trash. */ - - /* Save the remaining registers that we must treat as caller save. */ - ADDI sp, sp, -FRAME_SIZE - REG_S ra, sp, 0 * SZREG - REG_S a1, sp, 1 * SZREG - REG_S a2, sp, 2 * SZREG - REG_S a3, sp, 3 * SZREG - REG_S a4, sp, 4 * SZREG - REG_S a5, sp, 5 * SZREG - REG_S a6, sp, 6 * SZREG - REG_S a7, sp, 7 * SZREG - REG_S t3, sp, 8 * SZREG - REG_S t4, sp, 9 * SZREG - REG_S t5, sp, 10 * SZREG - REG_S t6, sp, 11 * SZREG - REG_S t7, sp, 12 * SZREG - REG_S t8, sp, 13 * SZREG - #ifndef __loongarch_soft_float - /* Save fcsr0 register. - Only one physical fcsr0 register, fcsr1-fcsr3 are aliases - of some fields in fcsr0. */ - movfcsr2gr t0, fcsr0 - st.w t0, sp, FRAME_SIZE + 24 /* Use the spare slot above t2 */ - - /* Whether support LASX. */ - la.global t0, _rtld_global_ro - REG_L t0, t0, GLRO_DL_HWCAP_OFFSET - andi t1, t0, HWCAP_LOONGARCH_LASX - beqz t1, .Llsx - - /* Save 256-bit vector registers. - FIXME: Without vector ABI, save all vector registers. */ - ADDI sp, sp, -FRAME_SIZE_LASX - xvst xr0, sp, 0*SZXREG - xvst xr1, sp, 1*SZXREG - xvst xr2, sp, 2*SZXREG - xvst xr3, sp, 3*SZXREG - xvst xr4, sp, 4*SZXREG - xvst xr5, sp, 5*SZXREG - xvst xr6, sp, 6*SZXREG - xvst xr7, sp, 7*SZXREG - xvst xr8, sp, 8*SZXREG - xvst xr9, sp, 9*SZXREG - xvst xr10, sp, 10*SZXREG - xvst xr11, sp, 11*SZXREG - xvst xr12, sp, 12*SZXREG - xvst xr13, sp, 13*SZXREG - xvst xr14, sp, 14*SZXREG - xvst xr15, sp, 15*SZXREG - xvst xr16, sp, 16*SZXREG - xvst xr17, sp, 17*SZXREG - xvst xr18, sp, 18*SZXREG - xvst xr19, sp, 19*SZXREG - xvst xr20, sp, 20*SZXREG - xvst xr21, sp, 21*SZXREG - xvst xr22, sp, 22*SZXREG - xvst xr23, sp, 23*SZXREG - xvst xr24, sp, 24*SZXREG - xvst xr25, sp, 25*SZXREG - xvst xr26, sp, 26*SZXREG - xvst xr27, sp, 27*SZXREG - xvst xr28, sp, 28*SZXREG - xvst xr29, sp, 29*SZXREG - xvst xr30, sp, 30*SZXREG - xvst xr31, sp, 31*SZXREG - b .Ltga - -.Llsx: - /* Whether support LSX. */ - andi t1, t0, HWCAP_LOONGARCH_LSX - beqz t1, .Lfloat - - /* Save 128-bit vector registers. */ - ADDI sp, sp, -FRAME_SIZE_LSX - vst vr0, sp, 0*SZVREG - vst vr1, sp, 1*SZVREG - vst vr2, sp, 2*SZVREG - vst vr3, sp, 3*SZVREG - vst vr4, sp, 4*SZVREG - vst vr5, sp, 5*SZVREG - vst vr6, sp, 6*SZVREG - vst vr7, sp, 7*SZVREG - vst vr8, sp, 8*SZVREG - vst vr9, sp, 9*SZVREG - vst vr10, sp, 10*SZVREG - vst vr11, sp, 11*SZVREG - vst vr12, sp, 12*SZVREG - vst vr13, sp, 13*SZVREG - vst vr14, sp, 14*SZVREG - vst vr15, sp, 15*SZVREG - vst vr16, sp, 16*SZVREG - vst vr17, sp, 17*SZVREG - vst vr18, sp, 18*SZVREG - vst vr19, sp, 19*SZVREG - vst vr20, sp, 20*SZVREG - vst vr21, sp, 21*SZVREG - vst vr22, sp, 22*SZVREG - vst vr23, sp, 23*SZVREG - vst vr24, sp, 24*SZVREG - vst vr25, sp, 25*SZVREG - vst vr26, sp, 26*SZVREG - vst vr27, sp, 27*SZVREG - vst vr28, sp, 28*SZVREG - vst vr29, sp, 29*SZVREG - vst vr30, sp, 30*SZVREG - vst vr31, sp, 31*SZVREG - b .Ltga - -.Lfloat: - /* Save float registers. */ - ADDI sp, sp, -FRAME_SIZE_FLOAT - FREG_S fa0, sp, 0*SZFREG - FREG_S fa1, sp, 1*SZFREG - FREG_S fa2, sp, 2*SZFREG - FREG_S fa3, sp, 3*SZFREG - FREG_S fa4, sp, 4*SZFREG - FREG_S fa5, sp, 5*SZFREG - FREG_S fa6, sp, 6*SZFREG - FREG_S fa7, sp, 7*SZFREG - FREG_S ft0, sp, 8*SZFREG - FREG_S ft1, sp, 9*SZFREG - FREG_S ft2, sp, 10*SZFREG - FREG_S ft3, sp, 11*SZFREG - FREG_S ft4, sp, 12*SZFREG - FREG_S ft5, sp, 13*SZFREG - FREG_S ft6, sp, 14*SZFREG - FREG_S ft7, sp, 15*SZFREG - FREG_S ft8, sp, 16*SZFREG - FREG_S ft9, sp, 17*SZFREG - FREG_S ft10, sp, 18*SZFREG - FREG_S ft11, sp, 19*SZFREG - FREG_S ft12, sp, 20*SZFREG - FREG_S ft13, sp, 21*SZFREG - FREG_S ft14, sp, 22*SZFREG - FREG_S ft15, sp, 23*SZFREG - -#endif /* #ifndef __loongarch_soft_float */ - -.Ltga: - bl HIDDEN_JUMPTARGET(__tls_get_addr) - ADDI a0, a0, -TLS_DTV_OFFSET - -#ifndef __loongarch_soft_float - - la.global t0, _rtld_global_ro - REG_L t0, t0, GLRO_DL_HWCAP_OFFSET - andi t1, t0, HWCAP_LOONGARCH_LASX - beqz t1, .Llsx1 - - /* Restore 256-bit vector registers. */ - xvld xr0, sp, 0*SZXREG - xvld xr1, sp, 1*SZXREG - xvld xr2, sp, 2*SZXREG - xvld xr3, sp, 3*SZXREG - xvld xr4, sp, 4*SZXREG - xvld xr5, sp, 5*SZXREG - xvld xr6, sp, 6*SZXREG - xvld xr7, sp, 7*SZXREG - xvld xr8, sp, 8*SZXREG - xvld xr9, sp, 9*SZXREG - xvld xr10, sp, 10*SZXREG - xvld xr11, sp, 11*SZXREG - xvld xr12, sp, 12*SZXREG - xvld xr13, sp, 13*SZXREG - xvld xr14, sp, 14*SZXREG - xvld xr15, sp, 15*SZXREG - xvld xr16, sp, 16*SZXREG - xvld xr17, sp, 17*SZXREG - xvld xr18, sp, 18*SZXREG - xvld xr19, sp, 19*SZXREG - xvld xr20, sp, 20*SZXREG - xvld xr21, sp, 21*SZXREG - xvld xr22, sp, 22*SZXREG - xvld xr23, sp, 23*SZXREG - xvld xr24, sp, 24*SZXREG - xvld xr25, sp, 25*SZXREG - xvld xr26, sp, 26*SZXREG - xvld xr27, sp, 27*SZXREG - xvld xr28, sp, 28*SZXREG - xvld xr29, sp, 29*SZXREG - xvld xr30, sp, 30*SZXREG - xvld xr31, sp, 31*SZXREG - ADDI sp, sp, FRAME_SIZE_LASX - b .Lfcsr - -.Llsx1: - andi t1, t0, HWCAP_LOONGARCH_LSX - beqz t1, .Lfloat1 - - /* Restore 128-bit vector registers. */ - vld vr0, sp, 0*SZVREG - vld vr1, sp, 1*SZVREG - vld vr2, sp, 2*SZVREG - vld vr3, sp, 3*SZVREG - vld vr4, sp, 4*SZVREG - vld vr5, sp, 5*SZVREG - vld vr6, sp, 6*SZVREG - vld vr7, sp, 7*SZVREG - vld vr8, sp, 8*SZVREG - vld vr9, sp, 9*SZVREG - vld vr10, sp, 10*SZVREG - vld vr11, sp, 11*SZVREG - vld vr12, sp, 12*SZVREG - vld vr13, sp, 13*SZVREG - vld vr14, sp, 14*SZVREG - vld vr15, sp, 15*SZVREG - vld vr16, sp, 16*SZVREG - vld vr17, sp, 17*SZVREG - vld vr18, sp, 18*SZVREG - vld vr19, sp, 19*SZVREG - vld vr20, sp, 20*SZVREG - vld vr21, sp, 21*SZVREG - vld vr22, sp, 22*SZVREG - vld vr23, sp, 23*SZVREG - vld vr24, sp, 24*SZVREG - vld vr25, sp, 25*SZVREG - vld vr26, sp, 26*SZVREG - vld vr27, sp, 27*SZVREG - vld vr28, sp, 28*SZVREG - vld vr29, sp, 29*SZVREG - vld vr30, sp, 30*SZVREG - vld vr31, sp, 31*SZVREG - ADDI sp, sp, FRAME_SIZE_LSX - b .Lfcsr - -.Lfloat1: - /* Restore float registers. */ - FREG_L fa0, sp, 0*SZFREG - FREG_L fa1, sp, 1*SZFREG - FREG_L fa2, sp, 2*SZFREG - FREG_L fa3, sp, 3*SZFREG - FREG_L fa4, sp, 4*SZFREG - FREG_L fa5, sp, 5*SZFREG - FREG_L fa6, sp, 6*SZFREG - FREG_L fa7, sp, 7*SZFREG - FREG_L ft0, sp, 8*SZFREG - FREG_L ft1, sp, 9*SZFREG - FREG_L ft2, sp, 10*SZFREG - FREG_L ft3, sp, 11*SZFREG - FREG_L ft4, sp, 12*SZFREG - FREG_L ft5, sp, 13*SZFREG - FREG_L ft6, sp, 14*SZFREG - FREG_L ft7, sp, 15*SZFREG - FREG_L ft8, sp, 16*SZFREG - FREG_L ft9, sp, 17*SZFREG - FREG_L ft10, sp, 18*SZFREG - FREG_L ft11, sp, 19*SZFREG - FREG_L ft12, sp, 20*SZFREG - FREG_L ft13, sp, 21*SZFREG - FREG_L ft14, sp, 22*SZFREG - FREG_L ft15, sp, 23*SZFREG - ADDI sp, sp, FRAME_SIZE_FLOAT - -.Lfcsr: - /* Restore fcsr0 register. */ - ld.w t0, sp, FRAME_SIZE + 24 - movgr2fcsr fcsr0, t0 +#define USE_LASX +#define _dl_tlsdesc_dynamic _dl_tlsdesc_dynamic_lasx +#define Lret Lret_lasx +#define Lslow Lslow_lasx +#include "dl-tlsdesc-dynamic.h" +#undef FRAME_SIZE +#undef V_REG_S +#undef V_REG_L +#undef V_SPACE +#undef V_REG +#undef V_REGS +#undef V_REGSZ +#undef USE_LASX +#undef _dl_tlsdesc_dynamic +#undef Lret +#undef Lslow + +#define USE_LSX +#define _dl_tlsdesc_dynamic _dl_tlsdesc_dynamic_lsx +#define Lret Lret_lsx +#define Lslow Lslow_lsx +#include "dl-tlsdesc-dynamic.h" +#undef FRAME_SIZE +#undef V_REG_S +#undef V_REG_L +#undef V_SPACE +#undef V_REG +#undef V_REGS +#undef V_REGSZ +#undef USE_LSX +#undef _dl_tlsdesc_dynamic +#undef Lret +#undef Lslow #endif /* #ifndef __loongarch_soft_float */ - REG_L ra, sp, 0 * SZREG - REG_L a1, sp, 1 * SZREG - REG_L a2, sp, 2 * SZREG - REG_L a3, sp, 3 * SZREG - REG_L a4, sp, 4 * SZREG - REG_L a5, sp, 5 * SZREG - REG_L a6, sp, 6 * SZREG - REG_L a7, sp, 7 * SZREG - REG_L t3, sp, 8 * SZREG - REG_L t4, sp, 9 * SZREG - REG_L t5, sp, 10 * SZREG - REG_L t6, sp, 11 * SZREG - REG_L t7, sp, 12 * SZREG - REG_L t8, sp, 13 * SZREG - ADDI sp, sp, FRAME_SIZE - - b .Lret - cfi_endproc - .size _dl_tlsdesc_dynamic, .-_dl_tlsdesc_dynamic - .hidden HIDDEN_JUMPTARGET(__tls_get_addr) +#include "dl-tlsdesc-dynamic.h" #endif /* #ifdef SHARED */ diff --git a/sysdeps/loongarch/dl-tlsdesc.h b/sysdeps/loongarch/dl-tlsdesc.h index ff8c69cb93..45c43a5b52 100644 --- a/sysdeps/loongarch/dl-tlsdesc.h +++ b/sysdeps/loongarch/dl-tlsdesc.h @@ -43,6 +43,10 @@ extern ptrdiff_t attribute_hidden _dl_tlsdesc_undefweak (struct tlsdesc *); #ifdef SHARED extern void *_dl_make_tlsdesc_dynamic (struct link_map *, size_t); +#ifndef __loongarch_soft_float +extern ptrdiff_t attribute_hidden _dl_tlsdesc_dynamic_lasx (struct tlsdesc *); +extern ptrdiff_t attribute_hidden _dl_tlsdesc_dynamic_lsx (struct tlsdesc *); +#endif extern ptrdiff_t attribute_hidden _dl_tlsdesc_dynamic (struct tlsdesc *); #endif diff --git a/sysdeps/loongarch/lp64/libm-test-ulps b/sysdeps/loongarch/lp64/libm-test-ulps index bdfd683454..833dca40e4 100644 --- a/sysdeps/loongarch/lp64/libm-test-ulps +++ b/sysdeps/loongarch/lp64/libm-test-ulps @@ -976,24 +976,24 @@ float: 1 ldouble: 3 Function: "exp10m1": -double: 2 -float: 1 -ldouble: 1 +double: 4 +float: 2 +ldouble: 3 Function: "exp10m1_downward": -double: 1 -float: 1 -ldouble: 3 +double: 3 +float: 3 +ldouble: 6 Function: "exp10m1_towardzero": -double: 1 -float: 1 -ldouble: 3 +double: 2 +float: 3 +ldouble: 6 Function: "exp10m1_upward": -double: 3 -float: 1 -ldouble: 3 +double: 5 +float: 3 +ldouble: 6 Function: "exp2": double: 1 @@ -1013,24 +1013,24 @@ float: 1 ldouble: 2 Function: "exp2m1": -double: 1 -float: 1 -ldouble: 1 +double: 2 +float: 2 +ldouble: 2 Function: "exp2m1_downward": -double: 1 -float: 1 -ldouble: 2 +double: 3 +float: 3 +ldouble: 3 Function: "exp2m1_towardzero": -double: 2 -float: 1 -ldouble: 2 +double: 3 +float: 2 +ldouble: 4 Function: "exp2m1_upward": -double: 1 -float: 1 -ldouble: 2 +double: 3 +float: 3 +ldouble: 5 Function: "exp_downward": double: 1 @@ -1206,24 +1206,24 @@ float: 2 ldouble: 1 Function: "log10p1": -double: 1 -float: 1 +double: 2 +float: 2 ldouble: 3 Function: "log10p1_downward": double: 2 -float: 1 -ldouble: 2 +float: 3 +ldouble: 4 Function: "log10p1_towardzero": -double: 2 +double: 3 float: 2 -ldouble: 2 +ldouble: 3 Function: "log10p1_upward": double: 2 -float: 1 -ldouble: 3 +float: 3 +ldouble: 4 Function: "log1p": double: 1 @@ -1263,8 +1263,8 @@ double: 3 ldouble: 1 Function: "log2p1": -double: 1 -float: 1 +double: 2 +float: 2 ldouble: 3 Function: "log2p1_downward": @@ -1278,9 +1278,9 @@ float: 2 ldouble: 2 Function: "log2p1_upward": -double: 1 +double: 2 float: 2 -ldouble: 2 +ldouble: 3 Function: "log_downward": ldouble: 1 diff --git a/sysdeps/loongarch/tlsdesc.sym b/sysdeps/loongarch/tlsdesc.sym index 213d0b3074..9f80fceca6 100644 --- a/sysdeps/loongarch/tlsdesc.sym +++ b/sysdeps/loongarch/tlsdesc.sym @@ -4,12 +4,6 @@ #include <link.h> #include <dl-tlsdesc.h> -#define SHARED 1 - -#include <ldsodefs.h> - -#define GLRO_offsetof(name) offsetof (struct rtld_global_ro, _##name) - -- -- Abuse tls.h macros to derive offsets relative to the thread register. @@ -23,6 +17,3 @@ DTV_COUNTER offsetof(dtv_t, counter) TLS_DTV_UNALLOCATED TLS_DTV_UNALLOCATED TLS_DTV_OFFSET TLS_DTV_OFFSET SIZE_OF_TCB sizeof(tcbhead_t) -GLRO_DL_HWCAP_OFFSET GLRO_offsetof (dl_hwcap) -HWCAP_LOONGARCH_LSX HWCAP_LOONGARCH_LSX -HWCAP_LOONGARCH_LASX HWCAP_LOONGARCH_LASX diff --git a/sysdeps/mach/hurd/i386/libc.abilist b/sysdeps/mach/hurd/i386/libc.abilist index 2adf98e71d..dc2c6b505b 100644 --- a/sysdeps/mach/hurd/i386/libc.abilist +++ b/sysdeps/mach/hurd/i386/libc.abilist @@ -29,7 +29,17 @@ GLIBC_2.11 mkostemps64 F GLIBC_2.11 mkstemps F GLIBC_2.11 mkstemps64 F GLIBC_2.12 __pthread_self F +GLIBC_2.12 pthread_attr_getdetachstate F +GLIBC_2.12 pthread_attr_getinheritsched F +GLIBC_2.12 pthread_attr_getschedparam F +GLIBC_2.12 pthread_attr_getschedpolicy F +GLIBC_2.12 pthread_attr_setdetachstate F +GLIBC_2.12 pthread_attr_setinheritsched F +GLIBC_2.12 pthread_attr_setschedpolicy F +GLIBC_2.12 pthread_equal F +GLIBC_2.12 pthread_getschedparam F GLIBC_2.12 pthread_self F +GLIBC_2.12 pthread_setschedparam F GLIBC_2.13 __fentry__ F GLIBC_2.14 syncfs F GLIBC_2.15 __fdelt_chk F diff --git a/sysdeps/mach/sleep.c b/sysdeps/mach/sleep.c index 572be150bc..aa750c13c5 100644 --- a/sysdeps/mach/sleep.c +++ b/sysdeps/mach/sleep.c @@ -43,6 +43,9 @@ __sleep (unsigned int seconds) after = time_now (); __mach_port_destroy (__mach_task_self (), recv); + if (after - before > seconds) + return 0; + return seconds - (after - before); } weak_alias (__sleep, sleep) diff --git a/sysdeps/mips/mips32/libm-test-ulps b/sysdeps/mips/mips32/libm-test-ulps index 1c27b6615b..d95230724b 100644 --- a/sysdeps/mips/mips32/libm-test-ulps +++ b/sysdeps/mips/mips32/libm-test-ulps @@ -778,20 +778,20 @@ double: 2 float: 1 Function: "exp10m1": -double: 2 -float: 1 +double: 3 +float: 2 Function: "exp10m1_downward": -double: 1 -float: 1 +double: 3 +float: 3 Function: "exp10m1_towardzero": -double: 1 -float: 1 +double: 2 +float: 3 Function: "exp10m1_upward": -double: 3 -float: 1 +double: 4 +float: 4 Function: "exp2": double: 1 @@ -810,20 +810,20 @@ double: 1 float: 1 Function: "exp2m1": -double: 1 -float: 1 +double: 2 +float: 2 Function: "exp2m1_downward": -double: 2 -float: 1 +double: 3 +float: 3 Function: "exp2m1_towardzero": -double: 2 -float: 1 +double: 3 +float: 2 Function: "exp2m1_upward": -double: 1 -float: 1 +double: 3 +float: 3 Function: "exp_downward": double: 1 @@ -967,19 +967,19 @@ float: 2 Function: "log10p1": double: 1 -float: 1 +float: 2 Function: "log10p1_downward": double: 2 -float: 1 +float: 3 Function: "log10p1_towardzero": -double: 2 +double: 3 float: 2 Function: "log10p1_upward": double: 2 -float: 1 +float: 3 Function: "log1p": double: 1 @@ -1015,7 +1015,7 @@ float: 3 Function: "log2p1": double: 1 -float: 1 +float: 2 ldouble: 1 Function: "log2p1_downward": @@ -1029,7 +1029,7 @@ float: 2 ldouble: 2 Function: "log2p1_upward": -double: 1 +double: 2 float: 2 ldouble: 1 diff --git a/sysdeps/mips/mips64/libm-test-ulps b/sysdeps/mips/mips64/libm-test-ulps index 902551b448..ce33d9b4b3 100644 --- a/sysdeps/mips/mips64/libm-test-ulps +++ b/sysdeps/mips/mips64/libm-test-ulps @@ -977,24 +977,24 @@ float: 1 ldouble: 3 Function: "exp10m1": -double: 2 -float: 1 -ldouble: 1 +double: 3 +float: 2 +ldouble: 3 Function: "exp10m1_downward": -double: 1 -float: 1 -ldouble: 3 +double: 3 +float: 3 +ldouble: 6 Function: "exp10m1_towardzero": -double: 1 -float: 1 -ldouble: 3 +double: 2 +float: 3 +ldouble: 6 Function: "exp10m1_upward": -double: 3 -float: 1 -ldouble: 3 +double: 4 +float: 4 +ldouble: 6 Function: "exp2": double: 1 @@ -1017,24 +1017,24 @@ float: 1 ldouble: 2 Function: "exp2m1": -double: 1 -float: 1 -ldouble: 1 - -Function: "exp2m1_downward": double: 2 -float: 1 +float: 2 ldouble: 2 +Function: "exp2m1_downward": +double: 3 +float: 3 +ldouble: 3 + Function: "exp2m1_towardzero": -double: 2 -float: 1 -ldouble: 2 +double: 3 +float: 2 +ldouble: 4 Function: "exp2m1_upward": -double: 1 -float: 1 -ldouble: 2 +double: 3 +float: 3 +ldouble: 5 Function: "exp_downward": double: 1 @@ -1212,23 +1212,23 @@ ldouble: 1 Function: "log10p1": double: 1 -float: 1 +float: 2 ldouble: 3 Function: "log10p1_downward": double: 2 -float: 1 -ldouble: 2 +float: 3 +ldouble: 4 Function: "log10p1_towardzero": -double: 2 +double: 3 float: 2 -ldouble: 2 +ldouble: 3 Function: "log10p1_upward": double: 2 -float: 1 -ldouble: 3 +float: 3 +ldouble: 4 Function: "log1p": double: 1 @@ -1272,7 +1272,7 @@ ldouble: 1 Function: "log2p1": double: 1 -float: 1 +float: 2 ldouble: 3 Function: "log2p1_downward": @@ -1286,9 +1286,9 @@ float: 2 ldouble: 2 Function: "log2p1_upward": -double: 1 +double: 2 float: 2 -ldouble: 2 +ldouble: 3 Function: "log_downward": float: 2 diff --git a/sysdeps/nptl/cancellation-pc-check.h b/sysdeps/nptl/cancellation-pc-check.h new file mode 100644 index 0000000000..cb38ad6819 --- /dev/null +++ b/sysdeps/nptl/cancellation-pc-check.h @@ -0,0 +1,54 @@ +/* Architecture specific code for pthread cancellation handling. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#ifndef _NPTL_CANCELLATION_PC_CHECK +#define _NPTL_CANCELLATION_PC_CHECK + +#include <sigcontextinfo.h> + +/* For syscalls with side-effects (e.g read that might return partial read), + the kernel cannot restart the syscall when interrupted by a signal, it must + return from the call with whatever partial result. In this case, the saved + program counter is set just after the syscall instruction, so the SIGCANCEL + handler should not act on cancellation. + + The __syscall_cancel_arch function, used for all cancellable syscalls, + contains two extra markers, __syscall_cancel_arch_start and + __syscall_cancel_arch_end. The former points to just before the initial + conditional branch that checks if the thread has received a cancellation + request, while former points to the instruction after the one responsible + to issue the syscall. + + The function check if the program counter (PC) from ucontext_t CTX is + within the start and then end boundary from the __syscall_cancel_arch + bridge. Return TRUE if the PC is within the boundary, meaning the + syscall does not have any side effects; or FALSE otherwise. */ + +static __always_inline bool +cancellation_pc_check (void *ctx) +{ + /* Both are defined in syscall_cancel.S. */ + extern const char __syscall_cancel_arch_start[1]; + extern const char __syscall_cancel_arch_end[1]; + + uintptr_t pc = sigcontext_get_pc (ctx); + return pc >= (uintptr_t) __syscall_cancel_arch_start + && pc < (uintptr_t) __syscall_cancel_arch_end; +} + +#endif diff --git a/sysdeps/nptl/lowlevellock-futex.h b/sysdeps/nptl/lowlevellock-futex.h index 278213a37b..c205806300 100644 --- a/sysdeps/nptl/lowlevellock-futex.h +++ b/sysdeps/nptl/lowlevellock-futex.h @@ -21,7 +21,6 @@ #ifndef __ASSEMBLER__ # include <sysdep.h> -# include <sysdep-cancel.h> # include <kernel-features.h> #endif @@ -120,21 +119,10 @@ nr_wake, nr_move, mutex, val) /* Like lll_futex_wait, but acting as a cancellable entrypoint. */ -# define lll_futex_wait_cancel(futexp, val, private) \ - ({ \ - int __oldtype = LIBC_CANCEL_ASYNC (); \ - long int __err = lll_futex_wait (futexp, val, LLL_SHARED); \ - LIBC_CANCEL_RESET (__oldtype); \ - __err; \ - }) - -/* Like lll_futex_timed_wait, but acting as a cancellable entrypoint. */ -# define lll_futex_timed_wait_cancel(futexp, val, timeout, private) \ - ({ \ - int __oldtype = LIBC_CANCEL_ASYNC (); \ - long int __err = lll_futex_timed_wait (futexp, val, timeout, private); \ - LIBC_CANCEL_RESET (__oldtype); \ - __err; \ +# define lll_futex_wait_cancel(futexp, val, private) \ + ({ \ + int __op = __lll_private_flag (FUTEX_WAIT, private); \ + INTERNAL_SYSCALL_CANCEL (futex, futexp, __op, val, NULL); \ }) #endif /* !__ASSEMBLER__ */ diff --git a/sysdeps/nptl/pthreadP.h b/sysdeps/nptl/pthreadP.h index 30e8a2d177..7d9b95e6ac 100644 --- a/sysdeps/nptl/pthreadP.h +++ b/sysdeps/nptl/pthreadP.h @@ -261,10 +261,12 @@ libc_hidden_proto (__pthread_unregister_cancel) /* Called when a thread reacts on a cancellation request. */ static inline void __attribute ((noreturn, always_inline)) -__do_cancel (void) +__do_cancel (void *result) { struct pthread *self = THREAD_SELF; + self->result = result; + /* Make sure we get no more cancellations. */ atomic_fetch_or_relaxed (&self->cancelhandling, EXITING_BITMASK); @@ -272,6 +274,13 @@ __do_cancel (void) THREAD_GETMEM (self, cleanup_jmp_buf)); } +extern long int __syscall_cancel_arch (volatile int *, __syscall_arg_t nr, + __syscall_arg_t arg1, __syscall_arg_t arg2, __syscall_arg_t arg3, + __syscall_arg_t arg4, __syscall_arg_t arg5, __syscall_arg_t arg6 + __SYSCALL_CANCEL7_ARCH_ARG_DEF) attribute_hidden; + +extern _Noreturn void __syscall_do_cancel (void) attribute_hidden; + /* Internal prototypes. */ diff --git a/sysdeps/powerpc/fpu/libm-test-ulps b/sysdeps/powerpc/fpu/libm-test-ulps index c6292a695b..2b5d2b940d 100644 --- a/sysdeps/powerpc/fpu/libm-test-ulps +++ b/sysdeps/powerpc/fpu/libm-test-ulps @@ -1185,28 +1185,28 @@ float128: 3 ldouble: 4 Function: "exp10m1": -double: 2 -float: 1 -float128: 1 -ldouble: 2 +double: 4 +float: 2 +float128: 3 +ldouble: 4 Function: "exp10m1_downward": -double: 1 -float: 1 -float128: 3 +double: 3 +float: 3 +float128: 6 ldouble: 9 Function: "exp10m1_towardzero": -double: 1 -float: 1 -float128: 3 +double: 2 +float: 3 +float128: 6 ldouble: 9 Function: "exp10m1_upward": -double: 3 -float: 1 -float128: 3 -ldouble: 4 +double: 5 +float: 3 +float128: 6 +ldouble: 8 Function: "exp2": double: 1 @@ -1230,28 +1230,28 @@ float128: 2 ldouble: 2 Function: "exp2m1": -double: 1 -float: 1 -float128: 1 -ldouble: 2 - -Function: "exp2m1_downward": -double: 1 -float: 1 +double: 2 +float: 2 float128: 2 ldouble: 3 +Function: "exp2m1_downward": +double: 3 +float: 3 +float128: 3 +ldouble: 7 + Function: "exp2m1_towardzero": -double: 2 -float: 1 -float128: 2 -ldouble: 3 +double: 3 +float: 2 +float128: 4 +ldouble: 8 Function: "exp2m1_upward": -double: 1 -float: 1 -float128: 2 -ldouble: 3 +double: 3 +float: 3 +float128: 5 +ldouble: 9 Function: "exp_downward": double: 1 @@ -1507,28 +1507,28 @@ float128: 1 ldouble: 2 Function: "log10p1": -double: 1 -float: 1 +double: 2 +float: 2 float128: 3 ldouble: 3 Function: "log10p1_downward": double: 2 -float: 1 -float128: 2 -ldouble: 4 +float: 3 +float128: 4 +ldouble: 6 Function: "log10p1_towardzero": -double: 2 +double: 3 float: 2 -float128: 2 -ldouble: 4 +float128: 3 +ldouble: 6 Function: "log10p1_upward": double: 2 -float: 1 -float128: 3 -ldouble: 4 +float: 3 +float128: 4 +ldouble: 6 Function: "log1p": double: 1 @@ -1579,8 +1579,8 @@ float128: 1 ldouble: 4 Function: "log2p1": -double: 1 -float: 1 +double: 2 +float: 2 float128: 3 ldouble: 2 @@ -1594,13 +1594,13 @@ Function: "log2p1_towardzero": double: 2 float: 2 float128: 2 -ldouble: 4 +ldouble: 5 Function: "log2p1_upward": -double: 1 +double: 2 float: 2 -float128: 2 -ldouble: 4 +float128: 3 +ldouble: 6 Function: "log_downward": float: 2 diff --git a/sysdeps/powerpc/nofpu/libm-test-ulps b/sysdeps/powerpc/nofpu/libm-test-ulps index fcebafc1d5..d56327ac33 100644 --- a/sysdeps/powerpc/nofpu/libm-test-ulps +++ b/sysdeps/powerpc/nofpu/libm-test-ulps @@ -986,24 +986,24 @@ float: 1 ldouble: 4 Function: "exp10m1": -double: 2 -float: 1 +double: 3 +float: 2 ldouble: 3 Function: "exp10m1_downward": -double: 1 -float: 1 +double: 3 +float: 3 ldouble: 9 Function: "exp10m1_towardzero": -double: 1 -float: 1 +double: 2 +float: 3 ldouble: 9 Function: "exp10m1_upward": -double: 3 -float: 1 -ldouble: 4 +double: 4 +float: 4 +ldouble: 10 Function: "exp2": double: 1 @@ -1026,24 +1026,24 @@ float: 1 ldouble: 2 Function: "exp2m1": -double: 1 -float: 1 +double: 2 +float: 2 ldouble: 2 Function: "exp2m1_downward": -double: 2 -float: 1 -ldouble: 3 +double: 3 +float: 3 +ldouble: 7 Function: "exp2m1_towardzero": -double: 2 -float: 1 -ldouble: 3 +double: 3 +float: 2 +ldouble: 8 Function: "exp2m1_upward": -double: 1 -float: 1 -ldouble: 3 +double: 3 +float: 3 +ldouble: 9 Function: "exp_downward": double: 1 @@ -1269,23 +1269,23 @@ ldouble: 2 Function: "log10p1": double: 1 -float: 1 +float: 2 ldouble: 3 Function: "log10p1_downward": double: 2 -float: 1 -ldouble: 4 +float: 3 +ldouble: 6 Function: "log10p1_towardzero": -double: 2 +double: 3 float: 2 -ldouble: 4 +ldouble: 6 Function: "log10p1_upward": double: 2 -float: 1 -ldouble: 3 +float: 3 +ldouble: 6 Function: "log1p": double: 1 @@ -1329,8 +1329,8 @@ ldouble: 4 Function: "log2p1": double: 1 -float: 1 -ldouble: 1 +float: 2 +ldouble: 2 Function: "log2p1_downward": double: 2 @@ -1340,12 +1340,12 @@ ldouble: 4 Function: "log2p1_towardzero": double: 2 float: 2 -ldouble: 4 +ldouble: 5 Function: "log2p1_upward": -double: 1 +double: 2 float: 2 -ldouble: 4 +ldouble: 6 Function: "log_downward": float: 2 diff --git a/sysdeps/powerpc/powerpc32/sysdep.h b/sysdeps/powerpc/powerpc32/sysdep.h index 62de4ca2e5..852a755c7c 100644 --- a/sysdeps/powerpc/powerpc32/sysdep.h +++ b/sysdeps/powerpc/powerpc32/sysdep.h @@ -104,6 +104,9 @@ GOT_LABEL: ; \ # define JUMPTARGET(name) name #endif +#define TAIL_CALL_NO_RETURN(__func) \ + b __func@local + #if defined SHARED && defined PIC && !defined NO_HIDDEN # undef HIDDEN_JUMPTARGET # define HIDDEN_JUMPTARGET(name) __GI_##name##@local diff --git a/sysdeps/powerpc/powerpc64/le/Makefile b/sysdeps/powerpc/powerpc64/le/Makefile index 9d568d4f44..b77775cf95 100644 --- a/sysdeps/powerpc/powerpc64/le/Makefile +++ b/sysdeps/powerpc/powerpc64/le/Makefile @@ -129,6 +129,10 @@ CFLAGS-tst-strtod-round.c += $(type-float128-CFLAGS) CFLAGS-tst-wcstod-round.c += $(type-float128-CFLAGS) CFLAGS-tst-strtod-nan-locale.c += $(type-float128-CFLAGS) CFLAGS-tst-wcstod-nan-locale.c += $(type-float128-CFLAGS) +CFLAGS-tst-strtod1i.c += $(type-float128-CFLAGS) +CFLAGS-tst-strtod3.c += $(type-float128-CFLAGS) +CFLAGS-tst-strtod4.c += $(type-float128-CFLAGS) +CFLAGS-tst-strtod5i.c += $(type-float128-CFLAGS) CFLAGS-tst-strtod6.c += $(type-float128-CFLAGS) CFLAGS-tst-strfrom.c += $(type-float128-CFLAGS) CFLAGS-tst-strfrom-locale.c += $(type-float128-CFLAGS) diff --git a/sysdeps/powerpc/powerpc64/le/power9/strcpy.S b/sysdeps/powerpc/powerpc64/le/power9/strcpy.S index 603bde1e39..2f50625a19 100644 --- a/sysdeps/powerpc/powerpc64/le/power9/strcpy.S +++ b/sysdeps/powerpc/powerpc64/le/power9/strcpy.S @@ -42,22 +42,48 @@ if USE_AS_STPCPY is defined. - The implementation can load bytes past a null terminator, but only - up to the next 16B boundary, so it never crosses a page. */ + This implementation never reads across a page boundary, but may + read beyond the NUL terminator. */ -/* Load quadword at addr+offset to vreg, check for null bytes, +/* Load 4 quadwords, merge into one VR for speed and check for NUL + and branch to label if NUL is found. */ +#define CHECK_64B(offset,addr,label) \ + lxv 32+v4,(offset+0)(addr); \ + lxv 32+v5,(offset+16)(addr); \ + lxv 32+v6,(offset+32)(addr); \ + lxv 32+v7,(offset+48)(addr); \ + vminub v14,v4,v5; \ + vminub v15,v6,v7; \ + vminub v16,v14,v15; \ + vcmpequb. v0,v16,v18; \ + beq cr6,$+12; \ + li r7,offset; \ + b L(label); \ + stxv 32+v4,(offset+0)(r11); \ + stxv 32+v5,(offset+16)(r11); \ + stxv 32+v6,(offset+32)(r11); \ + stxv 32+v7,(offset+48)(r11) + +/* Load quadword at addr+offset to vreg, check for NUL bytes, and branch to label if any are found. */ -#define CHECK16(vreg,offset,addr,label) \ - lxv vreg+32,offset(addr); \ - vcmpequb. v6,vreg,v18; \ +#define CHECK_16B(vreg,offset,addr,label) \ + lxv vreg+32,offset(addr); \ + vcmpequb. v15,vreg,v18; \ bne cr6,L(label); -.machine power9 +/* Store vreg2 with length if NUL is found. */ +#define STORE_WITH_LEN(vreg1,vreg2,reg) \ + vctzlsbb r8,vreg1; \ + addi r9,r8,1; \ + sldi r9,r9,56; \ + stxvl 32+vreg2,reg,r9; + +.machine power9 ENTRY_TOCLESS (FUNC_NAME, 4) CALL_MCOUNT 2 - vspltisb v18,0 /* Zeroes in v18 */ - vspltisb v19,-1 /* 0xFF bytes in v19 */ + vspltisb v18,0 /* Zeroes in v18. */ + vspltisb v19,-1 /* 0xFF bytes in v19. */ /* Next 16B-aligned address. Prepare address for L(loop). */ addi r5,r4,16 @@ -70,14 +96,11 @@ ENTRY_TOCLESS (FUNC_NAME, 4) lvsr v1,0,r4 vperm v0,v19,v0,v1 - vcmpequb. v6,v0,v18 /* 0xff if byte is NULL, 0x00 otherwise */ + vcmpequb. v6,v0,v18 /* 0xff if byte is NUL, 0x00 otherwise. */ beq cr6,L(no_null) - /* There's a null byte. */ - vctzlsbb r8,v6 /* Number of trailing zeroes */ - addi r9,r8,1 /* Add null byte. */ - sldi r10,r9,56 /* stxvl wants size in top 8 bits. */ - stxvl 32+v0,r3,r10 /* Partial store */ + /* There's a NUL byte. */ + STORE_WITH_LEN(v6,v0,r3) #ifdef USE_AS_STPCPY /* stpcpy returns the dest address plus the size not counting the @@ -87,17 +110,22 @@ ENTRY_TOCLESS (FUNC_NAME, 4) blr L(no_null): - sldi r10,r8,56 /* stxvl wants size in top 8 bits */ - stxvl 32+v0,r3,r10 /* Partial store */ + sldi r10,r8,56 /* stxvl wants size in top 8 bits. */ + stxvl 32+v0,r3,r10 /* Partial store. */ +/* The main loop is optimized for longer strings(> 512 bytes), + so checking the first bytes in 16B chunks benefits shorter + strings a lot. */ .p2align 4 -L(loop): - CHECK16(v0,0,r5,tail1) - CHECK16(v1,16,r5,tail2) - CHECK16(v2,32,r5,tail3) - CHECK16(v3,48,r5,tail4) - CHECK16(v4,64,r5,tail5) - CHECK16(v5,80,r5,tail6) +L(aligned): + CHECK_16B(v0,0,r5,tail1) + CHECK_16B(v1,16,r5,tail2) + CHECK_16B(v2,32,r5,tail3) + CHECK_16B(v3,48,r5,tail4) + CHECK_16B(v4,64,r5,tail5) + CHECK_16B(v5,80,r5,tail6) + CHECK_16B(v6,96,r5,tail7) + CHECK_16B(v7,112,r5,tail8) stxv 32+v0,0(r11) stxv 32+v1,16(r11) @@ -105,21 +133,146 @@ L(loop): stxv 32+v3,48(r11) stxv 32+v4,64(r11) stxv 32+v5,80(r11) + stxv 32+v6,96(r11) + stxv 32+v7,112(r11) - addi r5,r5,96 - addi r11,r11,96 + addi r11,r11,128 + + CHECK_16B(v0,128,r5,tail1) + CHECK_16B(v1,128+16,r5,tail2) + CHECK_16B(v2,128+32,r5,tail3) + CHECK_16B(v3,128+48,r5,tail4) + CHECK_16B(v4,128+64,r5,tail5) + CHECK_16B(v5,128+80,r5,tail6) + CHECK_16B(v6,128+96,r5,tail7) + CHECK_16B(v7,128+112,r5,tail8) + + stxv 32+v0,0(r11) + stxv 32+v1,16(r11) + stxv 32+v2,32(r11) + stxv 32+v3,48(r11) + stxv 32+v4,64(r11) + stxv 32+v5,80(r11) + stxv 32+v6,96(r11) + stxv 32+v7,112(r11) + + addi r11,r11,128 + + CHECK_16B(v0,256,r5,tail1) + CHECK_16B(v1,256+16,r5,tail2) + CHECK_16B(v2,256+32,r5,tail3) + CHECK_16B(v3,256+48,r5,tail4) + CHECK_16B(v4,256+64,r5,tail5) + CHECK_16B(v5,256+80,r5,tail6) + CHECK_16B(v6,256+96,r5,tail7) + CHECK_16B(v7,256+112,r5,tail8) + + stxv 32+v0,0(r11) + stxv 32+v1,16(r11) + stxv 32+v2,32(r11) + stxv 32+v3,48(r11) + stxv 32+v4,64(r11) + stxv 32+v5,80(r11) + stxv 32+v6,96(r11) + stxv 32+v7,112(r11) + + addi r11,r11,128 + + CHECK_16B(v0,384,r5,tail1) + CHECK_16B(v1,384+16,r5,tail2) + CHECK_16B(v2,384+32,r5,tail3) + CHECK_16B(v3,384+48,r5,tail4) + CHECK_16B(v4,384+64,r5,tail5) + CHECK_16B(v5,384+80,r5,tail6) + CHECK_16B(v6,384+96,r5,tail7) + CHECK_16B(v7,384+112,r5,tail8) + + stxv 32+v0,0(r11) + stxv 32+v1,16(r11) + stxv 32+v2,32(r11) + stxv 32+v3,48(r11) + stxv 32+v4,64(r11) + stxv 32+v5,80(r11) + stxv 32+v6,96(r11) + stxv 32+v7,112(r11) + + /* Align src pointer down to a 64B boundary. */ + addi r5,r4,512 + clrrdi r5,r5,6 + subf r7,r4,r5 + add r11,r3,r7 + +/* Switch to a more aggressive approach checking 64B each time. */ + .p2align 5 +L(strcpy_loop): + CHECK_64B(0,r5,tail_64b) + CHECK_64B(64,r5,tail_64b) + CHECK_64B(128,r5,tail_64b) + CHECK_64B(192,r5,tail_64b) + + CHECK_64B(256,r5,tail_64b) + CHECK_64B(256+64,r5,tail_64b) + CHECK_64B(256+128,r5,tail_64b) + CHECK_64B(256+192,r5,tail_64b) + addi r5,r5,512 + addi r11,r11,512 + + b L(strcpy_loop) + + .p2align 5 +L(tail_64b): + /* OK, we found a NUL byte. Let's look for it in the current 64-byte + block and mark it in its corresponding VR. */ + add r11,r11,r7 + vcmpequb. v8,v4,v18 + beq cr6,L(no_null_16B) + /* There's a NUL byte. */ + STORE_WITH_LEN(v8,v4,r11) +#ifdef USE_AS_STPCPY + add r3,r11,r8 +#endif + blr + +L(no_null_16B): + stxv 32+v4,0(r11) + vcmpequb. v8,v5,v18 + beq cr6,L(no_null_32B) + /* There's a NUL byte. */ + addi r11,r11,16 + STORE_WITH_LEN(v8,v5,r11) +#ifdef USE_AS_STPCPY + add r3,r11,r8 +#endif + blr - b L(loop) +L(no_null_32B): + stxv 32+v5,16(r11) + vcmpequb. v8,v6,v18 + beq cr6,L(no_null_48B) + /* There's a NUL byte. */ + addi r11,r11,32 + STORE_WITH_LEN(v8,v6,r11) +#ifdef USE_AS_STPCPY + add r3,r11,r8 +#endif + blr + +L(no_null_48B): + stxv 32+v6,32(r11) + vcmpequb. v8,v7,v18; + /* There's a NUL byte. */ + addi r11,r11,48 + STORE_WITH_LEN(v8,v7,r11) +#ifdef USE_AS_STPCPY + add r3,r11,r8 +#endif + blr .p2align 4 L(tail1): - vctzlsbb r8,v6 /* Number of trailing zeroes */ - addi r9,r8,1 /* Add null terminator */ - sldi r9,r9,56 /* stxvl wants size in top 8 bits */ - stxvl 32+v0,r11,r9 /* Partial store */ + /* There's a NUL byte. */ + STORE_WITH_LEN(v15,v0,r11) #ifdef USE_AS_STPCPY - /* stpcpy returns the dest address plus the size not counting the - final '\0'. */ add r3,r11,r8 #endif blr @@ -127,11 +280,9 @@ L(tail1): .p2align 4 L(tail2): stxv 32+v0,0(r11) - vctzlsbb r8,v6 - addi r9,r8,1 - sldi r9,r9,56 + /* There's a NUL byte. */ addi r11,r11,16 - stxvl 32+v1,r11,r9 + STORE_WITH_LEN(v15,v1,r11) #ifdef USE_AS_STPCPY add r3,r11,r8 #endif @@ -141,11 +292,8 @@ L(tail2): L(tail3): stxv 32+v0,0(r11) stxv 32+v1,16(r11) - vctzlsbb r8,v6 - addi r9,r8,1 - sldi r9,r9,56 addi r11,r11,32 - stxvl 32+v2,r11,r9 + STORE_WITH_LEN(v15,v2,r11) #ifdef USE_AS_STPCPY add r3,r11,r8 #endif @@ -156,11 +304,8 @@ L(tail4): stxv 32+v0,0(r11) stxv 32+v1,16(r11) stxv 32+v2,32(r11) - vctzlsbb r8,v6 - addi r9,r8,1 - sldi r9,r9,56 addi r11,r11,48 - stxvl 32+v3,r11,r9 + STORE_WITH_LEN(v15,v3,r11) #ifdef USE_AS_STPCPY add r3,r11,r8 #endif @@ -172,11 +317,8 @@ L(tail5): stxv 32+v1,16(r11) stxv 32+v2,32(r11) stxv 32+v3,48(r11) - vctzlsbb r8,v6 - addi r9,r8,1 - sldi r9,r9,56 addi r11,r11,64 - stxvl 32+v4,r11,r9 + STORE_WITH_LEN(v15,v4,r11) #ifdef USE_AS_STPCPY add r3,r11,r8 #endif @@ -189,11 +331,39 @@ L(tail6): stxv 32+v2,32(r11) stxv 32+v3,48(r11) stxv 32+v4,64(r11) - vctzlsbb r8,v6 - addi r9,r8,1 - sldi r9,r9,56 addi r11,r11,80 - stxvl 32+v5,r11,r9 + STORE_WITH_LEN(v15,v5,r11) +#ifdef USE_AS_STPCPY + add r3,r11,r8 +#endif + blr + + .p2align 4 +L(tail7): + stxv 32+v0,0(r11) + stxv 32+v1,16(r11) + stxv 32+v2,32(r11) + stxv 32+v3,48(r11) + stxv 32+v4,64(r11) + stxv 32+v5,80(r11) + addi r11,r11,96 + STORE_WITH_LEN(v15,v6,r11) +#ifdef USE_AS_STPCPY + add r3,r11,r8 +#endif + blr + + .p2align 4 +L(tail8): + stxv 32+v0,0(r11) + stxv 32+v1,16(r11) + stxv 32+v2,32(r11) + stxv 32+v3,48(r11) + stxv 32+v4,64(r11) + stxv 32+v5,80(r11) + stxv 32+v6,96(r11) + addi r11,r11,112 + STORE_WITH_LEN(v15,v7,r11) #ifdef USE_AS_STPCPY add r3,r11,r8 #endif diff --git a/sysdeps/powerpc/powerpc64/sysdep.h b/sysdeps/powerpc/powerpc64/sysdep.h index c363939e1a..c439b06121 100644 --- a/sysdeps/powerpc/powerpc64/sysdep.h +++ b/sysdeps/powerpc/powerpc64/sysdep.h @@ -353,6 +353,25 @@ LT_LABELSUFFIX(name,_name_end): ; \ DO_CALL (SYS_ify (syscall_name)) #ifdef SHARED +# define TAIL_CALL_NO_RETURN(__func) \ + b JUMPTARGET (NOTOC (__func)) +#else +# define TAIL_CALL_NO_RETURN(__func) \ + .ifdef .Local ## __func; \ + b .Local ## __func; \ + .else; \ +.Local ## __func: \ + mflr 0; \ + std 0,FRAME_LR_SAVE(1); \ + stdu 1,-FRAME_MIN_SIZE(1); \ + cfi_adjust_cfa_offset(FRAME_MIN_SIZE); \ + cfi_offset(lr,FRAME_LR_SAVE); \ + bl JUMPTARGET(__func); \ + nop; \ + .endif +#endif + +#ifdef SHARED #define TAIL_CALL_SYSCALL_ERROR \ b JUMPTARGET (NOTOC (__syscall_error)) #else diff --git a/sysdeps/pthread/tst-cancel2.c b/sysdeps/pthread/tst-cancel2.c index ac38b50115..b4f7098235 100644 --- a/sysdeps/pthread/tst-cancel2.c +++ b/sysdeps/pthread/tst-cancel2.c @@ -32,6 +32,10 @@ tf (void *arg) char buf[100000]; while (write (fd[1], buf, sizeof (buf)) > 0); + /* The write can return -1/EPIPE if the pipe was closed before the + thread calls write, which signals a side-effect that must be + signaled to the thread. */ + pthread_testcancel (); return (void *) 42l; } diff --git a/sysdeps/pthread/tst-setuid3.c b/sysdeps/pthread/tst-setuid3.c index 83f42a0ae5..3845ab03d3 100644 --- a/sysdeps/pthread/tst-setuid3.c +++ b/sysdeps/pthread/tst-setuid3.c @@ -15,24 +15,19 @@ License along with the GNU C Library; if not, see <https://www.gnu.org/licenses/>. */ -#include <stdio.h> #include <errno.h> #include <pthread.h> #include <stdbool.h> #include <unistd.h> +#include <support/check.h> + /* The test must run under a non-privileged user ID. */ static const uid_t test_uid = 1; static pthread_barrier_t barrier1; static pthread_barrier_t barrier2; -#define FAIL(fmt, ...) \ - do { printf ("FAIL: " fmt "\n", __VA_ARGS__); _exit (1); } while (0) - -#define FAIL_ERR(fmt, ...) \ - do { printf ("FAIL: " fmt ": %m\n", __VA_ARGS__); _exit (1); } while (0) - /* True if x is not a successful return code from pthread_barrier_wait. */ static inline bool is_invalid_barrier_ret (int x) @@ -45,10 +40,10 @@ thread_func (void *ctx __attribute__ ((unused))) { int ret = pthread_barrier_wait (&barrier1); if (is_invalid_barrier_ret (ret)) - FAIL ("pthread_barrier_wait (barrier1) (on thread): %d", ret); + FAIL_EXIT1 ("pthread_barrier_wait (barrier1) (on thread): %d", ret); ret = pthread_barrier_wait (&barrier2); if (is_invalid_barrier_ret (ret)) - FAIL ("pthread_barrier_wait (barrier2) (on thread): %d", ret); + FAIL_EXIT1 ("pthread_barrier_wait (barrier2) (on thread): %d", ret); return NULL; } @@ -59,13 +54,13 @@ setuid_failure (int phase) switch (ret) { case 0: - FAIL ("setuid succeeded unexpectedly in phase %d", phase); + FAIL_EXIT1 ("setuid succeeded unexpectedly in phase %d", phase); case -1: if (errno != EPERM) - FAIL_ERR ("setuid phase %d", phase); + FAIL_EXIT1 ("setuid phase %d: %m", phase); break; default: - FAIL ("invalid setuid return value in phase %d: %d", phase, ret); + FAIL_EXIT1 ("invalid setuid return value in phase %d: %d", phase, ret); } } @@ -74,42 +69,42 @@ do_test (void) { if (getuid () == 0) if (setuid (test_uid) != 0) - FAIL_ERR ("setuid (%u)", (unsigned) test_uid); + FAIL_EXIT1 ("setuid (%u): %m", (unsigned) test_uid); if (setuid (getuid ())) - FAIL_ERR ("setuid (%s)", "getuid ()"); + FAIL_EXIT1 ("setuid (%s): %m", "getuid ()"); setuid_failure (1); int ret = pthread_barrier_init (&barrier1, NULL, 2); if (ret != 0) - FAIL ("pthread_barrier_init (barrier1): %d", ret); + FAIL_EXIT1 ("pthread_barrier_init (barrier1): %d", ret); ret = pthread_barrier_init (&barrier2, NULL, 2); if (ret != 0) - FAIL ("pthread_barrier_init (barrier2): %d", ret); + FAIL_EXIT1 ("pthread_barrier_init (barrier2): %d", ret); pthread_t thread; ret = pthread_create (&thread, NULL, thread_func, NULL); if (ret != 0) - FAIL ("pthread_create: %d", ret); + FAIL_EXIT1 ("pthread_create: %d", ret); /* Ensure that the thread is running properly. */ ret = pthread_barrier_wait (&barrier1); if (is_invalid_barrier_ret (ret)) - FAIL ("pthread_barrier_wait (barrier1): %d", ret); + FAIL_EXIT1 ("pthread_barrier_wait (barrier1): %d", ret); setuid_failure (2); /* Check success case. */ if (setuid (getuid ()) != 0) - FAIL_ERR ("setuid (%s)", "getuid ()"); + FAIL_EXIT1 ("setuid (%s): %m", "getuid ()"); /* Shutdown. */ ret = pthread_barrier_wait (&barrier2); if (is_invalid_barrier_ret (ret)) - FAIL ("pthread_barrier_wait (barrier2): %d", ret); + FAIL_EXIT1 ("pthread_barrier_wait (barrier2): %d", ret); ret = pthread_join (thread, NULL); if (ret != 0) - FAIL ("pthread_join: %d", ret); + FAIL_EXIT1 ("pthread_join: %d", ret); return 0; } diff --git a/sysdeps/riscv/rvd/libm-test-ulps b/sysdeps/riscv/rvd/libm-test-ulps index b9df07dc8c..233186f29a 100644 --- a/sysdeps/riscv/rvd/libm-test-ulps +++ b/sysdeps/riscv/rvd/libm-test-ulps @@ -976,24 +976,24 @@ float: 1 ldouble: 3 Function: "exp10m1": -double: 2 -float: 1 -ldouble: 1 +double: 4 +float: 2 +ldouble: 3 Function: "exp10m1_downward": -double: 1 -float: 1 -ldouble: 3 +double: 3 +float: 3 +ldouble: 6 Function: "exp10m1_towardzero": -double: 1 -float: 1 -ldouble: 3 +double: 2 +float: 3 +ldouble: 6 Function: "exp10m1_upward": -double: 3 -float: 1 -ldouble: 3 +double: 5 +float: 3 +ldouble: 6 Function: "exp2": double: 1 @@ -1013,24 +1013,24 @@ float: 1 ldouble: 2 Function: "exp2m1": -double: 1 -float: 1 -ldouble: 1 +double: 2 +float: 2 +ldouble: 2 Function: "exp2m1_downward": -double: 1 -float: 1 -ldouble: 2 +double: 3 +float: 3 +ldouble: 3 Function: "exp2m1_towardzero": -double: 2 -float: 1 -ldouble: 2 +double: 3 +float: 2 +ldouble: 4 Function: "exp2m1_upward": -double: 1 -float: 1 -ldouble: 2 +double: 3 +float: 3 +ldouble: 5 Function: "exp_downward": double: 1 @@ -1206,24 +1206,24 @@ float: 2 ldouble: 1 Function: "log10p1": -double: 1 -float: 1 +double: 2 +float: 2 ldouble: 3 Function: "log10p1_downward": double: 2 -float: 1 -ldouble: 2 +float: 3 +ldouble: 4 Function: "log10p1_towardzero": -double: 2 +double: 3 float: 2 -ldouble: 2 +ldouble: 3 Function: "log10p1_upward": double: 2 -float: 1 -ldouble: 3 +float: 3 +ldouble: 4 Function: "log1p": double: 1 @@ -1263,8 +1263,8 @@ double: 3 ldouble: 1 Function: "log2p1": -double: 1 -float: 1 +double: 2 +float: 2 ldouble: 3 Function: "log2p1_downward": @@ -1278,9 +1278,9 @@ float: 2 ldouble: 2 Function: "log2p1_upward": -double: 1 +double: 2 float: 2 -ldouble: 2 +ldouble: 3 Function: "log_downward": ldouble: 1 diff --git a/sysdeps/s390/fpu/libm-test-ulps b/sysdeps/s390/fpu/libm-test-ulps index 4ffd344a4e..e10b5c69ae 100644 --- a/sysdeps/s390/fpu/libm-test-ulps +++ b/sysdeps/s390/fpu/libm-test-ulps @@ -977,24 +977,24 @@ float: 1 ldouble: 3 Function: "exp10m1": -double: 2 -float: 1 -ldouble: 1 +double: 4 +float: 2 +ldouble: 3 Function: "exp10m1_downward": -double: 1 -float: 1 -ldouble: 3 +double: 3 +float: 3 +ldouble: 6 Function: "exp10m1_towardzero": -double: 1 -float: 1 -ldouble: 3 +double: 2 +float: 3 +ldouble: 6 Function: "exp10m1_upward": -double: 3 -float: 1 -ldouble: 3 +double: 5 +float: 3 +ldouble: 6 Function: "exp2": double: 1 @@ -1014,24 +1014,24 @@ float: 1 ldouble: 2 Function: "exp2m1": -double: 1 -float: 1 -ldouble: 1 +double: 2 +float: 2 +ldouble: 2 Function: "exp2m1_downward": -double: 1 -float: 1 -ldouble: 2 +double: 3 +float: 3 +ldouble: 3 Function: "exp2m1_towardzero": -double: 2 -float: 1 -ldouble: 2 +double: 3 +float: 2 +ldouble: 4 Function: "exp2m1_upward": -double: 1 -float: 1 -ldouble: 2 +double: 3 +float: 3 +ldouble: 5 Function: "exp_downward": double: 1 @@ -1207,24 +1207,24 @@ float: 2 ldouble: 1 Function: "log10p1": -double: 1 -float: 1 +double: 2 +float: 2 ldouble: 3 Function: "log10p1_downward": double: 2 -float: 1 -ldouble: 2 +float: 3 +ldouble: 4 Function: "log10p1_towardzero": -double: 2 +double: 3 float: 2 -ldouble: 2 +ldouble: 3 Function: "log10p1_upward": double: 2 -float: 1 -ldouble: 3 +float: 3 +ldouble: 4 Function: "log1p": double: 1 @@ -1262,8 +1262,8 @@ double: 1 ldouble: 1 Function: "log2p1": -double: 1 -float: 1 +double: 2 +float: 2 ldouble: 3 Function: "log2p1_downward": @@ -1277,9 +1277,9 @@ float: 2 ldouble: 2 Function: "log2p1_upward": -double: 1 +double: 2 float: 2 -ldouble: 2 +ldouble: 3 Function: "log_downward": ldouble: 1 diff --git a/sysdeps/sh/sysdep.h b/sysdeps/sh/sysdep.h index 0c9e5626e9..377d29b950 100644 --- a/sysdeps/sh/sysdep.h +++ b/sysdeps/sh/sysdep.h @@ -24,6 +24,7 @@ #define ALIGNARG(log2) log2 #define ASM_SIZE_DIRECTIVE(name) .size name,.-name +#define L(label) .L##label #ifdef SHARED #define PLTJMP(_x) _x##@PLT diff --git a/sysdeps/sparc/fpu/libm-test-ulps b/sysdeps/sparc/fpu/libm-test-ulps index 23523f094f..1bb8b7c5f4 100644 --- a/sysdeps/sparc/fpu/libm-test-ulps +++ b/sysdeps/sparc/fpu/libm-test-ulps @@ -977,24 +977,24 @@ float: 1 ldouble: 3 Function: "exp10m1": -double: 2 -float: 1 -ldouble: 1 +double: 3 +float: 2 +ldouble: 3 Function: "exp10m1_downward": -double: 1 -float: 1 -ldouble: 3 +double: 3 +float: 3 +ldouble: 6 Function: "exp10m1_towardzero": -double: 1 -float: 1 -ldouble: 3 +double: 2 +float: 3 +ldouble: 6 Function: "exp10m1_upward": -double: 3 -float: 1 -ldouble: 3 +double: 4 +float: 4 +ldouble: 6 Function: "exp2": double: 1 @@ -1017,24 +1017,24 @@ float: 1 ldouble: 2 Function: "exp2m1": -double: 1 -float: 1 -ldouble: 1 - -Function: "exp2m1_downward": double: 2 -float: 1 +float: 2 ldouble: 2 +Function: "exp2m1_downward": +double: 3 +float: 3 +ldouble: 3 + Function: "exp2m1_towardzero": -double: 2 -float: 1 -ldouble: 2 +double: 3 +float: 2 +ldouble: 4 Function: "exp2m1_upward": -double: 1 -float: 1 -ldouble: 2 +double: 3 +float: 3 +ldouble: 5 Function: "exp_downward": double: 1 @@ -1212,23 +1212,23 @@ ldouble: 1 Function: "log10p1": double: 1 -float: 1 +float: 2 ldouble: 3 Function: "log10p1_downward": double: 2 -float: 1 -ldouble: 2 +float: 3 +ldouble: 4 Function: "log10p1_towardzero": -double: 2 +double: 3 float: 2 -ldouble: 2 +ldouble: 3 Function: "log10p1_upward": double: 2 -float: 1 -ldouble: 3 +float: 3 +ldouble: 4 Function: "log1p": double: 1 @@ -1272,7 +1272,7 @@ ldouble: 1 Function: "log2p1": double: 1 -float: 1 +float: 2 ldouble: 3 Function: "log2p1_downward": @@ -1286,9 +1286,9 @@ float: 2 ldouble: 2 Function: "log2p1_upward": -double: 1 +double: 2 float: 2 -ldouble: 2 +ldouble: 3 Function: "log_downward": float: 2 diff --git a/sysdeps/unix/sysdep.h b/sysdeps/unix/sysdep.h index a19e84165b..adc8d71f49 100644 --- a/sysdeps/unix/sysdep.h +++ b/sysdeps/unix/sysdep.h @@ -24,6 +24,9 @@ #define SYSCALL__(name, args) PSEUDO (__##name, name, args) #define SYSCALL(name, args) PSEUDO (name, name, args) +#ifndef __ASSEMBLER__ +# include <errno.h> + #define __SYSCALL_CONCAT_X(a,b) a##b #define __SYSCALL_CONCAT(a,b) __SYSCALL_CONCAT_X (a, b) @@ -108,42 +111,148 @@ #define INLINE_SYSCALL_CALL(...) \ __INLINE_SYSCALL_DISP (__INLINE_SYSCALL, __VA_ARGS__) -#if IS_IN (rtld) -/* All cancellation points are compiled out in the dynamic loader. */ -# define NO_SYSCALL_CANCEL_CHECKING 1 +#define __INTERNAL_SYSCALL_NCS0(name) \ + INTERNAL_SYSCALL_NCS (name, 0) +#define __INTERNAL_SYSCALL_NCS1(name, a1) \ + INTERNAL_SYSCALL_NCS (name, 1, a1) +#define __INTERNAL_SYSCALL_NCS2(name, a1, a2) \ + INTERNAL_SYSCALL_NCS (name, 2, a1, a2) +#define __INTERNAL_SYSCALL_NCS3(name, a1, a2, a3) \ + INTERNAL_SYSCALL_NCS (name, 3, a1, a2, a3) +#define __INTERNAL_SYSCALL_NCS4(name, a1, a2, a3, a4) \ + INTERNAL_SYSCALL_NCS (name, 4, a1, a2, a3, a4) +#define __INTERNAL_SYSCALL_NCS5(name, a1, a2, a3, a4, a5) \ + INTERNAL_SYSCALL_NCS (name, 5, a1, a2, a3, a4, a5) +#define __INTERNAL_SYSCALL_NCS6(name, a1, a2, a3, a4, a5, a6) \ + INTERNAL_SYSCALL_NCS (name, 6, a1, a2, a3, a4, a5, a6) +#define __INTERNAL_SYSCALL_NCS7(name, a1, a2, a3, a4, a5, a6, a7) \ + INTERNAL_SYSCALL_NCS (name, 7, a1, a2, a3, a4, a5, a6, a7) + +/* Issue a syscall defined by syscall number plus any other argument required. + It is similar to INTERNAL_SYSCALL_NCS macro, but without the need to pass + the expected argument number as third parameter. */ +#define INTERNAL_SYSCALL_NCS_CALL(...) \ + __INTERNAL_SYSCALL_DISP (__INTERNAL_SYSCALL_NCS, __VA_ARGS__) + +/* Cancellation macros. */ +#include <syscall_types.h> + +/* Adjust both the __syscall_cancel and the SYSCALL_CANCEL macro to support + 7 arguments instead of default 6 (curently only mip32). It avoid add + the requirement to each architecture to support 7 argument macros + {INTERNAL,INLINE}_SYSCALL. */ +#ifdef HAVE_CANCELABLE_SYSCALL_WITH_7_ARGS +# define __SYSCALL_CANCEL7_ARG_DEF __syscall_arg_t a7, +# define __SYSCALL_CANCEL7_ARCH_ARG_DEF ,__syscall_arg_t a7 +# define __SYSCALL_CANCEL7_ARG 0, +# define __SYSCALL_CANCEL7_ARG7 a7, +# define __SYSCALL_CANCEL7_ARCH_ARG7 , a7 #else -# define NO_SYSCALL_CANCEL_CHECKING SINGLE_THREAD_P +# define __SYSCALL_CANCEL7_ARG_DEF +# define __SYSCALL_CANCEL7_ARCH_ARG_DEF +# define __SYSCALL_CANCEL7_ARG +# define __SYSCALL_CANCEL7_ARG7 +# define __SYSCALL_CANCEL7_ARCH_ARG7 #endif +long int __internal_syscall_cancel (__syscall_arg_t a1, __syscall_arg_t a2, + __syscall_arg_t a3, __syscall_arg_t a4, + __syscall_arg_t a5, __syscall_arg_t a6, + __SYSCALL_CANCEL7_ARG_DEF + __syscall_arg_t nr) attribute_hidden; -#define SYSCALL_CANCEL(...) \ - ({ \ - long int sc_ret; \ - if (NO_SYSCALL_CANCEL_CHECKING) \ - sc_ret = INLINE_SYSCALL_CALL (__VA_ARGS__); \ - else \ - { \ - int sc_cancel_oldtype = LIBC_CANCEL_ASYNC (); \ - sc_ret = INLINE_SYSCALL_CALL (__VA_ARGS__); \ - LIBC_CANCEL_RESET (sc_cancel_oldtype); \ - } \ - sc_ret; \ - }) +long int __syscall_cancel (__syscall_arg_t arg1, __syscall_arg_t arg2, + __syscall_arg_t arg3, __syscall_arg_t arg4, + __syscall_arg_t arg5, __syscall_arg_t arg6, + __SYSCALL_CANCEL7_ARG_DEF + __syscall_arg_t nr) attribute_hidden; -/* Issue a syscall defined by syscall number plus any other argument - required. Any error will be returned unmodified (including errno). */ -#define INTERNAL_SYSCALL_CANCEL(...) \ - ({ \ - long int sc_ret; \ - if (NO_SYSCALL_CANCEL_CHECKING) \ - sc_ret = INTERNAL_SYSCALL_CALL (__VA_ARGS__); \ - else \ - { \ - int sc_cancel_oldtype = LIBC_CANCEL_ASYNC (); \ - sc_ret = INTERNAL_SYSCALL_CALL (__VA_ARGS__); \ - LIBC_CANCEL_RESET (sc_cancel_oldtype); \ - } \ - sc_ret; \ - }) +#define __SYSCALL_CANCEL0(name) \ + __syscall_cancel (0, 0, 0, 0, 0, 0, __SYSCALL_CANCEL7_ARG __NR_##name) +#define __SYSCALL_CANCEL1(name, a1) \ + __syscall_cancel (__SSC (a1), 0, 0, 0, 0, 0, \ + __SYSCALL_CANCEL7_ARG __NR_##name) +#define __SYSCALL_CANCEL2(name, a1, a2) \ + __syscall_cancel (__SSC (a1), __SSC (a2), 0, 0, 0, 0, \ + __SYSCALL_CANCEL7_ARG __NR_##name) +#define __SYSCALL_CANCEL3(name, a1, a2, a3) \ + __syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), 0, 0, 0, \ + __SYSCALL_CANCEL7_ARG __NR_##name) +#define __SYSCALL_CANCEL4(name, a1, a2, a3, a4) \ + __syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \ + __SSC(a4), 0, 0, __SYSCALL_CANCEL7_ARG __NR_##name) +#define __SYSCALL_CANCEL5(name, a1, a2, a3, a4, a5) \ + __syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC(a4), \ + __SSC (a5), 0, __SYSCALL_CANCEL7_ARG __NR_##name) +#define __SYSCALL_CANCEL6(name, a1, a2, a3, a4, a5, a6) \ + __syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC (a4), \ + __SSC (a5), __SSC (a6), __SYSCALL_CANCEL7_ARG \ + __NR_##name) +#define __SYSCALL_CANCEL7(name, a1, a2, a3, a4, a5, a6, a7) \ + __syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC (a4), \ + __SSC (a5), __SSC (a6), __SSC (a7), __NR_##name) + +#define __SYSCALL_CANCEL_NARGS_X(a,b,c,d,e,f,g,h,n,...) n +#define __SYSCALL_CANCEL_NARGS(...) \ + __SYSCALL_CANCEL_NARGS_X (__VA_ARGS__,7,6,5,4,3,2,1,0,) +#define __SYSCALL_CANCEL_CONCAT_X(a,b) a##b +#define __SYSCALL_CANCEL_CONCAT(a,b) __SYSCALL_CANCEL_CONCAT_X (a, b) +#define __SYSCALL_CANCEL_DISP(b,...) \ + __SYSCALL_CANCEL_CONCAT (b,__SYSCALL_CANCEL_NARGS(__VA_ARGS__))(__VA_ARGS__) + +/* Issue a cancellable syscall defined first argument plus any other argument + required. If and error occurs its value, the macro returns -1 and sets + errno accordingly. */ +#define __SYSCALL_CANCEL_CALL(...) \ + __SYSCALL_CANCEL_DISP (__SYSCALL_CANCEL, __VA_ARGS__) + +#define __INTERNAL_SYSCALL_CANCEL0(name) \ + __internal_syscall_cancel (0, 0, 0, 0, 0, 0, __SYSCALL_CANCEL7_ARG \ + __NR_##name) +#define __INTERNAL_SYSCALL_CANCEL1(name, a1) \ + __internal_syscall_cancel (__SSC (a1), 0, 0, 0, 0, 0, \ + __SYSCALL_CANCEL7_ARG __NR_##name) +#define __INTERNAL_SYSCALL_CANCEL2(name, a1, a2) \ + __internal_syscall_cancel (__SSC (a1), __SSC (a2), 0, 0, 0, 0, \ + __SYSCALL_CANCEL7_ARG __NR_##name) +#define __INTERNAL_SYSCALL_CANCEL3(name, a1, a2, a3) \ + __internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), 0, \ + 0, 0, __SYSCALL_CANCEL7_ARG __NR_##name) +#define __INTERNAL_SYSCALL_CANCEL4(name, a1, a2, a3, a4) \ + __internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \ + __SSC(a4), 0, 0, \ + __SYSCALL_CANCEL7_ARG __NR_##name) +#define __INTERNAL_SYSCALL_CANCEL5(name, a1, a2, a3, a4, a5) \ + __internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \ + __SSC(a4), __SSC (a5), 0, \ + __SYSCALL_CANCEL7_ARG __NR_##name) +#define __INTERNAL_SYSCALL_CANCEL6(name, a1, a2, a3, a4, a5, a6) \ + __internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \ + __SSC (a4), __SSC (a5), __SSC (a6), \ + __SYSCALL_CANCEL7_ARG __NR_##name) +#define __INTERNAL_SYSCALL_CANCEL7(name, a1, a2, a3, a4, a5, a6, a7) \ + __internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \ + __SSC (a4), __SSC (a5), __SSC (a6), \ + __SSC (a7), __NR_##name) + +/* Issue a cancellable syscall defined by syscall number NAME plus any other + argument required. If an error occurs its value is returned as an negative + number unmodified and errno is not set. */ +#define __INTERNAL_SYSCALL_CANCEL_CALL(...) \ + __SYSCALL_CANCEL_DISP (__INTERNAL_SYSCALL_CANCEL, __VA_ARGS__) + +#if IS_IN (rtld) +/* The loader does not need to handle thread cancellation, use direct + syscall instead. */ +# define INTERNAL_SYSCALL_CANCEL(...) INTERNAL_SYSCALL_CALL(__VA_ARGS__) +# define SYSCALL_CANCEL(...) INLINE_SYSCALL_CALL (__VA_ARGS__) +#else +# define INTERNAL_SYSCALL_CANCEL(...) \ + __INTERNAL_SYSCALL_CANCEL_CALL (__VA_ARGS__) +# define SYSCALL_CANCEL(...) \ + __SYSCALL_CANCEL_CALL (__VA_ARGS__) +#endif + +#endif /* __ASSEMBLER__ */ /* Machine-dependent sysdep.h files are expected to define the macro PSEUDO (function_name, syscall_name) to emit assembly code to define the diff --git a/sysdeps/unix/sysv/linux/Makefile b/sysdeps/unix/sysv/linux/Makefile index 097b5a26fc..7df51a325c 100644 --- a/sysdeps/unix/sysv/linux/Makefile +++ b/sysdeps/unix/sysv/linux/Makefile @@ -95,6 +95,8 @@ sysdep_routines += \ process_vm_writev \ pselect32 \ readahead \ + sched_getattr \ + sched_setattr \ setfsgid \ setfsuid \ setvmaname \ @@ -206,6 +208,7 @@ tests += \ tst-getauxval \ tst-gettid \ tst-gettid-kill \ + tst-linux-mremap1 \ tst-memfd_create \ tst-misalign-clone \ tst-mlock2 \ @@ -223,6 +226,7 @@ tests += \ tst-process_mrelease \ tst-quota \ tst-rlimit-infinity \ + tst-sched_setattr \ tst-scm_rights \ tst-sigtimedwait \ tst-sync_file_range \ @@ -394,6 +398,16 @@ $(objpfx)tst-mount-compile.out: ../sysdeps/unix/sysv/linux/tst-mount-compile.py < /dev/null > $@ 2>&1; $(evaluate-test) $(objpfx)tst-mount-compile.out: $(sysdeps-linux-python-deps) +tests-special += \ + $(objpfx)tst-sched-consts.out \ + # tests-special +$(objpfx)tst-sched-consts.out: ../sysdeps/unix/sysv/linux/tst-sched-consts.py + $(sysdeps-linux-python) \ + ../sysdeps/unix/sysv/linux/tst-sched-consts.py \ + $(sysdeps-linux-python-cc) \ + < /dev/null > $@ 2>&1; $(evaluate-test) +$(objpfx)tst-sched-consts.out: $(sysdeps-linux-python-deps) + tst-rseq-disable-ENV = GLIBC_TUNABLES=glibc.pthread.rseq=0 endif # $(subdir) == misc diff --git a/sysdeps/unix/sysv/linux/Versions b/sysdeps/unix/sysv/linux/Versions index 268ba1b6ac..213ff5f1fe 100644 --- a/sysdeps/unix/sysv/linux/Versions +++ b/sysdeps/unix/sysv/linux/Versions @@ -328,6 +328,10 @@ libc { posix_spawnattr_getcgroup_np; posix_spawnattr_setcgroup_np; } + GLIBC_2.41 { + sched_getattr; + sched_setattr; + } GLIBC_PRIVATE { # functions used in other libraries __syscall_rt_sigqueueinfo; diff --git a/sysdeps/unix/sysv/linux/aarch64/arch-syscall.h b/sysdeps/unix/sysv/linux/aarch64/arch-syscall.h index 7ee8a2167a..19b6316cb6 100644 --- a/sysdeps/unix/sysv/linux/aarch64/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/aarch64/arch-syscall.h @@ -158,6 +158,7 @@ #define __NR_mq_timedsend 182 #define __NR_mq_unlink 181 #define __NR_mremap 216 +#define __NR_mseal 462 #define __NR_msgctl 187 #define __NR_msgget 186 #define __NR_msgrcv 188 diff --git a/sysdeps/unix/sysv/linux/aarch64/libc.abilist b/sysdeps/unix/sysv/linux/aarch64/libc.abilist index 68eeca1c08..38db77e4f7 100644 --- a/sysdeps/unix/sysv/linux/aarch64/libc.abilist +++ b/sysdeps/unix/sysv/linux/aarch64/libc.abilist @@ -2748,3 +2748,5 @@ GLIBC_2.39 stdc_trailing_zeros_ui F GLIBC_2.39 stdc_trailing_zeros_ul F GLIBC_2.39 stdc_trailing_zeros_ull F GLIBC_2.39 stdc_trailing_zeros_us F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F diff --git a/sysdeps/unix/sysv/linux/aarch64/syscall_cancel.S b/sysdeps/unix/sysv/linux/aarch64/syscall_cancel.S new file mode 100644 index 0000000000..e91a431b36 --- /dev/null +++ b/sysdeps/unix/sysv/linux/aarch64/syscall_cancel.S @@ -0,0 +1,59 @@ +/* Cancellable syscall wrapper. Linux/AArch64 version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int [x0] __syscall_cancel_arch (int *cancelhandling [x0], + long int nr [x1], + long int arg1 [x2], + long int arg2 [x3], + long int arg3 [x4], + long int arg4 [x5], + long int arg5 [x6], + long int arg6 [x7]) */ + +ENTRY (__syscall_cancel_arch) + + .globl __syscall_cancel_arch_start +__syscall_cancel_arch_start: + + /* if (*cancelhandling & CANCELED_BITMASK) + __syscall_do_cancel() */ + ldr w0, [x0] + tbnz w0, TCB_CANCELED_BIT, 1f + + /* Issue a 6 argument syscall, the nr [x1] being the syscall + number. */ + mov x8, x1 + mov x0, x2 + mov x1, x3 + mov x2, x4 + mov x3, x5 + mov x4, x6 + mov x5, x7 + svc 0x0 + + .globl __syscall_cancel_arch_end +__syscall_cancel_arch_end: + ret + +1: + b __syscall_do_cancel + +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/alpha/arch-syscall.h b/sysdeps/unix/sysv/linux/alpha/arch-syscall.h index 0f4ea7670b..216a5575c6 100644 --- a/sysdeps/unix/sysv/linux/alpha/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/alpha/arch-syscall.h @@ -24,6 +24,7 @@ #define __NR_clock_nanosleep 422 #define __NR_clock_settime 419 #define __NR_clone 312 +#define __NR_clone3 545 #define __NR_close 6 #define __NR_close_range 546 #define __NR_connect 98 @@ -189,6 +190,7 @@ #define __NR_mq_timedsend 434 #define __NR_mq_unlink 433 #define __NR_mremap 341 +#define __NR_mseal 572 #define __NR_msgctl 200 #define __NR_msgget 201 #define __NR_msgrcv 202 diff --git a/sysdeps/unix/sysv/linux/alpha/libc.abilist b/sysdeps/unix/sysv/linux/alpha/libc.abilist index 34c187b721..637bfce9fb 100644 --- a/sysdeps/unix/sysv/linux/alpha/libc.abilist +++ b/sysdeps/unix/sysv/linux/alpha/libc.abilist @@ -3095,6 +3095,8 @@ GLIBC_2.4 wcstold F GLIBC_2.4 wcstold_l F GLIBC_2.4 wprintf F GLIBC_2.4 wscanf F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/alpha/syscall_cancel.S b/sysdeps/unix/sysv/linux/alpha/syscall_cancel.S new file mode 100644 index 0000000000..377eef48be --- /dev/null +++ b/sysdeps/unix/sysv/linux/alpha/syscall_cancel.S @@ -0,0 +1,80 @@ +/* Cancellable syscall wrapper. Linux/alpha version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int __syscall_cancel_arch (int *ch, + __syscall_arg_t nr, + __syscall_arg_t arg1, + __syscall_arg_t arg2, + __syscall_arg_t arg3, + __syscall_arg_t arg4, + __syscall_arg_t arg5, + __syscall_arg_t arg6) */ + + .set noreorder + .set noat + .set nomacro +ENTRY (__syscall_cancel_arch) + .frame sp, 16, ra, 0 + .mask 0x4000000,-16 + cfi_startproc + ldah gp, 0(t12) + lda gp, 0(gp) + lda sp, -16(sp) + cfi_def_cfa_offset (16) + mov a1, v0 + stq ra, 0(sp) + cfi_offset (26, -16) + .prologue 1 + + .global __syscall_cancel_arch_start +__syscall_cancel_arch_start: + ldl t0, 0(a0) + addl zero, t0, t0 + /* if (*ch & CANCELED_BITMASK) */ + and t0, TCB_CANCELED_BITMASK, t0 + bne t0, 1f + mov a2, a0 + mov a3, a1 + mov a4, a2 + ldq a4, 16(sp) + mov a5, a3 + ldq a5, 24(sp) + .set macro + callsys + .set nomacro + + .global __syscall_cancel_arch_end +__syscall_cancel_arch_end: + subq zero, v0, t0 + ldq ra, 0(sp) + cmovne a3, t0, v0 + lda sp, 16(sp) + cfi_remember_state + cfi_restore (26) + cfi_def_cfa_offset (0) + ret zero, (ra), 1 + .align 4 +1: + cfi_restore_state + ldq t12, __syscall_do_cancel(gp) !literal!2 + jsr ra, (t12), __syscall_do_cancel !lituse_jsr!2 + cfi_endproc +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/arc/arch-syscall.h b/sysdeps/unix/sysv/linux/arc/arch-syscall.h index 90359482a8..ea581b0a6d 100644 --- a/sysdeps/unix/sysv/linux/arc/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/arc/arch-syscall.h @@ -161,6 +161,7 @@ #define __NR_mq_timedsend_time64 418 #define __NR_mq_unlink 181 #define __NR_mremap 216 +#define __NR_mseal 462 #define __NR_msgctl 187 #define __NR_msgget 186 #define __NR_msgrcv 188 diff --git a/sysdeps/unix/sysv/linux/arc/libc.abilist b/sysdeps/unix/sysv/linux/arc/libc.abilist index 916c18ea94..4a305cf730 100644 --- a/sysdeps/unix/sysv/linux/arc/libc.abilist +++ b/sysdeps/unix/sysv/linux/arc/libc.abilist @@ -2509,3 +2509,5 @@ GLIBC_2.39 stdc_trailing_zeros_ui F GLIBC_2.39 stdc_trailing_zeros_ul F GLIBC_2.39 stdc_trailing_zeros_ull F GLIBC_2.39 stdc_trailing_zeros_us F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F diff --git a/sysdeps/unix/sysv/linux/arc/syscall_cancel.S b/sysdeps/unix/sysv/linux/arc/syscall_cancel.S new file mode 100644 index 0000000000..fa02af4163 --- /dev/null +++ b/sysdeps/unix/sysv/linux/arc/syscall_cancel.S @@ -0,0 +1,56 @@ +/* Cancellable syscall wrapper. Linux/ARC version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int __syscall_cancel_arch (int *cancelhandling, + __syscall_arg_t nr, + __syscall_arg_t arg1, + __syscall_arg_t arg2, + __syscall_arg_t arg3, + __syscall_arg_t arg4, + __syscall_arg_t arg5, + __syscall_arg_t arg6) */ + +ENTRY (__syscall_cancel_arch) + + .globl __syscall_cancel_arch_start +__syscall_cancel_arch_start: + ld_s r12,[r0] + bbit1 r12, TCB_CANCELED_BITMASK, 1f + mov_s r8, r1 + mov_s r0, r2 + mov_s r1, r3 + mov_s r2, r4 + mov_s r3, r5 + mov_s r4, r6 + mov_s r5, r7 + trap_s 0 + + .globl __syscall_cancel_arch_end +__syscall_cancel_arch_end: + j_s [blink] + + .align 4 +1: push_s blink + cfi_def_cfa_offset (4) + cfi_offset (31, -4) + bl @__syscall_do_cancel + +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/arm/arch-syscall.h b/sysdeps/unix/sysv/linux/arm/arch-syscall.h index 4930167a03..2809f52f94 100644 --- a/sysdeps/unix/sysv/linux/arm/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/arm/arch-syscall.h @@ -205,6 +205,7 @@ #define __NR_mq_timedsend_time64 418 #define __NR_mq_unlink 275 #define __NR_mremap 163 +#define __NR_mseal 462 #define __NR_msgctl 304 #define __NR_msgget 303 #define __NR_msgrcv 302 diff --git a/sysdeps/unix/sysv/linux/arm/be/libc.abilist b/sysdeps/unix/sysv/linux/arm/be/libc.abilist index ea95de282a..1d54f71b14 100644 --- a/sysdeps/unix/sysv/linux/arm/be/libc.abilist +++ b/sysdeps/unix/sysv/linux/arm/be/libc.abilist @@ -2801,6 +2801,8 @@ GLIBC_2.4 xdrstdio_create F GLIBC_2.4 xencrypt F GLIBC_2.4 xprt_register F GLIBC_2.4 xprt_unregister F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/arm/le/libc.abilist b/sysdeps/unix/sysv/linux/arm/le/libc.abilist index 1cdbc983e1..ff7e8bc40b 100644 --- a/sysdeps/unix/sysv/linux/arm/le/libc.abilist +++ b/sysdeps/unix/sysv/linux/arm/le/libc.abilist @@ -2798,6 +2798,8 @@ GLIBC_2.4 xdrstdio_create F GLIBC_2.4 xencrypt F GLIBC_2.4 xprt_register F GLIBC_2.4 xprt_unregister F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/arm/syscall_cancel.S b/sysdeps/unix/sysv/linux/arm/syscall_cancel.S new file mode 100644 index 0000000000..6b899306e3 --- /dev/null +++ b/sysdeps/unix/sysv/linux/arm/syscall_cancel.S @@ -0,0 +1,78 @@ +/* Cancellable syscall wrapper. Linux/arm version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int [r0] __syscall_cancel_arch (int *cancelhandling [r0], + long int nr [r1], + long int arg1 [r2], + long int arg2 [r3], + long int arg3 [SP], + long int arg4 [SP+4], + long int arg5 [SP+8], + long int arg6 [SP+12]) */ + + .syntax unified + +ENTRY (__syscall_cancel_arch) + .fnstart + mov ip, sp + stmfd sp!, {r4, r5, r6, r7, lr} + .save {r4, r5, r6, r7, lr} + + cfi_adjust_cfa_offset (20) + cfi_rel_offset (r4, 0) + cfi_rel_offset (r5, 4) + cfi_rel_offset (r6, 8) + cfi_rel_offset (r7, 12) + cfi_rel_offset (lr, 16) + + .globl __syscall_cancel_arch_start +__syscall_cancel_arch_start: + + /* if (*cancelhandling & CANCELED_BITMASK) + __syscall_do_cancel() */ + ldr r0, [r0] + tst r0, #TCB_CANCELED_BITMASK + bne 1f + + /* Issue a 6 argument syscall, the nr [r1] being the syscall + number. */ + mov r7, r1 + mov r0, r2 + mov r1, r3 + ldmfd ip, {r2, r3, r4, r5, r6} + svc 0x0 + + .globl __syscall_cancel_arch_end +__syscall_cancel_arch_end: + ldmfd sp!, {r4, r5, r6, r7, lr} + cfi_adjust_cfa_offset (-20) + cfi_restore (r4) + cfi_restore (r5) + cfi_restore (r6) + cfi_restore (r7) + cfi_restore (lr) + BX (lr) + +1: + ldmfd sp!, {r4, r5, r6, r7, lr} + b __syscall_do_cancel + .fnend +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/bits/fcntl-linux.h b/sysdeps/unix/sysv/linux/bits/fcntl-linux.h index 628612b885..61c100d984 100644 --- a/sysdeps/unix/sysv/linux/bits/fcntl-linux.h +++ b/sysdeps/unix/sysv/linux/bits/fcntl-linux.h @@ -200,6 +200,7 @@ # define F_SETLEASE 1024 /* Set a lease. */ # define F_GETLEASE 1025 /* Enquire what lease is active. */ # define F_NOTIFY 1026 /* Request notifications on a directory. */ +# define F_DUPFD_QUERY 1027 /* Compare two file descriptors for sameness. */ # define F_SETPIPE_SZ 1031 /* Set pipe page size array. */ # define F_GETPIPE_SZ 1032 /* Set pipe page size array. */ # define F_ADD_SEALS 1033 /* Add seals to file. */ diff --git a/sysdeps/unix/sysv/linux/bits/sched.h b/sysdeps/unix/sysv/linux/bits/sched.h index a53e1362a0..a02cb69de7 100644 --- a/sysdeps/unix/sysv/linux/bits/sched.h +++ b/sysdeps/unix/sysv/linux/bits/sched.h @@ -29,15 +29,58 @@ #define SCHED_FIFO 1 #define SCHED_RR 2 #ifdef __USE_GNU +# define SCHED_NORMAL SCHED_OTHER # define SCHED_BATCH 3 # define SCHED_ISO 4 # define SCHED_IDLE 5 # define SCHED_DEADLINE 6 +/* Flags that can be used in policy values. */ # define SCHED_RESET_ON_FORK 0x40000000 -#endif -#ifdef __USE_GNU +/* Flags for the sched_flags field in struct sched_attr. */ +#define SCHED_FLAG_RESET_ON_FORK 0x01 +#define SCHED_FLAG_RECLAIM 0x02 +#define SCHED_FLAG_DL_OVERRUN 0x04 +#define SCHED_FLAG_KEEP_POLICY 0x08 +#define SCHED_FLAG_KEEP_PARAMS 0x10 +#define SCHED_FLAG_UTIL_CLAMP_MIN 0x20 +#define SCHED_FLAG_UTIL_CLAMP_MAX 0x40 + +/* Combinations of sched_flags fields. */ +#define SCHED_FLAG_KEEP_ALL 0x18 +#define SCHED_FLAG_UTIL_CLAMP 0x60 + +/* Use "" to work around incorrect macro expansion of the + __has_include argument (GCC PR 80005). */ +# ifdef __has_include +# if __has_include ("linux/sched/types.h") +/* Some older Linux versions defined sched_param in <linux/sched/types.h>. */ +# define sched_param __glibc_mask_sched_param +# include <linux/sched/types.h> +# undef sched_param +# endif +# endif +# ifndef SCHED_ATTR_SIZE_VER0 +# include <linux/types.h> +# define SCHED_ATTR_SIZE_VER0 48 +# define SCHED_ATTR_SIZE_VER1 56 +struct sched_attr +{ + __u32 size; + __u32 sched_policy; + __u64 sched_flags; + __s32 sched_nice; + __u32 sched_priority; + __u64 sched_runtime; + __u64 sched_deadline; + __u64 sched_period; + __u32 sched_util_min; + __u32 sched_util_max; + /* Additional fields may be added at the end. */ +}; +# endif /* !SCHED_ATTR_SIZE_VER0 */ + /* Cloning flags. */ # define CSIGNAL 0x000000ff /* Signal mask to be sent at exit. */ # define CLONE_VM 0x00000100 /* Set if VM shared between processes. */ @@ -97,6 +140,17 @@ extern int getcpu (unsigned int *, unsigned int *) __THROW; /* Switch process to namespace of type NSTYPE indicated by FD. */ extern int setns (int __fd, int __nstype) __THROW; + +/* Apply the scheduling attributes from *ATTR to the process or thread TID. */ +int sched_setattr (pid_t tid, struct sched_attr *attr, unsigned int flags) + __THROW __nonnull ((2)); + +/* Obtain the scheduling attributes of the process or thread TID and + store it in *ATTR. */ +int sched_getattr (pid_t tid, struct sched_attr *attr, unsigned int size, + unsigned int flags) + __THROW __nonnull ((2)) __attr_access ((__write_only__, 2, 3)); + #endif __END_DECLS diff --git a/sysdeps/unix/sysv/linux/csky/arch-syscall.h b/sysdeps/unix/sysv/linux/csky/arch-syscall.h index 3f16a29f57..ede3551a00 100644 --- a/sysdeps/unix/sysv/linux/csky/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/csky/arch-syscall.h @@ -168,6 +168,7 @@ #define __NR_mq_timedsend_time64 418 #define __NR_mq_unlink 181 #define __NR_mremap 216 +#define __NR_mseal 462 #define __NR_msgctl 187 #define __NR_msgget 186 #define __NR_msgrcv 188 @@ -301,7 +302,7 @@ #define __NR_swapon 224 #define __NR_symlinkat 36 #define __NR_sync 81 -#define __NR_sync_file_range 84 +#define __NR_sync_file_range2 84 #define __NR_syncfs 267 #define __NR_sysinfo 179 #define __NR_syslog 116 diff --git a/sysdeps/unix/sysv/linux/csky/libc.abilist b/sysdeps/unix/sysv/linux/csky/libc.abilist index 96d45961e2..c3ed65467d 100644 --- a/sysdeps/unix/sysv/linux/csky/libc.abilist +++ b/sysdeps/unix/sysv/linux/csky/libc.abilist @@ -2785,3 +2785,5 @@ GLIBC_2.39 stdc_trailing_zeros_ui F GLIBC_2.39 stdc_trailing_zeros_ul F GLIBC_2.39 stdc_trailing_zeros_ull F GLIBC_2.39 stdc_trailing_zeros_us F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F diff --git a/sysdeps/unix/sysv/linux/csky/syscall_cancel.S b/sysdeps/unix/sysv/linux/csky/syscall_cancel.S new file mode 100644 index 0000000000..2989765f8c --- /dev/null +++ b/sysdeps/unix/sysv/linux/csky/syscall_cancel.S @@ -0,0 +1,114 @@ +/* Cancellable syscall wrapper. Linux/csky version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int __syscall_cancel_arch (int *cancelhandling, + __syscall_arg_t nr, + __syscall_arg_t arg1, + __syscall_arg_t arg2, + __syscall_arg_t arg3, + __syscall_arg_t arg4, + __syscall_arg_t arg5, + __syscall_arg_t arg6) */ + +#ifdef SHARED +# define STACK_ADJ 4 +#else +# define STACK_ADJ 0 +#endif + +ENTRY (__syscall_cancel_arch) + subi sp, sp, 16 + STACK_ADJ + cfi_def_cfa_offset (16 + STACK_ADJ) +#ifdef SHARED + st.w gb, (sp, 16) + lrw t1, 1f@GOTPC + cfi_offset (gb, -4) + grs gb, 1f +1: +#endif + st.w lr, (sp, 12) + st.w l3, (sp, 8) + st.w l1, (sp, 4) + st.w l0, (sp, 0) +#ifdef SHARED + addu gb, gb, t1 +#endif + subi sp, sp, 16 + cfi_def_cfa_offset (32 + STACK_ADJ) + cfi_offset (lr, -( 4 + STACK_ADJ)) + cfi_offset (l3, -( 8 + STACK_ADJ)) + cfi_offset (l1, -(12 + STACK_ADJ)) + cfi_offset (l0, -(16 + STACK_ADJ)) + + mov l3, a1 + mov a1, a3 + ld.w a3, (sp, 32 + STACK_ADJ) + st.w a3, (sp, 0) + ld.w a3, (sp, 36 + STACK_ADJ) + st.w a3, (sp, 4) + ld.w a3, (sp, 40 + STACK_ADJ) + st.w a3, (sp, 8) + ld.w a3, (sp, 44 + STACK_ADJ) + st.w a3, (sp, 12) + + .globl __syscall_cancel_arch_start +__syscall_cancel_arch_start: + ld.w t0, (a0, 0) + andi t0, t0, TCB_CANCELED_BITMASK + jbnez t0, 2f + mov a0, a2 + ld.w a3, (sp, 4) + ld.w a2, (sp, 0) + ld.w l0, (sp, 8) + ld.w l1, (sp, 12) + trap 0 + + .globl __syscall_cancel_arch_end +__syscall_cancel_arch_end: + addi sp, sp, 16 + cfi_remember_state + cfi_def_cfa_offset (16 + STACK_ADJ) +#ifdef SHARED + ld.w gb, (sp, 16) + cfi_restore (gb) +#endif + ld.w lr, (sp, 12) + cfi_restore (lr) + ld.w l3, (sp, 8) + cfi_restore (l3) + ld.w l1, (sp, 4) + cfi_restore (l1) + ld.w l0, (sp, 0) + cfi_restore (l0) + addi sp, sp, 16 + cfi_def_cfa_offset (0) + rts + +2: + cfi_restore_state +#ifdef SHARED + lrw a3, __syscall_do_cancel@GOTOFF + addu a3, a3, gb + jsr a3 +#else + jbsr __syscall_do_cancel +#endif +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/hppa/arch-syscall.h b/sysdeps/unix/sysv/linux/hppa/arch-syscall.h index a1b2c819d6..08b153f2cc 100644 --- a/sysdeps/unix/sysv/linux/hppa/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/hppa/arch-syscall.h @@ -197,6 +197,7 @@ #define __NR_mq_timedsend_time64 418 #define __NR_mq_unlink 230 #define __NR_mremap 163 +#define __NR_mseal 462 #define __NR_msgctl 191 #define __NR_msgget 190 #define __NR_msgrcv 189 diff --git a/sysdeps/unix/sysv/linux/hppa/libc.abilist b/sysdeps/unix/sysv/linux/hppa/libc.abilist index fbcd60c2b3..8de7644a59 100644 --- a/sysdeps/unix/sysv/linux/hppa/libc.abilist +++ b/sysdeps/unix/sysv/linux/hppa/libc.abilist @@ -2821,6 +2821,8 @@ GLIBC_2.4 sys_errlist D 0x400 GLIBC_2.4 sys_nerr D 0x4 GLIBC_2.4 unlinkat F GLIBC_2.4 unshare F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/hppa/syscall_cancel.S b/sysdeps/unix/sysv/linux/hppa/syscall_cancel.S new file mode 100644 index 0000000000..b9c19747ea --- /dev/null +++ b/sysdeps/unix/sysv/linux/hppa/syscall_cancel.S @@ -0,0 +1,81 @@ +/* Cancellable syscall wrapper. Linux/hppa version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library. If not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int __syscall_cancel_arch (int *cancelhandling, + long int nr, + long int arg1, + long int arg2, + long int arg3, + long int arg4, + long int arg5, + long int arg6) */ + + .text +ENTRY(__syscall_cancel_arch) + stw %r2,-20(%r30) + ldo 128(%r30),%r30 + cfi_def_cfa_offset (-128) + cfi_offset (2, -20) + ldw -180(%r30),%r28 + copy %r26,%r20 + stw %r28,-108(%r30) + ldw -184(%r30),%r28 + copy %r24,%r26 + stw %r28,-112(%r30) + ldw -188(%r30),%r28 + stw %r28,-116(%r30) + ldw -192(%r30),%r28 + stw %r4,-104(%r30) + stw %r28,-120(%r30) + copy %r25,%r28 + copy %r23,%r25 +#ifdef __PIC__ + stw %r19,-32(%r30) +#endif + cfi_offset (4, 24) + + .global __syscall_cancel_arch_start +__syscall_cancel_arch_start: + ldw 0(%r20),%r20 + bb,< %r20,31-TCB_CANCELED_BIT,1f + ldw -120(%r30),%r21 + ldw -116(%r30),%r22 + ldw -112(%r30),%r23 + ldw -108(%r30),%r24 + copy %r19, %r4 + ble 0x100(%sr2, %r0) + + .global __syscall_cancel_arch_end +__syscall_cancel_arch_end: + + copy %r28,%r20 + copy %r4,%r19 + + ldw -148(%r30),%r2 + ldw -104(%r30),%r4 + bv %r0(%r2) + ldo -128(%r30),%r30 +1: + bl __syscall_do_cancel,%r2 + nop + nop + +END(__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/i386/arch-syscall.h b/sysdeps/unix/sysv/linux/i386/arch-syscall.h index cc775432d6..500ca1ec70 100644 --- a/sysdeps/unix/sysv/linux/i386/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/i386/arch-syscall.h @@ -222,6 +222,7 @@ #define __NR_mq_timedsend_time64 418 #define __NR_mq_unlink 278 #define __NR_mremap 163 +#define __NR_mseal 462 #define __NR_msgctl 402 #define __NR_msgget 399 #define __NR_msgrcv 401 diff --git a/sysdeps/unix/sysv/linux/i386/libc.abilist b/sysdeps/unix/sysv/linux/i386/libc.abilist index c989b433c0..4fedf775d4 100644 --- a/sysdeps/unix/sysv/linux/i386/libc.abilist +++ b/sysdeps/unix/sysv/linux/i386/libc.abilist @@ -3005,6 +3005,8 @@ GLIBC_2.4 sys_errlist D 0x210 GLIBC_2.4 sys_nerr D 0x4 GLIBC_2.4 unlinkat F GLIBC_2.4 unshare F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/i386/syscall_cancel.S b/sysdeps/unix/sysv/linux/i386/syscall_cancel.S new file mode 100644 index 0000000000..46fb746da0 --- /dev/null +++ b/sysdeps/unix/sysv/linux/i386/syscall_cancel.S @@ -0,0 +1,104 @@ +/* Cancellable syscall wrapper. Linux/i686 version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int [eax] __syscall_cancel_arch (int *cancelhandling [SP], + long int nr [SP+4], + long int arg1 [SP+8], + long int arg2 [SP+12], + long int arg3 [SP+16], + long int arg4 [SP+20], + long int arg5 [SP+24], + long int arg6 [SP+28]) */ + +ENTRY (__syscall_cancel_arch) + pushl %ebp + cfi_def_cfa_offset (8) + cfi_offset (ebp, -8) + pushl %edi + cfi_def_cfa_offset (12) + cfi_offset (edi, -12) + pushl %esi + cfi_def_cfa_offset (16) + cfi_offset (esi, -16) + pushl %ebx + cfi_def_cfa_offset (20) + cfi_offset (ebx, -20) + + .global __syscall_cancel_arch_start +__syscall_cancel_arch_start: + + /* if (*cancelhandling & CANCELED_BITMASK) + __syscall_do_cancel() */ + movl 20(%esp), %eax + testb $TCB_CANCELED_BITMASK, (%eax) + jne 1f + + /* Issue a 6 argument syscall, the nr [%eax] being the syscall + number. */ + movl 24(%esp), %eax + movl 28(%esp), %ebx + movl 32(%esp), %ecx + movl 36(%esp), %edx + movl 40(%esp), %esi + movl 44(%esp), %edi + movl 48(%esp), %ebp + + /* We can not use the vDSO helper for syscall (__kernel_vsyscall) + because the returned PC from kernel will point to the vDSO page + instead of the expected __syscall_cancel_arch_{start,end} + marks. */ + int $0x80 + + .global __syscall_cancel_arch_end +__syscall_cancel_arch_end: + + popl %ebx + cfi_restore (ebx) + cfi_def_cfa_offset (16) + popl %esi + cfi_restore (esi) + cfi_def_cfa_offset (12) + popl %edi + cfi_restore (edi) + cfi_def_cfa_offset (8) + popl %ebp + cfi_restore (ebp) + cfi_def_cfa_offset (4) + ret + +1: + /* Although the __syscall_do_cancel do not return, we need to stack + being set correctly for unwind. */ + popl %ebx + cfi_restore (ebx) + cfi_def_cfa_offset (16) + popl %esi + cfi_restore (esi) + cfi_def_cfa_offset (12) + popl %edi + cfi_restore (edi) + cfi_def_cfa_offset (8) + popl %ebp + cfi_restore (ebp) + cfi_def_cfa_offset (4) + jmp __syscall_do_cancel + +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/loongarch/arch-syscall.h b/sysdeps/unix/sysv/linux/loongarch/arch-syscall.h index 56bb08718a..8bb82448a7 100644 --- a/sysdeps/unix/sysv/linux/loongarch/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/loongarch/arch-syscall.h @@ -155,6 +155,7 @@ #define __NR_mq_timedsend 182 #define __NR_mq_unlink 181 #define __NR_mremap 216 +#define __NR_mseal 462 #define __NR_msgctl 187 #define __NR_msgget 186 #define __NR_msgrcv 188 diff --git a/sysdeps/unix/sysv/linux/loongarch/lp64/libc.abilist b/sysdeps/unix/sysv/linux/loongarch/lp64/libc.abilist index 0023ec1fa1..0024282289 100644 --- a/sysdeps/unix/sysv/linux/loongarch/lp64/libc.abilist +++ b/sysdeps/unix/sysv/linux/loongarch/lp64/libc.abilist @@ -2269,3 +2269,5 @@ GLIBC_2.39 stdc_trailing_zeros_ui F GLIBC_2.39 stdc_trailing_zeros_ul F GLIBC_2.39 stdc_trailing_zeros_ull F GLIBC_2.39 stdc_trailing_zeros_us F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F diff --git a/sysdeps/unix/sysv/linux/loongarch/syscall_cancel.S b/sysdeps/unix/sysv/linux/loongarch/syscall_cancel.S new file mode 100644 index 0000000000..edea9632ff --- /dev/null +++ b/sysdeps/unix/sysv/linux/loongarch/syscall_cancel.S @@ -0,0 +1,50 @@ +/* Cancellable syscall wrapper. Linux/loongarch version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +ENTRY (__syscall_cancel_arch) + + .global __syscall_cancel_arch_start +__syscall_cancel_arch_start: + + /* if (*cancelhandling & CANCELED_BITMASK) + __syscall_do_cancel() */ + ld.w t0, a0, 0 + andi t0, t0, TCB_CANCELED_BITMASK + bnez t0, 1f + + /* Issue a 6 argument syscall. */ + move t1, a1 + move a0, a2 + move a1, a3 + move a2, a4 + move a3, a5 + move a4, a6 + move a5, a7 + move a7, t1 + syscall 0 + + .global __syscall_cancel_arch_end +__syscall_cancel_arch_end: + jr ra +1: + b __syscall_do_cancel + +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/m68k/arch-syscall.h b/sysdeps/unix/sysv/linux/m68k/arch-syscall.h index 79f277dd5b..4ab34f6228 100644 --- a/sysdeps/unix/sysv/linux/m68k/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/m68k/arch-syscall.h @@ -213,6 +213,7 @@ #define __NR_mq_timedsend_time64 418 #define __NR_mq_unlink 272 #define __NR_mremap 163 +#define __NR_mseal 462 #define __NR_msgctl 402 #define __NR_msgget 399 #define __NR_msgrcv 401 diff --git a/sysdeps/unix/sysv/linux/m68k/coldfire/libc.abilist b/sysdeps/unix/sysv/linux/m68k/coldfire/libc.abilist index d9bd6a9b56..142595eb3e 100644 --- a/sysdeps/unix/sysv/linux/m68k/coldfire/libc.abilist +++ b/sysdeps/unix/sysv/linux/m68k/coldfire/libc.abilist @@ -2781,6 +2781,8 @@ GLIBC_2.4 xdrstdio_create F GLIBC_2.4 xencrypt F GLIBC_2.4 xprt_register F GLIBC_2.4 xprt_unregister F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/m68k/m680x0/libc.abilist b/sysdeps/unix/sysv/linux/m68k/m680x0/libc.abilist index 439796d693..85e7746c10 100644 --- a/sysdeps/unix/sysv/linux/m68k/m680x0/libc.abilist +++ b/sysdeps/unix/sysv/linux/m68k/m680x0/libc.abilist @@ -2948,6 +2948,8 @@ GLIBC_2.4 sys_errlist D 0x210 GLIBC_2.4 sys_nerr D 0x4 GLIBC_2.4 unlinkat F GLIBC_2.4 unshare F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/m68k/syscall_cancel.S b/sysdeps/unix/sysv/linux/m68k/syscall_cancel.S new file mode 100644 index 0000000000..8923bcc71c --- /dev/null +++ b/sysdeps/unix/sysv/linux/m68k/syscall_cancel.S @@ -0,0 +1,84 @@ +/* Cancellable syscall wrapper. Linux/m68k version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int __syscall_cancel_arch (int *cancelhandling, + __syscall_arg_t nr, + __syscall_arg_t arg1, + __syscall_arg_t arg2, + __syscall_arg_t arg3, + __syscall_arg_t arg4, + __syscall_arg_t arg5, + __syscall_arg_t arg6) */ + + +ENTRY (__syscall_cancel_arch) +#ifdef __mcoldfire__ + lea (-16,%sp),%sp + movem.l %d2-%d5,(%sp) +#else + movem.l %d2-%d5,-(%sp) +#endif + cfi_def_cfa_offset (20) + cfi_offset (2, -20) + cfi_offset (3, -16) + cfi_offset (4, -12) + cfi_offset (5, -8) + + .global __syscall_cancel_arch_start +__syscall_cancel_arch_start: + + move.l 20(%sp),%a0 + move.l (%a0),%d0 +#ifdef __mcoldfire__ + move.w %d0,%ccr + jeq 1f +#else + btst #TCB_CANCELED_BIT,%d0 + jne 1f +#endif + + move.l 48(%sp),%a0 + move.l 44(%sp),%d5 + move.l 40(%sp),%d4 + move.l 36(%sp),%d3 + move.l 32(%sp),%d2 + move.l 28(%sp),%d1 + move.l 24(%sp),%d0 + trap #0 + + .global __syscall_cancel_arch_end +__syscall_cancel_arch_end: + +#ifdef __mcoldfire__ + movem.l (%sp),%d2-%d5 + lea (16,%sp),%sp +#else + movem.l (%sp)+,%d2-%d5 +#endif + rts + +1: +#ifdef PIC + bsr.l __syscall_do_cancel +#else + jsr __syscall_do_cancel +#endif +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/microblaze/arch-syscall.h b/sysdeps/unix/sysv/linux/microblaze/arch-syscall.h index 779d5d5d70..79e225e50c 100644 --- a/sysdeps/unix/sysv/linux/microblaze/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/microblaze/arch-syscall.h @@ -221,6 +221,7 @@ #define __NR_mq_timedsend_time64 418 #define __NR_mq_unlink 278 #define __NR_mremap 163 +#define __NR_mseal 462 #define __NR_msgctl 331 #define __NR_msgget 332 #define __NR_msgrcv 333 diff --git a/sysdeps/unix/sysv/linux/microblaze/be/libc.abilist b/sysdeps/unix/sysv/linux/microblaze/be/libc.abilist index 1069d3252c..91dc1b8378 100644 --- a/sysdeps/unix/sysv/linux/microblaze/be/libc.abilist +++ b/sysdeps/unix/sysv/linux/microblaze/be/libc.abilist @@ -2834,3 +2834,5 @@ GLIBC_2.39 stdc_trailing_zeros_ui F GLIBC_2.39 stdc_trailing_zeros_ul F GLIBC_2.39 stdc_trailing_zeros_ull F GLIBC_2.39 stdc_trailing_zeros_us F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F diff --git a/sysdeps/unix/sysv/linux/microblaze/le/libc.abilist b/sysdeps/unix/sysv/linux/microblaze/le/libc.abilist index 17abe08c8b..3440e90f6f 100644 --- a/sysdeps/unix/sysv/linux/microblaze/le/libc.abilist +++ b/sysdeps/unix/sysv/linux/microblaze/le/libc.abilist @@ -2831,3 +2831,5 @@ GLIBC_2.39 stdc_trailing_zeros_ui F GLIBC_2.39 stdc_trailing_zeros_ul F GLIBC_2.39 stdc_trailing_zeros_ull F GLIBC_2.39 stdc_trailing_zeros_us F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F diff --git a/sysdeps/unix/sysv/linux/microblaze/syscall_cancel.S b/sysdeps/unix/sysv/linux/microblaze/syscall_cancel.S new file mode 100644 index 0000000000..1f9d202bf5 --- /dev/null +++ b/sysdeps/unix/sysv/linux/microblaze/syscall_cancel.S @@ -0,0 +1,61 @@ +/* Cancellable syscall wrapper. Linux/microblaze version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int __syscall_cancel_arch (int *cancelhandling, + long int nr, + long int arg1, + long int arg2, + long int arg3, + long int arg4, + long int arg5, + long int arg6) */ + +ENTRY (__syscall_cancel_arch) + + .globl __syscall_cancel_arch_start +__syscall_cancel_arch_start: + + lwi r3,r5,0 + andi r3,r3,TCB_CANCELED_BITMASK + bneid r3,1f + addk r12,r6,r0 + + addk r5,r7,r0 + addk r6,r8,r0 + addk r7,r9,r0 + addk r8,r10,r0 + lwi r9,r1,56 + lwi r10,r1,60 + brki r14,8 + + .globl __syscall_cancel_arch_end +__syscall_cancel_arch_end: + + nop + lwi r15,r1,0 + rtsd r15,8 + addik r1,r1,28 + +1: + brlid r15, __syscall_do_cancel + nop + +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/mips/mips32/arch-syscall.h b/sysdeps/unix/sysv/linux/mips/mips32/arch-syscall.h index 86ffd5ce84..dadd7f3130 100644 --- a/sysdeps/unix/sysv/linux/mips/mips32/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/mips/mips32/arch-syscall.h @@ -211,6 +211,7 @@ #define __NR_mq_timedsend_time64 4418 #define __NR_mq_unlink 4272 #define __NR_mremap 4167 +#define __NR_mseal 4462 #define __NR_msgctl 4402 #define __NR_msgget 4399 #define __NR_msgrcv 4401 diff --git a/sysdeps/unix/sysv/linux/mips/mips32/fpu/libc.abilist b/sysdeps/unix/sysv/linux/mips/mips32/fpu/libc.abilist index 799e508950..5ee7b8c52f 100644 --- a/sysdeps/unix/sysv/linux/mips/mips32/fpu/libc.abilist +++ b/sysdeps/unix/sysv/linux/mips/mips32/fpu/libc.abilist @@ -2909,6 +2909,8 @@ GLIBC_2.4 renameat F GLIBC_2.4 symlinkat F GLIBC_2.4 unlinkat F GLIBC_2.4 unshare F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/mips/mips32/nofpu/libc.abilist b/sysdeps/unix/sysv/linux/mips/mips32/nofpu/libc.abilist index 1c10996cbc..6cb6328e7c 100644 --- a/sysdeps/unix/sysv/linux/mips/mips32/nofpu/libc.abilist +++ b/sysdeps/unix/sysv/linux/mips/mips32/nofpu/libc.abilist @@ -2907,6 +2907,8 @@ GLIBC_2.4 renameat F GLIBC_2.4 symlinkat F GLIBC_2.4 unlinkat F GLIBC_2.4 unshare F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/mips/mips32/syscall_cancel.S b/sysdeps/unix/sysv/linux/mips/mips32/syscall_cancel.S new file mode 100644 index 0000000000..eb3b2ed005 --- /dev/null +++ b/sysdeps/unix/sysv/linux/mips/mips32/syscall_cancel.S @@ -0,0 +1,128 @@ +/* Cancellable syscall wrapper. Linux/mips32 version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <sys/asm.h> +#include <descr-const.h> + +/* long int __syscall_cancel_arch (int *cancelhandling, + __syscall_arg_t nr, + __syscall_arg_t arg1, + __syscall_arg_t arg2, + __syscall_arg_t arg3, + __syscall_arg_t arg4, + __syscall_arg_t arg5, + __syscall_arg_t arg6, + __syscall_arg_t arg7) */ + +#define FRAME_SIZE 56 + +NESTED (__syscall_cancel_arch, FRAME_SIZE, fp) + .mask 0xc0070000,-SZREG + .fmask 0x00000000,0 + + PTR_ADDIU sp, -FRAME_SIZE + cfi_def_cfa_offset (FRAME_SIZE) + + sw fp, 48(sp) + sw ra, 52(sp) + sw s2, 44(sp) + sw s1, 40(sp) + sw s0, 36(sp) +#ifdef __PIC__ + .cprestore 16 +#endif + cfi_offset (ra, -4) + cfi_offset (fp, -8) + cfi_offset (s2, -12) + cfi_offset (s1, -16) + cfi_offset (s0, -20) + + move fp ,sp + cfi_def_cfa_register (fp) + + .globl __syscall_cancel_arch_start +__syscall_cancel_arch_start: + + lw v0, 0(a0) + andi v0, v0, TCB_CANCELED_BITMASK + bne v0, zero, 2f + + addiu sp, sp, -16 + addiu v0, sp, 16 + sw v0, 24(fp) + + move s0, a1 + move a0, a2 + move a1, a3 + lw a2, 72(fp) + lw a3, 76(fp) + lw v0, 84(fp) + lw s1, 80(fp) + lw s2, 88(fp) + + .set noreorder + subu sp, 32 + sw s1, 16(sp) + sw v0, 20(sp) + sw s2, 24(sp) + move v0, s0 + syscall + + .globl __syscall_cancel_arch_end +__syscall_cancel_arch_end: + addiu sp, sp, 32 + .set reorder + + beq a3, zero, 1f + subu v0, zero, v0 +1: + move sp, fp + cfi_remember_state + cfi_def_cfa_register (sp) + lw ra, 52(fp) + lw fp, 48(sp) + lw s2, 44(sp) + lw s1, 40(sp) + lw s0, 36(sp) + + .set noreorder + .set nomacro + jr ra + addiu sp,sp,FRAME_SIZE + + .set macro + .set reorder + + cfi_def_cfa_offset (0) + cfi_restore (s0) + cfi_restore (s1) + cfi_restore (s2) + cfi_restore (fp) + cfi_restore (ra) + +2: + cfi_restore_state +#ifdef __PIC__ + PTR_LA t9, __syscall_do_cancel + jalr t9 +#else + jal __syscall_do_cancel +#endif + +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/mips/mips32/sysdep.h b/sysdeps/unix/sysv/linux/mips/mips32/sysdep.h index 1827caf595..47a1b97351 100644 --- a/sysdeps/unix/sysv/linux/mips/mips32/sysdep.h +++ b/sysdeps/unix/sysv/linux/mips/mips32/sysdep.h @@ -18,6 +18,10 @@ #ifndef _LINUX_MIPS_MIPS32_SYSDEP_H #define _LINUX_MIPS_MIPS32_SYSDEP_H 1 +/* mips32 have cancelable syscalls with 7 arguments (currently only + sync_file_range). */ +#define HAVE_CANCELABLE_SYSCALL_WITH_7_ARGS 1 + /* There is some commonality. */ #include <sysdeps/unix/sysv/linux/mips/sysdep.h> #include <sysdeps/unix/sysv/linux/sysdep.h> diff --git a/sysdeps/unix/sysv/linux/mips/mips64/n32/arch-syscall.h b/sysdeps/unix/sysv/linux/mips/mips64/n32/arch-syscall.h index 5d37a686e5..db6b2d4609 100644 --- a/sysdeps/unix/sysv/linux/mips/mips64/n32/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/mips/mips64/n32/arch-syscall.h @@ -194,6 +194,7 @@ #define __NR_mq_timedsend_time64 6418 #define __NR_mq_unlink 6235 #define __NR_mremap 6024 +#define __NR_mseal 6462 #define __NR_msgctl 6069 #define __NR_msgget 6066 #define __NR_msgrcv 6068 diff --git a/sysdeps/unix/sysv/linux/mips/mips64/n32/libc.abilist b/sysdeps/unix/sysv/linux/mips/mips64/n32/libc.abilist index 03d9655f26..ae7474c0f0 100644 --- a/sysdeps/unix/sysv/linux/mips/mips64/n32/libc.abilist +++ b/sysdeps/unix/sysv/linux/mips/mips64/n32/libc.abilist @@ -2915,6 +2915,8 @@ GLIBC_2.4 renameat F GLIBC_2.4 symlinkat F GLIBC_2.4 unlinkat F GLIBC_2.4 unshare F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/mips/mips64/n32/syscall_types.h b/sysdeps/unix/sysv/linux/mips/mips64/n32/syscall_types.h new file mode 100644 index 0000000000..b3a8b0b634 --- /dev/null +++ b/sysdeps/unix/sysv/linux/mips/mips64/n32/syscall_types.h @@ -0,0 +1,28 @@ +/* Types and macros used for syscall issuing. MIPS64n32 version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#ifndef _SYSCALL_TYPES_H +#define _SYSCALL_TYPES_H + +typedef long long int __syscall_arg_t; + +/* Convert X to a long long, without losing any bits if it is one + already or warning if it is a 32-bit pointer. */ +#define __SSC(__x) ((__syscall_arg_t) (__typeof__ ((__x) - (__x))) (__x)) + +#endif diff --git a/sysdeps/unix/sysv/linux/mips/mips64/n64/arch-syscall.h b/sysdeps/unix/sysv/linux/mips/mips64/n64/arch-syscall.h index 9b1e846e76..b4129a4dbd 100644 --- a/sysdeps/unix/sysv/linux/mips/mips64/n64/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/mips/mips64/n64/arch-syscall.h @@ -183,6 +183,7 @@ #define __NR_mq_timedsend 5232 #define __NR_mq_unlink 5231 #define __NR_mremap 5024 +#define __NR_mseal 5462 #define __NR_msgctl 5069 #define __NR_msgget 5066 #define __NR_msgrcv 5068 diff --git a/sysdeps/unix/sysv/linux/mips/mips64/n64/libc.abilist b/sysdeps/unix/sysv/linux/mips/mips64/n64/libc.abilist index 05e402ed30..cdf040dec2 100644 --- a/sysdeps/unix/sysv/linux/mips/mips64/n64/libc.abilist +++ b/sysdeps/unix/sysv/linux/mips/mips64/n64/libc.abilist @@ -2817,6 +2817,8 @@ GLIBC_2.4 renameat F GLIBC_2.4 symlinkat F GLIBC_2.4 unlinkat F GLIBC_2.4 unshare F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/mips/mips64/syscall_cancel.S b/sysdeps/unix/sysv/linux/mips/mips64/syscall_cancel.S new file mode 100644 index 0000000000..cfc0596b6a --- /dev/null +++ b/sysdeps/unix/sysv/linux/mips/mips64/syscall_cancel.S @@ -0,0 +1,112 @@ +/* Cancellable syscall wrapper. Linux/mips64 version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <sys/asm.h> +#include <descr-const.h> + +/* long int __syscall_cancel_arch (int *cancelhandling, + __syscall_arg_t nr, + __syscall_arg_t arg1, + __syscall_arg_t arg2, + __syscall_arg_t arg3, + __syscall_arg_t arg4, + __syscall_arg_t arg5, + __syscall_arg_t arg6, + __syscall_arg_t arg7) */ + +#define FRAME_SIZE 32 + + .text +NESTED (__syscall_cancel_arch, FRAME_SIZE, ra) + .mask 0x90010000, -SZREG + .fmask 0x00000000, 0 + LONG_ADDIU sp, sp, -FRAME_SIZE + cfi_def_cfa_offset (FRAME_SIZE) + sd gp, 16(sp) + cfi_offset (gp, -16) + lui gp, %hi(%neg(%gp_rel(__syscall_cancel_arch))) + LONG_ADDU gp, gp, t9 + sd ra, 24(sp) + sd s0, 8(sp) + cfi_offset (ra, -8) + cfi_offset (s0, -24) + LONG_ADDIU gp, gp, %lo(%neg(%gp_rel(__syscall_cancel_arch))) + + .global __syscall_cancel_arch_start +__syscall_cancel_arch_start: + + lw v0, 0(a0) + andi v0, v0, TCB_CANCELED_BITMASK + .set noreorder + .set nomacro + bne v0, zero, 2f + move s0, a1 + .set macro + .set reorder + + move a0, a2 + move a1, a3 + move a2, a4 + move a3, a5 + move a4, a6 + move a5, a7 + + .set noreorder + move v0, s0 + syscall + .set reorder + + .global __syscall_cancel_arch_end +__syscall_cancel_arch_end: + + .set noreorder + .set nomacro +#if __mips_isa_rev >= 6 + beqzc $7, 1f +#else + bnel a3, zero, 1f +#endif + SUBU v0, zero, v0 + .set macro + .set reorder + +1: + ld ra, 24(sp) + ld gp, 16(sp) + ld s0, 8(sp) + + .set noreorder + .set nomacro + jr ra + LONG_ADDIU sp, sp, FRAME_SIZE + .set macro + .set reorder + + cfi_remember_state + cfi_def_cfa_offset (0) + cfi_restore (s0) + cfi_restore (gp) + cfi_restore (ra) + .align 3 +2: + cfi_restore_state + LONG_L t9, %got_disp(__syscall_do_cancel)(gp) + .reloc 3f, R_MIPS_JALR, __syscall_do_cancel +3: jalr t9 +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/mips/mips64/sysdep.h b/sysdeps/unix/sysv/linux/mips/mips64/sysdep.h index 0a1711dad2..0438bed23d 100644 --- a/sysdeps/unix/sysv/linux/mips/mips64/sysdep.h +++ b/sysdeps/unix/sysv/linux/mips/mips64/sysdep.h @@ -44,15 +44,7 @@ #undef HAVE_INTERNAL_BRK_ADDR_SYMBOL #define HAVE_INTERNAL_BRK_ADDR_SYMBOL 1 -#if _MIPS_SIM == _ABIN32 -/* Convert X to a long long, without losing any bits if it is one - already or warning if it is a 32-bit pointer. */ -# define ARGIFY(X) ((long long int) (__typeof__ ((X) - (X))) (X)) -typedef long long int __syscall_arg_t; -#else -# define ARGIFY(X) ((long int) (X)) -typedef long int __syscall_arg_t; -#endif +#include <syscall_types.h> /* Note that the original Linux syscall restart convention required the instruction immediately preceding SYSCALL to initialize $v0 with the @@ -120,7 +112,7 @@ typedef long int __syscall_arg_t; long int _sys_result; \ \ { \ - __syscall_arg_t _arg1 = ARGIFY (arg1); \ + __syscall_arg_t _arg1 = __SSC (arg1); \ register __syscall_arg_t __s0 asm ("$16") __attribute__ ((unused))\ = (number); \ register __syscall_arg_t __v0 asm ("$2"); \ @@ -144,8 +136,8 @@ typedef long int __syscall_arg_t; long int _sys_result; \ \ { \ - __syscall_arg_t _arg1 = ARGIFY (arg1); \ - __syscall_arg_t _arg2 = ARGIFY (arg2); \ + __syscall_arg_t _arg1 = __SSC (arg1); \ + __syscall_arg_t _arg2 = __SSC (arg2); \ register __syscall_arg_t __s0 asm ("$16") __attribute__ ((unused))\ = (number); \ register __syscall_arg_t __v0 asm ("$2"); \ @@ -170,9 +162,9 @@ typedef long int __syscall_arg_t; long int _sys_result; \ \ { \ - __syscall_arg_t _arg1 = ARGIFY (arg1); \ - __syscall_arg_t _arg2 = ARGIFY (arg2); \ - __syscall_arg_t _arg3 = ARGIFY (arg3); \ + __syscall_arg_t _arg1 = __SSC (arg1); \ + __syscall_arg_t _arg2 = __SSC (arg2); \ + __syscall_arg_t _arg3 = __SSC (arg3); \ register __syscall_arg_t __s0 asm ("$16") __attribute__ ((unused))\ = (number); \ register __syscall_arg_t __v0 asm ("$2"); \ @@ -199,10 +191,10 @@ typedef long int __syscall_arg_t; long int _sys_result; \ \ { \ - __syscall_arg_t _arg1 = ARGIFY (arg1); \ - __syscall_arg_t _arg2 = ARGIFY (arg2); \ - __syscall_arg_t _arg3 = ARGIFY (arg3); \ - __syscall_arg_t _arg4 = ARGIFY (arg4); \ + __syscall_arg_t _arg1 = __SSC (arg1); \ + __syscall_arg_t _arg2 = __SSC (arg2); \ + __syscall_arg_t _arg3 = __SSC (arg3); \ + __syscall_arg_t _arg4 = __SSC (arg4); \ register __syscall_arg_t __s0 asm ("$16") __attribute__ ((unused))\ = (number); \ register __syscall_arg_t __v0 asm ("$2"); \ @@ -229,11 +221,11 @@ typedef long int __syscall_arg_t; long int _sys_result; \ \ { \ - __syscall_arg_t _arg1 = ARGIFY (arg1); \ - __syscall_arg_t _arg2 = ARGIFY (arg2); \ - __syscall_arg_t _arg3 = ARGIFY (arg3); \ - __syscall_arg_t _arg4 = ARGIFY (arg4); \ - __syscall_arg_t _arg5 = ARGIFY (arg5); \ + __syscall_arg_t _arg1 = __SSC (arg1); \ + __syscall_arg_t _arg2 = __SSC (arg2); \ + __syscall_arg_t _arg3 = __SSC (arg3); \ + __syscall_arg_t _arg4 = __SSC (arg4); \ + __syscall_arg_t _arg5 = __SSC (arg5); \ register __syscall_arg_t __s0 asm ("$16") __attribute__ ((unused))\ = (number); \ register __syscall_arg_t __v0 asm ("$2"); \ @@ -261,12 +253,12 @@ typedef long int __syscall_arg_t; long int _sys_result; \ \ { \ - __syscall_arg_t _arg1 = ARGIFY (arg1); \ - __syscall_arg_t _arg2 = ARGIFY (arg2); \ - __syscall_arg_t _arg3 = ARGIFY (arg3); \ - __syscall_arg_t _arg4 = ARGIFY (arg4); \ - __syscall_arg_t _arg5 = ARGIFY (arg5); \ - __syscall_arg_t _arg6 = ARGIFY (arg6); \ + __syscall_arg_t _arg1 = __SSC (arg1); \ + __syscall_arg_t _arg2 = __SSC (arg2); \ + __syscall_arg_t _arg3 = __SSC (arg3); \ + __syscall_arg_t _arg4 = __SSC (arg4); \ + __syscall_arg_t _arg5 = __SSC (arg5); \ + __syscall_arg_t _arg6 = __SSC (arg6); \ register __syscall_arg_t __s0 asm ("$16") __attribute__ ((unused))\ = (number); \ register __syscall_arg_t __v0 asm ("$2"); \ diff --git a/sysdeps/unix/sysv/linux/mremap-failure.h b/sysdeps/unix/sysv/linux/mremap-failure.h new file mode 100644 index 0000000000..c99ab30ca9 --- /dev/null +++ b/sysdeps/unix/sysv/linux/mremap-failure.h @@ -0,0 +1,30 @@ +/* mremap failure handling. Linux version. + Copyright (C) 2024 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <support/check.h> + +/* Return exit value on mremap failure with errno ERR. */ + +static int +mremap_failure_exit (int err) +{ + if (err != EINVAL) + return EXIT_FAILURE; + + return EXIT_UNSUPPORTED; +} diff --git a/sysdeps/unix/sysv/linux/mremap.c b/sysdeps/unix/sysv/linux/mremap.c index 4f770799c4..1ada5c1f40 100644 --- a/sysdeps/unix/sysv/linux/mremap.c +++ b/sysdeps/unix/sysv/linux/mremap.c @@ -20,6 +20,12 @@ #include <sysdep.h> #include <stdarg.h> #include <stddef.h> +#include <errno.h> + +#define MREMAP_KNOWN_BITS \ + (MREMAP_MAYMOVE \ + | MREMAP_FIXED \ + | MREMAP_DONTUNMAP) void * __mremap (void *addr, size_t old_len, size_t new_len, int flags, ...) @@ -27,7 +33,13 @@ __mremap (void *addr, size_t old_len, size_t new_len, int flags, ...) va_list va; void *new_addr = NULL; - if (flags & MREMAP_FIXED) + if (flags & ~(MREMAP_KNOWN_BITS)) + { + __set_errno (EINVAL); + return MAP_FAILED; + } + + if (flags & (MREMAP_FIXED | MREMAP_DONTUNMAP)) { va_start (va, flags); new_addr = va_arg (va, void *); diff --git a/sysdeps/unix/sysv/linux/nios2/arch-syscall.h b/sysdeps/unix/sysv/linux/nios2/arch-syscall.h index abbc9ab6b0..f94e212995 100644 --- a/sysdeps/unix/sysv/linux/nios2/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/nios2/arch-syscall.h @@ -167,6 +167,7 @@ #define __NR_mq_timedsend_time64 418 #define __NR_mq_unlink 181 #define __NR_mremap 216 +#define __NR_mseal 462 #define __NR_msgctl 187 #define __NR_msgget 186 #define __NR_msgrcv 188 diff --git a/sysdeps/unix/sysv/linux/nios2/libc.abilist b/sysdeps/unix/sysv/linux/nios2/libc.abilist index 3aa81766aa..773d4c5873 100644 --- a/sysdeps/unix/sysv/linux/nios2/libc.abilist +++ b/sysdeps/unix/sysv/linux/nios2/libc.abilist @@ -2873,3 +2873,5 @@ GLIBC_2.39 stdc_trailing_zeros_ui F GLIBC_2.39 stdc_trailing_zeros_ul F GLIBC_2.39 stdc_trailing_zeros_ull F GLIBC_2.39 stdc_trailing_zeros_us F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F diff --git a/sysdeps/unix/sysv/linux/nios2/syscall_cancel.S b/sysdeps/unix/sysv/linux/nios2/syscall_cancel.S new file mode 100644 index 0000000000..19d0795886 --- /dev/null +++ b/sysdeps/unix/sysv/linux/nios2/syscall_cancel.S @@ -0,0 +1,95 @@ +/* Cancellable syscall wrapper. Linux/nios2 version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int __syscall_cancel_arch (int *cancelhandling, + __syscall_arg_t nr, + __syscall_arg_t arg1, + __syscall_arg_t arg2, + __syscall_arg_t arg3, + __syscall_arg_t arg4, + __syscall_arg_t arg5, + __syscall_arg_t arg6) */ + +ENTRY (__syscall_cancel_arch) +#ifdef SHARED + addi sp, sp, -8 + stw r22, 0(sp) + nextpc r22 +1: + movhi r8, %hiadj(_gp_got - 1b) + addi r8, r8, %lo(_gp_got - 1b) + stw ra, 4(sp) + add r22, r22, r8 +#else + addi sp, sp, -4 + cfi_def_cfa_offset (4) + stw ra, 0(sp) + cfi_offset (31, -4) +#endif + + .globl __syscall_cancel_arch_start +__syscall_cancel_arch_start: + ldw r3, 0(r4) + andi r3, r3, TCB_CANCELED_BITMASK + bne r3, zero, 3f + mov r10, r6 + mov r2, r5 +#ifdef SHARED +# define STACK_ADJ 4 +#else +# define STACK_ADJ 0 +#endif + ldw r9, (16 + STACK_ADJ)(sp) + mov r5, r7 + ldw r8, (12 + STACK_ADJ)(sp) + ldw r7, (8 + STACK_ADJ)(sp) + ldw r6, (4 + STACK_ADJ)(sp) + mov r4, r10 + trap + + .globl __syscall_cancel_arch_end +__syscall_cancel_arch_end: + beq r7, zero, 2f + sub r2, zero, r2 +2: +#ifdef SHARED + ldw ra, 4(sp) + ldw r22, 0(sp) + addi sp, sp, 8 +#else + ldw ra, (0 + STACK_ADJ)(sp) + cfi_remember_state + cfi_restore (31) + addi sp, sp, 4 + cfi_def_cfa_offset (0) +#endif + ret + +3: +#ifdef SHARED + ldw r2, %call(__syscall_do_cancel)(r22) + callr r2 +#else + cfi_restore_state + call __syscall_do_cancel +#endif + +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/or1k/arch-syscall.h b/sysdeps/unix/sysv/linux/or1k/arch-syscall.h index 7223a93673..2d21fa2085 100644 --- a/sysdeps/unix/sysv/linux/or1k/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/or1k/arch-syscall.h @@ -167,6 +167,7 @@ #define __NR_mq_timedsend_time64 418 #define __NR_mq_unlink 181 #define __NR_mremap 216 +#define __NR_mseal 462 #define __NR_msgctl 187 #define __NR_msgget 186 #define __NR_msgrcv 188 diff --git a/sysdeps/unix/sysv/linux/or1k/libc.abilist b/sysdeps/unix/sysv/linux/or1k/libc.abilist index 959e59e7e7..c356a11b1c 100644 --- a/sysdeps/unix/sysv/linux/or1k/libc.abilist +++ b/sysdeps/unix/sysv/linux/or1k/libc.abilist @@ -2259,3 +2259,5 @@ GLIBC_2.40 getcontext F GLIBC_2.40 makecontext F GLIBC_2.40 setcontext F GLIBC_2.40 swapcontext F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F diff --git a/sysdeps/unix/sysv/linux/or1k/syscall_cancel.S b/sysdeps/unix/sysv/linux/or1k/syscall_cancel.S new file mode 100644 index 0000000000..876f5e05ab --- /dev/null +++ b/sysdeps/unix/sysv/linux/or1k/syscall_cancel.S @@ -0,0 +1,63 @@ +/* Cancellable syscall wrapper. Linux/or1k version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +ENTRY (__syscall_cancel_arch) + l.addi r1, r1, -4 + cfi_def_cfa_offset (4) + l.sw 0(r1), r9 + cfi_offset (9, -4) + + .global __syscall_cancel_arch_start +__syscall_cancel_arch_start: + + /* if (*cancelhandling & CANCELED_BITMASK) + __syscall_do_cancel() */ + l.movhi r19, hi(0) + l.lwz r17, 0(r3) + l.andi r17, r17, 8 + l.sfeq r17, r19 + l.bnf 1f + + /* Issue a 6 argument syscall. */ + l.or r11, r4, r4 + l.or r3, r5, r5 + l.or r4, r6, r6 + l.or r5, r7, r7 + l.or r6, r8, r8 + l.lwz r7, 4(r1) + l.lwz r8, 8(r1) + l.sys 1 + l.nop + + .global __syscall_cancel_arch_end +__syscall_cancel_arch_end: + + l.lwz r9, 0(r1) + l.jr r9 + l.addi r1, r1, 4 + cfi_remember_state + cfi_def_cfa_offset (0) + cfi_restore (9) +1: + cfi_restore_state + l.jal __syscall_do_cancel + l.nop +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/powerpc/cancellation-pc-check.h b/sysdeps/unix/sysv/linux/powerpc/cancellation-pc-check.h new file mode 100644 index 0000000000..1175e1a070 --- /dev/null +++ b/sysdeps/unix/sysv/linux/powerpc/cancellation-pc-check.h @@ -0,0 +1,65 @@ +/* Architecture specific code for pthread cancellation handling. + Linux/PowerPC version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#ifndef _NPTL_CANCELLATION_PC_CHECK +#define _NPTL_CANCELLATION_PC_CHECK + +#include <sigcontextinfo.h> + +/* For syscalls with side-effects (e.g read that might return partial read), + the kernel cannot restart the syscall when interrupted by a signal, it must + return from the call with whatever partial result. In this case, the saved + program counter is set just after the syscall instruction, so the SIGCANCEL + handler should not act on cancellation. + + The __syscall_cancel_arch function, used for all cancellable syscalls, + contains two extra markers, __syscall_cancel_arch_start and + __syscall_cancel_arch_end. The former points to just before the initial + conditional branch that checks if the thread has received a cancellation + request, while former points to the instruction after the one responsible + to issue the syscall. + + The function check if the program counter (PC) from ucontext_t CTX is + within the start and then end boundary from the __syscall_cancel_arch + bridge. Return TRUE if the PC is within the boundary, meaning the + syscall does not have any side effects; or FALSE otherwise. */ + +static __always_inline bool +cancellation_pc_check (void *ctx) +{ + /* Both are defined in syscall_cancel.S. */ + extern const char __syscall_cancel_arch_start[1]; + extern const char __syscall_cancel_arch_end_sc[1]; +#if defined(USE_PPC_SVC) && defined(__powerpc64__) + extern const char __syscall_cancel_arch_end_svc[1]; +#endif + + uintptr_t pc = sigcontext_get_pc (ctx); + + return pc >= (uintptr_t) __syscall_cancel_arch_start +#if defined(USE_PPC_SVC) && defined(__powerpc64__) + && THREAD_GET_HWCAP() & PPC_FEATURE2_SCV + ? pc < (uintptr_t) __syscall_cancel_arch_end_sc + : pc < (uintptr_t) __syscall_cancel_arch_end_svc; +#else + && pc < (uintptr_t) __syscall_cancel_arch_end_sc; +#endif +} + +#endif diff --git a/sysdeps/unix/sysv/linux/powerpc/powerpc32/arch-syscall.h b/sysdeps/unix/sysv/linux/powerpc/powerpc32/arch-syscall.h index af0d2b121e..206d9fd656 100644 --- a/sysdeps/unix/sysv/linux/powerpc/powerpc32/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/powerpc/powerpc32/arch-syscall.h @@ -211,6 +211,7 @@ #define __NR_mq_timedsend_time64 418 #define __NR_mq_unlink 263 #define __NR_mremap 163 +#define __NR_mseal 462 #define __NR_msgctl 402 #define __NR_msgget 399 #define __NR_msgrcv 401 diff --git a/sysdeps/unix/sysv/linux/powerpc/powerpc32/fpu/libc.abilist b/sysdeps/unix/sysv/linux/powerpc/powerpc32/fpu/libc.abilist index 9714305608..7937f94cf0 100644 --- a/sysdeps/unix/sysv/linux/powerpc/powerpc32/fpu/libc.abilist +++ b/sysdeps/unix/sysv/linux/powerpc/powerpc32/fpu/libc.abilist @@ -3138,6 +3138,8 @@ GLIBC_2.4 wcstold F GLIBC_2.4 wcstold_l F GLIBC_2.4 wprintf F GLIBC_2.4 wscanf F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/powerpc/powerpc32/nofpu/libc.abilist b/sysdeps/unix/sysv/linux/powerpc/powerpc32/nofpu/libc.abilist index 0beb52c542..d6e35f31d2 100644 --- a/sysdeps/unix/sysv/linux/powerpc/powerpc32/nofpu/libc.abilist +++ b/sysdeps/unix/sysv/linux/powerpc/powerpc32/nofpu/libc.abilist @@ -3183,6 +3183,8 @@ GLIBC_2.4 wcstold F GLIBC_2.4 wcstold_l F GLIBC_2.4 wprintf F GLIBC_2.4 wscanf F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/powerpc/powerpc64/arch-syscall.h b/sysdeps/unix/sysv/linux/powerpc/powerpc64/arch-syscall.h index a4c70aa7fe..19f72a7f69 100644 --- a/sysdeps/unix/sysv/linux/powerpc/powerpc64/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/powerpc/powerpc64/arch-syscall.h @@ -195,6 +195,7 @@ #define __NR_mq_timedsend 264 #define __NR_mq_unlink 263 #define __NR_mremap 163 +#define __NR_mseal 462 #define __NR_msgctl 402 #define __NR_msgget 399 #define __NR_msgrcv 401 diff --git a/sysdeps/unix/sysv/linux/powerpc/powerpc64/be/libc.abilist b/sysdeps/unix/sysv/linux/powerpc/powerpc64/be/libc.abilist index cfc2ebd3ec..2268d6890d 100644 --- a/sysdeps/unix/sysv/linux/powerpc/powerpc64/be/libc.abilist +++ b/sysdeps/unix/sysv/linux/powerpc/powerpc64/be/libc.abilist @@ -2892,6 +2892,8 @@ GLIBC_2.4 wcstold F GLIBC_2.4 wcstold_l F GLIBC_2.4 wprintf F GLIBC_2.4 wscanf F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/powerpc/powerpc64/le/libc.abilist b/sysdeps/unix/sysv/linux/powerpc/powerpc64/le/libc.abilist index 8c9efc5a16..7f61b14bc8 100644 --- a/sysdeps/unix/sysv/linux/powerpc/powerpc64/le/libc.abilist +++ b/sysdeps/unix/sysv/linux/powerpc/powerpc64/le/libc.abilist @@ -2968,3 +2968,5 @@ GLIBC_2.39 stdc_trailing_zeros_ui F GLIBC_2.39 stdc_trailing_zeros_ul F GLIBC_2.39 stdc_trailing_zeros_ull F GLIBC_2.39 stdc_trailing_zeros_us F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F diff --git a/sysdeps/unix/sysv/linux/powerpc/syscall_cancel.S b/sysdeps/unix/sysv/linux/powerpc/syscall_cancel.S new file mode 100644 index 0000000000..1f119d0889 --- /dev/null +++ b/sysdeps/unix/sysv/linux/powerpc/syscall_cancel.S @@ -0,0 +1,86 @@ +/* Cancellable syscall wrapper. Linux/powerpc version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int [r3] __syscall_cancel_arch (int *cancelhandling [r3], + long int nr [r4], + long int arg1 [r5], + long int arg2 [r6], + long int arg3 [r7], + long int arg4 [r8], + long int arg5 [r9], + long int arg6 [r10]) */ + +ENTRY (__syscall_cancel_arch) + + .globl __syscall_cancel_arch_start +__syscall_cancel_arch_start: + + /* if (*cancelhandling & CANCELED_BITMASK) + __syscall_do_cancel() */ + lwz r0,0(r3) + andi. r0,r0,TCB_CANCELED_BITMASK + bne 1f + + /* Issue a 6 argument syscall, the nr [r4] being the syscall + number. */ + mr r0,r4 + mr r3,r5 + mr r4,r6 + mr r5,r7 + mr r6,r8 + mr r7,r9 + mr r8,r10 + +#if defined(USE_PPC_SVC) && defined(__powerpc64__) + CHECK_SCV_SUPPORT r9 0f + + stdu r1, -SCV_FRAME_SIZE(r1) + cfi_adjust_cfa_offset (SCV_FRAME_SIZE) + .machine "push" + .machine "power9" + scv 0 + .machine "pop" + .globl __syscall_cancel_arch_end_svc +__syscall_cancel_arch_end_svc: + ld r9, SCV_FRAME_SIZE + FRAME_LR_SAVE(r1) + mtlr r9 + addi r1, r1, SCV_FRAME_SIZE + cfi_restore (lr) + li r9, -4095 + cmpld r3, r9 + bnslr+ + neg r3,r3 + blr +0: +#endif + sc + .globl __syscall_cancel_arch_end_sc +__syscall_cancel_arch_end_sc: + bnslr+ + neg r3,r3 + blr + + /* Although the __syscall_do_cancel do not return, we need to stack + being set correctly for unwind. */ +1: + TAIL_CALL_NO_RETURN (__syscall_do_cancel) + +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/readdir_r.c b/sysdeps/unix/sysv/linux/readdir_r.c index ffd5262cf5..1d595688f7 100644 --- a/sysdeps/unix/sysv/linux/readdir_r.c +++ b/sysdeps/unix/sysv/linux/readdir_r.c @@ -25,14 +25,22 @@ __readdir_r (DIR *dirp, struct dirent *entry, struct dirent **result) { struct dirent *dp; size_t reclen; + int saved_errno = errno; __libc_lock_lock (dirp->lock); while (1) { + /* If errno is changed from 0, the NULL return value indicates + an actual error. It overrides a pending ENAMETOOLONG error. */ + __set_errno (0); dp = __readdir_unlocked (dirp); if (dp == NULL) - break; + { + if (errno != 0) + dirp->errcode = errno; + break; + } reclen = dp->d_reclen; if (reclen <= offsetof (struct dirent, d_name) + NAME_MAX + 1) @@ -61,6 +69,7 @@ __readdir_r (DIR *dirp, struct dirent *entry, struct dirent **result) __libc_lock_unlock (dirp->lock); + __set_errno (saved_errno); return dp != NULL ? 0 : dirp->errcode; } diff --git a/sysdeps/unix/sysv/linux/riscv/rv32/arch-syscall.h b/sysdeps/unix/sysv/linux/riscv/rv32/arch-syscall.h index 7315d164d6..eb9e57b028 100644 --- a/sysdeps/unix/sysv/linux/riscv/rv32/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/riscv/rv32/arch-syscall.h @@ -153,6 +153,7 @@ #define __NR_mq_timedsend_time64 418 #define __NR_mq_unlink 181 #define __NR_mremap 216 +#define __NR_mseal 462 #define __NR_msgctl 187 #define __NR_msgget 186 #define __NR_msgrcv 188 diff --git a/sysdeps/unix/sysv/linux/riscv/rv32/libc.abilist b/sysdeps/unix/sysv/linux/riscv/rv32/libc.abilist index 6397a9cb91..4187241f50 100644 --- a/sysdeps/unix/sysv/linux/riscv/rv32/libc.abilist +++ b/sysdeps/unix/sysv/linux/riscv/rv32/libc.abilist @@ -2512,3 +2512,5 @@ GLIBC_2.39 stdc_trailing_zeros_ul F GLIBC_2.39 stdc_trailing_zeros_ull F GLIBC_2.39 stdc_trailing_zeros_us F GLIBC_2.40 __riscv_hwprobe F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F diff --git a/sysdeps/unix/sysv/linux/riscv/rv64/arch-syscall.h b/sysdeps/unix/sysv/linux/riscv/rv64/arch-syscall.h index 31a1130db9..1eac18e582 100644 --- a/sysdeps/unix/sysv/linux/riscv/rv64/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/riscv/rv64/arch-syscall.h @@ -158,6 +158,7 @@ #define __NR_mq_timedsend 182 #define __NR_mq_unlink 181 #define __NR_mremap 216 +#define __NR_mseal 462 #define __NR_msgctl 187 #define __NR_msgget 186 #define __NR_msgrcv 188 diff --git a/sysdeps/unix/sysv/linux/riscv/rv64/libc.abilist b/sysdeps/unix/sysv/linux/riscv/rv64/libc.abilist index 71bbf94f66..8935beccac 100644 --- a/sysdeps/unix/sysv/linux/riscv/rv64/libc.abilist +++ b/sysdeps/unix/sysv/linux/riscv/rv64/libc.abilist @@ -2712,3 +2712,5 @@ GLIBC_2.39 stdc_trailing_zeros_ul F GLIBC_2.39 stdc_trailing_zeros_ull F GLIBC_2.39 stdc_trailing_zeros_us F GLIBC_2.40 __riscv_hwprobe F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F diff --git a/sysdeps/unix/sysv/linux/riscv/syscall_cancel.S b/sysdeps/unix/sysv/linux/riscv/syscall_cancel.S new file mode 100644 index 0000000000..742c748d09 --- /dev/null +++ b/sysdeps/unix/sysv/linux/riscv/syscall_cancel.S @@ -0,0 +1,67 @@ +/* Cancellable syscall wrapper. Linux/riscv version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int __syscall_cancel_arch (int *cancelhandling, + __syscall_arg_t nr, + __syscall_arg_t arg1, + __syscall_arg_t arg2, + __syscall_arg_t arg3, + __syscall_arg_t arg4, + __syscall_arg_t arg5, + __syscall_arg_t arg6) */ + +#ifdef SHARED + .option pic +#else + .option nopic +#endif + +ENTRY (__syscall_cancel_arch) + + .globl __syscall_cancel_arch_start +__syscall_cancel_arch_start: + lw t1, 0(a0) + /* if (*ch & CANCELED_BITMASK) */ + andi t1, t1, TCB_CANCELED_BITMASK + bne t1, zero, 1f + + mv t3, a1 + mv a0, a2 + mv a1, a3 + mv a2, a4 + mv a3, a5 + mv a4, a6 + mv a5, a7 + mv a7, t3 + scall + + .globl __syscall_cancel_arch_end +__syscall_cancel_arch_end: + ret + +1: + addi sp, sp, -16 + cfi_def_cfa_offset (16) + REG_S ra, (16-SZREG)(sp) + cfi_offset (ra, -SZREG) + call __syscall_do_cancel + +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/s390/s390-32/arch-syscall.h b/sysdeps/unix/sysv/linux/s390/s390-32/arch-syscall.h index cf8569304d..464eca58b2 100644 --- a/sysdeps/unix/sysv/linux/s390/s390-32/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/s390/s390-32/arch-syscall.h @@ -214,6 +214,7 @@ #define __NR_mq_timedsend_time64 418 #define __NR_mq_unlink 272 #define __NR_mremap 163 +#define __NR_mseal 462 #define __NR_msgctl 402 #define __NR_msgget 399 #define __NR_msgrcv 401 diff --git a/sysdeps/unix/sysv/linux/s390/s390-32/libc.abilist b/sysdeps/unix/sysv/linux/s390/s390-32/libc.abilist index a7467e2850..e69dc7ccf6 100644 --- a/sysdeps/unix/sysv/linux/s390/s390-32/libc.abilist +++ b/sysdeps/unix/sysv/linux/s390/s390-32/libc.abilist @@ -3136,6 +3136,8 @@ GLIBC_2.4 wcstold F GLIBC_2.4 wcstold_l F GLIBC_2.4 wprintf F GLIBC_2.4 wscanf F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/s390/s390-32/syscall_cancel.S b/sysdeps/unix/sysv/linux/s390/s390-32/syscall_cancel.S new file mode 100644 index 0000000000..9e0ad2a635 --- /dev/null +++ b/sysdeps/unix/sysv/linux/s390/s390-32/syscall_cancel.S @@ -0,0 +1,62 @@ +/* Cancellable syscall wrapper. Linux/s390 version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int __syscall_cancel_arch (int *cancelhandling, + __syscall_arg_t nr, + __syscall_arg_t arg1, + __syscall_arg_t arg2, + __syscall_arg_t arg3, + __syscall_arg_t arg4, + __syscall_arg_t arg5, + __syscall_arg_t arg6) */ + +ENTRY (__syscall_cancel_arch) + stm %r6,%r7,24(%r15) + cfi_offset (%r6, -72) + cfi_offset (%r7, -68) + + .globl __syscall_cancel_arch_start +__syscall_cancel_arch_start: + /* if (*cancelhandling & CANCELED_BITMASK) + __syscall_do_cancel() */ + tm 3(%r2),TCB_CANCELED_BITMASK + jne 1f + + /* Issue a 6 argument syscall, the nr [%r1] being the syscall + number. */ + lr %r1,%r3 + lr %r2,%r4 + lr %r3,%r5 + lr %r4,%r6 + lm %r5,%r7,96(%r15) + svc 0 + + .globl __syscall_cancel_arch_end +__syscall_cancel_arch_end: + lm %r6,%r7,24(%r15) + cfi_remember_state + cfi_restore (%r7) + cfi_restore (%r6) + br %r14 +1: + cfi_restore_state + jg __syscall_do_cancel +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/s390/s390-64/arch-syscall.h b/sysdeps/unix/sysv/linux/s390/s390-64/arch-syscall.h index f3536ed03f..57842702fd 100644 --- a/sysdeps/unix/sysv/linux/s390/s390-64/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/s390/s390-64/arch-syscall.h @@ -185,6 +185,7 @@ #define __NR_mq_timedsend 273 #define __NR_mq_unlink 272 #define __NR_mremap 163 +#define __NR_mseal 462 #define __NR_msgctl 402 #define __NR_msgget 399 #define __NR_msgrcv 401 diff --git a/sysdeps/unix/sysv/linux/s390/s390-64/libc.abilist b/sysdeps/unix/sysv/linux/s390/s390-64/libc.abilist index fd1cb2972d..7d860001d8 100644 --- a/sysdeps/unix/sysv/linux/s390/s390-64/libc.abilist +++ b/sysdeps/unix/sysv/linux/s390/s390-64/libc.abilist @@ -2929,6 +2929,8 @@ GLIBC_2.4 wcstold F GLIBC_2.4 wcstold_l F GLIBC_2.4 wprintf F GLIBC_2.4 wscanf F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/s390/s390-64/syscall_cancel.S b/sysdeps/unix/sysv/linux/s390/s390-64/syscall_cancel.S new file mode 100644 index 0000000000..e1620add6a --- /dev/null +++ b/sysdeps/unix/sysv/linux/s390/s390-64/syscall_cancel.S @@ -0,0 +1,62 @@ +/* Cancellable syscall wrapper. Linux/s390x version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int __syscall_cancel_arch (int *cancelhandling, + __syscall_arg_t nr, + __syscall_arg_t arg1, + __syscall_arg_t arg2, + __syscall_arg_t arg3, + __syscall_arg_t arg4, + __syscall_arg_t arg5, + __syscall_arg_t arg6) */ + +ENTRY (__syscall_cancel_arch) + stmg %r6,%r7,48(%r15) + cfi_offset (%r6, -112) + cfi_offset (%r7, -104) + + .globl __syscall_cancel_arch_start +__syscall_cancel_arch_start: + /* if (*cancelhandling & CANCELED_BITMASK) + __syscall_do_cancel() */ + tm 3(%r2),TCB_CANCELED_BITMASK + jne 1f + + /* Issue a 6 argument syscall, the nr [%r1] being the syscall + number. */ + lgr %r1,%r3 + lgr %r2,%r4 + lgr %r3,%r5 + lgr %r4,%r6 + lmg %r5,%r7,160(%r15) + svc 0 + + .globl __syscall_cancel_arch_end +__syscall_cancel_arch_end: + lmg %r6,%r7,48(%r15) + cfi_remember_state + cfi_restore (%r7) + cfi_restore (%r6) + br %r14 +1: + cfi_restore_state + jg __syscall_do_cancel +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/sched_getattr.c b/sysdeps/unix/sysv/linux/sched_getattr.c new file mode 100644 index 0000000000..e6b9970fcb --- /dev/null +++ b/sysdeps/unix/sysv/linux/sched_getattr.c @@ -0,0 +1,27 @@ +/* Reading scheduling policy and attributes. + Copyright (C) 2024 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation; either version 2.1 of the + License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <sched.h> +#include <sysdep.h> + +int +sched_getattr (pid_t pid, struct sched_attr *attr, unsigned int size, + unsigned int flags) +{ + return INLINE_SYSCALL_CALL (sched_getattr, pid, attr, size, flags); +} diff --git a/sysdeps/unix/sysv/linux/sched_setattr.c b/sysdeps/unix/sysv/linux/sched_setattr.c new file mode 100644 index 0000000000..25403fb454 --- /dev/null +++ b/sysdeps/unix/sysv/linux/sched_setattr.c @@ -0,0 +1,26 @@ +/* Setting scheduling policy and attributes. + Copyright (C) 2024 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation; either version 2.1 of the + License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <sched.h> +#include <sysdep.h> + +int +sched_setattr (pid_t pid, struct sched_attr *attr, unsigned int flags) +{ + return INLINE_SYSCALL_CALL (sched_setattr, pid, attr, flags); +} diff --git a/sysdeps/unix/sysv/linux/sh/arch-syscall.h b/sysdeps/unix/sysv/linux/sh/arch-syscall.h index 0c88bf10c7..165ba017c7 100644 --- a/sysdeps/unix/sysv/linux/sh/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/sh/arch-syscall.h @@ -206,6 +206,7 @@ #define __NR_mq_timedsend_time64 418 #define __NR_mq_unlink 278 #define __NR_mremap 163 +#define __NR_mseal 462 #define __NR_msgctl 402 #define __NR_msgget 399 #define __NR_msgrcv 401 @@ -378,6 +379,7 @@ #define __NR_symlinkat 304 #define __NR_sync 36 #define __NR_sync_file_range 314 +#define __NR_sync_file_range2 388 #define __NR_syncfs 362 #define __NR_sysfs 135 #define __NR_sysinfo 116 diff --git a/sysdeps/unix/sysv/linux/sh/be/libc.abilist b/sysdeps/unix/sysv/linux/sh/be/libc.abilist index ff6e6b1a13..fcb8161841 100644 --- a/sysdeps/unix/sysv/linux/sh/be/libc.abilist +++ b/sysdeps/unix/sysv/linux/sh/be/libc.abilist @@ -2828,6 +2828,8 @@ GLIBC_2.4 sys_errlist D 0x210 GLIBC_2.4 sys_nerr D 0x4 GLIBC_2.4 unlinkat F GLIBC_2.4 unshare F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/sh/le/libc.abilist b/sysdeps/unix/sysv/linux/sh/le/libc.abilist index 449d92bbc5..3fd078d125 100644 --- a/sysdeps/unix/sysv/linux/sh/le/libc.abilist +++ b/sysdeps/unix/sysv/linux/sh/le/libc.abilist @@ -2825,6 +2825,8 @@ GLIBC_2.4 sys_errlist D 0x210 GLIBC_2.4 sys_nerr D 0x4 GLIBC_2.4 unlinkat F GLIBC_2.4 unshare F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/sh/syscall_cancel.S b/sysdeps/unix/sysv/linux/sh/syscall_cancel.S new file mode 100644 index 0000000000..2afd23928d --- /dev/null +++ b/sysdeps/unix/sysv/linux/sh/syscall_cancel.S @@ -0,0 +1,126 @@ +/* Cancellable syscall wrapper. Linux/sh version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int __syscall_cancel_arch (int *cancelhandling, + long int nr, + long int arg1, + long int arg2, + long int arg3, + long int arg4, + long int arg5, + long int arg6) */ + +ENTRY (__syscall_cancel_arch) + +#ifdef SHARED + mov.l r12,@-r15 + cfi_def_cfa_offset (4) + cfi_offset (12, -4) + mova L(GT),r0 + mov.l L(GT),r12 + sts.l pr,@-r15 + cfi_def_cfa_offset (8) + cfi_offset (17, -8) + add r0,r12 +#else + sts.l pr,@-r15 + cfi_def_cfa_offset (4) + cfi_offset (17, -4) +#endif + + .globl __syscall_cancel_arch_start +__syscall_cancel_arch_start: + + /* if (*cancelhandling & CANCELED_BITMASK) + __syscall_do_cancel() */ + mov.l @r4,r0 + tst #TCB_CANCELED_BITMASK,r0 + bf/s 1f + + /* Issue a 6 argument syscall. */ + mov r5,r3 + mov r6,r4 + mov r7,r5 +#ifdef SHARED + mov.l @(8,r15),r6 + mov.l @(12,r15),r7 + mov.l @(16,r15),r0 + mov.l @(20,r15),r1 +#else + mov.l @(4,r15),r6 + mov.l @(8,r15),r7 + mov.l @(12,r15),r0 + mov.l @(16,r15),r1 +#endif + trapa #0x16 + + .globl __syscall_cancel_arch_end +__syscall_cancel_arch_end: + + /* The additional or is a workaround for a hardware issue: + http://documentation.renesas.com/eng/products/mpumcu/tu/tnsh7456ae.pdf + */ + or r0,r0 + or r0,r0 + or r0,r0 + or r0,r0 + or r0,r0 + + lds.l @r15+,pr + cfi_remember_state + cfi_restore (17) +#ifdef SHARED + cfi_def_cfa_offset (4) + rts + mov.l @r15+,r12 + cfi_def_cfa_offset (0) + cfi_restore (12) + .align 1 +1: + cfi_restore_state + mov.l L(SC),r1 + bsrf r1 +L(M): + nop + + .align 2 +L(GT): + .long _GLOBAL_OFFSET_TABLE_ +L(SC): + .long __syscall_do_cancel-(L(M)+2) +#else + cfi_def_cfa_offset (0) + rts + nop + + .align 1 +1: + cfi_restore_state + mov.l 2f,r1 + jsr @r1 + nop + + .align 2 +2: + .long __syscall_do_cancel +#endif + +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/socketcall.h b/sysdeps/unix/sysv/linux/socketcall.h index 537fa43678..0efa5ee9e4 100644 --- a/sysdeps/unix/sysv/linux/socketcall.h +++ b/sysdeps/unix/sysv/linux/socketcall.h @@ -88,14 +88,33 @@ sc_ret; \ }) - -#define SOCKETCALL_CANCEL(name, args...) \ - ({ \ - int oldtype = LIBC_CANCEL_ASYNC (); \ - long int sc_ret = __SOCKETCALL (SOCKOP_##name, args); \ - LIBC_CANCEL_RESET (oldtype); \ - sc_ret; \ - }) +#define __SOCKETCALL_CANCEL1(__name, __a1) \ + SYSCALL_CANCEL (socketcall, __name, \ + ((long int [1]) { (long int) __a1 })) +#define __SOCKETCALL_CANCEL2(__name, __a1, __a2) \ + SYSCALL_CANCEL (socketcall, __name, \ + ((long int [2]) { (long int) __a1, (long int) __a2 })) +#define __SOCKETCALL_CANCEL3(__name, __a1, __a2, __a3) \ + SYSCALL_CANCEL (socketcall, __name, \ + ((long int [3]) { (long int) __a1, (long int) __a2, (long int) __a3 })) +#define __SOCKETCALL_CANCEL4(__name, __a1, __a2, __a3, __a4) \ + SYSCALL_CANCEL (socketcall, __name, \ + ((long int [4]) { (long int) __a1, (long int) __a2, (long int) __a3, \ + (long int) __a4 })) +#define __SOCKETCALL_CANCEL5(__name, __a1, __a2, __a3, __a4, __a5) \ + SYSCALL_CANCEL (socketcall, __name, \ + ((long int [5]) { (long int) __a1, (long int) __a2, (long int) __a3, \ + (long int) __a4, (long int) __a5 })) +#define __SOCKETCALL_CANCEL6(__name, __a1, __a2, __a3, __a4, __a5, __a6) \ + SYSCALL_CANCEL (socketcall, __name, \ + ((long int [6]) { (long int) __a1, (long int) __a2, (long int) __a3, \ + (long int) __a4, (long int) __a5, (long int) __a6 })) + +#define __SOCKETCALL_CANCEL(...) __SOCKETCALL_DISP (__SOCKETCALL_CANCEL,\ + __VA_ARGS__) + +#define SOCKETCALL_CANCEL(name, args...) \ + __SOCKETCALL_CANCEL (SOCKOP_##name, args) #endif /* sys/socketcall.h */ diff --git a/sysdeps/unix/sysv/linux/sparc/sparc32/arch-syscall.h b/sysdeps/unix/sysv/linux/sparc/sparc32/arch-syscall.h index 19fa614624..3bad6f102f 100644 --- a/sysdeps/unix/sysv/linux/sparc/sparc32/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/sparc/sparc32/arch-syscall.h @@ -211,6 +211,7 @@ #define __NR_mq_timedsend_time64 418 #define __NR_mq_unlink 274 #define __NR_mremap 250 +#define __NR_mseal 462 #define __NR_msgctl 402 #define __NR_msgget 399 #define __NR_msgrcv 401 diff --git a/sysdeps/unix/sysv/linux/sparc/sparc32/clone.S b/sysdeps/unix/sysv/linux/sparc/sparc32/clone.S index 748d25fcfe..c9cf9bb055 100644 --- a/sysdeps/unix/sysv/linux/sparc/sparc32/clone.S +++ b/sysdeps/unix/sysv/linux/sparc/sparc32/clone.S @@ -28,6 +28,9 @@ .text ENTRY (__clone) save %sp,-96,%sp + save %sp,-96,%sp + flushw + restore cfi_def_cfa_register(%fp) cfi_window_save cfi_register(%o7, %i7) diff --git a/sysdeps/unix/sysv/linux/sparc/sparc32/libc.abilist b/sysdeps/unix/sysv/linux/sparc/sparc32/libc.abilist index e615be759a..1ce1fe9da7 100644 --- a/sysdeps/unix/sysv/linux/sparc/sparc32/libc.abilist +++ b/sysdeps/unix/sysv/linux/sparc/sparc32/libc.abilist @@ -3157,6 +3157,8 @@ GLIBC_2.4 wcstold F GLIBC_2.4 wcstold_l F GLIBC_2.4 wprintf F GLIBC_2.4 wscanf F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/sparc/sparc32/syscall_cancel.S b/sysdeps/unix/sysv/linux/sparc/sparc32/syscall_cancel.S new file mode 100644 index 0000000000..aa5c658ce1 --- /dev/null +++ b/sysdeps/unix/sysv/linux/sparc/sparc32/syscall_cancel.S @@ -0,0 +1,71 @@ +/* Cancellable syscall wrapper. Linux/sparc32 version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int __syscall_cancel_arch (int *cancelhandling, + long int nr, + long int arg1, + long int arg2, + long int arg3, + long int arg4, + long int arg5, + long int arg6) */ + +ENTRY (__syscall_cancel_arch) + save %sp, -96, %sp + + cfi_window_save + cfi_register (%o7, %i7) + cfi_def_cfa_register (%fp) + + .globl __syscall_cancel_arch_start +__syscall_cancel_arch_start: + + /* if (*cancelhandling & CANCELED_BITMASK) + __syscall_do_cancel() */ + ld [%i0], %g2 + andcc %g2, TCB_CANCELED_BITMASK, %g0 + bne,pn %icc, 2f + /* Issue a 6 argument syscall. */ + mov %i1, %g1 + mov %i2, %o0 + mov %i3, %o1 + mov %i4, %o2 + mov %i5, %o3 + ld [%fp+92], %o4 + ld [%fp+96], %o5 + ta 0x10 + + .globl __syscall_cancel_arch_end +__syscall_cancel_arch_end: + bcc 1f + nop + sub %g0, %o0, %o0 +1: + mov %o0, %i0 + return %i7+8 + nop + +2: + call __syscall_do_cancel, 0 + nop + nop + +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/sparc/sparc64/arch-syscall.h b/sysdeps/unix/sysv/linux/sparc/sparc64/arch-syscall.h index 18516f20cb..98e1437920 100644 --- a/sysdeps/unix/sysv/linux/sparc/sparc64/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/sparc/sparc64/arch-syscall.h @@ -192,6 +192,7 @@ #define __NR_mq_timedsend 275 #define __NR_mq_unlink 274 #define __NR_mremap 250 +#define __NR_mseal 462 #define __NR_msgctl 402 #define __NR_msgget 399 #define __NR_msgrcv 401 diff --git a/sysdeps/unix/sysv/linux/sparc/sparc64/clone.S b/sysdeps/unix/sysv/linux/sparc/sparc64/clone.S index e5ff2cf1a0..370d51fda2 100644 --- a/sysdeps/unix/sysv/linux/sparc/sparc64/clone.S +++ b/sysdeps/unix/sysv/linux/sparc/sparc64/clone.S @@ -32,6 +32,9 @@ ENTRY (__clone) save %sp, -192, %sp + save %sp, -192, %sp + flushw + restore cfi_def_cfa_register(%fp) cfi_window_save cfi_register(%o7, %i7) diff --git a/sysdeps/unix/sysv/linux/sparc/sparc64/libc.abilist b/sysdeps/unix/sysv/linux/sparc/sparc64/libc.abilist index bd36431dd7..07507b86f6 100644 --- a/sysdeps/unix/sysv/linux/sparc/sparc64/libc.abilist +++ b/sysdeps/unix/sysv/linux/sparc/sparc64/libc.abilist @@ -2793,6 +2793,8 @@ GLIBC_2.4 sys_errlist D 0x430 GLIBC_2.4 sys_nerr D 0x4 GLIBC_2.4 unlinkat F GLIBC_2.4 unshare F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/sparc/sparc64/syscall_cancel.S b/sysdeps/unix/sysv/linux/sparc/sparc64/syscall_cancel.S new file mode 100644 index 0000000000..21b0728d5a --- /dev/null +++ b/sysdeps/unix/sysv/linux/sparc/sparc64/syscall_cancel.S @@ -0,0 +1,74 @@ +/* Cancellable syscall wrapper. Linux/sparc64 version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + + .register %g2, #scratch + +/* long int __syscall_cancel_arch (int *cancelhandling, + long int nr, + long int arg1, + long int arg2, + long int arg3, + long int arg4, + long int arg5, + long int arg6) */ + +ENTRY (__syscall_cancel_arch) + save %sp, -176, %sp + + cfi_window_save + cfi_register (%o7, %i7) + cfi_def_cfa_register (%fp) + + .globl __syscall_cancel_arch_start +__syscall_cancel_arch_start: + + /* if (*cancelhandling & CANCELED_BITMASK) + __syscall_do_cancel() */ + lduw [%i0], %g2 + andcc %g2, TCB_CANCELED_BITMASK, %g0 + bne,pn %xcc, 2f + /* Issue a 6 argument syscall. */ + mov %i1, %g1 + mov %i2, %o0 + mov %i3, %o1 + mov %i4, %o2 + mov %i5, %o3 + ldx [%fp + STACK_BIAS + 176], %o4 + ldx [%fp + STACK_BIAS + 184], %o5 + ta 0x6d + + .global __syscall_cancel_arch_end +__syscall_cancel_arch_end: + + bcc,pt %xcc, 1f + nop + sub %g0, %o0, %o0 +1: + mov %o0, %i0 + return %i7+8 + nop + +2: + call __syscall_do_cancel, 0 + nop + nop + +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/syscall-names.list b/sysdeps/unix/sysv/linux/syscall-names.list index 672d39eaad..7871f93b94 100644 --- a/sysdeps/unix/sysv/linux/syscall-names.list +++ b/sysdeps/unix/sysv/linux/syscall-names.list @@ -21,8 +21,8 @@ # This file can list all potential system calls. The names are only # used if the installed kernel headers also provide them. -# The list of system calls is current as of Linux 6.9. -kernel 6.9 +# The list of system calls is current as of Linux 6.10. +kernel 6.10 FAST_atomic_update FAST_cmpxchg @@ -287,6 +287,7 @@ mq_timedsend mq_timedsend_time64 mq_unlink mremap +mseal msgctl msgget msgrcv diff --git a/sysdeps/unix/sysv/linux/syscall_cancel.c b/sysdeps/unix/sysv/linux/syscall_cancel.c new file mode 100644 index 0000000000..5fa0706486 --- /dev/null +++ b/sysdeps/unix/sysv/linux/syscall_cancel.c @@ -0,0 +1,73 @@ +/* Pthread cancellation syscall bridge. Default Linux version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <pthreadP.h> + +#warning "This implementation should be use just as reference or for bootstrapping" + +/* This is the generic version of the cancellable syscall code which + adds the label guards (__syscall_cancel_arch_{start,end}) used on SIGCANCEL + handler to check if the cancelled syscall have side-effects that need to be + returned to the caller. + + This implementation should be used as a reference one to document the + implementation constraints: + + 1. The __syscall_cancel_arch_start should point just before the test + that thread is already cancelled, + 2. The __syscall_cancel_arch_end should point to the immediate next + instruction after the syscall one. + 3. It should return the syscall value or a negative result if is has + failed, similar to INTERNAL_SYSCALL_CALL. + + The __syscall_cancel_arch_end one is because the kernel will signal + interrupted syscall with side effects by setting the signal frame program + counter (on the ucontext_t third argument from SA_SIGINFO signal handler) + right after the syscall instruction. + + For some architecture, the INTERNAL_SYSCALL_NCS macro use more instructions + to get the error condition from kernel (as for powerpc and sparc that + checks for the conditional register), or uses an out of the line helper + (ARM thumb), or uses a kernel helper gate (i686 or ia64). In this case + the architecture should either adjust the macro or provide a custom + __syscall_cancel_arch implementation. */ + +long int +__syscall_cancel_arch (volatile int *ch, __syscall_arg_t nr, + __syscall_arg_t a1, __syscall_arg_t a2, + __syscall_arg_t a3, __syscall_arg_t a4, + __syscall_arg_t a5, __syscall_arg_t a6 + __SYSCALL_CANCEL7_ARG_DEF) +{ +#define ADD_LABEL(__label) \ + asm volatile ( \ + ".global " __label "\t\n" \ + __label ":\n"); + + ADD_LABEL ("__syscall_cancel_arch_start"); + if (__glibc_unlikely (*ch & CANCELED_BITMASK)) + __syscall_do_cancel(); + + long int result = INTERNAL_SYSCALL_NCS_CALL (nr, a1, a2, a3, a4, a5, a6 + __SYSCALL_CANCEL7_ARG7); + ADD_LABEL ("__syscall_cancel_arch_end"); + if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result))) + return -INTERNAL_SYSCALL_ERRNO (result); + return result; +} diff --git a/sysdeps/unix/sysv/linux/sysdep-cancel.h b/sysdeps/unix/sysv/linux/sysdep-cancel.h index c48a50fa88..3f1543fec2 100644 --- a/sysdeps/unix/sysv/linux/sysdep-cancel.h +++ b/sysdeps/unix/sysv/linux/sysdep-cancel.h @@ -21,17 +21,5 @@ #define _SYSDEP_CANCEL_H #include <sysdep.h> -#include <tls.h> -#include <errno.h> - -/* Set cancellation mode to asynchronous. */ -extern int __pthread_enable_asynccancel (void); -libc_hidden_proto (__pthread_enable_asynccancel) -#define LIBC_CANCEL_ASYNC() __pthread_enable_asynccancel () - -/* Reset to previous cancellation mode. */ -extern void __pthread_disable_asynccancel (int oldtype); -libc_hidden_proto (__pthread_disable_asynccancel) -#define LIBC_CANCEL_RESET(oldtype) __pthread_disable_asynccancel (oldtype) #endif diff --git a/sysdeps/unix/sysv/linux/tst-linux-mremap1.c b/sysdeps/unix/sysv/linux/tst-linux-mremap1.c new file mode 100644 index 0000000000..408e8af2ab --- /dev/null +++ b/sysdeps/unix/sysv/linux/tst-linux-mremap1.c @@ -0,0 +1,63 @@ +/* Test mremap with MREMAP_DONTUNMAP. + Copyright (C) 2024 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <errno.h> +#include <sys/mman.h> +#include <support/xstdlib.h> +#include <support/xunistd.h> +#include <support/check.h> +#include <support/test-driver.h> +#include <mremap-failure.h> + +static int +do_test (void) +{ + size_t old_size = getpagesize (); + size_t new_size = old_size; + char *old_addr = xmmap (NULL, old_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); + old_addr[0] = 1; + old_addr[old_size - 1] = 2; + + /* Create an available 64-page mmap region. */ + size_t fixed_size = old_size * 64; + char *fixed_addr = xmmap (NULL, fixed_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); + xmunmap (fixed_addr, fixed_size); + + /* Add 3 * pagesize. */ + fixed_size += 3 * old_size; + + /* Test MREMAP_DONTUNMAP. It should return FIXED_ADDR created above. */ + char *new_addr = mremap (old_addr, old_size, new_size, + MREMAP_DONTUNMAP | MREMAP_MAYMOVE, + fixed_addr); + if (new_addr == MAP_FAILED) + return mremap_failure_exit (errno); + TEST_VERIFY_EXIT (fixed_addr == new_addr); + old_addr[0] = 3; + old_addr[old_size - 1] = 4; + new_addr[0] = 1; + new_addr[new_size - 1] = 2; + xmunmap (new_addr, new_size); + xmunmap (old_addr, old_size); + + return 0; +} + +#include <support/test-driver.c> diff --git a/sysdeps/unix/sysv/linux/tst-mman-consts.py b/sysdeps/unix/sysv/linux/tst-mman-consts.py index 441261c945..a1137eb1d5 100644 --- a/sysdeps/unix/sysv/linux/tst-mman-consts.py +++ b/sysdeps/unix/sysv/linux/tst-mman-consts.py @@ -33,7 +33,7 @@ def main(): help='C compiler (including options) to use') args = parser.parse_args() linux_version_headers = glibcsyscalls.linux_kernel_version(args.cc) - linux_version_glibc = (6, 9) + linux_version_glibc = (6, 10) sys.exit(glibcextract.compare_macro_consts( '#define _GNU_SOURCE 1\n' '#include <sys/mman.h>\n', diff --git a/sysdeps/unix/sysv/linux/tst-mount-consts.py b/sysdeps/unix/sysv/linux/tst-mount-consts.py index c4a67221c1..675f1790b6 100755 --- a/sysdeps/unix/sysv/linux/tst-mount-consts.py +++ b/sysdeps/unix/sysv/linux/tst-mount-consts.py @@ -39,10 +39,10 @@ def main(): sys.exit (77) linux_version_headers = glibcsyscalls.linux_kernel_version(args.cc) - # Constants in glibc were updated to match Linux v6.9. When glibc + # Constants in glibc were updated to match Linux v6.10. When glibc # constants are updated this value should be updated to match the # released kernel version from which the constants were taken. - linux_version_glibc = (6, 9) + linux_version_glibc = (6, 10) def check(cte, exclude=None): return glibcextract.compare_macro_consts( '#include <sys/mount.h>\n', diff --git a/sysdeps/unix/sysv/linux/tst-pidfd-consts.py b/sysdeps/unix/sysv/linux/tst-pidfd-consts.py index 6f05291949..9824fd214d 100644 --- a/sysdeps/unix/sysv/linux/tst-pidfd-consts.py +++ b/sysdeps/unix/sysv/linux/tst-pidfd-consts.py @@ -39,7 +39,7 @@ def main(): sys.exit (77) linux_version_headers = glibcsyscalls.linux_kernel_version(args.cc) - linux_version_glibc = (6, 9) + linux_version_glibc = (6, 10) sys.exit(glibcextract.compare_macro_consts( '#include <sys/pidfd.h>\n', '#include <asm/fcntl.h>\n' diff --git a/sysdeps/unix/sysv/linux/tst-pkey.c b/sysdeps/unix/sysv/linux/tst-pkey.c index 46f55666da..297a7fbc02 100644 --- a/sysdeps/unix/sysv/linux/tst-pkey.c +++ b/sysdeps/unix/sysv/linux/tst-pkey.c @@ -210,7 +210,7 @@ do_test (void) if (errno == ENOSYS) FAIL_UNSUPPORTED ("glibc does not support memory protection keys"); - FAIL_EXIT1 ("pkey_alloc: %m"); + FAIL_EXIT1 ("pkey_get: %m"); } for (int i = 1; i < key_count; ++i) { diff --git a/sysdeps/unix/sysv/linux/tst-rseq-nptl.c b/sysdeps/unix/sysv/linux/tst-rseq-nptl.c index b9f498473d..2a246c4593 100644 --- a/sysdeps/unix/sysv/linux/tst-rseq-nptl.c +++ b/sysdeps/unix/sysv/linux/tst-rseq-nptl.c @@ -28,6 +28,11 @@ #include <sys/rseq.h> #include <unistd.h> +/* Set this in 'do_test' only so as to invoke the destructor test in + the test process only and not 'support_test_main' parent. Otherwise + the test harness may hang in the destructor if something goes wrong. */ +static int run_destructor_test; + #ifdef RSEQ_SIG # include <array_length.h> # include <errno.h> @@ -236,6 +241,9 @@ do_rseq_test (void) static void __attribute__ ((destructor)) do_rseq_destructor_test (void) { + if (!run_destructor_test) + return; + /* Cannot use deferred failure reporting after main returns. */ if (do_rseq_test ()) FAIL_EXIT1 ("rseq not registered within destructor"); @@ -254,6 +262,7 @@ do_rseq_test (void) static int do_test (void) { + run_destructor_test = 1; return do_rseq_test (); } diff --git a/sysdeps/unix/sysv/linux/tst-sched-consts.py b/sysdeps/unix/sysv/linux/tst-sched-consts.py new file mode 100644 index 0000000000..70071dcd97 --- /dev/null +++ b/sysdeps/unix/sysv/linux/tst-sched-consts.py @@ -0,0 +1,56 @@ +#!/usr/bin/python3 +# Test that glibc's sched.h constants match the kernel's. +# Copyright (C) 2018-2024 Free Software Foundation, Inc. +# This file is part of the GNU C Library. +# +# The GNU C Library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# The GNU C Library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with the GNU C Library; if not, see +# <https://www.gnu.org/licenses/>. + +import argparse +import sys + +import glibcextract +import glibcsyscalls + + +def main(): + """The main entry point.""" + parser = argparse.ArgumentParser( + description="Test that glibc's sched.h constants " + "match the kernel's.") + parser.add_argument('--cc', metavar='CC', + help='C compiler (including options) to use') + args = parser.parse_args() + linux_version_headers = glibcsyscalls.linux_kernel_version(args.cc) + linux_version_glibc = (6, 10) + sys.exit(glibcextract.compare_macro_consts( + '#define _GNU_SOURCE 1\n' + '#include <sched.h>\n', + '#define _GNU_SOURCE 1\n' + '#include <linux/sched.h>\n' + '#include <linux/sched/types.h>\n', + args.cc, + 'SCHED_.*', + # SCHED_ISO is reserved, but not implemented in the kernel. + # SCHED_OTHER is the standard name for SCHED_NORMAL. + # SCHED_FLAG_ALL will receive more and more flags, so + # exposing it to userspace does not seem useful. + 'SCHED_ISO' + '|SCHED_OTHER' + '|SCHED_FLAG_ALL', + linux_version_glibc > linux_version_headers, + linux_version_headers > linux_version_glibc)) + +if __name__ == '__main__': + main() diff --git a/sysdeps/unix/sysv/linux/tst-sched_setattr.c b/sysdeps/unix/sysv/linux/tst-sched_setattr.c new file mode 100644 index 0000000000..fbb73c31a7 --- /dev/null +++ b/sysdeps/unix/sysv/linux/tst-sched_setattr.c @@ -0,0 +1,107 @@ +/* Tests for sched_setattr and sched_getattr. + Copyright (C) 2024 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation; either version 2.1 of the + License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <sched.h> + +#include <errno.h> +#include <stddef.h> +#include <string.h> +#include <support/check.h> +#include <sys/resource.h> +#include <unistd.h> + +/* Padding struct to detect unexpected writes. */ +union +{ + struct sched_attr attr; + /* Hopefully the kernel will never need as much. */ + unsigned char padding[4096]; +} u; + +static void +check_unused (void) +{ + TEST_VERIFY (u.attr.size < sizeof (u)); + for (unsigned int i = u.attr.size; i < sizeof (u); ++i) + TEST_COMPARE (u.padding[i], 0xcc); +} + +static int +do_test (void) +{ + _Static_assert (SCHED_OTHER == SCHED_NORMAL, + "SCHED_OTHER, SCHED_NORMAL values"); + TEST_VERIFY (sizeof (struct sched_attr) < sizeof (u)); + + /* Check that reading and re-applying the current policy works. */ + memset (&u, 0xcc, sizeof (u)); + /* Compiler barrier to bypass write access attribute. */ + volatile unsigned int size = sizeof (u); + TEST_COMPARE (sched_getattr (0, (struct sched_attr *) &u, size, 0), 0); + check_unused (); + TEST_COMPARE (sched_setattr (0, &u.attr, 0), 0); /* Apply unchanged. */ + + /* Try to switch to the SCHED_OTHER policy. */ + memset (&u, 0, sizeof (u)); + u.attr.size = sizeof (u); /* With padding, kernel should accept zeroes. */ + u.attr.sched_policy = SCHED_OTHER; /* Should be the default. */ + { + errno = 0; + int prio = getpriority (PRIO_PROCESS, 0); + if (errno != 0) + prio = 0; + u.attr.sched_nice = prio; + } + TEST_COMPARE (sched_setattr (0, &u.attr, 0), 0); + + /* Non-zero values not known to the kernel result in an E2BIG error. */ + memset (&u, 0, sizeof (u)); + TEST_COMPARE (sched_getattr (0, (struct sched_attr *) &u, size, 0), 0); + u.padding[u.attr.size] = 0xcc; + u.attr.size = sizeof (u); + errno = 0; + TEST_COMPARE (sched_setattr (0, &u.attr, 0), -1); + TEST_COMPARE (errno, E2BIG); + + memset (&u, 0xcc, sizeof (u)); + TEST_COMPARE (sched_getattr (0, (struct sched_attr *) &u, size, 0), 0); + TEST_COMPARE (u.attr.sched_policy, SCHED_OTHER); + check_unused (); + + /* Raise the niceless level to 19 and observe its effect. */ + TEST_COMPARE (nice (19), 19); + TEST_COMPARE (sched_getattr (0, &u.attr, sizeof (u.attr), 0), 0); + TEST_COMPARE (u.attr.sched_policy, SCHED_OTHER); + TEST_COMPARE (u.attr.sched_nice, 19); + check_unused (); + + /* Invalid buffer arguments result in EINVAL (not EFAULT). */ + { + errno = 0; + void *volatile null_pointer = NULL; /* compiler barrier. */ + TEST_COMPARE (sched_setattr (0, null_pointer, 0), -1); + TEST_COMPARE (errno, EINVAL); + errno = 0; + TEST_COMPARE (sched_getattr (0, null_pointer, size, 0), -1); + TEST_COMPARE (errno, EINVAL); + } + + return 0; +} + +#include <support/test-driver.c> diff --git a/sysdeps/unix/sysv/linux/tst-skeleton-affinity.c b/sysdeps/unix/sysv/linux/tst-skeleton-affinity.c index 31a15b3ad7..2f921ed397 100644 --- a/sysdeps/unix/sysv/linux/tst-skeleton-affinity.c +++ b/sysdeps/unix/sysv/linux/tst-skeleton-affinity.c @@ -157,7 +157,7 @@ test_size (const struct conf *conf, size_t size) if (setaffinity (kernel_size, initial_set) < 0) { printf ("error: size %zu: setaffinity: %m\n", size); - return true; + return false; } /* Use one-CPU set to test switching between CPUs. */ diff --git a/sysdeps/unix/sysv/linux/x86_64/64/arch-syscall.h b/sysdeps/unix/sysv/linux/x86_64/64/arch-syscall.h index b122216013..5d86e75dd5 100644 --- a/sysdeps/unix/sysv/linux/x86_64/64/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/x86_64/64/arch-syscall.h @@ -189,6 +189,7 @@ #define __NR_mq_timedsend 242 #define __NR_mq_unlink 241 #define __NR_mremap 25 +#define __NR_mseal 462 #define __NR_msgctl 71 #define __NR_msgget 68 #define __NR_msgrcv 70 diff --git a/sysdeps/unix/sysv/linux/x86_64/64/libc.abilist b/sysdeps/unix/sysv/linux/x86_64/64/libc.abilist index aea7848ed6..5acf49dbe8 100644 --- a/sysdeps/unix/sysv/linux/x86_64/64/libc.abilist +++ b/sysdeps/unix/sysv/linux/x86_64/64/libc.abilist @@ -2744,6 +2744,8 @@ GLIBC_2.4 sys_errlist D 0x420 GLIBC_2.4 sys_nerr D 0x4 GLIBC_2.4 unlinkat F GLIBC_2.4 unshare F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F GLIBC_2.5 __readlinkat_chk F GLIBC_2.5 inet6_opt_append F GLIBC_2.5 inet6_opt_find F diff --git a/sysdeps/unix/sysv/linux/x86_64/dl-cet.h b/sysdeps/unix/sysv/linux/x86_64/dl-cet.h index 1fe3133406..b4f7e6c9cd 100644 --- a/sysdeps/unix/sysv/linux/x86_64/dl-cet.h +++ b/sysdeps/unix/sysv/linux/x86_64/dl-cet.h @@ -92,9 +92,9 @@ dl_cet_ibt_enabled (void) # Pass GL(dl_x86_feature_1) to _dl_cet_setup_features.\n\ movl %edx, %edi\n\ # Align stack for the _dl_cet_setup_features call.\n\ - andq $-16, %rsp\n\ + and $-16, %" RSP_LP "\n\ call _dl_cet_setup_features\n\ # Restore %rax and %rsp from %r12 and %r13.\n\ - movq %r12, %rax\n\ - movq %r13, %rsp\n\ + mov %" R12_LP ", %" RAX_LP "\n\ + mov %" R13_LP ", %" RSP_LP "\n\ " diff --git a/sysdeps/unix/sysv/linux/x86_64/syscall_cancel.S b/sysdeps/unix/sysv/linux/x86_64/syscall_cancel.S new file mode 100644 index 0000000000..cda9d20a83 --- /dev/null +++ b/sysdeps/unix/sysv/linux/x86_64/syscall_cancel.S @@ -0,0 +1,57 @@ +/* Cancellable syscall wrapper. Linux/x86_64 version. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <descr-const.h> + +/* long int [rax] __syscall_cancel_arch (volatile int *cancelhandling [%rdi], + __syscall_arg_t nr [%rsi], + __syscall_arg_t arg1 [%rdx], + __syscall_arg_t arg2 [%rcx], + __syscall_arg_t arg3 [%r8], + __syscall_arg_t arg4 [%r9], + __syscall_arg_t arg5 [SP+8], + __syscall_arg_t arg6 [SP+16]) */ + +ENTRY (__syscall_cancel_arch) + .globl __syscall_cancel_arch_start +__syscall_cancel_arch_start: + + /* if (*cancelhandling & CANCELED_BITMASK) + __syscall_do_cancel() */ + mov (%rdi),%eax + testb $TCB_CANCELED_BITMASK, (%rdi) + jne __syscall_do_cancel + + /* Issue a 6 argument syscall, the nr [%rax] being the syscall + number. */ + mov %rdi,%r11 + mov %rsi,%rax + mov %rdx,%rdi + mov %rcx,%rsi + mov %r8,%rdx + mov %r9,%r10 + mov 8(%rsp),%r8 + mov 16(%rsp),%r9 + mov %r11,8(%rsp) + syscall + + .globl __syscall_cancel_arch_end +__syscall_cancel_arch_end: + ret +END (__syscall_cancel_arch) diff --git a/sysdeps/unix/sysv/linux/x86_64/x32/arch-syscall.h b/sysdeps/unix/sysv/linux/x86_64/x32/arch-syscall.h index 3040a47d72..6c35068d8e 100644 --- a/sysdeps/unix/sysv/linux/x86_64/x32/arch-syscall.h +++ b/sysdeps/unix/sysv/linux/x86_64/x32/arch-syscall.h @@ -155,6 +155,7 @@ #define __NR_lsm_set_self_attr 1073742284 #define __NR_lstat 1073741830 #define __NR_madvise 1073741852 +#define __NR_map_shadow_stack 1073742277 #define __NR_mbind 1073742061 #define __NR_membarrier 1073742148 #define __NR_memfd_create 1073742143 @@ -182,6 +183,7 @@ #define __NR_mq_timedsend 1073742066 #define __NR_mq_unlink 1073742065 #define __NR_mremap 1073741849 +#define __NR_mseal 1073742286 #define __NR_msgctl 1073741895 #define __NR_msgget 1073741892 #define __NR_msgrcv 1073741894 diff --git a/sysdeps/unix/sysv/linux/x86_64/x32/fixup-asm-unistd.h b/sysdeps/unix/sysv/linux/x86_64/x32/fixup-asm-unistd.h index 98124169e6..47fa8af4ce 100644 --- a/sysdeps/unix/sysv/linux/x86_64/x32/fixup-asm-unistd.h +++ b/sysdeps/unix/sysv/linux/x86_64/x32/fixup-asm-unistd.h @@ -15,6 +15,10 @@ License along with the GNU C Library; if not, see <http://www.gnu.org/licenses/>. */ +#ifndef __NR_map_shadow_stack +# define __NR_map_shadow_stack 1073742277 +#endif + /* X32 uses the same 64-bit syscall interface for set_thread_area. */ #ifndef __NR_set_thread_area # define __NR_set_thread_area 1073742029 diff --git a/sysdeps/unix/sysv/linux/x86_64/x32/libc.abilist b/sysdeps/unix/sysv/linux/x86_64/x32/libc.abilist index 4ab3681914..02d1bb97dc 100644 --- a/sysdeps/unix/sysv/linux/x86_64/x32/libc.abilist +++ b/sysdeps/unix/sysv/linux/x86_64/x32/libc.abilist @@ -2763,3 +2763,5 @@ GLIBC_2.39 stdc_trailing_zeros_ui F GLIBC_2.39 stdc_trailing_zeros_ul F GLIBC_2.39 stdc_trailing_zeros_ull F GLIBC_2.39 stdc_trailing_zeros_us F +GLIBC_2.41 sched_getattr F +GLIBC_2.41 sched_setattr F diff --git a/sysdeps/generic/dl-fptr.h b/sysdeps/unix/sysv/linux/x86_64/x32/syscall_types.h index 71cfb62411..ac2019751d 100644 --- a/sysdeps/generic/dl-fptr.h +++ b/sysdeps/unix/sysv/linux/x86_64/x32/syscall_types.h @@ -1,5 +1,5 @@ -/* Function descriptors. Generic version. - Copyright (C) 1995-2024 Free Software Foundation, Inc. +/* Types and macros used for syscall issuing. x86_64/x32 version. + Copyright (C) 2023 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -16,30 +16,19 @@ License along with the GNU C Library; if not, see <https://www.gnu.org/licenses/>. */ -#ifndef dl_fptr_h -#define dl_fptr_h 1 +#ifndef _SYSCALL_TYPES_H +#define _SYSCALL_TYPES_H -/* An FDESC is a function descriptor. */ +#include <libc-diag.h> -struct fdesc - { - ElfW(Addr) ip; /* code entry point */ - ElfW(Addr) gp; /* global pointer */ - }; +typedef long long int __syscall_arg_t; -struct fdesc_table - { - struct fdesc_table *next; - unsigned int len; /* # of entries in fdesc table */ - volatile unsigned int first_unused; /* index of first available entry */ - struct fdesc fdesc[0]; - }; +/* Syscall arguments for x32 follows x86_64 ABI, however pointers are 32 bits + should be zero extended. */ +#define __SSC(__x) \ + ({ \ + TYPEFY (__x, __tmp) = ARGIFY (__x); \ + (__syscall_arg_t) __tmp; \ + }) -struct link_map; - -extern ElfW(Addr) _dl_boot_fptr_table []; - -extern ElfW(Addr) _dl_make_fptr (struct link_map *, const ElfW(Sym) *, - ElfW(Addr)); - -#endif /* !dl_fptr_h */ +#endif diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c index c096dd390a..1f30e237f5 100644 --- a/sysdeps/x86/cpu-features.c +++ b/sysdeps/x86/cpu-features.c @@ -756,6 +756,12 @@ init_cpu_features (struct cpu_features *cpu_features) unsigned int stepping = 0; enum cpu_features_kind kind; + /* Default is avoid non-temporal memset for non Intel/AMD/Hygon hardware. This is, + as of writing this, we only have benchmarks indicatings it profitability + on Intel/AMD/Hygon. */ + cpu_features->preferred[index_arch_Avoid_Non_Temporal_Memset] + |= bit_arch_Avoid_Non_Temporal_Memset; + cpu_features->cachesize_non_temporal_divisor = 4; #if !HAS_CPUID if (__get_cpuid_max (0, 0) == 0) @@ -781,6 +787,11 @@ init_cpu_features (struct cpu_features *cpu_features) update_active (cpu_features); + /* Benchmarks indicate non-temporal memset can be profitable on Intel + hardware. */ + cpu_features->preferred[index_arch_Avoid_Non_Temporal_Memset] + &= ~bit_arch_Avoid_Non_Temporal_Memset; + if (family == 0x06) { model += extended_model; @@ -879,6 +890,7 @@ init_cpu_features (struct cpu_features *cpu_features) non-temporal on all Skylake servers. */ cpu_features->preferred[index_arch_Avoid_Non_Temporal_Memset] |= bit_arch_Avoid_Non_Temporal_Memset; + /* fallthrough */ case INTEL_BIGCORE_COMETLAKE: case INTEL_BIGCORE_SKYLAKE: case INTEL_BIGCORE_KABYLAKE: @@ -974,9 +986,8 @@ https://www.intel.com/content/www/us/en/support/articles/000059422/processors.ht cpu_features->preferred[index_arch_Avoid_Short_Distance_REP_MOVSB] |= bit_arch_Avoid_Short_Distance_REP_MOVSB; } - /* This spells out "AuthenticAMD" or "HygonGenuine". */ - else if ((ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65) - || (ebx == 0x6f677948 && ecx == 0x656e6975 && edx == 0x6e65476e)) + /* This spells out "AuthenticAMD". */ + else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65) { unsigned int extended_model; @@ -991,6 +1002,11 @@ https://www.intel.com/content/www/us/en/support/articles/000059422/processors.ht ecx = cpu_features->features[CPUID_INDEX_1].cpuid.ecx; + /* Benchmarks indicate non-temporal memset can be profitable on AMD + hardware. */ + cpu_features->preferred[index_arch_Avoid_Non_Temporal_Memset] + &= ~bit_arch_Avoid_Non_Temporal_Memset; + if (CPU_FEATURE_USABLE_P (cpu_features, AVX)) { /* Since the FMA4 bit is in CPUID_INDEX_80000001 and @@ -1073,6 +1089,7 @@ https://www.intel.com/content/www/us/en/support/articles/000059422/processors.ht /* Yongfeng and Shijidadao mircoarch tuning. */ case 0x5b: cpu_features->cachesize_non_temporal_divisor = 2; + /* fallthrough */ case 0x6b: cpu_features->preferred[index_arch_AVX_Fast_Unaligned_Load] &= ~bit_arch_AVX_Fast_Unaligned_Load; @@ -1086,6 +1103,25 @@ https://www.intel.com/content/www/us/en/support/articles/000059422/processors.ht } } } + /* This spells out "HygonGenuine". */ + else if (ebx == 0x6f677948 && ecx == 0x656e6975 && edx == 0x6e65476e) + { + unsigned int extended_model; + + kind = arch_kind_hygon; + + get_common_indices (cpu_features, &family, &model, &extended_model, + &stepping); + + get_extended_indices (cpu_features); + + update_active (cpu_features); + + /* Benchmarks indicate non-temporal memset can be profitable on Hygon + hardware. */ + cpu_features->preferred[index_arch_Avoid_Non_Temporal_Memset] + &= ~bit_arch_Avoid_Non_Temporal_Memset; + } else { kind = arch_kind_other; @@ -1101,6 +1137,10 @@ https://www.intel.com/content/www/us/en/support/articles/000059422/processors.ht if (CPU_FEATURES_CPU_P (cpu_features, CMOV)) cpu_features->preferred[index_arch_I686] |= bit_arch_I686; + /* No ERMS, we want to avoid stosb for memset. */ + if (!CPU_FEATURE_USABLE_P (cpu_features, ERMS)) + cpu_features->preferred[index_arch_Avoid_STOSB] |= bit_arch_Avoid_STOSB; + #if !HAS_CPUID no_cpuid: #endif diff --git a/sysdeps/x86/cpu-tunables.c b/sysdeps/x86/cpu-tunables.c index ae9dcd6180..cd36de2d8b 100644 --- a/sysdeps/x86/cpu-tunables.c +++ b/sysdeps/x86/cpu-tunables.c @@ -193,6 +193,8 @@ TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp) 11); CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, Prefer_FSRM, 11); + CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, Avoid_STOSB, + 11); CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH (n, cpu_features, Slow_SSE4_2, SSE4_2, @@ -248,6 +250,7 @@ TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp) CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, Avoid_Non_Temporal_Memset, 25); } + break; case 26: { CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h index a1c03b8903..e9579505a3 100644 --- a/sysdeps/x86/dl-cacheinfo.h +++ b/sysdeps/x86/dl-cacheinfo.h @@ -567,6 +567,48 @@ handle_zhaoxin (int name) return 0; } +static long int __attribute__ ((noinline)) +handle_hygon (int name) +{ + unsigned int eax; + unsigned int ebx; + unsigned int ecx; + unsigned int edx; + unsigned int count = 0x1; + + if (name >= _SC_LEVEL3_CACHE_SIZE) + count = 0x3; + else if (name >= _SC_LEVEL2_CACHE_SIZE) + count = 0x2; + else if (name >= _SC_LEVEL1_DCACHE_SIZE) + count = 0x0; + + /* Use __cpuid__ '0x8000_001D' to compute cache details. */ + __cpuid_count (0x8000001D, count, eax, ebx, ecx, edx); + + switch (name) + { + case _SC_LEVEL1_ICACHE_ASSOC: + case _SC_LEVEL1_DCACHE_ASSOC: + case _SC_LEVEL2_CACHE_ASSOC: + case _SC_LEVEL3_CACHE_ASSOC: + return ((ebx >> 22) & 0x3ff) + 1; + case _SC_LEVEL1_ICACHE_LINESIZE: + case _SC_LEVEL1_DCACHE_LINESIZE: + case _SC_LEVEL2_CACHE_LINESIZE: + case _SC_LEVEL3_CACHE_LINESIZE: + return (ebx & 0xfff) + 1; + case _SC_LEVEL1_ICACHE_SIZE: + case _SC_LEVEL1_DCACHE_SIZE: + case _SC_LEVEL2_CACHE_SIZE: + case _SC_LEVEL3_CACHE_SIZE: + return (((ebx >> 22) & 0x3ff) + 1) * ((ebx & 0xfff) + 1) * (ecx + 1); + default: + __builtin_unreachable (); + } + return -1; +} + static void get_common_cache_info (long int *shared_ptr, long int * shared_per_thread_ptr, unsigned int *threads_ptr, long int core) @@ -889,6 +931,24 @@ dl_init_cacheinfo (struct cpu_features *cpu_features) shared_per_thread = shared; } + else if (cpu_features->basic.kind == arch_kind_hygon) + { + data = handle_hygon (_SC_LEVEL1_DCACHE_SIZE); + shared = handle_hygon (_SC_LEVEL3_CACHE_SIZE); + shared_per_thread = shared; + + level1_icache_size = handle_hygon (_SC_LEVEL1_ICACHE_SIZE); + level1_icache_linesize = handle_hygon (_SC_LEVEL1_ICACHE_LINESIZE); + level1_dcache_size = data; + level1_dcache_assoc = handle_hygon (_SC_LEVEL1_DCACHE_ASSOC); + level1_dcache_linesize = handle_hygon (_SC_LEVEL1_DCACHE_LINESIZE); + level2_cache_size = handle_hygon (_SC_LEVEL2_CACHE_SIZE);; + level2_cache_assoc = handle_hygon (_SC_LEVEL2_CACHE_ASSOC); + level2_cache_linesize = handle_hygon (_SC_LEVEL2_CACHE_LINESIZE); + level3_cache_size = shared; + level3_cache_assoc = handle_hygon (_SC_LEVEL3_CACHE_ASSOC); + level3_cache_linesize = handle_hygon (_SC_LEVEL3_CACHE_LINESIZE); + } cpu_features->level1_icache_size = level1_icache_size; cpu_features->level1_icache_linesize = level1_icache_linesize; @@ -988,14 +1048,6 @@ dl_init_cacheinfo (struct cpu_features *cpu_features) if (CPU_FEATURE_USABLE_P (cpu_features, FSRM)) rep_movsb_threshold = 2112; - /* Non-temporal stores are more performant on Intel and AMD hardware above - non_temporal_threshold. Enable this for both Intel and AMD hardware. */ - unsigned long int memset_non_temporal_threshold = SIZE_MAX; - if (!CPU_FEATURES_ARCH_P (cpu_features, Avoid_Non_Temporal_Memset) - && (cpu_features->basic.kind == arch_kind_intel - || cpu_features->basic.kind == arch_kind_amd)) - memset_non_temporal_threshold = non_temporal_threshold; - /* For AMD CPUs that support ERMS (Zen3+), REP MOVSB is in a lot of cases slower than the vectorized path (and for some alignments, it is really slow, check BZ #30994). */ @@ -1017,6 +1069,13 @@ dl_init_cacheinfo (struct cpu_features *cpu_features) if (tunable_size != 0) shared = tunable_size; + /* Non-temporal stores are more performant on some hardware above + non_temporal_threshold. Currently Prefer_Non_Temporal is set for for both + Intel, AMD and Hygon hardware. */ + unsigned long int memset_non_temporal_threshold = SIZE_MAX; + if (!CPU_FEATURES_ARCH_P (cpu_features, Avoid_Non_Temporal_Memset)) + memset_non_temporal_threshold = non_temporal_threshold; + tunable_size = TUNABLE_GET (x86_non_temporal_threshold, long int, NULL); if (tunable_size > minimum_non_temporal_threshold && tunable_size <= maximum_non_temporal_threshold) @@ -1042,18 +1101,42 @@ dl_init_cacheinfo (struct cpu_features *cpu_features) slightly better than ERMS. */ rep_stosb_threshold = SIZE_MAX; + /* + For memset, the non-temporal implementation is only accessed through the + stosb code. ie: + ``` + if (size >= rep_stosb_thresh) + { + if (size >= non_temporal_thresh) + { + do_non_temporal (); + } + do_stosb (); + } + do_normal_vec_loop (); + ``` + So if we prefer non-temporal, set `rep_stosb_thresh = non_temporal_thresh` + to enable the implementation. If `rep_stosb_thresh = non_temporal_thresh`, + `rep stosb` will never be used. + */ + TUNABLE_SET_WITH_BOUNDS (x86_memset_non_temporal_threshold, + memset_non_temporal_threshold, + minimum_non_temporal_threshold, SIZE_MAX); + /* Do `rep_stosb_thresh = non_temporal_thresh` after setting/getting the + final value of `x86_memset_non_temporal_threshold`. In some cases this can + be a matter of correctness. */ + if (CPU_FEATURES_ARCH_P (cpu_features, Avoid_STOSB)) + rep_stosb_threshold + = TUNABLE_GET (x86_memset_non_temporal_threshold, long int, NULL); + TUNABLE_SET_WITH_BOUNDS (x86_rep_stosb_threshold, rep_stosb_threshold, 1, + SIZE_MAX); TUNABLE_SET_WITH_BOUNDS (x86_data_cache_size, data, 0, SIZE_MAX); TUNABLE_SET_WITH_BOUNDS (x86_shared_cache_size, shared, 0, SIZE_MAX); TUNABLE_SET_WITH_BOUNDS (x86_non_temporal_threshold, non_temporal_threshold, minimum_non_temporal_threshold, maximum_non_temporal_threshold); - TUNABLE_SET_WITH_BOUNDS (x86_memset_non_temporal_threshold, - memset_non_temporal_threshold, - minimum_non_temporal_threshold, SIZE_MAX); TUNABLE_SET_WITH_BOUNDS (x86_rep_movsb_threshold, rep_movsb_threshold, minimum_rep_movsb_threshold, SIZE_MAX); - TUNABLE_SET_WITH_BOUNDS (x86_rep_stosb_threshold, rep_stosb_threshold, 1, - SIZE_MAX); unsigned long int rep_movsb_stop_threshold; /* Setting the upper bound of ERMS to the computed value of diff --git a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def index 61bbbc2e89..2a58000147 100644 --- a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def +++ b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def @@ -34,3 +34,4 @@ BIT (MathVec_Prefer_No_AVX512) BIT (Prefer_FSRM) BIT (Avoid_Short_Distance_REP_MOVSB) BIT (Avoid_Non_Temporal_Memset) +BIT (Avoid_STOSB) diff --git a/sysdeps/x86/include/cpu-features.h b/sysdeps/x86/include/cpu-features.h index aaae44f0e1..b8e790c8fd 100644 --- a/sysdeps/x86/include/cpu-features.h +++ b/sysdeps/x86/include/cpu-features.h @@ -881,6 +881,7 @@ enum cpu_features_kind arch_kind_intel, arch_kind_amd, arch_kind_zhaoxin, + arch_kind_hygon, arch_kind_other }; diff --git a/sysdeps/x86/tst-hwcap-tunables.c b/sysdeps/x86/tst-hwcap-tunables.c index 94307283d7..1920f5057e 100644 --- a/sysdeps/x86/tst-hwcap-tunables.c +++ b/sysdeps/x86/tst-hwcap-tunables.c @@ -60,7 +60,8 @@ static const struct test_t /* Disable everything. */ "-Prefer_ERMS,-Prefer_FSRM,-AVX,-AVX2,-AVX512F,-AVX512VL," "-SSE4_1,-SSE4_2,-SSSE3,-Fast_Unaligned_Load,-ERMS," - "-AVX_Fast_Unaligned_Load,-Avoid_Non_Temporal_Memset", + "-AVX_Fast_Unaligned_Load,-Avoid_Non_Temporal_Memset," + "-Avoid_STOSB", test_1, array_length (test_1) }, @@ -68,7 +69,8 @@ static const struct test_t /* Same as before, but with some empty suboptions. */ ",-,-Prefer_ERMS,-Prefer_FSRM,-AVX,-AVX2,-AVX512F,-AVX512VL," "-SSE4_1,-SSE4_2,-SSSE3,-Fast_Unaligned_Load,,-," - "-ERMS,-AVX_Fast_Unaligned_Load,-Avoid_Non_Temporal_Memset,-,", + "-ERMS,-AVX_Fast_Unaligned_Load,-Avoid_Non_Temporal_Memset," + "-Avoid_STOSB,-,", test_1, array_length (test_1) } diff --git a/sysdeps/x86_64/dl-machine.h b/sysdeps/x86_64/dl-machine.h index a6de3793e4..4f12955875 100644 --- a/sysdeps/x86_64/dl-machine.h +++ b/sysdeps/x86_64/dl-machine.h @@ -139,37 +139,37 @@ elf_machine_runtime_setup (struct link_map *l, struct r_scope_elem *scope[], .globl _start\n\ .globl _dl_start_user\n\ _start:\n\ - movq %rsp, %rdi\n\ + mov %" RSP_LP ", %" RDI_LP "\n\ call _dl_start\n\ _dl_start_user:\n\ # Save the user entry point address in %r12.\n\ - movq %rax, %r12\n\ + mov %" RAX_LP ", %" R12_LP "\n\ # Save %rsp value in %r13.\n\ - movq %rsp, %r13\n\ + mov %" RSP_LP ", % " R13_LP "\n\ "\ RTLD_START_ENABLE_X86_FEATURES \ "\ # Read the original argument count.\n\ - movq (%rsp), %rdx\n\ + mov (%rsp), %" RDX_LP "\n\ # Call _dl_init (struct link_map *main_map, int argc, char **argv, char **env)\n\ # argc -> rsi\n\ - movq %rdx, %rsi\n\ + mov %" RDX_LP ", %" RSI_LP "\n\ # And align stack for the _dl_init call. \n\ - andq $-16, %rsp\n\ + and $-16, %" RSP_LP "\n\ # _dl_loaded -> rdi\n\ - movq _rtld_local(%rip), %rdi\n\ + mov _rtld_local(%rip), %" RDI_LP "\n\ # env -> rcx\n\ - leaq 16(%r13,%rdx,8), %rcx\n\ + lea 2*" LP_SIZE "(%r13,%rdx," LP_SIZE "), %" RCX_LP "\n\ # argv -> rdx\n\ - leaq 8(%r13), %rdx\n\ + lea " LP_SIZE "(%r13), %" RDX_LP "\n\ # Clear %rbp to mark outermost frame obviously even for constructors.\n\ xorl %ebp, %ebp\n\ # Call the function to run the initializers.\n\ call _dl_init\n\ # Pass our finalizer function to the user in %rdx, as per ELF ABI.\n\ - leaq _dl_fini(%rip), %rdx\n\ + lea _dl_fini(%rip), %" RDX_LP "\n\ # And make sure %rsp points to argc stored on the stack.\n\ - movq %r13, %rsp\n\ + mov %" R13_LP ", %" RSP_LP "\n\ # Jump to the user's entry point.\n\ jmp *%r12\n\ .previous\n\ @@ -234,8 +234,13 @@ elf_machine_plt_value (struct link_map *map, const ElfW(Rela) *reloc, /* Names of the architecture-specific auditing callback functions. */ +#ifdef __LP64__ #define ARCH_LA_PLTENTER x86_64_gnu_pltenter #define ARCH_LA_PLTEXIT x86_64_gnu_pltexit +#else +#define ARCH_LA_PLTENTER x32_gnu_pltenter +#define ARCH_LA_PLTEXIT x32_gnu_pltexit +#endif #endif /* !dl_machine_h */ diff --git a/sysdeps/x86_64/fpu/libm-test-ulps b/sysdeps/x86_64/fpu/libm-test-ulps index 3592dfae1d..c2e36dcbdf 100644 --- a/sysdeps/x86_64/fpu/libm-test-ulps +++ b/sysdeps/x86_64/fpu/libm-test-ulps @@ -1430,28 +1430,28 @@ Function: "exp10_vlen8_avx2": float: 1 Function: "exp10m1": -double: 2 -float: 1 -float128: 1 -ldouble: 1 +double: 4 +float: 2 +float128: 3 +ldouble: 4 Function: "exp10m1_downward": -double: 1 -float: 1 -float128: 3 -ldouble: 2 +double: 3 +float: 3 +float128: 6 +ldouble: 6 Function: "exp10m1_towardzero": -double: 1 -float: 1 -float128: 3 -ldouble: 2 +double: 2 +float: 3 +float128: 6 +ldouble: 5 Function: "exp10m1_upward": -double: 3 -float: 1 -float128: 3 -ldouble: 2 +double: 5 +float: 4 +float128: 6 +ldouble: 6 Function: "exp2": double: 1 @@ -1498,28 +1498,28 @@ Function: "exp2_vlen8_avx2": float: 1 Function: "exp2m1": -double: 1 -float: 1 -float128: 1 -ldouble: 1 - -Function: "exp2m1_downward": double: 2 -float: 1 +float: 2 float128: 2 ldouble: 3 +Function: "exp2m1_downward": +double: 3 +float: 3 +float128: 3 +ldouble: 6 + Function: "exp2m1_towardzero": -double: 2 -float: 1 -float128: 2 -ldouble: 3 +double: 3 +float: 2 +float128: 4 +ldouble: 5 Function: "exp2m1_upward": -double: 1 -float: 1 -float128: 2 -ldouble: 3 +double: 3 +float: 3 +float128: 5 +ldouble: 6 Function: "exp_downward": double: 1 @@ -1808,28 +1808,28 @@ Function: "log10_vlen8_avx2": float: 1 Function: "log10p1": -double: 1 -float: 1 +double: 2 +float: 2 float128: 3 -ldouble: 2 +ldouble: 4 Function: "log10p1_downward": double: 2 -float: 1 -float128: 2 -ldouble: 4 +float: 3 +float128: 4 +ldouble: 8 Function: "log10p1_towardzero": -double: 2 +double: 3 float: 2 -float128: 2 -ldouble: 4 +float128: 3 +ldouble: 8 Function: "log10p1_upward": double: 2 -float: 1 -float128: 3 -ldouble: 3 +float: 3 +float128: 4 +ldouble: 6 Function: "log1p": double: 1 @@ -1920,10 +1920,10 @@ Function: "log2_vlen8_avx2": float: 1 Function: "log2p1": -double: 1 -float: 1 +double: 2 +float: 2 float128: 3 -ldouble: 2 +ldouble: 4 Function: "log2p1_downward": double: 2 @@ -1938,9 +1938,9 @@ float128: 2 ldouble: 4 Function: "log2p1_upward": -double: 1 +double: 2 float: 2 -float128: 2 +float128: 3 ldouble: 5 Function: "log_downward": diff --git a/sysdeps/x86_64/multiarch/ifunc-memset.h b/sysdeps/x86_64/multiarch/ifunc-memset.h index 7a637ef7ca..8dc3d7ab5a 100644 --- a/sysdeps/x86_64/multiarch/ifunc-memset.h +++ b/sysdeps/x86_64/multiarch/ifunc-memset.h @@ -46,6 +46,13 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned_erms) attribute_hidden; +static inline int +prefer_erms_nt_impl (const struct cpu_features *cpu_features) +{ + return CPU_FEATURE_USABLE_P (cpu_features, ERMS) + || !CPU_FEATURES_ARCH_P (cpu_features, Avoid_Non_Temporal_Memset); +} + static inline void * IFUNC_SELECTOR (void) { @@ -61,7 +68,7 @@ IFUNC_SELECTOR (void) && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, BMI2)) { - if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) + if (prefer_erms_nt_impl (cpu_features)) return OPTIMIZE (avx512_unaligned_erms); return OPTIMIZE (avx512_unaligned); @@ -76,7 +83,7 @@ IFUNC_SELECTOR (void) && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, BMI2)) { - if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) + if (prefer_erms_nt_impl (cpu_features)) return OPTIMIZE (evex_unaligned_erms); return OPTIMIZE (evex_unaligned); @@ -84,7 +91,7 @@ IFUNC_SELECTOR (void) if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) { - if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) + if (prefer_erms_nt_impl (cpu_features)) return OPTIMIZE (avx2_unaligned_erms_rtm); return OPTIMIZE (avx2_unaligned_rtm); @@ -93,14 +100,15 @@ IFUNC_SELECTOR (void) if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER, !)) { - if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) + if (prefer_erms_nt_impl (cpu_features)) return OPTIMIZE (avx2_unaligned_erms); return OPTIMIZE (avx2_unaligned); } } - if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) + if (CPU_FEATURE_USABLE_P (cpu_features, ERMS) + || !CPU_FEATURES_ARCH_P (cpu_features, Avoid_Non_Temporal_Memset)) return OPTIMIZE (sse2_unaligned_erms); return OPTIMIZE (sse2_unaligned); diff --git a/sysdeps/x86_64/multiarch/strchr-evex-base.S b/sysdeps/x86_64/multiarch/strchr-evex-base.S index 04e2c0e79e..3a0b7c9d64 100644 --- a/sysdeps/x86_64/multiarch/strchr-evex-base.S +++ b/sysdeps/x86_64/multiarch/strchr-evex-base.S @@ -124,13 +124,13 @@ L(page_cross): VPCMPNE %VMM(1), %VMM(0), %k1 VPTEST %VMM(1), %VMM(1), %k0{%k1} KMOV %k0, %VRAX -# ifdef USE_AS_WCSCHR + sar %cl, %VRAX +#ifdef USE_AS_WCSCHR sub $VEC_MATCH_MASK, %VRAX -# else +#else inc %VRAX -# endif +#endif /* Ignore number of character for alignment adjustment. */ - shr %cl, %VRAX jz L(align_more) bsf %VRAX, %VRAX diff --git a/sysdeps/x86_64/multiarch/strnlen-evex-base.S b/sysdeps/x86_64/multiarch/strnlen-evex-base.S new file mode 100644 index 0000000000..1c2cfdfe06 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strnlen-evex-base.S @@ -0,0 +1,462 @@ +/* strnlen/wcsnlen optimized with 256/512-bit EVEX instructions. + Copyright (C) 2022-2024 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + + +#include <isa-level.h> + +#if ISA_SHOULD_BUILD (4) + +# include <sysdep.h> + +#ifdef USE_AS_WCSLEN +# define VPCMPEQ vpcmpeqd +# define VPTESTN vptestnmd +# define VPMINU vpminud +# define CHAR_SIZE 4 +#else +# define VPCMPEQ vpcmpeqb +# define VPTESTN vptestnmb +# define VPMINU vpminub +# define CHAR_SIZE 1 +#endif + +#define XZERO VMM_128(0) +#define VZERO VMM(0) +#define PAGE_SIZE 4096 +#define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) + +#if CHAR_PER_VEC == 32 +# define SUB_SHORT(imm, reg) subb $(imm), %VGPR_SZ(reg, 8) +#else +# define SUB_SHORT(imm, reg) subl $(imm), %VGPR_SZ(reg, 32) +#endif + +#ifdef USE_AS_WCSLEN +/* For wide-character, we care more about limitting code size + than optimally aligning targets, so just cap nop padding + reasonably low. */ +# define P2ALIGN(...) .p2align 4,, 6 +# define P2ALIGN_CLAMPED(...) P2ALIGN(__VA_ARGS__) +#else +# define P2ALIGN(x) .p2align x +# define P2ALIGN_CLAMPED(x, y) .p2align x,, y +#endif + + .section SECTION(.text), "ax", @progbits + /* Aligning entry point to 64 byte, provides better performance for + one vector length string. */ +ENTRY_P2ALIGN(STRNLEN, 6) + /* rdi is pointer to array, rsi is the upper limit. */ + + /* Check zero length. */ + test %RSI_LP, %RSI_LP + jz L(zero) + +#ifdef __ILP32__ + /* Clear the upper 32 bits. */ + movl %esi, %esi +#endif + + vpxorq %XZERO, %XZERO, %XZERO + + /* Check that we won't cross a page boundary with our first load. */ + movl %edi, %eax + shll $20, %eax + cmpl $((PAGE_SIZE - VEC_SIZE) << 20), %eax + ja L(crosses_page_boundary) + + /* Check the first VEC_SIZE bytes. Each bit in K0 represents a + null byte. */ + VPCMPEQ (%rdi), %VZERO, %k0 + KMOV %k0, %VRCX + + /* If src (rcx) is zero, bsf does not change the result. NB: + Must use 64-bit bsf here so that upper bits of len are not + cleared. */ + movq %rsi, %rax + bsfq %rcx, %rax + + /* If rax > CHAR_PER_VEC then rcx must have been zero (no null + CHAR) and rsi must be > CHAR_PER_VEC. */ + cmpq $CHAR_PER_VEC, %rax + ja L(more_1x_vec) + + /* Check if first match in bounds. */ + cmpq %rax, %rsi + cmovb %esi, %eax + ret + +#if VEC_SIZE == 32 + P2ALIGN_CLAMPED(4, 2) +L(zero): +L(max_0): + movl %esi, %eax + ret +#endif + + P2ALIGN_CLAMPED(4, 10) +L(more_1x_vec): +L(cross_page_continue): + /* After this calculation, rax stores the number of elements + left to be processed The complexity comes from the fact some + elements get read twice due to alignment and we need to be + sure we don't count them twice (else, it would just be rsi - + CHAR_PER_VEC). */ + +#ifdef USE_AS_WCSLEN + /* Need to compute directly for wcslen as CHAR_SIZE * rsi can + overflow. */ + movq %rdi, %rax + andq $(VEC_SIZE * -1), %rdi + subq %rdi, %rax + sarq $2, %rax + leaq -(CHAR_PER_VEC * 1)(%rax, %rsi), %rax +#else + /* Calculate ptr + N - VEC_SIZE, then mask off the low bits, + then subtract ptr to get the new aligned limit value. */ + leaq (VEC_SIZE * -1)(%rsi, %rdi), %rax + andq $(VEC_SIZE * -1), %rdi + subq %rdi, %rax +#endif + + VPCMPEQ VEC_SIZE(%rdi), %VZERO, %k0 + + /* Checking here is faster for 256-bit but not 512-bit */ +#if VEC_SIZE == 0 + KMOV %k0, %VRDX + test %VRDX, %VRDX + jnz L(last_vec_check) +#endif + + cmpq $(CHAR_PER_VEC * 2), %rax + ja L(more_2x_vec) + +L(last_2x_vec_or_less): + + /* Checking here is faster for 512-bit but not 256-bit */ +#if VEC_SIZE != 0 + KMOV %k0, %VRDX + test %VRDX, %VRDX + jnz L(last_vec_check) +#endif + + /* Check for the end of data. */ + SUB_SHORT (CHAR_PER_VEC, rax) + jbe L(max_0) + + /* Check the final remaining vector. */ + VPCMPEQ (VEC_SIZE * 2)(%rdi), %VZERO, %k0 + KMOV %k0, %VRDX + test %VRDX, %VRDX +#if VEC_SIZE == 32 + jz L(max_0) +#else + jnz L(last_vec_check) + P2ALIGN_CLAMPED(4, 2) +L(zero): +L(max_0): + movl %esi, %eax + ret + +#endif + P2ALIGN_CLAMPED(4, 4) +L(last_vec_check): + bsf %VRDX, %VRDX + sub %eax, %edx + lea (%rsi, %rdx), %eax + cmovae %esi, %eax + ret + + +#if VEC_SIZE == 32 + P2ALIGN_CLAMPED(4, 8) +#endif +L(last_4x_vec_or_less): + addl $(CHAR_PER_VEC * -4), %eax + VPCMPEQ (VEC_SIZE * 5)(%rdi), %VZERO, %k0 + +#if VEC_SIZE == 64 + KMOV %k0, %VRDX + test %VRDX, %VRDX + jnz L(last_vec_check) +#endif + + subq $(VEC_SIZE * -4), %rdi + cmpl $(CHAR_PER_VEC * 2), %eax + jbe L(last_2x_vec_or_less) + + P2ALIGN_CLAMPED(4, 6) +L(more_2x_vec): + /* Remaining length >= 2 * CHAR_PER_VEC so do VEC0/VEC1 without + rechecking bounds. */ + + /* Already checked in 256-bit case */ +#if VEC_SIZE != 0 + KMOV %k0, %VRDX + + test %VRDX, %VRDX + jnz L(first_vec_x1) +#endif + + VPCMPEQ (VEC_SIZE * 2)(%rdi), %VZERO, %k0 + KMOV %k0, %VRDX + + test %VRDX, %VRDX + jnz L(first_vec_x2) + + cmpq $(CHAR_PER_VEC * 4), %rax + ja L(more_4x_vec) + + + VPCMPEQ (VEC_SIZE * 3)(%rdi), %VZERO, %k0 + KMOV %k0, %VRDX + addl $(CHAR_PER_VEC * -2), %eax + test %VRDX, %VRDX + jnz L(last_vec_check) + + subb $(CHAR_PER_VEC), %al + jbe L(max_1) + + VPCMPEQ (VEC_SIZE * 4)(%rdi), %VZERO, %k0 + KMOV %k0, %VRDX + + test %VRDX, %VRDX + jnz L(last_vec_check) +L(max_1): + movl %esi, %eax + ret + + + P2ALIGN_CLAMPED(4, 14) +L(first_vec_x2): +#if VEC_SIZE == 64 + /* If VEC_SIZE == 64 we can fit logic for full return label in + spare bytes before next cache line. */ + bsf %VRDX, %VRDX + sub %eax, %esi + leal (CHAR_PER_VEC * 1)(%rsi, %rdx), %eax + ret + P2ALIGN_CLAMPED(4, 6) +#else + addl $CHAR_PER_VEC, %esi +#endif +L(first_vec_x1): + bsf %VRDX, %VRDX + sub %eax, %esi + leal (CHAR_PER_VEC * 0)(%rsi, %rdx), %eax + ret + +#if VEC_SIZE == 64 + P2ALIGN_CLAMPED(4, 6) +L(first_vec_x4): +# if VEC_SIZE == 64 + /* If VEC_SIZE == 64 we can fit logic for full return label in + spare bytes before next cache line. */ + bsf %VRDX, %VRDX + sub %eax, %esi + leal (CHAR_PER_VEC * 3)(%rsi, %rdx), %eax + ret + P2ALIGN_CLAMPED(4, 6) +# else + addl $CHAR_PER_VEC, %esi +# endif +L(first_vec_x3): + bsf %VRDX, %VRDX + sub %eax, %esi + leal (CHAR_PER_VEC * 2)(%rsi, %rdx), %eax + ret +#endif + + P2ALIGN_CLAMPED(6, 20) +L(more_4x_vec): + VPCMPEQ (VEC_SIZE * 3)(%rdi), %VZERO, %k0 + KMOV %k0, %VRDX + test %VRDX, %VRDX + jnz L(first_vec_x3) + + VPCMPEQ (VEC_SIZE * 4)(%rdi), %VZERO, %k0 + KMOV %k0, %VRDX + test %VRDX, %VRDX + jnz L(first_vec_x4) + + /* Check if at last VEC_SIZE * 4 length before aligning for the + loop. */ + cmpq $(CHAR_PER_VEC * 8), %rax + jbe L(last_4x_vec_or_less) + + + /* Compute number of words checked after aligning. */ +#ifdef USE_AS_WCSLEN + /* Need to compute directly for wcslen as CHAR_SIZE * rsi can + overflow. */ + leaq (VEC_SIZE * -3)(%rdi), %rdx +#else + leaq (VEC_SIZE * -3)(%rdi, %rax), %rax +#endif + + subq $(VEC_SIZE * -1), %rdi + + /* Align data to VEC_SIZE * 4. */ +#if VEC_SIZE == 64 + /* Saves code size. No evex512 processor has partial register + stalls. If that change this can be replaced with `andq + $-(VEC_SIZE * 4), %rdi`. */ + xorb %dil, %dil +#else + andq $-(VEC_SIZE * 4), %rdi +#endif + +#ifdef USE_AS_WCSLEN + subq %rdi, %rdx + sarq $2, %rdx + addq %rdx, %rax +#else + subq %rdi, %rax +#endif + + // mov %rdi, %rdx + + P2ALIGN(6) +L(loop): + /* VPMINU and VPCMP combination provide better performance as + compared to alternative combinations. */ + VMOVA (VEC_SIZE * 4)(%rdi), %VMM(1) + VPMINU (VEC_SIZE * 5)(%rdi), %VMM(1), %VMM(2) + VMOVA (VEC_SIZE * 6)(%rdi), %VMM(3) + VPMINU (VEC_SIZE * 7)(%rdi), %VMM(3), %VMM(4) + + VPTESTN %VMM(2), %VMM(2), %k0 + VPTESTN %VMM(4), %VMM(4), %k1 + + subq $-(VEC_SIZE * 4), %rdi + KORTEST %k0, %k1 + + jnz L(loopend) + subq $(CHAR_PER_VEC * 4), %rax + ja L(loop) + mov %rsi, %rax + ret + + +#if VEC_SIZE == 32 + P2ALIGN_CLAMPED(4, 6) +L(first_vec_x4): +# if VEC_SIZE == 64 + /* If VEC_SIZE == 64 we can fit logic for full return label in + spare bytes before next cache line. */ + bsf %VRDX, %VRDX + sub %eax, %esi + leal (CHAR_PER_VEC * 3)(%rsi, %rdx), %eax + ret + P2ALIGN_CLAMPED(4, 6) +# else + addl $CHAR_PER_VEC, %esi +# endif +L(first_vec_x3): + bsf %VRDX, %VRDX + sub %eax, %esi + leal (CHAR_PER_VEC * 2)(%rsi, %rdx), %eax + ret +#endif + + + P2ALIGN_CLAMPED(4, 11) +L(loopend): + /* We found a null terminator in one of the 4 vectors. */ + + /* Check the first vector. */ + movq %rax, %r8 + VPTESTN %VMM(1), %VMM(1), %k2 + KMOV %k2, %VRCX + bsf %rcx, %r8 + + cmpq $(CHAR_PER_VEC), %r8 + jbe L(end_vec) + + /* Check the second vector. */ + subq $(CHAR_PER_VEC), %rax + movq %rax, %r8 + KMOV %k0, %VRCX + bsf %rcx, %r8 + + cmpq $(CHAR_PER_VEC), %r8 + jbe L(end_vec) + + /* Check the third vector. */ + subq $(CHAR_PER_VEC), %rax + movq %rax, %r8 + VPTESTN %VMM(3), %VMM(3), %k2 + KMOV %k2, %VRCX + bsf %rcx, %r8 + + cmpq $(CHAR_PER_VEC), %r8 + jbe L(end_vec) + + /* It is in the fourth vector. */ + subq $(CHAR_PER_VEC), %rax + movq %rax, %r8 + KMOV %k1, %VRCX + bsf %rcx, %r8 + + P2ALIGN_CLAMPED(4, 3) +L(end_vec): + /* Get the number that has been processed. */ + movq %rsi, %rcx + subq %rax, %rcx + + /* Add that to the offset we found the null terminator at. */ + leaq (%r8, %rcx), %rax + + /* Take the min of that and the limit. */ + cmpq %rsi, %rax + cmovnb %rsi, %rax + ret + + P2ALIGN_CLAMPED(4, 11) +L(crosses_page_boundary): + /* Align data backwards to VEC_SIZE. */ + shrl $20, %eax + movq %rdi, %rcx + andq $-VEC_SIZE, %rcx + VPCMPEQ (%rcx), %VZERO, %k0 + + KMOV %k0, %VRCX +#ifdef USE_AS_WCSLEN + shrl $2, %eax + andl $(CHAR_PER_VEC - 1), %eax +#endif + /* By this point rax contains number of bytes we need to skip. */ + shrx %VRAX, %VRCX, %VRCX + + /* Calculates CHAR_PER_VEC - eax and stores in eax. */ + negl %eax + andl $(CHAR_PER_VEC - 1), %eax + + movq %rsi, %rdx + bsf %VRCX, %VRDX + cmpq %rax, %rdx + ja L(cross_page_continue) + + /* The vector had a null terminator or we are at the limit. */ + movl %edx, %eax + cmpq %rdx, %rsi + cmovb %esi, %eax + ret + +END(STRNLEN) +#endif diff --git a/sysdeps/x86_64/multiarch/strnlen-evex.S b/sysdeps/x86_64/multiarch/strnlen-evex.S index 91b16830eb..c41288906c 100644 --- a/sysdeps/x86_64/multiarch/strnlen-evex.S +++ b/sysdeps/x86_64/multiarch/strnlen-evex.S @@ -1,423 +1,7 @@ -/* strnlen/wcsnlen optimized with 256-bit EVEX instructions. - Copyright (C) 2022-2024 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include <isa-level.h> -#include <sysdep.h> - -#if ISA_SHOULD_BUILD (4) - -# ifndef VEC_SIZE -# include "x86-evex256-vecs.h" -# endif - - -# ifndef STRNLEN -# define STRNLEN __strnlen_evex -# endif - -# ifdef USE_AS_WCSLEN -# define VPCMPEQ vpcmpeqd -# define VPCMPNEQ vpcmpneqd -# define VPTESTN vptestnmd -# define VPTEST vptestmd -# define VPMINU vpminud -# define CHAR_SIZE 4 - -# else -# define VPCMPEQ vpcmpeqb -# define VPCMPNEQ vpcmpneqb -# define VPTESTN vptestnmb -# define VPTEST vptestmb -# define VPMINU vpminub -# define CHAR_SIZE 1 - -# define REG_WIDTH VEC_SIZE -# endif - -# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) - -# include "reg-macros.h" - -# if CHAR_PER_VEC == 32 -# define SUB_SHORT(imm, reg) subb $(imm), %VGPR_SZ(reg, 8) -# else -# define SUB_SHORT(imm, reg) subl $(imm), %VGPR_SZ(reg, 32) -# endif - - - -# if CHAR_PER_VEC == 64 -# define FALLTHROUGH_RETURN_OFFSET (CHAR_PER_VEC * 3) -# else -# define FALLTHROUGH_RETURN_OFFSET (CHAR_PER_VEC * 2) -# endif - - -# define XZERO VMM_128(0) -# define VZERO VMM(0) -# define PAGE_SIZE 4096 - - .section SECTION(.text), "ax", @progbits -ENTRY_P2ALIGN (STRNLEN, 6) - /* Check zero length. */ - test %RSI_LP, %RSI_LP - jz L(zero) -# ifdef __ILP32__ - /* Clear the upper 32 bits. */ - movl %esi, %esi -# endif - - movl %edi, %eax - vpxorq %XZERO, %XZERO, %XZERO - andl $(PAGE_SIZE - 1), %eax - cmpl $(PAGE_SIZE - VEC_SIZE), %eax - ja L(cross_page_boundary) - - /* Check the first VEC_SIZE bytes. Each bit in K0 represents a - null byte. */ - VPCMPEQ (%rdi), %VZERO, %k0 - - KMOV %k0, %VRCX - movq %rsi, %rax - - /* If src (rcx) is zero, bsf does not change the result. NB: - Must use 64-bit bsf here so that upper bits of len are not - cleared. */ - bsfq %rcx, %rax - /* If rax > CHAR_PER_VEC then rcx must have been zero (no null - CHAR) and rsi must be > CHAR_PER_VEC. */ - cmpq $CHAR_PER_VEC, %rax - ja L(more_1x_vec) - /* Check if first match in bounds. */ - cmpq %rax, %rsi - cmovb %esi, %eax - ret - - -# if CHAR_PER_VEC != 32 - .p2align 4,, 2 -L(zero): -L(max_0): - movl %esi, %eax - ret -# endif - - /* Aligned more for strnlen compares remaining length vs 2 * - CHAR_PER_VEC, 4 * CHAR_PER_VEC, and 8 * CHAR_PER_VEC before - going to the loop. */ - .p2align 4,, 10 -L(more_1x_vec): -L(cross_page_continue): - /* Compute number of words checked after aligning. */ -# ifdef USE_AS_WCSLEN - /* Need to compute directly for wcslen as CHAR_SIZE * rsi can - overflow. */ - movq %rdi, %rax - andq $(VEC_SIZE * -1), %rdi - subq %rdi, %rax - sarq $2, %rax - leaq -(CHAR_PER_VEC * 1)(%rax, %rsi), %rax -# else - leaq (VEC_SIZE * -1)(%rsi, %rdi), %rax - andq $(VEC_SIZE * -1), %rdi - subq %rdi, %rax -# endif - - - VPCMPEQ VEC_SIZE(%rdi), %VZERO, %k0 - - cmpq $(CHAR_PER_VEC * 2), %rax - ja L(more_2x_vec) - -L(last_2x_vec_or_less): - KMOV %k0, %VRDX - test %VRDX, %VRDX - jnz L(last_vec_check) - - /* Check the end of data. */ - SUB_SHORT (CHAR_PER_VEC, rax) - jbe L(max_0) - VPCMPEQ (VEC_SIZE * 2)(%rdi), %VZERO, %k0 - KMOV %k0, %VRDX - test %VRDX, %VRDX - jz L(max_0) - /* Best place for LAST_VEC_CHECK if ZMM. */ - .p2align 4,, 8 -L(last_vec_check): - bsf %VRDX, %VRDX - sub %eax, %edx - lea (%rsi, %rdx), %eax - cmovae %esi, %eax - ret - -# if CHAR_PER_VEC == 32 - .p2align 4,, 2 -L(zero): -L(max_0): - movl %esi, %eax - ret -# endif - - .p2align 4,, 8 -L(last_4x_vec_or_less): - addl $(CHAR_PER_VEC * -4), %eax - VPCMPEQ (VEC_SIZE * 5)(%rdi), %VZERO, %k0 - subq $(VEC_SIZE * -4), %rdi - cmpl $(CHAR_PER_VEC * 2), %eax - jbe L(last_2x_vec_or_less) - - .p2align 4,, 6 -L(more_2x_vec): - /* Remaining length >= 2 * CHAR_PER_VEC so do VEC0/VEC1 without - rechecking bounds. */ - - KMOV %k0, %VRDX - - test %VRDX, %VRDX - jnz L(first_vec_x1) - - VPCMPEQ (VEC_SIZE * 2)(%rdi), %VZERO, %k0 - KMOV %k0, %VRDX - test %VRDX, %VRDX - jnz L(first_vec_x2) - - cmpq $(CHAR_PER_VEC * 4), %rax - ja L(more_4x_vec) - - - VPCMPEQ (VEC_SIZE * 3)(%rdi), %VZERO, %k0 - KMOV %k0, %VRDX - addl $(CHAR_PER_VEC * -2), %eax - test %VRDX, %VRDX - jnz L(last_vec_check) - - subl $(CHAR_PER_VEC), %eax - jbe L(max_1) - - VPCMPEQ (VEC_SIZE * 4)(%rdi), %VZERO, %k0 - KMOV %k0, %VRDX - - test %VRDX, %VRDX - jnz L(last_vec_check) -L(max_1): - movl %esi, %eax - ret - - .p2align 4,, 3 -L(first_vec_x2): -# if VEC_SIZE == 64 - /* If VEC_SIZE == 64 we can fit logic for full return label in - spare bytes before next cache line. */ - bsf %VRDX, %VRDX - sub %eax, %esi - leal (CHAR_PER_VEC * 1)(%rsi, %rdx), %eax - ret - .p2align 4,, 6 -# else - addl $CHAR_PER_VEC, %esi -# endif -L(first_vec_x1): - bsf %VRDX, %VRDX - sub %eax, %esi - leal (CHAR_PER_VEC * 0)(%rsi, %rdx), %eax - ret - - - .p2align 4,, 6 -L(first_vec_x4): -# if VEC_SIZE == 64 - /* If VEC_SIZE == 64 we can fit logic for full return label in - spare bytes before next cache line. */ - bsf %VRDX, %VRDX - sub %eax, %esi - leal (CHAR_PER_VEC * 3)(%rsi, %rdx), %eax - ret - .p2align 4,, 6 -# else - addl $CHAR_PER_VEC, %esi -# endif -L(first_vec_x3): - bsf %VRDX, %VRDX - sub %eax, %esi - leal (CHAR_PER_VEC * 2)(%rsi, %rdx), %eax - ret - - .p2align 4,, 5 -L(more_4x_vec): - VPCMPEQ (VEC_SIZE * 3)(%rdi), %VZERO, %k0 - KMOV %k0, %VRDX - test %VRDX, %VRDX - jnz L(first_vec_x3) - - VPCMPEQ (VEC_SIZE * 4)(%rdi), %VZERO, %k0 - KMOV %k0, %VRDX - test %VRDX, %VRDX - jnz L(first_vec_x4) - - /* Check if at last VEC_SIZE * 4 length before aligning for the - loop. */ - cmpq $(CHAR_PER_VEC * 8), %rax - jbe L(last_4x_vec_or_less) - - - /* Compute number of words checked after aligning. */ -# ifdef USE_AS_WCSLEN - /* Need to compute directly for wcslen as CHAR_SIZE * rsi can - overflow. */ - leaq (VEC_SIZE * -3)(%rdi), %rdx -# else - leaq (VEC_SIZE * -3)(%rdi, %rax), %rax -# endif - - subq $(VEC_SIZE * -1), %rdi - - /* Align data to VEC_SIZE * 4. */ -# if VEC_SIZE == 64 - /* Saves code size. No evex512 processor has partial register - stalls. If that change this can be replaced with `andq - $-(VEC_SIZE * 4), %rdi`. */ - xorb %dil, %dil -# else - andq $-(VEC_SIZE * 4), %rdi -# endif - -# ifdef USE_AS_WCSLEN - subq %rdi, %rdx - sarq $2, %rdx - addq %rdx, %rax -# else - subq %rdi, %rax -# endif - /* Compare 4 * VEC at a time forward. */ - .p2align 4,, 11 -L(loop_4x_vec): - VMOVA (VEC_SIZE * 4)(%rdi), %VMM(1) - VPMINU (VEC_SIZE * 5)(%rdi), %VMM(1), %VMM(2) - VMOVA (VEC_SIZE * 6)(%rdi), %VMM(3) - VPMINU (VEC_SIZE * 7)(%rdi), %VMM(3), %VMM(4) - VPTESTN %VMM(2), %VMM(2), %k0 - VPTESTN %VMM(4), %VMM(4), %k2 - subq $-(VEC_SIZE * 4), %rdi - /* Break if at end of length. */ - subq $(CHAR_PER_VEC * 4), %rax - jbe L(loop_len_end) - - - KORTEST %k0, %k2 - jz L(loop_4x_vec) - - -L(loop_last_4x_vec): - movq %rsi, %rcx - subq %rax, %rsi - VPTESTN %VMM(1), %VMM(1), %k1 - KMOV %k1, %VRDX - test %VRDX, %VRDX - jnz L(last_vec_x0) - - KMOV %k0, %VRDX - test %VRDX, %VRDX - jnz L(last_vec_x1) - - VPTESTN %VMM(3), %VMM(3), %k0 - - /* Separate logic for VEC_SIZE == 64 and VEC_SIZE == 32 for - returning last 2x VEC. For VEC_SIZE == 64 we test each VEC - individually, for VEC_SIZE == 32 we combine them in a single - 64-bit GPR. */ -# if CHAR_PER_VEC == 64 - KMOV %k0, %VRDX - test %VRDX, %VRDX - jnz L(last_vec_x2) - KMOV %k2, %VRDX -# else - /* We can only combine last 2x VEC masks if CHAR_PER_VEC <= 32. - */ - kmovd %k2, %edx - kmovd %k0, %eax - salq $CHAR_PER_VEC, %rdx - orq %rax, %rdx -# endif - - /* first_vec_x3 for strlen-ZMM and first_vec_x2 for strlen-YMM. - */ - bsfq %rdx, %rdx - leaq (FALLTHROUGH_RETURN_OFFSET - CHAR_PER_VEC * 4)(%rsi, %rdx), %rax - cmpq %rax, %rcx - cmovb %rcx, %rax - ret - - /* Handle last 4x VEC after loop. All VECs have been loaded. */ - .p2align 4,, 4 -L(loop_len_end): - KORTEST %k0, %k2 - jnz L(loop_last_4x_vec) - movq %rsi, %rax - ret - - -# if CHAR_PER_VEC == 64 - /* Since we can't combine the last 2x VEC for VEC_SIZE == 64 - need return label for it. */ - .p2align 4,, 8 -L(last_vec_x2): - bsf %VRDX, %VRDX - leaq (CHAR_PER_VEC * -2)(%rsi, %rdx), %rax - cmpq %rax, %rcx - cmovb %rcx, %rax - ret -# endif - - - .p2align 4,, 10 -L(last_vec_x1): - addq $CHAR_PER_VEC, %rsi -L(last_vec_x0): - bsf %VRDX, %VRDX - leaq (CHAR_PER_VEC * -4)(%rsi, %rdx), %rax - cmpq %rax, %rcx - cmovb %rcx, %rax - ret - - - .p2align 4,, 8 -L(cross_page_boundary): - /* Align data to VEC_SIZE. */ - movq %rdi, %rcx - andq $-VEC_SIZE, %rcx - VPCMPEQ (%rcx), %VZERO, %k0 - - KMOV %k0, %VRCX -# ifdef USE_AS_WCSLEN - shrl $2, %eax - andl $(CHAR_PER_VEC - 1), %eax -# endif - shrx %VRAX, %VRCX, %VRCX - - negl %eax - andl $(CHAR_PER_VEC - 1), %eax - movq %rsi, %rdx - bsf %VRCX, %VRDX - cmpq %rax, %rdx - ja L(cross_page_continue) - movl %edx, %eax - cmpq %rdx, %rsi - cmovb %esi, %eax - ret -END (STRNLEN) +#ifndef STRNLEN +#define STRNLEN __strnlen_evex #endif + +#include "x86-evex256-vecs.h" +#include "reg-macros.h" +#include "strnlen-evex-base.S" diff --git a/sysdeps/x86_64/multiarch/strnlen-evex512.S b/sysdeps/x86_64/multiarch/strnlen-evex512.S index f8e55883bb..8ef54078f8 100644 --- a/sysdeps/x86_64/multiarch/strnlen-evex512.S +++ b/sysdeps/x86_64/multiarch/strnlen-evex512.S @@ -1,264 +1,7 @@ -/* Placeholder function, not used by any processor at the moment. - Copyright (C) 2022-2024 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - #ifndef STRNLEN #define STRNLEN __strnlen_evex512 #endif #include "x86-evex512-vecs.h" #include "reg-macros.h" - -#include <isa-level.h> - -#if ISA_SHOULD_BUILD (4) - -# include <sysdep.h> - -# ifdef USE_AS_WCSLEN -# define VPCMPEQ vpcmpeqd -# define VPTESTN vptestnmd -# define VPMINU vpminud -# define CHAR_SIZE 4 -# else -# define VPCMPEQ vpcmpeqb -# define VPTESTN vptestnmb -# define VPMINU vpminub -# define CHAR_SIZE 1 -# endif - -# define PAGE_SIZE 4096 -# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) - - .section SECTION(.text),"ax",@progbits -/* Aligning entry point to 64 byte, provides better performance for - one vector length string. */ -ENTRY_P2ALIGN (STRNLEN, 6) - /* Check zero length. */ - test %RSI_LP, %RSI_LP - jz L(ret_max) -# ifdef __ILP32__ - /* Clear the upper 32 bits. */ - movl %esi, %esi -# endif - - movl %edi, %eax - vpxorq %VMM_128(0), %VMM_128(0), %VMM_128(0) - sall $20, %eax - cmpl $((PAGE_SIZE - VEC_SIZE) << 20), %eax - ja L(page_cross) - - /* Compare [w]char for null, mask bit will be set for match. */ - VPCMPEQ (%rdi), %VMM(0), %k0 - KMOV %k0, %VRCX - /* Store max length in rax. */ - mov %rsi, %rax - /* If rcx is 0, rax will have max length. We can not use VRCX - and VRAX here for evex256 because, upper 32 bits may be - undefined for ecx and eax. */ - bsfq %rcx, %rax - cmp $CHAR_PER_VEC, %rax - ja L(align_more) - cmpq %rax, %rsi - cmovb %esi, %eax - ret - - /* At this point vector max length reached. */ - .p2align 4,,3 -L(ret_max): - movq %rsi, %rax - ret - -L(align_more): - mov %rdi, %rax - /* Align rax to VEC_SIZE. */ - andq $-VEC_SIZE, %rax - movq %rdi, %rdx - subq %rax, %rdx -# ifdef USE_AS_WCSLEN - shr $2, %VRDX -# endif - /* At this point rdx contains [w]chars already compared. */ - leaq -CHAR_PER_VEC(%rsi, %rdx), %rdx - /* At this point rdx contains number of w[char] needs to go. - Now onwards rdx will keep decrementing with each compare. */ - - /* Loop unroll 4 times for 4 vector loop. */ - VPCMPEQ VEC_SIZE(%rax), %VMM(0), %k0 - subq $-VEC_SIZE, %rax - KMOV %k0, %VRCX - test %VRCX, %VRCX - jnz L(ret_vec_x1) - - subq $CHAR_PER_VEC, %rdx - jbe L(ret_max) - - VPCMPEQ VEC_SIZE(%rax), %VMM(0), %k0 - KMOV %k0, %VRCX - test %VRCX, %VRCX - jnz L(ret_vec_x2) - - subq $CHAR_PER_VEC, %rdx - jbe L(ret_max) - - VPCMPEQ (VEC_SIZE * 2)(%rax), %VMM(0), %k0 - KMOV %k0, %VRCX - test %VRCX, %VRCX - jnz L(ret_vec_x3) - - subq $CHAR_PER_VEC, %rdx - jbe L(ret_max) - - VPCMPEQ (VEC_SIZE * 3)(%rax), %VMM(0), %k0 - KMOV %k0, %VRCX - test %VRCX, %VRCX - jnz L(ret_vec_x4) - - subq $CHAR_PER_VEC, %rdx - jbe L(ret_max) - /* Save pointer before 4 x VEC_SIZE alignment. */ - movq %rax, %rcx - - /* Align address to VEC_SIZE * 4 for loop. */ - andq $-(VEC_SIZE * 4), %rax - - subq %rax, %rcx -# ifdef USE_AS_WCSLEN - shr $2, %VRCX -# endif - /* rcx contains number of [w]char will be recompared due to - alignment fixes. rdx must be incremented by rcx to offset - alignment adjustment. */ - addq %rcx, %rdx - /* Need jump as we don't want to add/subtract rdx for first - iteration of 4 x VEC_SIZE aligned loop. */ - - .p2align 4,,11 -L(loop): - /* VPMINU and VPCMP combination provide better performance as - compared to alternative combinations. */ - VMOVA (VEC_SIZE * 4)(%rax), %VMM(1) - VPMINU (VEC_SIZE * 5)(%rax), %VMM(1), %VMM(2) - VMOVA (VEC_SIZE * 6)(%rax), %VMM(3) - VPMINU (VEC_SIZE * 7)(%rax), %VMM(3), %VMM(4) - - VPTESTN %VMM(2), %VMM(2), %k0 - VPTESTN %VMM(4), %VMM(4), %k1 - - subq $-(VEC_SIZE * 4), %rax - KORTEST %k0, %k1 - - jnz L(loopend) - subq $(CHAR_PER_VEC * 4), %rdx - ja L(loop) - mov %rsi, %rax - ret - -L(loopend): - - VPTESTN %VMM(1), %VMM(1), %k2 - KMOV %k2, %VRCX - test %VRCX, %VRCX - jnz L(ret_vec_x1) - - KMOV %k0, %VRCX - /* At this point, if k0 is non zero, null char must be in the - second vector. */ - test %VRCX, %VRCX - jnz L(ret_vec_x2) - - VPTESTN %VMM(3), %VMM(3), %k3 - KMOV %k3, %VRCX - test %VRCX, %VRCX - jnz L(ret_vec_x3) - /* At this point null [w]char must be in the fourth vector so no - need to check. */ - KMOV %k1, %VRCX - - /* Fourth, third, second vector terminating are pretty much - same, implemented this way to avoid branching and reuse code - from pre loop exit condition. */ -L(ret_vec_x4): - bsf %VRCX, %VRCX - subq %rdi, %rax -# ifdef USE_AS_WCSLEN - subq $-(VEC_SIZE * 3), %rax - shrq $2, %rax - addq %rcx, %rax -# else - leaq (VEC_SIZE * 3)(%rcx, %rax), %rax -# endif - - cmpq %rsi, %rax - cmovnb %rsi, %rax - ret - -L(ret_vec_x3): - bsf %VRCX, %VRCX - subq %rdi, %rax -# ifdef USE_AS_WCSLEN - subq $-(VEC_SIZE * 2), %rax - shrq $2, %rax - addq %rcx, %rax -# else - leaq (VEC_SIZE * 2)(%rcx, %rax), %rax -# endif - cmpq %rsi, %rax - cmovnb %rsi, %rax - ret - -L(ret_vec_x2): - subq $-VEC_SIZE, %rax -L(ret_vec_x1): - bsf %VRCX, %VRCX - subq %rdi, %rax -# ifdef USE_AS_WCSLEN - shrq $2, %rax -# endif - addq %rcx, %rax - cmpq %rsi, %rax - cmovnb %rsi, %rax - ret - -L(page_cross): - mov %rdi, %rax - movl %edi, %ecx - andl $(VEC_SIZE - 1), %ecx -# ifdef USE_AS_WCSLEN - sarl $2, %ecx -# endif - /* ecx contains number of w[char] to be skipped as a result - of address alignment. */ - andq $-VEC_SIZE, %rax - VPCMPEQ (%rax), %VMM(0), %k0 - KMOV %k0, %VRDX - /* Ignore number of character for alignment adjustment. */ - shr %cl, %VRDX - jnz L(page_cross_end) - movl $CHAR_PER_VEC, %eax - sub %ecx, %eax - cmp %rax, %rsi - ja L(align_more) - -L(page_cross_end): - bsf %VRDX, %VRAX - cmpq %rsi, %rax - cmovnb %esi, %eax - ret - -END (STRNLEN) -#endif +#include "strnlen-evex-base.S" \ No newline at end of file diff --git a/sysdeps/x86_64/nptl/tcb-offsets.sym b/sysdeps/x86_64/nptl/tcb-offsets.sym index 2bbd563a6c..988a4b8593 100644 --- a/sysdeps/x86_64/nptl/tcb-offsets.sym +++ b/sysdeps/x86_64/nptl/tcb-offsets.sym @@ -13,6 +13,3 @@ MULTIPLE_THREADS_OFFSET offsetof (tcbhead_t, multiple_threads) POINTER_GUARD offsetof (tcbhead_t, pointer_guard) FEATURE_1_OFFSET offsetof (tcbhead_t, feature_1) SSP_BASE_OFFSET offsetof (tcbhead_t, ssp_base) - --- Not strictly offsets, but these values are also used in the TCB. -TCB_CANCELED_BITMASK CANCELED_BITMASK diff --git a/sysdeps/x86_64/x32/dl-machine.h b/sysdeps/x86_64/x32/dl-machine.h deleted file mode 100644 index c35cee9261..0000000000 --- a/sysdeps/x86_64/x32/dl-machine.h +++ /dev/null @@ -1,76 +0,0 @@ -/* Machine-dependent ELF dynamic relocation inline functions. x32 version. - Copyright (C) 2012-2024 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -/* Must allow <sysdeps/x86_64/dl-machine.h> to be included more than once. - See #ifdef RESOLVE_MAP in sysdeps/x86_64/dl-machine.h. */ -#include <sysdeps/x86_64/dl-machine.h> - -#ifndef _X32_DL_MACHINE_H -#define _X32_DL_MACHINE_H - -#undef ARCH_LA_PLTENTER -#undef ARCH_LA_PLTEXIT -#undef RTLD_START - -/* Names of the architecture-specific auditing callback functions. */ -#define ARCH_LA_PLTENTER x32_gnu_pltenter -#define ARCH_LA_PLTEXIT x32_gnu_pltexit - -/* Initial entry point code for the dynamic linker. - The C function `_dl_start' is the real entry point; - its return value is the user program's entry point. */ -#define RTLD_START asm ("\n\ -.text\n\ - .p2align 4\n\ -.globl _start\n\ -.globl _dl_start_user\n\ -_start:\n\ - movl %esp, %edi\n\ - call _dl_start\n\ -_dl_start_user:\n\ - # Save the user entry point address in %r12.\n\ - movl %eax, %r12d\n\ - # Read the original argument count.\n\ - movl (%rsp), %edx\n\ - # Call _dl_init (struct link_map *main_map, int argc, char **argv, char **env)\n\ - # argc -> rsi\n\ - movl %edx, %esi\n\ - # Save %rsp value in %r13.\n\ - movl %esp, %r13d\n\ - # And align stack for the _dl_init call.\n\ - and $-16, %esp\n\ - # _dl_loaded -> rdi\n\ - movl _rtld_local(%rip), %edi\n\ - # env -> rcx\n\ - lea 8(%r13,%rdx,4), %ecx\n\ - # argv -> rdx\n\ - lea 4(%r13), %edx\n\ - # Clear %rbp to mark outermost frame obviously even for constructors.\n\ - xorl %ebp, %ebp\n\ - # Call the function to run the initializers.\n\ - call _dl_init\n\ - # Pass our finalizer function to the user in %rdx, as per ELF ABI.\n\ - lea _dl_fini(%rip), %edx\n\ - # And make sure %rsp points to argc stored on the stack.\n\ - movl %r13d, %esp\n\ - # Jump to the user's entry point.\n\ - jmp *%r12\n\ -.previous\n\ -"); - -#endif /* !_X32_DL_MACHINE_H */ |