aboutsummaryrefslogtreecommitdiffstats
path: root/main/linux-vanilla/0001-arm64-Avoid-redundant-type-conversions-in-xchg-and-c.patch
diff options
context:
space:
mode:
Diffstat (limited to 'main/linux-vanilla/0001-arm64-Avoid-redundant-type-conversions-in-xchg-and-c.patch')
-rw-r--r--main/linux-vanilla/0001-arm64-Avoid-redundant-type-conversions-in-xchg-and-c.patch355
1 files changed, 355 insertions, 0 deletions
diff --git a/main/linux-vanilla/0001-arm64-Avoid-redundant-type-conversions-in-xchg-and-c.patch b/main/linux-vanilla/0001-arm64-Avoid-redundant-type-conversions-in-xchg-and-c.patch
new file mode 100644
index 00000000000..2441864dfb7
--- /dev/null
+++ b/main/linux-vanilla/0001-arm64-Avoid-redundant-type-conversions-in-xchg-and-c.patch
@@ -0,0 +1,355 @@
+From 7b7b95eca1c3c2d6e5302b813b2b8470d004dedb Mon Sep 17 00:00:00 2001
+From: Will Deacon <will.deacon@arm.com>
+Date: Thu, 13 Sep 2018 13:30:45 +0100
+Subject: [PATCH 1/2] arm64: Avoid redundant type conversions in xchg() and
+ cmpxchg()
+
+Our atomic instructions (either LSE atomics of LDXR/STXR sequences)
+natively support byte, half-word, word and double-word memory accesses
+so there is no need to mask the data register prior to being stored.
+
+Signed-off-by: Will Deacon <will.deacon@arm.com>
+(cherry picked from commit 5ef3fe4cecdf82fdd71ce78988403963d01444d4)
+---
+ arch/arm64/include/asm/atomic_ll_sc.h | 53 ++++++------
+ arch/arm64/include/asm/atomic_lse.h | 46 +++++-----
+ arch/arm64/include/asm/cmpxchg.h | 116 +++++++++++++-------------
+ 3 files changed, 108 insertions(+), 107 deletions(-)
+
+diff --git a/arch/arm64/include/asm/atomic_ll_sc.h b/arch/arm64/include/asm/atomic_ll_sc.h
+index f5a2d09afb38..f02d3bf7b9e6 100644
+--- a/arch/arm64/include/asm/atomic_ll_sc.h
++++ b/arch/arm64/include/asm/atomic_ll_sc.h
+@@ -248,48 +248,49 @@ __LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v))
+ }
+ __LL_SC_EXPORT(atomic64_dec_if_positive);
+
+-#define __CMPXCHG_CASE(w, sz, name, mb, acq, rel, cl) \
+-__LL_SC_INLINE unsigned long \
+-__LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \
+- unsigned long old, \
+- unsigned long new)) \
++#define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl) \
++__LL_SC_INLINE u##sz \
++__LL_SC_PREFIX(__cmpxchg_case_##name##sz(volatile void *ptr, \
++ unsigned long old, \
++ u##sz new)) \
+ { \
+- unsigned long tmp, oldval; \
++ unsigned long tmp; \
++ u##sz oldval; \
+ \
+ asm volatile( \
+ " prfm pstl1strm, %[v]\n" \
+- "1: ld" #acq "xr" #sz "\t%" #w "[oldval], %[v]\n" \
++ "1: ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n" \
+ " eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \
+ " cbnz %" #w "[tmp], 2f\n" \
+- " st" #rel "xr" #sz "\t%w[tmp], %" #w "[new], %[v]\n" \
++ " st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n" \
+ " cbnz %w[tmp], 1b\n" \
+ " " #mb "\n" \
+ "2:" \
+ : [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \
+- [v] "+Q" (*(unsigned long *)ptr) \
++ [v] "+Q" (*(u##sz *)ptr) \
+ : [old] "Lr" (old), [new] "r" (new) \
+ : cl); \
+ \
+ return oldval; \
+ } \
+-__LL_SC_EXPORT(__cmpxchg_case_##name);
++__LL_SC_EXPORT(__cmpxchg_case_##name##sz);
+
+-__CMPXCHG_CASE(w, b, 1, , , , )
+-__CMPXCHG_CASE(w, h, 2, , , , )
+-__CMPXCHG_CASE(w, , 4, , , , )
+-__CMPXCHG_CASE( , , 8, , , , )
+-__CMPXCHG_CASE(w, b, acq_1, , a, , "memory")
+-__CMPXCHG_CASE(w, h, acq_2, , a, , "memory")
+-__CMPXCHG_CASE(w, , acq_4, , a, , "memory")
+-__CMPXCHG_CASE( , , acq_8, , a, , "memory")
+-__CMPXCHG_CASE(w, b, rel_1, , , l, "memory")
+-__CMPXCHG_CASE(w, h, rel_2, , , l, "memory")
+-__CMPXCHG_CASE(w, , rel_4, , , l, "memory")
+-__CMPXCHG_CASE( , , rel_8, , , l, "memory")
+-__CMPXCHG_CASE(w, b, mb_1, dmb ish, , l, "memory")
+-__CMPXCHG_CASE(w, h, mb_2, dmb ish, , l, "memory")
+-__CMPXCHG_CASE(w, , mb_4, dmb ish, , l, "memory")
+-__CMPXCHG_CASE( , , mb_8, dmb ish, , l, "memory")
++__CMPXCHG_CASE(w, b, , 8, , , , )
++__CMPXCHG_CASE(w, h, , 16, , , , )
++__CMPXCHG_CASE(w, , , 32, , , , )
++__CMPXCHG_CASE( , , , 64, , , , )
++__CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory")
++__CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory")
++__CMPXCHG_CASE(w, , acq_, 32, , a, , "memory")
++__CMPXCHG_CASE( , , acq_, 64, , a, , "memory")
++__CMPXCHG_CASE(w, b, rel_, 8, , , l, "memory")
++__CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory")
++__CMPXCHG_CASE(w, , rel_, 32, , , l, "memory")
++__CMPXCHG_CASE( , , rel_, 64, , , l, "memory")
++__CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory")
++__CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory")
++__CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory")
++__CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory")
+
+ #undef __CMPXCHG_CASE
+
+diff --git a/arch/arm64/include/asm/atomic_lse.h b/arch/arm64/include/asm/atomic_lse.h
+index eab3de4f2ad2..80cadc789f1a 100644
+--- a/arch/arm64/include/asm/atomic_lse.h
++++ b/arch/arm64/include/asm/atomic_lse.h
+@@ -480,24 +480,24 @@ static inline long atomic64_dec_if_positive(atomic64_t *v)
+
+ #define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op)
+
+-#define __CMPXCHG_CASE(w, sz, name, mb, cl...) \
+-static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \
+- unsigned long old, \
+- unsigned long new) \
++#define __CMPXCHG_CASE(w, sfx, name, sz, mb, cl...) \
++static inline u##sz __cmpxchg_case_##name##sz(volatile void *ptr, \
++ unsigned long old, \
++ u##sz new) \
+ { \
+ register unsigned long x0 asm ("x0") = (unsigned long)ptr; \
+ register unsigned long x1 asm ("x1") = old; \
+- register unsigned long x2 asm ("x2") = new; \
++ register u##sz x2 asm ("x2") = new; \
+ \
+ asm volatile( \
+ __LSE_PREAMBLE \
+ ARM64_LSE_ATOMIC_INSN( \
+ /* LL/SC */ \
+- __LL_SC_CMPXCHG(name) \
++ __LL_SC_CMPXCHG(name##sz) \
+ __nops(2), \
+ /* LSE atomics */ \
+ " mov " #w "30, %" #w "[old]\n" \
+- " cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n" \
++ " cas" #mb #sfx "\t" #w "30, %" #w "[new], %[v]\n" \
+ " mov %" #w "[ret], " #w "30") \
+ : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \
+ : [old] "r" (x1), [new] "r" (x2) \
+@@ -506,22 +506,22 @@ static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \
+ return x0; \
+ }
+
+-__CMPXCHG_CASE(w, b, 1, )
+-__CMPXCHG_CASE(w, h, 2, )
+-__CMPXCHG_CASE(w, , 4, )
+-__CMPXCHG_CASE(x, , 8, )
+-__CMPXCHG_CASE(w, b, acq_1, a, "memory")
+-__CMPXCHG_CASE(w, h, acq_2, a, "memory")
+-__CMPXCHG_CASE(w, , acq_4, a, "memory")
+-__CMPXCHG_CASE(x, , acq_8, a, "memory")
+-__CMPXCHG_CASE(w, b, rel_1, l, "memory")
+-__CMPXCHG_CASE(w, h, rel_2, l, "memory")
+-__CMPXCHG_CASE(w, , rel_4, l, "memory")
+-__CMPXCHG_CASE(x, , rel_8, l, "memory")
+-__CMPXCHG_CASE(w, b, mb_1, al, "memory")
+-__CMPXCHG_CASE(w, h, mb_2, al, "memory")
+-__CMPXCHG_CASE(w, , mb_4, al, "memory")
+-__CMPXCHG_CASE(x, , mb_8, al, "memory")
++__CMPXCHG_CASE(w, b, , 8, )
++__CMPXCHG_CASE(w, h, , 16, )
++__CMPXCHG_CASE(w, , , 32, )
++__CMPXCHG_CASE(x, , , 64, )
++__CMPXCHG_CASE(w, b, acq_, 8, a, "memory")
++__CMPXCHG_CASE(w, h, acq_, 16, a, "memory")
++__CMPXCHG_CASE(w, , acq_, 32, a, "memory")
++__CMPXCHG_CASE(x, , acq_, 64, a, "memory")
++__CMPXCHG_CASE(w, b, rel_, 8, l, "memory")
++__CMPXCHG_CASE(w, h, rel_, 16, l, "memory")
++__CMPXCHG_CASE(w, , rel_, 32, l, "memory")
++__CMPXCHG_CASE(x, , rel_, 64, l, "memory")
++__CMPXCHG_CASE(w, b, mb_, 8, al, "memory")
++__CMPXCHG_CASE(w, h, mb_, 16, al, "memory")
++__CMPXCHG_CASE(w, , mb_, 32, al, "memory")
++__CMPXCHG_CASE(x, , mb_, 64, al, "memory")
+
+ #undef __LL_SC_CMPXCHG
+ #undef __CMPXCHG_CASE
+diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
+index d8b01c7c9cd3..94ccb3bfbd61 100644
+--- a/arch/arm64/include/asm/cmpxchg.h
++++ b/arch/arm64/include/asm/cmpxchg.h
+@@ -30,46 +30,46 @@
+ * barrier case is generated as release+dmb for the former and
+ * acquire+release for the latter.
+ */
+-#define __XCHG_CASE(w, sz, name, mb, nop_lse, acq, acq_lse, rel, cl) \
+-static inline unsigned long __xchg_case_##name(unsigned long x, \
+- volatile void *ptr) \
+-{ \
+- unsigned long ret, tmp; \
+- \
+- asm volatile(ARM64_LSE_ATOMIC_INSN( \
+- /* LL/SC */ \
+- " prfm pstl1strm, %2\n" \
+- "1: ld" #acq "xr" #sz "\t%" #w "0, %2\n" \
+- " st" #rel "xr" #sz "\t%w1, %" #w "3, %2\n" \
+- " cbnz %w1, 1b\n" \
+- " " #mb, \
+- /* LSE atomics */ \
+- " swp" #acq_lse #rel #sz "\t%" #w "3, %" #w "0, %2\n" \
+- __nops(3) \
+- " " #nop_lse) \
+- : "=&r" (ret), "=&r" (tmp), "+Q" (*(unsigned long *)ptr) \
+- : "r" (x) \
+- : cl); \
+- \
+- return ret; \
++#define __XCHG_CASE(w, sfx, name, sz, mb, nop_lse, acq, acq_lse, rel, cl) \
++static inline u##sz __xchg_case_##name##sz(u##sz x, volatile void *ptr) \
++{ \
++ u##sz ret; \
++ unsigned long tmp; \
++ \
++ asm volatile(ARM64_LSE_ATOMIC_INSN( \
++ /* LL/SC */ \
++ " prfm pstl1strm, %2\n" \
++ "1: ld" #acq "xr" #sfx "\t%" #w "0, %2\n" \
++ " st" #rel "xr" #sfx "\t%w1, %" #w "3, %2\n" \
++ " cbnz %w1, 1b\n" \
++ " " #mb, \
++ /* LSE atomics */ \
++ " swp" #acq_lse #rel #sfx "\t%" #w "3, %" #w "0, %2\n" \
++ __nops(3) \
++ " " #nop_lse) \
++ : "=&r" (ret), "=&r" (tmp), "+Q" (*(u##sz *)ptr) \
++ : "r" (x) \
++ : cl); \
++ \
++ return ret; \
+ }
+
+-__XCHG_CASE(w, b, 1, , , , , , )
+-__XCHG_CASE(w, h, 2, , , , , , )
+-__XCHG_CASE(w, , 4, , , , , , )
+-__XCHG_CASE( , , 8, , , , , , )
+-__XCHG_CASE(w, b, acq_1, , , a, a, , "memory")
+-__XCHG_CASE(w, h, acq_2, , , a, a, , "memory")
+-__XCHG_CASE(w, , acq_4, , , a, a, , "memory")
+-__XCHG_CASE( , , acq_8, , , a, a, , "memory")
+-__XCHG_CASE(w, b, rel_1, , , , , l, "memory")
+-__XCHG_CASE(w, h, rel_2, , , , , l, "memory")
+-__XCHG_CASE(w, , rel_4, , , , , l, "memory")
+-__XCHG_CASE( , , rel_8, , , , , l, "memory")
+-__XCHG_CASE(w, b, mb_1, dmb ish, nop, , a, l, "memory")
+-__XCHG_CASE(w, h, mb_2, dmb ish, nop, , a, l, "memory")
+-__XCHG_CASE(w, , mb_4, dmb ish, nop, , a, l, "memory")
+-__XCHG_CASE( , , mb_8, dmb ish, nop, , a, l, "memory")
++__XCHG_CASE(w, b, , 8, , , , , , )
++__XCHG_CASE(w, h, , 16, , , , , , )
++__XCHG_CASE(w, , , 32, , , , , , )
++__XCHG_CASE( , , , 64, , , , , , )
++__XCHG_CASE(w, b, acq_, 8, , , a, a, , "memory")
++__XCHG_CASE(w, h, acq_, 16, , , a, a, , "memory")
++__XCHG_CASE(w, , acq_, 32, , , a, a, , "memory")
++__XCHG_CASE( , , acq_, 64, , , a, a, , "memory")
++__XCHG_CASE(w, b, rel_, 8, , , , , l, "memory")
++__XCHG_CASE(w, h, rel_, 16, , , , , l, "memory")
++__XCHG_CASE(w, , rel_, 32, , , , , l, "memory")
++__XCHG_CASE( , , rel_, 64, , , , , l, "memory")
++__XCHG_CASE(w, b, mb_, 8, dmb ish, nop, , a, l, "memory")
++__XCHG_CASE(w, h, mb_, 16, dmb ish, nop, , a, l, "memory")
++__XCHG_CASE(w, , mb_, 32, dmb ish, nop, , a, l, "memory")
++__XCHG_CASE( , , mb_, 64, dmb ish, nop, , a, l, "memory")
+
+ #undef __XCHG_CASE
+
+@@ -80,13 +80,13 @@ static __always_inline unsigned long __xchg##sfx(unsigned long x, \
+ { \
+ switch (size) { \
+ case 1: \
+- return __xchg_case##sfx##_1(x, ptr); \
++ return __xchg_case##sfx##_8(x, ptr); \
+ case 2: \
+- return __xchg_case##sfx##_2(x, ptr); \
++ return __xchg_case##sfx##_16(x, ptr); \
+ case 4: \
+- return __xchg_case##sfx##_4(x, ptr); \
++ return __xchg_case##sfx##_32(x, ptr); \
+ case 8: \
+- return __xchg_case##sfx##_8(x, ptr); \
++ return __xchg_case##sfx##_64(x, ptr); \
+ default: \
+ BUILD_BUG(); \
+ } \
+@@ -123,13 +123,13 @@ static __always_inline unsigned long __cmpxchg##sfx(volatile void *ptr, \
+ { \
+ switch (size) { \
+ case 1: \
+- return __cmpxchg_case##sfx##_1(ptr, (u8)old, new); \
++ return __cmpxchg_case##sfx##_8(ptr, (u8)old, new); \
+ case 2: \
+- return __cmpxchg_case##sfx##_2(ptr, (u16)old, new); \
++ return __cmpxchg_case##sfx##_16(ptr, (u16)old, new); \
+ case 4: \
+- return __cmpxchg_case##sfx##_4(ptr, old, new); \
++ return __cmpxchg_case##sfx##_32(ptr, old, new); \
+ case 8: \
+- return __cmpxchg_case##sfx##_8(ptr, old, new); \
++ return __cmpxchg_case##sfx##_64(ptr, old, new); \
+ default: \
+ BUILD_BUG(); \
+ } \
+@@ -197,16 +197,16 @@ __CMPXCHG_GEN(_mb)
+ __ret; \
+ })
+
+-#define __CMPWAIT_CASE(w, sz, name) \
+-static inline void __cmpwait_case_##name(volatile void *ptr, \
+- unsigned long val) \
++#define __CMPWAIT_CASE(w, sfx, sz) \
++static inline void __cmpwait_case_##sz(volatile void *ptr, \
++ unsigned long val) \
+ { \
+ unsigned long tmp; \
+ \
+ asm volatile( \
+ " sevl\n" \
+ " wfe\n" \
+- " ldxr" #sz "\t%" #w "[tmp], %[v]\n" \
++ " ldxr" #sfx "\t%" #w "[tmp], %[v]\n" \
+ " eor %" #w "[tmp], %" #w "[tmp], %" #w "[val]\n" \
+ " cbnz %" #w "[tmp], 1f\n" \
+ " wfe\n" \
+@@ -215,10 +215,10 @@ static inline void __cmpwait_case_##name(volatile void *ptr, \
+ : [val] "r" (val)); \
+ }
+
+-__CMPWAIT_CASE(w, b, 1);
+-__CMPWAIT_CASE(w, h, 2);
+-__CMPWAIT_CASE(w, , 4);
+-__CMPWAIT_CASE( , , 8);
++__CMPWAIT_CASE(w, b, 8);
++__CMPWAIT_CASE(w, h, 16);
++__CMPWAIT_CASE(w, , 32);
++__CMPWAIT_CASE( , , 64);
+
+ #undef __CMPWAIT_CASE
+
+@@ -229,13 +229,13 @@ static __always_inline void __cmpwait##sfx(volatile void *ptr, \
+ { \
+ switch (size) { \
+ case 1: \
+- return __cmpwait_case##sfx##_1(ptr, (u8)val); \
++ return __cmpwait_case##sfx##_8(ptr, (u8)val); \
+ case 2: \
+- return __cmpwait_case##sfx##_2(ptr, (u16)val); \
++ return __cmpwait_case##sfx##_16(ptr, (u16)val); \
+ case 4: \
+- return __cmpwait_case##sfx##_4(ptr, val); \
++ return __cmpwait_case##sfx##_32(ptr, val); \
+ case 8: \
+- return __cmpwait_case##sfx##_8(ptr, val); \
++ return __cmpwait_case##sfx##_64(ptr, val); \
+ default: \
+ BUILD_BUG(); \
+ } \
+--
+2.30.1
+
generated by cgit v1.2.3 (git 2.41.0) at 2024-04-25 10:44:53 +0000