[PATCH] Clean up read write lock assembly
- Move the slow path fallbacks to their own assembly files
This makes them much easier to read and is needed for the next change.
- Add CFI annotations for unwinding (XXX need review)
- Remove constant case which can never happen with out of line spinlocks
- Use patchable LOCK prefixes
- Don't use lock sections anymore for inline code because they can't
be expressed by the unwinder (this adds one taken jump to the lock
fast path)
Cc: jbeulich@novell.com
Signed-off-by: Andi Kleen <ak@suse.de>
diff --git a/include/asm-x86_64/rwlock.h b/include/asm-x86_64/rwlock.h
index dea0e94..28a080d 100644
--- a/include/asm-x86_64/rwlock.h
+++ b/include/asm-x86_64/rwlock.h
@@ -18,69 +18,21 @@
#ifndef _ASM_X86_64_RWLOCK_H
#define _ASM_X86_64_RWLOCK_H
-#include <linux/stringify.h>
-
#define RW_LOCK_BIAS 0x01000000
-#define RW_LOCK_BIAS_STR "0x01000000"
+#define RW_LOCK_BIAS_STR "0x01000000"
-#define __build_read_lock_ptr(rw, helper) \
+#define __build_read_lock(rw) \
asm volatile(LOCK_PREFIX "subl $1,(%0)\n\t" \
- "js 2f\n" \
+ "jns 1f\n" \
+ "call __read_lock_failed\n" \
"1:\n" \
- LOCK_SECTION_START("") \
- "2:\tcall " helper "\n\t" \
- "jmp 1b\n" \
- LOCK_SECTION_END \
- ::"a" (rw) : "memory")
+ ::"D" (rw), "i" (RW_LOCK_BIAS) : "memory")
-#define __build_read_lock_const(rw, helper) \
- asm volatile(LOCK_PREFIX "subl $1,%0\n\t" \
- "js 2f\n" \
+#define __build_write_lock(rw) \
+ asm volatile(LOCK_PREFIX "subl %1,(%0)\n\t" \
+ "jz 1f\n" \
+ "\tcall __write_lock_failed\n\t" \
"1:\n" \
- LOCK_SECTION_START("") \
- "2:\tpushq %%rax\n\t" \
- "leaq %0,%%rax\n\t" \
- "call " helper "\n\t" \
- "popq %%rax\n\t" \
- "jmp 1b\n" \
- LOCK_SECTION_END \
- :"=m" (*((volatile int *)rw))::"memory")
-
-#define __build_read_lock(rw, helper) do { \
- if (__builtin_constant_p(rw)) \
- __build_read_lock_const(rw, helper); \
- else \
- __build_read_lock_ptr(rw, helper); \
- } while (0)
-
-#define __build_write_lock_ptr(rw, helper) \
- asm volatile(LOCK_PREFIX "subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" \
- "jnz 2f\n" \
- "1:\n" \
- LOCK_SECTION_START("") \
- "2:\tcall " helper "\n\t" \
- "jmp 1b\n" \
- LOCK_SECTION_END \
- ::"a" (rw) : "memory")
-
-#define __build_write_lock_const(rw, helper) \
- asm volatile(LOCK_PREFIX "subl $" RW_LOCK_BIAS_STR ",%0\n\t" \
- "jnz 2f\n" \
- "1:\n" \
- LOCK_SECTION_START("") \
- "2:\tpushq %%rax\n\t" \
- "leaq %0,%%rax\n\t" \
- "call " helper "\n\t" \
- "popq %%rax\n\t" \
- "jmp 1b\n" \
- LOCK_SECTION_END \
- :"=m" (*((volatile long *)rw))::"memory")
-
-#define __build_write_lock(rw, helper) do { \
- if (__builtin_constant_p(rw)) \
- __build_write_lock_const(rw, helper); \
- else \
- __build_write_lock_ptr(rw, helper); \
- } while (0)
+ ::"D" (rw), "i" (RW_LOCK_BIAS) : "memory")
#endif