[PATCH] mm: pagefault_{disable,enable}()
Introduce pagefault_{disable,enable}() and use these where previously we did
manual preempt increments/decrements to make the pagefault handler do the
atomic thing.
Currently they still rely on the increased preempt count, but do not rely on
the disabled preemption, this might go away in the future.
(NOTE: the extra barrier() in pagefault_disable might fix some holes on
machines which have too many registers for their own good)
[heiko.carstens@de.ibm.com: s390 fix]
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
diff --git a/include/asm-frv/highmem.h b/include/asm-frv/highmem.h
index 0f390f4..ff4d6cd 100644
--- a/include/asm-frv/highmem.h
+++ b/include/asm-frv/highmem.h
@@ -115,7 +115,7 @@
{
unsigned long paddr;
- inc_preempt_count();
+ pagefault_disable();
paddr = page_to_phys(page);
switch (type) {
@@ -170,8 +170,7 @@
default:
BUG();
}
- dec_preempt_count();
- preempt_check_resched();
+ pagefault_enable();
}
#endif /* !__ASSEMBLY__ */
diff --git a/include/asm-generic/futex.h b/include/asm-generic/futex.h
index df893c1..f422df0 100644
--- a/include/asm-generic/futex.h
+++ b/include/asm-generic/futex.h
@@ -21,7 +21,7 @@
if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
return -EFAULT;
- inc_preempt_count();
+ pagefault_disable();
switch (op) {
case FUTEX_OP_SET:
@@ -33,7 +33,7 @@
ret = -ENOSYS;
}
- dec_preempt_count();
+ pagefault_enable();
if (!ret) {
switch (cmp) {
diff --git a/include/asm-i386/futex.h b/include/asm-i386/futex.h
index 946d97c..438ef0e 100644
--- a/include/asm-i386/futex.h
+++ b/include/asm-i386/futex.h
@@ -56,7 +56,7 @@
if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
return -EFAULT;
- inc_preempt_count();
+ pagefault_disable();
if (op == FUTEX_OP_SET)
__futex_atomic_op1("xchgl %0, %2", ret, oldval, uaddr, oparg);
@@ -88,7 +88,7 @@
}
}
- dec_preempt_count();
+ pagefault_enable();
if (!ret) {
switch (cmp) {
diff --git a/include/asm-ia64/futex.h b/include/asm-ia64/futex.h
index 07d77f3..8a98a26 100644
--- a/include/asm-ia64/futex.h
+++ b/include/asm-ia64/futex.h
@@ -59,7 +59,7 @@
if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
return -EFAULT;
- inc_preempt_count();
+ pagefault_disable();
switch (op) {
case FUTEX_OP_SET:
@@ -83,7 +83,7 @@
ret = -ENOSYS;
}
- dec_preempt_count();
+ pagefault_enable();
if (!ret) {
switch (cmp) {
diff --git a/include/asm-mips/futex.h b/include/asm-mips/futex.h
index 927a216..47e5679 100644
--- a/include/asm-mips/futex.h
+++ b/include/asm-mips/futex.h
@@ -88,7 +88,7 @@
if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
return -EFAULT;
- inc_preempt_count();
+ pagefault_disable();
switch (op) {
case FUTEX_OP_SET:
@@ -115,7 +115,7 @@
ret = -ENOSYS;
}
- dec_preempt_count();
+ pagefault_enable();
if (!ret) {
switch (cmp) {
diff --git a/include/asm-parisc/futex.h b/include/asm-parisc/futex.h
index d84bbb2..dbee6e6 100644
--- a/include/asm-parisc/futex.h
+++ b/include/asm-parisc/futex.h
@@ -21,7 +21,7 @@
if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
return -EFAULT;
- inc_preempt_count();
+ pagefault_disable();
switch (op) {
case FUTEX_OP_SET:
@@ -33,7 +33,7 @@
ret = -ENOSYS;
}
- dec_preempt_count();
+ pagefault_enable();
if (!ret) {
switch (cmp) {
diff --git a/include/asm-powerpc/futex.h b/include/asm-powerpc/futex.h
index 936422e..3f3673f 100644
--- a/include/asm-powerpc/futex.h
+++ b/include/asm-powerpc/futex.h
@@ -43,7 +43,7 @@
if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
return -EFAULT;
- inc_preempt_count();
+ pagefault_disable();
switch (op) {
case FUTEX_OP_SET:
@@ -65,7 +65,7 @@
ret = -ENOSYS;
}
- dec_preempt_count();
+ pagefault_enable();
if (!ret) {
switch (cmp) {
diff --git a/include/asm-ppc/highmem.h b/include/asm-ppc/highmem.h
index 1d2c4ef..f7b21ee 100644
--- a/include/asm-ppc/highmem.h
+++ b/include/asm-ppc/highmem.h
@@ -79,7 +79,7 @@
unsigned long vaddr;
/* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */
- inc_preempt_count();
+ pagefault_disable();
if (!PageHighMem(page))
return page_address(page);
@@ -101,8 +101,7 @@
unsigned int idx = type + KM_TYPE_NR*smp_processor_id();
if (vaddr < KMAP_FIX_BEGIN) { // FIXME
- dec_preempt_count();
- preempt_check_resched();
+ pagefault_enable();
return;
}
@@ -115,8 +114,7 @@
pte_clear(&init_mm, vaddr, kmap_pte+idx);
flush_tlb_page(NULL, vaddr);
#endif
- dec_preempt_count();
- preempt_check_resched();
+ pagefault_enable();
}
static inline struct page *kmap_atomic_to_page(void *ptr)
diff --git a/include/asm-sparc64/futex.h b/include/asm-sparc64/futex.h
index 7392fc4..876312f 100644
--- a/include/asm-sparc64/futex.h
+++ b/include/asm-sparc64/futex.h
@@ -45,7 +45,7 @@
if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))
oparg = 1 << oparg;
- inc_preempt_count();
+ pagefault_disable();
switch (op) {
case FUTEX_OP_SET:
@@ -67,7 +67,7 @@
ret = -ENOSYS;
}
- dec_preempt_count();
+ pagefault_enable();
if (!ret) {
switch (cmp) {
diff --git a/include/asm-x86_64/futex.h b/include/asm-x86_64/futex.h
index 9804bf0..5cdfb08 100644
--- a/include/asm-x86_64/futex.h
+++ b/include/asm-x86_64/futex.h
@@ -55,7 +55,7 @@
if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
return -EFAULT;
- inc_preempt_count();
+ pagefault_disable();
switch (op) {
case FUTEX_OP_SET:
@@ -78,7 +78,7 @@
ret = -ENOSYS;
}
- dec_preempt_count();
+ pagefault_enable();
if (!ret) {
switch (cmp) {
diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h
index a48d7f1..67918c2 100644
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -1,8 +1,43 @@
#ifndef __LINUX_UACCESS_H__
#define __LINUX_UACCESS_H__
+#include <linux/preempt.h>
#include <asm/uaccess.h>
+/*
+ * These routines enable/disable the pagefault handler in that
+ * it will not take any locks and go straight to the fixup table.
+ *
+ * They have great resemblance to the preempt_disable/enable calls
+ * and in fact they are identical; this is because currently there is
+ * no other way to make the pagefault handlers do this. So we do
+ * disable preemption but we don't necessarily care about that.
+ */
+static inline void pagefault_disable(void)
+{
+ inc_preempt_count();
+ /*
+ * make sure to have issued the store before a pagefault
+ * can hit.
+ */
+ barrier();
+}
+
+static inline void pagefault_enable(void)
+{
+ /*
+ * make sure to issue those last loads/stores before enabling
+ * the pagefault handler again.
+ */
+ barrier();
+ dec_preempt_count();
+ /*
+ * make sure we do..
+ */
+ barrier();
+ preempt_check_resched();
+}
+
#ifndef ARCH_HAS_NOCACHE_UACCESS
static inline unsigned long __copy_from_user_inatomic_nocache(void *to,
@@ -35,9 +70,9 @@
({ \
long ret; \
\
- inc_preempt_count(); \
+ pagefault_disable(); \
ret = __get_user(retval, addr); \
- dec_preempt_count(); \
+ pagefault_enable(); \
ret; \
})