Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 1 | /* |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 2 | * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3 | * Licensed under the GPL |
| 4 | */ |
| 5 | |
Jeff Dike | 8192ab4 | 2008-02-04 22:30:53 -0800 | [diff] [blame] | 6 | #include <linux/mm.h> |
Al Viro | 73395a0 | 2011-08-18 20:14:10 +0100 | [diff] [blame] | 7 | #include <linux/module.h> |
Jeff Dike | 8192ab4 | 2008-02-04 22:30:53 -0800 | [diff] [blame] | 8 | #include <linux/sched.h> |
| 9 | #include <asm/pgtable.h> |
| 10 | #include <asm/tlbflush.h> |
Al Viro | 37185b3 | 2012-10-08 03:27:32 +0100 | [diff] [blame] | 11 | #include <as-layout.h> |
| 12 | #include <mem_user.h> |
| 13 | #include <os.h> |
| 14 | #include <skas.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 15 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 16 | struct host_vm_change { |
| 17 | struct host_vm_op { |
| 18 | enum { NONE, MMAP, MUNMAP, MPROTECT } type; |
| 19 | union { |
| 20 | struct { |
| 21 | unsigned long addr; |
| 22 | unsigned long len; |
| 23 | unsigned int prot; |
| 24 | int fd; |
| 25 | __u64 offset; |
| 26 | } mmap; |
| 27 | struct { |
| 28 | unsigned long addr; |
| 29 | unsigned long len; |
| 30 | } munmap; |
| 31 | struct { |
| 32 | unsigned long addr; |
| 33 | unsigned long len; |
| 34 | unsigned int prot; |
| 35 | } mprotect; |
| 36 | } u; |
| 37 | } ops[1]; |
| 38 | int index; |
| 39 | struct mm_id *id; |
| 40 | void *data; |
| 41 | int force; |
| 42 | }; |
| 43 | |
| 44 | #define INIT_HVC(mm, force) \ |
| 45 | ((struct host_vm_change) \ |
| 46 | { .ops = { { .type = NONE } }, \ |
| 47 | .id = &mm->context.id, \ |
| 48 | .data = NULL, \ |
| 49 | .index = 0, \ |
| 50 | .force = force }) |
| 51 | |
| 52 | static int do_ops(struct host_vm_change *hvc, int end, |
| 53 | int finished) |
| 54 | { |
| 55 | struct host_vm_op *op; |
| 56 | int i, ret = 0; |
| 57 | |
| 58 | for (i = 0; i < end && !ret; i++) { |
| 59 | op = &hvc->ops[i]; |
Jeff Dike | c5d4bb1 | 2008-02-04 22:31:14 -0800 | [diff] [blame] | 60 | switch (op->type) { |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 61 | case MMAP: |
| 62 | ret = map(hvc->id, op->u.mmap.addr, op->u.mmap.len, |
| 63 | op->u.mmap.prot, op->u.mmap.fd, |
| 64 | op->u.mmap.offset, finished, &hvc->data); |
| 65 | break; |
| 66 | case MUNMAP: |
| 67 | ret = unmap(hvc->id, op->u.munmap.addr, |
| 68 | op->u.munmap.len, finished, &hvc->data); |
| 69 | break; |
| 70 | case MPROTECT: |
| 71 | ret = protect(hvc->id, op->u.mprotect.addr, |
| 72 | op->u.mprotect.len, op->u.mprotect.prot, |
| 73 | finished, &hvc->data); |
| 74 | break; |
| 75 | default: |
| 76 | printk(KERN_ERR "Unknown op type %d in do_ops\n", |
| 77 | op->type); |
Richard Weinberger | 62179d4 | 2012-04-13 15:54:01 +0200 | [diff] [blame] | 78 | BUG(); |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 79 | break; |
| 80 | } |
| 81 | } |
| 82 | |
| 83 | return ret; |
| 84 | } |
| 85 | |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 86 | static int add_mmap(unsigned long virt, unsigned long phys, unsigned long len, |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 87 | unsigned int prot, struct host_vm_change *hvc) |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 88 | { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 89 | __u64 offset; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 90 | struct host_vm_op *last; |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 91 | int fd, ret = 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 92 | |
| 93 | fd = phys_mapping(phys, &offset); |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 94 | if (hvc->index != 0) { |
| 95 | last = &hvc->ops[hvc->index - 1]; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 96 | if ((last->type == MMAP) && |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 97 | (last->u.mmap.addr + last->u.mmap.len == virt) && |
Jeff Dike | 16dd07b | 2007-05-06 14:51:48 -0700 | [diff] [blame] | 98 | (last->u.mmap.prot == prot) && (last->u.mmap.fd == fd) && |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 99 | (last->u.mmap.offset + last->u.mmap.len == offset)) { |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 100 | last->u.mmap.len += len; |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 101 | return 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 102 | } |
| 103 | } |
| 104 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 105 | if (hvc->index == ARRAY_SIZE(hvc->ops)) { |
| 106 | ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0); |
| 107 | hvc->index = 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 108 | } |
| 109 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 110 | hvc->ops[hvc->index++] = ((struct host_vm_op) |
| 111 | { .type = MMAP, |
| 112 | .u = { .mmap = { .addr = virt, |
| 113 | .len = len, |
| 114 | .prot = prot, |
| 115 | .fd = fd, |
| 116 | .offset = offset } |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 117 | } }); |
| 118 | return ret; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 119 | } |
| 120 | |
| 121 | static int add_munmap(unsigned long addr, unsigned long len, |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 122 | struct host_vm_change *hvc) |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 123 | { |
| 124 | struct host_vm_op *last; |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 125 | int ret = 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 126 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 127 | if (hvc->index != 0) { |
| 128 | last = &hvc->ops[hvc->index - 1]; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 129 | if ((last->type == MUNMAP) && |
| 130 | (last->u.munmap.addr + last->u.mmap.len == addr)) { |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 131 | last->u.munmap.len += len; |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 132 | return 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 133 | } |
| 134 | } |
| 135 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 136 | if (hvc->index == ARRAY_SIZE(hvc->ops)) { |
| 137 | ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0); |
| 138 | hvc->index = 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 139 | } |
| 140 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 141 | hvc->ops[hvc->index++] = ((struct host_vm_op) |
| 142 | { .type = MUNMAP, |
| 143 | .u = { .munmap = { .addr = addr, |
| 144 | .len = len } } }); |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 145 | return ret; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 146 | } |
| 147 | |
Jeff Dike | 16dd07b | 2007-05-06 14:51:48 -0700 | [diff] [blame] | 148 | static int add_mprotect(unsigned long addr, unsigned long len, |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 149 | unsigned int prot, struct host_vm_change *hvc) |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 150 | { |
| 151 | struct host_vm_op *last; |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 152 | int ret = 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 153 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 154 | if (hvc->index != 0) { |
| 155 | last = &hvc->ops[hvc->index - 1]; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 156 | if ((last->type == MPROTECT) && |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 157 | (last->u.mprotect.addr + last->u.mprotect.len == addr) && |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 158 | (last->u.mprotect.prot == prot)) { |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 159 | last->u.mprotect.len += len; |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 160 | return 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 161 | } |
| 162 | } |
| 163 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 164 | if (hvc->index == ARRAY_SIZE(hvc->ops)) { |
| 165 | ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0); |
| 166 | hvc->index = 0; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 167 | } |
| 168 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 169 | hvc->ops[hvc->index++] = ((struct host_vm_op) |
| 170 | { .type = MPROTECT, |
| 171 | .u = { .mprotect = { .addr = addr, |
| 172 | .len = len, |
| 173 | .prot = prot } } }); |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 174 | return ret; |
Jeff Dike | c560049 | 2005-09-03 15:57:36 -0700 | [diff] [blame] | 175 | } |
| 176 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 177 | #define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1)) |
| 178 | |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 179 | static inline int update_pte_range(pmd_t *pmd, unsigned long addr, |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 180 | unsigned long end, |
| 181 | struct host_vm_change *hvc) |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 182 | { |
| 183 | pte_t *pte; |
Jeff Dike | 16dd07b | 2007-05-06 14:51:48 -0700 | [diff] [blame] | 184 | int r, w, x, prot, ret = 0; |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 185 | |
| 186 | pte = pte_offset_kernel(pmd, addr); |
| 187 | do { |
Jeff Dike | 3963333 | 2008-02-04 22:31:01 -0800 | [diff] [blame] | 188 | if ((addr >= STUB_START) && (addr < STUB_END)) |
| 189 | continue; |
| 190 | |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 191 | r = pte_read(*pte); |
| 192 | w = pte_write(*pte); |
| 193 | x = pte_exec(*pte); |
| 194 | if (!pte_young(*pte)) { |
| 195 | r = 0; |
| 196 | w = 0; |
Jeff Dike | 0b4e273 | 2008-02-04 22:31:07 -0800 | [diff] [blame] | 197 | } else if (!pte_dirty(*pte)) |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 198 | w = 0; |
Jeff Dike | 0b4e273 | 2008-02-04 22:31:07 -0800 | [diff] [blame] | 199 | |
Jeff Dike | 16dd07b | 2007-05-06 14:51:48 -0700 | [diff] [blame] | 200 | prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) | |
| 201 | (x ? UM_PROT_EXEC : 0)); |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 202 | if (hvc->force || pte_newpage(*pte)) { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 203 | if (pte_present(*pte)) |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 204 | ret = add_mmap(addr, pte_val(*pte) & PAGE_MASK, |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 205 | PAGE_SIZE, prot, hvc); |
Jeff Dike | 0b4e273 | 2008-02-04 22:31:07 -0800 | [diff] [blame] | 206 | else |
| 207 | ret = add_munmap(addr, PAGE_SIZE, hvc); |
| 208 | } else if (pte_newprot(*pte)) |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 209 | ret = add_mprotect(addr, PAGE_SIZE, prot, hvc); |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 210 | *pte = pte_mkuptodate(*pte); |
Jeff Dike | 909e90d | 2008-02-04 22:31:06 -0800 | [diff] [blame] | 211 | } while (pte++, addr += PAGE_SIZE, ((addr < end) && !ret)); |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 212 | return ret; |
| 213 | } |
| 214 | |
| 215 | static inline int update_pmd_range(pud_t *pud, unsigned long addr, |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 216 | unsigned long end, |
| 217 | struct host_vm_change *hvc) |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 218 | { |
| 219 | pmd_t *pmd; |
| 220 | unsigned long next; |
| 221 | int ret = 0; |
| 222 | |
| 223 | pmd = pmd_offset(pud, addr); |
| 224 | do { |
| 225 | next = pmd_addr_end(addr, end); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 226 | if (!pmd_present(*pmd)) { |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 227 | if (hvc->force || pmd_newpage(*pmd)) { |
| 228 | ret = add_munmap(addr, next - addr, hvc); |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 229 | pmd_mkuptodate(*pmd); |
| 230 | } |
| 231 | } |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 232 | else ret = update_pte_range(pmd, addr, next, hvc); |
Jeff Dike | 909e90d | 2008-02-04 22:31:06 -0800 | [diff] [blame] | 233 | } while (pmd++, addr = next, ((addr < end) && !ret)); |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 234 | return ret; |
| 235 | } |
| 236 | |
| 237 | static inline int update_pud_range(pgd_t *pgd, unsigned long addr, |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 238 | unsigned long end, |
| 239 | struct host_vm_change *hvc) |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 240 | { |
| 241 | pud_t *pud; |
| 242 | unsigned long next; |
| 243 | int ret = 0; |
| 244 | |
| 245 | pud = pud_offset(pgd, addr); |
| 246 | do { |
| 247 | next = pud_addr_end(addr, end); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 248 | if (!pud_present(*pud)) { |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 249 | if (hvc->force || pud_newpage(*pud)) { |
| 250 | ret = add_munmap(addr, next - addr, hvc); |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 251 | pud_mkuptodate(*pud); |
| 252 | } |
| 253 | } |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 254 | else ret = update_pmd_range(pud, addr, next, hvc); |
Jeff Dike | 909e90d | 2008-02-04 22:31:06 -0800 | [diff] [blame] | 255 | } while (pud++, addr = next, ((addr < end) && !ret)); |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 256 | return ret; |
| 257 | } |
| 258 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 259 | void fix_range_common(struct mm_struct *mm, unsigned long start_addr, |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 260 | unsigned long end_addr, int force) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 261 | { |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 262 | pgd_t *pgd; |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 263 | struct host_vm_change hvc; |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 264 | unsigned long addr = start_addr, next; |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 265 | int ret = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 266 | |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 267 | hvc = INIT_HVC(mm, force); |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 268 | pgd = pgd_offset(mm, addr); |
| 269 | do { |
| 270 | next = pgd_addr_end(addr, end_addr); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 271 | if (!pgd_present(*pgd)) { |
| 272 | if (force || pgd_newpage(*pgd)) { |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 273 | ret = add_munmap(addr, next - addr, &hvc); |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 274 | pgd_mkuptodate(*pgd); |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 275 | } |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 276 | } |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 277 | else ret = update_pud_range(pgd, addr, next, &hvc); |
Jeff Dike | 909e90d | 2008-02-04 22:31:06 -0800 | [diff] [blame] | 278 | } while (pgd++, addr = next, ((addr < end_addr) && !ret)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 279 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 280 | if (!ret) |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 281 | ret = do_ops(&hvc, hvc.index, 1); |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 282 | |
Jeff Dike | 7f0536f | 2007-05-06 14:51:30 -0700 | [diff] [blame] | 283 | /* This is not an else because ret is modified above */ |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 284 | if (ret) { |
| 285 | printk(KERN_ERR "fix_range_common: failed, killing current " |
| 286 | "process\n"); |
Bodo Stroesser | 07bf731 | 2005-09-03 15:57:50 -0700 | [diff] [blame] | 287 | force_sig(SIGKILL, current); |
| 288 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 289 | } |
| 290 | |
Al Viro | c75d053 | 2011-08-18 20:07:49 +0100 | [diff] [blame] | 291 | static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 292 | { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 293 | struct mm_struct *mm; |
| 294 | pgd_t *pgd; |
| 295 | pud_t *pud; |
| 296 | pmd_t *pmd; |
| 297 | pte_t *pte; |
| 298 | unsigned long addr, last; |
| 299 | int updated = 0, err; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 300 | |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 301 | mm = &init_mm; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 302 | for (addr = start; addr < end;) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 303 | pgd = pgd_offset(mm, addr); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 304 | if (!pgd_present(*pgd)) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 305 | last = ADD_ROUND(addr, PGDIR_SIZE); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 306 | if (last > end) |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 307 | last = end; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 308 | if (pgd_newpage(*pgd)) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 309 | updated = 1; |
| 310 | err = os_unmap_memory((void *) addr, |
| 311 | last - addr); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 312 | if (err < 0) |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 313 | panic("munmap failed, errno = %d\n", |
| 314 | -err); |
| 315 | } |
| 316 | addr = last; |
| 317 | continue; |
| 318 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 319 | |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 320 | pud = pud_offset(pgd, addr); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 321 | if (!pud_present(*pud)) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 322 | last = ADD_ROUND(addr, PUD_SIZE); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 323 | if (last > end) |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 324 | last = end; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 325 | if (pud_newpage(*pud)) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 326 | updated = 1; |
| 327 | err = os_unmap_memory((void *) addr, |
| 328 | last - addr); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 329 | if (err < 0) |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 330 | panic("munmap failed, errno = %d\n", |
| 331 | -err); |
| 332 | } |
| 333 | addr = last; |
| 334 | continue; |
| 335 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 336 | |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 337 | pmd = pmd_offset(pud, addr); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 338 | if (!pmd_present(*pmd)) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 339 | last = ADD_ROUND(addr, PMD_SIZE); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 340 | if (last > end) |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 341 | last = end; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 342 | if (pmd_newpage(*pmd)) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 343 | updated = 1; |
| 344 | err = os_unmap_memory((void *) addr, |
| 345 | last - addr); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 346 | if (err < 0) |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 347 | panic("munmap failed, errno = %d\n", |
| 348 | -err); |
| 349 | } |
| 350 | addr = last; |
| 351 | continue; |
| 352 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 353 | |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 354 | pte = pte_offset_kernel(pmd, addr); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 355 | if (!pte_present(*pte) || pte_newpage(*pte)) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 356 | updated = 1; |
| 357 | err = os_unmap_memory((void *) addr, |
| 358 | PAGE_SIZE); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 359 | if (err < 0) |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 360 | panic("munmap failed, errno = %d\n", |
| 361 | -err); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 362 | if (pte_present(*pte)) |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 363 | map_memory(addr, |
| 364 | pte_val(*pte) & PAGE_MASK, |
| 365 | PAGE_SIZE, 1, 1, 1); |
| 366 | } |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 367 | else if (pte_newprot(*pte)) { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 368 | updated = 1; |
| 369 | os_protect_memory((void *) addr, PAGE_SIZE, 1, 1, 1); |
| 370 | } |
| 371 | addr += PAGE_SIZE; |
| 372 | } |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 373 | return updated; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 374 | } |
| 375 | |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 376 | void flush_tlb_page(struct vm_area_struct *vma, unsigned long address) |
| 377 | { |
| 378 | pgd_t *pgd; |
| 379 | pud_t *pud; |
| 380 | pmd_t *pmd; |
| 381 | pte_t *pte; |
| 382 | struct mm_struct *mm = vma->vm_mm; |
| 383 | void *flush = NULL; |
| 384 | int r, w, x, prot, err = 0; |
| 385 | struct mm_id *mm_id; |
| 386 | |
| 387 | address &= PAGE_MASK; |
| 388 | pgd = pgd_offset(mm, address); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 389 | if (!pgd_present(*pgd)) |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 390 | goto kill; |
| 391 | |
| 392 | pud = pud_offset(pgd, address); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 393 | if (!pud_present(*pud)) |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 394 | goto kill; |
| 395 | |
| 396 | pmd = pmd_offset(pud, address); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 397 | if (!pmd_present(*pmd)) |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 398 | goto kill; |
| 399 | |
| 400 | pte = pte_offset_kernel(pmd, address); |
| 401 | |
| 402 | r = pte_read(*pte); |
| 403 | w = pte_write(*pte); |
| 404 | x = pte_exec(*pte); |
| 405 | if (!pte_young(*pte)) { |
| 406 | r = 0; |
| 407 | w = 0; |
| 408 | } else if (!pte_dirty(*pte)) { |
| 409 | w = 0; |
| 410 | } |
| 411 | |
Jeff Dike | 6c738ff | 2007-10-16 01:27:06 -0700 | [diff] [blame] | 412 | mm_id = &mm->context.id; |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 413 | prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) | |
| 414 | (x ? UM_PROT_EXEC : 0)); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 415 | if (pte_newpage(*pte)) { |
| 416 | if (pte_present(*pte)) { |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 417 | unsigned long long offset; |
| 418 | int fd; |
| 419 | |
| 420 | fd = phys_mapping(pte_val(*pte) & PAGE_MASK, &offset); |
| 421 | err = map(mm_id, address, PAGE_SIZE, prot, fd, offset, |
| 422 | 1, &flush); |
| 423 | } |
| 424 | else err = unmap(mm_id, address, PAGE_SIZE, 1, &flush); |
| 425 | } |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 426 | else if (pte_newprot(*pte)) |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 427 | err = protect(mm_id, address, PAGE_SIZE, prot, 1, &flush); |
| 428 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 429 | if (err) |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 430 | goto kill; |
| 431 | |
| 432 | *pte = pte_mkuptodate(*pte); |
| 433 | |
| 434 | return; |
| 435 | |
| 436 | kill: |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 437 | printk(KERN_ERR "Failed to flush page for address 0x%lx\n", address); |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 438 | force_sig(SIGKILL, current); |
| 439 | } |
| 440 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 441 | pgd_t *pgd_offset_proc(struct mm_struct *mm, unsigned long address) |
| 442 | { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 443 | return pgd_offset(mm, address); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 444 | } |
| 445 | |
| 446 | pud_t *pud_offset_proc(pgd_t *pgd, unsigned long address) |
| 447 | { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 448 | return pud_offset(pgd, address); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 449 | } |
| 450 | |
| 451 | pmd_t *pmd_offset_proc(pud_t *pud, unsigned long address) |
| 452 | { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 453 | return pmd_offset(pud, address); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 454 | } |
| 455 | |
| 456 | pte_t *pte_offset_proc(pmd_t *pmd, unsigned long address) |
| 457 | { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 458 | return pte_offset_kernel(pmd, address); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 459 | } |
| 460 | |
| 461 | pte_t *addr_pte(struct task_struct *task, unsigned long addr) |
| 462 | { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 463 | pgd_t *pgd = pgd_offset(task->mm, addr); |
| 464 | pud_t *pud = pud_offset(pgd, addr); |
| 465 | pmd_t *pmd = pmd_offset(pud, addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 466 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 467 | return pte_offset_map(pmd, addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 468 | } |
| 469 | |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 470 | void flush_tlb_all(void) |
| 471 | { |
Jeff Dike | 5e1f65a | 2006-09-25 23:33:01 -0700 | [diff] [blame] | 472 | flush_tlb_mm(current->mm); |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 473 | } |
| 474 | |
| 475 | void flush_tlb_kernel_range(unsigned long start, unsigned long end) |
| 476 | { |
Jeff Dike | 6aa802c | 2007-10-16 01:26:56 -0700 | [diff] [blame] | 477 | flush_tlb_kernel_range_common(start, end); |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 478 | } |
| 479 | |
| 480 | void flush_tlb_kernel_vm(void) |
| 481 | { |
Jeff Dike | 6aa802c | 2007-10-16 01:26:56 -0700 | [diff] [blame] | 482 | flush_tlb_kernel_range_common(start_vm, end_vm); |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 483 | } |
| 484 | |
| 485 | void __flush_tlb_one(unsigned long addr) |
| 486 | { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 487 | flush_tlb_kernel_range_common(addr, addr + PAGE_SIZE); |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 488 | } |
| 489 | |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 490 | static void fix_range(struct mm_struct *mm, unsigned long start_addr, |
| 491 | unsigned long end_addr, int force) |
| 492 | { |
Jeff Dike | 1466abf | 2007-10-16 01:27:12 -0700 | [diff] [blame] | 493 | fix_range_common(mm, start_addr, end_addr, force); |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 494 | } |
| 495 | |
| 496 | void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, |
| 497 | unsigned long end) |
| 498 | { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 499 | if (vma->vm_mm == NULL) |
| 500 | flush_tlb_kernel_range_common(start, end); |
| 501 | else fix_range(vma->vm_mm, start, end, 0); |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 502 | } |
Al Viro | 73395a0 | 2011-08-18 20:14:10 +0100 | [diff] [blame] | 503 | EXPORT_SYMBOL(flush_tlb_range); |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 504 | |
Jeff Dike | 0b4e273 | 2008-02-04 22:31:07 -0800 | [diff] [blame] | 505 | void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, |
| 506 | unsigned long end) |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 507 | { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 508 | /* |
| 509 | * Don't bother flushing if this address space is about to be |
| 510 | * destroyed. |
| 511 | */ |
| 512 | if (atomic_read(&mm->mm_users) == 0) |
| 513 | return; |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 514 | |
Jeff Dike | 0b4e273 | 2008-02-04 22:31:07 -0800 | [diff] [blame] | 515 | fix_range(mm, start, end, 0); |
| 516 | } |
| 517 | |
| 518 | void flush_tlb_mm(struct mm_struct *mm) |
| 519 | { |
| 520 | struct vm_area_struct *vma = mm->mmap; |
| 521 | |
| 522 | while (vma != NULL) { |
| 523 | fix_range(mm, vma->vm_start, vma->vm_end, 0); |
| 524 | vma = vma->vm_next; |
| 525 | } |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 526 | } |
| 527 | |
| 528 | void force_flush_all(void) |
| 529 | { |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 530 | struct mm_struct *mm = current->mm; |
| 531 | struct vm_area_struct *vma = mm->mmap; |
| 532 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 533 | while (vma != NULL) { |
Jeff Dike | 77bf440 | 2007-10-16 01:26:58 -0700 | [diff] [blame] | 534 | fix_range(mm, vma->vm_start, vma->vm_end, 1); |
| 535 | vma = vma->vm_next; |
| 536 | } |
Jeff Dike | d67b569 | 2005-07-07 17:56:49 -0700 | [diff] [blame] | 537 | } |