Adrian Bunk | 88278ca | 2008-05-19 16:53:02 -0700 | [diff] [blame] | 1 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | * unaligned.c: Unaligned load/store trap handling with special |
| 3 | * cases for the kernel to do them more quickly. |
| 4 | * |
| 5 | * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) |
| 6 | * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz) |
| 7 | */ |
| 8 | |
| 9 | |
| 10 | #include <linux/kernel.h> |
| 11 | #include <linux/sched.h> |
| 12 | #include <linux/mm.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 13 | #include <asm/ptrace.h> |
| 14 | #include <asm/processor.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 15 | #include <asm/uaccess.h> |
| 16 | #include <linux/smp.h> |
David S. Miller | 121dd5f | 2009-12-11 01:07:53 -0800 | [diff] [blame] | 17 | #include <linux/perf_event.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 18 | |
| 19 | enum direction { |
| 20 | load, /* ld, ldd, ldh, ldsh */ |
| 21 | store, /* st, std, sth, stsh */ |
| 22 | both, /* Swap, ldstub, etc. */ |
| 23 | fpload, |
| 24 | fpstore, |
| 25 | invalid, |
| 26 | }; |
| 27 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 28 | static inline enum direction decode_direction(unsigned int insn) |
| 29 | { |
| 30 | unsigned long tmp = (insn >> 21) & 1; |
| 31 | |
| 32 | if(!tmp) |
| 33 | return load; |
| 34 | else { |
| 35 | if(((insn>>19)&0x3f) == 15) |
| 36 | return both; |
| 37 | else |
| 38 | return store; |
| 39 | } |
| 40 | } |
| 41 | |
| 42 | /* 8 = double-word, 4 = word, 2 = half-word */ |
| 43 | static inline int decode_access_size(unsigned int insn) |
| 44 | { |
| 45 | insn = (insn >> 19) & 3; |
| 46 | |
| 47 | if(!insn) |
| 48 | return 4; |
| 49 | else if(insn == 3) |
| 50 | return 8; |
| 51 | else if(insn == 2) |
| 52 | return 2; |
| 53 | else { |
| 54 | printk("Impossible unaligned trap. insn=%08x\n", insn); |
| 55 | die_if_kernel("Byte sized unaligned access?!?!", current->thread.kregs); |
| 56 | return 4; /* just to keep gcc happy. */ |
| 57 | } |
| 58 | } |
| 59 | |
| 60 | /* 0x400000 = signed, 0 = unsigned */ |
| 61 | static inline int decode_signedness(unsigned int insn) |
| 62 | { |
| 63 | return (insn & 0x400000); |
| 64 | } |
| 65 | |
| 66 | static inline void maybe_flush_windows(unsigned int rs1, unsigned int rs2, |
| 67 | unsigned int rd) |
| 68 | { |
| 69 | if(rs2 >= 16 || rs1 >= 16 || rd >= 16) { |
| 70 | /* Wheee... */ |
| 71 | __asm__ __volatile__("save %sp, -0x40, %sp\n\t" |
| 72 | "save %sp, -0x40, %sp\n\t" |
| 73 | "save %sp, -0x40, %sp\n\t" |
| 74 | "save %sp, -0x40, %sp\n\t" |
| 75 | "save %sp, -0x40, %sp\n\t" |
| 76 | "save %sp, -0x40, %sp\n\t" |
| 77 | "save %sp, -0x40, %sp\n\t" |
| 78 | "restore; restore; restore; restore;\n\t" |
| 79 | "restore; restore; restore;\n\t"); |
| 80 | } |
| 81 | } |
| 82 | |
| 83 | static inline int sign_extend_imm13(int imm) |
| 84 | { |
| 85 | return imm << 19 >> 19; |
| 86 | } |
| 87 | |
| 88 | static inline unsigned long fetch_reg(unsigned int reg, struct pt_regs *regs) |
| 89 | { |
Sam Ravnborg | 4d7b92a | 2009-01-02 19:32:59 -0800 | [diff] [blame] | 90 | struct reg_window32 *win; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 91 | |
| 92 | if(reg < 16) |
| 93 | return (!reg ? 0 : regs->u_regs[reg]); |
| 94 | |
| 95 | /* Ho hum, the slightly complicated case. */ |
Sam Ravnborg | 4d7b92a | 2009-01-02 19:32:59 -0800 | [diff] [blame] | 96 | win = (struct reg_window32 *) regs->u_regs[UREG_FP]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 97 | return win->locals[reg - 16]; /* yes, I know what this does... */ |
| 98 | } |
| 99 | |
| 100 | static inline unsigned long safe_fetch_reg(unsigned int reg, struct pt_regs *regs) |
| 101 | { |
Sam Ravnborg | 4d7b92a | 2009-01-02 19:32:59 -0800 | [diff] [blame] | 102 | struct reg_window32 __user *win; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 103 | unsigned long ret; |
| 104 | |
| 105 | if (reg < 16) |
| 106 | return (!reg ? 0 : regs->u_regs[reg]); |
| 107 | |
| 108 | /* Ho hum, the slightly complicated case. */ |
Sam Ravnborg | 4d7b92a | 2009-01-02 19:32:59 -0800 | [diff] [blame] | 109 | win = (struct reg_window32 __user *) regs->u_regs[UREG_FP]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 110 | |
| 111 | if ((unsigned long)win & 3) |
| 112 | return -1; |
| 113 | |
| 114 | if (get_user(ret, &win->locals[reg - 16])) |
| 115 | return -1; |
| 116 | |
| 117 | return ret; |
| 118 | } |
| 119 | |
| 120 | static inline unsigned long *fetch_reg_addr(unsigned int reg, struct pt_regs *regs) |
| 121 | { |
Sam Ravnborg | 4d7b92a | 2009-01-02 19:32:59 -0800 | [diff] [blame] | 122 | struct reg_window32 *win; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 123 | |
| 124 | if(reg < 16) |
| 125 | return ®s->u_regs[reg]; |
Sam Ravnborg | 4d7b92a | 2009-01-02 19:32:59 -0800 | [diff] [blame] | 126 | win = (struct reg_window32 *) regs->u_regs[UREG_FP]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 127 | return &win->locals[reg - 16]; |
| 128 | } |
| 129 | |
| 130 | static unsigned long compute_effective_address(struct pt_regs *regs, |
| 131 | unsigned int insn) |
| 132 | { |
| 133 | unsigned int rs1 = (insn >> 14) & 0x1f; |
| 134 | unsigned int rs2 = insn & 0x1f; |
| 135 | unsigned int rd = (insn >> 25) & 0x1f; |
| 136 | |
| 137 | if(insn & 0x2000) { |
| 138 | maybe_flush_windows(rs1, 0, rd); |
| 139 | return (fetch_reg(rs1, regs) + sign_extend_imm13(insn)); |
| 140 | } else { |
| 141 | maybe_flush_windows(rs1, rs2, rd); |
| 142 | return (fetch_reg(rs1, regs) + fetch_reg(rs2, regs)); |
| 143 | } |
| 144 | } |
| 145 | |
| 146 | unsigned long safe_compute_effective_address(struct pt_regs *regs, |
| 147 | unsigned int insn) |
| 148 | { |
| 149 | unsigned int rs1 = (insn >> 14) & 0x1f; |
| 150 | unsigned int rs2 = insn & 0x1f; |
| 151 | unsigned int rd = (insn >> 25) & 0x1f; |
| 152 | |
| 153 | if(insn & 0x2000) { |
| 154 | maybe_flush_windows(rs1, 0, rd); |
| 155 | return (safe_fetch_reg(rs1, regs) + sign_extend_imm13(insn)); |
| 156 | } else { |
| 157 | maybe_flush_windows(rs1, rs2, rd); |
| 158 | return (safe_fetch_reg(rs1, regs) + safe_fetch_reg(rs2, regs)); |
| 159 | } |
| 160 | } |
| 161 | |
| 162 | /* This is just to make gcc think panic does return... */ |
| 163 | static void unaligned_panic(char *str) |
| 164 | { |
| 165 | panic(str); |
| 166 | } |
| 167 | |
David S. Miller | f0e98c3 | 2008-03-03 15:01:05 -0800 | [diff] [blame] | 168 | /* una_asm.S */ |
| 169 | extern int do_int_load(unsigned long *dest_reg, int size, |
| 170 | unsigned long *saddr, int is_signed); |
| 171 | extern int __do_int_store(unsigned long *dst_addr, int size, |
| 172 | unsigned long *src_val); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 173 | |
David S. Miller | f0e98c3 | 2008-03-03 15:01:05 -0800 | [diff] [blame] | 174 | static int do_int_store(int reg_num, int size, unsigned long *dst_addr, |
| 175 | struct pt_regs *regs) |
| 176 | { |
| 177 | unsigned long zero[2] = { 0, 0 }; |
| 178 | unsigned long *src_val; |
| 179 | |
| 180 | if (reg_num) |
| 181 | src_val = fetch_reg_addr(reg_num, regs); |
| 182 | else { |
| 183 | src_val = &zero[0]; |
| 184 | if (size == 8) |
| 185 | zero[1] = fetch_reg(1, regs); |
| 186 | } |
| 187 | return __do_int_store(dst_addr, size, src_val); |
| 188 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 189 | |
| 190 | extern void smp_capture(void); |
| 191 | extern void smp_release(void); |
| 192 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 193 | static inline void advance(struct pt_regs *regs) |
| 194 | { |
| 195 | regs->pc = regs->npc; |
| 196 | regs->npc += 4; |
| 197 | } |
| 198 | |
| 199 | static inline int floating_point_load_or_store_p(unsigned int insn) |
| 200 | { |
| 201 | return (insn >> 24) & 1; |
| 202 | } |
| 203 | |
| 204 | static inline int ok_for_kernel(unsigned int insn) |
| 205 | { |
| 206 | return !floating_point_load_or_store_p(insn); |
| 207 | } |
| 208 | |
David S. Miller | f0e98c3 | 2008-03-03 15:01:05 -0800 | [diff] [blame] | 209 | static void kernel_mna_trap_fault(struct pt_regs *regs, unsigned int insn) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 210 | { |
| 211 | unsigned long g2 = regs->u_regs [UREG_G2]; |
| 212 | unsigned long fixup = search_extables_range(regs->pc, &g2); |
| 213 | |
| 214 | if (!fixup) { |
| 215 | unsigned long address = compute_effective_address(regs, insn); |
| 216 | if(address < PAGE_SIZE) { |
| 217 | printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference in mna handler"); |
| 218 | } else |
| 219 | printk(KERN_ALERT "Unable to handle kernel paging request in mna handler"); |
| 220 | printk(KERN_ALERT " at virtual address %08lx\n",address); |
| 221 | printk(KERN_ALERT "current->{mm,active_mm}->context = %08lx\n", |
| 222 | (current->mm ? current->mm->context : |
| 223 | current->active_mm->context)); |
| 224 | printk(KERN_ALERT "current->{mm,active_mm}->pgd = %08lx\n", |
| 225 | (current->mm ? (unsigned long) current->mm->pgd : |
| 226 | (unsigned long) current->active_mm->pgd)); |
| 227 | die_if_kernel("Oops", regs); |
| 228 | /* Not reached */ |
| 229 | } |
| 230 | regs->pc = fixup; |
| 231 | regs->npc = regs->pc + 4; |
| 232 | regs->u_regs [UREG_G2] = g2; |
| 233 | } |
| 234 | |
| 235 | asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn) |
| 236 | { |
| 237 | enum direction dir = decode_direction(insn); |
| 238 | int size = decode_access_size(insn); |
| 239 | |
| 240 | if(!ok_for_kernel(insn) || dir == both) { |
| 241 | printk("Unsupported unaligned load/store trap for kernel at <%08lx>.\n", |
| 242 | regs->pc); |
| 243 | unaligned_panic("Wheee. Kernel does fpu/atomic unaligned load/store."); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 244 | } else { |
| 245 | unsigned long addr = compute_effective_address(regs, insn); |
David S. Miller | f0e98c3 | 2008-03-03 15:01:05 -0800 | [diff] [blame] | 246 | int err; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 247 | |
Peter Zijlstra | a8b0ca1 | 2011-06-27 14:41:57 +0200 | [diff] [blame] | 248 | perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); |
David S. Miller | f0e98c3 | 2008-03-03 15:01:05 -0800 | [diff] [blame] | 249 | switch (dir) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 250 | case load: |
David S. Miller | f0e98c3 | 2008-03-03 15:01:05 -0800 | [diff] [blame] | 251 | err = do_int_load(fetch_reg_addr(((insn>>25)&0x1f), |
| 252 | regs), |
| 253 | size, (unsigned long *) addr, |
| 254 | decode_signedness(insn)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 255 | break; |
| 256 | |
| 257 | case store: |
David S. Miller | f0e98c3 | 2008-03-03 15:01:05 -0800 | [diff] [blame] | 258 | err = do_int_store(((insn>>25)&0x1f), size, |
| 259 | (unsigned long *) addr, regs); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 260 | break; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 261 | default: |
| 262 | panic("Impossible kernel unaligned trap."); |
| 263 | /* Not reached... */ |
| 264 | } |
David S. Miller | f0e98c3 | 2008-03-03 15:01:05 -0800 | [diff] [blame] | 265 | if (err) |
| 266 | kernel_mna_trap_fault(regs, insn); |
| 267 | else |
| 268 | advance(regs); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 269 | } |
| 270 | } |
| 271 | |
| 272 | static inline int ok_for_user(struct pt_regs *regs, unsigned int insn, |
| 273 | enum direction dir) |
| 274 | { |
| 275 | unsigned int reg; |
| 276 | int check = (dir == load) ? VERIFY_READ : VERIFY_WRITE; |
| 277 | int size = ((insn >> 19) & 3) == 3 ? 8 : 4; |
| 278 | |
| 279 | if ((regs->pc | regs->npc) & 3) |
| 280 | return 0; |
| 281 | |
| 282 | /* Must access_ok() in all the necessary places. */ |
| 283 | #define WINREG_ADDR(regnum) \ |
| 284 | ((void __user *)(((unsigned long *)regs->u_regs[UREG_FP])+(regnum))) |
| 285 | |
| 286 | reg = (insn >> 25) & 0x1f; |
| 287 | if (reg >= 16) { |
| 288 | if (!access_ok(check, WINREG_ADDR(reg - 16), size)) |
| 289 | return -EFAULT; |
| 290 | } |
| 291 | reg = (insn >> 14) & 0x1f; |
| 292 | if (reg >= 16) { |
| 293 | if (!access_ok(check, WINREG_ADDR(reg - 16), size)) |
| 294 | return -EFAULT; |
| 295 | } |
| 296 | if (!(insn & 0x2000)) { |
| 297 | reg = (insn & 0x1f); |
| 298 | if (reg >= 16) { |
| 299 | if (!access_ok(check, WINREG_ADDR(reg - 16), size)) |
| 300 | return -EFAULT; |
| 301 | } |
| 302 | } |
| 303 | #undef WINREG_ADDR |
| 304 | return 0; |
| 305 | } |
| 306 | |
David S. Miller | f0e98c3 | 2008-03-03 15:01:05 -0800 | [diff] [blame] | 307 | static void user_mna_trap_fault(struct pt_regs *regs, unsigned int insn) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 308 | { |
| 309 | siginfo_t info; |
| 310 | |
| 311 | info.si_signo = SIGBUS; |
| 312 | info.si_errno = 0; |
| 313 | info.si_code = BUS_ADRALN; |
| 314 | info.si_addr = (void __user *)safe_compute_effective_address(regs, insn); |
| 315 | info.si_trapno = 0; |
| 316 | send_sig_info(SIGBUS, &info, current); |
| 317 | } |
| 318 | |
| 319 | asmlinkage void user_unaligned_trap(struct pt_regs *regs, unsigned int insn) |
| 320 | { |
| 321 | enum direction dir; |
| 322 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 323 | if(!(current->thread.flags & SPARC_FLAG_UNALIGNED) || |
| 324 | (((insn >> 30) & 3) != 3)) |
| 325 | goto kill_user; |
| 326 | dir = decode_direction(insn); |
| 327 | if(!ok_for_user(regs, insn, dir)) { |
| 328 | goto kill_user; |
| 329 | } else { |
David S. Miller | f0e98c3 | 2008-03-03 15:01:05 -0800 | [diff] [blame] | 330 | int err, size = decode_access_size(insn); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 331 | unsigned long addr; |
| 332 | |
| 333 | if(floating_point_load_or_store_p(insn)) { |
| 334 | printk("User FPU load/store unaligned unsupported.\n"); |
| 335 | goto kill_user; |
| 336 | } |
| 337 | |
| 338 | addr = compute_effective_address(regs, insn); |
Peter Zijlstra | a8b0ca1 | 2011-06-27 14:41:57 +0200 | [diff] [blame] | 339 | perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 340 | switch(dir) { |
| 341 | case load: |
David S. Miller | f0e98c3 | 2008-03-03 15:01:05 -0800 | [diff] [blame] | 342 | err = do_int_load(fetch_reg_addr(((insn>>25)&0x1f), |
| 343 | regs), |
| 344 | size, (unsigned long *) addr, |
| 345 | decode_signedness(insn)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 346 | break; |
| 347 | |
| 348 | case store: |
David S. Miller | f0e98c3 | 2008-03-03 15:01:05 -0800 | [diff] [blame] | 349 | err = do_int_store(((insn>>25)&0x1f), size, |
| 350 | (unsigned long *) addr, regs); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 351 | break; |
| 352 | |
| 353 | case both: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 354 | /* |
| 355 | * This was supported in 2.4. However, we question |
| 356 | * the value of SWAP instruction across word boundaries. |
| 357 | */ |
| 358 | printk("Unaligned SWAP unsupported.\n"); |
David S. Miller | f0e98c3 | 2008-03-03 15:01:05 -0800 | [diff] [blame] | 359 | err = -EFAULT; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 360 | break; |
| 361 | |
| 362 | default: |
| 363 | unaligned_panic("Impossible user unaligned trap."); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 364 | goto out; |
| 365 | } |
David S. Miller | f0e98c3 | 2008-03-03 15:01:05 -0800 | [diff] [blame] | 366 | if (err) |
| 367 | goto kill_user; |
| 368 | else |
| 369 | advance(regs); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 370 | goto out; |
| 371 | } |
| 372 | |
| 373 | kill_user: |
| 374 | user_mna_trap_fault(regs, insn); |
| 375 | out: |
David S. Miller | b19f820 | 2010-09-08 20:57:59 -0700 | [diff] [blame] | 376 | ; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 377 | } |