Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This file contains the code that gets mapped at the upper end of each task's text |
| 3 | * region. For now, it contains the signal trampoline code only. |
| 4 | * |
| 5 | * Copyright (C) 1999-2003 Hewlett-Packard Co |
| 6 | * David Mosberger-Tang <davidm@hpl.hp.com> |
| 7 | */ |
| 8 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9 | |
| 10 | #include <asm/asmmacro.h> |
| 11 | #include <asm/errno.h> |
Sam Ravnborg | 39e01cb | 2005-09-09 22:03:13 +0200 | [diff] [blame] | 12 | #include <asm/asm-offsets.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 13 | #include <asm/sigcontext.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 14 | #include <asm/unistd.h> |
David Howells | c140d87 | 2012-03-28 18:30:02 +0100 | [diff] [blame] | 15 | #include <asm/kregs.h> |
| 16 | #include <asm/page.h> |
Isaku Yamahata | c431251 | 2009-03-04 21:05:45 +0900 | [diff] [blame] | 17 | #include "paravirt_inst.h" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 18 | |
| 19 | /* |
| 20 | * We can't easily refer to symbols inside the kernel. To avoid full runtime relocation, |
| 21 | * complications with the linker (which likes to create PLT stubs for branches |
| 22 | * to targets outside the shared object) and to avoid multi-phase kernel builds, we |
| 23 | * simply create minimalistic "patch lists" in special ELF sections. |
| 24 | */ |
Denys Vlasenko | dafb932 | 2010-02-20 01:03:42 +0100 | [diff] [blame] | 25 | .section ".data..patch.fsyscall_table", "a" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 26 | .previous |
| 27 | #define LOAD_FSYSCALL_TABLE(reg) \ |
| 28 | [1:] movl reg=0; \ |
Denys Vlasenko | dafb932 | 2010-02-20 01:03:42 +0100 | [diff] [blame] | 29 | .xdata4 ".data..patch.fsyscall_table", 1b-. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 30 | |
Denys Vlasenko | dafb932 | 2010-02-20 01:03:42 +0100 | [diff] [blame] | 31 | .section ".data..patch.brl_fsys_bubble_down", "a" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 32 | .previous |
| 33 | #define BRL_COND_FSYS_BUBBLE_DOWN(pr) \ |
| 34 | [1:](pr)brl.cond.sptk 0; \ |
Christian Kandeler | c6255e9 | 2007-07-09 16:19:11 +0200 | [diff] [blame] | 35 | ;; \ |
Denys Vlasenko | dafb932 | 2010-02-20 01:03:42 +0100 | [diff] [blame] | 36 | .xdata4 ".data..patch.brl_fsys_bubble_down", 1b-. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 37 | |
| 38 | GLOBAL_ENTRY(__kernel_syscall_via_break) |
| 39 | .prologue |
| 40 | .altrp b6 |
| 41 | .body |
| 42 | /* |
| 43 | * Note: for (fast) syscall restart to work, the break instruction must be |
| 44 | * the first one in the bundle addressed by syscall_via_break. |
| 45 | */ |
| 46 | { .mib |
| 47 | break 0x100000 |
| 48 | nop.i 0 |
| 49 | br.ret.sptk.many b6 |
| 50 | } |
| 51 | END(__kernel_syscall_via_break) |
| 52 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 53 | # define ARG0_OFF (16 + IA64_SIGFRAME_ARG0_OFFSET) |
| 54 | # define ARG1_OFF (16 + IA64_SIGFRAME_ARG1_OFFSET) |
| 55 | # define ARG2_OFF (16 + IA64_SIGFRAME_ARG2_OFFSET) |
| 56 | # define SIGHANDLER_OFF (16 + IA64_SIGFRAME_HANDLER_OFFSET) |
| 57 | # define SIGCONTEXT_OFF (16 + IA64_SIGFRAME_SIGCONTEXT_OFFSET) |
| 58 | |
| 59 | # define FLAGS_OFF IA64_SIGCONTEXT_FLAGS_OFFSET |
| 60 | # define CFM_OFF IA64_SIGCONTEXT_CFM_OFFSET |
| 61 | # define FR6_OFF IA64_SIGCONTEXT_FR6_OFFSET |
| 62 | # define BSP_OFF IA64_SIGCONTEXT_AR_BSP_OFFSET |
| 63 | # define RNAT_OFF IA64_SIGCONTEXT_AR_RNAT_OFFSET |
| 64 | # define UNAT_OFF IA64_SIGCONTEXT_AR_UNAT_OFFSET |
| 65 | # define FPSR_OFF IA64_SIGCONTEXT_AR_FPSR_OFFSET |
| 66 | # define PR_OFF IA64_SIGCONTEXT_PR_OFFSET |
| 67 | # define RP_OFF IA64_SIGCONTEXT_IP_OFFSET |
| 68 | # define SP_OFF IA64_SIGCONTEXT_R12_OFFSET |
| 69 | # define RBS_BASE_OFF IA64_SIGCONTEXT_RBS_BASE_OFFSET |
| 70 | # define LOADRS_OFF IA64_SIGCONTEXT_LOADRS_OFFSET |
| 71 | # define base0 r2 |
| 72 | # define base1 r3 |
| 73 | /* |
| 74 | * When we get here, the memory stack looks like this: |
| 75 | * |
| 76 | * +===============================+ |
| 77 | * | | |
| 78 | * // struct sigframe // |
| 79 | * | | |
| 80 | * +-------------------------------+ <-- sp+16 |
| 81 | * | 16 byte of scratch | |
| 82 | * | space | |
| 83 | * +-------------------------------+ <-- sp |
| 84 | * |
| 85 | * The register stack looks _exactly_ the way it looked at the time the signal |
| 86 | * occurred. In other words, we're treading on a potential mine-field: each |
| 87 | * incoming general register may be a NaT value (including sp, in which case the |
| 88 | * process ends up dying with a SIGSEGV). |
| 89 | * |
| 90 | * The first thing need to do is a cover to get the registers onto the backing |
| 91 | * store. Once that is done, we invoke the signal handler which may modify some |
| 92 | * of the machine state. After returning from the signal handler, we return |
| 93 | * control to the previous context by executing a sigreturn system call. A signal |
| 94 | * handler may call the rt_sigreturn() function to directly return to a given |
| 95 | * sigcontext. However, the user-level sigreturn() needs to do much more than |
| 96 | * calling the rt_sigreturn() system call as it needs to unwind the stack to |
| 97 | * restore preserved registers that may have been saved on the signal handler's |
| 98 | * call stack. |
| 99 | */ |
| 100 | |
| 101 | #define SIGTRAMP_SAVES \ |
| 102 | .unwabi 3, 's'; /* mark this as a sigtramp handler (saves scratch regs) */ \ |
| 103 | .unwabi @svr4, 's'; /* backwards compatibility with old unwinders (remove in v2.7) */ \ |
| 104 | .savesp ar.unat, UNAT_OFF+SIGCONTEXT_OFF; \ |
| 105 | .savesp ar.fpsr, FPSR_OFF+SIGCONTEXT_OFF; \ |
| 106 | .savesp pr, PR_OFF+SIGCONTEXT_OFF; \ |
| 107 | .savesp rp, RP_OFF+SIGCONTEXT_OFF; \ |
| 108 | .savesp ar.pfs, CFM_OFF+SIGCONTEXT_OFF; \ |
| 109 | .vframesp SP_OFF+SIGCONTEXT_OFF |
| 110 | |
| 111 | GLOBAL_ENTRY(__kernel_sigtramp) |
| 112 | // describe the state that is active when we get here: |
| 113 | .prologue |
| 114 | SIGTRAMP_SAVES |
| 115 | .body |
| 116 | |
| 117 | .label_state 1 |
| 118 | |
| 119 | adds base0=SIGHANDLER_OFF,sp |
| 120 | adds base1=RBS_BASE_OFF+SIGCONTEXT_OFF,sp |
| 121 | br.call.sptk.many rp=1f |
| 122 | 1: |
| 123 | ld8 r17=[base0],(ARG0_OFF-SIGHANDLER_OFF) // get pointer to signal handler's plabel |
| 124 | ld8 r15=[base1] // get address of new RBS base (or NULL) |
| 125 | cover // push args in interrupted frame onto backing store |
| 126 | ;; |
| 127 | cmp.ne p1,p0=r15,r0 // do we need to switch rbs? (note: pr is saved by kernel) |
| 128 | mov.m r9=ar.bsp // fetch ar.bsp |
| 129 | .spillsp.p p1, ar.rnat, RNAT_OFF+SIGCONTEXT_OFF |
| 130 | (p1) br.cond.spnt setup_rbs // yup -> (clobbers p8, r14-r16, and r18-r20) |
| 131 | back_from_setup_rbs: |
| 132 | alloc r8=ar.pfs,0,0,3,0 |
| 133 | ld8 out0=[base0],16 // load arg0 (signum) |
| 134 | adds base1=(ARG1_OFF-(RBS_BASE_OFF+SIGCONTEXT_OFF)),base1 |
| 135 | ;; |
| 136 | ld8 out1=[base1] // load arg1 (siginfop) |
| 137 | ld8 r10=[r17],8 // get signal handler entry point |
| 138 | ;; |
| 139 | ld8 out2=[base0] // load arg2 (sigcontextp) |
| 140 | ld8 gp=[r17] // get signal handler's global pointer |
| 141 | adds base0=(BSP_OFF+SIGCONTEXT_OFF),sp |
| 142 | ;; |
| 143 | .spillsp ar.bsp, BSP_OFF+SIGCONTEXT_OFF |
| 144 | st8 [base0]=r9 // save sc_ar_bsp |
| 145 | adds base0=(FR6_OFF+SIGCONTEXT_OFF),sp |
| 146 | adds base1=(FR6_OFF+16+SIGCONTEXT_OFF),sp |
| 147 | ;; |
| 148 | stf.spill [base0]=f6,32 |
| 149 | stf.spill [base1]=f7,32 |
| 150 | ;; |
| 151 | stf.spill [base0]=f8,32 |
| 152 | stf.spill [base1]=f9,32 |
| 153 | mov b6=r10 |
| 154 | ;; |
| 155 | stf.spill [base0]=f10,32 |
| 156 | stf.spill [base1]=f11,32 |
| 157 | ;; |
| 158 | stf.spill [base0]=f12,32 |
| 159 | stf.spill [base1]=f13,32 |
| 160 | ;; |
| 161 | stf.spill [base0]=f14,32 |
| 162 | stf.spill [base1]=f15,32 |
| 163 | br.call.sptk.many rp=b6 // call the signal handler |
| 164 | .ret0: adds base0=(BSP_OFF+SIGCONTEXT_OFF),sp |
| 165 | ;; |
| 166 | ld8 r15=[base0] // fetch sc_ar_bsp |
| 167 | mov r14=ar.bsp |
| 168 | ;; |
| 169 | cmp.ne p1,p0=r14,r15 // do we need to restore the rbs? |
| 170 | (p1) br.cond.spnt restore_rbs // yup -> (clobbers r14-r18, f6 & f7) |
| 171 | ;; |
| 172 | back_from_restore_rbs: |
| 173 | adds base0=(FR6_OFF+SIGCONTEXT_OFF),sp |
| 174 | adds base1=(FR6_OFF+16+SIGCONTEXT_OFF),sp |
| 175 | ;; |
| 176 | ldf.fill f6=[base0],32 |
| 177 | ldf.fill f7=[base1],32 |
| 178 | ;; |
| 179 | ldf.fill f8=[base0],32 |
| 180 | ldf.fill f9=[base1],32 |
| 181 | ;; |
| 182 | ldf.fill f10=[base0],32 |
| 183 | ldf.fill f11=[base1],32 |
| 184 | ;; |
| 185 | ldf.fill f12=[base0],32 |
| 186 | ldf.fill f13=[base1],32 |
| 187 | ;; |
| 188 | ldf.fill f14=[base0],32 |
| 189 | ldf.fill f15=[base1],32 |
| 190 | mov r15=__NR_rt_sigreturn |
| 191 | .restore sp // pop .prologue |
| 192 | break __BREAK_SYSCALL |
| 193 | |
| 194 | .prologue |
| 195 | SIGTRAMP_SAVES |
| 196 | setup_rbs: |
| 197 | mov ar.rsc=0 // put RSE into enforced lazy mode |
| 198 | ;; |
| 199 | .save ar.rnat, r19 |
| 200 | mov r19=ar.rnat // save RNaT before switching backing store area |
| 201 | adds r14=(RNAT_OFF+SIGCONTEXT_OFF),sp |
| 202 | |
| 203 | mov r18=ar.bspstore |
| 204 | mov ar.bspstore=r15 // switch over to new register backing store area |
| 205 | ;; |
| 206 | |
| 207 | .spillsp ar.rnat, RNAT_OFF+SIGCONTEXT_OFF |
| 208 | st8 [r14]=r19 // save sc_ar_rnat |
| 209 | .body |
| 210 | mov.m r16=ar.bsp // sc_loadrs <- (new bsp - new bspstore) << 16 |
| 211 | adds r14=(LOADRS_OFF+SIGCONTEXT_OFF),sp |
| 212 | ;; |
| 213 | invala |
| 214 | sub r15=r16,r15 |
| 215 | extr.u r20=r18,3,6 |
| 216 | ;; |
| 217 | mov ar.rsc=0xf // set RSE into eager mode, pl 3 |
| 218 | cmp.eq p8,p0=63,r20 |
| 219 | shl r15=r15,16 |
| 220 | ;; |
| 221 | st8 [r14]=r15 // save sc_loadrs |
| 222 | (p8) st8 [r18]=r19 // if bspstore points at RNaT slot, store RNaT there now |
| 223 | .restore sp // pop .prologue |
| 224 | br.cond.sptk back_from_setup_rbs |
| 225 | |
| 226 | .prologue |
| 227 | SIGTRAMP_SAVES |
| 228 | .spillsp ar.rnat, RNAT_OFF+SIGCONTEXT_OFF |
| 229 | .body |
| 230 | restore_rbs: |
| 231 | // On input: |
| 232 | // r14 = bsp1 (bsp at the time of return from signal handler) |
| 233 | // r15 = bsp0 (bsp at the time the signal occurred) |
| 234 | // |
| 235 | // Here, we need to calculate bspstore0, the value that ar.bspstore needs |
| 236 | // to be set to, based on bsp0 and the size of the dirty partition on |
| 237 | // the alternate stack (sc_loadrs >> 16). This can be done with the |
| 238 | // following algorithm: |
| 239 | // |
| 240 | // bspstore0 = rse_skip_regs(bsp0, -rse_num_regs(bsp1 - (loadrs >> 19), bsp1)); |
| 241 | // |
| 242 | // This is what the code below does. |
| 243 | // |
| 244 | alloc r2=ar.pfs,0,0,0,0 // alloc null frame |
| 245 | adds r16=(LOADRS_OFF+SIGCONTEXT_OFF),sp |
| 246 | adds r18=(RNAT_OFF+SIGCONTEXT_OFF),sp |
| 247 | ;; |
| 248 | ld8 r17=[r16] |
| 249 | ld8 r16=[r18] // get new rnat |
| 250 | extr.u r18=r15,3,6 // r18 <- rse_slot_num(bsp0) |
| 251 | ;; |
| 252 | mov ar.rsc=r17 // put RSE into enforced lazy mode |
| 253 | shr.u r17=r17,16 |
| 254 | ;; |
| 255 | sub r14=r14,r17 // r14 (bspstore1) <- bsp1 - (sc_loadrs >> 16) |
| 256 | shr.u r17=r17,3 // r17 <- (sc_loadrs >> 19) |
| 257 | ;; |
| 258 | loadrs // restore dirty partition |
| 259 | extr.u r14=r14,3,6 // r14 <- rse_slot_num(bspstore1) |
| 260 | ;; |
| 261 | add r14=r14,r17 // r14 <- rse_slot_num(bspstore1) + (sc_loadrs >> 19) |
| 262 | ;; |
| 263 | shr.u r14=r14,6 // r14 <- (rse_slot_num(bspstore1) + (sc_loadrs >> 19))/0x40 |
| 264 | ;; |
| 265 | sub r14=r14,r17 // r14 <- -rse_num_regs(bspstore1, bsp1) |
| 266 | movl r17=0x8208208208208209 |
| 267 | ;; |
| 268 | add r18=r18,r14 // r18 (delta) <- rse_slot_num(bsp0) - rse_num_regs(bspstore1,bsp1) |
| 269 | setf.sig f7=r17 |
| 270 | cmp.lt p7,p0=r14,r0 // p7 <- (r14 < 0)? |
| 271 | ;; |
| 272 | (p7) adds r18=-62,r18 // delta -= 62 |
| 273 | ;; |
| 274 | setf.sig f6=r18 |
| 275 | ;; |
| 276 | xmpy.h f6=f6,f7 |
| 277 | ;; |
| 278 | getf.sig r17=f6 |
| 279 | ;; |
| 280 | add r17=r17,r18 |
| 281 | shr r18=r18,63 |
| 282 | ;; |
| 283 | shr r17=r17,5 |
| 284 | ;; |
| 285 | sub r17=r17,r18 // r17 = delta/63 |
| 286 | ;; |
| 287 | add r17=r14,r17 // r17 <- delta/63 - rse_num_regs(bspstore1, bsp1) |
| 288 | ;; |
| 289 | shladd r15=r17,3,r15 // r15 <- bsp0 + 8*(delta/63 - rse_num_regs(bspstore1, bsp1)) |
| 290 | ;; |
| 291 | mov ar.bspstore=r15 // switch back to old register backing store area |
| 292 | ;; |
| 293 | mov ar.rnat=r16 // restore RNaT |
| 294 | mov ar.rsc=0xf // (will be restored later on from sc_ar_rsc) |
| 295 | // invala not necessary as that will happen when returning to user-mode |
| 296 | br.cond.sptk back_from_restore_rbs |
| 297 | END(__kernel_sigtramp) |
Isaku Yamahata | 53129c5 | 2009-03-04 21:05:44 +0900 | [diff] [blame] | 298 | |
| 299 | /* |
| 300 | * On entry: |
| 301 | * r11 = saved ar.pfs |
| 302 | * r15 = system call # |
| 303 | * b0 = saved return address |
| 304 | * b6 = return address |
| 305 | * On exit: |
| 306 | * r11 = saved ar.pfs |
| 307 | * r15 = system call # |
| 308 | * b0 = saved return address |
| 309 | * all other "scratch" registers: undefined |
| 310 | * all "preserved" registers: same as on entry |
| 311 | */ |
| 312 | |
| 313 | GLOBAL_ENTRY(__kernel_syscall_via_epc) |
| 314 | .prologue |
| 315 | .altrp b6 |
| 316 | .body |
| 317 | { |
| 318 | /* |
| 319 | * Note: the kernel cannot assume that the first two instructions in this |
| 320 | * bundle get executed. The remaining code must be safe even if |
| 321 | * they do not get executed. |
| 322 | */ |
| 323 | adds r17=-1024,r15 // A |
| 324 | mov r10=0 // A default to successful syscall execution |
| 325 | epc // B causes split-issue |
| 326 | } |
| 327 | ;; |
Isaku Yamahata | c431251 | 2009-03-04 21:05:45 +0900 | [diff] [blame] | 328 | RSM_PSR_BE_I(r20, r22) // M2 (5 cyc to srlz.d) |
Isaku Yamahata | 53129c5 | 2009-03-04 21:05:44 +0900 | [diff] [blame] | 329 | LOAD_FSYSCALL_TABLE(r14) // X |
| 330 | ;; |
| 331 | mov r16=IA64_KR(CURRENT) // M2 (12 cyc) |
| 332 | shladd r18=r17,3,r14 // A |
| 333 | mov r19=NR_syscalls-1 // A |
| 334 | ;; |
| 335 | lfetch [r18] // M0|1 |
Isaku Yamahata | c431251 | 2009-03-04 21:05:45 +0900 | [diff] [blame] | 336 | MOV_FROM_PSR(p0, r29, r8) // M2 (12 cyc) |
Isaku Yamahata | 53129c5 | 2009-03-04 21:05:44 +0900 | [diff] [blame] | 337 | // If r17 is a NaT, p6 will be zero |
| 338 | cmp.geu p6,p7=r19,r17 // A (sysnr > 0 && sysnr < 1024+NR_syscalls)? |
| 339 | ;; |
| 340 | mov r21=ar.fpsr // M2 (12 cyc) |
| 341 | tnat.nz p10,p9=r15 // I0 |
| 342 | mov.i r26=ar.pfs // I0 (would stall anyhow due to srlz.d...) |
| 343 | ;; |
| 344 | srlz.d // M0 (forces split-issue) ensure PSR.BE==0 |
| 345 | (p6) ld8 r18=[r18] // M0|1 |
| 346 | nop.i 0 |
| 347 | ;; |
| 348 | nop.m 0 |
| 349 | (p6) tbit.z.unc p8,p0=r18,0 // I0 (dual-issues with "mov b7=r18"!) |
| 350 | nop.i 0 |
| 351 | ;; |
Isaku Yamahata | c431251 | 2009-03-04 21:05:45 +0900 | [diff] [blame] | 352 | SSM_PSR_I(p8, p14, r25) |
Isaku Yamahata | 53129c5 | 2009-03-04 21:05:44 +0900 | [diff] [blame] | 353 | (p6) mov b7=r18 // I0 |
| 354 | (p8) br.dptk.many b7 // B |
| 355 | |
| 356 | mov r27=ar.rsc // M2 (12 cyc) |
| 357 | /* |
| 358 | * brl.cond doesn't work as intended because the linker would convert this branch |
| 359 | * into a branch to a PLT. Perhaps there will be a way to avoid this with some |
| 360 | * future version of the linker. In the meantime, we just use an indirect branch |
| 361 | * instead. |
| 362 | */ |
| 363 | #ifdef CONFIG_ITANIUM |
| 364 | (p6) add r14=-8,r14 // r14 <- addr of fsys_bubble_down entry |
| 365 | ;; |
| 366 | (p6) ld8 r14=[r14] // r14 <- fsys_bubble_down |
| 367 | ;; |
| 368 | (p6) mov b7=r14 |
| 369 | (p6) br.sptk.many b7 |
| 370 | #else |
| 371 | BRL_COND_FSYS_BUBBLE_DOWN(p6) |
| 372 | #endif |
Isaku Yamahata | c431251 | 2009-03-04 21:05:45 +0900 | [diff] [blame] | 373 | SSM_PSR_I(p0, p14, r10) |
Isaku Yamahata | 53129c5 | 2009-03-04 21:05:44 +0900 | [diff] [blame] | 374 | mov r10=-1 |
| 375 | (p10) mov r8=EINVAL |
| 376 | (p9) mov r8=ENOSYS |
| 377 | FSYS_RETURN |
Isaku Yamahata | c431251 | 2009-03-04 21:05:45 +0900 | [diff] [blame] | 378 | |
| 379 | #ifdef CONFIG_PARAVIRT |
| 380 | /* |
| 381 | * padd to make the size of this symbol constant |
| 382 | * independent of paravirtualization. |
| 383 | */ |
| 384 | .align PAGE_SIZE / 8 |
| 385 | #endif |
Isaku Yamahata | 53129c5 | 2009-03-04 21:05:44 +0900 | [diff] [blame] | 386 | END(__kernel_syscall_via_epc) |