blob: 21dbe6bdb8edc661d729f656a02a5f1cf01a18c1 [file] [log] [blame]
Chris Zankel5a0015d2005-06-23 22:01:16 -07001/*
2 * arch/xtensa/kernel/entry.S
3 *
4 * Low-level exception handling
5 *
6 * This file is subject to the terms and conditions of the GNU General Public
7 * License. See the file "COPYING" in the main directory of this archive
8 * for more details.
9 *
Marc Gauthier2d1c6452013-01-05 04:57:17 +040010 * Copyright (C) 2004 - 2008 by Tensilica Inc.
Chris Zankel5a0015d2005-06-23 22:01:16 -070011 *
12 * Chris Zankel <chris@zankel.net>
13 *
14 */
15
16#include <linux/linkage.h>
Sam Ravnborg0013a852005-09-09 20:57:26 +020017#include <asm/asm-offsets.h>
Chris Zankel5a0015d2005-06-23 22:01:16 -070018#include <asm/processor.h>
Chris Zankel4573e392010-05-02 01:05:13 -070019#include <asm/coprocessor.h>
Chris Zankel5a0015d2005-06-23 22:01:16 -070020#include <asm/thread_info.h>
21#include <asm/uaccess.h>
22#include <asm/unistd.h>
23#include <asm/ptrace.h>
24#include <asm/current.h>
25#include <asm/pgtable.h>
26#include <asm/page.h>
27#include <asm/signal.h>
Chris Zankel173d6682006-12-10 02:18:48 -080028#include <asm/tlbflush.h>
Chris Zankel367b8112008-11-06 06:40:46 -080029#include <variant/tie-asm.h>
Chris Zankel5a0015d2005-06-23 22:01:16 -070030
31/* Unimplemented features. */
32
Chris Zankel5a0015d2005-06-23 22:01:16 -070033#undef KERNEL_STACK_OVERFLOW_CHECK
Chris Zankel5a0015d2005-06-23 22:01:16 -070034
35/* Not well tested.
36 *
37 * - fast_coprocessor
38 */
39
40/*
41 * Macro to find first bit set in WINDOWBASE from the left + 1
42 *
43 * 100....0 -> 1
44 * 010....0 -> 2
45 * 000....1 -> WSBITS
46 */
47
48 .macro ffs_ws bit mask
49
50#if XCHAL_HAVE_NSA
51 nsau \bit, \mask # 32-WSBITS ... 31 (32 iff 0)
52 addi \bit, \bit, WSBITS - 32 + 1 # uppest bit set -> return 1
53#else
54 movi \bit, WSBITS
55#if WSBITS > 16
56 _bltui \mask, 0x10000, 99f
57 addi \bit, \bit, -16
58 extui \mask, \mask, 16, 16
59#endif
60#if WSBITS > 8
6199: _bltui \mask, 0x100, 99f
62 addi \bit, \bit, -8
63 srli \mask, \mask, 8
64#endif
6599: _bltui \mask, 0x10, 99f
66 addi \bit, \bit, -4
67 srli \mask, \mask, 4
6899: _bltui \mask, 0x4, 99f
69 addi \bit, \bit, -2
70 srli \mask, \mask, 2
7199: _bltui \mask, 0x2, 99f
72 addi \bit, \bit, -1
7399:
74
75#endif
76 .endm
77
78/* ----------------- DEFAULT FIRST LEVEL EXCEPTION HANDLERS ----------------- */
79
80/*
81 * First-level exception handler for user exceptions.
82 * Save some special registers, extra states and all registers in the AR
83 * register file that were in use in the user task, and jump to the common
84 * exception code.
85 * We save SAR (used to calculate WMASK), and WB and WS (we don't have to
86 * save them for kernel exceptions).
87 *
88 * Entry condition for user_exception:
89 *
90 * a0: trashed, original value saved on stack (PT_AREG0)
91 * a1: a1
92 * a2: new stack pointer, original value in depc
Max Filippov99d50402013-07-03 20:23:28 +040093 * a3: a3
Chris Zankel5a0015d2005-06-23 22:01:16 -070094 * depc: a2, original value saved on stack (PT_DEPC)
Max Filippov99d50402013-07-03 20:23:28 +040095 * excsave1: dispatch table
Chris Zankel5a0015d2005-06-23 22:01:16 -070096 *
97 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
98 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
99 *
100 * Entry condition for _user_exception:
101 *
102 * a0-a3 and depc have been saved to PT_AREG0...PT_AREG3 and PT_DEPC
103 * excsave has been restored, and
104 * stack pointer (a1) has been set.
105 *
Daniel Mack3ad2f3f2010-02-03 08:01:28 +0800106 * Note: _user_exception might be at an odd address. Don't use call0..call12
Chris Zankel5a0015d2005-06-23 22:01:16 -0700107 */
108
109ENTRY(user_exception)
110
Max Filippov99d50402013-07-03 20:23:28 +0400111 /* Save a1, a2, a3, and set SP. */
Chris Zankel5a0015d2005-06-23 22:01:16 -0700112
Max Filippovbc5378f2012-10-15 03:55:38 +0400113 rsr a0, depc
Chris Zankel5a0015d2005-06-23 22:01:16 -0700114 s32i a1, a2, PT_AREG1
115 s32i a0, a2, PT_AREG2
116 s32i a3, a2, PT_AREG3
117 mov a1, a2
118
119 .globl _user_exception
120_user_exception:
121
122 /* Save SAR and turn off single stepping */
123
124 movi a2, 0
Max Filippovbc5378f2012-10-15 03:55:38 +0400125 rsr a3, sar
126 xsr a2, icountlevel
Chris Zankel5a0015d2005-06-23 22:01:16 -0700127 s32i a3, a1, PT_SAR
Chris Zankel29c4dfd2007-05-31 17:49:32 -0700128 s32i a2, a1, PT_ICOUNTLEVEL
Chris Zankel5a0015d2005-06-23 22:01:16 -0700129
Chris Zankelc50842d2013-02-23 19:35:57 -0800130#if XCHAL_HAVE_THREADPTR
131 rur a2, threadptr
132 s32i a2, a1, PT_THREADPTR
133#endif
134
Chris Zankel5a0015d2005-06-23 22:01:16 -0700135 /* Rotate ws so that the current windowbase is at bit0. */
136 /* Assume ws = xxwww1yyyy. Rotate ws right, so that a2 = yyyyxxwww1 */
137
Max Filippovbc5378f2012-10-15 03:55:38 +0400138 rsr a2, windowbase
139 rsr a3, windowstart
Chris Zankel5a0015d2005-06-23 22:01:16 -0700140 ssr a2
141 s32i a2, a1, PT_WINDOWBASE
142 s32i a3, a1, PT_WINDOWSTART
143 slli a2, a3, 32-WSBITS
144 src a2, a3, a2
145 srli a2, a2, 32-WSBITS
146 s32i a2, a1, PT_WMASK # needed for restoring registers
147
148 /* Save only live registers. */
149
150 _bbsi.l a2, 1, 1f
151 s32i a4, a1, PT_AREG4
152 s32i a5, a1, PT_AREG5
153 s32i a6, a1, PT_AREG6
154 s32i a7, a1, PT_AREG7
155 _bbsi.l a2, 2, 1f
156 s32i a8, a1, PT_AREG8
157 s32i a9, a1, PT_AREG9
158 s32i a10, a1, PT_AREG10
159 s32i a11, a1, PT_AREG11
160 _bbsi.l a2, 3, 1f
161 s32i a12, a1, PT_AREG12
162 s32i a13, a1, PT_AREG13
163 s32i a14, a1, PT_AREG14
164 s32i a15, a1, PT_AREG15
165 _bnei a2, 1, 1f # only one valid frame?
166
167 /* Only one valid frame, skip saving regs. */
168
169 j 2f
170
171 /* Save the remaining registers.
172 * We have to save all registers up to the first '1' from
173 * the right, except the current frame (bit 0).
174 * Assume a2 is: 001001000110001
Chris Zankel66569202007-08-22 10:14:51 -0700175 * All register frames starting from the top field to the marked '1'
Chris Zankel5a0015d2005-06-23 22:01:16 -0700176 * must be saved.
177 */
178
1791: addi a3, a2, -1 # eliminate '1' in bit 0: yyyyxxww0
180 neg a3, a3 # yyyyxxww0 -> YYYYXXWW1+1
181 and a3, a3, a2 # max. only one bit is set
182
183 /* Find number of frames to save */
184
185 ffs_ws a0, a3 # number of frames to the '1' from left
186
187 /* Store information into WMASK:
188 * bits 0..3: xxx1 masked lower 4 bits of the rotated windowstart,
189 * bits 4...: number of valid 4-register frames
190 */
191
192 slli a3, a0, 4 # number of frames to save in bits 8..4
193 extui a2, a2, 0, 4 # mask for the first 16 registers
194 or a2, a3, a2
195 s32i a2, a1, PT_WMASK # needed when we restore the reg-file
196
197 /* Save 4 registers at a time */
198
1991: rotw -1
200 s32i a0, a5, PT_AREG_END - 16
201 s32i a1, a5, PT_AREG_END - 12
202 s32i a2, a5, PT_AREG_END - 8
203 s32i a3, a5, PT_AREG_END - 4
204 addi a0, a4, -1
205 addi a1, a5, -16
206 _bnez a0, 1b
207
208 /* WINDOWBASE still in SAR! */
209
Max Filippovbc5378f2012-10-15 03:55:38 +0400210 rsr a2, sar # original WINDOWBASE
Chris Zankel5a0015d2005-06-23 22:01:16 -0700211 movi a3, 1
212 ssl a2
213 sll a3, a3
Max Filippovbc5378f2012-10-15 03:55:38 +0400214 wsr a3, windowstart # set corresponding WINDOWSTART bit
215 wsr a2, windowbase # and WINDOWSTART
Chris Zankel5a0015d2005-06-23 22:01:16 -0700216 rsync
217
218 /* We are back to the original stack pointer (a1) */
219
Chris Zankelc658eac2008-02-12 13:17:07 -08002202: /* Now, jump to the common exception handler. */
Chris Zankel5a0015d2005-06-23 22:01:16 -0700221
222 j common_exception
223
Chris Zankeld1538c42012-11-16 16:16:20 -0800224ENDPROC(user_exception)
Chris Zankel5a0015d2005-06-23 22:01:16 -0700225
226/*
227 * First-level exit handler for kernel exceptions
228 * Save special registers and the live window frame.
229 * Note: Even though we changes the stack pointer, we don't have to do a
230 * MOVSP here, as we do that when we return from the exception.
231 * (See comment in the kernel exception exit code)
232 *
233 * Entry condition for kernel_exception:
234 *
235 * a0: trashed, original value saved on stack (PT_AREG0)
236 * a1: a1
237 * a2: new stack pointer, original in DEPC
Max Filippov99d50402013-07-03 20:23:28 +0400238 * a3: a3
Chris Zankel5a0015d2005-06-23 22:01:16 -0700239 * depc: a2, original value saved on stack (PT_DEPC)
Max Filippov99d50402013-07-03 20:23:28 +0400240 * excsave_1: dispatch table
Chris Zankel5a0015d2005-06-23 22:01:16 -0700241 *
242 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
243 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
244 *
245 * Entry condition for _kernel_exception:
246 *
247 * a0-a3 and depc have been saved to PT_AREG0...PT_AREG3 and PT_DEPC
248 * excsave has been restored, and
249 * stack pointer (a1) has been set.
250 *
Daniel Mack3ad2f3f2010-02-03 08:01:28 +0800251 * Note: _kernel_exception might be at an odd address. Don't use call0..call12
Chris Zankel5a0015d2005-06-23 22:01:16 -0700252 */
253
254ENTRY(kernel_exception)
255
Max Filippov99d50402013-07-03 20:23:28 +0400256 /* Save a1, a2, a3, and set SP. */
Chris Zankel5a0015d2005-06-23 22:01:16 -0700257
Max Filippovbc5378f2012-10-15 03:55:38 +0400258 rsr a0, depc # get a2
Chris Zankel5a0015d2005-06-23 22:01:16 -0700259 s32i a1, a2, PT_AREG1
260 s32i a0, a2, PT_AREG2
261 s32i a3, a2, PT_AREG3
262 mov a1, a2
263
264 .globl _kernel_exception
265_kernel_exception:
266
267 /* Save SAR and turn off single stepping */
268
269 movi a2, 0
Max Filippovbc5378f2012-10-15 03:55:38 +0400270 rsr a3, sar
271 xsr a2, icountlevel
Chris Zankel5a0015d2005-06-23 22:01:16 -0700272 s32i a3, a1, PT_SAR
Chris Zankel29c4dfd2007-05-31 17:49:32 -0700273 s32i a2, a1, PT_ICOUNTLEVEL
Chris Zankel5a0015d2005-06-23 22:01:16 -0700274
275 /* Rotate ws so that the current windowbase is at bit0. */
276 /* Assume ws = xxwww1yyyy. Rotate ws right, so that a2 = yyyyxxwww1 */
277
Max Filippovbc5378f2012-10-15 03:55:38 +0400278 rsr a2, windowbase # don't need to save these, we only
279 rsr a3, windowstart # need shifted windowstart: windowmask
Chris Zankel5a0015d2005-06-23 22:01:16 -0700280 ssr a2
281 slli a2, a3, 32-WSBITS
282 src a2, a3, a2
283 srli a2, a2, 32-WSBITS
284 s32i a2, a1, PT_WMASK # needed for kernel_exception_exit
285
286 /* Save only the live window-frame */
287
288 _bbsi.l a2, 1, 1f
289 s32i a4, a1, PT_AREG4
290 s32i a5, a1, PT_AREG5
291 s32i a6, a1, PT_AREG6
292 s32i a7, a1, PT_AREG7
293 _bbsi.l a2, 2, 1f
294 s32i a8, a1, PT_AREG8
295 s32i a9, a1, PT_AREG9
296 s32i a10, a1, PT_AREG10
297 s32i a11, a1, PT_AREG11
298 _bbsi.l a2, 3, 1f
299 s32i a12, a1, PT_AREG12
300 s32i a13, a1, PT_AREG13
301 s32i a14, a1, PT_AREG14
302 s32i a15, a1, PT_AREG15
303
3041:
305
306#ifdef KERNEL_STACK_OVERFLOW_CHECK
307
308 /* Stack overflow check, for debugging */
309 extui a2, a1, TASK_SIZE_BITS,XX
310 movi a3, SIZE??
311 _bge a2, a3, out_of_stack_panic
312
313#endif
314
315/*
316 * This is the common exception handler.
317 * We get here from the user exception handler or simply by falling through
318 * from the kernel exception handler.
319 * Save the remaining special registers, switch to kernel mode, and jump
320 * to the second-level exception handler.
321 *
322 */
323
324common_exception:
325
Chris Zankel29c4dfd2007-05-31 17:49:32 -0700326 /* Save some registers, disable loops and clear the syscall flag. */
Chris Zankel5a0015d2005-06-23 22:01:16 -0700327
Max Filippovbc5378f2012-10-15 03:55:38 +0400328 rsr a2, debugcause
329 rsr a3, epc1
Chris Zankel5a0015d2005-06-23 22:01:16 -0700330 s32i a2, a1, PT_DEBUGCAUSE
331 s32i a3, a1, PT_PC
332
Chris Zankel29c4dfd2007-05-31 17:49:32 -0700333 movi a2, -1
Max Filippovbc5378f2012-10-15 03:55:38 +0400334 rsr a3, excvaddr
Chris Zankel29c4dfd2007-05-31 17:49:32 -0700335 s32i a2, a1, PT_SYSCALL
Chris Zankel5a0015d2005-06-23 22:01:16 -0700336 movi a2, 0
337 s32i a3, a1, PT_EXCVADDR
Max Filippovbc5378f2012-10-15 03:55:38 +0400338 xsr a2, lcount
Chris Zankel5a0015d2005-06-23 22:01:16 -0700339 s32i a2, a1, PT_LCOUNT
340
341 /* It is now save to restore the EXC_TABLE_FIXUP variable. */
342
Max Filippovbc5378f2012-10-15 03:55:38 +0400343 rsr a0, exccause
Chris Zankel5a0015d2005-06-23 22:01:16 -0700344 movi a3, 0
Max Filippovbc5378f2012-10-15 03:55:38 +0400345 rsr a2, excsave1
Chris Zankel5a0015d2005-06-23 22:01:16 -0700346 s32i a0, a1, PT_EXCCAUSE
347 s32i a3, a2, EXC_TABLE_FIXUP
348
349 /* All unrecoverable states are saved on stack, now, and a1 is valid,
350 * so we can allow exceptions and interrupts (*) again.
351 * Set PS(EXCM = 0, UM = 0, RING = 0, OWB = 0, WOE = 1, INTLEVEL = X)
352 *
Max Filippov895666a2013-03-26 02:51:43 +0400353 * (*) We only allow interrupts if they were previously enabled and
354 * we're not handling an IRQ
Chris Zankel5a0015d2005-06-23 22:01:16 -0700355 */
356
Max Filippovbc5378f2012-10-15 03:55:38 +0400357 rsr a3, ps
Max Filippov895666a2013-03-26 02:51:43 +0400358 addi a0, a0, -EXCCAUSE_LEVEL1_INTERRUPT
359 movi a2, LOCKLEVEL
Marc Gauthier2d1c6452013-01-05 04:57:17 +0400360 extui a3, a3, PS_INTLEVEL_SHIFT, PS_INTLEVEL_WIDTH
361 # a3 = PS.INTLEVEL
Max Filippov895666a2013-03-26 02:51:43 +0400362 moveqz a3, a2, a0 # a3 = LOCKLEVEL iff interrupt
Chris Zankel173d6682006-12-10 02:18:48 -0800363 movi a2, 1 << PS_WOE_BIT
Chris Zankel5a0015d2005-06-23 22:01:16 -0700364 or a3, a3, a2
Max Filippovbc5378f2012-10-15 03:55:38 +0400365 rsr a0, exccause
366 xsr a3, ps
Chris Zankel5a0015d2005-06-23 22:01:16 -0700367
368 s32i a3, a1, PT_PS # save ps
369
Max Filippovbc5378f2012-10-15 03:55:38 +0400370 /* Save lbeg, lend */
Chris Zankel5a0015d2005-06-23 22:01:16 -0700371
Max Filippovbc5378f2012-10-15 03:55:38 +0400372 rsr a2, lbeg
373 rsr a3, lend
Chris Zankel5a0015d2005-06-23 22:01:16 -0700374 s32i a2, a1, PT_LBEG
375 s32i a3, a1, PT_LEND
376
Max Filippov733536b2012-11-15 06:25:48 +0400377 /* Save SCOMPARE1 */
378
379#if XCHAL_HAVE_S32C1I
380 rsr a2, scompare1
381 s32i a2, a1, PT_SCOMPARE1
382#endif
383
Chris Zankelc658eac2008-02-12 13:17:07 -0800384 /* Save optional registers. */
385
386 save_xtregs_opt a1 a2 a4 a5 a6 a7 PT_XTREGS_OPT
387
Max Filippovc92931b2013-03-31 06:32:42 +0400388#ifdef CONFIG_TRACE_IRQFLAGS
389 l32i a4, a1, PT_DEPC
390 /* Double exception means we came here with an exception
391 * while PS.EXCM was set, i.e. interrupts disabled.
392 */
393 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
394 l32i a4, a1, PT_EXCCAUSE
395 bnei a4, EXCCAUSE_LEVEL1_INTERRUPT, 1f
396 /* We came here with an interrupt means interrupts were enabled
397 * and we've just disabled them.
398 */
399 movi a4, trace_hardirqs_off
400 callx4 a4
4011:
402#endif
403
Chris Zankel5a0015d2005-06-23 22:01:16 -0700404 /* Go to second-level dispatcher. Set up parameters to pass to the
405 * exception handler and call the exception handler.
406 */
407
Max Filippov99d50402013-07-03 20:23:28 +0400408 rsr a4, excsave1
Chris Zankel5a0015d2005-06-23 22:01:16 -0700409 mov a6, a1 # pass stack frame
410 mov a7, a0 # pass EXCCAUSE
411 addx4 a4, a0, a4
412 l32i a4, a4, EXC_TABLE_DEFAULT # load handler
413
414 /* Call the second-level handler */
415
416 callx4 a4
417
418 /* Jump here for exception exit */
danne6ffe172008-05-21 17:43:50 -0700419 .global common_exception_return
Chris Zankel5a0015d2005-06-23 22:01:16 -0700420common_exception_return:
421
Max Filippovc92931b2013-03-31 06:32:42 +04004221:
Max Filippovaea8e7c2013-08-26 15:12:58 +0400423 rsil a2, LOCKLEVEL
Max Filippovc92931b2013-03-31 06:32:42 +0400424
Chris Zankel5a0015d2005-06-23 22:01:16 -0700425 /* Jump if we are returning from kernel exceptions. */
426
Max Filippovaea8e7c2013-08-26 15:12:58 +0400427 l32i a3, a1, PT_PS
Max Filippov16c5bec2013-08-26 15:16:23 +0400428 GET_THREAD_INFO(a2, a1)
429 l32i a4, a2, TI_FLAGS
430 _bbci.l a3, PS_UM_BIT, 6f
Chris Zankel5a0015d2005-06-23 22:01:16 -0700431
432 /* Specific to a user exception exit:
433 * We need to check some flags for signal handling and rescheduling,
434 * and have to restore WB and WS, extra states, and all registers
435 * in the register file that were in use in the user task.
Chris Zankele1088432008-01-22 00:45:25 -0800436 * Note that we don't disable interrupts here.
Chris Zankel5a0015d2005-06-23 22:01:16 -0700437 */
438
Chris Zankel5a0015d2005-06-23 22:01:16 -0700439 _bbsi.l a4, TIF_NEED_RESCHED, 3f
Al Viroa53bb242012-04-24 02:30:16 -0400440 _bbsi.l a4, TIF_NOTIFY_RESUME, 2f
Max Filippova99e07e2013-05-15 19:34:05 +0400441 _bbci.l a4, TIF_SIGPENDING, 5f
Chris Zankel5a0015d2005-06-23 22:01:16 -0700442
Al Viroa53bb242012-04-24 02:30:16 -04004432: l32i a4, a1, PT_DEPC
Chris Zankel5a0015d2005-06-23 22:01:16 -0700444 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 4f
Chris Zankel5a0015d2005-06-23 22:01:16 -0700445
Chris Zankele1088432008-01-22 00:45:25 -0800446 /* Call do_signal() */
447
Max Filippovaea8e7c2013-08-26 15:12:58 +0400448 rsil a2, 0
Al Viroa53bb242012-04-24 02:30:16 -0400449 movi a4, do_notify_resume # int do_notify_resume(struct pt_regs*)
Chris Zankel5a0015d2005-06-23 22:01:16 -0700450 mov a6, a1
Chris Zankel5a0015d2005-06-23 22:01:16 -0700451 callx4 a4
452 j 1b
453
Chris Zankele1088432008-01-22 00:45:25 -08004543: /* Reschedule */
Chris Zankel5a0015d2005-06-23 22:01:16 -0700455
Max Filippovaea8e7c2013-08-26 15:12:58 +0400456 rsil a2, 0
Chris Zankel5a0015d2005-06-23 22:01:16 -0700457 movi a4, schedule # void schedule (void)
458 callx4 a4
459 j 1b
460
Max Filippov16c5bec2013-08-26 15:16:23 +0400461#ifdef CONFIG_PREEMPT
4626:
463 _bbci.l a4, TIF_NEED_RESCHED, 4f
464
465 /* Check current_thread_info->preempt_count */
466
467 l32i a4, a2, TI_PRE_COUNT
468 bnez a4, 4f
469 movi a4, preempt_schedule_irq
470 callx4 a4
471 j 1b
472#endif
473
Max Filippova99e07e2013-05-15 19:34:05 +04004745:
475#ifdef CONFIG_DEBUG_TLB_SANITY
476 l32i a4, a1, PT_DEPC
477 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 4f
478 movi a4, check_tlb_sanity
479 callx4 a4
480#endif
Max Filippov16c5bec2013-08-26 15:16:23 +04004816:
Max Filippovaea8e7c2013-08-26 15:12:58 +04004824:
483#ifdef CONFIG_TRACE_IRQFLAGS
484 l32i a4, a1, PT_DEPC
485 /* Double exception means we came here with an exception
486 * while PS.EXCM was set, i.e. interrupts disabled.
487 */
488 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
489 l32i a4, a1, PT_EXCCAUSE
490 bnei a4, EXCCAUSE_LEVEL1_INTERRUPT, 1f
491 /* We came here with an interrupt means interrupts were enabled
492 * and we'll reenable them on return.
493 */
494 movi a4, trace_hardirqs_on
495 callx4 a4
4961:
497#endif
498 /* Restore optional registers. */
Chris Zankel5a0015d2005-06-23 22:01:16 -0700499
Chris Zankele1088432008-01-22 00:45:25 -0800500 load_xtregs_opt a1 a2 a4 a5 a6 a7 PT_XTREGS_OPT
501
Max Filippov733536b2012-11-15 06:25:48 +0400502 /* Restore SCOMPARE1 */
503
504#if XCHAL_HAVE_S32C1I
505 l32i a2, a1, PT_SCOMPARE1
506 wsr a2, scompare1
507#endif
Max Filippovbc5378f2012-10-15 03:55:38 +0400508 wsr a3, ps /* disable interrupts */
Chris Zankele1088432008-01-22 00:45:25 -0800509
510 _bbci.l a3, PS_UM_BIT, kernel_exception_exit
511
512user_exception_exit:
513
514 /* Restore the state of the task and return from the exception. */
Chris Zankel5a0015d2005-06-23 22:01:16 -0700515
Chris Zankel5a0015d2005-06-23 22:01:16 -0700516 /* Switch to the user thread WINDOWBASE. Save SP temporarily in DEPC */
517
518 l32i a2, a1, PT_WINDOWBASE
519 l32i a3, a1, PT_WINDOWSTART
Max Filippovbc5378f2012-10-15 03:55:38 +0400520 wsr a1, depc # use DEPC as temp storage
521 wsr a3, windowstart # restore WINDOWSTART
Chris Zankel5a0015d2005-06-23 22:01:16 -0700522 ssr a2 # preserve user's WB in the SAR
Max Filippovbc5378f2012-10-15 03:55:38 +0400523 wsr a2, windowbase # switch to user's saved WB
Chris Zankel5a0015d2005-06-23 22:01:16 -0700524 rsync
Max Filippovbc5378f2012-10-15 03:55:38 +0400525 rsr a1, depc # restore stack pointer
Chris Zankel5a0015d2005-06-23 22:01:16 -0700526 l32i a2, a1, PT_WMASK # register frames saved (in bits 4...9)
527 rotw -1 # we restore a4..a7
528 _bltui a6, 16, 1f # only have to restore current window?
529
530 /* The working registers are a0 and a3. We are restoring to
531 * a4..a7. Be careful not to destroy what we have just restored.
532 * Note: wmask has the format YYYYM:
533 * Y: number of registers saved in groups of 4
534 * M: 4 bit mask of first 16 registers
535 */
536
537 mov a2, a6
538 mov a3, a5
539
5402: rotw -1 # a0..a3 become a4..a7
541 addi a3, a7, -4*4 # next iteration
542 addi a2, a6, -16 # decrementing Y in WMASK
543 l32i a4, a3, PT_AREG_END + 0
544 l32i a5, a3, PT_AREG_END + 4
545 l32i a6, a3, PT_AREG_END + 8
546 l32i a7, a3, PT_AREG_END + 12
547 _bgeui a2, 16, 2b
548
549 /* Clear unrestored registers (don't leak anything to user-land */
550
Max Filippovbc5378f2012-10-15 03:55:38 +04005511: rsr a0, windowbase
552 rsr a3, sar
Chris Zankel5a0015d2005-06-23 22:01:16 -0700553 sub a3, a0, a3
554 beqz a3, 2f
555 extui a3, a3, 0, WBBITS
556
5571: rotw -1
558 addi a3, a7, -1
559 movi a4, 0
560 movi a5, 0
561 movi a6, 0
562 movi a7, 0
563 bgei a3, 1, 1b
564
565 /* We are back were we were when we started.
566 * Note: a2 still contains WMASK (if we've returned to the original
567 * frame where we had loaded a2), or at least the lower 4 bits
568 * (if we have restored WSBITS-1 frames).
569 */
570
Chris Zankelc50842d2013-02-23 19:35:57 -0800571#if XCHAL_HAVE_THREADPTR
572 l32i a3, a1, PT_THREADPTR
573 wur a3, threadptr
574#endif
575
Chris Zankel5a0015d2005-06-23 22:01:16 -07005762: j common_exception_exit
577
578 /* This is the kernel exception exit.
579 * We avoided to do a MOVSP when we entered the exception, but we
580 * have to do it here.
581 */
582
583kernel_exception_exit:
584
Chris Zankel5a0015d2005-06-23 22:01:16 -0700585 /* Check if we have to do a movsp.
586 *
587 * We only have to do a movsp if the previous window-frame has
588 * been spilled to the *temporary* exception stack instead of the
589 * task's stack. This is the case if the corresponding bit in
590 * WINDOWSTART for the previous window-frame was set before
591 * (not spilled) but is zero now (spilled).
592 * If this bit is zero, all other bits except the one for the
593 * current window frame are also zero. So, we can use a simple test:
594 * 'and' WINDOWSTART and WINDOWSTART-1:
595 *
596 * (XXXXXX1[0]* - 1) AND XXXXXX1[0]* = XXXXXX0[0]*
597 *
598 * The result is zero only if one bit was set.
599 *
600 * (Note: We might have gone through several task switches before
601 * we come back to the current task, so WINDOWBASE might be
602 * different from the time the exception occurred.)
603 */
604
605 /* Test WINDOWSTART before and after the exception.
606 * We actually have WMASK, so we only have to test if it is 1 or not.
607 */
608
609 l32i a2, a1, PT_WMASK
610 _beqi a2, 1, common_exception_exit # Spilled before exception,jump
611
612 /* Test WINDOWSTART now. If spilled, do the movsp */
613
Max Filippovbc5378f2012-10-15 03:55:38 +0400614 rsr a3, windowstart
Chris Zankel5a0015d2005-06-23 22:01:16 -0700615 addi a0, a3, -1
616 and a3, a3, a0
617 _bnez a3, common_exception_exit
618
619 /* Do a movsp (we returned from a call4, so we have at least a0..a7) */
620
621 addi a0, a1, -16
622 l32i a3, a0, 0
623 l32i a4, a0, 4
624 s32i a3, a1, PT_SIZE+0
625 s32i a4, a1, PT_SIZE+4
626 l32i a3, a0, 8
627 l32i a4, a0, 12
628 s32i a3, a1, PT_SIZE+8
629 s32i a4, a1, PT_SIZE+12
630
631 /* Common exception exit.
632 * We restore the special register and the current window frame, and
633 * return from the exception.
634 *
635 * Note: We expect a2 to hold PT_WMASK
636 */
637
638common_exception_exit:
639
Chris Zankelc658eac2008-02-12 13:17:07 -0800640 /* Restore address registers. */
641
Chris Zankel5a0015d2005-06-23 22:01:16 -0700642 _bbsi.l a2, 1, 1f
643 l32i a4, a1, PT_AREG4
644 l32i a5, a1, PT_AREG5
645 l32i a6, a1, PT_AREG6
646 l32i a7, a1, PT_AREG7
647 _bbsi.l a2, 2, 1f
648 l32i a8, a1, PT_AREG8
649 l32i a9, a1, PT_AREG9
650 l32i a10, a1, PT_AREG10
651 l32i a11, a1, PT_AREG11
652 _bbsi.l a2, 3, 1f
653 l32i a12, a1, PT_AREG12
654 l32i a13, a1, PT_AREG13
655 l32i a14, a1, PT_AREG14
656 l32i a15, a1, PT_AREG15
657
658 /* Restore PC, SAR */
659
6601: l32i a2, a1, PT_PC
661 l32i a3, a1, PT_SAR
Max Filippovbc5378f2012-10-15 03:55:38 +0400662 wsr a2, epc1
663 wsr a3, sar
Chris Zankel5a0015d2005-06-23 22:01:16 -0700664
665 /* Restore LBEG, LEND, LCOUNT */
666
667 l32i a2, a1, PT_LBEG
668 l32i a3, a1, PT_LEND
Max Filippovbc5378f2012-10-15 03:55:38 +0400669 wsr a2, lbeg
Chris Zankel5a0015d2005-06-23 22:01:16 -0700670 l32i a2, a1, PT_LCOUNT
Max Filippovbc5378f2012-10-15 03:55:38 +0400671 wsr a3, lend
672 wsr a2, lcount
Chris Zankel5a0015d2005-06-23 22:01:16 -0700673
Chris Zankel29c4dfd2007-05-31 17:49:32 -0700674 /* We control single stepping through the ICOUNTLEVEL register. */
675
676 l32i a2, a1, PT_ICOUNTLEVEL
677 movi a3, -2
Max Filippovbc5378f2012-10-15 03:55:38 +0400678 wsr a2, icountlevel
679 wsr a3, icount
Chris Zankel29c4dfd2007-05-31 17:49:32 -0700680
Chris Zankel5a0015d2005-06-23 22:01:16 -0700681 /* Check if it was double exception. */
682
683 l32i a0, a1, PT_DEPC
684 l32i a3, a1, PT_AREG3
Marc Gauthier2d1c6452013-01-05 04:57:17 +0400685 l32i a2, a1, PT_AREG2
Max Filippov895666a2013-03-26 02:51:43 +0400686 _bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
Marc Gauthier2d1c6452013-01-05 04:57:17 +0400687
Chris Zankel5a0015d2005-06-23 22:01:16 -0700688 /* Restore a0...a3 and return */
689
690 l32i a0, a1, PT_AREG0
691 l32i a1, a1, PT_AREG1
692 rfe
693
Max Filippov895666a2013-03-26 02:51:43 +04006941: wsr a0, depc
695 l32i a0, a1, PT_AREG0
696 l32i a1, a1, PT_AREG1
697 rfde
Chris Zankel5a0015d2005-06-23 22:01:16 -0700698
Chris Zankeld1538c42012-11-16 16:16:20 -0800699ENDPROC(kernel_exception)
700
Chris Zankel5a0015d2005-06-23 22:01:16 -0700701/*
702 * Debug exception handler.
703 *
704 * Currently, we don't support KGDB, so only user application can be debugged.
705 *
706 * When we get here, a0 is trashed and saved to excsave[debuglevel]
707 */
708
709ENTRY(debug_exception)
710
Max Filippovbc5378f2012-10-15 03:55:38 +0400711 rsr a0, SREG_EPS + XCHAL_DEBUGLEVEL
Chris Zankel173d6682006-12-10 02:18:48 -0800712 bbsi.l a0, PS_EXCM_BIT, 1f # exception mode
Chris Zankel5a0015d2005-06-23 22:01:16 -0700713
Max Filippovbc5378f2012-10-15 03:55:38 +0400714 /* Set EPC1 and EXCCAUSE */
Chris Zankel5a0015d2005-06-23 22:01:16 -0700715
Max Filippovbc5378f2012-10-15 03:55:38 +0400716 wsr a2, depc # save a2 temporarily
717 rsr a2, SREG_EPC + XCHAL_DEBUGLEVEL
718 wsr a2, epc1
Chris Zankel5a0015d2005-06-23 22:01:16 -0700719
720 movi a2, EXCCAUSE_MAPPED_DEBUG
Max Filippovbc5378f2012-10-15 03:55:38 +0400721 wsr a2, exccause
Chris Zankel5a0015d2005-06-23 22:01:16 -0700722
723 /* Restore PS to the value before the debug exc but with PS.EXCM set.*/
724
Chris Zankel173d6682006-12-10 02:18:48 -0800725 movi a2, 1 << PS_EXCM_BIT
Chris Zankel5a0015d2005-06-23 22:01:16 -0700726 or a2, a0, a2
727 movi a0, debug_exception # restore a3, debug jump vector
Max Filippovbc5378f2012-10-15 03:55:38 +0400728 wsr a2, ps
729 xsr a0, SREG_EXCSAVE + XCHAL_DEBUGLEVEL
Chris Zankel5a0015d2005-06-23 22:01:16 -0700730
731 /* Switch to kernel/user stack, restore jump vector, and save a0 */
732
Chris Zankel173d6682006-12-10 02:18:48 -0800733 bbsi.l a2, PS_UM_BIT, 2f # jump if user mode
Chris Zankel5a0015d2005-06-23 22:01:16 -0700734
735 addi a2, a1, -16-PT_SIZE # assume kernel stack
736 s32i a0, a2, PT_AREG0
737 movi a0, 0
738 s32i a1, a2, PT_AREG1
739 s32i a0, a2, PT_DEPC # mark it as a regular exception
Max Filippovbc5378f2012-10-15 03:55:38 +0400740 xsr a0, depc
Chris Zankel5a0015d2005-06-23 22:01:16 -0700741 s32i a3, a2, PT_AREG3
742 s32i a0, a2, PT_AREG2
743 mov a1, a2
744 j _kernel_exception
745
Max Filippovbc5378f2012-10-15 03:55:38 +04007462: rsr a2, excsave1
Chris Zankel5a0015d2005-06-23 22:01:16 -0700747 l32i a2, a2, EXC_TABLE_KSTK # load kernel stack pointer
748 s32i a0, a2, PT_AREG0
749 movi a0, 0
750 s32i a1, a2, PT_AREG1
751 s32i a0, a2, PT_DEPC
Max Filippovbc5378f2012-10-15 03:55:38 +0400752 xsr a0, depc
Chris Zankel5a0015d2005-06-23 22:01:16 -0700753 s32i a3, a2, PT_AREG3
754 s32i a0, a2, PT_AREG2
755 mov a1, a2
756 j _user_exception
757
758 /* Debug exception while in exception mode. */
7591: j 1b // FIXME!!
760
Chris Zankeld1538c42012-11-16 16:16:20 -0800761ENDPROC(debug_exception)
Chris Zankel5a0015d2005-06-23 22:01:16 -0700762
763/*
764 * We get here in case of an unrecoverable exception.
765 * The only thing we can do is to be nice and print a panic message.
766 * We only produce a single stack frame for panic, so ???
767 *
768 *
769 * Entry conditions:
770 *
771 * - a0 contains the caller address; original value saved in excsave1.
772 * - the original a0 contains a valid return address (backtrace) or 0.
773 * - a2 contains a valid stackpointer
774 *
775 * Notes:
776 *
777 * - If the stack pointer could be invalid, the caller has to setup a
778 * dummy stack pointer (e.g. the stack of the init_task)
779 *
780 * - If the return address could be invalid, the caller has to set it
781 * to 0, so the backtrace would stop.
782 *
783 */
784 .align 4
785unrecoverable_text:
786 .ascii "Unrecoverable error in exception handler\0"
787
788ENTRY(unrecoverable_exception)
789
790 movi a0, 1
791 movi a1, 0
792
Max Filippovbc5378f2012-10-15 03:55:38 +0400793 wsr a0, windowstart
794 wsr a1, windowbase
Chris Zankel5a0015d2005-06-23 22:01:16 -0700795 rsync
796
Marc Gauthier2d1c6452013-01-05 04:57:17 +0400797 movi a1, (1 << PS_WOE_BIT) | LOCKLEVEL
Max Filippovbc5378f2012-10-15 03:55:38 +0400798 wsr a1, ps
Chris Zankel5a0015d2005-06-23 22:01:16 -0700799 rsync
800
801 movi a1, init_task
802 movi a0, 0
803 addi a1, a1, PT_REGS_OFFSET
804
805 movi a4, panic
806 movi a6, unrecoverable_text
807
808 callx4 a4
809
8101: j 1b
811
Chris Zankeld1538c42012-11-16 16:16:20 -0800812ENDPROC(unrecoverable_exception)
Chris Zankel5a0015d2005-06-23 22:01:16 -0700813
814/* -------------------------- FAST EXCEPTION HANDLERS ----------------------- */
815
816/*
817 * Fast-handler for alloca exceptions
818 *
819 * The ALLOCA handler is entered when user code executes the MOVSP
820 * instruction and the caller's frame is not in the register file.
Chris Zankel5a0015d2005-06-23 22:01:16 -0700821 *
Max Filippovfff96d62013-07-15 02:02:24 +0400822 * This algorithm was taken from the Ross Morley's RTOS Porting Layer:
823 *
824 * /home/ross/rtos/porting/XtensaRTOS-PortingLayer-20090507/xtensa_vectors.S
825 *
826 * It leverages the existing window spill/fill routines and their support for
827 * double exceptions. The 'movsp' instruction will only cause an exception if
828 * the next window needs to be loaded. In fact this ALLOCA exception may be
829 * replaced at some point by changing the hardware to do a underflow exception
830 * of the proper size instead.
831 *
832 * This algorithm simply backs out the register changes started by the user
833 * excpetion handler, makes it appear that we have started a window underflow
834 * by rotating the window back and then setting the old window base (OWB) in
835 * the 'ps' register with the rolled back window base. The 'movsp' instruction
836 * will be re-executed and this time since the next window frames is in the
837 * active AR registers it won't cause an exception.
838 *
839 * If the WindowUnderflow code gets a TLB miss the page will get mapped
840 * the the partial windeowUnderflow will be handeled in the double exception
841 * handler.
Chris Zankel5a0015d2005-06-23 22:01:16 -0700842 *
843 * Entry condition:
844 *
845 * a0: trashed, original value saved on stack (PT_AREG0)
846 * a1: a1
847 * a2: new stack pointer, original in DEPC
Max Filippov99d50402013-07-03 20:23:28 +0400848 * a3: a3
Chris Zankel5a0015d2005-06-23 22:01:16 -0700849 * depc: a2, original value saved on stack (PT_DEPC)
Max Filippov99d50402013-07-03 20:23:28 +0400850 * excsave_1: dispatch table
Chris Zankel5a0015d2005-06-23 22:01:16 -0700851 *
852 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
853 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
854 */
855
Chris Zankel5a0015d2005-06-23 22:01:16 -0700856ENTRY(fast_alloca)
Max Filippovfff96d62013-07-15 02:02:24 +0400857 rsr a0, windowbase
858 rotw -1
859 rsr a2, ps
860 extui a3, a2, PS_OWB_SHIFT, PS_OWB_WIDTH
861 xor a3, a3, a4
862 l32i a4, a6, PT_AREG0
863 l32i a1, a6, PT_DEPC
864 rsr a6, depc
865 wsr a1, depc
866 slli a3, a3, PS_OWB_SHIFT
867 xor a2, a2, a3
868 wsr a2, ps
869 rsync
Chris Zankel5a0015d2005-06-23 22:01:16 -0700870
Max Filippovfff96d62013-07-15 02:02:24 +0400871 _bbci.l a4, 31, 4f
872 rotw -1
873 _bbci.l a8, 30, 8f
874 rotw -1
875 j _WindowUnderflow12
8768: j _WindowUnderflow8
8774: j _WindowUnderflow4
Chris Zankeld1538c42012-11-16 16:16:20 -0800878ENDPROC(fast_alloca)
Chris Zankel5a0015d2005-06-23 22:01:16 -0700879
880/*
881 * fast system calls.
882 *
883 * WARNING: The kernel doesn't save the entire user context before
884 * handling a fast system call. These functions are small and short,
885 * usually offering some functionality not available to user tasks.
886 *
887 * BE CAREFUL TO PRESERVE THE USER'S CONTEXT.
888 *
889 * Entry condition:
890 *
891 * a0: trashed, original value saved on stack (PT_AREG0)
892 * a1: a1
893 * a2: new stack pointer, original in DEPC
Max Filippov99d50402013-07-03 20:23:28 +0400894 * a3: a3
Chris Zankel5a0015d2005-06-23 22:01:16 -0700895 * depc: a2, original value saved on stack (PT_DEPC)
Max Filippov99d50402013-07-03 20:23:28 +0400896 * excsave_1: dispatch table
Chris Zankel5a0015d2005-06-23 22:01:16 -0700897 */
898
899ENTRY(fast_syscall_kernel)
900
901 /* Skip syscall. */
902
Max Filippovbc5378f2012-10-15 03:55:38 +0400903 rsr a0, epc1
Chris Zankel5a0015d2005-06-23 22:01:16 -0700904 addi a0, a0, 3
Max Filippovbc5378f2012-10-15 03:55:38 +0400905 wsr a0, epc1
Chris Zankel5a0015d2005-06-23 22:01:16 -0700906
907 l32i a0, a2, PT_DEPC
908 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable
909
Max Filippovbc5378f2012-10-15 03:55:38 +0400910 rsr a0, depc # get syscall-nr
Chris Zankel5a0015d2005-06-23 22:01:16 -0700911 _beqz a0, fast_syscall_spill_registers
Chris Zankelfc4fb2a2006-12-10 02:18:52 -0800912 _beqi a0, __NR_xtensa, fast_syscall_xtensa
Chris Zankel5a0015d2005-06-23 22:01:16 -0700913
914 j kernel_exception
915
Chris Zankeld1538c42012-11-16 16:16:20 -0800916ENDPROC(fast_syscall_kernel)
917
Chris Zankel5a0015d2005-06-23 22:01:16 -0700918ENTRY(fast_syscall_user)
919
920 /* Skip syscall. */
921
Max Filippovbc5378f2012-10-15 03:55:38 +0400922 rsr a0, epc1
Chris Zankel5a0015d2005-06-23 22:01:16 -0700923 addi a0, a0, 3
Max Filippovbc5378f2012-10-15 03:55:38 +0400924 wsr a0, epc1
Chris Zankel5a0015d2005-06-23 22:01:16 -0700925
926 l32i a0, a2, PT_DEPC
927 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable
928
Max Filippovbc5378f2012-10-15 03:55:38 +0400929 rsr a0, depc # get syscall-nr
Chris Zankel5a0015d2005-06-23 22:01:16 -0700930 _beqz a0, fast_syscall_spill_registers
Chris Zankelfc4fb2a2006-12-10 02:18:52 -0800931 _beqi a0, __NR_xtensa, fast_syscall_xtensa
Chris Zankel5a0015d2005-06-23 22:01:16 -0700932
933 j user_exception
934
Chris Zankeld1538c42012-11-16 16:16:20 -0800935ENDPROC(fast_syscall_user)
936
Chris Zankel5a0015d2005-06-23 22:01:16 -0700937ENTRY(fast_syscall_unrecoverable)
938
Chris Zankelc4c45942012-11-28 16:53:51 -0800939 /* Restore all states. */
Chris Zankel5a0015d2005-06-23 22:01:16 -0700940
Chris Zankelc4c45942012-11-28 16:53:51 -0800941 l32i a0, a2, PT_AREG0 # restore a0
942 xsr a2, depc # restore a2, depc
Chris Zankel5a0015d2005-06-23 22:01:16 -0700943
Chris Zankelc4c45942012-11-28 16:53:51 -0800944 wsr a0, excsave1
945 movi a0, unrecoverable_exception
946 callx0 a0
Chris Zankel5a0015d2005-06-23 22:01:16 -0700947
Chris Zankeld1538c42012-11-16 16:16:20 -0800948ENDPROC(fast_syscall_unrecoverable)
Chris Zankel5a0015d2005-06-23 22:01:16 -0700949
950/*
951 * sysxtensa syscall handler
952 *
Chris Zankelfc4fb2a2006-12-10 02:18:52 -0800953 * int sysxtensa (SYS_XTENSA_ATOMIC_SET, ptr, val, unused);
954 * int sysxtensa (SYS_XTENSA_ATOMIC_ADD, ptr, val, unused);
955 * int sysxtensa (SYS_XTENSA_ATOMIC_EXG_ADD, ptr, val, unused);
956 * int sysxtensa (SYS_XTENSA_ATOMIC_CMP_SWP, ptr, oldval, newval);
957 * a2 a6 a3 a4 a5
Chris Zankel5a0015d2005-06-23 22:01:16 -0700958 *
959 * Entry condition:
960 *
Chris Zankelfc4fb2a2006-12-10 02:18:52 -0800961 * a0: a2 (syscall-nr), original value saved on stack (PT_AREG0)
Chris Zankel5a0015d2005-06-23 22:01:16 -0700962 * a1: a1
Chris Zankelfc4fb2a2006-12-10 02:18:52 -0800963 * a2: new stack pointer, original in a0 and DEPC
Max Filippov99d50402013-07-03 20:23:28 +0400964 * a3: a3
Chris Zankelfc4fb2a2006-12-10 02:18:52 -0800965 * a4..a15: unchanged
Chris Zankel5a0015d2005-06-23 22:01:16 -0700966 * depc: a2, original value saved on stack (PT_DEPC)
Max Filippov99d50402013-07-03 20:23:28 +0400967 * excsave_1: dispatch table
Chris Zankel5a0015d2005-06-23 22:01:16 -0700968 *
969 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
970 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
971 *
972 * Note: we don't have to save a2; a2 holds the return value
973 *
974 * We use the two macros TRY and CATCH:
975 *
976 * TRY adds an entry to the __ex_table fixup table for the immediately
977 * following instruction.
978 *
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300979 * CATCH catches any exception that occurred at one of the preceding TRY
Chris Zankel5a0015d2005-06-23 22:01:16 -0700980 * statements and continues from there
981 *
982 * Usage TRY l32i a0, a1, 0
983 * <other code>
984 * done: rfe
985 * CATCH <set return code>
986 * j done
987 */
988
989#define TRY \
990 .section __ex_table, "a"; \
991 .word 66f, 67f; \
992 .text; \
99366:
994
995#define CATCH \
99667:
997
Chris Zankelfc4fb2a2006-12-10 02:18:52 -0800998ENTRY(fast_syscall_xtensa)
Chris Zankel5a0015d2005-06-23 22:01:16 -0700999
Chris Zankelfc4fb2a2006-12-10 02:18:52 -08001000 s32i a7, a2, PT_AREG7 # we need an additional register
Chris Zankel5a0015d2005-06-23 22:01:16 -07001001 movi a7, 4 # sizeof(unsigned int)
Chris Zankelfc4fb2a2006-12-10 02:18:52 -08001002 access_ok a3, a7, a0, a2, .Leac # a0: scratch reg, a2: sp
Chris Zankel5a0015d2005-06-23 22:01:16 -07001003
Chris Zankelfc4fb2a2006-12-10 02:18:52 -08001004 addi a6, a6, -1 # assuming SYS_XTENSA_ATOMIC_SET = 1
1005 _bgeui a6, SYS_XTENSA_COUNT - 1, .Lill
1006 _bnei a6, SYS_XTENSA_ATOMIC_CMP_SWP - 1, .Lnswp
Chris Zankel5a0015d2005-06-23 22:01:16 -07001007
Chris Zankelfc4fb2a2006-12-10 02:18:52 -08001008 /* Fall through for ATOMIC_CMP_SWP. */
Chris Zankel5a0015d2005-06-23 22:01:16 -07001009
1010.Lswp: /* Atomic compare and swap */
1011
Chris Zankelfc4fb2a2006-12-10 02:18:52 -08001012TRY l32i a0, a3, 0 # read old value
1013 bne a0, a4, 1f # same as old value? jump
1014TRY s32i a5, a3, 0 # different, modify value
1015 l32i a7, a2, PT_AREG7 # restore a7
1016 l32i a0, a2, PT_AREG0 # restore a0
1017 movi a2, 1 # and return 1
1018 addi a6, a6, 1 # restore a6 (really necessary?)
1019 rfe
Chris Zankel5a0015d2005-06-23 22:01:16 -07001020
Chris Zankelfc4fb2a2006-12-10 02:18:52 -080010211: l32i a7, a2, PT_AREG7 # restore a7
1022 l32i a0, a2, PT_AREG0 # restore a0
1023 movi a2, 0 # return 0 (note that we cannot set
1024 addi a6, a6, 1 # restore a6 (really necessary?)
1025 rfe
Chris Zankel5a0015d2005-06-23 22:01:16 -07001026
Chris Zankelfc4fb2a2006-12-10 02:18:52 -08001027.Lnswp: /* Atomic set, add, and exg_add. */
Chris Zankel5a0015d2005-06-23 22:01:16 -07001028
Chris Zankelfc4fb2a2006-12-10 02:18:52 -08001029TRY l32i a7, a3, 0 # orig
1030 add a0, a4, a7 # + arg
1031 moveqz a0, a4, a6 # set
1032TRY s32i a0, a3, 0 # write new value
Chris Zankel5a0015d2005-06-23 22:01:16 -07001033
Chris Zankelfc4fb2a2006-12-10 02:18:52 -08001034 mov a0, a2
Chris Zankel5a0015d2005-06-23 22:01:16 -07001035 mov a2, a7
Chris Zankelfc4fb2a2006-12-10 02:18:52 -08001036 l32i a7, a0, PT_AREG7 # restore a7
1037 l32i a0, a0, PT_AREG0 # restore a0
1038 addi a6, a6, 1 # restore a6 (really necessary?)
Chris Zankel5a0015d2005-06-23 22:01:16 -07001039 rfe
1040
1041CATCH
Chris Zankelfc4fb2a2006-12-10 02:18:52 -08001042.Leac: l32i a7, a2, PT_AREG7 # restore a7
1043 l32i a0, a2, PT_AREG0 # restore a0
1044 movi a2, -EFAULT
1045 rfe
1046
1047.Lill: l32i a7, a2, PT_AREG0 # restore a7
1048 l32i a0, a2, PT_AREG0 # restore a0
1049 movi a2, -EINVAL
1050 rfe
1051
Chris Zankeld1538c42012-11-16 16:16:20 -08001052ENDPROC(fast_syscall_xtensa)
Chris Zankel5a0015d2005-06-23 22:01:16 -07001053
1054
1055/* fast_syscall_spill_registers.
1056 *
1057 * Entry condition:
1058 *
1059 * a0: trashed, original value saved on stack (PT_AREG0)
1060 * a1: a1
1061 * a2: new stack pointer, original in DEPC
Max Filippov99d50402013-07-03 20:23:28 +04001062 * a3: a3
Chris Zankel5a0015d2005-06-23 22:01:16 -07001063 * depc: a2, original value saved on stack (PT_DEPC)
Max Filippov99d50402013-07-03 20:23:28 +04001064 * excsave_1: dispatch table
Chris Zankel5a0015d2005-06-23 22:01:16 -07001065 *
1066 * Note: We assume the stack pointer is EXC_TABLE_KSTK in the fixup handler.
Chris Zankel5a0015d2005-06-23 22:01:16 -07001067 */
1068
1069ENTRY(fast_syscall_spill_registers)
1070
1071 /* Register a FIXUP handler (pass current wb as a parameter) */
1072
Max Filippov99d50402013-07-03 20:23:28 +04001073 xsr a3, excsave1
Chris Zankel5a0015d2005-06-23 22:01:16 -07001074 movi a0, fast_syscall_spill_registers_fixup
1075 s32i a0, a3, EXC_TABLE_FIXUP
Max Filippovbc5378f2012-10-15 03:55:38 +04001076 rsr a0, windowbase
Chris Zankel5a0015d2005-06-23 22:01:16 -07001077 s32i a0, a3, EXC_TABLE_PARAM
Max Filippov99d50402013-07-03 20:23:28 +04001078 xsr a3, excsave1 # restore a3 and excsave_1
Chris Zankel5a0015d2005-06-23 22:01:16 -07001079
Max Filippov99d50402013-07-03 20:23:28 +04001080 /* Save a3, a4 and SAR on stack. */
Chris Zankel5a0015d2005-06-23 22:01:16 -07001081
Max Filippovbc5378f2012-10-15 03:55:38 +04001082 rsr a0, sar
Chris Zankel5a0015d2005-06-23 22:01:16 -07001083 s32i a3, a2, PT_AREG3
Chris Zankelc658eac2008-02-12 13:17:07 -08001084 s32i a4, a2, PT_AREG4
1085 s32i a0, a2, PT_AREG5 # store SAR to PT_AREG5
Chris Zankel5a0015d2005-06-23 22:01:16 -07001086
1087 /* The spill routine might clobber a7, a11, and a15. */
1088
Chris Zankelc658eac2008-02-12 13:17:07 -08001089 s32i a7, a2, PT_AREG7
1090 s32i a11, a2, PT_AREG11
1091 s32i a15, a2, PT_AREG15
Chris Zankel5a0015d2005-06-23 22:01:16 -07001092
Chris Zankelc658eac2008-02-12 13:17:07 -08001093 call0 _spill_registers # destroys a3, a4, and SAR
Chris Zankel5a0015d2005-06-23 22:01:16 -07001094
1095 /* Advance PC, restore registers and SAR, and return from exception. */
1096
Chris Zankelc658eac2008-02-12 13:17:07 -08001097 l32i a3, a2, PT_AREG5
1098 l32i a4, a2, PT_AREG4
Chris Zankel5a0015d2005-06-23 22:01:16 -07001099 l32i a0, a2, PT_AREG0
Max Filippovbc5378f2012-10-15 03:55:38 +04001100 wsr a3, sar
Chris Zankel5a0015d2005-06-23 22:01:16 -07001101 l32i a3, a2, PT_AREG3
1102
1103 /* Restore clobbered registers. */
1104
Chris Zankelc658eac2008-02-12 13:17:07 -08001105 l32i a7, a2, PT_AREG7
1106 l32i a11, a2, PT_AREG11
1107 l32i a15, a2, PT_AREG15
Chris Zankel5a0015d2005-06-23 22:01:16 -07001108
1109 movi a2, 0
1110 rfe
1111
Chris Zankeld1538c42012-11-16 16:16:20 -08001112ENDPROC(fast_syscall_spill_registers)
1113
Chris Zankel5a0015d2005-06-23 22:01:16 -07001114/* Fixup handler.
1115 *
1116 * We get here if the spill routine causes an exception, e.g. tlb miss.
1117 * We basically restore WINDOWBASE and WINDOWSTART to the condition when
1118 * we entered the spill routine and jump to the user exception handler.
1119 *
1120 * a0: value of depc, original value in depc
1121 * a2: trashed, original value in EXC_TABLE_DOUBLE_SAVE
1122 * a3: exctable, original value in excsave1
1123 */
1124
Max Filippov244066f2013-10-15 02:22:42 +04001125ENTRY(fast_syscall_spill_registers_fixup)
Chris Zankel5a0015d2005-06-23 22:01:16 -07001126
Max Filippovbc5378f2012-10-15 03:55:38 +04001127 rsr a2, windowbase # get current windowbase (a2 is saved)
1128 xsr a0, depc # restore depc and a0
Chris Zankel5a0015d2005-06-23 22:01:16 -07001129 ssl a2 # set shift (32 - WB)
1130
1131 /* We need to make sure the current registers (a0-a3) are preserved.
1132 * To do this, we simply set the bit for the current window frame
1133 * in WS, so that the exception handlers save them to the task stack.
1134 */
1135
Max Filippov99d50402013-07-03 20:23:28 +04001136 xsr a3, excsave1 # get spill-mask
Max Filippov244066f2013-10-15 02:22:42 +04001137 slli a3, a3, 1 # shift left by one
Chris Zankel5a0015d2005-06-23 22:01:16 -07001138
Max Filippov244066f2013-10-15 02:22:42 +04001139 slli a2, a3, 32-WSBITS
1140 src a2, a3, a2 # a2 = xxwww1yyxxxwww1yy......
Max Filippovbc5378f2012-10-15 03:55:38 +04001141 wsr a2, windowstart # set corrected windowstart
Chris Zankel5a0015d2005-06-23 22:01:16 -07001142
Max Filippov244066f2013-10-15 02:22:42 +04001143 srli a3, a3, 1
1144 rsr a2, excsave1
1145 l32i a2, a2, EXC_TABLE_DOUBLE_SAVE # restore a2
1146 xsr a2, excsave1
1147 s32i a3, a2, EXC_TABLE_DOUBLE_SAVE # save a3
1148 l32i a3, a2, EXC_TABLE_PARAM # original WB (in user task)
1149 xsr a2, excsave1
Chris Zankel5a0015d2005-06-23 22:01:16 -07001150
1151 /* Return to the original (user task) WINDOWBASE.
1152 * We leave the following frame behind:
1153 * a0, a1, a2 same
Max Filippov244066f2013-10-15 02:22:42 +04001154 * a3: trashed (saved in EXC_TABLE_DOUBLE_SAVE)
Chris Zankel5a0015d2005-06-23 22:01:16 -07001155 * depc: depc (we have to return to that address)
Max Filippov244066f2013-10-15 02:22:42 +04001156 * excsave_1: exctable
Chris Zankel5a0015d2005-06-23 22:01:16 -07001157 */
1158
Max Filippovbc5378f2012-10-15 03:55:38 +04001159 wsr a3, windowbase
Chris Zankel5a0015d2005-06-23 22:01:16 -07001160 rsync
1161
1162 /* We are now in the original frame when we entered _spill_registers:
1163 * a0: return address
1164 * a1: used, stack pointer
1165 * a2: kernel stack pointer
Max Filippov244066f2013-10-15 02:22:42 +04001166 * a3: available
Chris Zankel5a0015d2005-06-23 22:01:16 -07001167 * depc: exception address
Max Filippov244066f2013-10-15 02:22:42 +04001168 * excsave: exctable
Chris Zankel5a0015d2005-06-23 22:01:16 -07001169 * Note: This frame might be the same as above.
1170 */
1171
Chris Zankel5a0015d2005-06-23 22:01:16 -07001172 /* Setup stack pointer. */
1173
1174 addi a2, a2, -PT_USER_SIZE
1175 s32i a0, a2, PT_AREG0
1176
1177 /* Make sure we return to this fixup handler. */
1178
1179 movi a3, fast_syscall_spill_registers_fixup_return
1180 s32i a3, a2, PT_DEPC # setup depc
1181
1182 /* Jump to the exception handler. */
1183
Max Filippov99d50402013-07-03 20:23:28 +04001184 rsr a3, excsave1
Max Filippovbc5378f2012-10-15 03:55:38 +04001185 rsr a0, exccause
Chris Zankelc4c45942012-11-28 16:53:51 -08001186 addx4 a0, a0, a3 # find entry in table
1187 l32i a0, a0, EXC_TABLE_FAST_USER # load handler
Max Filippov244066f2013-10-15 02:22:42 +04001188 l32i a3, a3, EXC_TABLE_DOUBLE_SAVE
Chris Zankelc4c45942012-11-28 16:53:51 -08001189 jx a0
Chris Zankel5a0015d2005-06-23 22:01:16 -07001190
Max Filippov244066f2013-10-15 02:22:42 +04001191ENDPROC(fast_syscall_spill_registers_fixup)
1192
1193ENTRY(fast_syscall_spill_registers_fixup_return)
Chris Zankel5a0015d2005-06-23 22:01:16 -07001194
1195 /* When we return here, all registers have been restored (a2: DEPC) */
1196
Max Filippovbc5378f2012-10-15 03:55:38 +04001197 wsr a2, depc # exception address
Chris Zankel5a0015d2005-06-23 22:01:16 -07001198
1199 /* Restore fixup handler. */
1200
Max Filippov244066f2013-10-15 02:22:42 +04001201 rsr a2, excsave1
1202 s32i a3, a2, EXC_TABLE_DOUBLE_SAVE
1203 movi a3, fast_syscall_spill_registers_fixup
1204 s32i a3, a2, EXC_TABLE_FIXUP
1205 rsr a3, windowbase
1206 s32i a3, a2, EXC_TABLE_PARAM
1207 l32i a2, a2, EXC_TABLE_KSTK
Chris Zankel5a0015d2005-06-23 22:01:16 -07001208
Chris Zankel5a0015d2005-06-23 22:01:16 -07001209 /* Load WB at the time the exception occurred. */
1210
Max Filippovbc5378f2012-10-15 03:55:38 +04001211 rsr a3, sar # WB is still in SAR
Chris Zankel5a0015d2005-06-23 22:01:16 -07001212 neg a3, a3
Max Filippovbc5378f2012-10-15 03:55:38 +04001213 wsr a3, windowbase
Chris Zankel5a0015d2005-06-23 22:01:16 -07001214 rsync
1215
Max Filippov244066f2013-10-15 02:22:42 +04001216 rsr a3, excsave1
1217 l32i a3, a3, EXC_TABLE_DOUBLE_SAVE
1218
Chris Zankel5a0015d2005-06-23 22:01:16 -07001219 rfde
1220
Max Filippov244066f2013-10-15 02:22:42 +04001221ENDPROC(fast_syscall_spill_registers_fixup_return)
Chris Zankel5a0015d2005-06-23 22:01:16 -07001222
1223/*
1224 * spill all registers.
1225 *
1226 * This is not a real function. The following conditions must be met:
1227 *
1228 * - must be called with call0.
Chris Zankelc658eac2008-02-12 13:17:07 -08001229 * - uses a3, a4 and SAR.
Chris Zankel5a0015d2005-06-23 22:01:16 -07001230 * - the last 'valid' register of each frame are clobbered.
1231 * - the caller must have registered a fixup handler
1232 * (or be inside a critical section)
1233 * - PS_EXCM must be set (PS_WOE cleared?)
1234 */
1235
1236ENTRY(_spill_registers)
1237
1238 /*
1239 * Rotate ws so that the current windowbase is at bit 0.
1240 * Assume ws = xxxwww1yy (www1 current window frame).
Chris Zankelc658eac2008-02-12 13:17:07 -08001241 * Rotate ws right so that a4 = yyxxxwww1.
Chris Zankel5a0015d2005-06-23 22:01:16 -07001242 */
1243
Max Filippovbc5378f2012-10-15 03:55:38 +04001244 rsr a4, windowbase
1245 rsr a3, windowstart # a3 = xxxwww1yy
Chris Zankelc658eac2008-02-12 13:17:07 -08001246 ssr a4 # holds WB
1247 slli a4, a3, WSBITS
1248 or a3, a3, a4 # a3 = xxxwww1yyxxxwww1yy
Chris Zankelea0b6b02008-01-09 09:22:36 -08001249 srl a3, a3 # a3 = 00xxxwww1yyxxxwww1
Chris Zankel5a0015d2005-06-23 22:01:16 -07001250
1251 /* We are done if there are no more than the current register frame. */
1252
Chris Zankel50c07162007-11-14 13:47:02 -08001253 extui a3, a3, 1, WSBITS-1 # a3 = 0yyxxxwww
Chris Zankelc658eac2008-02-12 13:17:07 -08001254 movi a4, (1 << (WSBITS-1))
Chris Zankel5a0015d2005-06-23 22:01:16 -07001255 _beqz a3, .Lnospill # only one active frame? jump
1256
1257 /* We want 1 at the top, so that we return to the current windowbase */
1258
Chris Zankelc658eac2008-02-12 13:17:07 -08001259 or a3, a3, a4 # 1yyxxxwww
Chris Zankel5a0015d2005-06-23 22:01:16 -07001260
1261 /* Skip empty frames - get 'oldest' WINDOWSTART-bit. */
1262
Max Filippovbc5378f2012-10-15 03:55:38 +04001263 wsr a3, windowstart # save shifted windowstart
Chris Zankelc658eac2008-02-12 13:17:07 -08001264 neg a4, a3
1265 and a3, a4, a3 # first bit set from right: 000010000
Chris Zankel5a0015d2005-06-23 22:01:16 -07001266
Chris Zankelc658eac2008-02-12 13:17:07 -08001267 ffs_ws a4, a3 # a4: shifts to skip empty frames
Chris Zankel5a0015d2005-06-23 22:01:16 -07001268 movi a3, WSBITS
Chris Zankelc658eac2008-02-12 13:17:07 -08001269 sub a4, a3, a4 # WSBITS-a4:number of 0-bits from right
1270 ssr a4 # save in SAR for later.
Chris Zankel5a0015d2005-06-23 22:01:16 -07001271
Max Filippovbc5378f2012-10-15 03:55:38 +04001272 rsr a3, windowbase
Chris Zankelc658eac2008-02-12 13:17:07 -08001273 add a3, a3, a4
Max Filippovbc5378f2012-10-15 03:55:38 +04001274 wsr a3, windowbase
Chris Zankel5a0015d2005-06-23 22:01:16 -07001275 rsync
1276
Max Filippovbc5378f2012-10-15 03:55:38 +04001277 rsr a3, windowstart
Chris Zankel5a0015d2005-06-23 22:01:16 -07001278 srl a3, a3 # shift windowstart
1279
1280 /* WB is now just one frame below the oldest frame in the register
1281 window. WS is shifted so the oldest frame is in bit 0, thus, WB
1282 and WS differ by one 4-register frame. */
1283
1284 /* Save frames. Depending what call was used (call4, call8, call12),
1285 * we have to save 4,8. or 12 registers.
1286 */
1287
1288 _bbsi.l a3, 1, .Lc4
1289 _bbsi.l a3, 2, .Lc8
1290
1291 /* Special case: we have a call12-frame starting at a4. */
1292
1293 _bbci.l a3, 3, .Lc12 # bit 3 shouldn't be zero! (Jump to Lc12 first)
1294
1295 s32e a4, a1, -16 # a1 is valid with an empty spill area
1296 l32e a4, a5, -12
1297 s32e a8, a4, -48
1298 mov a8, a4
1299 l32e a4, a1, -16
1300 j .Lc12c
1301
Chris Zankel50c07162007-11-14 13:47:02 -08001302.Lnospill:
Chris Zankelea0b6b02008-01-09 09:22:36 -08001303 ret
Chris Zankel50c07162007-11-14 13:47:02 -08001304
Chris Zankel5a0015d2005-06-23 22:01:16 -07001305.Lloop: _bbsi.l a3, 1, .Lc4
1306 _bbci.l a3, 2, .Lc12
1307
1308.Lc8: s32e a4, a13, -16
1309 l32e a4, a5, -12
1310 s32e a8, a4, -32
1311 s32e a5, a13, -12
1312 s32e a6, a13, -8
1313 s32e a7, a13, -4
1314 s32e a9, a4, -28
1315 s32e a10, a4, -24
1316 s32e a11, a4, -20
1317
1318 srli a11, a3, 2 # shift windowbase by 2
1319 rotw 2
1320 _bnei a3, 1, .Lloop
1321
1322.Lexit: /* Done. Do the final rotation, set WS, and return. */
1323
1324 rotw 1
Max Filippovbc5378f2012-10-15 03:55:38 +04001325 rsr a3, windowbase
Chris Zankel5a0015d2005-06-23 22:01:16 -07001326 ssl a3
1327 movi a3, 1
1328 sll a3, a3
Max Filippovbc5378f2012-10-15 03:55:38 +04001329 wsr a3, windowstart
Chris Zankelea0b6b02008-01-09 09:22:36 -08001330 ret
Chris Zankel5a0015d2005-06-23 22:01:16 -07001331
1332.Lc4: s32e a4, a9, -16
1333 s32e a5, a9, -12
1334 s32e a6, a9, -8
1335 s32e a7, a9, -4
1336
1337 srli a7, a3, 1
1338 rotw 1
1339 _bnei a3, 1, .Lloop
1340 j .Lexit
1341
1342.Lc12: _bbci.l a3, 3, .Linvalid_mask # bit 2 shouldn't be zero!
1343
1344 /* 12-register frame (call12) */
1345
1346 l32e a2, a5, -12
1347 s32e a8, a2, -48
1348 mov a8, a2
1349
1350.Lc12c: s32e a9, a8, -44
1351 s32e a10, a8, -40
1352 s32e a11, a8, -36
1353 s32e a12, a8, -32
1354 s32e a13, a8, -28
1355 s32e a14, a8, -24
1356 s32e a15, a8, -20
1357 srli a15, a3, 3
1358
1359 /* The stack pointer for a4..a7 is out of reach, so we rotate the
1360 * window, grab the stackpointer, and rotate back.
1361 * Alternatively, we could also use the following approach, but that
1362 * makes the fixup routine much more complicated:
1363 * rotw 1
1364 * s32e a0, a13, -16
1365 * ...
1366 * rotw 2
1367 */
1368
1369 rotw 1
1370 mov a5, a13
1371 rotw -1
1372
1373 s32e a4, a9, -16
1374 s32e a5, a9, -12
1375 s32e a6, a9, -8
1376 s32e a7, a9, -4
1377
1378 rotw 3
1379
1380 _beqi a3, 1, .Lexit
1381 j .Lloop
1382
1383.Linvalid_mask:
1384
1385 /* We get here because of an unrecoverable error in the window
1386 * registers. If we are in user space, we kill the application,
1387 * however, this condition is unrecoverable in kernel space.
1388 */
1389
Max Filippovbc5378f2012-10-15 03:55:38 +04001390 rsr a0, ps
Chris Zankel173d6682006-12-10 02:18:48 -08001391 _bbci.l a0, PS_UM_BIT, 1f
Chris Zankel5a0015d2005-06-23 22:01:16 -07001392
Chris Zankelc4c45942012-11-28 16:53:51 -08001393 /* User space: Setup a dummy frame and kill application.
Chris Zankel5a0015d2005-06-23 22:01:16 -07001394 * Note: We assume EXC_TABLE_KSTK contains a valid stack pointer.
1395 */
1396
1397 movi a0, 1
1398 movi a1, 0
1399
Max Filippovbc5378f2012-10-15 03:55:38 +04001400 wsr a0, windowstart
1401 wsr a1, windowbase
Chris Zankel5a0015d2005-06-23 22:01:16 -07001402 rsync
1403
1404 movi a0, 0
1405
Max Filippov99d50402013-07-03 20:23:28 +04001406 rsr a3, excsave1
Chris Zankel5a0015d2005-06-23 22:01:16 -07001407 l32i a1, a3, EXC_TABLE_KSTK
Chris Zankel5a0015d2005-06-23 22:01:16 -07001408
Marc Gauthier2d1c6452013-01-05 04:57:17 +04001409 movi a4, (1 << PS_WOE_BIT) | LOCKLEVEL
Max Filippovbc5378f2012-10-15 03:55:38 +04001410 wsr a4, ps
Chris Zankel5a0015d2005-06-23 22:01:16 -07001411 rsync
1412
1413 movi a6, SIGSEGV
1414 movi a4, do_exit
1415 callx4 a4
1416
14171: /* Kernel space: PANIC! */
1418
Max Filippovbc5378f2012-10-15 03:55:38 +04001419 wsr a0, excsave1
Chris Zankel5a0015d2005-06-23 22:01:16 -07001420 movi a0, unrecoverable_exception
1421 callx0 a0 # should not return
14221: j 1b
1423
Chris Zankeld1538c42012-11-16 16:16:20 -08001424ENDPROC(_spill_registers)
1425
Johannes Weinere5083a62009-03-04 16:21:31 +01001426#ifdef CONFIG_MMU
Chris Zankel5a0015d2005-06-23 22:01:16 -07001427/*
1428 * We should never get here. Bail out!
1429 */
1430
1431ENTRY(fast_second_level_miss_double_kernel)
1432
14331: movi a0, unrecoverable_exception
1434 callx0 a0 # should not return
14351: j 1b
1436
Chris Zankeld1538c42012-11-16 16:16:20 -08001437ENDPROC(fast_second_level_miss_double_kernel)
1438
Chris Zankel5a0015d2005-06-23 22:01:16 -07001439/* First-level entry handler for user, kernel, and double 2nd-level
1440 * TLB miss exceptions. Note that for now, user and kernel miss
1441 * exceptions share the same entry point and are handled identically.
1442 *
1443 * An old, less-efficient C version of this function used to exist.
1444 * We include it below, interleaved as comments, for reference.
1445 *
1446 * Entry condition:
1447 *
1448 * a0: trashed, original value saved on stack (PT_AREG0)
1449 * a1: a1
1450 * a2: new stack pointer, original in DEPC
Max Filippov99d50402013-07-03 20:23:28 +04001451 * a3: a3
Chris Zankel5a0015d2005-06-23 22:01:16 -07001452 * depc: a2, original value saved on stack (PT_DEPC)
Max Filippov99d50402013-07-03 20:23:28 +04001453 * excsave_1: dispatch table
Chris Zankel5a0015d2005-06-23 22:01:16 -07001454 *
1455 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
1456 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
1457 */
1458
1459ENTRY(fast_second_level_miss)
1460
Max Filippov99d50402013-07-03 20:23:28 +04001461 /* Save a1 and a3. Note: we don't expect a double exception. */
Chris Zankel5a0015d2005-06-23 22:01:16 -07001462
1463 s32i a1, a2, PT_AREG1
Max Filippov99d50402013-07-03 20:23:28 +04001464 s32i a3, a2, PT_AREG3
Chris Zankel5a0015d2005-06-23 22:01:16 -07001465
1466 /* We need to map the page of PTEs for the user task. Find
1467 * the pointer to that page. Also, it's possible for tsk->mm
1468 * to be NULL while tsk->active_mm is nonzero if we faulted on
1469 * a vmalloc address. In that rare case, we must use
1470 * active_mm instead to avoid a fault in this handler. See
1471 *
1472 * http://mail.nl.linux.org/linux-mm/2002-08/msg00258.html
1473 * (or search Internet on "mm vs. active_mm")
1474 *
1475 * if (!mm)
1476 * mm = tsk->active_mm;
1477 * pgd = pgd_offset (mm, regs->excvaddr);
1478 * pmd = pmd_offset (pgd, regs->excvaddr);
1479 * pmdval = *pmd;
1480 */
1481
1482 GET_CURRENT(a1,a2)
1483 l32i a0, a1, TASK_MM # tsk->mm
1484 beqz a0, 9f
1485
Max Filippovbc5378f2012-10-15 03:55:38 +040014868: rsr a3, excvaddr # fault address
Chris Zankel01858d12007-08-06 23:57:57 -07001487 _PGD_OFFSET(a0, a3, a1)
Chris Zankel5a0015d2005-06-23 22:01:16 -07001488 l32i a0, a0, 0 # read pmdval
Chris Zankel5a0015d2005-06-23 22:01:16 -07001489 beqz a0, 2f
1490
1491 /* Read ptevaddr and convert to top of page-table page.
1492 *
1493 * vpnval = read_ptevaddr_register() & PAGE_MASK;
1494 * vpnval += DTLB_WAY_PGTABLE;
1495 * pteval = mk_pte (virt_to_page(pmd_val(pmdval)), PAGE_KERNEL);
1496 * write_dtlb_entry (pteval, vpnval);
1497 *
1498 * The messy computation for 'pteval' above really simplifies
1499 * into the following:
1500 *
Chris Zankel66569202007-08-22 10:14:51 -07001501 * pteval = ((pmdval - PAGE_OFFSET) & PAGE_MASK) | PAGE_DIRECTORY
Chris Zankel5a0015d2005-06-23 22:01:16 -07001502 */
1503
Chris Zankel39070cb2012-10-17 23:08:20 -07001504 movi a1, (-PAGE_OFFSET) & 0xffffffff
Chris Zankel5a0015d2005-06-23 22:01:16 -07001505 add a0, a0, a1 # pmdval - PAGE_OFFSET
1506 extui a1, a0, 0, PAGE_SHIFT # ... & PAGE_MASK
1507 xor a0, a0, a1
1508
Chris Zankel01858d12007-08-06 23:57:57 -07001509 movi a1, _PAGE_DIRECTORY
Chris Zankel5a0015d2005-06-23 22:01:16 -07001510 or a0, a0, a1 # ... | PAGE_DIRECTORY
1511
Chris Zankel01858d12007-08-06 23:57:57 -07001512 /*
Chris Zankel66569202007-08-22 10:14:51 -07001513 * We utilize all three wired-ways (7-9) to hold pmd translations.
Chris Zankel01858d12007-08-06 23:57:57 -07001514 * Memory regions are mapped to the DTLBs according to bits 28 and 29.
1515 * This allows to map the three most common regions to three different
1516 * DTLBs:
1517 * 0,1 -> way 7 program (0040.0000) and virtual (c000.0000)
1518 * 2 -> way 8 shared libaries (2000.0000)
1519 * 3 -> way 0 stack (3000.0000)
1520 */
Chris Zankel5a0015d2005-06-23 22:01:16 -07001521
Chris Zankel01858d12007-08-06 23:57:57 -07001522 extui a3, a3, 28, 2 # addr. bit 28 and 29 0,1,2,3
Max Filippovbc5378f2012-10-15 03:55:38 +04001523 rsr a1, ptevaddr
Chris Zankel01858d12007-08-06 23:57:57 -07001524 addx2 a3, a3, a3 # -> 0,3,6,9
1525 srli a1, a1, PAGE_SHIFT
1526 extui a3, a3, 2, 2 # -> 0,0,1,2
1527 slli a1, a1, PAGE_SHIFT # ptevaddr & PAGE_MASK
1528 addi a3, a3, DTLB_WAY_PGD
1529 add a1, a1, a3 # ... + way_number
1530
15313: wdtlb a0, a1
Chris Zankel5a0015d2005-06-23 22:01:16 -07001532 dsync
1533
1534 /* Exit critical section. */
1535
Max Filippov99d50402013-07-03 20:23:28 +040015364: rsr a3, excsave1
Chris Zankel5a0015d2005-06-23 22:01:16 -07001537 movi a0, 0
1538 s32i a0, a3, EXC_TABLE_FIXUP
1539
1540 /* Restore the working registers, and return. */
1541
1542 l32i a0, a2, PT_AREG0
1543 l32i a1, a2, PT_AREG1
Max Filippov99d50402013-07-03 20:23:28 +04001544 l32i a3, a2, PT_AREG3
Chris Zankel5a0015d2005-06-23 22:01:16 -07001545 l32i a2, a2, PT_DEPC
Chris Zankel5a0015d2005-06-23 22:01:16 -07001546
1547 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
1548
1549 /* Restore excsave1 and return. */
1550
Max Filippovbc5378f2012-10-15 03:55:38 +04001551 rsr a2, depc
Chris Zankel5a0015d2005-06-23 22:01:16 -07001552 rfe
1553
1554 /* Return from double exception. */
1555
Max Filippovbc5378f2012-10-15 03:55:38 +040015561: xsr a2, depc
Chris Zankel5a0015d2005-06-23 22:01:16 -07001557 esync
1558 rfde
1559
15609: l32i a0, a1, TASK_ACTIVE_MM # unlikely case mm == 0
1561 j 8b
1562
Chris Zankel66569202007-08-22 10:14:51 -07001563#if (DCACHE_WAY_SIZE > PAGE_SIZE)
1564
15652: /* Special case for cache aliasing.
1566 * We (should) only get here if a clear_user_page, copy_user_page
1567 * or the aliased cache flush functions got preemptively interrupted
1568 * by another task. Re-establish temporary mapping to the
1569 * TLBTEMP_BASE areas.
1570 */
1571
1572 /* We shouldn't be in a double exception */
1573
1574 l32i a0, a2, PT_DEPC
1575 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 2f
1576
1577 /* Make sure the exception originated in the special functions */
1578
1579 movi a0, __tlbtemp_mapping_start
Max Filippovbc5378f2012-10-15 03:55:38 +04001580 rsr a3, epc1
Chris Zankel66569202007-08-22 10:14:51 -07001581 bltu a3, a0, 2f
1582 movi a0, __tlbtemp_mapping_end
1583 bgeu a3, a0, 2f
1584
1585 /* Check if excvaddr was in one of the TLBTEMP_BASE areas. */
1586
1587 movi a3, TLBTEMP_BASE_1
Max Filippovbc5378f2012-10-15 03:55:38 +04001588 rsr a0, excvaddr
Chris Zankel66569202007-08-22 10:14:51 -07001589 bltu a0, a3, 2f
1590
1591 addi a1, a0, -(2 << (DCACHE_ALIAS_ORDER + PAGE_SHIFT))
1592 bgeu a1, a3, 2f
1593
1594 /* Check if we have to restore an ITLB mapping. */
1595
1596 movi a1, __tlbtemp_mapping_itlb
Max Filippovbc5378f2012-10-15 03:55:38 +04001597 rsr a3, epc1
Chris Zankel66569202007-08-22 10:14:51 -07001598 sub a3, a3, a1
1599
1600 /* Calculate VPN */
1601
1602 movi a1, PAGE_MASK
1603 and a1, a1, a0
1604
1605 /* Jump for ITLB entry */
1606
1607 bgez a3, 1f
1608
1609 /* We can use up to two TLBTEMP areas, one for src and one for dst. */
1610
1611 extui a3, a0, PAGE_SHIFT + DCACHE_ALIAS_ORDER, 1
1612 add a1, a3, a1
1613
1614 /* PPN is in a6 for the first TLBTEMP area and in a7 for the second. */
1615
1616 mov a0, a6
1617 movnez a0, a7, a3
1618 j 3b
1619
1620 /* ITLB entry. We only use dst in a6. */
1621
16221: witlb a6, a1
1623 isync
1624 j 4b
1625
1626
1627#endif // DCACHE_WAY_SIZE > PAGE_SIZE
1628
1629
Chris Zankel5a0015d2005-06-23 22:01:16 -070016302: /* Invalid PGD, default exception handling */
1631
Max Filippovbc5378f2012-10-15 03:55:38 +04001632 rsr a1, depc
Chris Zankel5a0015d2005-06-23 22:01:16 -07001633 s32i a1, a2, PT_AREG2
Chris Zankel5a0015d2005-06-23 22:01:16 -07001634 mov a1, a2
1635
Max Filippovbc5378f2012-10-15 03:55:38 +04001636 rsr a2, ps
Chris Zankel173d6682006-12-10 02:18:48 -08001637 bbsi.l a2, PS_UM_BIT, 1f
Chris Zankel5a0015d2005-06-23 22:01:16 -07001638 j _kernel_exception
16391: j _user_exception
1640
Chris Zankeld1538c42012-11-16 16:16:20 -08001641ENDPROC(fast_second_level_miss)
Chris Zankel5a0015d2005-06-23 22:01:16 -07001642
1643/*
1644 * StoreProhibitedException
1645 *
1646 * Update the pte and invalidate the itlb mapping for this pte.
1647 *
1648 * Entry condition:
1649 *
1650 * a0: trashed, original value saved on stack (PT_AREG0)
1651 * a1: a1
1652 * a2: new stack pointer, original in DEPC
Max Filippov99d50402013-07-03 20:23:28 +04001653 * a3: a3
Chris Zankel5a0015d2005-06-23 22:01:16 -07001654 * depc: a2, original value saved on stack (PT_DEPC)
Max Filippov99d50402013-07-03 20:23:28 +04001655 * excsave_1: dispatch table
Chris Zankel5a0015d2005-06-23 22:01:16 -07001656 *
1657 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
1658 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
1659 */
1660
1661ENTRY(fast_store_prohibited)
1662
Max Filippov99d50402013-07-03 20:23:28 +04001663 /* Save a1 and a3. */
Chris Zankel5a0015d2005-06-23 22:01:16 -07001664
1665 s32i a1, a2, PT_AREG1
Max Filippov99d50402013-07-03 20:23:28 +04001666 s32i a3, a2, PT_AREG3
Chris Zankel5a0015d2005-06-23 22:01:16 -07001667
1668 GET_CURRENT(a1,a2)
1669 l32i a0, a1, TASK_MM # tsk->mm
1670 beqz a0, 9f
1671
Max Filippovbc5378f2012-10-15 03:55:38 +040016728: rsr a1, excvaddr # fault address
Max Filippov99d50402013-07-03 20:23:28 +04001673 _PGD_OFFSET(a0, a1, a3)
Chris Zankel5a0015d2005-06-23 22:01:16 -07001674 l32i a0, a0, 0
Chris Zankel5a0015d2005-06-23 22:01:16 -07001675 beqz a0, 2f
1676
Max Filippov51fc41a2013-05-18 23:34:30 +04001677 /*
1678 * Note that we test _PAGE_WRITABLE_BIT only if PTE is present
1679 * and is not PAGE_NONE. See pgtable.h for possible PTE layouts.
1680 */
Chris Zankel01858d12007-08-06 23:57:57 -07001681
Max Filippov99d50402013-07-03 20:23:28 +04001682 _PTE_OFFSET(a0, a1, a3)
1683 l32i a3, a0, 0 # read pteval
Max Filippov51fc41a2013-05-18 23:34:30 +04001684 movi a1, _PAGE_CA_INVALID
Max Filippov99d50402013-07-03 20:23:28 +04001685 ball a3, a1, 2f
1686 bbci.l a3, _PAGE_WRITABLE_BIT, 2f
Chris Zankel5a0015d2005-06-23 22:01:16 -07001687
Chris Zankel01858d12007-08-06 23:57:57 -07001688 movi a1, _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_HW_WRITE
Max Filippov99d50402013-07-03 20:23:28 +04001689 or a3, a3, a1
Max Filippovbc5378f2012-10-15 03:55:38 +04001690 rsr a1, excvaddr
Max Filippov99d50402013-07-03 20:23:28 +04001691 s32i a3, a0, 0
Chris Zankel5a0015d2005-06-23 22:01:16 -07001692
1693 /* We need to flush the cache if we have page coloring. */
1694#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK
1695 dhwb a0, 0
1696#endif
1697 pdtlb a0, a1
Max Filippov99d50402013-07-03 20:23:28 +04001698 wdtlb a3, a0
Chris Zankel5a0015d2005-06-23 22:01:16 -07001699
1700 /* Exit critical section. */
1701
1702 movi a0, 0
Max Filippov99d50402013-07-03 20:23:28 +04001703 rsr a3, excsave1
Chris Zankel5a0015d2005-06-23 22:01:16 -07001704 s32i a0, a3, EXC_TABLE_FIXUP
1705
1706 /* Restore the working registers, and return. */
1707
Max Filippov99d50402013-07-03 20:23:28 +04001708 l32i a3, a2, PT_AREG3
Chris Zankel5a0015d2005-06-23 22:01:16 -07001709 l32i a1, a2, PT_AREG1
1710 l32i a0, a2, PT_AREG0
1711 l32i a2, a2, PT_DEPC
1712
Chris Zankel5a0015d2005-06-23 22:01:16 -07001713 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
1714
Max Filippovbc5378f2012-10-15 03:55:38 +04001715 rsr a2, depc
Chris Zankel5a0015d2005-06-23 22:01:16 -07001716 rfe
1717
1718 /* Double exception. Restore FIXUP handler and return. */
1719
Max Filippovbc5378f2012-10-15 03:55:38 +040017201: xsr a2, depc
Chris Zankel5a0015d2005-06-23 22:01:16 -07001721 esync
1722 rfde
1723
17249: l32i a0, a1, TASK_ACTIVE_MM # unlikely case mm == 0
1725 j 8b
1726
17272: /* If there was a problem, handle fault in C */
1728
Max Filippov99d50402013-07-03 20:23:28 +04001729 rsr a3, depc # still holds a2
1730 s32i a3, a2, PT_AREG2
Chris Zankel5a0015d2005-06-23 22:01:16 -07001731 mov a1, a2
1732
Max Filippovbc5378f2012-10-15 03:55:38 +04001733 rsr a2, ps
Chris Zankel173d6682006-12-10 02:18:48 -08001734 bbsi.l a2, PS_UM_BIT, 1f
Chris Zankel5a0015d2005-06-23 22:01:16 -07001735 j _kernel_exception
17361: j _user_exception
Chris Zankeld1538c42012-11-16 16:16:20 -08001737
1738ENDPROC(fast_store_prohibited)
1739
Johannes Weinere5083a62009-03-04 16:21:31 +01001740#endif /* CONFIG_MMU */
Chris Zankel5a0015d2005-06-23 22:01:16 -07001741
Chris Zankel5a0015d2005-06-23 22:01:16 -07001742/*
Chris Zankelfc4fb2a2006-12-10 02:18:52 -08001743 * System Calls.
1744 *
1745 * void system_call (struct pt_regs* regs, int exccause)
1746 * a2 a3
1747 */
1748
1749ENTRY(system_call)
Chris Zankeld1538c42012-11-16 16:16:20 -08001750
Chris Zankelfc4fb2a2006-12-10 02:18:52 -08001751 entry a1, 32
1752
1753 /* regs->syscall = regs->areg[2] */
1754
1755 l32i a3, a2, PT_AREG2
1756 mov a6, a2
1757 movi a4, do_syscall_trace_enter
1758 s32i a3, a2, PT_SYSCALL
1759 callx4 a4
1760
1761 /* syscall = sys_call_table[syscall_nr] */
1762
1763 movi a4, sys_call_table;
1764 movi a5, __NR_syscall_count
1765 movi a6, -ENOSYS
1766 bgeu a3, a5, 1f
1767
1768 addx4 a4, a3, a4
1769 l32i a4, a4, 0
1770 movi a5, sys_ni_syscall;
1771 beq a4, a5, 1f
1772
1773 /* Load args: arg0 - arg5 are passed via regs. */
1774
1775 l32i a6, a2, PT_AREG6
1776 l32i a7, a2, PT_AREG3
1777 l32i a8, a2, PT_AREG4
1778 l32i a9, a2, PT_AREG5
1779 l32i a10, a2, PT_AREG8
1780 l32i a11, a2, PT_AREG9
1781
1782 /* Pass one additional argument to the syscall: pt_regs (on stack) */
1783 s32i a2, a1, 0
1784
1785 callx4 a4
1786
17871: /* regs->areg[2] = return_value */
1788
1789 s32i a6, a2, PT_AREG2
1790 movi a4, do_syscall_trace_leave
1791 mov a6, a2
1792 callx4 a4
1793 retw
1794
Chris Zankeld1538c42012-11-16 16:16:20 -08001795ENDPROC(system_call)
1796
Chris Zankelfc4fb2a2006-12-10 02:18:52 -08001797
1798/*
Chris Zankel5a0015d2005-06-23 22:01:16 -07001799 * Task switch.
1800 *
1801 * struct task* _switch_to (struct task* prev, struct task* next)
1802 * a2 a2 a3
1803 */
1804
1805ENTRY(_switch_to)
1806
1807 entry a1, 16
1808
Chris Zankelc658eac2008-02-12 13:17:07 -08001809 mov a12, a2 # preserve 'prev' (a2)
1810 mov a13, a3 # and 'next' (a3)
Chris Zankel5a0015d2005-06-23 22:01:16 -07001811
Chris Zankelc658eac2008-02-12 13:17:07 -08001812 l32i a4, a2, TASK_THREAD_INFO
1813 l32i a5, a3, TASK_THREAD_INFO
Chris Zankel5a0015d2005-06-23 22:01:16 -07001814
Chris Zankelc658eac2008-02-12 13:17:07 -08001815 save_xtregs_user a4 a6 a8 a9 a10 a11 THREAD_XTREGS_USER
Chris Zankel5a0015d2005-06-23 22:01:16 -07001816
Chris Zankelc658eac2008-02-12 13:17:07 -08001817 s32i a0, a12, THREAD_RA # save return address
1818 s32i a1, a12, THREAD_SP # save stack pointer
1819
1820 /* Disable ints while we manipulate the stack pointer. */
1821
1822 movi a14, (1 << PS_EXCM_BIT) | LOCKLEVEL
Max Filippovbc5378f2012-10-15 03:55:38 +04001823 xsr a14, ps
1824 rsr a3, excsave1
Chris Zankel5a0015d2005-06-23 22:01:16 -07001825 rsync
1826 s32i a3, a3, EXC_TABLE_FIXUP /* enter critical section */
1827
Chris Zankelc658eac2008-02-12 13:17:07 -08001828 /* Switch CPENABLE */
1829
1830#if (XTENSA_HAVE_COPROCESSORS || XTENSA_HAVE_IO_PORTS)
1831 l32i a3, a5, THREAD_CPENABLE
Max Filippovbc5378f2012-10-15 03:55:38 +04001832 xsr a3, cpenable
Chris Zankelc658eac2008-02-12 13:17:07 -08001833 s32i a3, a4, THREAD_CPENABLE
1834#endif
1835
1836 /* Flush register file. */
1837
1838 call0 _spill_registers # destroys a3, a4, and SAR
Chris Zankel5a0015d2005-06-23 22:01:16 -07001839
1840 /* Set kernel stack (and leave critical section)
1841 * Note: It's save to set it here. The stack will not be overwritten
1842 * because the kernel stack will only be loaded again after
1843 * we return from kernel space.
1844 */
1845
Max Filippovbc5378f2012-10-15 03:55:38 +04001846 rsr a3, excsave1 # exc_table
Chris Zankelc658eac2008-02-12 13:17:07 -08001847 movi a6, 0
1848 addi a7, a5, PT_REGS_OFFSET
1849 s32i a6, a3, EXC_TABLE_FIXUP
1850 s32i a7, a3, EXC_TABLE_KSTK
Chris Zankel5a0015d2005-06-23 22:01:16 -07001851
Chris Zankelc50842d2013-02-23 19:35:57 -08001852 /* restore context of the task 'next' */
Chris Zankel5a0015d2005-06-23 22:01:16 -07001853
Chris Zankelc658eac2008-02-12 13:17:07 -08001854 l32i a0, a13, THREAD_RA # restore return address
1855 l32i a1, a13, THREAD_SP # restore stack pointer
Chris Zankel5a0015d2005-06-23 22:01:16 -07001856
Chris Zankelc658eac2008-02-12 13:17:07 -08001857 load_xtregs_user a5 a6 a8 a9 a10 a11 THREAD_XTREGS_USER
1858
Max Filippovbc5378f2012-10-15 03:55:38 +04001859 wsr a14, ps
Chris Zankelc658eac2008-02-12 13:17:07 -08001860 mov a2, a12 # return 'prev'
Chris Zankel5a0015d2005-06-23 22:01:16 -07001861 rsync
1862
1863 retw
1864
Chris Zankeld1538c42012-11-16 16:16:20 -08001865ENDPROC(_switch_to)
Chris Zankel5a0015d2005-06-23 22:01:16 -07001866
1867ENTRY(ret_from_fork)
1868
1869 /* void schedule_tail (struct task_struct *prev)
1870 * Note: prev is still in a6 (return value from fake call4 frame)
1871 */
1872 movi a4, schedule_tail
1873 callx4 a4
1874
Chris Zankelfc4fb2a2006-12-10 02:18:52 -08001875 movi a4, do_syscall_trace_leave
1876 mov a6, a1
Chris Zankel5a0015d2005-06-23 22:01:16 -07001877 callx4 a4
1878
1879 j common_exception_return
1880
Chris Zankeld1538c42012-11-16 16:16:20 -08001881ENDPROC(ret_from_fork)
1882
Max Filippov3306a722012-10-25 11:10:50 +04001883/*
1884 * Kernel thread creation helper
1885 * On entry, set up by copy_thread: a2 = thread_fn, a3 = thread_fn arg
1886 * left from _switch_to: a6 = prev
1887 */
1888ENTRY(ret_from_kernel_thread)
1889
1890 call4 schedule_tail
1891 mov a6, a3
1892 callx4 a2
Max Filippovf0a1bf02012-10-25 11:10:51 +04001893 j common_exception_return
Max Filippov3306a722012-10-25 11:10:50 +04001894
1895ENDPROC(ret_from_kernel_thread)