Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2010 Tilera Corporation. All Rights Reserved. |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or |
| 5 | * modify it under the terms of the GNU General Public License |
| 6 | * as published by the Free Software Foundation, version 2. |
| 7 | * |
| 8 | * This program is distributed in the hope that it will be useful, but |
| 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
| 10 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or |
| 11 | * NON INFRINGEMENT. See the GNU General Public License for |
| 12 | * more details. |
| 13 | */ |
| 14 | |
| 15 | #include <linux/sched.h> |
| 16 | #include <linux/preempt.h> |
| 17 | #include <linux/module.h> |
| 18 | #include <linux/fs.h> |
| 19 | #include <linux/kprobes.h> |
| 20 | #include <linux/elfcore.h> |
| 21 | #include <linux/tick.h> |
| 22 | #include <linux/init.h> |
| 23 | #include <linux/mm.h> |
| 24 | #include <linux/compat.h> |
| 25 | #include <linux/hardirq.h> |
| 26 | #include <linux/syscalls.h> |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 27 | #include <linux/kernel.h> |
Chris Metcalf | 313ce67 | 2011-05-02 14:50:06 -0400 | [diff] [blame] | 28 | #include <linux/tracehook.h> |
| 29 | #include <linux/signal.h> |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 30 | #include <asm/stack.h> |
Paul Gortmaker | 34f2c0a | 2012-04-01 16:38:46 -0400 | [diff] [blame] | 31 | #include <asm/switch_to.h> |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 32 | #include <asm/homecache.h> |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 33 | #include <asm/syscalls.h> |
Chris Metcalf | 313ce67 | 2011-05-02 14:50:06 -0400 | [diff] [blame] | 34 | #include <asm/traps.h> |
David Howells | bd119c6 | 2012-03-28 18:30:03 +0100 | [diff] [blame] | 35 | #include <asm/setup.h> |
Chris Metcalf | 2f9ac29 | 2013-08-06 16:04:13 -0400 | [diff] [blame] | 36 | #include <asm/uaccess.h> |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 37 | #ifdef CONFIG_HARDWALL |
| 38 | #include <asm/hardwall.h> |
| 39 | #endif |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 40 | #include <arch/chip.h> |
| 41 | #include <arch/abi.h> |
David Howells | bd119c6 | 2012-03-28 18:30:03 +0100 | [diff] [blame] | 42 | #include <arch/sim_def.h> |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 43 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 44 | /* |
| 45 | * Use the (x86) "idle=poll" option to prefer low latency when leaving the |
| 46 | * idle loop over low power while in the idle loop, e.g. if we have |
| 47 | * one thread per core and we want to get threads out of futex waits fast. |
| 48 | */ |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 49 | static int __init idle_setup(char *str) |
| 50 | { |
| 51 | if (!str) |
| 52 | return -EINVAL; |
| 53 | |
| 54 | if (!strcmp(str, "poll")) { |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 55 | pr_info("using polling idle threads.\n"); |
Thomas Gleixner | 0dc8153 | 2013-03-21 22:50:01 +0100 | [diff] [blame] | 56 | cpu_idle_poll_ctrl(true); |
| 57 | return 0; |
| 58 | } else if (!strcmp(str, "halt")) { |
| 59 | return 0; |
| 60 | } |
| 61 | return -1; |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 62 | } |
| 63 | early_param("idle", idle_setup); |
| 64 | |
Thomas Gleixner | 0dc8153 | 2013-03-21 22:50:01 +0100 | [diff] [blame] | 65 | void arch_cpu_idle(void) |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 66 | { |
Thomas Gleixner | 0dc8153 | 2013-03-21 22:50:01 +0100 | [diff] [blame] | 67 | __get_cpu_var(irq_stat).idle_timestamp = jiffies; |
| 68 | _cpu_idle(); |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 69 | } |
| 70 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 71 | /* |
Thomas Gleixner | d909a81 | 2012-05-05 15:05:47 +0000 | [diff] [blame] | 72 | * Release a thread_info structure |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 73 | */ |
Thomas Gleixner | d909a81 | 2012-05-05 15:05:47 +0000 | [diff] [blame] | 74 | void arch_release_thread_info(struct thread_info *info) |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 75 | { |
| 76 | struct single_step_state *step_state = info->step_state; |
| 77 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 78 | if (step_state) { |
| 79 | |
| 80 | /* |
| 81 | * FIXME: we don't munmap step_state->buffer |
| 82 | * because the mm_struct for this process (info->task->mm) |
| 83 | * has already been zeroed in exit_mm(). Keeping a |
| 84 | * reference to it here seems like a bad move, so this |
| 85 | * means we can't munmap() the buffer, and therefore if we |
| 86 | * ptrace multiple threads in a process, we will slowly |
| 87 | * leak user memory. (Note that as soon as the last |
| 88 | * thread in a process dies, we will reclaim all user |
| 89 | * memory including single-step buffers in the usual way.) |
| 90 | * We should either assign a kernel VA to this buffer |
| 91 | * somehow, or we should associate the buffer(s) with the |
| 92 | * mm itself so we can clean them up that way. |
| 93 | */ |
| 94 | kfree(step_state); |
| 95 | } |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 96 | } |
| 97 | |
| 98 | static void save_arch_state(struct thread_struct *t); |
| 99 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 100 | int copy_thread(unsigned long clone_flags, unsigned long sp, |
Al Viro | afa86fc | 2012-10-22 22:51:14 -0400 | [diff] [blame] | 101 | unsigned long arg, struct task_struct *p) |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 102 | { |
Chris Metcalf | e69ddd3 | 2013-01-04 11:37:13 -0500 | [diff] [blame] | 103 | struct pt_regs *childregs = task_pt_regs(p); |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 104 | unsigned long ksp; |
Chris Metcalf | 0f8b983 | 2012-10-19 16:25:12 -0400 | [diff] [blame] | 105 | unsigned long *callee_regs; |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 106 | |
| 107 | /* |
Chris Metcalf | 0f8b983 | 2012-10-19 16:25:12 -0400 | [diff] [blame] | 108 | * Set up the stack and stack pointer appropriately for the |
| 109 | * new child to find itself woken up in __switch_to(). |
| 110 | * The callee-saved registers must be on the stack to be read; |
| 111 | * the new task will then jump to assembly support to handle |
| 112 | * calling schedule_tail(), etc., and (for userspace tasks) |
| 113 | * returning to the context set up in the pt_regs. |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 114 | */ |
Chris Metcalf | 0f8b983 | 2012-10-19 16:25:12 -0400 | [diff] [blame] | 115 | ksp = (unsigned long) childregs; |
| 116 | ksp -= C_ABI_SAVE_AREA_SIZE; /* interrupt-entry save area */ |
| 117 | ((long *)ksp)[0] = ((long *)ksp)[1] = 0; |
| 118 | ksp -= CALLEE_SAVED_REGS_COUNT * sizeof(unsigned long); |
| 119 | callee_regs = (unsigned long *)ksp; |
| 120 | ksp -= C_ABI_SAVE_AREA_SIZE; /* __switch_to() save area */ |
| 121 | ((long *)ksp)[0] = ((long *)ksp)[1] = 0; |
| 122 | p->thread.ksp = ksp; |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 123 | |
Chris Metcalf | 0f8b983 | 2012-10-19 16:25:12 -0400 | [diff] [blame] | 124 | /* Record the pid of the task that created this one. */ |
| 125 | p->thread.creator_pid = current->pid; |
| 126 | |
Al Viro | 008f179 | 2012-10-23 23:32:21 -0400 | [diff] [blame] | 127 | if (unlikely(p->flags & PF_KTHREAD)) { |
Chris Metcalf | 0f8b983 | 2012-10-19 16:25:12 -0400 | [diff] [blame] | 128 | /* kernel thread */ |
| 129 | memset(childregs, 0, sizeof(struct pt_regs)); |
| 130 | memset(&callee_regs[2], 0, |
| 131 | (CALLEE_SAVED_REGS_COUNT - 2) * sizeof(unsigned long)); |
| 132 | callee_regs[0] = sp; /* r30 = function */ |
| 133 | callee_regs[1] = arg; /* r31 = arg */ |
| 134 | childregs->ex1 = PL_ICS_EX1(KERNEL_PL, 0); |
| 135 | p->thread.pc = (unsigned long) ret_from_kernel_thread; |
| 136 | return 0; |
| 137 | } |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 138 | |
| 139 | /* |
| 140 | * Start new thread in ret_from_fork so it schedules properly |
| 141 | * and then return from interrupt like the parent. |
| 142 | */ |
| 143 | p->thread.pc = (unsigned long) ret_from_fork; |
| 144 | |
Chris Metcalf | 0f8b983 | 2012-10-19 16:25:12 -0400 | [diff] [blame] | 145 | /* |
| 146 | * Do not clone step state from the parent; each thread |
| 147 | * must make its own lazily. |
| 148 | */ |
| 149 | task_thread_info(p)->step_state = NULL; |
| 150 | |
Chris Metcalf | 2f9ac29 | 2013-08-06 16:04:13 -0400 | [diff] [blame] | 151 | #ifdef __tilegx__ |
| 152 | /* |
| 153 | * Do not clone unalign jit fixup from the parent; each thread |
| 154 | * must allocate its own on demand. |
| 155 | */ |
| 156 | task_thread_info(p)->unalign_jit_base = NULL; |
| 157 | #endif |
| 158 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 159 | /* |
| 160 | * Copy the registers onto the kernel stack so the |
| 161 | * return-from-interrupt code will reload it into registers. |
| 162 | */ |
Al Viro | 008f179 | 2012-10-23 23:32:21 -0400 | [diff] [blame] | 163 | *childregs = *current_pt_regs(); |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 164 | childregs->regs[0] = 0; /* return value is zero */ |
Al Viro | 008f179 | 2012-10-23 23:32:21 -0400 | [diff] [blame] | 165 | if (sp) |
| 166 | childregs->sp = sp; /* override with new user stack pointer */ |
| 167 | memcpy(callee_regs, &childregs->regs[CALLEE_SAVED_FIRST_REG], |
Chris Metcalf | 0f8b983 | 2012-10-19 16:25:12 -0400 | [diff] [blame] | 168 | CALLEE_SAVED_REGS_COUNT * sizeof(unsigned long)); |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 169 | |
Al Viro | 008f179 | 2012-10-23 23:32:21 -0400 | [diff] [blame] | 170 | /* Save user stack top pointer so we can ID the stack vm area later. */ |
| 171 | p->thread.usp0 = childregs->sp; |
| 172 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 173 | /* |
Chris Metcalf | bc4cf2b | 2010-12-14 15:57:49 -0500 | [diff] [blame] | 174 | * If CLONE_SETTLS is set, set "tp" in the new task to "r4", |
| 175 | * which is passed in as arg #5 to sys_clone(). |
| 176 | */ |
| 177 | if (clone_flags & CLONE_SETTLS) |
Al Viro | 008f179 | 2012-10-23 23:32:21 -0400 | [diff] [blame] | 178 | childregs->tp = childregs->regs[4]; |
Chris Metcalf | bc4cf2b | 2010-12-14 15:57:49 -0500 | [diff] [blame] | 179 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 180 | |
| 181 | #if CHIP_HAS_TILE_DMA() |
| 182 | /* |
| 183 | * No DMA in the new thread. We model this on the fact that |
| 184 | * fork() clears the pending signals, alarms, and aio for the child. |
| 185 | */ |
| 186 | memset(&p->thread.tile_dma_state, 0, sizeof(struct tile_dma_state)); |
| 187 | memset(&p->thread.dma_async_tlb, 0, sizeof(struct async_tlb)); |
| 188 | #endif |
| 189 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 190 | /* New thread has its miscellaneous processor state bits clear. */ |
| 191 | p->thread.proc_status = 0; |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 192 | |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 193 | #ifdef CONFIG_HARDWALL |
| 194 | /* New thread does not own any networks. */ |
Chris Metcalf | b8ace08 | 2012-03-30 16:01:48 -0400 | [diff] [blame] | 195 | memset(&p->thread.hardwall[0], 0, |
| 196 | sizeof(struct hardwall_task) * HARDWALL_TYPES); |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 197 | #endif |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 198 | |
| 199 | |
| 200 | /* |
| 201 | * Start the new thread with the current architecture state |
| 202 | * (user interrupt masks, etc.). |
| 203 | */ |
| 204 | save_arch_state(&p->thread); |
| 205 | |
| 206 | return 0; |
| 207 | } |
| 208 | |
Chris Metcalf | 2f9ac29 | 2013-08-06 16:04:13 -0400 | [diff] [blame] | 209 | int set_unalign_ctl(struct task_struct *tsk, unsigned int val) |
| 210 | { |
| 211 | task_thread_info(tsk)->align_ctl = val; |
| 212 | return 0; |
| 213 | } |
| 214 | |
| 215 | int get_unalign_ctl(struct task_struct *tsk, unsigned long adr) |
| 216 | { |
| 217 | return put_user(task_thread_info(tsk)->align_ctl, |
| 218 | (unsigned int __user *)adr); |
| 219 | } |
| 220 | |
Chris Metcalf | 4036c7d | 2013-08-10 12:22:43 -0400 | [diff] [blame] | 221 | static struct task_struct corrupt_current = { .comm = "<corrupt>" }; |
| 222 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 223 | /* |
| 224 | * Return "current" if it looks plausible, or else a pointer to a dummy. |
| 225 | * This can be helpful if we are just trying to emit a clean panic. |
| 226 | */ |
| 227 | struct task_struct *validate_current(void) |
| 228 | { |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 229 | struct task_struct *tsk = current; |
| 230 | if (unlikely((unsigned long)tsk < PAGE_OFFSET || |
Chris Metcalf | b287f69 | 2012-03-29 14:02:52 -0400 | [diff] [blame] | 231 | (high_memory && (void *)tsk > high_memory) || |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 232 | ((unsigned long)tsk & (__alignof__(*tsk) - 1)) != 0)) { |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 233 | pr_err("Corrupt 'current' %p (sp %#lx)\n", tsk, stack_pointer); |
Chris Metcalf | 4036c7d | 2013-08-10 12:22:43 -0400 | [diff] [blame] | 234 | tsk = &corrupt_current; |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 235 | } |
| 236 | return tsk; |
| 237 | } |
| 238 | |
| 239 | /* Take and return the pointer to the previous task, for schedule_tail(). */ |
| 240 | struct task_struct *sim_notify_fork(struct task_struct *prev) |
| 241 | { |
| 242 | struct task_struct *tsk = current; |
| 243 | __insn_mtspr(SPR_SIM_CONTROL, SIM_CONTROL_OS_FORK_PARENT | |
| 244 | (tsk->thread.creator_pid << _SIM_CONTROL_OPERATOR_BITS)); |
| 245 | __insn_mtspr(SPR_SIM_CONTROL, SIM_CONTROL_OS_FORK | |
| 246 | (tsk->pid << _SIM_CONTROL_OPERATOR_BITS)); |
| 247 | return prev; |
| 248 | } |
| 249 | |
| 250 | int dump_task_regs(struct task_struct *tsk, elf_gregset_t *regs) |
| 251 | { |
| 252 | struct pt_regs *ptregs = task_pt_regs(tsk); |
| 253 | elf_core_copy_regs(regs, ptregs); |
| 254 | return 1; |
| 255 | } |
| 256 | |
| 257 | #if CHIP_HAS_TILE_DMA() |
| 258 | |
| 259 | /* Allow user processes to access the DMA SPRs */ |
| 260 | void grant_dma_mpls(void) |
| 261 | { |
Chris Metcalf | a78c942 | 2010-10-14 16:23:03 -0400 | [diff] [blame] | 262 | #if CONFIG_KERNEL_PL == 2 |
| 263 | __insn_mtspr(SPR_MPL_DMA_CPL_SET_1, 1); |
| 264 | __insn_mtspr(SPR_MPL_DMA_NOTIFY_SET_1, 1); |
| 265 | #else |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 266 | __insn_mtspr(SPR_MPL_DMA_CPL_SET_0, 1); |
| 267 | __insn_mtspr(SPR_MPL_DMA_NOTIFY_SET_0, 1); |
Chris Metcalf | a78c942 | 2010-10-14 16:23:03 -0400 | [diff] [blame] | 268 | #endif |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 269 | } |
| 270 | |
| 271 | /* Forbid user processes from accessing the DMA SPRs */ |
| 272 | void restrict_dma_mpls(void) |
| 273 | { |
Chris Metcalf | a78c942 | 2010-10-14 16:23:03 -0400 | [diff] [blame] | 274 | #if CONFIG_KERNEL_PL == 2 |
| 275 | __insn_mtspr(SPR_MPL_DMA_CPL_SET_2, 1); |
| 276 | __insn_mtspr(SPR_MPL_DMA_NOTIFY_SET_2, 1); |
| 277 | #else |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 278 | __insn_mtspr(SPR_MPL_DMA_CPL_SET_1, 1); |
| 279 | __insn_mtspr(SPR_MPL_DMA_NOTIFY_SET_1, 1); |
Chris Metcalf | a78c942 | 2010-10-14 16:23:03 -0400 | [diff] [blame] | 280 | #endif |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 281 | } |
| 282 | |
| 283 | /* Pause the DMA engine, then save off its state registers. */ |
| 284 | static void save_tile_dma_state(struct tile_dma_state *dma) |
| 285 | { |
| 286 | unsigned long state = __insn_mfspr(SPR_DMA_USER_STATUS); |
| 287 | unsigned long post_suspend_state; |
| 288 | |
| 289 | /* If we're running, suspend the engine. */ |
| 290 | if ((state & DMA_STATUS_MASK) == SPR_DMA_STATUS__RUNNING_MASK) |
| 291 | __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__SUSPEND_MASK); |
| 292 | |
| 293 | /* |
| 294 | * Wait for the engine to idle, then save regs. Note that we |
| 295 | * want to record the "running" bit from before suspension, |
| 296 | * and the "done" bit from after, so that we can properly |
| 297 | * distinguish a case where the user suspended the engine from |
| 298 | * the case where the kernel suspended as part of the context |
| 299 | * swap. |
| 300 | */ |
| 301 | do { |
| 302 | post_suspend_state = __insn_mfspr(SPR_DMA_USER_STATUS); |
| 303 | } while (post_suspend_state & SPR_DMA_STATUS__BUSY_MASK); |
| 304 | |
| 305 | dma->src = __insn_mfspr(SPR_DMA_SRC_ADDR); |
| 306 | dma->src_chunk = __insn_mfspr(SPR_DMA_SRC_CHUNK_ADDR); |
| 307 | dma->dest = __insn_mfspr(SPR_DMA_DST_ADDR); |
| 308 | dma->dest_chunk = __insn_mfspr(SPR_DMA_DST_CHUNK_ADDR); |
| 309 | dma->strides = __insn_mfspr(SPR_DMA_STRIDE); |
| 310 | dma->chunk_size = __insn_mfspr(SPR_DMA_CHUNK_SIZE); |
| 311 | dma->byte = __insn_mfspr(SPR_DMA_BYTE); |
| 312 | dma->status = (state & SPR_DMA_STATUS__RUNNING_MASK) | |
| 313 | (post_suspend_state & SPR_DMA_STATUS__DONE_MASK); |
| 314 | } |
| 315 | |
| 316 | /* Restart a DMA that was running before we were context-switched out. */ |
| 317 | static void restore_tile_dma_state(struct thread_struct *t) |
| 318 | { |
| 319 | const struct tile_dma_state *dma = &t->tile_dma_state; |
| 320 | |
| 321 | /* |
| 322 | * The only way to restore the done bit is to run a zero |
| 323 | * length transaction. |
| 324 | */ |
| 325 | if ((dma->status & SPR_DMA_STATUS__DONE_MASK) && |
| 326 | !(__insn_mfspr(SPR_DMA_USER_STATUS) & SPR_DMA_STATUS__DONE_MASK)) { |
| 327 | __insn_mtspr(SPR_DMA_BYTE, 0); |
| 328 | __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK); |
| 329 | while (__insn_mfspr(SPR_DMA_USER_STATUS) & |
| 330 | SPR_DMA_STATUS__BUSY_MASK) |
| 331 | ; |
| 332 | } |
| 333 | |
| 334 | __insn_mtspr(SPR_DMA_SRC_ADDR, dma->src); |
| 335 | __insn_mtspr(SPR_DMA_SRC_CHUNK_ADDR, dma->src_chunk); |
| 336 | __insn_mtspr(SPR_DMA_DST_ADDR, dma->dest); |
| 337 | __insn_mtspr(SPR_DMA_DST_CHUNK_ADDR, dma->dest_chunk); |
| 338 | __insn_mtspr(SPR_DMA_STRIDE, dma->strides); |
| 339 | __insn_mtspr(SPR_DMA_CHUNK_SIZE, dma->chunk_size); |
| 340 | __insn_mtspr(SPR_DMA_BYTE, dma->byte); |
| 341 | |
| 342 | /* |
| 343 | * Restart the engine if we were running and not done. |
| 344 | * Clear a pending async DMA fault that we were waiting on return |
| 345 | * to user space to execute, since we expect the DMA engine |
| 346 | * to regenerate those faults for us now. Note that we don't |
| 347 | * try to clear the TIF_ASYNC_TLB flag, since it's relatively |
| 348 | * harmless if set, and it covers both DMA and the SN processor. |
| 349 | */ |
| 350 | if ((dma->status & DMA_STATUS_MASK) == SPR_DMA_STATUS__RUNNING_MASK) { |
| 351 | t->dma_async_tlb.fault_num = 0; |
| 352 | __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK); |
| 353 | } |
| 354 | } |
| 355 | |
| 356 | #endif |
| 357 | |
| 358 | static void save_arch_state(struct thread_struct *t) |
| 359 | { |
| 360 | #if CHIP_HAS_SPLIT_INTR_MASK() |
| 361 | t->interrupt_mask = __insn_mfspr(SPR_INTERRUPT_MASK_0_0) | |
| 362 | ((u64)__insn_mfspr(SPR_INTERRUPT_MASK_0_1) << 32); |
| 363 | #else |
| 364 | t->interrupt_mask = __insn_mfspr(SPR_INTERRUPT_MASK_0); |
| 365 | #endif |
| 366 | t->ex_context[0] = __insn_mfspr(SPR_EX_CONTEXT_0_0); |
| 367 | t->ex_context[1] = __insn_mfspr(SPR_EX_CONTEXT_0_1); |
| 368 | t->system_save[0] = __insn_mfspr(SPR_SYSTEM_SAVE_0_0); |
| 369 | t->system_save[1] = __insn_mfspr(SPR_SYSTEM_SAVE_0_1); |
| 370 | t->system_save[2] = __insn_mfspr(SPR_SYSTEM_SAVE_0_2); |
| 371 | t->system_save[3] = __insn_mfspr(SPR_SYSTEM_SAVE_0_3); |
| 372 | t->intctrl_0 = __insn_mfspr(SPR_INTCTRL_0_STATUS); |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 373 | t->proc_status = __insn_mfspr(SPR_PROC_STATUS); |
Chris Metcalf | a802fc6 | 2010-09-15 11:16:10 -0400 | [diff] [blame] | 374 | #if !CHIP_HAS_FIXED_INTVEC_BASE() |
| 375 | t->interrupt_vector_base = __insn_mfspr(SPR_INTERRUPT_VECTOR_BASE_0); |
| 376 | #endif |
Chris Metcalf | a802fc6 | 2010-09-15 11:16:10 -0400 | [diff] [blame] | 377 | t->tile_rtf_hwm = __insn_mfspr(SPR_TILE_RTF_HWM); |
Chris Metcalf | a802fc6 | 2010-09-15 11:16:10 -0400 | [diff] [blame] | 378 | #if CHIP_HAS_DSTREAM_PF() |
| 379 | t->dstream_pf = __insn_mfspr(SPR_DSTREAM_PF); |
| 380 | #endif |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 381 | } |
| 382 | |
| 383 | static void restore_arch_state(const struct thread_struct *t) |
| 384 | { |
| 385 | #if CHIP_HAS_SPLIT_INTR_MASK() |
| 386 | __insn_mtspr(SPR_INTERRUPT_MASK_0_0, (u32) t->interrupt_mask); |
| 387 | __insn_mtspr(SPR_INTERRUPT_MASK_0_1, t->interrupt_mask >> 32); |
| 388 | #else |
| 389 | __insn_mtspr(SPR_INTERRUPT_MASK_0, t->interrupt_mask); |
| 390 | #endif |
| 391 | __insn_mtspr(SPR_EX_CONTEXT_0_0, t->ex_context[0]); |
| 392 | __insn_mtspr(SPR_EX_CONTEXT_0_1, t->ex_context[1]); |
| 393 | __insn_mtspr(SPR_SYSTEM_SAVE_0_0, t->system_save[0]); |
| 394 | __insn_mtspr(SPR_SYSTEM_SAVE_0_1, t->system_save[1]); |
| 395 | __insn_mtspr(SPR_SYSTEM_SAVE_0_2, t->system_save[2]); |
| 396 | __insn_mtspr(SPR_SYSTEM_SAVE_0_3, t->system_save[3]); |
| 397 | __insn_mtspr(SPR_INTCTRL_0_STATUS, t->intctrl_0); |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 398 | __insn_mtspr(SPR_PROC_STATUS, t->proc_status); |
Chris Metcalf | a802fc6 | 2010-09-15 11:16:10 -0400 | [diff] [blame] | 399 | #if !CHIP_HAS_FIXED_INTVEC_BASE() |
| 400 | __insn_mtspr(SPR_INTERRUPT_VECTOR_BASE_0, t->interrupt_vector_base); |
| 401 | #endif |
Chris Metcalf | a802fc6 | 2010-09-15 11:16:10 -0400 | [diff] [blame] | 402 | __insn_mtspr(SPR_TILE_RTF_HWM, t->tile_rtf_hwm); |
Chris Metcalf | a802fc6 | 2010-09-15 11:16:10 -0400 | [diff] [blame] | 403 | #if CHIP_HAS_DSTREAM_PF() |
| 404 | __insn_mtspr(SPR_DSTREAM_PF, t->dstream_pf); |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 405 | #endif |
| 406 | } |
| 407 | |
| 408 | |
| 409 | void _prepare_arch_switch(struct task_struct *next) |
| 410 | { |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 411 | #if CHIP_HAS_TILE_DMA() |
| 412 | struct tile_dma_state *dma = ¤t->thread.tile_dma_state; |
| 413 | if (dma->enabled) |
| 414 | save_tile_dma_state(dma); |
| 415 | #endif |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 416 | } |
| 417 | |
| 418 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 419 | struct task_struct *__sched _switch_to(struct task_struct *prev, |
| 420 | struct task_struct *next) |
| 421 | { |
| 422 | /* DMA state is already saved; save off other arch state. */ |
| 423 | save_arch_state(&prev->thread); |
| 424 | |
| 425 | #if CHIP_HAS_TILE_DMA() |
| 426 | /* |
| 427 | * Restore DMA in new task if desired. |
| 428 | * Note that it is only safe to restart here since interrupts |
| 429 | * are disabled, so we can't take any DMATLB miss or access |
| 430 | * interrupts before we have finished switching stacks. |
| 431 | */ |
| 432 | if (next->thread.tile_dma_state.enabled) { |
| 433 | restore_tile_dma_state(&next->thread); |
| 434 | grant_dma_mpls(); |
| 435 | } else { |
| 436 | restrict_dma_mpls(); |
| 437 | } |
| 438 | #endif |
| 439 | |
| 440 | /* Restore other arch state. */ |
| 441 | restore_arch_state(&next->thread); |
| 442 | |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 443 | #ifdef CONFIG_HARDWALL |
| 444 | /* Enable or disable access to the network registers appropriately. */ |
Chris Metcalf | b8ace08 | 2012-03-30 16:01:48 -0400 | [diff] [blame] | 445 | hardwall_switch_tasks(prev, next); |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 446 | #endif |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 447 | |
| 448 | /* |
| 449 | * Switch kernel SP, PC, and callee-saved registers. |
| 450 | * In the context of the new task, return the old task pointer |
| 451 | * (i.e. the task that actually called __switch_to). |
Chris Metcalf | a78c942 | 2010-10-14 16:23:03 -0400 | [diff] [blame] | 452 | * Pass the value to use for SYSTEM_SAVE_K_0 when we reset our sp. |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 453 | */ |
| 454 | return __switch_to(prev, next, next_current_ksp0(next)); |
| 455 | } |
| 456 | |
Chris Metcalf | 313ce67 | 2011-05-02 14:50:06 -0400 | [diff] [blame] | 457 | /* |
| 458 | * This routine is called on return from interrupt if any of the |
| 459 | * TIF_WORK_MASK flags are set in thread_info->flags. It is |
| 460 | * entered with interrupts disabled so we don't miss an event |
| 461 | * that modified the thread_info flags. If any flag is set, we |
| 462 | * handle it and return, and the calling assembly code will |
| 463 | * re-disable interrupts, reload the thread flags, and call back |
| 464 | * if more flags need to be handled. |
| 465 | * |
| 466 | * We return whether we need to check the thread_info flags again |
| 467 | * or not. Note that we don't clear TIF_SINGLESTEP here, so it's |
| 468 | * important that it be tested last, and then claim that we don't |
| 469 | * need to recheck the flags. |
| 470 | */ |
| 471 | int do_work_pending(struct pt_regs *regs, u32 thread_info_flags) |
| 472 | { |
Chris Metcalf | fc327e2 | 2012-04-28 18:51:43 -0400 | [diff] [blame] | 473 | /* If we enter in kernel mode, do nothing and exit the caller loop. */ |
| 474 | if (!user_mode(regs)) |
| 475 | return 0; |
| 476 | |
Chris Metcalf | c19c6c9 | 2012-10-12 15:38:54 -0400 | [diff] [blame] | 477 | /* Enable interrupts; they are disabled again on return to caller. */ |
| 478 | local_irq_enable(); |
| 479 | |
Chris Metcalf | 313ce67 | 2011-05-02 14:50:06 -0400 | [diff] [blame] | 480 | if (thread_info_flags & _TIF_NEED_RESCHED) { |
| 481 | schedule(); |
| 482 | return 1; |
| 483 | } |
Chris Metcalf | d7c9661 | 2013-08-15 16:23:24 -0400 | [diff] [blame] | 484 | #if CHIP_HAS_TILE_DMA() |
Chris Metcalf | 313ce67 | 2011-05-02 14:50:06 -0400 | [diff] [blame] | 485 | if (thread_info_flags & _TIF_ASYNC_TLB) { |
| 486 | do_async_page_fault(regs); |
| 487 | return 1; |
| 488 | } |
| 489 | #endif |
| 490 | if (thread_info_flags & _TIF_SIGPENDING) { |
| 491 | do_signal(regs); |
| 492 | return 1; |
| 493 | } |
| 494 | if (thread_info_flags & _TIF_NOTIFY_RESUME) { |
| 495 | clear_thread_flag(TIF_NOTIFY_RESUME); |
| 496 | tracehook_notify_resume(regs); |
Chris Metcalf | 313ce67 | 2011-05-02 14:50:06 -0400 | [diff] [blame] | 497 | return 1; |
| 498 | } |
| 499 | if (thread_info_flags & _TIF_SINGLESTEP) { |
Chris Metcalf | fc327e2 | 2012-04-28 18:51:43 -0400 | [diff] [blame] | 500 | single_step_once(regs); |
Chris Metcalf | 313ce67 | 2011-05-02 14:50:06 -0400 | [diff] [blame] | 501 | return 0; |
| 502 | } |
| 503 | panic("work_pending: bad flags %#x\n", thread_info_flags); |
| 504 | } |
| 505 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 506 | unsigned long get_wchan(struct task_struct *p) |
| 507 | { |
| 508 | struct KBacktraceIterator kbt; |
| 509 | |
| 510 | if (!p || p == current || p->state == TASK_RUNNING) |
| 511 | return 0; |
| 512 | |
| 513 | for (KBacktraceIterator_init(&kbt, p, NULL); |
| 514 | !KBacktraceIterator_end(&kbt); |
| 515 | KBacktraceIterator_next(&kbt)) { |
| 516 | if (!in_sched_functions(kbt.it.pc)) |
| 517 | return kbt.it.pc; |
| 518 | } |
| 519 | |
| 520 | return 0; |
| 521 | } |
| 522 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 523 | /* Flush thread state. */ |
| 524 | void flush_thread(void) |
| 525 | { |
| 526 | /* Nothing */ |
| 527 | } |
| 528 | |
| 529 | /* |
| 530 | * Free current thread data structures etc.. |
| 531 | */ |
| 532 | void exit_thread(void) |
| 533 | { |
Chris Metcalf | 7d93771 | 2013-07-23 17:18:05 -0400 | [diff] [blame] | 534 | #ifdef CONFIG_HARDWALL |
| 535 | /* |
| 536 | * Remove the task from the list of tasks that are associated |
| 537 | * with any live hardwalls. (If the task that is exiting held |
| 538 | * the last reference to a hardwall fd, it would already have |
| 539 | * been released and deactivated at this point.) |
| 540 | */ |
| 541 | hardwall_deactivate_all(current); |
| 542 | #endif |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 543 | } |
| 544 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 545 | void show_regs(struct pt_regs *regs) |
| 546 | { |
| 547 | struct task_struct *tsk = validate_current(); |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 548 | int i; |
| 549 | |
| 550 | pr_err("\n"); |
Chris Metcalf | 4036c7d | 2013-08-10 12:22:43 -0400 | [diff] [blame] | 551 | if (tsk != &corrupt_current) |
| 552 | show_regs_print_info(KERN_ERR); |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 553 | #ifdef __tilegx__ |
Chris Metcalf | dadf78b | 2013-08-07 12:03:53 -0400 | [diff] [blame] | 554 | for (i = 0; i < 17; i++) |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 555 | pr_err(" r%-2d: "REGFMT" r%-2d: "REGFMT" r%-2d: "REGFMT"\n", |
Chris Metcalf | dadf78b | 2013-08-07 12:03:53 -0400 | [diff] [blame] | 556 | i, regs->regs[i], i+18, regs->regs[i+18], |
| 557 | i+36, regs->regs[i+36]); |
| 558 | pr_err(" r17: "REGFMT" r35: "REGFMT" tp : "REGFMT"\n", |
| 559 | regs->regs[17], regs->regs[35], regs->tp); |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 560 | pr_err(" sp : "REGFMT" lr : "REGFMT"\n", regs->sp, regs->lr); |
| 561 | #else |
Chris Metcalf | dadf78b | 2013-08-07 12:03:53 -0400 | [diff] [blame] | 562 | for (i = 0; i < 13; i++) |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 563 | pr_err(" r%-2d: "REGFMT" r%-2d: "REGFMT |
| 564 | " r%-2d: "REGFMT" r%-2d: "REGFMT"\n", |
Chris Metcalf | dadf78b | 2013-08-07 12:03:53 -0400 | [diff] [blame] | 565 | i, regs->regs[i], i+14, regs->regs[i+14], |
| 566 | i+27, regs->regs[i+27], i+40, regs->regs[i+40]); |
| 567 | pr_err(" r13: "REGFMT" tp : "REGFMT" sp : "REGFMT" lr : "REGFMT"\n", |
| 568 | regs->regs[13], regs->tp, regs->sp, regs->lr); |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 569 | #endif |
| 570 | pr_err(" pc : "REGFMT" ex1: %ld faultnum: %ld\n", |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 571 | regs->pc, regs->ex1, regs->faultnum); |
| 572 | |
| 573 | dump_stack_regs(regs); |
| 574 | } |