Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This program is free software; you can redistribute it and/or |
| 3 | * modify it under the terms of the GNU General Public License |
| 4 | * as published by the Free Software Foundation; either version 2 |
| 5 | * of the License, or (at your option) any later version. |
| 6 | * |
| 7 | * This program is distributed in the hope that it will be useful, |
| 8 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 9 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 10 | * GNU General Public License for more details. |
| 11 | * |
| 12 | * You should have received a copy of the GNU General Public License |
| 13 | * along with this program; if not, write to the Free Software |
| 14 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. |
| 15 | * |
| 16 | * Copyright (C) 2000, 2001 Kanoj Sarcar |
| 17 | * Copyright (C) 2000, 2001 Ralf Baechle |
| 18 | * Copyright (C) 2000, 2001 Silicon Graphics, Inc. |
| 19 | * Copyright (C) 2000, 2001, 2003 Broadcom Corporation |
| 20 | */ |
| 21 | #include <linux/cache.h> |
| 22 | #include <linux/delay.h> |
| 23 | #include <linux/init.h> |
| 24 | #include <linux/interrupt.h> |
| 25 | #include <linux/spinlock.h> |
| 26 | #include <linux/threads.h> |
| 27 | #include <linux/module.h> |
| 28 | #include <linux/time.h> |
| 29 | #include <linux/timex.h> |
| 30 | #include <linux/sched.h> |
| 31 | #include <linux/cpumask.h> |
Rojhalat Ibrahim | 1e35aab | 2006-02-20 13:35:27 +0000 | [diff] [blame] | 32 | #include <linux/cpu.h> |
Alexey Dobriyan | 4e950f6 | 2007-07-30 02:36:13 +0400 | [diff] [blame] | 33 | #include <linux/err.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 34 | |
| 35 | #include <asm/atomic.h> |
| 36 | #include <asm/cpu.h> |
| 37 | #include <asm/processor.h> |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 38 | #include <asm/r4k-timer.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 39 | #include <asm/system.h> |
| 40 | #include <asm/mmu_context.h> |
Ralf Baechle | 7bcf771 | 2007-10-11 23:46:09 +0100 | [diff] [blame] | 41 | #include <asm/time.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 42 | |
Ralf Baechle | 41c594a | 2006-04-05 09:45:45 +0100 | [diff] [blame] | 43 | #ifdef CONFIG_MIPS_MT_SMTC |
| 44 | #include <asm/mipsmtregs.h> |
| 45 | #endif /* CONFIG_MIPS_MT_SMTC */ |
| 46 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 47 | cpumask_t phys_cpu_present_map; /* Bitmask of available CPUs */ |
| 48 | volatile cpumask_t cpu_callin_map; /* Bitmask of started secondaries */ |
| 49 | cpumask_t cpu_online_map; /* Bitmask of currently online CPUs */ |
| 50 | int __cpu_number_map[NR_CPUS]; /* Map physical to logical */ |
| 51 | int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */ |
| 52 | |
| 53 | EXPORT_SYMBOL(phys_cpu_present_map); |
| 54 | EXPORT_SYMBOL(cpu_online_map); |
| 55 | |
Robert P. J. Day | b3f6df9 | 2007-05-25 14:32:28 -0400 | [diff] [blame] | 56 | extern void cpu_idle(void); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 57 | |
Ralf Baechle | 0ab7aef | 2007-03-02 20:42:04 +0000 | [diff] [blame] | 58 | /* Number of TCs (or siblings in Intel speak) per CPU core */ |
| 59 | int smp_num_siblings = 1; |
| 60 | EXPORT_SYMBOL(smp_num_siblings); |
| 61 | |
| 62 | /* representing the TCs (or siblings in Intel speak) of each logical CPU */ |
| 63 | cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly; |
| 64 | EXPORT_SYMBOL(cpu_sibling_map); |
| 65 | |
| 66 | /* representing cpus for which sibling maps can be computed */ |
| 67 | static cpumask_t cpu_sibling_setup_map; |
| 68 | |
| 69 | static inline void set_cpu_sibling_map(int cpu) |
| 70 | { |
| 71 | int i; |
| 72 | |
| 73 | cpu_set(cpu, cpu_sibling_setup_map); |
| 74 | |
| 75 | if (smp_num_siblings > 1) { |
| 76 | for_each_cpu_mask(i, cpu_sibling_setup_map) { |
| 77 | if (cpu_data[cpu].core == cpu_data[i].core) { |
| 78 | cpu_set(i, cpu_sibling_map[cpu]); |
| 79 | cpu_set(cpu, cpu_sibling_map[i]); |
| 80 | } |
| 81 | } |
| 82 | } else |
| 83 | cpu_set(cpu, cpu_sibling_map[cpu]); |
| 84 | } |
| 85 | |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 86 | struct plat_smp_ops *mp_ops; |
| 87 | |
| 88 | __cpuinit void register_smp_ops(struct plat_smp_ops *ops) |
| 89 | { |
Thiemo Seufer | 83738e3 | 2008-05-06 11:21:22 +0100 | [diff] [blame] | 90 | if (mp_ops) |
| 91 | printk(KERN_WARNING "Overriding previously set SMP ops\n"); |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 92 | |
| 93 | mp_ops = ops; |
| 94 | } |
| 95 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 96 | /* |
| 97 | * First C code run on the secondary CPUs after being started up by |
| 98 | * the master. |
| 99 | */ |
Ralf Baechle | 4ebd523 | 2007-05-31 16:15:01 +0100 | [diff] [blame] | 100 | asmlinkage __cpuinit void start_secondary(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 101 | { |
Nick Piggin | 5bfb5d6 | 2005-11-08 21:39:01 -0800 | [diff] [blame] | 102 | unsigned int cpu; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 103 | |
Ralf Baechle | 41c594a | 2006-04-05 09:45:45 +0100 | [diff] [blame] | 104 | #ifdef CONFIG_MIPS_MT_SMTC |
| 105 | /* Only do cpu_probe for first TC of CPU */ |
| 106 | if ((read_c0_tcbind() & TCBIND_CURTC) == 0) |
| 107 | #endif /* CONFIG_MIPS_MT_SMTC */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 108 | cpu_probe(); |
| 109 | cpu_report(); |
| 110 | per_cpu_trap_init(); |
Ralf Baechle | 7bcf771 | 2007-10-11 23:46:09 +0100 | [diff] [blame] | 111 | mips_clockevent_init(); |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 112 | mp_ops->init_secondary(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 113 | |
| 114 | /* |
| 115 | * XXX parity protection should be folded in here when it's converted |
| 116 | * to an option instead of something based on .cputype |
| 117 | */ |
| 118 | |
| 119 | calibrate_delay(); |
Nick Piggin | 5bfb5d6 | 2005-11-08 21:39:01 -0800 | [diff] [blame] | 120 | preempt_disable(); |
| 121 | cpu = smp_processor_id(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 122 | cpu_data[cpu].udelay_val = loops_per_jiffy; |
| 123 | |
Manfred Spraul | e545a61 | 2008-09-07 16:57:22 +0200 | [diff] [blame^] | 124 | notify_cpu_starting(cpu); |
| 125 | |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 126 | mp_ops->smp_finish(); |
Ralf Baechle | 0ab7aef | 2007-03-02 20:42:04 +0000 | [diff] [blame] | 127 | set_cpu_sibling_map(cpu); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 128 | |
| 129 | cpu_set(cpu, cpu_callin_map); |
| 130 | |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 131 | synchronise_count_slave(); |
| 132 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 133 | cpu_idle(); |
| 134 | } |
| 135 | |
Jens Axboe | 2f304c0 | 2008-06-17 10:45:23 +0200 | [diff] [blame] | 136 | void arch_send_call_function_ipi(cpumask_t mask) |
| 137 | { |
| 138 | mp_ops->send_ipi_mask(mask, SMP_CALL_FUNCTION); |
| 139 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 140 | |
| 141 | /* |
Jens Axboe | 2f304c0 | 2008-06-17 10:45:23 +0200 | [diff] [blame] | 142 | * We reuse the same vector for the single IPI |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 143 | */ |
Jens Axboe | 2f304c0 | 2008-06-17 10:45:23 +0200 | [diff] [blame] | 144 | void arch_send_call_function_single_ipi(int cpu) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 145 | { |
Jens Axboe | 2f304c0 | 2008-06-17 10:45:23 +0200 | [diff] [blame] | 146 | mp_ops->send_ipi_mask(cpumask_of_cpu(cpu), SMP_CALL_FUNCTION); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 147 | } |
| 148 | |
Jens Axboe | 2f304c0 | 2008-06-17 10:45:23 +0200 | [diff] [blame] | 149 | /* |
| 150 | * Call into both interrupt handlers, as we share the IPI for them |
| 151 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 152 | void smp_call_function_interrupt(void) |
| 153 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 154 | irq_enter(); |
Jens Axboe | 2f304c0 | 2008-06-17 10:45:23 +0200 | [diff] [blame] | 155 | generic_smp_call_function_single_interrupt(); |
| 156 | generic_smp_call_function_interrupt(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 157 | irq_exit(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 158 | } |
| 159 | |
| 160 | static void stop_this_cpu(void *dummy) |
| 161 | { |
| 162 | /* |
| 163 | * Remove this CPU: |
| 164 | */ |
| 165 | cpu_clear(smp_processor_id(), cpu_online_map); |
| 166 | local_irq_enable(); /* May need to service _machine_restart IPI */ |
| 167 | for (;;); /* Wait if available. */ |
| 168 | } |
| 169 | |
| 170 | void smp_send_stop(void) |
| 171 | { |
Jens Axboe | 8691e5a | 2008-06-06 11:18:06 +0200 | [diff] [blame] | 172 | smp_call_function(stop_this_cpu, NULL, 0); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 173 | } |
| 174 | |
| 175 | void __init smp_cpus_done(unsigned int max_cpus) |
| 176 | { |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 177 | mp_ops->cpus_done(); |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 178 | synchronise_count_master(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 179 | } |
| 180 | |
| 181 | /* called from main before smp_init() */ |
| 182 | void __init smp_prepare_cpus(unsigned int max_cpus) |
| 183 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 184 | init_new_context(current, &init_mm); |
| 185 | current_thread_info()->cpu = 0; |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 186 | mp_ops->prepare_cpus(max_cpus); |
Ralf Baechle | 0ab7aef | 2007-03-02 20:42:04 +0000 | [diff] [blame] | 187 | set_cpu_sibling_map(0); |
Ralf Baechle | 320e6ab | 2006-05-22 14:24:04 +0100 | [diff] [blame] | 188 | #ifndef CONFIG_HOTPLUG_CPU |
| 189 | cpu_present_map = cpu_possible_map; |
| 190 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 191 | } |
| 192 | |
| 193 | /* preload SMP state for boot cpu */ |
| 194 | void __devinit smp_prepare_boot_cpu(void) |
| 195 | { |
| 196 | /* |
| 197 | * This assumes that bootup is always handled by the processor |
| 198 | * with the logic and physical number 0. |
| 199 | */ |
| 200 | __cpu_number_map[0] = 0; |
| 201 | __cpu_logical_map[0] = 0; |
| 202 | cpu_set(0, phys_cpu_present_map); |
| 203 | cpu_set(0, cpu_online_map); |
| 204 | cpu_set(0, cpu_callin_map); |
| 205 | } |
| 206 | |
| 207 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 208 | * Called once for each "cpu_possible(cpu)". Needs to spin up the cpu |
| 209 | * and keep control until "cpu_online(cpu)" is set. Note: cpu is |
| 210 | * physical, not logical. |
| 211 | */ |
Gautham R Shenoy | b282b6f | 2007-01-10 23:15:34 -0800 | [diff] [blame] | 212 | int __cpuinit __cpu_up(unsigned int cpu) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 213 | { |
Ralf Baechle | b727a60 | 2005-02-22 21:18:01 +0000 | [diff] [blame] | 214 | struct task_struct *idle; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 215 | |
Ralf Baechle | b727a60 | 2005-02-22 21:18:01 +0000 | [diff] [blame] | 216 | /* |
| 217 | * Processor goes to start_secondary(), sets online flag |
| 218 | * The following code is purely to make sure |
| 219 | * Linux can schedule processes on this slave. |
| 220 | */ |
| 221 | idle = fork_idle(cpu); |
| 222 | if (IS_ERR(idle)) |
| 223 | panic(KERN_ERR "Fork failed for CPU %d", cpu); |
| 224 | |
Ralf Baechle | 87353d8 | 2007-11-19 12:23:51 +0000 | [diff] [blame] | 225 | mp_ops->boot_secondary(cpu, idle); |
Ralf Baechle | b727a60 | 2005-02-22 21:18:01 +0000 | [diff] [blame] | 226 | |
| 227 | /* |
| 228 | * Trust is futile. We should really have timeouts ... |
| 229 | */ |
| 230 | while (!cpu_isset(cpu, cpu_callin_map)) |
| 231 | udelay(100); |
| 232 | |
| 233 | cpu_set(cpu, cpu_online_map); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 234 | |
| 235 | return 0; |
| 236 | } |
| 237 | |
| 238 | /* Not really SMP stuff ... */ |
| 239 | int setup_profiling_timer(unsigned int multiplier) |
| 240 | { |
| 241 | return 0; |
| 242 | } |
| 243 | |
| 244 | static void flush_tlb_all_ipi(void *info) |
| 245 | { |
| 246 | local_flush_tlb_all(); |
| 247 | } |
| 248 | |
| 249 | void flush_tlb_all(void) |
| 250 | { |
Jens Axboe | 15c8b6c | 2008-05-09 09:39:44 +0200 | [diff] [blame] | 251 | on_each_cpu(flush_tlb_all_ipi, NULL, 1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 252 | } |
| 253 | |
| 254 | static void flush_tlb_mm_ipi(void *mm) |
| 255 | { |
| 256 | local_flush_tlb_mm((struct mm_struct *)mm); |
| 257 | } |
| 258 | |
| 259 | /* |
Ralf Baechle | 2596935 | 2006-06-22 22:42:32 +0100 | [diff] [blame] | 260 | * Special Variant of smp_call_function for use by TLB functions: |
| 261 | * |
| 262 | * o No return value |
| 263 | * o collapses to normal function call on UP kernels |
| 264 | * o collapses to normal function call on systems with a single shared |
| 265 | * primary cache. |
| 266 | * o CONFIG_MIPS_MT_SMTC currently implies there is only one physical core. |
| 267 | */ |
| 268 | static inline void smp_on_other_tlbs(void (*func) (void *info), void *info) |
| 269 | { |
| 270 | #ifndef CONFIG_MIPS_MT_SMTC |
Jens Axboe | 8691e5a | 2008-06-06 11:18:06 +0200 | [diff] [blame] | 271 | smp_call_function(func, info, 1); |
Ralf Baechle | 2596935 | 2006-06-22 22:42:32 +0100 | [diff] [blame] | 272 | #endif |
| 273 | } |
| 274 | |
| 275 | static inline void smp_on_each_tlb(void (*func) (void *info), void *info) |
| 276 | { |
| 277 | preempt_disable(); |
| 278 | |
| 279 | smp_on_other_tlbs(func, info); |
| 280 | func(info); |
| 281 | |
| 282 | preempt_enable(); |
| 283 | } |
| 284 | |
| 285 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 286 | * The following tlb flush calls are invoked when old translations are |
| 287 | * being torn down, or pte attributes are changing. For single threaded |
| 288 | * address spaces, a new context is obtained on the current cpu, and tlb |
| 289 | * context on other cpus are invalidated to force a new context allocation |
| 290 | * at switch_mm time, should the mm ever be used on other cpus. For |
| 291 | * multithreaded address spaces, intercpu interrupts have to be sent. |
| 292 | * Another case where intercpu interrupts are required is when the target |
| 293 | * mm might be active on another cpu (eg debuggers doing the flushes on |
| 294 | * behalf of debugees, kswapd stealing pages from another process etc). |
| 295 | * Kanoj 07/00. |
| 296 | */ |
| 297 | |
| 298 | void flush_tlb_mm(struct mm_struct *mm) |
| 299 | { |
| 300 | preempt_disable(); |
| 301 | |
| 302 | if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { |
Ralf Baechle | c50cade | 2007-10-04 16:57:08 +0100 | [diff] [blame] | 303 | smp_on_other_tlbs(flush_tlb_mm_ipi, mm); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 304 | } else { |
Ralf Baechle | b5eb551 | 2007-10-03 19:16:57 +0100 | [diff] [blame] | 305 | cpumask_t mask = cpu_online_map; |
| 306 | unsigned int cpu; |
| 307 | |
| 308 | cpu_clear(smp_processor_id(), mask); |
Ralf Baechle | ece8a9e | 2007-10-12 15:03:38 +0100 | [diff] [blame] | 309 | for_each_cpu_mask(cpu, mask) |
Ralf Baechle | b5eb551 | 2007-10-03 19:16:57 +0100 | [diff] [blame] | 310 | if (cpu_context(cpu, mm)) |
| 311 | cpu_context(cpu, mm) = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 312 | } |
| 313 | local_flush_tlb_mm(mm); |
| 314 | |
| 315 | preempt_enable(); |
| 316 | } |
| 317 | |
| 318 | struct flush_tlb_data { |
| 319 | struct vm_area_struct *vma; |
| 320 | unsigned long addr1; |
| 321 | unsigned long addr2; |
| 322 | }; |
| 323 | |
| 324 | static void flush_tlb_range_ipi(void *info) |
| 325 | { |
Ralf Baechle | c50cade | 2007-10-04 16:57:08 +0100 | [diff] [blame] | 326 | struct flush_tlb_data *fd = info; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 327 | |
| 328 | local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2); |
| 329 | } |
| 330 | |
| 331 | void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) |
| 332 | { |
| 333 | struct mm_struct *mm = vma->vm_mm; |
| 334 | |
| 335 | preempt_disable(); |
| 336 | if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { |
Ralf Baechle | 89a8a5a | 2007-10-04 18:18:52 +0100 | [diff] [blame] | 337 | struct flush_tlb_data fd = { |
| 338 | .vma = vma, |
| 339 | .addr1 = start, |
| 340 | .addr2 = end, |
| 341 | }; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 342 | |
Ralf Baechle | c50cade | 2007-10-04 16:57:08 +0100 | [diff] [blame] | 343 | smp_on_other_tlbs(flush_tlb_range_ipi, &fd); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 344 | } else { |
Ralf Baechle | b5eb551 | 2007-10-03 19:16:57 +0100 | [diff] [blame] | 345 | cpumask_t mask = cpu_online_map; |
| 346 | unsigned int cpu; |
| 347 | |
| 348 | cpu_clear(smp_processor_id(), mask); |
Ralf Baechle | ece8a9e | 2007-10-12 15:03:38 +0100 | [diff] [blame] | 349 | for_each_cpu_mask(cpu, mask) |
Ralf Baechle | b5eb551 | 2007-10-03 19:16:57 +0100 | [diff] [blame] | 350 | if (cpu_context(cpu, mm)) |
| 351 | cpu_context(cpu, mm) = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 352 | } |
| 353 | local_flush_tlb_range(vma, start, end); |
| 354 | preempt_enable(); |
| 355 | } |
| 356 | |
| 357 | static void flush_tlb_kernel_range_ipi(void *info) |
| 358 | { |
Ralf Baechle | c50cade | 2007-10-04 16:57:08 +0100 | [diff] [blame] | 359 | struct flush_tlb_data *fd = info; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 360 | |
| 361 | local_flush_tlb_kernel_range(fd->addr1, fd->addr2); |
| 362 | } |
| 363 | |
| 364 | void flush_tlb_kernel_range(unsigned long start, unsigned long end) |
| 365 | { |
Ralf Baechle | 89a8a5a | 2007-10-04 18:18:52 +0100 | [diff] [blame] | 366 | struct flush_tlb_data fd = { |
| 367 | .addr1 = start, |
| 368 | .addr2 = end, |
| 369 | }; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 370 | |
Jens Axboe | 15c8b6c | 2008-05-09 09:39:44 +0200 | [diff] [blame] | 371 | on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 372 | } |
| 373 | |
| 374 | static void flush_tlb_page_ipi(void *info) |
| 375 | { |
Ralf Baechle | c50cade | 2007-10-04 16:57:08 +0100 | [diff] [blame] | 376 | struct flush_tlb_data *fd = info; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 377 | |
| 378 | local_flush_tlb_page(fd->vma, fd->addr1); |
| 379 | } |
| 380 | |
| 381 | void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) |
| 382 | { |
| 383 | preempt_disable(); |
| 384 | if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) { |
Ralf Baechle | 89a8a5a | 2007-10-04 18:18:52 +0100 | [diff] [blame] | 385 | struct flush_tlb_data fd = { |
| 386 | .vma = vma, |
| 387 | .addr1 = page, |
| 388 | }; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 389 | |
Ralf Baechle | c50cade | 2007-10-04 16:57:08 +0100 | [diff] [blame] | 390 | smp_on_other_tlbs(flush_tlb_page_ipi, &fd); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 391 | } else { |
Ralf Baechle | b5eb551 | 2007-10-03 19:16:57 +0100 | [diff] [blame] | 392 | cpumask_t mask = cpu_online_map; |
| 393 | unsigned int cpu; |
| 394 | |
| 395 | cpu_clear(smp_processor_id(), mask); |
Ralf Baechle | ece8a9e | 2007-10-12 15:03:38 +0100 | [diff] [blame] | 396 | for_each_cpu_mask(cpu, mask) |
Ralf Baechle | b5eb551 | 2007-10-03 19:16:57 +0100 | [diff] [blame] | 397 | if (cpu_context(cpu, vma->vm_mm)) |
| 398 | cpu_context(cpu, vma->vm_mm) = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 399 | } |
| 400 | local_flush_tlb_page(vma, page); |
| 401 | preempt_enable(); |
| 402 | } |
| 403 | |
| 404 | static void flush_tlb_one_ipi(void *info) |
| 405 | { |
| 406 | unsigned long vaddr = (unsigned long) info; |
| 407 | |
| 408 | local_flush_tlb_one(vaddr); |
| 409 | } |
| 410 | |
| 411 | void flush_tlb_one(unsigned long vaddr) |
| 412 | { |
Ralf Baechle | 2596935 | 2006-06-22 22:42:32 +0100 | [diff] [blame] | 413 | smp_on_each_tlb(flush_tlb_one_ipi, (void *) vaddr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 414 | } |
| 415 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 416 | EXPORT_SYMBOL(flush_tlb_page); |
| 417 | EXPORT_SYMBOL(flush_tlb_one); |