blob: 02e62806501256ad63f322d26461bd34f8bcc4e9 [file] [log] [blame]
Chris Metcalf867e3592010-05-28 23:09:12 -04001/*
2 * Copyright 2010 Tilera Corporation. All Rights Reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 * NON INFRINGEMENT. See the GNU General Public License for
12 * more details.
13 */
14
15#include <linux/module.h>
16#include <linux/seq_file.h>
17#include <linux/interrupt.h>
18#include <linux/irq.h>
19#include <linux/kernel_stat.h>
20#include <linux/uaccess.h>
21#include <hv/drv_pcie_rc_intf.h>
Chris Metcalffb702b92010-06-25 16:41:11 -040022#include <arch/spr_def.h>
23#include <asm/traps.h>
24
25/* Bit-flag stored in irq_desc->chip_data to indicate HW-cleared irqs. */
26#define IS_HW_CLEARED 1
Chris Metcalf867e3592010-05-28 23:09:12 -040027
28/*
Chris Metcalf5d966112010-11-01 15:24:29 -040029 * The set of interrupts we enable for arch_local_irq_enable().
Chris Metcalf867e3592010-05-28 23:09:12 -040030 * This is initialized to have just a single interrupt that the kernel
31 * doesn't actually use as a sentinel. During kernel init,
32 * interrupts are added as the kernel gets prepared to support them.
33 * NOTE: we could probably initialize them all statically up front.
34 */
35DEFINE_PER_CPU(unsigned long long, interrupts_enabled_mask) =
36 INITIAL_INTERRUPTS_ENABLED;
37EXPORT_PER_CPU_SYMBOL(interrupts_enabled_mask);
38
Chris Metcalffb702b92010-06-25 16:41:11 -040039/* Define per-tile device interrupt statistics state. */
Chris Metcalf867e3592010-05-28 23:09:12 -040040DEFINE_PER_CPU(irq_cpustat_t, irq_stat) ____cacheline_internodealigned_in_smp;
41EXPORT_PER_CPU_SYMBOL(irq_stat);
42
Chris Metcalffb702b92010-06-25 16:41:11 -040043/*
44 * Define per-tile irq disable mask; the hardware/HV only has a single
45 * mask that we use to implement both masking and disabling.
46 */
47static DEFINE_PER_CPU(unsigned long, irq_disable_mask)
48 ____cacheline_internodealigned_in_smp;
Chris Metcalf867e3592010-05-28 23:09:12 -040049
50/*
Chris Metcalffb702b92010-06-25 16:41:11 -040051 * Per-tile IRQ nesting depth. Used to make sure we enable newly
52 * enabled IRQs before exiting the outermost interrupt.
53 */
54static DEFINE_PER_CPU(int, irq_depth);
55
56/* State for allocating IRQs on Gx. */
57#if CHIP_HAS_IPI()
58static unsigned long available_irqs = ~(1UL << IRQ_RESCHEDULE);
59static DEFINE_SPINLOCK(available_irqs_lock);
60#endif
61
62#if CHIP_HAS_IPI()
63/* Use SPRs to manipulate device interrupts. */
Chris Metcalfa78c9422010-10-14 16:23:03 -040064#define mask_irqs(irq_mask) __insn_mtspr(SPR_IPI_MASK_SET_K, irq_mask)
65#define unmask_irqs(irq_mask) __insn_mtspr(SPR_IPI_MASK_RESET_K, irq_mask)
66#define clear_irqs(irq_mask) __insn_mtspr(SPR_IPI_EVENT_RESET_K, irq_mask)
Chris Metcalffb702b92010-06-25 16:41:11 -040067#else
68/* Use HV to manipulate device interrupts. */
69#define mask_irqs(irq_mask) hv_disable_intr(irq_mask)
70#define unmask_irqs(irq_mask) hv_enable_intr(irq_mask)
71#define clear_irqs(irq_mask) hv_clear_intr(irq_mask)
72#endif
73
74/*
75 * The interrupt handling path, implemented in terms of HV interrupt
76 * emulation on TILE64 and TILEPro, and IPI hardware on TILE-Gx.
Chris Metcalf867e3592010-05-28 23:09:12 -040077 */
78void tile_dev_intr(struct pt_regs *regs, int intnum)
79{
Chris Metcalffb702b92010-06-25 16:41:11 -040080 int depth = __get_cpu_var(irq_depth)++;
81 unsigned long original_irqs;
82 unsigned long remaining_irqs;
83 struct pt_regs *old_regs;
Chris Metcalf867e3592010-05-28 23:09:12 -040084
Chris Metcalffb702b92010-06-25 16:41:11 -040085#if CHIP_HAS_IPI()
Chris Metcalf867e3592010-05-28 23:09:12 -040086 /*
Chris Metcalffb702b92010-06-25 16:41:11 -040087 * Pending interrupts are listed in an SPR. We might be
88 * nested, so be sure to only handle irqs that weren't already
89 * masked by a previous interrupt. Then, mask out the ones
90 * we're going to handle.
Chris Metcalf867e3592010-05-28 23:09:12 -040091 */
Chris Metcalfa78c9422010-10-14 16:23:03 -040092 unsigned long masked = __insn_mfspr(SPR_IPI_MASK_K);
93 original_irqs = __insn_mfspr(SPR_IPI_EVENT_K) & ~masked;
94 __insn_mtspr(SPR_IPI_MASK_SET_K, original_irqs);
Chris Metcalffb702b92010-06-25 16:41:11 -040095#else
96 /*
97 * Hypervisor performs the equivalent of the Gx code above and
98 * then puts the pending interrupt mask into a system save reg
99 * for us to find.
100 */
Chris Metcalfa78c9422010-10-14 16:23:03 -0400101 original_irqs = __insn_mfspr(SPR_SYSTEM_SAVE_K_3);
Chris Metcalffb702b92010-06-25 16:41:11 -0400102#endif
103 remaining_irqs = original_irqs;
Chris Metcalf867e3592010-05-28 23:09:12 -0400104
105 /* Track time spent here in an interrupt context. */
Chris Metcalffb702b92010-06-25 16:41:11 -0400106 old_regs = set_irq_regs(regs);
Chris Metcalf867e3592010-05-28 23:09:12 -0400107 irq_enter();
108
109#ifdef CONFIG_DEBUG_STACKOVERFLOW
110 /* Debugging check for stack overflow: less than 1/8th stack free? */
111 {
112 long sp = stack_pointer - (long) current_thread_info();
113 if (unlikely(sp < (sizeof(struct thread_info) + STACK_WARN))) {
Chris Metcalffb702b92010-06-25 16:41:11 -0400114 pr_emerg("tile_dev_intr: "
Chris Metcalf867e3592010-05-28 23:09:12 -0400115 "stack overflow: %ld\n",
116 sp - sizeof(struct thread_info));
117 dump_stack();
118 }
119 }
120#endif
Chris Metcalffb702b92010-06-25 16:41:11 -0400121 while (remaining_irqs) {
122 unsigned long irq = __ffs(remaining_irqs);
123 remaining_irqs &= ~(1UL << irq);
Chris Metcalf867e3592010-05-28 23:09:12 -0400124
Chris Metcalffb702b92010-06-25 16:41:11 -0400125 /* Count device irqs; Linux IPIs are counted elsewhere. */
126 if (irq != IRQ_RESCHEDULE)
127 __get_cpu_var(irq_stat).irq_dev_intr_count++;
Chris Metcalf867e3592010-05-28 23:09:12 -0400128
Chris Metcalffb702b92010-06-25 16:41:11 -0400129 generic_handle_irq(irq);
Chris Metcalf867e3592010-05-28 23:09:12 -0400130 }
131
132 /*
Chris Metcalffb702b92010-06-25 16:41:11 -0400133 * If we weren't nested, turn on all enabled interrupts,
134 * including any that were reenabled during interrupt
135 * handling.
136 */
137 if (depth == 0)
138 unmask_irqs(~__get_cpu_var(irq_disable_mask));
139
140 __get_cpu_var(irq_depth)--;
141
142 /*
Chris Metcalf867e3592010-05-28 23:09:12 -0400143 * Track time spent against the current process again and
144 * process any softirqs if they are waiting.
145 */
146 irq_exit();
147 set_irq_regs(old_regs);
148}
149
150
Chris Metcalffb702b92010-06-25 16:41:11 -0400151/*
152 * Remove an irq from the disabled mask. If we're in an interrupt
153 * context, defer enabling the HW interrupt until we leave.
154 */
Chris Metcalf0c905472011-12-01 12:58:19 -0500155static void tile_irq_chip_enable(struct irq_data *d)
Chris Metcalf867e3592010-05-28 23:09:12 -0400156{
Chris Metcalf0c905472011-12-01 12:58:19 -0500157 get_cpu_var(irq_disable_mask) &= ~(1UL << d->irq);
Chris Metcalffb702b92010-06-25 16:41:11 -0400158 if (__get_cpu_var(irq_depth) == 0)
Chris Metcalf0c905472011-12-01 12:58:19 -0500159 unmask_irqs(1UL << d->irq);
Chris Metcalffb702b92010-06-25 16:41:11 -0400160 put_cpu_var(irq_disable_mask);
161}
Chris Metcalffb702b92010-06-25 16:41:11 -0400162
163/*
164 * Add an irq to the disabled mask. We disable the HW interrupt
165 * immediately so that there's no possibility of it firing. If we're
166 * in an interrupt context, the return path is careful to avoid
167 * unmasking a newly disabled interrupt.
168 */
Chris Metcalf0c905472011-12-01 12:58:19 -0500169static void tile_irq_chip_disable(struct irq_data *d)
Chris Metcalffb702b92010-06-25 16:41:11 -0400170{
Chris Metcalf0c905472011-12-01 12:58:19 -0500171 get_cpu_var(irq_disable_mask) |= (1UL << d->irq);
172 mask_irqs(1UL << d->irq);
Chris Metcalffb702b92010-06-25 16:41:11 -0400173 put_cpu_var(irq_disable_mask);
174}
Chris Metcalffb702b92010-06-25 16:41:11 -0400175
176/* Mask an interrupt. */
Thomas Gleixnerf5b42c92011-02-06 23:04:40 +0000177static void tile_irq_chip_mask(struct irq_data *d)
Chris Metcalffb702b92010-06-25 16:41:11 -0400178{
Thomas Gleixnerf5b42c92011-02-06 23:04:40 +0000179 mask_irqs(1UL << d->irq);
Chris Metcalf867e3592010-05-28 23:09:12 -0400180}
181
182/* Unmask an interrupt. */
Thomas Gleixnerf5b42c92011-02-06 23:04:40 +0000183static void tile_irq_chip_unmask(struct irq_data *d)
Chris Metcalf867e3592010-05-28 23:09:12 -0400184{
Thomas Gleixnerf5b42c92011-02-06 23:04:40 +0000185 unmask_irqs(1UL << d->irq);
Chris Metcalf867e3592010-05-28 23:09:12 -0400186}
187
188/*
Chris Metcalffb702b92010-06-25 16:41:11 -0400189 * Clear an interrupt before processing it so that any new assertions
190 * will trigger another irq.
Chris Metcalf867e3592010-05-28 23:09:12 -0400191 */
Thomas Gleixnerf5b42c92011-02-06 23:04:40 +0000192static void tile_irq_chip_ack(struct irq_data *d)
Chris Metcalf867e3592010-05-28 23:09:12 -0400193{
Thomas Gleixnerf5b42c92011-02-06 23:04:40 +0000194 if ((unsigned long)irq_data_get_irq_chip_data(d) != IS_HW_CLEARED)
195 clear_irqs(1UL << d->irq);
Chris Metcalf867e3592010-05-28 23:09:12 -0400196}
197
198/*
Chris Metcalffb702b92010-06-25 16:41:11 -0400199 * For per-cpu interrupts, we need to avoid unmasking any interrupts
200 * that we disabled via disable_percpu_irq().
Chris Metcalf867e3592010-05-28 23:09:12 -0400201 */
Thomas Gleixnerf5b42c92011-02-06 23:04:40 +0000202static void tile_irq_chip_eoi(struct irq_data *d)
Chris Metcalf867e3592010-05-28 23:09:12 -0400203{
Thomas Gleixnerf5b42c92011-02-06 23:04:40 +0000204 if (!(__get_cpu_var(irq_disable_mask) & (1UL << d->irq)))
205 unmask_irqs(1UL << d->irq);
Chris Metcalf867e3592010-05-28 23:09:12 -0400206}
207
Chris Metcalffb702b92010-06-25 16:41:11 -0400208static struct irq_chip tile_irq_chip = {
Thomas Gleixnerd1ea13c2010-09-23 18:40:07 +0200209 .name = "tile_irq_chip",
Chris Metcalf0c905472011-12-01 12:58:19 -0500210 .irq_enable = tile_irq_chip_enable,
211 .irq_disable = tile_irq_chip_disable,
Thomas Gleixnerf5b42c92011-02-06 23:04:40 +0000212 .irq_ack = tile_irq_chip_ack,
213 .irq_eoi = tile_irq_chip_eoi,
214 .irq_mask = tile_irq_chip_mask,
215 .irq_unmask = tile_irq_chip_unmask,
Chris Metcalf867e3592010-05-28 23:09:12 -0400216};
217
218void __init init_IRQ(void)
219{
Chris Metcalffb702b92010-06-25 16:41:11 -0400220 ipi_init();
Chris Metcalf867e3592010-05-28 23:09:12 -0400221}
222
Chris Metcalffb702b92010-06-25 16:41:11 -0400223void __cpuinit setup_irq_regs(void)
Chris Metcalf867e3592010-05-28 23:09:12 -0400224{
Chris Metcalffb702b92010-06-25 16:41:11 -0400225 /* Enable interrupt delivery. */
226 unmask_irqs(~0UL);
227#if CHIP_HAS_IPI()
Chris Metcalf5d966112010-11-01 15:24:29 -0400228 arch_local_irq_unmask(INT_IPI_K);
Chris Metcalffb702b92010-06-25 16:41:11 -0400229#endif
Chris Metcalf867e3592010-05-28 23:09:12 -0400230}
231
Chris Metcalffb702b92010-06-25 16:41:11 -0400232void tile_irq_activate(unsigned int irq, int tile_irq_type)
Chris Metcalf867e3592010-05-28 23:09:12 -0400233{
234 /*
Chris Metcalffb702b92010-06-25 16:41:11 -0400235 * We use handle_level_irq() by default because the pending
236 * interrupt vector (whether modeled by the HV on TILE64 and
237 * TILEPro or implemented in hardware on TILE-Gx) has
238 * level-style semantics for each bit. An interrupt fires
239 * whenever a bit is high, not just at edges.
Chris Metcalf867e3592010-05-28 23:09:12 -0400240 */
Chris Metcalffb702b92010-06-25 16:41:11 -0400241 irq_flow_handler_t handle = handle_level_irq;
242 if (tile_irq_type == TILE_IRQ_PERCPU)
243 handle = handle_percpu_irq;
Thomas Gleixner1919d642011-03-25 14:21:16 +0000244 irq_set_chip_and_handler(irq, &tile_irq_chip, handle);
Chris Metcalffb702b92010-06-25 16:41:11 -0400245
246 /*
247 * Flag interrupts that are hardware-cleared so that ack()
248 * won't clear them.
249 */
250 if (tile_irq_type == TILE_IRQ_HW_CLEAR)
Thomas Gleixner1919d642011-03-25 14:21:16 +0000251 irq_set_chip_data(irq, (void *)IS_HW_CLEARED);
Chris Metcalf867e3592010-05-28 23:09:12 -0400252}
Chris Metcalffb702b92010-06-25 16:41:11 -0400253EXPORT_SYMBOL(tile_irq_activate);
254
Chris Metcalf867e3592010-05-28 23:09:12 -0400255
256void ack_bad_irq(unsigned int irq)
257{
Chris Metcalffb702b92010-06-25 16:41:11 -0400258 pr_err("unexpected IRQ trap at vector %02x\n", irq);
Chris Metcalf867e3592010-05-28 23:09:12 -0400259}
260
261/*
262 * Generic, controller-independent functions:
263 */
264
Chris Metcalffb702b92010-06-25 16:41:11 -0400265#if CHIP_HAS_IPI()
266int create_irq(void)
267{
268 unsigned long flags;
269 int result;
270
271 spin_lock_irqsave(&available_irqs_lock, flags);
272 if (available_irqs == 0)
273 result = -ENOMEM;
274 else {
275 result = __ffs(available_irqs);
276 available_irqs &= ~(1UL << result);
277 dynamic_irq_init(result);
278 }
279 spin_unlock_irqrestore(&available_irqs_lock, flags);
280
281 return result;
282}
283EXPORT_SYMBOL(create_irq);
284
285void destroy_irq(unsigned int irq)
286{
287 unsigned long flags;
288
289 spin_lock_irqsave(&available_irqs_lock, flags);
290 available_irqs |= (1UL << irq);
291 dynamic_irq_cleanup(irq);
292 spin_unlock_irqrestore(&available_irqs_lock, flags);
293}
294EXPORT_SYMBOL(destroy_irq);
295#endif