blob: 347f17edad77bab15f1e5ca286a833a3a8392906 [file] [log] [blame]
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001/*
2 * Xen event channels
3 *
4 * Xen models interrupts with abstract event channels. Because each
5 * domain gets 1024 event channels, but NR_IRQ is not that large, we
6 * must dynamically map irqs<->event channels. The event channels
7 * interface with the rest of the kernel by defining a xen interrupt
8 * chip. When an event is recieved, it is mapped to an irq and sent
9 * through the normal interrupt processing path.
10 *
11 * There are four kinds of events which can be mapped to an event
12 * channel:
13 *
14 * 1. Inter-domain notifications. This includes all the virtual
15 * device events, since they're driven by front-ends in another domain
16 * (typically dom0).
17 * 2. VIRQs, typically used for timers. These are per-cpu events.
18 * 3. IPIs.
19 * 4. Hardware interrupts. Not supported at present.
20 *
21 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
22 */
23
24#include <linux/linkage.h>
25#include <linux/interrupt.h>
26#include <linux/irq.h>
27#include <linux/module.h>
28#include <linux/string.h>
Christophe Saout28e08862009-01-11 11:46:23 -080029#include <linux/bootmem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090030#include <linux/slab.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070031
Sheng Yang38e20b02010-05-14 12:40:51 +010032#include <asm/desc.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070033#include <asm/ptrace.h>
34#include <asm/irq.h>
Jeremy Fitzhardinge792dc4f2009-02-06 14:09:43 -080035#include <asm/idle.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070036#include <asm/sync_bitops.h>
37#include <asm/xen/hypercall.h>
Adrian Bunk8d1b8752007-07-20 00:31:44 -070038#include <asm/xen/hypervisor.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070039
Sheng Yang38e20b02010-05-14 12:40:51 +010040#include <xen/xen.h>
41#include <xen/hvm.h>
Isaku Yamahatae04d0d02008-04-02 10:53:55 -070042#include <xen/xen-ops.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070043#include <xen/events.h>
44#include <xen/interface/xen.h>
45#include <xen/interface/event_channel.h>
Sheng Yang38e20b02010-05-14 12:40:51 +010046#include <xen/interface/hvm/hvm_op.h>
47#include <xen/interface/hvm/params.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070048
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070049/*
50 * This lock protects updates to the following mapping and reference-count
51 * arrays. The lock does not need to be acquired to read the mapping tables.
52 */
53static DEFINE_SPINLOCK(irq_mapping_update_lock);
54
55/* IRQ <-> VIRQ mapping. */
Tejun Heo204fba42009-06-24 15:13:45 +090056static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1};
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070057
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -070058/* IRQ <-> IPI mapping */
Tejun Heo204fba42009-06-24 15:13:45 +090059static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1};
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -070060
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -080061/* Interrupt types. */
62enum xen_irq_type {
Jeremy Fitzhardinged77bbd42009-02-06 14:09:45 -080063 IRQT_UNBOUND = 0,
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -070064 IRQT_PIRQ,
65 IRQT_VIRQ,
66 IRQT_IPI,
67 IRQT_EVTCHN
68};
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070069
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -080070/*
71 * Packed IRQ information:
72 * type - enum xen_irq_type
73 * event channel - irq->event channel mapping
74 * cpu - cpu this event channel is bound to
75 * index - type-specific information:
76 * PIRQ - vector, with MSB being "needs EIO"
77 * VIRQ - virq number
78 * IPI - IPI vector
79 * EVTCHN -
80 */
81struct irq_info
82{
83 enum xen_irq_type type; /* type */
84 unsigned short evtchn; /* event channel */
85 unsigned short cpu; /* cpu bound */
86
87 union {
88 unsigned short virq;
89 enum ipi_vector ipi;
90 struct {
91 unsigned short gsi;
92 unsigned short vector;
93 } pirq;
94 } u;
95};
96
97static struct irq_info irq_info[NR_IRQS];
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070098
99static int evtchn_to_irq[NR_EVENT_CHANNELS] = {
100 [0 ... NR_EVENT_CHANNELS-1] = -1
101};
Mike Travisc7a35892009-01-10 21:58:11 -0800102struct cpu_evtchn_s {
103 unsigned long bits[NR_EVENT_CHANNELS/BITS_PER_LONG];
104};
105static struct cpu_evtchn_s *cpu_evtchn_mask_p;
106static inline unsigned long *cpu_evtchn_mask(int cpu)
107{
108 return cpu_evtchn_mask_p[cpu].bits;
109}
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700110
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700111/* Xen will never allocate port zero for any purpose. */
112#define VALID_EVTCHN(chn) ((chn) != 0)
113
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700114static struct irq_chip xen_dynamic_chip;
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -0700115static struct irq_chip xen_percpu_chip;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700116
117/* Constructor for packed IRQ information. */
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800118static struct irq_info mk_unbound_info(void)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700119{
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800120 return (struct irq_info) { .type = IRQT_UNBOUND };
121}
122
123static struct irq_info mk_evtchn_info(unsigned short evtchn)
124{
Ian Campbell90af9512009-02-06 16:55:58 -0800125 return (struct irq_info) { .type = IRQT_EVTCHN, .evtchn = evtchn,
126 .cpu = 0 };
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800127}
128
129static struct irq_info mk_ipi_info(unsigned short evtchn, enum ipi_vector ipi)
130{
131 return (struct irq_info) { .type = IRQT_IPI, .evtchn = evtchn,
Ian Campbell90af9512009-02-06 16:55:58 -0800132 .cpu = 0, .u.ipi = ipi };
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800133}
134
135static struct irq_info mk_virq_info(unsigned short evtchn, unsigned short virq)
136{
137 return (struct irq_info) { .type = IRQT_VIRQ, .evtchn = evtchn,
Ian Campbell90af9512009-02-06 16:55:58 -0800138 .cpu = 0, .u.virq = virq };
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800139}
140
141static struct irq_info mk_pirq_info(unsigned short evtchn,
142 unsigned short gsi, unsigned short vector)
143{
144 return (struct irq_info) { .type = IRQT_PIRQ, .evtchn = evtchn,
Ian Campbell90af9512009-02-06 16:55:58 -0800145 .cpu = 0, .u.pirq = { .gsi = gsi, .vector = vector } };
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700146}
147
148/*
149 * Accessors for packed IRQ information.
150 */
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800151static struct irq_info *info_for_irq(unsigned irq)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700152{
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800153 return &irq_info[irq];
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700154}
155
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800156static unsigned int evtchn_from_irq(unsigned irq)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700157{
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800158 return info_for_irq(irq)->evtchn;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700159}
160
Ian Campbelld4c04532009-02-06 19:20:31 -0800161unsigned irq_from_evtchn(unsigned int evtchn)
162{
163 return evtchn_to_irq[evtchn];
164}
165EXPORT_SYMBOL_GPL(irq_from_evtchn);
166
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800167static enum ipi_vector ipi_from_irq(unsigned irq)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700168{
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800169 struct irq_info *info = info_for_irq(irq);
170
171 BUG_ON(info == NULL);
172 BUG_ON(info->type != IRQT_IPI);
173
174 return info->u.ipi;
175}
176
177static unsigned virq_from_irq(unsigned irq)
178{
179 struct irq_info *info = info_for_irq(irq);
180
181 BUG_ON(info == NULL);
182 BUG_ON(info->type != IRQT_VIRQ);
183
184 return info->u.virq;
185}
186
187static unsigned gsi_from_irq(unsigned irq)
188{
189 struct irq_info *info = info_for_irq(irq);
190
191 BUG_ON(info == NULL);
192 BUG_ON(info->type != IRQT_PIRQ);
193
194 return info->u.pirq.gsi;
195}
196
197static unsigned vector_from_irq(unsigned irq)
198{
199 struct irq_info *info = info_for_irq(irq);
200
201 BUG_ON(info == NULL);
202 BUG_ON(info->type != IRQT_PIRQ);
203
204 return info->u.pirq.vector;
205}
206
207static enum xen_irq_type type_from_irq(unsigned irq)
208{
209 return info_for_irq(irq)->type;
210}
211
212static unsigned cpu_from_irq(unsigned irq)
213{
214 return info_for_irq(irq)->cpu;
215}
216
217static unsigned int cpu_from_evtchn(unsigned int evtchn)
218{
219 int irq = evtchn_to_irq[evtchn];
220 unsigned ret = 0;
221
222 if (irq != -1)
223 ret = cpu_from_irq(irq);
224
225 return ret;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700226}
227
228static inline unsigned long active_evtchns(unsigned int cpu,
229 struct shared_info *sh,
230 unsigned int idx)
231{
232 return (sh->evtchn_pending[idx] &
Mike Travisc7a35892009-01-10 21:58:11 -0800233 cpu_evtchn_mask(cpu)[idx] &
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700234 ~sh->evtchn_mask[idx]);
235}
236
237static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu)
238{
239 int irq = evtchn_to_irq[chn];
240
241 BUG_ON(irq == -1);
242#ifdef CONFIG_SMP
Mike Travis7f7ace02009-01-10 21:58:08 -0800243 cpumask_copy(irq_to_desc(irq)->affinity, cpumask_of(cpu));
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700244#endif
245
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800246 __clear_bit(chn, cpu_evtchn_mask(cpu_from_irq(irq)));
Mike Travisc7a35892009-01-10 21:58:11 -0800247 __set_bit(chn, cpu_evtchn_mask(cpu));
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700248
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800249 irq_info[irq].cpu = cpu;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700250}
251
252static void init_evtchn_cpu_bindings(void)
253{
254#ifdef CONFIG_SMP
Thomas Gleixner10e58082008-10-16 14:19:04 +0200255 struct irq_desc *desc;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700256 int i;
Thomas Gleixner10e58082008-10-16 14:19:04 +0200257
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700258 /* By default all event channels notify CPU#0. */
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -0800259 for_each_irq_desc(i, desc) {
Mike Travis7f7ace02009-01-10 21:58:08 -0800260 cpumask_copy(desc->affinity, cpumask_of(0));
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -0800261 }
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700262#endif
263
Ian Campbellb0097ad2010-10-08 16:59:12 +0100264 memset(cpu_evtchn_mask(0), ~0, sizeof(struct cpu_evtchn_s));
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700265}
266
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700267static inline void clear_evtchn(int port)
268{
269 struct shared_info *s = HYPERVISOR_shared_info;
270 sync_clear_bit(port, &s->evtchn_pending[0]);
271}
272
273static inline void set_evtchn(int port)
274{
275 struct shared_info *s = HYPERVISOR_shared_info;
276 sync_set_bit(port, &s->evtchn_pending[0]);
277}
278
Jeremy Fitzhardinge168d2f42008-08-20 17:02:18 -0700279static inline int test_evtchn(int port)
280{
281 struct shared_info *s = HYPERVISOR_shared_info;
282 return sync_test_bit(port, &s->evtchn_pending[0]);
283}
284
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700285
286/**
287 * notify_remote_via_irq - send event to remote end of event channel via irq
288 * @irq: irq of event channel to send event to
289 *
290 * Unlike notify_remote_via_evtchn(), this is safe to use across
291 * save/restore. Notifications on a broken connection are silently
292 * dropped.
293 */
294void notify_remote_via_irq(int irq)
295{
296 int evtchn = evtchn_from_irq(irq);
297
298 if (VALID_EVTCHN(evtchn))
299 notify_remote_via_evtchn(evtchn);
300}
301EXPORT_SYMBOL_GPL(notify_remote_via_irq);
302
303static void mask_evtchn(int port)
304{
305 struct shared_info *s = HYPERVISOR_shared_info;
306 sync_set_bit(port, &s->evtchn_mask[0]);
307}
308
309static void unmask_evtchn(int port)
310{
311 struct shared_info *s = HYPERVISOR_shared_info;
312 unsigned int cpu = get_cpu();
313
314 BUG_ON(!irqs_disabled());
315
316 /* Slow path (hypercall) if this is a non-local port. */
317 if (unlikely(cpu != cpu_from_evtchn(port))) {
318 struct evtchn_unmask unmask = { .port = port };
319 (void)HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask);
320 } else {
321 struct vcpu_info *vcpu_info = __get_cpu_var(xen_vcpu);
322
323 sync_clear_bit(port, &s->evtchn_mask[0]);
324
325 /*
326 * The following is basically the equivalent of
327 * 'hw_resend_irq'. Just like a real IO-APIC we 'lose
328 * the interrupt edge' if the channel is masked.
329 */
330 if (sync_test_bit(port, &s->evtchn_pending[0]) &&
331 !sync_test_and_set_bit(port / BITS_PER_LONG,
332 &vcpu_info->evtchn_pending_sel))
333 vcpu_info->evtchn_upcall_pending = 1;
334 }
335
336 put_cpu();
337}
338
339static int find_unbound_irq(void)
340{
Thomas Gleixner77dff1c2010-09-29 17:37:10 +0200341 struct irq_data *data;
342 int irq, res;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700343
Stefano Stabellini99ad1982010-05-14 12:41:20 +0100344 for (irq = 0; irq < nr_irqs; irq++) {
Thomas Gleixner77dff1c2010-09-29 17:37:10 +0200345 data = irq_get_irq_data(irq);
Stefano Stabellini99ad1982010-05-14 12:41:20 +0100346 /* only 0->15 have init'd desc; handle irq > 16 */
Thomas Gleixner77dff1c2010-09-29 17:37:10 +0200347 if (!data)
Stefano Stabellini99ad1982010-05-14 12:41:20 +0100348 break;
Thomas Gleixner77dff1c2010-09-29 17:37:10 +0200349 if (data->chip == &no_irq_chip)
Stefano Stabellini99ad1982010-05-14 12:41:20 +0100350 break;
Thomas Gleixner77dff1c2010-09-29 17:37:10 +0200351 if (data->chip != &xen_dynamic_chip)
Stefano Stabellini99ad1982010-05-14 12:41:20 +0100352 continue;
Jeremy Fitzhardinged77bbd42009-02-06 14:09:45 -0800353 if (irq_info[irq].type == IRQT_UNBOUND)
Thomas Gleixner77dff1c2010-09-29 17:37:10 +0200354 return irq;
Stefano Stabellini99ad1982010-05-14 12:41:20 +0100355 }
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700356
Yinghai Lu5a15d7e2008-08-19 20:49:57 -0700357 if (irq == nr_irqs)
358 panic("No available IRQ to bind to: increase nr_irqs!\n");
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700359
Thomas Gleixner77dff1c2010-09-29 17:37:10 +0200360 res = irq_alloc_desc_at(irq, 0);
Jeremy Fitzhardinge6f8a0ed2008-12-17 13:42:29 -0800361
Thomas Gleixner77dff1c2010-09-29 17:37:10 +0200362 if (WARN_ON(res != irq))
363 return -1;
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800364
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700365 return irq;
366}
367
Jeremy Fitzhardingeb536b4b2007-07-17 18:37:06 -0700368int bind_evtchn_to_irq(unsigned int evtchn)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700369{
370 int irq;
371
372 spin_lock(&irq_mapping_update_lock);
373
374 irq = evtchn_to_irq[evtchn];
375
376 if (irq == -1) {
377 irq = find_unbound_irq();
378
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700379 set_irq_chip_and_handler_name(irq, &xen_dynamic_chip,
Jeremy Fitzhardinge3588fe22010-08-27 17:30:24 -0700380 handle_fasteoi_irq, "event");
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700381
382 evtchn_to_irq[evtchn] = irq;
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800383 irq_info[irq] = mk_evtchn_info(evtchn);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700384 }
385
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700386 spin_unlock(&irq_mapping_update_lock);
387
388 return irq;
389}
Jeremy Fitzhardingeb536b4b2007-07-17 18:37:06 -0700390EXPORT_SYMBOL_GPL(bind_evtchn_to_irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700391
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700392static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
393{
394 struct evtchn_bind_ipi bind_ipi;
395 int evtchn, irq;
396
397 spin_lock(&irq_mapping_update_lock);
398
399 irq = per_cpu(ipi_to_irq, cpu)[ipi];
Ian Campbell90af9512009-02-06 16:55:58 -0800400
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700401 if (irq == -1) {
402 irq = find_unbound_irq();
403 if (irq < 0)
404 goto out;
405
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -0700406 set_irq_chip_and_handler_name(irq, &xen_percpu_chip,
407 handle_percpu_irq, "ipi");
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700408
409 bind_ipi.vcpu = cpu;
410 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
411 &bind_ipi) != 0)
412 BUG();
413 evtchn = bind_ipi.port;
414
415 evtchn_to_irq[evtchn] = irq;
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800416 irq_info[irq] = mk_ipi_info(evtchn, ipi);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700417 per_cpu(ipi_to_irq, cpu)[ipi] = irq;
418
419 bind_evtchn_to_cpu(evtchn, cpu);
420 }
421
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700422 out:
423 spin_unlock(&irq_mapping_update_lock);
424 return irq;
425}
426
427
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700428static int bind_virq_to_irq(unsigned int virq, unsigned int cpu)
429{
430 struct evtchn_bind_virq bind_virq;
431 int evtchn, irq;
432
433 spin_lock(&irq_mapping_update_lock);
434
435 irq = per_cpu(virq_to_irq, cpu)[virq];
436
437 if (irq == -1) {
Jeremy Fitzhardingea52521f2010-09-22 15:28:52 -0700438 irq = find_unbound_irq();
439
440 set_irq_chip_and_handler_name(irq, &xen_percpu_chip,
441 handle_percpu_irq, "virq");
442
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700443 bind_virq.virq = virq;
444 bind_virq.vcpu = cpu;
445 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
446 &bind_virq) != 0)
447 BUG();
448 evtchn = bind_virq.port;
449
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700450 evtchn_to_irq[evtchn] = irq;
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800451 irq_info[irq] = mk_virq_info(evtchn, virq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700452
453 per_cpu(virq_to_irq, cpu)[virq] = irq;
454
455 bind_evtchn_to_cpu(evtchn, cpu);
456 }
457
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700458 spin_unlock(&irq_mapping_update_lock);
459
460 return irq;
461}
462
463static void unbind_from_irq(unsigned int irq)
464{
465 struct evtchn_close close;
466 int evtchn = evtchn_from_irq(irq);
467
468 spin_lock(&irq_mapping_update_lock);
469
Jeremy Fitzhardinged77bbd42009-02-06 14:09:45 -0800470 if (VALID_EVTCHN(evtchn)) {
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700471 close.port = evtchn;
472 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
473 BUG();
474
475 switch (type_from_irq(irq)) {
476 case IRQT_VIRQ:
477 per_cpu(virq_to_irq, cpu_from_evtchn(evtchn))
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800478 [virq_from_irq(irq)] = -1;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700479 break;
Alex Nixond68d82a2008-08-22 11:52:15 +0100480 case IRQT_IPI:
481 per_cpu(ipi_to_irq, cpu_from_evtchn(evtchn))
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800482 [ipi_from_irq(irq)] = -1;
Alex Nixond68d82a2008-08-22 11:52:15 +0100483 break;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700484 default:
485 break;
486 }
487
488 /* Closed ports are implicitly re-bound to VCPU0. */
489 bind_evtchn_to_cpu(evtchn, 0);
490
491 evtchn_to_irq[evtchn] = -1;
Ian Campbellfed5ea82009-12-01 16:15:30 +0000492 }
493
494 if (irq_info[irq].type != IRQT_UNBOUND) {
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800495 irq_info[irq] = mk_unbound_info();
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700496
Thomas Gleixner77dff1c2010-09-29 17:37:10 +0200497 irq_free_desc(irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700498 }
499
500 spin_unlock(&irq_mapping_update_lock);
501}
502
503int bind_evtchn_to_irqhandler(unsigned int evtchn,
Jeff Garzik7c239972007-10-19 03:12:20 -0400504 irq_handler_t handler,
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700505 unsigned long irqflags,
506 const char *devname, void *dev_id)
507{
508 unsigned int irq;
509 int retval;
510
511 irq = bind_evtchn_to_irq(evtchn);
512 retval = request_irq(irq, handler, irqflags, devname, dev_id);
513 if (retval != 0) {
514 unbind_from_irq(irq);
515 return retval;
516 }
517
518 return irq;
519}
520EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler);
521
522int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
Jeff Garzik7c239972007-10-19 03:12:20 -0400523 irq_handler_t handler,
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700524 unsigned long irqflags, const char *devname, void *dev_id)
525{
526 unsigned int irq;
527 int retval;
528
529 irq = bind_virq_to_irq(virq, cpu);
530 retval = request_irq(irq, handler, irqflags, devname, dev_id);
531 if (retval != 0) {
532 unbind_from_irq(irq);
533 return retval;
534 }
535
536 return irq;
537}
538EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler);
539
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700540int bind_ipi_to_irqhandler(enum ipi_vector ipi,
541 unsigned int cpu,
542 irq_handler_t handler,
543 unsigned long irqflags,
544 const char *devname,
545 void *dev_id)
546{
547 int irq, retval;
548
549 irq = bind_ipi_to_irq(ipi, cpu);
550 if (irq < 0)
551 return irq;
552
Ian Campbell4877c732010-07-29 11:16:35 +0100553 irqflags |= IRQF_NO_SUSPEND;
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700554 retval = request_irq(irq, handler, irqflags, devname, dev_id);
555 if (retval != 0) {
556 unbind_from_irq(irq);
557 return retval;
558 }
559
560 return irq;
561}
562
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700563void unbind_from_irqhandler(unsigned int irq, void *dev_id)
564{
565 free_irq(irq, dev_id);
566 unbind_from_irq(irq);
567}
568EXPORT_SYMBOL_GPL(unbind_from_irqhandler);
569
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700570void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
571{
572 int irq = per_cpu(ipi_to_irq, cpu)[vector];
573 BUG_ON(irq < 0);
574 notify_remote_via_irq(irq);
575}
576
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -0700577irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
578{
579 struct shared_info *sh = HYPERVISOR_shared_info;
580 int cpu = smp_processor_id();
Ian Campbellcb52e6d2010-10-15 11:52:46 +0100581 unsigned long *cpu_evtchn = cpu_evtchn_mask(cpu);
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -0700582 int i;
583 unsigned long flags;
584 static DEFINE_SPINLOCK(debug_lock);
Ian Campbellcb52e6d2010-10-15 11:52:46 +0100585 struct vcpu_info *v;
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -0700586
587 spin_lock_irqsave(&debug_lock, flags);
588
Ian Campbellcb52e6d2010-10-15 11:52:46 +0100589 printk("\nvcpu %d\n ", cpu);
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -0700590
591 for_each_online_cpu(i) {
Ian Campbellcb52e6d2010-10-15 11:52:46 +0100592 int pending;
593 v = per_cpu(xen_vcpu, i);
594 pending = (get_irq_regs() && i == cpu)
595 ? xen_irqs_disabled(get_irq_regs())
596 : v->evtchn_upcall_mask;
597 printk("%d: masked=%d pending=%d event_sel %0*lx\n ", i,
598 pending, v->evtchn_upcall_pending,
599 (int)(sizeof(v->evtchn_pending_sel)*2),
600 v->evtchn_pending_sel);
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -0700601 }
Ian Campbellcb52e6d2010-10-15 11:52:46 +0100602 v = per_cpu(xen_vcpu, cpu);
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -0700603
Ian Campbellcb52e6d2010-10-15 11:52:46 +0100604 printk("\npending:\n ");
605 for (i = ARRAY_SIZE(sh->evtchn_pending)-1; i >= 0; i--)
606 printk("%0*lx%s", (int)sizeof(sh->evtchn_pending[0])*2,
607 sh->evtchn_pending[i],
608 i % 8 == 0 ? "\n " : " ");
609 printk("\nglobal mask:\n ");
610 for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
611 printk("%0*lx%s",
612 (int)(sizeof(sh->evtchn_mask[0])*2),
613 sh->evtchn_mask[i],
614 i % 8 == 0 ? "\n " : " ");
615
616 printk("\nglobally unmasked:\n ");
617 for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
618 printk("%0*lx%s", (int)(sizeof(sh->evtchn_mask[0])*2),
619 sh->evtchn_pending[i] & ~sh->evtchn_mask[i],
620 i % 8 == 0 ? "\n " : " ");
621
622 printk("\nlocal cpu%d mask:\n ", cpu);
623 for (i = (NR_EVENT_CHANNELS/BITS_PER_LONG)-1; i >= 0; i--)
624 printk("%0*lx%s", (int)(sizeof(cpu_evtchn[0])*2),
625 cpu_evtchn[i],
626 i % 8 == 0 ? "\n " : " ");
627
628 printk("\nlocally unmasked:\n ");
629 for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--) {
630 unsigned long pending = sh->evtchn_pending[i]
631 & ~sh->evtchn_mask[i]
632 & cpu_evtchn[i];
633 printk("%0*lx%s", (int)(sizeof(sh->evtchn_mask[0])*2),
634 pending, i % 8 == 0 ? "\n " : " ");
635 }
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -0700636
637 printk("\npending list:\n");
Ian Campbellcb52e6d2010-10-15 11:52:46 +0100638 for (i = 0; i < NR_EVENT_CHANNELS; i++) {
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -0700639 if (sync_test_bit(i, sh->evtchn_pending)) {
Ian Campbellcb52e6d2010-10-15 11:52:46 +0100640 int word_idx = i / BITS_PER_LONG;
641 printk(" %d: event %d -> irq %d%s%s%s\n",
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800642 cpu_from_evtchn(i), i,
Ian Campbellcb52e6d2010-10-15 11:52:46 +0100643 evtchn_to_irq[i],
644 sync_test_bit(word_idx, &v->evtchn_pending_sel)
645 ? "" : " l2-clear",
646 !sync_test_bit(i, sh->evtchn_mask)
647 ? "" : " globally-masked",
648 sync_test_bit(i, cpu_evtchn)
649 ? "" : " locally-masked");
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -0700650 }
651 }
652
653 spin_unlock_irqrestore(&debug_lock, flags);
654
655 return IRQ_HANDLED;
656}
657
Tejun Heo245b2e72009-06-24 15:13:48 +0900658static DEFINE_PER_CPU(unsigned, xed_nesting_count);
659
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700660/*
661 * Search the CPUs pending events bitmasks. For each one found, map
662 * the event number to an irq, and feed it into do_IRQ() for
663 * handling.
664 *
665 * Xen uses a two-level bitmap to speed searching. The first level is
666 * a bitset of words which contain pending event bits. The second
667 * level is a bitset of pending events themselves.
668 */
Sheng Yang38e20b02010-05-14 12:40:51 +0100669static void __xen_evtchn_do_upcall(void)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700670{
671 int cpu = get_cpu();
672 struct shared_info *s = HYPERVISOR_shared_info;
673 struct vcpu_info *vcpu_info = __get_cpu_var(xen_vcpu);
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -0700674 unsigned count;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700675
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -0700676 do {
677 unsigned long pending_words;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700678
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -0700679 vcpu_info->evtchn_upcall_pending = 0;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700680
Tejun Heo245b2e72009-06-24 15:13:48 +0900681 if (__get_cpu_var(xed_nesting_count)++)
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -0700682 goto out;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700683
Isaku Yamahatae849c3e2008-04-02 10:53:56 -0700684#ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */
685 /* Clear master flag /before/ clearing selector flag. */
Isaku Yamahata6673cf62008-06-16 14:58:13 -0700686 wmb();
Isaku Yamahatae849c3e2008-04-02 10:53:56 -0700687#endif
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -0700688 pending_words = xchg(&vcpu_info->evtchn_pending_sel, 0);
689 while (pending_words != 0) {
690 unsigned long pending_bits;
691 int word_idx = __ffs(pending_words);
692 pending_words &= ~(1UL << word_idx);
693
694 while ((pending_bits = active_evtchns(cpu, s, word_idx)) != 0) {
695 int bit_idx = __ffs(pending_bits);
696 int port = (word_idx * BITS_PER_LONG) + bit_idx;
697 int irq = evtchn_to_irq[port];
Eric W. Biedermanca4dbc62010-02-17 18:49:54 -0800698 struct irq_desc *desc;
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -0700699
Jeremy Fitzhardinge3588fe22010-08-27 17:30:24 -0700700 mask_evtchn(port);
701 clear_evtchn(port);
702
Eric W. Biedermanca4dbc62010-02-17 18:49:54 -0800703 if (irq != -1) {
704 desc = irq_to_desc(irq);
705 if (desc)
706 generic_handle_irq_desc(irq, desc);
707 }
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700708 }
709 }
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700710
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -0700711 BUG_ON(!irqs_disabled());
712
Tejun Heo245b2e72009-06-24 15:13:48 +0900713 count = __get_cpu_var(xed_nesting_count);
714 __get_cpu_var(xed_nesting_count) = 0;
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100715 } while (count != 1 || vcpu_info->evtchn_upcall_pending);
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -0700716
717out:
Jeremy Fitzhardinge3445a8f2009-02-06 14:09:46 -0800718
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700719 put_cpu();
720}
721
Sheng Yang38e20b02010-05-14 12:40:51 +0100722void xen_evtchn_do_upcall(struct pt_regs *regs)
723{
724 struct pt_regs *old_regs = set_irq_regs(regs);
725
726 exit_idle();
727 irq_enter();
728
729 __xen_evtchn_do_upcall();
730
731 irq_exit();
732 set_irq_regs(old_regs);
733}
734
735void xen_hvm_evtchn_do_upcall(void)
736{
737 __xen_evtchn_do_upcall();
738}
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100739EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall);
Sheng Yang38e20b02010-05-14 12:40:51 +0100740
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +0100741/* Rebind a new event channel to an existing irq. */
742void rebind_evtchn_irq(int evtchn, int irq)
743{
Jeremy Fitzhardinged77bbd42009-02-06 14:09:45 -0800744 struct irq_info *info = info_for_irq(irq);
745
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +0100746 /* Make sure the irq is masked, since the new event channel
747 will also be masked. */
748 disable_irq(irq);
749
750 spin_lock(&irq_mapping_update_lock);
751
752 /* After resume the irq<->evtchn mappings are all cleared out */
753 BUG_ON(evtchn_to_irq[evtchn] != -1);
754 /* Expect irq to have been bound before,
Jeremy Fitzhardinged77bbd42009-02-06 14:09:45 -0800755 so there should be a proper type */
756 BUG_ON(info->type == IRQT_UNBOUND);
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +0100757
758 evtchn_to_irq[evtchn] = irq;
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800759 irq_info[irq] = mk_evtchn_info(evtchn);
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +0100760
761 spin_unlock(&irq_mapping_update_lock);
762
763 /* new event channels are always bound to cpu 0 */
Rusty Russell0de26522008-12-13 21:20:26 +1030764 irq_set_affinity(irq, cpumask_of(0));
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +0100765
766 /* Unmask the event channel. */
767 enable_irq(irq);
768}
769
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700770/* Rebind an evtchn so that it gets delivered to a specific cpu */
Yinghai Lud5dedd42009-04-27 17:59:21 -0700771static int rebind_irq_to_cpu(unsigned irq, unsigned tcpu)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700772{
773 struct evtchn_bind_vcpu bind_vcpu;
774 int evtchn = evtchn_from_irq(irq);
775
Stefano Stabellini183d03c2010-05-17 17:08:21 +0100776 /* events delivered via platform PCI interrupts are always
777 * routed to vcpu 0 */
778 if (!VALID_EVTCHN(evtchn) ||
779 (xen_hvm_domain() && !xen_have_vector_callback))
Yinghai Lud5dedd42009-04-27 17:59:21 -0700780 return -1;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700781
782 /* Send future instances of this interrupt to other vcpu. */
783 bind_vcpu.port = evtchn;
784 bind_vcpu.vcpu = tcpu;
785
786 /*
787 * If this fails, it usually just indicates that we're dealing with a
788 * virq or IPI channel, which don't actually need to be rebound. Ignore
789 * it, but don't do the xenlinux-level rebind in that case.
790 */
791 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0)
792 bind_evtchn_to_cpu(evtchn, tcpu);
Yinghai Lud5dedd42009-04-27 17:59:21 -0700793
794 return 0;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700795}
796
Yinghai Lud5dedd42009-04-27 17:59:21 -0700797static int set_affinity_irq(unsigned irq, const struct cpumask *dest)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700798{
Rusty Russell0de26522008-12-13 21:20:26 +1030799 unsigned tcpu = cpumask_first(dest);
Yinghai Lud5dedd42009-04-27 17:59:21 -0700800
801 return rebind_irq_to_cpu(irq, tcpu);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700802}
803
Isaku Yamahata642e0c82008-04-02 10:53:57 -0700804int resend_irq_on_evtchn(unsigned int irq)
805{
806 int masked, evtchn = evtchn_from_irq(irq);
807 struct shared_info *s = HYPERVISOR_shared_info;
808
809 if (!VALID_EVTCHN(evtchn))
810 return 1;
811
812 masked = sync_test_and_set_bit(evtchn, s->evtchn_mask);
813 sync_set_bit(evtchn, s->evtchn_pending);
814 if (!masked)
815 unmask_evtchn(evtchn);
816
817 return 1;
818}
819
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700820static void enable_dynirq(unsigned int irq)
821{
822 int evtchn = evtchn_from_irq(irq);
823
824 if (VALID_EVTCHN(evtchn))
825 unmask_evtchn(evtchn);
826}
827
828static void disable_dynirq(unsigned int irq)
829{
830 int evtchn = evtchn_from_irq(irq);
831
832 if (VALID_EVTCHN(evtchn))
833 mask_evtchn(evtchn);
834}
835
836static void ack_dynirq(unsigned int irq)
837{
838 int evtchn = evtchn_from_irq(irq);
839
Jeremy Fitzhardinge3588fe22010-08-27 17:30:24 -0700840 move_masked_irq(irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700841
842 if (VALID_EVTCHN(evtchn))
Jeremy Fitzhardinge3588fe22010-08-27 17:30:24 -0700843 unmask_evtchn(evtchn);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700844}
845
846static int retrigger_dynirq(unsigned int irq)
847{
848 int evtchn = evtchn_from_irq(irq);
Jeremy Fitzhardingeee8fa1c2008-03-17 16:37:19 -0700849 struct shared_info *sh = HYPERVISOR_shared_info;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700850 int ret = 0;
851
852 if (VALID_EVTCHN(evtchn)) {
Jeremy Fitzhardingeee8fa1c2008-03-17 16:37:19 -0700853 int masked;
854
855 masked = sync_test_and_set_bit(evtchn, sh->evtchn_mask);
856 sync_set_bit(evtchn, sh->evtchn_pending);
857 if (!masked)
858 unmask_evtchn(evtchn);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700859 ret = 1;
860 }
861
862 return ret;
863}
864
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100865static void restore_cpu_virqs(unsigned int cpu)
866{
867 struct evtchn_bind_virq bind_virq;
868 int virq, irq, evtchn;
869
870 for (virq = 0; virq < NR_VIRQS; virq++) {
871 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1)
872 continue;
873
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800874 BUG_ON(virq_from_irq(irq) != virq);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100875
876 /* Get a new binding from Xen. */
877 bind_virq.virq = virq;
878 bind_virq.vcpu = cpu;
879 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
880 &bind_virq) != 0)
881 BUG();
882 evtchn = bind_virq.port;
883
884 /* Record the new mapping. */
885 evtchn_to_irq[evtchn] = irq;
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800886 irq_info[irq] = mk_virq_info(evtchn, virq);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100887 bind_evtchn_to_cpu(evtchn, cpu);
888
889 /* Ready for use. */
890 unmask_evtchn(evtchn);
891 }
892}
893
894static void restore_cpu_ipis(unsigned int cpu)
895{
896 struct evtchn_bind_ipi bind_ipi;
897 int ipi, irq, evtchn;
898
899 for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) {
900 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1)
901 continue;
902
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800903 BUG_ON(ipi_from_irq(irq) != ipi);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100904
905 /* Get a new binding from Xen. */
906 bind_ipi.vcpu = cpu;
907 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
908 &bind_ipi) != 0)
909 BUG();
910 evtchn = bind_ipi.port;
911
912 /* Record the new mapping. */
913 evtchn_to_irq[evtchn] = irq;
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800914 irq_info[irq] = mk_ipi_info(evtchn, ipi);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100915 bind_evtchn_to_cpu(evtchn, cpu);
916
917 /* Ready for use. */
918 unmask_evtchn(evtchn);
919
920 }
921}
922
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -0700923/* Clear an irq's pending state, in preparation for polling on it */
924void xen_clear_irq_pending(int irq)
925{
926 int evtchn = evtchn_from_irq(irq);
927
928 if (VALID_EVTCHN(evtchn))
929 clear_evtchn(evtchn);
930}
931
Jeremy Fitzhardinge168d2f42008-08-20 17:02:18 -0700932void xen_set_irq_pending(int irq)
933{
934 int evtchn = evtchn_from_irq(irq);
935
936 if (VALID_EVTCHN(evtchn))
937 set_evtchn(evtchn);
938}
939
940bool xen_test_irq_pending(int irq)
941{
942 int evtchn = evtchn_from_irq(irq);
943 bool ret = false;
944
945 if (VALID_EVTCHN(evtchn))
946 ret = test_evtchn(evtchn);
947
948 return ret;
949}
950
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -0700951/* Poll waiting for an irq to become pending. In the usual case, the
952 irq will be disabled so it won't deliver an interrupt. */
953void xen_poll_irq(int irq)
954{
955 evtchn_port_t evtchn = evtchn_from_irq(irq);
956
957 if (VALID_EVTCHN(evtchn)) {
958 struct sched_poll poll;
959
960 poll.nr_ports = 1;
961 poll.timeout = 0;
Isaku Yamahataff3c5362008-10-14 17:50:44 -0700962 set_xen_guest_handle(poll.ports, &evtchn);
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -0700963
964 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0)
965 BUG();
966 }
967}
968
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100969void xen_irq_resume(void)
970{
971 unsigned int cpu, irq, evtchn;
972
973 init_evtchn_cpu_bindings();
974
975 /* New event-channel space is not 'live' yet. */
976 for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
977 mask_evtchn(evtchn);
978
979 /* No IRQ <-> event-channel mappings. */
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -0800980 for (irq = 0; irq < nr_irqs; irq++)
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100981 irq_info[irq].evtchn = 0; /* zap event-channel binding */
982
983 for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
984 evtchn_to_irq[evtchn] = -1;
985
986 for_each_possible_cpu(cpu) {
987 restore_cpu_virqs(cpu);
988 restore_cpu_ipis(cpu);
989 }
990}
991
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700992static struct irq_chip xen_dynamic_chip __read_mostly = {
993 .name = "xen-dyn",
Jeremy Fitzhardinge54a353a2009-02-06 14:09:42 -0800994
995 .disable = disable_dynirq,
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700996 .mask = disable_dynirq,
997 .unmask = enable_dynirq,
Jeremy Fitzhardinge54a353a2009-02-06 14:09:42 -0800998
Jeremy Fitzhardinge3588fe22010-08-27 17:30:24 -0700999 .eoi = ack_dynirq,
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001000 .set_affinity = set_affinity_irq,
1001 .retrigger = retrigger_dynirq,
1002};
1003
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -07001004static struct irq_chip xen_percpu_chip __read_mostly = {
1005 .name = "xen-percpu",
1006
1007 .disable = disable_dynirq,
1008 .mask = disable_dynirq,
1009 .unmask = enable_dynirq,
1010
1011 .ack = ack_dynirq,
1012};
1013
Sheng Yang38e20b02010-05-14 12:40:51 +01001014int xen_set_callback_via(uint64_t via)
1015{
1016 struct xen_hvm_param a;
1017 a.domid = DOMID_SELF;
1018 a.index = HVM_PARAM_CALLBACK_IRQ;
1019 a.value = via;
1020 return HYPERVISOR_hvm_op(HVMOP_set_param, &a);
1021}
1022EXPORT_SYMBOL_GPL(xen_set_callback_via);
1023
Stefano Stabellinica65f9f2010-07-29 14:37:48 +01001024#ifdef CONFIG_XEN_PVHVM
Sheng Yang38e20b02010-05-14 12:40:51 +01001025/* Vector callbacks are better than PCI interrupts to receive event
1026 * channel notifications because we can receive vector callbacks on any
1027 * vcpu and we don't need PCI support or APIC interactions. */
1028void xen_callback_vector(void)
1029{
1030 int rc;
1031 uint64_t callback_via;
1032 if (xen_have_vector_callback) {
1033 callback_via = HVM_CALLBACK_VECTOR(XEN_HVM_EVTCHN_CALLBACK);
1034 rc = xen_set_callback_via(callback_via);
1035 if (rc) {
1036 printk(KERN_ERR "Request for Xen HVM callback vector"
1037 " failed.\n");
1038 xen_have_vector_callback = 0;
1039 return;
1040 }
1041 printk(KERN_INFO "Xen HVM callback vector for event delivery is "
1042 "enabled\n");
1043 /* in the restore case the vector has already been allocated */
1044 if (!test_bit(XEN_HVM_EVTCHN_CALLBACK, used_vectors))
1045 alloc_intr_gate(XEN_HVM_EVTCHN_CALLBACK, xen_hvm_callback_vector);
1046 }
1047}
Stefano Stabellinica65f9f2010-07-29 14:37:48 +01001048#else
1049void xen_callback_vector(void) {}
1050#endif
Sheng Yang38e20b02010-05-14 12:40:51 +01001051
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001052void __init xen_init_IRQ(void)
1053{
1054 int i;
Mike Travisc7a35892009-01-10 21:58:11 -08001055
Pekka Enberga70c3522009-07-01 11:51:18 +03001056 cpu_evtchn_mask_p = kcalloc(nr_cpu_ids, sizeof(struct cpu_evtchn_s),
1057 GFP_KERNEL);
Christophe Saout28e08862009-01-11 11:46:23 -08001058 BUG_ON(cpu_evtchn_mask_p == NULL);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001059
1060 init_evtchn_cpu_bindings();
1061
1062 /* No event channels are 'live' right now. */
1063 for (i = 0; i < NR_EVENT_CHANNELS; i++)
1064 mask_evtchn(i);
1065
Sheng Yang38e20b02010-05-14 12:40:51 +01001066 if (xen_hvm_domain()) {
1067 xen_callback_vector();
1068 native_init_IRQ();
1069 } else {
1070 irq_ctx_init(smp_processor_id());
1071 }
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001072}