Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1 | /* |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 2 | * Performance events x86 architecture code |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 3 | * |
Ingo Molnar | 9814451 | 2009-04-29 14:52:50 +0200 | [diff] [blame] | 4 | * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> |
| 5 | * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar |
| 6 | * Copyright (C) 2009 Jaswinder Singh Rajput |
| 7 | * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter |
| 8 | * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> |
Markus Metzger | 30dd568 | 2009-07-21 15:56:48 +0200 | [diff] [blame] | 9 | * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com> |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 10 | * Copyright (C) 2009 Google, Inc., Stephane Eranian |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 11 | * |
| 12 | * For licencing details see kernel-base/COPYING |
| 13 | */ |
| 14 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 15 | #include <linux/perf_event.h> |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 16 | #include <linux/capability.h> |
| 17 | #include <linux/notifier.h> |
| 18 | #include <linux/hardirq.h> |
| 19 | #include <linux/kprobes.h> |
Thomas Gleixner | 4ac1329 | 2008-12-09 21:43:39 +0100 | [diff] [blame] | 20 | #include <linux/module.h> |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 21 | #include <linux/kdebug.h> |
| 22 | #include <linux/sched.h> |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 23 | #include <linux/uaccess.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 24 | #include <linux/slab.h> |
Markus Metzger | 30dd568 | 2009-07-21 15:56:48 +0200 | [diff] [blame] | 25 | #include <linux/cpu.h> |
Peter Zijlstra | 272d30b | 2010-01-22 16:32:17 +0100 | [diff] [blame] | 26 | #include <linux/bitops.h> |
Peter Zijlstra | 0c9d42e | 2011-11-20 23:30:47 +0100 | [diff] [blame] | 27 | #include <linux/device.h> |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 28 | |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 29 | #include <asm/apic.h> |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 30 | #include <asm/stacktrace.h> |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 31 | #include <asm/nmi.h> |
Lin Ming | 6909262 | 2011-03-03 10:34:50 +0800 | [diff] [blame] | 32 | #include <asm/smp.h> |
Robert Richter | c8e5910 | 2011-04-16 02:27:55 +0200 | [diff] [blame] | 33 | #include <asm/alternative.h> |
Peter Zijlstra | e3f3541 | 2011-11-21 11:43:53 +0100 | [diff] [blame] | 34 | #include <asm/timer.h> |
Peter Zijlstra | d07bdfd | 2012-07-10 09:42:15 +0200 | [diff] [blame] | 35 | #include <asm/desc.h> |
| 36 | #include <asm/ldt.h> |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 37 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 38 | #include "perf_event.h" |
| 39 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 40 | struct x86_pmu x86_pmu __read_mostly; |
Stephane Eranian | efc9f05 | 2011-06-06 16:57:03 +0200 | [diff] [blame] | 41 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 42 | DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { |
Peter Zijlstra | b0f3f28 | 2009-03-05 18:08:27 +0100 | [diff] [blame] | 43 | .enabled = 1, |
| 44 | }; |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 45 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 46 | u64 __read_mostly hw_cache_event_ids |
Ingo Molnar | 8326f44 | 2009-06-05 20:22:46 +0200 | [diff] [blame] | 47 | [PERF_COUNT_HW_CACHE_MAX] |
| 48 | [PERF_COUNT_HW_CACHE_OP_MAX] |
| 49 | [PERF_COUNT_HW_CACHE_RESULT_MAX]; |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 50 | u64 __read_mostly hw_cache_extra_regs |
Andi Kleen | e994d7d | 2011-03-03 10:34:48 +0800 | [diff] [blame] | 51 | [PERF_COUNT_HW_CACHE_MAX] |
| 52 | [PERF_COUNT_HW_CACHE_OP_MAX] |
| 53 | [PERF_COUNT_HW_CACHE_RESULT_MAX]; |
Ingo Molnar | 8326f44 | 2009-06-05 20:22:46 +0200 | [diff] [blame] | 54 | |
Jaswinder Singh Rajput | f87ad35 | 2009-02-27 20:15:14 +0530 | [diff] [blame] | 55 | /* |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 56 | * Propagate event elapsed time into the generic event. |
| 57 | * Can only be executed on the CPU where the event is active. |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 58 | * Returns the delta events processed. |
| 59 | */ |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 60 | u64 x86_perf_event_update(struct perf_event *event) |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 61 | { |
Peter Zijlstra | cc2ad4b | 2010-03-02 20:18:39 +0100 | [diff] [blame] | 62 | struct hw_perf_event *hwc = &event->hw; |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 63 | int shift = 64 - x86_pmu.cntval_bits; |
Peter Zijlstra | ec3232b | 2009-05-13 09:45:19 +0200 | [diff] [blame] | 64 | u64 prev_raw_count, new_raw_count; |
Peter Zijlstra | cc2ad4b | 2010-03-02 20:18:39 +0100 | [diff] [blame] | 65 | int idx = hwc->idx; |
Peter Zijlstra | ec3232b | 2009-05-13 09:45:19 +0200 | [diff] [blame] | 66 | s64 delta; |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 67 | |
Robert Richter | 15c7ad5 | 2012-06-20 20:46:33 +0200 | [diff] [blame] | 68 | if (idx == INTEL_PMC_IDX_FIXED_BTS) |
Markus Metzger | 30dd568 | 2009-07-21 15:56:48 +0200 | [diff] [blame] | 69 | return 0; |
| 70 | |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 71 | /* |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 72 | * Careful: an NMI might modify the previous event value. |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 73 | * |
| 74 | * Our tactic to handle this is to first atomically read and |
| 75 | * exchange a new raw count - then add that new-prev delta |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 76 | * count to the generic event atomically: |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 77 | */ |
| 78 | again: |
Peter Zijlstra | e785059 | 2010-05-21 14:43:08 +0200 | [diff] [blame] | 79 | prev_raw_count = local64_read(&hwc->prev_count); |
Vince Weaver | c48b605 | 2012-03-01 17:28:14 -0500 | [diff] [blame] | 80 | rdpmcl(hwc->event_base_rdpmc, new_raw_count); |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 81 | |
Peter Zijlstra | e785059 | 2010-05-21 14:43:08 +0200 | [diff] [blame] | 82 | if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 83 | new_raw_count) != prev_raw_count) |
| 84 | goto again; |
| 85 | |
| 86 | /* |
| 87 | * Now we have the new raw value and have updated the prev |
| 88 | * timestamp already. We can now calculate the elapsed delta |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 89 | * (event-)time and add that to the generic event. |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 90 | * |
| 91 | * Careful, not all hw sign-extends above the physical width |
Peter Zijlstra | ec3232b | 2009-05-13 09:45:19 +0200 | [diff] [blame] | 92 | * of the count. |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 93 | */ |
Peter Zijlstra | ec3232b | 2009-05-13 09:45:19 +0200 | [diff] [blame] | 94 | delta = (new_raw_count << shift) - (prev_raw_count << shift); |
| 95 | delta >>= shift; |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 96 | |
Peter Zijlstra | e785059 | 2010-05-21 14:43:08 +0200 | [diff] [blame] | 97 | local64_add(delta, &event->count); |
| 98 | local64_sub(delta, &hwc->period_left); |
Robert Richter | 4b7bfd0 | 2009-04-29 12:47:22 +0200 | [diff] [blame] | 99 | |
| 100 | return new_raw_count; |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 101 | } |
| 102 | |
Andi Kleen | a7e3ed1 | 2011-03-03 10:34:47 +0800 | [diff] [blame] | 103 | /* |
| 104 | * Find and validate any extra registers to set up. |
| 105 | */ |
| 106 | static int x86_pmu_extra_regs(u64 config, struct perf_event *event) |
| 107 | { |
Stephane Eranian | efc9f05 | 2011-06-06 16:57:03 +0200 | [diff] [blame] | 108 | struct hw_perf_event_extra *reg; |
Andi Kleen | a7e3ed1 | 2011-03-03 10:34:47 +0800 | [diff] [blame] | 109 | struct extra_reg *er; |
| 110 | |
Stephane Eranian | efc9f05 | 2011-06-06 16:57:03 +0200 | [diff] [blame] | 111 | reg = &event->hw.extra_reg; |
Andi Kleen | a7e3ed1 | 2011-03-03 10:34:47 +0800 | [diff] [blame] | 112 | |
| 113 | if (!x86_pmu.extra_regs) |
| 114 | return 0; |
| 115 | |
| 116 | for (er = x86_pmu.extra_regs; er->msr; er++) { |
| 117 | if (er->event != (config & er->config_mask)) |
| 118 | continue; |
| 119 | if (event->attr.config1 & ~er->valid_mask) |
| 120 | return -EINVAL; |
Stephane Eranian | efc9f05 | 2011-06-06 16:57:03 +0200 | [diff] [blame] | 121 | |
| 122 | reg->idx = er->idx; |
| 123 | reg->config = event->attr.config1; |
| 124 | reg->reg = er->msr; |
Andi Kleen | a7e3ed1 | 2011-03-03 10:34:47 +0800 | [diff] [blame] | 125 | break; |
| 126 | } |
| 127 | return 0; |
| 128 | } |
| 129 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 130 | static atomic_t active_events; |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 131 | static DEFINE_MUTEX(pmc_reserve_mutex); |
| 132 | |
Robert Richter | b27ea29 | 2010-03-17 12:49:10 +0100 | [diff] [blame] | 133 | #ifdef CONFIG_X86_LOCAL_APIC |
| 134 | |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 135 | static bool reserve_pmc_hardware(void) |
| 136 | { |
| 137 | int i; |
| 138 | |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 139 | for (i = 0; i < x86_pmu.num_counters; i++) { |
Robert Richter | 41bf498 | 2011-02-02 17:40:57 +0100 | [diff] [blame] | 140 | if (!reserve_perfctr_nmi(x86_pmu_event_addr(i))) |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 141 | goto perfctr_fail; |
| 142 | } |
| 143 | |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 144 | for (i = 0; i < x86_pmu.num_counters; i++) { |
Robert Richter | 41bf498 | 2011-02-02 17:40:57 +0100 | [diff] [blame] | 145 | if (!reserve_evntsel_nmi(x86_pmu_config_addr(i))) |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 146 | goto eventsel_fail; |
| 147 | } |
| 148 | |
| 149 | return true; |
| 150 | |
| 151 | eventsel_fail: |
| 152 | for (i--; i >= 0; i--) |
Robert Richter | 41bf498 | 2011-02-02 17:40:57 +0100 | [diff] [blame] | 153 | release_evntsel_nmi(x86_pmu_config_addr(i)); |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 154 | |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 155 | i = x86_pmu.num_counters; |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 156 | |
| 157 | perfctr_fail: |
| 158 | for (i--; i >= 0; i--) |
Robert Richter | 41bf498 | 2011-02-02 17:40:57 +0100 | [diff] [blame] | 159 | release_perfctr_nmi(x86_pmu_event_addr(i)); |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 160 | |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 161 | return false; |
| 162 | } |
| 163 | |
| 164 | static void release_pmc_hardware(void) |
| 165 | { |
| 166 | int i; |
| 167 | |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 168 | for (i = 0; i < x86_pmu.num_counters; i++) { |
Robert Richter | 41bf498 | 2011-02-02 17:40:57 +0100 | [diff] [blame] | 169 | release_perfctr_nmi(x86_pmu_event_addr(i)); |
| 170 | release_evntsel_nmi(x86_pmu_config_addr(i)); |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 171 | } |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 172 | } |
| 173 | |
Robert Richter | b27ea29 | 2010-03-17 12:49:10 +0100 | [diff] [blame] | 174 | #else |
| 175 | |
| 176 | static bool reserve_pmc_hardware(void) { return true; } |
| 177 | static void release_pmc_hardware(void) {} |
| 178 | |
| 179 | #endif |
| 180 | |
Don Zickus | 33c6d6a | 2010-11-22 16:55:23 -0500 | [diff] [blame] | 181 | static bool check_hw_exists(void) |
| 182 | { |
George Dunlap | a5ebe0b | 2013-04-03 15:46:28 +0100 | [diff] [blame] | 183 | u64 val, val_fail, val_new= ~0; |
| 184 | int i, reg, reg_fail, ret = 0; |
| 185 | int bios_fail = 0; |
Don Zickus | 33c6d6a | 2010-11-22 16:55:23 -0500 | [diff] [blame] | 186 | |
Peter Zijlstra | 4407204 | 2010-12-08 15:56:23 +0100 | [diff] [blame] | 187 | /* |
| 188 | * Check to see if the BIOS enabled any of the counters, if so |
| 189 | * complain and bail. |
| 190 | */ |
| 191 | for (i = 0; i < x86_pmu.num_counters; i++) { |
Robert Richter | 41bf498 | 2011-02-02 17:40:57 +0100 | [diff] [blame] | 192 | reg = x86_pmu_config_addr(i); |
Peter Zijlstra | 4407204 | 2010-12-08 15:56:23 +0100 | [diff] [blame] | 193 | ret = rdmsrl_safe(reg, &val); |
| 194 | if (ret) |
| 195 | goto msr_fail; |
George Dunlap | a5ebe0b | 2013-04-03 15:46:28 +0100 | [diff] [blame] | 196 | if (val & ARCH_PERFMON_EVENTSEL_ENABLE) { |
| 197 | bios_fail = 1; |
| 198 | val_fail = val; |
| 199 | reg_fail = reg; |
| 200 | } |
Peter Zijlstra | 4407204 | 2010-12-08 15:56:23 +0100 | [diff] [blame] | 201 | } |
| 202 | |
| 203 | if (x86_pmu.num_counters_fixed) { |
| 204 | reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; |
| 205 | ret = rdmsrl_safe(reg, &val); |
| 206 | if (ret) |
| 207 | goto msr_fail; |
| 208 | for (i = 0; i < x86_pmu.num_counters_fixed; i++) { |
George Dunlap | a5ebe0b | 2013-04-03 15:46:28 +0100 | [diff] [blame] | 209 | if (val & (0x03 << i*4)) { |
| 210 | bios_fail = 1; |
| 211 | val_fail = val; |
| 212 | reg_fail = reg; |
| 213 | } |
Peter Zijlstra | 4407204 | 2010-12-08 15:56:23 +0100 | [diff] [blame] | 214 | } |
| 215 | } |
| 216 | |
| 217 | /* |
Andre Przywara | bffd5fc | 2012-10-09 17:38:35 +0200 | [diff] [blame] | 218 | * Read the current value, change it and read it back to see if it |
| 219 | * matches, this is needed to detect certain hardware emulators |
| 220 | * (qemu/kvm) that don't trap on the MSR access and always return 0s. |
Peter Zijlstra | 4407204 | 2010-12-08 15:56:23 +0100 | [diff] [blame] | 221 | */ |
Robert Richter | f285f92 | 2012-06-20 20:46:36 +0200 | [diff] [blame] | 222 | reg = x86_pmu_event_addr(0); |
Andre Przywara | bffd5fc | 2012-10-09 17:38:35 +0200 | [diff] [blame] | 223 | if (rdmsrl_safe(reg, &val)) |
| 224 | goto msr_fail; |
| 225 | val ^= 0xffffUL; |
Robert Richter | f285f92 | 2012-06-20 20:46:36 +0200 | [diff] [blame] | 226 | ret = wrmsrl_safe(reg, val); |
| 227 | ret |= rdmsrl_safe(reg, &val_new); |
Don Zickus | 33c6d6a | 2010-11-22 16:55:23 -0500 | [diff] [blame] | 228 | if (ret || val != val_new) |
Peter Zijlstra | 4407204 | 2010-12-08 15:56:23 +0100 | [diff] [blame] | 229 | goto msr_fail; |
Don Zickus | 33c6d6a | 2010-11-22 16:55:23 -0500 | [diff] [blame] | 230 | |
Ingo Molnar | 45daae5 | 2011-03-25 10:24:23 +0100 | [diff] [blame] | 231 | /* |
| 232 | * We still allow the PMU driver to operate: |
| 233 | */ |
George Dunlap | a5ebe0b | 2013-04-03 15:46:28 +0100 | [diff] [blame] | 234 | if (bios_fail) { |
| 235 | printk(KERN_CONT "Broken BIOS detected, complain to your hardware vendor.\n"); |
| 236 | printk(KERN_ERR FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", reg_fail, val_fail); |
| 237 | } |
Ingo Molnar | 45daae5 | 2011-03-25 10:24:23 +0100 | [diff] [blame] | 238 | |
| 239 | return true; |
Peter Zijlstra | 4407204 | 2010-12-08 15:56:23 +0100 | [diff] [blame] | 240 | |
| 241 | msr_fail: |
| 242 | printk(KERN_CONT "Broken PMU hardware detected, using software events only.\n"); |
Robert Richter | f285f92 | 2012-06-20 20:46:36 +0200 | [diff] [blame] | 243 | printk(KERN_ERR "Failed to access perfctr msr (MSR %x is %Lx)\n", reg, val_new); |
Ingo Molnar | 45daae5 | 2011-03-25 10:24:23 +0100 | [diff] [blame] | 244 | |
Peter Zijlstra | 4407204 | 2010-12-08 15:56:23 +0100 | [diff] [blame] | 245 | return false; |
Don Zickus | 33c6d6a | 2010-11-22 16:55:23 -0500 | [diff] [blame] | 246 | } |
| 247 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 248 | static void hw_perf_event_destroy(struct perf_event *event) |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 249 | { |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 250 | if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) { |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 251 | release_pmc_hardware(); |
Peter Zijlstra | ca03770 | 2010-03-02 19:52:12 +0100 | [diff] [blame] | 252 | release_ds_buffers(); |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 253 | mutex_unlock(&pmc_reserve_mutex); |
| 254 | } |
| 255 | } |
| 256 | |
Robert Richter | 85cf9db | 2009-04-29 12:47:20 +0200 | [diff] [blame] | 257 | static inline int x86_pmu_initialized(void) |
| 258 | { |
| 259 | return x86_pmu.handle_irq != NULL; |
| 260 | } |
| 261 | |
Ingo Molnar | 8326f44 | 2009-06-05 20:22:46 +0200 | [diff] [blame] | 262 | static inline int |
Andi Kleen | e994d7d | 2011-03-03 10:34:48 +0800 | [diff] [blame] | 263 | set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event) |
Ingo Molnar | 8326f44 | 2009-06-05 20:22:46 +0200 | [diff] [blame] | 264 | { |
Andi Kleen | e994d7d | 2011-03-03 10:34:48 +0800 | [diff] [blame] | 265 | struct perf_event_attr *attr = &event->attr; |
Ingo Molnar | 8326f44 | 2009-06-05 20:22:46 +0200 | [diff] [blame] | 266 | unsigned int cache_type, cache_op, cache_result; |
| 267 | u64 config, val; |
| 268 | |
| 269 | config = attr->config; |
| 270 | |
| 271 | cache_type = (config >> 0) & 0xff; |
| 272 | if (cache_type >= PERF_COUNT_HW_CACHE_MAX) |
| 273 | return -EINVAL; |
| 274 | |
| 275 | cache_op = (config >> 8) & 0xff; |
| 276 | if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) |
| 277 | return -EINVAL; |
| 278 | |
| 279 | cache_result = (config >> 16) & 0xff; |
| 280 | if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) |
| 281 | return -EINVAL; |
| 282 | |
| 283 | val = hw_cache_event_ids[cache_type][cache_op][cache_result]; |
| 284 | |
| 285 | if (val == 0) |
| 286 | return -ENOENT; |
| 287 | |
| 288 | if (val == -1) |
| 289 | return -EINVAL; |
| 290 | |
| 291 | hwc->config |= val; |
Andi Kleen | e994d7d | 2011-03-03 10:34:48 +0800 | [diff] [blame] | 292 | attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result]; |
| 293 | return x86_pmu_extra_regs(val, event); |
Ingo Molnar | 8326f44 | 2009-06-05 20:22:46 +0200 | [diff] [blame] | 294 | } |
| 295 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 296 | int x86_setup_perfctr(struct perf_event *event) |
Robert Richter | c1726f3 | 2010-04-13 22:23:11 +0200 | [diff] [blame] | 297 | { |
| 298 | struct perf_event_attr *attr = &event->attr; |
| 299 | struct hw_perf_event *hwc = &event->hw; |
| 300 | u64 config; |
| 301 | |
Franck Bui-Huu | 6c7e550 | 2010-11-23 16:21:43 +0100 | [diff] [blame] | 302 | if (!is_sampling_event(event)) { |
Robert Richter | c1726f3 | 2010-04-13 22:23:11 +0200 | [diff] [blame] | 303 | hwc->sample_period = x86_pmu.max_period; |
| 304 | hwc->last_period = hwc->sample_period; |
Peter Zijlstra | e785059 | 2010-05-21 14:43:08 +0200 | [diff] [blame] | 305 | local64_set(&hwc->period_left, hwc->sample_period); |
Robert Richter | c1726f3 | 2010-04-13 22:23:11 +0200 | [diff] [blame] | 306 | } else { |
| 307 | /* |
| 308 | * If we have a PMU initialized but no APIC |
| 309 | * interrupts, we cannot sample hardware |
| 310 | * events (user-space has to fall back and |
| 311 | * sample via a hrtimer based software event): |
| 312 | */ |
| 313 | if (!x86_pmu.apic) |
| 314 | return -EOPNOTSUPP; |
| 315 | } |
| 316 | |
| 317 | if (attr->type == PERF_TYPE_RAW) |
Peter Zijlstra | ed13ec5 | 2011-11-14 10:03:25 +0100 | [diff] [blame] | 318 | return x86_pmu_extra_regs(event->attr.config, event); |
Robert Richter | c1726f3 | 2010-04-13 22:23:11 +0200 | [diff] [blame] | 319 | |
| 320 | if (attr->type == PERF_TYPE_HW_CACHE) |
Andi Kleen | e994d7d | 2011-03-03 10:34:48 +0800 | [diff] [blame] | 321 | return set_ext_hw_attr(hwc, event); |
Robert Richter | c1726f3 | 2010-04-13 22:23:11 +0200 | [diff] [blame] | 322 | |
| 323 | if (attr->config >= x86_pmu.max_events) |
| 324 | return -EINVAL; |
| 325 | |
| 326 | /* |
| 327 | * The generic map: |
| 328 | */ |
| 329 | config = x86_pmu.event_map(attr->config); |
| 330 | |
| 331 | if (config == 0) |
| 332 | return -ENOENT; |
| 333 | |
| 334 | if (config == -1LL) |
| 335 | return -EINVAL; |
| 336 | |
| 337 | /* |
| 338 | * Branch tracing: |
| 339 | */ |
Peter Zijlstra | 18a073a | 2011-04-26 13:24:33 +0200 | [diff] [blame] | 340 | if (attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS && |
| 341 | !attr->freq && hwc->sample_period == 1) { |
Robert Richter | c1726f3 | 2010-04-13 22:23:11 +0200 | [diff] [blame] | 342 | /* BTS is not supported by this architecture. */ |
Peter Zijlstra | 6809b6e | 2010-10-19 14:22:50 +0200 | [diff] [blame] | 343 | if (!x86_pmu.bts_active) |
Robert Richter | c1726f3 | 2010-04-13 22:23:11 +0200 | [diff] [blame] | 344 | return -EOPNOTSUPP; |
| 345 | |
| 346 | /* BTS is currently only allowed for user-mode. */ |
| 347 | if (!attr->exclude_kernel) |
| 348 | return -EOPNOTSUPP; |
| 349 | } |
| 350 | |
| 351 | hwc->config |= config; |
| 352 | |
| 353 | return 0; |
| 354 | } |
Robert Richter | 4261e0e | 2010-04-13 22:23:10 +0200 | [diff] [blame] | 355 | |
Stephane Eranian | ff3fb51 | 2012-02-09 23:20:54 +0100 | [diff] [blame] | 356 | /* |
| 357 | * check that branch_sample_type is compatible with |
| 358 | * settings needed for precise_ip > 1 which implies |
| 359 | * using the LBR to capture ALL taken branches at the |
| 360 | * priv levels of the measurement |
| 361 | */ |
| 362 | static inline int precise_br_compat(struct perf_event *event) |
| 363 | { |
| 364 | u64 m = event->attr.branch_sample_type; |
| 365 | u64 b = 0; |
| 366 | |
| 367 | /* must capture all branches */ |
| 368 | if (!(m & PERF_SAMPLE_BRANCH_ANY)) |
| 369 | return 0; |
| 370 | |
| 371 | m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER; |
| 372 | |
| 373 | if (!event->attr.exclude_user) |
| 374 | b |= PERF_SAMPLE_BRANCH_USER; |
| 375 | |
| 376 | if (!event->attr.exclude_kernel) |
| 377 | b |= PERF_SAMPLE_BRANCH_KERNEL; |
| 378 | |
| 379 | /* |
| 380 | * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86 |
| 381 | */ |
| 382 | |
| 383 | return m == b; |
| 384 | } |
| 385 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 386 | int x86_pmu_hw_config(struct perf_event *event) |
Cyrill Gorcunov | a072738 | 2010-03-11 19:54:39 +0300 | [diff] [blame] | 387 | { |
Peter Zijlstra | ab60834 | 2010-04-08 23:03:20 +0200 | [diff] [blame] | 388 | if (event->attr.precise_ip) { |
| 389 | int precise = 0; |
| 390 | |
| 391 | /* Support for constant skid */ |
Peter Zijlstra | c93dc84 | 2012-06-08 14:50:50 +0200 | [diff] [blame] | 392 | if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) { |
Peter Zijlstra | ab60834 | 2010-04-08 23:03:20 +0200 | [diff] [blame] | 393 | precise++; |
| 394 | |
Peter Zijlstra | 5553be2 | 2010-10-19 14:38:11 +0200 | [diff] [blame] | 395 | /* Support for IP fixup */ |
| 396 | if (x86_pmu.lbr_nr) |
| 397 | precise++; |
| 398 | } |
Peter Zijlstra | ab60834 | 2010-04-08 23:03:20 +0200 | [diff] [blame] | 399 | |
| 400 | if (event->attr.precise_ip > precise) |
| 401 | return -EOPNOTSUPP; |
Stephane Eranian | ff3fb51 | 2012-02-09 23:20:54 +0100 | [diff] [blame] | 402 | /* |
| 403 | * check that PEBS LBR correction does not conflict with |
| 404 | * whatever the user is asking with attr->branch_sample_type |
| 405 | */ |
Andi Kleen | 130768b | 2013-06-17 17:36:47 -0700 | [diff] [blame] | 406 | if (event->attr.precise_ip > 1 && |
| 407 | x86_pmu.intel_cap.pebs_format < 2) { |
Stephane Eranian | ff3fb51 | 2012-02-09 23:20:54 +0100 | [diff] [blame] | 408 | u64 *br_type = &event->attr.branch_sample_type; |
| 409 | |
| 410 | if (has_branch_stack(event)) { |
| 411 | if (!precise_br_compat(event)) |
| 412 | return -EOPNOTSUPP; |
| 413 | |
| 414 | /* branch_sample_type is compatible */ |
| 415 | |
| 416 | } else { |
| 417 | /* |
| 418 | * user did not specify branch_sample_type |
| 419 | * |
| 420 | * For PEBS fixups, we capture all |
| 421 | * the branches at the priv level of the |
| 422 | * event. |
| 423 | */ |
| 424 | *br_type = PERF_SAMPLE_BRANCH_ANY; |
| 425 | |
| 426 | if (!event->attr.exclude_user) |
| 427 | *br_type |= PERF_SAMPLE_BRANCH_USER; |
| 428 | |
| 429 | if (!event->attr.exclude_kernel) |
| 430 | *br_type |= PERF_SAMPLE_BRANCH_KERNEL; |
| 431 | } |
| 432 | } |
Peter Zijlstra | ab60834 | 2010-04-08 23:03:20 +0200 | [diff] [blame] | 433 | } |
| 434 | |
Cyrill Gorcunov | a072738 | 2010-03-11 19:54:39 +0300 | [diff] [blame] | 435 | /* |
| 436 | * Generate PMC IRQs: |
| 437 | * (keep 'enabled' bit clear for now) |
| 438 | */ |
Peter Zijlstra | b4cdc5c | 2010-03-30 17:00:06 +0200 | [diff] [blame] | 439 | event->hw.config = ARCH_PERFMON_EVENTSEL_INT; |
Cyrill Gorcunov | a072738 | 2010-03-11 19:54:39 +0300 | [diff] [blame] | 440 | |
| 441 | /* |
| 442 | * Count user and OS events unless requested not to |
| 443 | */ |
Peter Zijlstra | b4cdc5c | 2010-03-30 17:00:06 +0200 | [diff] [blame] | 444 | if (!event->attr.exclude_user) |
| 445 | event->hw.config |= ARCH_PERFMON_EVENTSEL_USR; |
| 446 | if (!event->attr.exclude_kernel) |
| 447 | event->hw.config |= ARCH_PERFMON_EVENTSEL_OS; |
| 448 | |
| 449 | if (event->attr.type == PERF_TYPE_RAW) |
| 450 | event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK; |
Cyrill Gorcunov | a072738 | 2010-03-11 19:54:39 +0300 | [diff] [blame] | 451 | |
Robert Richter | 9d0fcba6 | 2010-04-13 22:23:12 +0200 | [diff] [blame] | 452 | return x86_setup_perfctr(event); |
Cyrill Gorcunov | a072738 | 2010-03-11 19:54:39 +0300 | [diff] [blame] | 453 | } |
| 454 | |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 455 | /* |
Peter Zijlstra | 0d48696 | 2009-06-02 19:22:16 +0200 | [diff] [blame] | 456 | * Setup the hardware configuration for a given attr_type |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 457 | */ |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 458 | static int __x86_pmu_event_init(struct perf_event *event) |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 459 | { |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 460 | int err; |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 461 | |
Robert Richter | 85cf9db | 2009-04-29 12:47:20 +0200 | [diff] [blame] | 462 | if (!x86_pmu_initialized()) |
| 463 | return -ENODEV; |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 464 | |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 465 | err = 0; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 466 | if (!atomic_inc_not_zero(&active_events)) { |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 467 | mutex_lock(&pmc_reserve_mutex); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 468 | if (atomic_read(&active_events) == 0) { |
Markus Metzger | 30dd568 | 2009-07-21 15:56:48 +0200 | [diff] [blame] | 469 | if (!reserve_pmc_hardware()) |
| 470 | err = -EBUSY; |
Peter Zijlstra | f80c9e3 | 2010-10-19 14:50:02 +0200 | [diff] [blame] | 471 | else |
| 472 | reserve_ds_buffers(); |
Markus Metzger | 30dd568 | 2009-07-21 15:56:48 +0200 | [diff] [blame] | 473 | } |
| 474 | if (!err) |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 475 | atomic_inc(&active_events); |
Peter Zijlstra | 4e935e4 | 2009-03-30 19:07:16 +0200 | [diff] [blame] | 476 | mutex_unlock(&pmc_reserve_mutex); |
| 477 | } |
| 478 | if (err) |
| 479 | return err; |
| 480 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 481 | event->destroy = hw_perf_event_destroy; |
Peter Zijlstra | a1792cdac | 2009-09-09 10:04:47 +0200 | [diff] [blame] | 482 | |
Robert Richter | 4261e0e | 2010-04-13 22:23:10 +0200 | [diff] [blame] | 483 | event->hw.idx = -1; |
| 484 | event->hw.last_cpu = -1; |
| 485 | event->hw.last_tag = ~0ULL; |
Stephane Eranian | b690081 | 2009-10-06 16:42:09 +0200 | [diff] [blame] | 486 | |
Stephane Eranian | efc9f05 | 2011-06-06 16:57:03 +0200 | [diff] [blame] | 487 | /* mark unused */ |
| 488 | event->hw.extra_reg.idx = EXTRA_REG_NONE; |
Stephane Eranian | b36817e | 2012-02-09 23:20:53 +0100 | [diff] [blame] | 489 | event->hw.branch_reg.idx = EXTRA_REG_NONE; |
| 490 | |
Robert Richter | 9d0fcba6 | 2010-04-13 22:23:12 +0200 | [diff] [blame] | 491 | return x86_pmu.hw_config(event); |
Robert Richter | 4261e0e | 2010-04-13 22:23:10 +0200 | [diff] [blame] | 492 | } |
| 493 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 494 | void x86_pmu_disable_all(void) |
Jaswinder Singh Rajput | f87ad35 | 2009-02-27 20:15:14 +0530 | [diff] [blame] | 495 | { |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 496 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); |
Peter Zijlstra | 9e35ad3 | 2009-05-13 16:21:38 +0200 | [diff] [blame] | 497 | int idx; |
Peter Zijlstra | b0f3f28 | 2009-03-05 18:08:27 +0100 | [diff] [blame] | 498 | |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 499 | for (idx = 0; idx < x86_pmu.num_counters; idx++) { |
Peter Zijlstra | b0f3f28 | 2009-03-05 18:08:27 +0100 | [diff] [blame] | 500 | u64 val; |
| 501 | |
Robert Richter | 43f6201 | 2009-04-29 16:55:56 +0200 | [diff] [blame] | 502 | if (!test_bit(idx, cpuc->active_mask)) |
Robert Richter | 4295ee6 | 2009-04-29 12:47:01 +0200 | [diff] [blame] | 503 | continue; |
Robert Richter | 41bf498 | 2011-02-02 17:40:57 +0100 | [diff] [blame] | 504 | rdmsrl(x86_pmu_config_addr(idx), val); |
Robert Richter | bb1165d | 2010-03-01 14:21:23 +0100 | [diff] [blame] | 505 | if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE)) |
Robert Richter | 4295ee6 | 2009-04-29 12:47:01 +0200 | [diff] [blame] | 506 | continue; |
Robert Richter | bb1165d | 2010-03-01 14:21:23 +0100 | [diff] [blame] | 507 | val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; |
Robert Richter | 41bf498 | 2011-02-02 17:40:57 +0100 | [diff] [blame] | 508 | wrmsrl(x86_pmu_config_addr(idx), val); |
Jaswinder Singh Rajput | f87ad35 | 2009-02-27 20:15:14 +0530 | [diff] [blame] | 509 | } |
Jaswinder Singh Rajput | f87ad35 | 2009-02-27 20:15:14 +0530 | [diff] [blame] | 510 | } |
| 511 | |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 512 | static void x86_pmu_disable(struct pmu *pmu) |
Jaswinder Singh Rajput | b56a380 | 2009-02-27 18:09:09 +0530 | [diff] [blame] | 513 | { |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 514 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); |
| 515 | |
Robert Richter | 85cf9db | 2009-04-29 12:47:20 +0200 | [diff] [blame] | 516 | if (!x86_pmu_initialized()) |
Peter Zijlstra | 9e35ad3 | 2009-05-13 16:21:38 +0200 | [diff] [blame] | 517 | return; |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 518 | |
Peter Zijlstra | 1a6e21f | 2010-01-27 23:07:47 +0100 | [diff] [blame] | 519 | if (!cpuc->enabled) |
| 520 | return; |
| 521 | |
| 522 | cpuc->n_added = 0; |
| 523 | cpuc->enabled = 0; |
| 524 | barrier(); |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 525 | |
| 526 | x86_pmu.disable_all(); |
Jaswinder Singh Rajput | b56a380 | 2009-02-27 18:09:09 +0530 | [diff] [blame] | 527 | } |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 528 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 529 | void x86_pmu_enable_all(int added) |
Jaswinder Singh Rajput | f87ad35 | 2009-02-27 20:15:14 +0530 | [diff] [blame] | 530 | { |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 531 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); |
Jaswinder Singh Rajput | f87ad35 | 2009-02-27 20:15:14 +0530 | [diff] [blame] | 532 | int idx; |
| 533 | |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 534 | for (idx = 0; idx < x86_pmu.num_counters; idx++) { |
Robert Richter | d45dd92 | 2011-02-02 17:40:56 +0100 | [diff] [blame] | 535 | struct hw_perf_event *hwc = &cpuc->events[idx]->hw; |
Peter Zijlstra | b0f3f28 | 2009-03-05 18:08:27 +0100 | [diff] [blame] | 536 | |
Robert Richter | 43f6201 | 2009-04-29 16:55:56 +0200 | [diff] [blame] | 537 | if (!test_bit(idx, cpuc->active_mask)) |
Robert Richter | 4295ee6 | 2009-04-29 12:47:01 +0200 | [diff] [blame] | 538 | continue; |
Peter Zijlstra | 984b838 | 2009-07-10 09:59:56 +0200 | [diff] [blame] | 539 | |
Robert Richter | d45dd92 | 2011-02-02 17:40:56 +0100 | [diff] [blame] | 540 | __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); |
Jaswinder Singh Rajput | f87ad35 | 2009-02-27 20:15:14 +0530 | [diff] [blame] | 541 | } |
| 542 | } |
| 543 | |
Peter Zijlstra | 51b0fe3 | 2010-06-11 13:35:57 +0200 | [diff] [blame] | 544 | static struct pmu pmu; |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 545 | |
| 546 | static inline int is_x86_event(struct perf_event *event) |
| 547 | { |
| 548 | return event->pmu == &pmu; |
| 549 | } |
| 550 | |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 551 | /* |
| 552 | * Event scheduler state: |
| 553 | * |
| 554 | * Assign events iterating over all events and counters, beginning |
| 555 | * with events with least weights first. Keep the current iterator |
| 556 | * state in struct sched_state. |
| 557 | */ |
| 558 | struct sched_state { |
| 559 | int weight; |
| 560 | int event; /* event index */ |
| 561 | int counter; /* counter index */ |
| 562 | int unassigned; /* number of events to be assigned left */ |
| 563 | unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; |
| 564 | }; |
| 565 | |
Robert Richter | bc1738f | 2011-11-18 12:35:22 +0100 | [diff] [blame] | 566 | /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */ |
| 567 | #define SCHED_STATES_MAX 2 |
| 568 | |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 569 | struct perf_sched { |
| 570 | int max_weight; |
| 571 | int max_events; |
Andrew Hunter | 43b45780 | 2013-05-23 11:07:03 -0700 | [diff] [blame] | 572 | struct perf_event **events; |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 573 | struct sched_state state; |
Robert Richter | bc1738f | 2011-11-18 12:35:22 +0100 | [diff] [blame] | 574 | int saved_states; |
| 575 | struct sched_state saved[SCHED_STATES_MAX]; |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 576 | }; |
| 577 | |
| 578 | /* |
| 579 | * Initialize interator that runs through all events and counters. |
| 580 | */ |
Andrew Hunter | 43b45780 | 2013-05-23 11:07:03 -0700 | [diff] [blame] | 581 | static void perf_sched_init(struct perf_sched *sched, struct perf_event **events, |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 582 | int num, int wmin, int wmax) |
| 583 | { |
| 584 | int idx; |
| 585 | |
| 586 | memset(sched, 0, sizeof(*sched)); |
| 587 | sched->max_events = num; |
| 588 | sched->max_weight = wmax; |
Andrew Hunter | 43b45780 | 2013-05-23 11:07:03 -0700 | [diff] [blame] | 589 | sched->events = events; |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 590 | |
| 591 | for (idx = 0; idx < num; idx++) { |
Andrew Hunter | 43b45780 | 2013-05-23 11:07:03 -0700 | [diff] [blame] | 592 | if (events[idx]->hw.constraint->weight == wmin) |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 593 | break; |
| 594 | } |
| 595 | |
| 596 | sched->state.event = idx; /* start with min weight */ |
| 597 | sched->state.weight = wmin; |
| 598 | sched->state.unassigned = num; |
| 599 | } |
| 600 | |
Robert Richter | bc1738f | 2011-11-18 12:35:22 +0100 | [diff] [blame] | 601 | static void perf_sched_save_state(struct perf_sched *sched) |
| 602 | { |
| 603 | if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX)) |
| 604 | return; |
| 605 | |
| 606 | sched->saved[sched->saved_states] = sched->state; |
| 607 | sched->saved_states++; |
| 608 | } |
| 609 | |
| 610 | static bool perf_sched_restore_state(struct perf_sched *sched) |
| 611 | { |
| 612 | if (!sched->saved_states) |
| 613 | return false; |
| 614 | |
| 615 | sched->saved_states--; |
| 616 | sched->state = sched->saved[sched->saved_states]; |
| 617 | |
| 618 | /* continue with next counter: */ |
| 619 | clear_bit(sched->state.counter++, sched->state.used); |
| 620 | |
| 621 | return true; |
| 622 | } |
| 623 | |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 624 | /* |
| 625 | * Select a counter for the current event to schedule. Return true on |
| 626 | * success. |
| 627 | */ |
Robert Richter | bc1738f | 2011-11-18 12:35:22 +0100 | [diff] [blame] | 628 | static bool __perf_sched_find_counter(struct perf_sched *sched) |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 629 | { |
| 630 | struct event_constraint *c; |
| 631 | int idx; |
| 632 | |
| 633 | if (!sched->state.unassigned) |
| 634 | return false; |
| 635 | |
| 636 | if (sched->state.event >= sched->max_events) |
| 637 | return false; |
| 638 | |
Andrew Hunter | 43b45780 | 2013-05-23 11:07:03 -0700 | [diff] [blame] | 639 | c = sched->events[sched->state.event]->hw.constraint; |
Peter Zijlstra | 4defea8 | 2011-11-10 15:15:42 +0100 | [diff] [blame] | 640 | /* Prefer fixed purpose counters */ |
Robert Richter | 15c7ad5 | 2012-06-20 20:46:33 +0200 | [diff] [blame] | 641 | if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) { |
| 642 | idx = INTEL_PMC_IDX_FIXED; |
Akinobu Mita | 307b1cd | 2012-03-23 15:02:03 -0700 | [diff] [blame] | 643 | for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) { |
Peter Zijlstra | 4defea8 | 2011-11-10 15:15:42 +0100 | [diff] [blame] | 644 | if (!__test_and_set_bit(idx, sched->state.used)) |
| 645 | goto done; |
| 646 | } |
| 647 | } |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 648 | /* Grab the first unused counter starting with idx */ |
| 649 | idx = sched->state.counter; |
Robert Richter | 15c7ad5 | 2012-06-20 20:46:33 +0200 | [diff] [blame] | 650 | for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) { |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 651 | if (!__test_and_set_bit(idx, sched->state.used)) |
Peter Zijlstra | 4defea8 | 2011-11-10 15:15:42 +0100 | [diff] [blame] | 652 | goto done; |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 653 | } |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 654 | |
Peter Zijlstra | 4defea8 | 2011-11-10 15:15:42 +0100 | [diff] [blame] | 655 | return false; |
| 656 | |
| 657 | done: |
| 658 | sched->state.counter = idx; |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 659 | |
Robert Richter | bc1738f | 2011-11-18 12:35:22 +0100 | [diff] [blame] | 660 | if (c->overlap) |
| 661 | perf_sched_save_state(sched); |
| 662 | |
| 663 | return true; |
| 664 | } |
| 665 | |
| 666 | static bool perf_sched_find_counter(struct perf_sched *sched) |
| 667 | { |
| 668 | while (!__perf_sched_find_counter(sched)) { |
| 669 | if (!perf_sched_restore_state(sched)) |
| 670 | return false; |
| 671 | } |
| 672 | |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 673 | return true; |
| 674 | } |
| 675 | |
| 676 | /* |
| 677 | * Go through all unassigned events and find the next one to schedule. |
| 678 | * Take events with the least weight first. Return true on success. |
| 679 | */ |
| 680 | static bool perf_sched_next_event(struct perf_sched *sched) |
| 681 | { |
| 682 | struct event_constraint *c; |
| 683 | |
| 684 | if (!sched->state.unassigned || !--sched->state.unassigned) |
| 685 | return false; |
| 686 | |
| 687 | do { |
| 688 | /* next event */ |
| 689 | sched->state.event++; |
| 690 | if (sched->state.event >= sched->max_events) { |
| 691 | /* next weight */ |
| 692 | sched->state.event = 0; |
| 693 | sched->state.weight++; |
| 694 | if (sched->state.weight > sched->max_weight) |
| 695 | return false; |
| 696 | } |
Andrew Hunter | 43b45780 | 2013-05-23 11:07:03 -0700 | [diff] [blame] | 697 | c = sched->events[sched->state.event]->hw.constraint; |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 698 | } while (c->weight != sched->state.weight); |
| 699 | |
| 700 | sched->state.counter = 0; /* start with first counter */ |
| 701 | |
| 702 | return true; |
| 703 | } |
| 704 | |
| 705 | /* |
| 706 | * Assign a counter for each event. |
| 707 | */ |
Andrew Hunter | 43b45780 | 2013-05-23 11:07:03 -0700 | [diff] [blame] | 708 | int perf_assign_events(struct perf_event **events, int n, |
Yan, Zheng | 4b4969b | 2012-06-15 14:31:30 +0800 | [diff] [blame] | 709 | int wmin, int wmax, int *assign) |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 710 | { |
| 711 | struct perf_sched sched; |
| 712 | |
Andrew Hunter | 43b45780 | 2013-05-23 11:07:03 -0700 | [diff] [blame] | 713 | perf_sched_init(&sched, events, n, wmin, wmax); |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 714 | |
| 715 | do { |
| 716 | if (!perf_sched_find_counter(&sched)) |
| 717 | break; /* failed */ |
| 718 | if (assign) |
| 719 | assign[sched.state.event] = sched.state.counter; |
| 720 | } while (perf_sched_next_event(&sched)); |
| 721 | |
| 722 | return sched.state.unassigned; |
| 723 | } |
| 724 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 725 | int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 726 | { |
Andrew Hunter | 43b45780 | 2013-05-23 11:07:03 -0700 | [diff] [blame] | 727 | struct event_constraint *c; |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 728 | unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; |
Stephane Eranian | 2f7f73a | 2013-06-20 18:42:54 +0200 | [diff] [blame] | 729 | struct perf_event *e; |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 730 | int i, wmin, wmax, num = 0; |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 731 | struct hw_perf_event *hwc; |
| 732 | |
| 733 | bitmap_zero(used_mask, X86_PMC_IDX_MAX); |
| 734 | |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 735 | for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) { |
Andrew Hunter | 43b45780 | 2013-05-23 11:07:03 -0700 | [diff] [blame] | 736 | hwc = &cpuc->event_list[i]->hw; |
Peter Zijlstra | b622d64 | 2010-02-01 15:36:30 +0100 | [diff] [blame] | 737 | c = x86_pmu.get_event_constraints(cpuc, cpuc->event_list[i]); |
Andrew Hunter | 43b45780 | 2013-05-23 11:07:03 -0700 | [diff] [blame] | 738 | hwc->constraint = c; |
| 739 | |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 740 | wmin = min(wmin, c->weight); |
| 741 | wmax = max(wmax, c->weight); |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 742 | } |
| 743 | |
| 744 | /* |
Stephane Eranian | 8113070 | 2010-01-21 17:39:01 +0200 | [diff] [blame] | 745 | * fastpath, try to reuse previous register |
| 746 | */ |
Peter Zijlstra | c933c1a | 2010-01-22 16:40:12 +0100 | [diff] [blame] | 747 | for (i = 0; i < n; i++) { |
Stephane Eranian | 8113070 | 2010-01-21 17:39:01 +0200 | [diff] [blame] | 748 | hwc = &cpuc->event_list[i]->hw; |
Andrew Hunter | 43b45780 | 2013-05-23 11:07:03 -0700 | [diff] [blame] | 749 | c = hwc->constraint; |
Stephane Eranian | 8113070 | 2010-01-21 17:39:01 +0200 | [diff] [blame] | 750 | |
| 751 | /* never assigned */ |
| 752 | if (hwc->idx == -1) |
| 753 | break; |
| 754 | |
| 755 | /* constraint still honored */ |
Peter Zijlstra | 63b1464 | 2010-01-22 16:32:17 +0100 | [diff] [blame] | 756 | if (!test_bit(hwc->idx, c->idxmsk)) |
Stephane Eranian | 8113070 | 2010-01-21 17:39:01 +0200 | [diff] [blame] | 757 | break; |
| 758 | |
| 759 | /* not already used */ |
| 760 | if (test_bit(hwc->idx, used_mask)) |
| 761 | break; |
| 762 | |
Peter Zijlstra | 34538ee | 2010-03-02 21:16:55 +0100 | [diff] [blame] | 763 | __set_bit(hwc->idx, used_mask); |
Stephane Eranian | 8113070 | 2010-01-21 17:39:01 +0200 | [diff] [blame] | 764 | if (assign) |
| 765 | assign[i] = hwc->idx; |
| 766 | } |
Stephane Eranian | 8113070 | 2010-01-21 17:39:01 +0200 | [diff] [blame] | 767 | |
Robert Richter | 1e2ad28 | 2011-11-18 12:35:21 +0100 | [diff] [blame] | 768 | /* slow path */ |
| 769 | if (i != n) |
Andrew Hunter | 43b45780 | 2013-05-23 11:07:03 -0700 | [diff] [blame] | 770 | num = perf_assign_events(cpuc->event_list, n, wmin, |
| 771 | wmax, assign); |
Stephane Eranian | 8113070 | 2010-01-21 17:39:01 +0200 | [diff] [blame] | 772 | |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 773 | /* |
Stephane Eranian | 2f7f73a | 2013-06-20 18:42:54 +0200 | [diff] [blame] | 774 | * Mark the event as committed, so we do not put_constraint() |
| 775 | * in case new events are added and fail scheduling. |
| 776 | */ |
| 777 | if (!num && assign) { |
| 778 | for (i = 0; i < n; i++) { |
| 779 | e = cpuc->event_list[i]; |
| 780 | e->hw.flags |= PERF_X86_EVENT_COMMITTED; |
| 781 | } |
| 782 | } |
| 783 | /* |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 784 | * scheduling failed or is just a simulation, |
| 785 | * free resources if necessary |
| 786 | */ |
| 787 | if (!assign || num) { |
| 788 | for (i = 0; i < n; i++) { |
Stephane Eranian | 2f7f73a | 2013-06-20 18:42:54 +0200 | [diff] [blame] | 789 | e = cpuc->event_list[i]; |
| 790 | /* |
| 791 | * do not put_constraint() on comitted events, |
| 792 | * because they are good to go |
| 793 | */ |
| 794 | if ((e->hw.flags & PERF_X86_EVENT_COMMITTED)) |
| 795 | continue; |
| 796 | |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 797 | if (x86_pmu.put_event_constraints) |
Stephane Eranian | 2f7f73a | 2013-06-20 18:42:54 +0200 | [diff] [blame] | 798 | x86_pmu.put_event_constraints(cpuc, e); |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 799 | } |
| 800 | } |
Peter Zijlstra | aa2bc1a | 2011-11-09 17:56:37 +0100 | [diff] [blame] | 801 | return num ? -EINVAL : 0; |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 802 | } |
| 803 | |
| 804 | /* |
| 805 | * dogrp: true if must collect siblings events (group) |
| 806 | * returns total number of events and error code |
| 807 | */ |
| 808 | static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp) |
| 809 | { |
| 810 | struct perf_event *event; |
| 811 | int n, max_count; |
| 812 | |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 813 | max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed; |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 814 | |
| 815 | /* current number of events already accepted */ |
| 816 | n = cpuc->n_events; |
| 817 | |
| 818 | if (is_x86_event(leader)) { |
| 819 | if (n >= max_count) |
Peter Zijlstra | aa2bc1a | 2011-11-09 17:56:37 +0100 | [diff] [blame] | 820 | return -EINVAL; |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 821 | cpuc->event_list[n] = leader; |
| 822 | n++; |
| 823 | } |
| 824 | if (!dogrp) |
| 825 | return n; |
| 826 | |
| 827 | list_for_each_entry(event, &leader->sibling_list, group_entry) { |
| 828 | if (!is_x86_event(event) || |
Stephane Eranian | 8113070 | 2010-01-21 17:39:01 +0200 | [diff] [blame] | 829 | event->state <= PERF_EVENT_STATE_OFF) |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 830 | continue; |
| 831 | |
| 832 | if (n >= max_count) |
Peter Zijlstra | aa2bc1a | 2011-11-09 17:56:37 +0100 | [diff] [blame] | 833 | return -EINVAL; |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 834 | |
| 835 | cpuc->event_list[n] = event; |
| 836 | n++; |
| 837 | } |
| 838 | return n; |
| 839 | } |
| 840 | |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 841 | static inline void x86_assign_hw_event(struct perf_event *event, |
Stephane Eranian | 447a194 | 2010-02-01 14:50:01 +0200 | [diff] [blame] | 842 | struct cpu_hw_events *cpuc, int i) |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 843 | { |
Stephane Eranian | 447a194 | 2010-02-01 14:50:01 +0200 | [diff] [blame] | 844 | struct hw_perf_event *hwc = &event->hw; |
| 845 | |
| 846 | hwc->idx = cpuc->assign[i]; |
| 847 | hwc->last_cpu = smp_processor_id(); |
| 848 | hwc->last_tag = ++cpuc->tags[i]; |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 849 | |
Robert Richter | 15c7ad5 | 2012-06-20 20:46:33 +0200 | [diff] [blame] | 850 | if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) { |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 851 | hwc->config_base = 0; |
| 852 | hwc->event_base = 0; |
Robert Richter | 15c7ad5 | 2012-06-20 20:46:33 +0200 | [diff] [blame] | 853 | } else if (hwc->idx >= INTEL_PMC_IDX_FIXED) { |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 854 | hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; |
Robert Richter | 15c7ad5 | 2012-06-20 20:46:33 +0200 | [diff] [blame] | 855 | hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED); |
| 856 | hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30; |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 857 | } else { |
Robert Richter | 73d6e52 | 2011-02-02 17:40:59 +0100 | [diff] [blame] | 858 | hwc->config_base = x86_pmu_config_addr(hwc->idx); |
| 859 | hwc->event_base = x86_pmu_event_addr(hwc->idx); |
Jacob Shin | 0fbdad0 | 2013-02-06 11:26:28 -0600 | [diff] [blame] | 860 | hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx); |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 861 | } |
| 862 | } |
| 863 | |
Stephane Eranian | 447a194 | 2010-02-01 14:50:01 +0200 | [diff] [blame] | 864 | static inline int match_prev_assignment(struct hw_perf_event *hwc, |
| 865 | struct cpu_hw_events *cpuc, |
| 866 | int i) |
| 867 | { |
| 868 | return hwc->idx == cpuc->assign[i] && |
| 869 | hwc->last_cpu == smp_processor_id() && |
| 870 | hwc->last_tag == cpuc->tags[i]; |
| 871 | } |
| 872 | |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 873 | static void x86_pmu_start(struct perf_event *event, int flags); |
Peter Zijlstra | 2e84187 | 2010-01-25 15:58:43 +0100 | [diff] [blame] | 874 | |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 875 | static void x86_pmu_enable(struct pmu *pmu) |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 876 | { |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 877 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); |
| 878 | struct perf_event *event; |
| 879 | struct hw_perf_event *hwc; |
Peter Zijlstra | 11164cd | 2010-03-26 14:08:44 +0100 | [diff] [blame] | 880 | int i, added = cpuc->n_added; |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 881 | |
Robert Richter | 85cf9db | 2009-04-29 12:47:20 +0200 | [diff] [blame] | 882 | if (!x86_pmu_initialized()) |
Ingo Molnar | 2b9ff0d | 2008-12-14 18:36:30 +0100 | [diff] [blame] | 883 | return; |
Peter Zijlstra | 1a6e21f | 2010-01-27 23:07:47 +0100 | [diff] [blame] | 884 | |
| 885 | if (cpuc->enabled) |
| 886 | return; |
| 887 | |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 888 | if (cpuc->n_added) { |
Peter Zijlstra | 19925ce | 2010-03-06 13:20:40 +0100 | [diff] [blame] | 889 | int n_running = cpuc->n_events - cpuc->n_added; |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 890 | /* |
| 891 | * apply assignment obtained either from |
| 892 | * hw_perf_group_sched_in() or x86_pmu_enable() |
| 893 | * |
| 894 | * step1: save events moving to new counters |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 895 | */ |
Peter Zijlstra | 19925ce | 2010-03-06 13:20:40 +0100 | [diff] [blame] | 896 | for (i = 0; i < n_running; i++) { |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 897 | event = cpuc->event_list[i]; |
| 898 | hwc = &event->hw; |
| 899 | |
Stephane Eranian | 447a194 | 2010-02-01 14:50:01 +0200 | [diff] [blame] | 900 | /* |
| 901 | * we can avoid reprogramming counter if: |
| 902 | * - assigned same counter as last time |
| 903 | * - running on same CPU as last time |
| 904 | * - no other event has used the counter since |
| 905 | */ |
| 906 | if (hwc->idx == -1 || |
| 907 | match_prev_assignment(hwc, cpuc, i)) |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 908 | continue; |
| 909 | |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 910 | /* |
| 911 | * Ensure we don't accidentally enable a stopped |
| 912 | * counter simply because we rescheduled. |
| 913 | */ |
| 914 | if (hwc->state & PERF_HES_STOPPED) |
| 915 | hwc->state |= PERF_HES_ARCH; |
| 916 | |
| 917 | x86_pmu_stop(event, PERF_EF_UPDATE); |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 918 | } |
| 919 | |
Peter Zijlstra | c347a2f | 2014-02-24 12:26:21 +0100 | [diff] [blame] | 920 | /* |
| 921 | * step2: reprogram moved events into new counters |
| 922 | */ |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 923 | for (i = 0; i < cpuc->n_events; i++) { |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 924 | event = cpuc->event_list[i]; |
| 925 | hwc = &event->hw; |
| 926 | |
Peter Zijlstra | 45e16a6 | 2010-03-11 13:40:30 +0100 | [diff] [blame] | 927 | if (!match_prev_assignment(hwc, cpuc, i)) |
Stephane Eranian | 447a194 | 2010-02-01 14:50:01 +0200 | [diff] [blame] | 928 | x86_assign_hw_event(event, cpuc, i); |
Peter Zijlstra | 45e16a6 | 2010-03-11 13:40:30 +0100 | [diff] [blame] | 929 | else if (i < n_running) |
| 930 | continue; |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 931 | |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 932 | if (hwc->state & PERF_HES_ARCH) |
| 933 | continue; |
| 934 | |
| 935 | x86_pmu_start(event, PERF_EF_RELOAD); |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 936 | } |
| 937 | cpuc->n_added = 0; |
| 938 | perf_events_lapic_init(); |
| 939 | } |
Peter Zijlstra | 1a6e21f | 2010-01-27 23:07:47 +0100 | [diff] [blame] | 940 | |
| 941 | cpuc->enabled = 1; |
| 942 | barrier(); |
| 943 | |
Peter Zijlstra | 11164cd | 2010-03-26 14:08:44 +0100 | [diff] [blame] | 944 | x86_pmu.enable_all(added); |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 945 | } |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 946 | |
Tejun Heo | 245b2e7 | 2009-06-24 15:13:48 +0900 | [diff] [blame] | 947 | static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 948 | |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 949 | /* |
| 950 | * Set the next IRQ period, based on the hwc->period_left value. |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 951 | * To be called with the event disabled in hw: |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 952 | */ |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 953 | int x86_perf_event_set_period(struct perf_event *event) |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 954 | { |
Peter Zijlstra | 07088ed | 2010-03-02 20:16:01 +0100 | [diff] [blame] | 955 | struct hw_perf_event *hwc = &event->hw; |
Peter Zijlstra | e785059 | 2010-05-21 14:43:08 +0200 | [diff] [blame] | 956 | s64 left = local64_read(&hwc->period_left); |
Peter Zijlstra | e4abb5d | 2009-06-02 16:08:20 +0200 | [diff] [blame] | 957 | s64 period = hwc->sample_period; |
Peter Zijlstra | 7645a24 | 2010-03-08 13:51:31 +0100 | [diff] [blame] | 958 | int ret = 0, idx = hwc->idx; |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 959 | |
Robert Richter | 15c7ad5 | 2012-06-20 20:46:33 +0200 | [diff] [blame] | 960 | if (idx == INTEL_PMC_IDX_FIXED_BTS) |
Markus Metzger | 30dd568 | 2009-07-21 15:56:48 +0200 | [diff] [blame] | 961 | return 0; |
| 962 | |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 963 | /* |
André Goddard Rosa | af901ca | 2009-11-14 13:09:05 -0200 | [diff] [blame] | 964 | * If we are way outside a reasonable range then just skip forward: |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 965 | */ |
| 966 | if (unlikely(left <= -period)) { |
| 967 | left = period; |
Peter Zijlstra | e785059 | 2010-05-21 14:43:08 +0200 | [diff] [blame] | 968 | local64_set(&hwc->period_left, left); |
Peter Zijlstra | 9e350de | 2009-06-10 21:34:59 +0200 | [diff] [blame] | 969 | hwc->last_period = period; |
Peter Zijlstra | e4abb5d | 2009-06-02 16:08:20 +0200 | [diff] [blame] | 970 | ret = 1; |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 971 | } |
| 972 | |
| 973 | if (unlikely(left <= 0)) { |
| 974 | left += period; |
Peter Zijlstra | e785059 | 2010-05-21 14:43:08 +0200 | [diff] [blame] | 975 | local64_set(&hwc->period_left, left); |
Peter Zijlstra | 9e350de | 2009-06-10 21:34:59 +0200 | [diff] [blame] | 976 | hwc->last_period = period; |
Peter Zijlstra | e4abb5d | 2009-06-02 16:08:20 +0200 | [diff] [blame] | 977 | ret = 1; |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 978 | } |
Ingo Molnar | 1c80f4b | 2009-05-15 08:25:22 +0200 | [diff] [blame] | 979 | /* |
Ingo Molnar | dfc6509 | 2009-09-21 11:31:35 +0200 | [diff] [blame] | 980 | * Quirk: certain CPUs dont like it if just 1 hw_event is left: |
Ingo Molnar | 1c80f4b | 2009-05-15 08:25:22 +0200 | [diff] [blame] | 981 | */ |
| 982 | if (unlikely(left < 2)) |
| 983 | left = 2; |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 984 | |
Peter Zijlstra | e4abb5d | 2009-06-02 16:08:20 +0200 | [diff] [blame] | 985 | if (left > x86_pmu.max_period) |
| 986 | left = x86_pmu.max_period; |
| 987 | |
Tejun Heo | 245b2e7 | 2009-06-24 15:13:48 +0900 | [diff] [blame] | 988 | per_cpu(pmc_prev_left[idx], smp_processor_id()) = left; |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 989 | |
| 990 | /* |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 991 | * The hw event starts counting from this event offset, |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 992 | * mark it to be able to extra future deltas: |
| 993 | */ |
Peter Zijlstra | e785059 | 2010-05-21 14:43:08 +0200 | [diff] [blame] | 994 | local64_set(&hwc->prev_count, (u64)-left); |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 995 | |
Robert Richter | 73d6e52 | 2011-02-02 17:40:59 +0100 | [diff] [blame] | 996 | wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); |
Cyrill Gorcunov | 68aa00a | 2010-06-03 01:23:04 +0400 | [diff] [blame] | 997 | |
| 998 | /* |
| 999 | * Due to erratum on certan cpu we need |
| 1000 | * a second write to be sure the register |
| 1001 | * is updated properly |
| 1002 | */ |
| 1003 | if (x86_pmu.perfctr_second_write) { |
Robert Richter | 73d6e52 | 2011-02-02 17:40:59 +0100 | [diff] [blame] | 1004 | wrmsrl(hwc->event_base, |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 1005 | (u64)(-left) & x86_pmu.cntval_mask); |
Cyrill Gorcunov | 68aa00a | 2010-06-03 01:23:04 +0400 | [diff] [blame] | 1006 | } |
Peter Zijlstra | e4abb5d | 2009-06-02 16:08:20 +0200 | [diff] [blame] | 1007 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1008 | perf_event_update_userpage(event); |
Peter Zijlstra | 194002b | 2009-06-22 16:35:24 +0200 | [diff] [blame] | 1009 | |
Peter Zijlstra | e4abb5d | 2009-06-02 16:08:20 +0200 | [diff] [blame] | 1010 | return ret; |
Ingo Molnar | 2f18d1e | 2008-12-22 11:10:42 +0100 | [diff] [blame] | 1011 | } |
| 1012 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 1013 | void x86_pmu_enable_event(struct perf_event *event) |
Robert Richter | 7c90cc4 | 2009-04-29 12:47:18 +0200 | [diff] [blame] | 1014 | { |
Tejun Heo | 0a3aee0 | 2010-12-18 16:28:55 +0100 | [diff] [blame] | 1015 | if (__this_cpu_read(cpu_hw_events.enabled)) |
Robert Richter | 31fa58a | 2010-04-13 22:23:14 +0200 | [diff] [blame] | 1016 | __x86_pmu_enable_event(&event->hw, |
| 1017 | ARCH_PERFMON_EVENTSEL_ENABLE); |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1018 | } |
| 1019 | |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 1020 | /* |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1021 | * Add a single event to the PMU. |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1022 | * |
| 1023 | * The event is added to the group of enabled events |
| 1024 | * but only if it can be scehduled with existing events. |
Peter Zijlstra | fe9081c | 2009-10-08 11:56:07 +0200 | [diff] [blame] | 1025 | */ |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1026 | static int x86_pmu_add(struct perf_event *event, int flags) |
Peter Zijlstra | fe9081c | 2009-10-08 11:56:07 +0200 | [diff] [blame] | 1027 | { |
| 1028 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1029 | struct hw_perf_event *hwc; |
| 1030 | int assign[X86_PMC_IDX_MAX]; |
| 1031 | int n, n0, ret; |
Peter Zijlstra | fe9081c | 2009-10-08 11:56:07 +0200 | [diff] [blame] | 1032 | |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1033 | hwc = &event->hw; |
Peter Zijlstra | fe9081c | 2009-10-08 11:56:07 +0200 | [diff] [blame] | 1034 | |
Peter Zijlstra | 33696fc | 2010-06-14 08:49:00 +0200 | [diff] [blame] | 1035 | perf_pmu_disable(event->pmu); |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1036 | n0 = cpuc->n_events; |
Peter Zijlstra | 24cd7f5 | 2010-06-11 17:32:03 +0200 | [diff] [blame] | 1037 | ret = n = collect_events(cpuc, event, false); |
| 1038 | if (ret < 0) |
| 1039 | goto out; |
Ingo Molnar | 53b441a | 2009-05-25 21:41:28 +0200 | [diff] [blame] | 1040 | |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1041 | hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; |
| 1042 | if (!(flags & PERF_EF_START)) |
| 1043 | hwc->state |= PERF_HES_ARCH; |
| 1044 | |
Lin Ming | 4d1c52b | 2010-04-23 13:56:12 +0800 | [diff] [blame] | 1045 | /* |
| 1046 | * If group events scheduling transaction was started, |
Lucas De Marchi | 0d2eb44 | 2011-03-17 16:24:16 -0300 | [diff] [blame] | 1047 | * skip the schedulability test here, it will be performed |
Peter Zijlstra | c347a2f | 2014-02-24 12:26:21 +0100 | [diff] [blame] | 1048 | * at commit time (->commit_txn) as a whole. |
Lin Ming | 4d1c52b | 2010-04-23 13:56:12 +0800 | [diff] [blame] | 1049 | */ |
Peter Zijlstra | 8d2cacb | 2010-05-25 17:49:05 +0200 | [diff] [blame] | 1050 | if (cpuc->group_flag & PERF_EVENT_TXN) |
Peter Zijlstra | 24cd7f5 | 2010-06-11 17:32:03 +0200 | [diff] [blame] | 1051 | goto done_collect; |
Lin Ming | 4d1c52b | 2010-04-23 13:56:12 +0800 | [diff] [blame] | 1052 | |
Cyrill Gorcunov | a072738 | 2010-03-11 19:54:39 +0300 | [diff] [blame] | 1053 | ret = x86_pmu.schedule_events(cpuc, n, assign); |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1054 | if (ret) |
Peter Zijlstra | 24cd7f5 | 2010-06-11 17:32:03 +0200 | [diff] [blame] | 1055 | goto out; |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1056 | /* |
| 1057 | * copy new assignment, now we know it is possible |
| 1058 | * will be used by hw_perf_enable() |
| 1059 | */ |
| 1060 | memcpy(cpuc->assign, assign, n*sizeof(int)); |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1061 | |
Peter Zijlstra | 24cd7f5 | 2010-06-11 17:32:03 +0200 | [diff] [blame] | 1062 | done_collect: |
Peter Zijlstra | c347a2f | 2014-02-24 12:26:21 +0100 | [diff] [blame] | 1063 | /* |
| 1064 | * Commit the collect_events() state. See x86_pmu_del() and |
| 1065 | * x86_pmu_*_txn(). |
| 1066 | */ |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1067 | cpuc->n_events = n; |
Peter Zijlstra | 356e1f2 | 2010-03-06 13:49:56 +0100 | [diff] [blame] | 1068 | cpuc->n_added += n - n0; |
Stephane Eranian | 90151c35 | 2010-05-25 16:23:10 +0200 | [diff] [blame] | 1069 | cpuc->n_txn += n - n0; |
Ingo Molnar | 7e2ae34 | 2008-12-09 11:40:46 +0100 | [diff] [blame] | 1070 | |
Peter Zijlstra | 24cd7f5 | 2010-06-11 17:32:03 +0200 | [diff] [blame] | 1071 | ret = 0; |
| 1072 | out: |
Peter Zijlstra | 33696fc | 2010-06-14 08:49:00 +0200 | [diff] [blame] | 1073 | perf_pmu_enable(event->pmu); |
Peter Zijlstra | 24cd7f5 | 2010-06-11 17:32:03 +0200 | [diff] [blame] | 1074 | return ret; |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1075 | } |
| 1076 | |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1077 | static void x86_pmu_start(struct perf_event *event, int flags) |
Stephane Eranian | d76a081 | 2010-02-08 17:06:01 +0200 | [diff] [blame] | 1078 | { |
Peter Zijlstra | c08053e | 2010-03-06 13:19:24 +0100 | [diff] [blame] | 1079 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); |
| 1080 | int idx = event->hw.idx; |
| 1081 | |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1082 | if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) |
| 1083 | return; |
Stephane Eranian | d76a081 | 2010-02-08 17:06:01 +0200 | [diff] [blame] | 1084 | |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1085 | if (WARN_ON_ONCE(idx == -1)) |
| 1086 | return; |
| 1087 | |
| 1088 | if (flags & PERF_EF_RELOAD) { |
| 1089 | WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); |
| 1090 | x86_perf_event_set_period(event); |
| 1091 | } |
| 1092 | |
| 1093 | event->hw.state = 0; |
| 1094 | |
Peter Zijlstra | c08053e | 2010-03-06 13:19:24 +0100 | [diff] [blame] | 1095 | cpuc->events[idx] = event; |
| 1096 | __set_bit(idx, cpuc->active_mask); |
Robert Richter | 63e6be6 | 2010-09-15 18:20:34 +0200 | [diff] [blame] | 1097 | __set_bit(idx, cpuc->running); |
Peter Zijlstra | aff3d91 | 2010-03-02 20:32:08 +0100 | [diff] [blame] | 1098 | x86_pmu.enable(event); |
Peter Zijlstra | c08053e | 2010-03-06 13:19:24 +0100 | [diff] [blame] | 1099 | perf_event_update_userpage(event); |
Peter Zijlstra | a78ac32 | 2009-05-25 17:39:05 +0200 | [diff] [blame] | 1100 | } |
| 1101 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1102 | void perf_event_print_debug(void) |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1103 | { |
Ingo Molnar | 2f18d1e | 2008-12-22 11:10:42 +0100 | [diff] [blame] | 1104 | u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed; |
Peter Zijlstra | ca03770 | 2010-03-02 19:52:12 +0100 | [diff] [blame] | 1105 | u64 pebs; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1106 | struct cpu_hw_events *cpuc; |
Peter Zijlstra | 5bb9efe | 2009-05-13 08:12:51 +0200 | [diff] [blame] | 1107 | unsigned long flags; |
Ingo Molnar | 1e12567 | 2008-12-09 12:18:18 +0100 | [diff] [blame] | 1108 | int cpu, idx; |
| 1109 | |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 1110 | if (!x86_pmu.num_counters) |
Ingo Molnar | 1e12567 | 2008-12-09 12:18:18 +0100 | [diff] [blame] | 1111 | return; |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1112 | |
Peter Zijlstra | 5bb9efe | 2009-05-13 08:12:51 +0200 | [diff] [blame] | 1113 | local_irq_save(flags); |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1114 | |
| 1115 | cpu = smp_processor_id(); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1116 | cpuc = &per_cpu(cpu_hw_events, cpu); |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1117 | |
Robert Richter | faa28ae | 2009-04-29 12:47:13 +0200 | [diff] [blame] | 1118 | if (x86_pmu.version >= 2) { |
Jaswinder Singh Rajput | a1ef58f | 2009-02-28 18:45:39 +0530 | [diff] [blame] | 1119 | rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl); |
| 1120 | rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status); |
| 1121 | rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow); |
| 1122 | rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed); |
Peter Zijlstra | ca03770 | 2010-03-02 19:52:12 +0100 | [diff] [blame] | 1123 | rdmsrl(MSR_IA32_PEBS_ENABLE, pebs); |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1124 | |
Jaswinder Singh Rajput | a1ef58f | 2009-02-28 18:45:39 +0530 | [diff] [blame] | 1125 | pr_info("\n"); |
| 1126 | pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl); |
| 1127 | pr_info("CPU#%d: status: %016llx\n", cpu, status); |
| 1128 | pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow); |
| 1129 | pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed); |
Peter Zijlstra | ca03770 | 2010-03-02 19:52:12 +0100 | [diff] [blame] | 1130 | pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs); |
Jaswinder Singh Rajput | f87ad35 | 2009-02-27 20:15:14 +0530 | [diff] [blame] | 1131 | } |
Peter Zijlstra | 7645a24 | 2010-03-08 13:51:31 +0100 | [diff] [blame] | 1132 | pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask); |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1133 | |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 1134 | for (idx = 0; idx < x86_pmu.num_counters; idx++) { |
Robert Richter | 41bf498 | 2011-02-02 17:40:57 +0100 | [diff] [blame] | 1135 | rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl); |
| 1136 | rdmsrl(x86_pmu_event_addr(idx), pmc_count); |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1137 | |
Tejun Heo | 245b2e7 | 2009-06-24 15:13:48 +0900 | [diff] [blame] | 1138 | prev_left = per_cpu(pmc_prev_left[idx], cpu); |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1139 | |
Jaswinder Singh Rajput | a1ef58f | 2009-02-28 18:45:39 +0530 | [diff] [blame] | 1140 | pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n", |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1141 | cpu, idx, pmc_ctrl); |
Jaswinder Singh Rajput | a1ef58f | 2009-02-28 18:45:39 +0530 | [diff] [blame] | 1142 | pr_info("CPU#%d: gen-PMC%d count: %016llx\n", |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1143 | cpu, idx, pmc_count); |
Jaswinder Singh Rajput | a1ef58f | 2009-02-28 18:45:39 +0530 | [diff] [blame] | 1144 | pr_info("CPU#%d: gen-PMC%d left: %016llx\n", |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 1145 | cpu, idx, prev_left); |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1146 | } |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 1147 | for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) { |
Ingo Molnar | 2f18d1e | 2008-12-22 11:10:42 +0100 | [diff] [blame] | 1148 | rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count); |
| 1149 | |
Jaswinder Singh Rajput | a1ef58f | 2009-02-28 18:45:39 +0530 | [diff] [blame] | 1150 | pr_info("CPU#%d: fixed-PMC%d count: %016llx\n", |
Ingo Molnar | 2f18d1e | 2008-12-22 11:10:42 +0100 | [diff] [blame] | 1151 | cpu, idx, pmc_count); |
| 1152 | } |
Peter Zijlstra | 5bb9efe | 2009-05-13 08:12:51 +0200 | [diff] [blame] | 1153 | local_irq_restore(flags); |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1154 | } |
| 1155 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 1156 | void x86_pmu_stop(struct perf_event *event, int flags) |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1157 | { |
Stephane Eranian | d76a081 | 2010-02-08 17:06:01 +0200 | [diff] [blame] | 1158 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1159 | struct hw_perf_event *hwc = &event->hw; |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1160 | |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1161 | if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) { |
| 1162 | x86_pmu.disable(event); |
| 1163 | cpuc->events[hwc->idx] = NULL; |
| 1164 | WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); |
| 1165 | hwc->state |= PERF_HES_STOPPED; |
| 1166 | } |
Peter Zijlstra | 71e2d28 | 2010-03-08 17:51:33 +0100 | [diff] [blame] | 1167 | |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1168 | if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { |
| 1169 | /* |
| 1170 | * Drain the remaining delta count out of a event |
| 1171 | * that we are disabling: |
| 1172 | */ |
| 1173 | x86_perf_event_update(event); |
| 1174 | hwc->state |= PERF_HES_UPTODATE; |
| 1175 | } |
Peter Zijlstra | 2e84187 | 2010-01-25 15:58:43 +0100 | [diff] [blame] | 1176 | } |
| 1177 | |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1178 | static void x86_pmu_del(struct perf_event *event, int flags) |
Peter Zijlstra | 2e84187 | 2010-01-25 15:58:43 +0100 | [diff] [blame] | 1179 | { |
| 1180 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); |
| 1181 | int i; |
| 1182 | |
Stephane Eranian | 90151c35 | 2010-05-25 16:23:10 +0200 | [diff] [blame] | 1183 | /* |
Stephane Eranian | 2f7f73a | 2013-06-20 18:42:54 +0200 | [diff] [blame] | 1184 | * event is descheduled |
| 1185 | */ |
| 1186 | event->hw.flags &= ~PERF_X86_EVENT_COMMITTED; |
| 1187 | |
| 1188 | /* |
Stephane Eranian | 90151c35 | 2010-05-25 16:23:10 +0200 | [diff] [blame] | 1189 | * If we're called during a txn, we don't need to do anything. |
| 1190 | * The events never got scheduled and ->cancel_txn will truncate |
| 1191 | * the event_list. |
Peter Zijlstra | c347a2f | 2014-02-24 12:26:21 +0100 | [diff] [blame] | 1192 | * |
| 1193 | * XXX assumes any ->del() called during a TXN will only be on |
| 1194 | * an event added during that same TXN. |
Stephane Eranian | 90151c35 | 2010-05-25 16:23:10 +0200 | [diff] [blame] | 1195 | */ |
Peter Zijlstra | 8d2cacb | 2010-05-25 17:49:05 +0200 | [diff] [blame] | 1196 | if (cpuc->group_flag & PERF_EVENT_TXN) |
Stephane Eranian | 90151c35 | 2010-05-25 16:23:10 +0200 | [diff] [blame] | 1197 | return; |
| 1198 | |
Peter Zijlstra | c347a2f | 2014-02-24 12:26:21 +0100 | [diff] [blame] | 1199 | /* |
| 1200 | * Not a TXN, therefore cleanup properly. |
| 1201 | */ |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1202 | x86_pmu_stop(event, PERF_EF_UPDATE); |
Peter Zijlstra | 194002b | 2009-06-22 16:35:24 +0200 | [diff] [blame] | 1203 | |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1204 | for (i = 0; i < cpuc->n_events; i++) { |
Peter Zijlstra | c347a2f | 2014-02-24 12:26:21 +0100 | [diff] [blame] | 1205 | if (event == cpuc->event_list[i]) |
Peter Zijlstra | 6c9687a | 2010-01-25 11:57:25 +0100 | [diff] [blame] | 1206 | break; |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1207 | } |
Peter Zijlstra | c347a2f | 2014-02-24 12:26:21 +0100 | [diff] [blame] | 1208 | |
| 1209 | if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */ |
| 1210 | return; |
| 1211 | |
| 1212 | /* If we have a newly added event; make sure to decrease n_added. */ |
| 1213 | if (i >= cpuc->n_events - cpuc->n_added) |
| 1214 | --cpuc->n_added; |
| 1215 | |
| 1216 | if (x86_pmu.put_event_constraints) |
| 1217 | x86_pmu.put_event_constraints(cpuc, event); |
| 1218 | |
| 1219 | /* Delete the array entry. */ |
| 1220 | while (++i < cpuc->n_events) |
| 1221 | cpuc->event_list[i-1] = cpuc->event_list[i]; |
| 1222 | --cpuc->n_events; |
| 1223 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1224 | perf_event_update_userpage(event); |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1225 | } |
| 1226 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 1227 | int x86_pmu_handle_irq(struct pt_regs *regs) |
Robert Richter | a29aa8a | 2009-04-29 12:47:21 +0200 | [diff] [blame] | 1228 | { |
Peter Zijlstra | df1a132 | 2009-06-10 21:02:22 +0200 | [diff] [blame] | 1229 | struct perf_sample_data data; |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1230 | struct cpu_hw_events *cpuc; |
| 1231 | struct perf_event *event; |
Vince Weaver | 11d1578 | 2009-07-08 17:46:14 -0400 | [diff] [blame] | 1232 | int idx, handled = 0; |
Ingo Molnar | 9029a5e | 2009-05-15 08:26:20 +0200 | [diff] [blame] | 1233 | u64 val; |
| 1234 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1235 | cpuc = &__get_cpu_var(cpu_hw_events); |
Robert Richter | a29aa8a | 2009-04-29 12:47:21 +0200 | [diff] [blame] | 1236 | |
Don Zickus | 2bce5da | 2011-04-27 06:32:33 -0400 | [diff] [blame] | 1237 | /* |
| 1238 | * Some chipsets need to unmask the LVTPC in a particular spot |
| 1239 | * inside the nmi handler. As a result, the unmasking was pushed |
| 1240 | * into all the nmi handlers. |
| 1241 | * |
| 1242 | * This generic handler doesn't seem to have any issues where the |
| 1243 | * unmasking occurs so it was left at the top. |
| 1244 | */ |
| 1245 | apic_write(APIC_LVTPC, APIC_DM_NMI); |
| 1246 | |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 1247 | for (idx = 0; idx < x86_pmu.num_counters; idx++) { |
Robert Richter | 63e6be6 | 2010-09-15 18:20:34 +0200 | [diff] [blame] | 1248 | if (!test_bit(idx, cpuc->active_mask)) { |
| 1249 | /* |
| 1250 | * Though we deactivated the counter some cpus |
| 1251 | * might still deliver spurious interrupts still |
| 1252 | * in flight. Catch them: |
| 1253 | */ |
| 1254 | if (__test_and_clear_bit(idx, cpuc->running)) |
| 1255 | handled++; |
Robert Richter | a29aa8a | 2009-04-29 12:47:21 +0200 | [diff] [blame] | 1256 | continue; |
Robert Richter | 63e6be6 | 2010-09-15 18:20:34 +0200 | [diff] [blame] | 1257 | } |
Peter Zijlstra | 962bf7a | 2009-05-13 13:21:36 +0200 | [diff] [blame] | 1258 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1259 | event = cpuc->events[idx]; |
Peter Zijlstra | a4016a7 | 2009-05-14 14:52:17 +0200 | [diff] [blame] | 1260 | |
Peter Zijlstra | cc2ad4b | 2010-03-02 20:18:39 +0100 | [diff] [blame] | 1261 | val = x86_perf_event_update(event); |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 1262 | if (val & (1ULL << (x86_pmu.cntval_bits - 1))) |
Peter Zijlstra | 48e22d5 | 2009-05-25 17:39:04 +0200 | [diff] [blame] | 1263 | continue; |
Peter Zijlstra | 962bf7a | 2009-05-13 13:21:36 +0200 | [diff] [blame] | 1264 | |
Peter Zijlstra | 9e350de | 2009-06-10 21:34:59 +0200 | [diff] [blame] | 1265 | /* |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1266 | * event overflow |
Peter Zijlstra | 9e350de | 2009-06-10 21:34:59 +0200 | [diff] [blame] | 1267 | */ |
Robert Richter | 4177c42 | 2010-09-02 15:07:48 -0400 | [diff] [blame] | 1268 | handled++; |
Robert Richter | fd0d000 | 2012-04-02 20:19:08 +0200 | [diff] [blame] | 1269 | perf_sample_data_init(&data, 0, event->hw.last_period); |
Peter Zijlstra | 9e350de | 2009-06-10 21:34:59 +0200 | [diff] [blame] | 1270 | |
Peter Zijlstra | 07088ed | 2010-03-02 20:16:01 +0100 | [diff] [blame] | 1271 | if (!x86_perf_event_set_period(event)) |
Peter Zijlstra | e4abb5d | 2009-06-02 16:08:20 +0200 | [diff] [blame] | 1272 | continue; |
| 1273 | |
Peter Zijlstra | a8b0ca1 | 2011-06-27 14:41:57 +0200 | [diff] [blame] | 1274 | if (perf_event_overflow(event, &data, regs)) |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1275 | x86_pmu_stop(event, 0); |
Robert Richter | a29aa8a | 2009-04-29 12:47:21 +0200 | [diff] [blame] | 1276 | } |
Peter Zijlstra | 962bf7a | 2009-05-13 13:21:36 +0200 | [diff] [blame] | 1277 | |
Peter Zijlstra | 9e350de | 2009-06-10 21:34:59 +0200 | [diff] [blame] | 1278 | if (handled) |
| 1279 | inc_irq_stat(apic_perf_irqs); |
| 1280 | |
Robert Richter | a29aa8a | 2009-04-29 12:47:21 +0200 | [diff] [blame] | 1281 | return handled; |
| 1282 | } |
Robert Richter | 39d81ea | 2009-04-29 12:47:05 +0200 | [diff] [blame] | 1283 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1284 | void perf_events_lapic_init(void) |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1285 | { |
Ingo Molnar | 04da8a4 | 2009-08-11 10:40:08 +0200 | [diff] [blame] | 1286 | if (!x86_pmu.apic || !x86_pmu_initialized()) |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1287 | return; |
Robert Richter | 85cf9db | 2009-04-29 12:47:20 +0200 | [diff] [blame] | 1288 | |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1289 | /* |
Yong Wang | c323d95 | 2009-05-29 13:28:35 +0800 | [diff] [blame] | 1290 | * Always use NMI for PMU |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1291 | */ |
Yong Wang | c323d95 | 2009-05-29 13:28:35 +0800 | [diff] [blame] | 1292 | apic_write(APIC_LVTPC, APIC_DM_NMI); |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1293 | } |
| 1294 | |
| 1295 | static int __kprobes |
Don Zickus | 9c48f1c | 2011-09-30 15:06:21 -0400 | [diff] [blame] | 1296 | perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1297 | { |
Dave Hansen | 14c63f1 | 2013-06-21 08:51:36 -0700 | [diff] [blame] | 1298 | u64 start_clock; |
| 1299 | u64 finish_clock; |
Peter Zijlstra | e8a923c | 2013-10-17 15:32:10 +0200 | [diff] [blame] | 1300 | int ret; |
Dave Hansen | 14c63f1 | 2013-06-21 08:51:36 -0700 | [diff] [blame] | 1301 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1302 | if (!atomic_read(&active_events)) |
Don Zickus | 9c48f1c | 2011-09-30 15:06:21 -0400 | [diff] [blame] | 1303 | return NMI_DONE; |
Peter Zijlstra | 63a809a | 2009-05-01 12:23:17 +0200 | [diff] [blame] | 1304 | |
Peter Zijlstra | e8a923c | 2013-10-17 15:32:10 +0200 | [diff] [blame] | 1305 | start_clock = sched_clock(); |
Dave Hansen | 14c63f1 | 2013-06-21 08:51:36 -0700 | [diff] [blame] | 1306 | ret = x86_pmu.handle_irq(regs); |
Peter Zijlstra | e8a923c | 2013-10-17 15:32:10 +0200 | [diff] [blame] | 1307 | finish_clock = sched_clock(); |
Dave Hansen | 14c63f1 | 2013-06-21 08:51:36 -0700 | [diff] [blame] | 1308 | |
| 1309 | perf_sample_event_took(finish_clock - start_clock); |
| 1310 | |
| 1311 | return ret; |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1312 | } |
| 1313 | |
Kevin Winchester | de0428a | 2011-08-30 20:41:05 -0300 | [diff] [blame] | 1314 | struct event_constraint emptyconstraint; |
| 1315 | struct event_constraint unconstrained; |
Jaswinder Singh Rajput | f87ad35 | 2009-02-27 20:15:14 +0530 | [diff] [blame] | 1316 | |
Paul Gortmaker | 148f9bb | 2013-06-18 18:23:59 -0400 | [diff] [blame] | 1317 | static int |
Peter Zijlstra | 3f6da39 | 2010-03-05 13:01:18 +0100 | [diff] [blame] | 1318 | x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) |
| 1319 | { |
| 1320 | unsigned int cpu = (long)hcpu; |
Peter Zijlstra | 7fdba1c | 2011-07-22 13:41:54 +0200 | [diff] [blame] | 1321 | struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); |
Peter Zijlstra | b38b24e | 2010-03-23 19:31:15 +0100 | [diff] [blame] | 1322 | int ret = NOTIFY_OK; |
Peter Zijlstra | 3f6da39 | 2010-03-05 13:01:18 +0100 | [diff] [blame] | 1323 | |
| 1324 | switch (action & ~CPU_TASKS_FROZEN) { |
| 1325 | case CPU_UP_PREPARE: |
Peter Zijlstra | 7fdba1c | 2011-07-22 13:41:54 +0200 | [diff] [blame] | 1326 | cpuc->kfree_on_online = NULL; |
Peter Zijlstra | 3f6da39 | 2010-03-05 13:01:18 +0100 | [diff] [blame] | 1327 | if (x86_pmu.cpu_prepare) |
Peter Zijlstra | b38b24e | 2010-03-23 19:31:15 +0100 | [diff] [blame] | 1328 | ret = x86_pmu.cpu_prepare(cpu); |
Peter Zijlstra | 3f6da39 | 2010-03-05 13:01:18 +0100 | [diff] [blame] | 1329 | break; |
| 1330 | |
| 1331 | case CPU_STARTING: |
Peter Zijlstra | 0c9d42e | 2011-11-20 23:30:47 +0100 | [diff] [blame] | 1332 | if (x86_pmu.attr_rdpmc) |
| 1333 | set_in_cr4(X86_CR4_PCE); |
Peter Zijlstra | 3f6da39 | 2010-03-05 13:01:18 +0100 | [diff] [blame] | 1334 | if (x86_pmu.cpu_starting) |
| 1335 | x86_pmu.cpu_starting(cpu); |
| 1336 | break; |
| 1337 | |
Peter Zijlstra | 7fdba1c | 2011-07-22 13:41:54 +0200 | [diff] [blame] | 1338 | case CPU_ONLINE: |
| 1339 | kfree(cpuc->kfree_on_online); |
| 1340 | break; |
| 1341 | |
Peter Zijlstra | 3f6da39 | 2010-03-05 13:01:18 +0100 | [diff] [blame] | 1342 | case CPU_DYING: |
| 1343 | if (x86_pmu.cpu_dying) |
| 1344 | x86_pmu.cpu_dying(cpu); |
| 1345 | break; |
| 1346 | |
Peter Zijlstra | b38b24e | 2010-03-23 19:31:15 +0100 | [diff] [blame] | 1347 | case CPU_UP_CANCELED: |
Peter Zijlstra | 3f6da39 | 2010-03-05 13:01:18 +0100 | [diff] [blame] | 1348 | case CPU_DEAD: |
| 1349 | if (x86_pmu.cpu_dead) |
| 1350 | x86_pmu.cpu_dead(cpu); |
| 1351 | break; |
| 1352 | |
| 1353 | default: |
| 1354 | break; |
| 1355 | } |
| 1356 | |
Peter Zijlstra | b38b24e | 2010-03-23 19:31:15 +0100 | [diff] [blame] | 1357 | return ret; |
Peter Zijlstra | 3f6da39 | 2010-03-05 13:01:18 +0100 | [diff] [blame] | 1358 | } |
| 1359 | |
Cyrill Gorcunov | 1255803 | 2009-12-10 19:56:34 +0300 | [diff] [blame] | 1360 | static void __init pmu_check_apic(void) |
| 1361 | { |
| 1362 | if (cpu_has_apic) |
| 1363 | return; |
| 1364 | |
| 1365 | x86_pmu.apic = 0; |
| 1366 | pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n"); |
| 1367 | pr_info("no hardware sampling interrupt available.\n"); |
| 1368 | } |
| 1369 | |
Jiri Olsa | 641cc93 | 2012-03-15 20:09:14 +0100 | [diff] [blame] | 1370 | static struct attribute_group x86_pmu_format_group = { |
| 1371 | .name = "format", |
| 1372 | .attrs = NULL, |
| 1373 | }; |
| 1374 | |
Jiri Olsa | 8300daa | 2012-10-10 14:53:12 +0200 | [diff] [blame] | 1375 | /* |
| 1376 | * Remove all undefined events (x86_pmu.event_map(id) == 0) |
| 1377 | * out of events_attr attributes. |
| 1378 | */ |
| 1379 | static void __init filter_events(struct attribute **attrs) |
| 1380 | { |
Stephane Eranian | 3a54aaa | 2013-01-24 16:10:26 +0100 | [diff] [blame] | 1381 | struct device_attribute *d; |
| 1382 | struct perf_pmu_events_attr *pmu_attr; |
Jiri Olsa | 8300daa | 2012-10-10 14:53:12 +0200 | [diff] [blame] | 1383 | int i, j; |
| 1384 | |
| 1385 | for (i = 0; attrs[i]; i++) { |
Stephane Eranian | 3a54aaa | 2013-01-24 16:10:26 +0100 | [diff] [blame] | 1386 | d = (struct device_attribute *)attrs[i]; |
| 1387 | pmu_attr = container_of(d, struct perf_pmu_events_attr, attr); |
| 1388 | /* str trumps id */ |
| 1389 | if (pmu_attr->event_str) |
| 1390 | continue; |
Jiri Olsa | 8300daa | 2012-10-10 14:53:12 +0200 | [diff] [blame] | 1391 | if (x86_pmu.event_map(i)) |
| 1392 | continue; |
| 1393 | |
| 1394 | for (j = i; attrs[j]; j++) |
| 1395 | attrs[j] = attrs[j + 1]; |
| 1396 | |
| 1397 | /* Check the shifted attr. */ |
| 1398 | i--; |
| 1399 | } |
| 1400 | } |
| 1401 | |
Andi Kleen | 1a6461b | 2013-01-24 16:10:25 +0100 | [diff] [blame] | 1402 | /* Merge two pointer arrays */ |
| 1403 | static __init struct attribute **merge_attr(struct attribute **a, struct attribute **b) |
| 1404 | { |
| 1405 | struct attribute **new; |
| 1406 | int j, i; |
| 1407 | |
| 1408 | for (j = 0; a[j]; j++) |
| 1409 | ; |
| 1410 | for (i = 0; b[i]; i++) |
| 1411 | j++; |
| 1412 | j++; |
| 1413 | |
| 1414 | new = kmalloc(sizeof(struct attribute *) * j, GFP_KERNEL); |
| 1415 | if (!new) |
| 1416 | return NULL; |
| 1417 | |
| 1418 | j = 0; |
| 1419 | for (i = 0; a[i]; i++) |
| 1420 | new[j++] = a[i]; |
| 1421 | for (i = 0; b[i]; i++) |
| 1422 | new[j++] = b[i]; |
| 1423 | new[j] = NULL; |
| 1424 | |
| 1425 | return new; |
| 1426 | } |
| 1427 | |
Stephane Eranian | f20093e | 2013-01-24 16:10:32 +0100 | [diff] [blame] | 1428 | ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, |
Jiri Olsa | a474739 | 2012-10-10 14:53:11 +0200 | [diff] [blame] | 1429 | char *page) |
| 1430 | { |
| 1431 | struct perf_pmu_events_attr *pmu_attr = \ |
| 1432 | container_of(attr, struct perf_pmu_events_attr, attr); |
Jiri Olsa | a474739 | 2012-10-10 14:53:11 +0200 | [diff] [blame] | 1433 | u64 config = x86_pmu.event_map(pmu_attr->id); |
Stephane Eranian | 3a54aaa | 2013-01-24 16:10:26 +0100 | [diff] [blame] | 1434 | |
| 1435 | /* string trumps id */ |
| 1436 | if (pmu_attr->event_str) |
| 1437 | return sprintf(page, "%s", pmu_attr->event_str); |
| 1438 | |
Jiri Olsa | a474739 | 2012-10-10 14:53:11 +0200 | [diff] [blame] | 1439 | return x86_pmu.events_sysfs_show(page, config); |
| 1440 | } |
| 1441 | |
Jiri Olsa | a474739 | 2012-10-10 14:53:11 +0200 | [diff] [blame] | 1442 | EVENT_ATTR(cpu-cycles, CPU_CYCLES ); |
| 1443 | EVENT_ATTR(instructions, INSTRUCTIONS ); |
| 1444 | EVENT_ATTR(cache-references, CACHE_REFERENCES ); |
| 1445 | EVENT_ATTR(cache-misses, CACHE_MISSES ); |
| 1446 | EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS ); |
| 1447 | EVENT_ATTR(branch-misses, BRANCH_MISSES ); |
| 1448 | EVENT_ATTR(bus-cycles, BUS_CYCLES ); |
| 1449 | EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND ); |
| 1450 | EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND ); |
| 1451 | EVENT_ATTR(ref-cycles, REF_CPU_CYCLES ); |
| 1452 | |
| 1453 | static struct attribute *empty_attrs; |
| 1454 | |
Peter Huewe | 95d18aa | 2012-10-29 21:48:17 +0100 | [diff] [blame] | 1455 | static struct attribute *events_attr[] = { |
Jiri Olsa | a474739 | 2012-10-10 14:53:11 +0200 | [diff] [blame] | 1456 | EVENT_PTR(CPU_CYCLES), |
| 1457 | EVENT_PTR(INSTRUCTIONS), |
| 1458 | EVENT_PTR(CACHE_REFERENCES), |
| 1459 | EVENT_PTR(CACHE_MISSES), |
| 1460 | EVENT_PTR(BRANCH_INSTRUCTIONS), |
| 1461 | EVENT_PTR(BRANCH_MISSES), |
| 1462 | EVENT_PTR(BUS_CYCLES), |
| 1463 | EVENT_PTR(STALLED_CYCLES_FRONTEND), |
| 1464 | EVENT_PTR(STALLED_CYCLES_BACKEND), |
| 1465 | EVENT_PTR(REF_CPU_CYCLES), |
| 1466 | NULL, |
| 1467 | }; |
| 1468 | |
| 1469 | static struct attribute_group x86_pmu_events_group = { |
| 1470 | .name = "events", |
| 1471 | .attrs = events_attr, |
| 1472 | }; |
| 1473 | |
Jiri Olsa | 0bf79d4 | 2012-10-10 14:53:14 +0200 | [diff] [blame] | 1474 | ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event) |
Jiri Olsa | 43c032f | 2012-10-10 14:53:13 +0200 | [diff] [blame] | 1475 | { |
Jiri Olsa | 43c032f | 2012-10-10 14:53:13 +0200 | [diff] [blame] | 1476 | u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8; |
| 1477 | u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24; |
| 1478 | bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE); |
| 1479 | bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL); |
| 1480 | bool any = (config & ARCH_PERFMON_EVENTSEL_ANY); |
| 1481 | bool inv = (config & ARCH_PERFMON_EVENTSEL_INV); |
| 1482 | ssize_t ret; |
| 1483 | |
| 1484 | /* |
| 1485 | * We have whole page size to spend and just little data |
| 1486 | * to write, so we can safely use sprintf. |
| 1487 | */ |
| 1488 | ret = sprintf(page, "event=0x%02llx", event); |
| 1489 | |
| 1490 | if (umask) |
| 1491 | ret += sprintf(page + ret, ",umask=0x%02llx", umask); |
| 1492 | |
| 1493 | if (edge) |
| 1494 | ret += sprintf(page + ret, ",edge"); |
| 1495 | |
| 1496 | if (pc) |
| 1497 | ret += sprintf(page + ret, ",pc"); |
| 1498 | |
| 1499 | if (any) |
| 1500 | ret += sprintf(page + ret, ",any"); |
| 1501 | |
| 1502 | if (inv) |
| 1503 | ret += sprintf(page + ret, ",inv"); |
| 1504 | |
| 1505 | if (cmask) |
| 1506 | ret += sprintf(page + ret, ",cmask=0x%02llx", cmask); |
| 1507 | |
| 1508 | ret += sprintf(page + ret, "\n"); |
| 1509 | |
| 1510 | return ret; |
| 1511 | } |
| 1512 | |
Yinghai Lu | dda9911 | 2011-01-21 15:30:01 -0800 | [diff] [blame] | 1513 | static int __init init_hw_perf_events(void) |
Jaswinder Singh Rajput | b56a380 | 2009-02-27 18:09:09 +0530 | [diff] [blame] | 1514 | { |
Peter Zijlstra | c1d6f42 | 2011-12-06 14:07:15 +0100 | [diff] [blame] | 1515 | struct x86_pmu_quirk *quirk; |
Robert Richter | 72eae04 | 2009-04-29 12:47:10 +0200 | [diff] [blame] | 1516 | int err; |
| 1517 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1518 | pr_info("Performance Events: "); |
Ingo Molnar | 1123e3a | 2009-05-29 11:25:09 +0200 | [diff] [blame] | 1519 | |
Jaswinder Singh Rajput | b56a380 | 2009-02-27 18:09:09 +0530 | [diff] [blame] | 1520 | switch (boot_cpu_data.x86_vendor) { |
| 1521 | case X86_VENDOR_INTEL: |
Robert Richter | 72eae04 | 2009-04-29 12:47:10 +0200 | [diff] [blame] | 1522 | err = intel_pmu_init(); |
Jaswinder Singh Rajput | b56a380 | 2009-02-27 18:09:09 +0530 | [diff] [blame] | 1523 | break; |
Jaswinder Singh Rajput | f87ad35 | 2009-02-27 20:15:14 +0530 | [diff] [blame] | 1524 | case X86_VENDOR_AMD: |
Robert Richter | 72eae04 | 2009-04-29 12:47:10 +0200 | [diff] [blame] | 1525 | err = amd_pmu_init(); |
Jaswinder Singh Rajput | f87ad35 | 2009-02-27 20:15:14 +0530 | [diff] [blame] | 1526 | break; |
Robert Richter | 4138960 | 2009-04-29 12:47:00 +0200 | [diff] [blame] | 1527 | default: |
Ingo Molnar | 8a3da6c7 | 2013-09-28 15:48:48 +0200 | [diff] [blame] | 1528 | err = -ENOTSUPP; |
Jaswinder Singh Rajput | b56a380 | 2009-02-27 18:09:09 +0530 | [diff] [blame] | 1529 | } |
Ingo Molnar | 1123e3a | 2009-05-29 11:25:09 +0200 | [diff] [blame] | 1530 | if (err != 0) { |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1531 | pr_cont("no PMU driver, software events only.\n"); |
Peter Zijlstra | 004417a | 2010-11-25 18:38:29 +0100 | [diff] [blame] | 1532 | return 0; |
Ingo Molnar | 1123e3a | 2009-05-29 11:25:09 +0200 | [diff] [blame] | 1533 | } |
Jaswinder Singh Rajput | b56a380 | 2009-02-27 18:09:09 +0530 | [diff] [blame] | 1534 | |
Cyrill Gorcunov | 1255803 | 2009-12-10 19:56:34 +0300 | [diff] [blame] | 1535 | pmu_check_apic(); |
| 1536 | |
Don Zickus | 33c6d6a | 2010-11-22 16:55:23 -0500 | [diff] [blame] | 1537 | /* sanity check that the hardware exists or is emulated */ |
Peter Zijlstra | 4407204 | 2010-12-08 15:56:23 +0100 | [diff] [blame] | 1538 | if (!check_hw_exists()) |
Peter Zijlstra | 004417a | 2010-11-25 18:38:29 +0100 | [diff] [blame] | 1539 | return 0; |
Don Zickus | 33c6d6a | 2010-11-22 16:55:23 -0500 | [diff] [blame] | 1540 | |
Ingo Molnar | 1123e3a | 2009-05-29 11:25:09 +0200 | [diff] [blame] | 1541 | pr_cont("%s PMU driver.\n", x86_pmu.name); |
Robert Richter | faa28ae | 2009-04-29 12:47:13 +0200 | [diff] [blame] | 1542 | |
Peter Zijlstra | e97df76 | 2014-02-05 20:48:51 +0100 | [diff] [blame] | 1543 | x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */ |
| 1544 | |
Peter Zijlstra | c1d6f42 | 2011-12-06 14:07:15 +0100 | [diff] [blame] | 1545 | for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next) |
| 1546 | quirk->func(); |
Peter Zijlstra | 3c44780 | 2010-03-04 21:49:01 +0100 | [diff] [blame] | 1547 | |
Robert Richter | a1eac7a | 2012-06-20 20:46:34 +0200 | [diff] [blame] | 1548 | if (!x86_pmu.intel_ctrl) |
| 1549 | x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1; |
Ingo Molnar | 862a1a5 | 2008-12-17 13:09:20 +0100 | [diff] [blame] | 1550 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1551 | perf_events_lapic_init(); |
Don Zickus | 9c48f1c | 2011-09-30 15:06:21 -0400 | [diff] [blame] | 1552 | register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI"); |
Ingo Molnar | 1123e3a | 2009-05-29 11:25:09 +0200 | [diff] [blame] | 1553 | |
Peter Zijlstra | 63b1464 | 2010-01-22 16:32:17 +0100 | [diff] [blame] | 1554 | unconstrained = (struct event_constraint) |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 1555 | __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1, |
Stephane Eranian | 9fac2cf | 2013-01-24 16:10:27 +0100 | [diff] [blame] | 1556 | 0, x86_pmu.num_counters, 0, 0); |
Peter Zijlstra | 63b1464 | 2010-01-22 16:32:17 +0100 | [diff] [blame] | 1557 | |
Jiri Olsa | 641cc93 | 2012-03-15 20:09:14 +0100 | [diff] [blame] | 1558 | x86_pmu_format_group.attrs = x86_pmu.format_attrs; |
Peter Zijlstra | 0c9d42e | 2011-11-20 23:30:47 +0100 | [diff] [blame] | 1559 | |
Stephane Eranian | f20093e | 2013-01-24 16:10:32 +0100 | [diff] [blame] | 1560 | if (x86_pmu.event_attrs) |
| 1561 | x86_pmu_events_group.attrs = x86_pmu.event_attrs; |
| 1562 | |
Jiri Olsa | a474739 | 2012-10-10 14:53:11 +0200 | [diff] [blame] | 1563 | if (!x86_pmu.events_sysfs_show) |
| 1564 | x86_pmu_events_group.attrs = &empty_attrs; |
Jiri Olsa | 8300daa | 2012-10-10 14:53:12 +0200 | [diff] [blame] | 1565 | else |
| 1566 | filter_events(x86_pmu_events_group.attrs); |
Jiri Olsa | a474739 | 2012-10-10 14:53:11 +0200 | [diff] [blame] | 1567 | |
Andi Kleen | 1a6461b | 2013-01-24 16:10:25 +0100 | [diff] [blame] | 1568 | if (x86_pmu.cpu_events) { |
| 1569 | struct attribute **tmp; |
| 1570 | |
| 1571 | tmp = merge_attr(x86_pmu_events_group.attrs, x86_pmu.cpu_events); |
| 1572 | if (!WARN_ON(!tmp)) |
| 1573 | x86_pmu_events_group.attrs = tmp; |
| 1574 | } |
| 1575 | |
Ingo Molnar | 57c0c15 | 2009-09-21 12:20:38 +0200 | [diff] [blame] | 1576 | pr_info("... version: %d\n", x86_pmu.version); |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 1577 | pr_info("... bit width: %d\n", x86_pmu.cntval_bits); |
| 1578 | pr_info("... generic registers: %d\n", x86_pmu.num_counters); |
| 1579 | pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask); |
Ingo Molnar | 57c0c15 | 2009-09-21 12:20:38 +0200 | [diff] [blame] | 1580 | pr_info("... max period: %016Lx\n", x86_pmu.max_period); |
Robert Richter | 948b1bb | 2010-03-29 18:36:50 +0200 | [diff] [blame] | 1581 | pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed); |
Robert Richter | d6dc0b4 | 2010-03-17 12:49:13 +0100 | [diff] [blame] | 1582 | pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl); |
Peter Zijlstra | 3f6da39 | 2010-03-05 13:01:18 +0100 | [diff] [blame] | 1583 | |
Peter Zijlstra | 2e80a82 | 2010-11-17 23:17:36 +0100 | [diff] [blame] | 1584 | perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); |
Peter Zijlstra | 3f6da39 | 2010-03-05 13:01:18 +0100 | [diff] [blame] | 1585 | perf_cpu_notifier(x86_pmu_notifier); |
Peter Zijlstra | 004417a | 2010-11-25 18:38:29 +0100 | [diff] [blame] | 1586 | |
| 1587 | return 0; |
Ingo Molnar | 241771e | 2008-12-03 10:39:53 +0100 | [diff] [blame] | 1588 | } |
Peter Zijlstra | 004417a | 2010-11-25 18:38:29 +0100 | [diff] [blame] | 1589 | early_initcall(init_hw_perf_events); |
Ingo Molnar | 621a01e | 2008-12-11 12:46:46 +0100 | [diff] [blame] | 1590 | |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1591 | static inline void x86_pmu_read(struct perf_event *event) |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 1592 | { |
Peter Zijlstra | cc2ad4b | 2010-03-02 20:18:39 +0100 | [diff] [blame] | 1593 | x86_perf_event_update(event); |
Ingo Molnar | ee06094 | 2008-12-13 09:00:03 +0100 | [diff] [blame] | 1594 | } |
| 1595 | |
Lin Ming | 4d1c52b | 2010-04-23 13:56:12 +0800 | [diff] [blame] | 1596 | /* |
| 1597 | * Start group events scheduling transaction |
| 1598 | * Set the flag to make pmu::enable() not perform the |
| 1599 | * schedulability test, it will be performed at commit time |
| 1600 | */ |
Peter Zijlstra | 51b0fe3 | 2010-06-11 13:35:57 +0200 | [diff] [blame] | 1601 | static void x86_pmu_start_txn(struct pmu *pmu) |
Lin Ming | 4d1c52b | 2010-04-23 13:56:12 +0800 | [diff] [blame] | 1602 | { |
Peter Zijlstra | 33696fc | 2010-06-14 08:49:00 +0200 | [diff] [blame] | 1603 | perf_pmu_disable(pmu); |
Tejun Heo | 0a3aee0 | 2010-12-18 16:28:55 +0100 | [diff] [blame] | 1604 | __this_cpu_or(cpu_hw_events.group_flag, PERF_EVENT_TXN); |
| 1605 | __this_cpu_write(cpu_hw_events.n_txn, 0); |
Lin Ming | 4d1c52b | 2010-04-23 13:56:12 +0800 | [diff] [blame] | 1606 | } |
| 1607 | |
| 1608 | /* |
| 1609 | * Stop group events scheduling transaction |
| 1610 | * Clear the flag and pmu::enable() will perform the |
| 1611 | * schedulability test. |
| 1612 | */ |
Peter Zijlstra | 51b0fe3 | 2010-06-11 13:35:57 +0200 | [diff] [blame] | 1613 | static void x86_pmu_cancel_txn(struct pmu *pmu) |
Lin Ming | 4d1c52b | 2010-04-23 13:56:12 +0800 | [diff] [blame] | 1614 | { |
Tejun Heo | 0a3aee0 | 2010-12-18 16:28:55 +0100 | [diff] [blame] | 1615 | __this_cpu_and(cpu_hw_events.group_flag, ~PERF_EVENT_TXN); |
Stephane Eranian | 90151c35 | 2010-05-25 16:23:10 +0200 | [diff] [blame] | 1616 | /* |
Peter Zijlstra | c347a2f | 2014-02-24 12:26:21 +0100 | [diff] [blame] | 1617 | * Truncate collected array by the number of events added in this |
| 1618 | * transaction. See x86_pmu_add() and x86_pmu_*_txn(). |
Stephane Eranian | 90151c35 | 2010-05-25 16:23:10 +0200 | [diff] [blame] | 1619 | */ |
Tejun Heo | 0a3aee0 | 2010-12-18 16:28:55 +0100 | [diff] [blame] | 1620 | __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn)); |
| 1621 | __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn)); |
Peter Zijlstra | 33696fc | 2010-06-14 08:49:00 +0200 | [diff] [blame] | 1622 | perf_pmu_enable(pmu); |
Lin Ming | 4d1c52b | 2010-04-23 13:56:12 +0800 | [diff] [blame] | 1623 | } |
| 1624 | |
| 1625 | /* |
| 1626 | * Commit group events scheduling transaction |
| 1627 | * Perform the group schedulability test as a whole |
| 1628 | * Return 0 if success |
Peter Zijlstra | c347a2f | 2014-02-24 12:26:21 +0100 | [diff] [blame] | 1629 | * |
| 1630 | * Does not cancel the transaction on failure; expects the caller to do this. |
Lin Ming | 4d1c52b | 2010-04-23 13:56:12 +0800 | [diff] [blame] | 1631 | */ |
Peter Zijlstra | 51b0fe3 | 2010-06-11 13:35:57 +0200 | [diff] [blame] | 1632 | static int x86_pmu_commit_txn(struct pmu *pmu) |
Lin Ming | 4d1c52b | 2010-04-23 13:56:12 +0800 | [diff] [blame] | 1633 | { |
| 1634 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); |
| 1635 | int assign[X86_PMC_IDX_MAX]; |
| 1636 | int n, ret; |
| 1637 | |
| 1638 | n = cpuc->n_events; |
| 1639 | |
| 1640 | if (!x86_pmu_initialized()) |
| 1641 | return -EAGAIN; |
| 1642 | |
| 1643 | ret = x86_pmu.schedule_events(cpuc, n, assign); |
| 1644 | if (ret) |
| 1645 | return ret; |
| 1646 | |
| 1647 | /* |
| 1648 | * copy new assignment, now we know it is possible |
| 1649 | * will be used by hw_perf_enable() |
| 1650 | */ |
| 1651 | memcpy(cpuc->assign, assign, n*sizeof(int)); |
| 1652 | |
Peter Zijlstra | 8d2cacb | 2010-05-25 17:49:05 +0200 | [diff] [blame] | 1653 | cpuc->group_flag &= ~PERF_EVENT_TXN; |
Peter Zijlstra | 33696fc | 2010-06-14 08:49:00 +0200 | [diff] [blame] | 1654 | perf_pmu_enable(pmu); |
Lin Ming | 4d1c52b | 2010-04-23 13:56:12 +0800 | [diff] [blame] | 1655 | return 0; |
| 1656 | } |
Stephane Eranian | cd8a38d | 2011-06-06 16:57:08 +0200 | [diff] [blame] | 1657 | /* |
| 1658 | * a fake_cpuc is used to validate event groups. Due to |
| 1659 | * the extra reg logic, we need to also allocate a fake |
| 1660 | * per_core and per_cpu structure. Otherwise, group events |
| 1661 | * using extra reg may conflict without the kernel being |
| 1662 | * able to catch this when the last event gets added to |
| 1663 | * the group. |
| 1664 | */ |
| 1665 | static void free_fake_cpuc(struct cpu_hw_events *cpuc) |
| 1666 | { |
| 1667 | kfree(cpuc->shared_regs); |
| 1668 | kfree(cpuc); |
| 1669 | } |
| 1670 | |
| 1671 | static struct cpu_hw_events *allocate_fake_cpuc(void) |
| 1672 | { |
| 1673 | struct cpu_hw_events *cpuc; |
| 1674 | int cpu = raw_smp_processor_id(); |
| 1675 | |
| 1676 | cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL); |
| 1677 | if (!cpuc) |
| 1678 | return ERR_PTR(-ENOMEM); |
| 1679 | |
| 1680 | /* only needed, if we have extra_regs */ |
| 1681 | if (x86_pmu.extra_regs) { |
| 1682 | cpuc->shared_regs = allocate_shared_regs(cpu); |
| 1683 | if (!cpuc->shared_regs) |
| 1684 | goto error; |
| 1685 | } |
Peter Zijlstra | b430f7c | 2012-06-05 15:30:31 +0200 | [diff] [blame] | 1686 | cpuc->is_fake = 1; |
Stephane Eranian | cd8a38d | 2011-06-06 16:57:08 +0200 | [diff] [blame] | 1687 | return cpuc; |
| 1688 | error: |
| 1689 | free_fake_cpuc(cpuc); |
| 1690 | return ERR_PTR(-ENOMEM); |
| 1691 | } |
Lin Ming | 4d1c52b | 2010-04-23 13:56:12 +0800 | [diff] [blame] | 1692 | |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1693 | /* |
Peter Zijlstra | ca03770 | 2010-03-02 19:52:12 +0100 | [diff] [blame] | 1694 | * validate that we can schedule this event |
| 1695 | */ |
| 1696 | static int validate_event(struct perf_event *event) |
| 1697 | { |
| 1698 | struct cpu_hw_events *fake_cpuc; |
| 1699 | struct event_constraint *c; |
| 1700 | int ret = 0; |
| 1701 | |
Stephane Eranian | cd8a38d | 2011-06-06 16:57:08 +0200 | [diff] [blame] | 1702 | fake_cpuc = allocate_fake_cpuc(); |
| 1703 | if (IS_ERR(fake_cpuc)) |
| 1704 | return PTR_ERR(fake_cpuc); |
Peter Zijlstra | ca03770 | 2010-03-02 19:52:12 +0100 | [diff] [blame] | 1705 | |
| 1706 | c = x86_pmu.get_event_constraints(fake_cpuc, event); |
| 1707 | |
| 1708 | if (!c || !c->weight) |
Peter Zijlstra | aa2bc1a | 2011-11-09 17:56:37 +0100 | [diff] [blame] | 1709 | ret = -EINVAL; |
Peter Zijlstra | ca03770 | 2010-03-02 19:52:12 +0100 | [diff] [blame] | 1710 | |
| 1711 | if (x86_pmu.put_event_constraints) |
| 1712 | x86_pmu.put_event_constraints(fake_cpuc, event); |
| 1713 | |
Stephane Eranian | cd8a38d | 2011-06-06 16:57:08 +0200 | [diff] [blame] | 1714 | free_fake_cpuc(fake_cpuc); |
Peter Zijlstra | ca03770 | 2010-03-02 19:52:12 +0100 | [diff] [blame] | 1715 | |
| 1716 | return ret; |
| 1717 | } |
| 1718 | |
| 1719 | /* |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1720 | * validate a single event group |
| 1721 | * |
| 1722 | * validation include: |
Ingo Molnar | 184f412 | 2010-01-27 08:39:39 +0100 | [diff] [blame] | 1723 | * - check events are compatible which each other |
| 1724 | * - events do not compete for the same counter |
| 1725 | * - number of events <= number of counters |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1726 | * |
| 1727 | * validation ensures the group can be loaded onto the |
| 1728 | * PMU if it was the only group available. |
| 1729 | */ |
Peter Zijlstra | fe9081c | 2009-10-08 11:56:07 +0200 | [diff] [blame] | 1730 | static int validate_group(struct perf_event *event) |
| 1731 | { |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1732 | struct perf_event *leader = event->group_leader; |
Peter Zijlstra | 502568d | 2010-01-22 14:35:46 +0100 | [diff] [blame] | 1733 | struct cpu_hw_events *fake_cpuc; |
Peter Zijlstra | aa2bc1a | 2011-11-09 17:56:37 +0100 | [diff] [blame] | 1734 | int ret = -EINVAL, n; |
Peter Zijlstra | fe9081c | 2009-10-08 11:56:07 +0200 | [diff] [blame] | 1735 | |
Stephane Eranian | cd8a38d | 2011-06-06 16:57:08 +0200 | [diff] [blame] | 1736 | fake_cpuc = allocate_fake_cpuc(); |
| 1737 | if (IS_ERR(fake_cpuc)) |
| 1738 | return PTR_ERR(fake_cpuc); |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1739 | /* |
| 1740 | * the event is not yet connected with its |
| 1741 | * siblings therefore we must first collect |
| 1742 | * existing siblings, then add the new event |
| 1743 | * before we can simulate the scheduling |
| 1744 | */ |
Peter Zijlstra | 502568d | 2010-01-22 14:35:46 +0100 | [diff] [blame] | 1745 | n = collect_events(fake_cpuc, leader, true); |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1746 | if (n < 0) |
Stephane Eranian | cd8a38d | 2011-06-06 16:57:08 +0200 | [diff] [blame] | 1747 | goto out; |
Peter Zijlstra | fe9081c | 2009-10-08 11:56:07 +0200 | [diff] [blame] | 1748 | |
Peter Zijlstra | 502568d | 2010-01-22 14:35:46 +0100 | [diff] [blame] | 1749 | fake_cpuc->n_events = n; |
| 1750 | n = collect_events(fake_cpuc, event, false); |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1751 | if (n < 0) |
Stephane Eranian | cd8a38d | 2011-06-06 16:57:08 +0200 | [diff] [blame] | 1752 | goto out; |
Peter Zijlstra | fe9081c | 2009-10-08 11:56:07 +0200 | [diff] [blame] | 1753 | |
Peter Zijlstra | 502568d | 2010-01-22 14:35:46 +0100 | [diff] [blame] | 1754 | fake_cpuc->n_events = n; |
Stephane Eranian | 1da53e0 | 2010-01-18 10:58:01 +0200 | [diff] [blame] | 1755 | |
Cyrill Gorcunov | a072738 | 2010-03-11 19:54:39 +0300 | [diff] [blame] | 1756 | ret = x86_pmu.schedule_events(fake_cpuc, n, NULL); |
Peter Zijlstra | 502568d | 2010-01-22 14:35:46 +0100 | [diff] [blame] | 1757 | |
Peter Zijlstra | 502568d | 2010-01-22 14:35:46 +0100 | [diff] [blame] | 1758 | out: |
Stephane Eranian | cd8a38d | 2011-06-06 16:57:08 +0200 | [diff] [blame] | 1759 | free_fake_cpuc(fake_cpuc); |
Peter Zijlstra | 502568d | 2010-01-22 14:35:46 +0100 | [diff] [blame] | 1760 | return ret; |
Peter Zijlstra | fe9081c | 2009-10-08 11:56:07 +0200 | [diff] [blame] | 1761 | } |
| 1762 | |
Yinghai Lu | dda9911 | 2011-01-21 15:30:01 -0800 | [diff] [blame] | 1763 | static int x86_pmu_event_init(struct perf_event *event) |
Ingo Molnar | 621a01e | 2008-12-11 12:46:46 +0100 | [diff] [blame] | 1764 | { |
Peter Zijlstra | 51b0fe3 | 2010-06-11 13:35:57 +0200 | [diff] [blame] | 1765 | struct pmu *tmp; |
Ingo Molnar | 621a01e | 2008-12-11 12:46:46 +0100 | [diff] [blame] | 1766 | int err; |
| 1767 | |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1768 | switch (event->attr.type) { |
| 1769 | case PERF_TYPE_RAW: |
| 1770 | case PERF_TYPE_HARDWARE: |
| 1771 | case PERF_TYPE_HW_CACHE: |
| 1772 | break; |
| 1773 | |
| 1774 | default: |
| 1775 | return -ENOENT; |
| 1776 | } |
| 1777 | |
| 1778 | err = __x86_pmu_event_init(event); |
Peter Zijlstra | fe9081c | 2009-10-08 11:56:07 +0200 | [diff] [blame] | 1779 | if (!err) { |
Stephane Eranian | 8113070 | 2010-01-21 17:39:01 +0200 | [diff] [blame] | 1780 | /* |
| 1781 | * we temporarily connect event to its pmu |
| 1782 | * such that validate_group() can classify |
| 1783 | * it as an x86 event using is_x86_event() |
| 1784 | */ |
| 1785 | tmp = event->pmu; |
| 1786 | event->pmu = &pmu; |
| 1787 | |
Peter Zijlstra | fe9081c | 2009-10-08 11:56:07 +0200 | [diff] [blame] | 1788 | if (event->group_leader != event) |
| 1789 | err = validate_group(event); |
Peter Zijlstra | ca03770 | 2010-03-02 19:52:12 +0100 | [diff] [blame] | 1790 | else |
| 1791 | err = validate_event(event); |
Stephane Eranian | 8113070 | 2010-01-21 17:39:01 +0200 | [diff] [blame] | 1792 | |
| 1793 | event->pmu = tmp; |
Peter Zijlstra | fe9081c | 2009-10-08 11:56:07 +0200 | [diff] [blame] | 1794 | } |
Peter Zijlstra | a1792cdac | 2009-09-09 10:04:47 +0200 | [diff] [blame] | 1795 | if (err) { |
Ingo Molnar | cdd6c48 | 2009-09-21 12:02:48 +0200 | [diff] [blame] | 1796 | if (event->destroy) |
| 1797 | event->destroy(event); |
Peter Zijlstra | a1792cdac | 2009-09-09 10:04:47 +0200 | [diff] [blame] | 1798 | } |
Ingo Molnar | 621a01e | 2008-12-11 12:46:46 +0100 | [diff] [blame] | 1799 | |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1800 | return err; |
Ingo Molnar | 621a01e | 2008-12-11 12:46:46 +0100 | [diff] [blame] | 1801 | } |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 1802 | |
Peter Zijlstra | fe4a330 | 2011-11-20 20:44:06 +0100 | [diff] [blame] | 1803 | static int x86_pmu_event_idx(struct perf_event *event) |
| 1804 | { |
| 1805 | int idx = event->hw.idx; |
| 1806 | |
Peter Zijlstra | c720620 | 2012-03-22 17:26:36 +0100 | [diff] [blame] | 1807 | if (!x86_pmu.attr_rdpmc) |
| 1808 | return 0; |
| 1809 | |
Robert Richter | 15c7ad5 | 2012-06-20 20:46:33 +0200 | [diff] [blame] | 1810 | if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) { |
| 1811 | idx -= INTEL_PMC_IDX_FIXED; |
Peter Zijlstra | fe4a330 | 2011-11-20 20:44:06 +0100 | [diff] [blame] | 1812 | idx |= 1 << 30; |
| 1813 | } |
| 1814 | |
| 1815 | return idx + 1; |
| 1816 | } |
| 1817 | |
Peter Zijlstra | 0c9d42e | 2011-11-20 23:30:47 +0100 | [diff] [blame] | 1818 | static ssize_t get_attr_rdpmc(struct device *cdev, |
| 1819 | struct device_attribute *attr, |
| 1820 | char *buf) |
| 1821 | { |
| 1822 | return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc); |
| 1823 | } |
| 1824 | |
| 1825 | static void change_rdpmc(void *info) |
| 1826 | { |
| 1827 | bool enable = !!(unsigned long)info; |
| 1828 | |
| 1829 | if (enable) |
| 1830 | set_in_cr4(X86_CR4_PCE); |
| 1831 | else |
| 1832 | clear_in_cr4(X86_CR4_PCE); |
| 1833 | } |
| 1834 | |
| 1835 | static ssize_t set_attr_rdpmc(struct device *cdev, |
| 1836 | struct device_attribute *attr, |
| 1837 | const char *buf, size_t count) |
| 1838 | { |
Shuah Khan | e2b297f | 2012-06-10 21:13:41 -0600 | [diff] [blame] | 1839 | unsigned long val; |
| 1840 | ssize_t ret; |
| 1841 | |
| 1842 | ret = kstrtoul(buf, 0, &val); |
| 1843 | if (ret) |
| 1844 | return ret; |
Peter Zijlstra | 0c9d42e | 2011-11-20 23:30:47 +0100 | [diff] [blame] | 1845 | |
Peter Zijlstra | e97df76 | 2014-02-05 20:48:51 +0100 | [diff] [blame] | 1846 | if (x86_pmu.attr_rdpmc_broken) |
| 1847 | return -ENOTSUPP; |
| 1848 | |
Peter Zijlstra | 0c9d42e | 2011-11-20 23:30:47 +0100 | [diff] [blame] | 1849 | if (!!val != !!x86_pmu.attr_rdpmc) { |
| 1850 | x86_pmu.attr_rdpmc = !!val; |
Peter Zijlstra | 0e9f220 | 2014-02-05 11:19:56 +0100 | [diff] [blame] | 1851 | on_each_cpu(change_rdpmc, (void *)val, 1); |
Peter Zijlstra | 0c9d42e | 2011-11-20 23:30:47 +0100 | [diff] [blame] | 1852 | } |
| 1853 | |
| 1854 | return count; |
| 1855 | } |
| 1856 | |
| 1857 | static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc); |
| 1858 | |
| 1859 | static struct attribute *x86_pmu_attrs[] = { |
| 1860 | &dev_attr_rdpmc.attr, |
| 1861 | NULL, |
| 1862 | }; |
| 1863 | |
| 1864 | static struct attribute_group x86_pmu_attr_group = { |
| 1865 | .attrs = x86_pmu_attrs, |
| 1866 | }; |
| 1867 | |
| 1868 | static const struct attribute_group *x86_pmu_attr_groups[] = { |
| 1869 | &x86_pmu_attr_group, |
Jiri Olsa | 641cc93 | 2012-03-15 20:09:14 +0100 | [diff] [blame] | 1870 | &x86_pmu_format_group, |
Jiri Olsa | a474739 | 2012-10-10 14:53:11 +0200 | [diff] [blame] | 1871 | &x86_pmu_events_group, |
Peter Zijlstra | 0c9d42e | 2011-11-20 23:30:47 +0100 | [diff] [blame] | 1872 | NULL, |
| 1873 | }; |
| 1874 | |
Stephane Eranian | d010b33 | 2012-02-09 23:21:00 +0100 | [diff] [blame] | 1875 | static void x86_pmu_flush_branch_stack(void) |
| 1876 | { |
| 1877 | if (x86_pmu.flush_branch_stack) |
| 1878 | x86_pmu.flush_branch_stack(); |
| 1879 | } |
| 1880 | |
Peter Zijlstra | c93dc84 | 2012-06-08 14:50:50 +0200 | [diff] [blame] | 1881 | void perf_check_microcode(void) |
| 1882 | { |
| 1883 | if (x86_pmu.check_microcode) |
| 1884 | x86_pmu.check_microcode(); |
| 1885 | } |
| 1886 | EXPORT_SYMBOL_GPL(perf_check_microcode); |
| 1887 | |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1888 | static struct pmu pmu = { |
Stephane Eranian | d010b33 | 2012-02-09 23:21:00 +0100 | [diff] [blame] | 1889 | .pmu_enable = x86_pmu_enable, |
| 1890 | .pmu_disable = x86_pmu_disable, |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1891 | |
Peter Zijlstra | c93dc84 | 2012-06-08 14:50:50 +0200 | [diff] [blame] | 1892 | .attr_groups = x86_pmu_attr_groups, |
Peter Zijlstra | 0c9d42e | 2011-11-20 23:30:47 +0100 | [diff] [blame] | 1893 | |
Peter Zijlstra | c93dc84 | 2012-06-08 14:50:50 +0200 | [diff] [blame] | 1894 | .event_init = x86_pmu_event_init, |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1895 | |
Stephane Eranian | d010b33 | 2012-02-09 23:21:00 +0100 | [diff] [blame] | 1896 | .add = x86_pmu_add, |
| 1897 | .del = x86_pmu_del, |
| 1898 | .start = x86_pmu_start, |
| 1899 | .stop = x86_pmu_stop, |
| 1900 | .read = x86_pmu_read, |
Peter Zijlstra | a4eaf7f | 2010-06-16 14:37:10 +0200 | [diff] [blame] | 1901 | |
Peter Zijlstra | c93dc84 | 2012-06-08 14:50:50 +0200 | [diff] [blame] | 1902 | .start_txn = x86_pmu_start_txn, |
| 1903 | .cancel_txn = x86_pmu_cancel_txn, |
| 1904 | .commit_txn = x86_pmu_commit_txn, |
Peter Zijlstra | fe4a330 | 2011-11-20 20:44:06 +0100 | [diff] [blame] | 1905 | |
Peter Zijlstra | c93dc84 | 2012-06-08 14:50:50 +0200 | [diff] [blame] | 1906 | .event_idx = x86_pmu_event_idx, |
Stephane Eranian | d010b33 | 2012-02-09 23:21:00 +0100 | [diff] [blame] | 1907 | .flush_branch_stack = x86_pmu_flush_branch_stack, |
Peter Zijlstra | b0a873e | 2010-06-11 13:35:08 +0200 | [diff] [blame] | 1908 | }; |
| 1909 | |
Peter Zijlstra | c720620 | 2012-03-22 17:26:36 +0100 | [diff] [blame] | 1910 | void arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now) |
Peter Zijlstra | e3f3541 | 2011-11-21 11:43:53 +0100 | [diff] [blame] | 1911 | { |
Peter Zijlstra | 20d1c86 | 2013-11-29 15:40:29 +0100 | [diff] [blame] | 1912 | struct cyc2ns_data *data; |
| 1913 | |
Peter Zijlstra | fa73158 | 2013-09-19 10:16:42 +0200 | [diff] [blame] | 1914 | userpg->cap_user_time = 0; |
| 1915 | userpg->cap_user_time_zero = 0; |
| 1916 | userpg->cap_user_rdpmc = x86_pmu.attr_rdpmc; |
Peter Zijlstra | c720620 | 2012-03-22 17:26:36 +0100 | [diff] [blame] | 1917 | userpg->pmc_width = x86_pmu.cntval_bits; |
| 1918 | |
Peter Zijlstra | 35af99e | 2013-11-28 19:38:42 +0100 | [diff] [blame] | 1919 | if (!sched_clock_stable()) |
Peter Zijlstra | e3f3541 | 2011-11-21 11:43:53 +0100 | [diff] [blame] | 1920 | return; |
| 1921 | |
Peter Zijlstra | 20d1c86 | 2013-11-29 15:40:29 +0100 | [diff] [blame] | 1922 | data = cyc2ns_read_begin(); |
| 1923 | |
Peter Zijlstra | fa73158 | 2013-09-19 10:16:42 +0200 | [diff] [blame] | 1924 | userpg->cap_user_time = 1; |
Peter Zijlstra | 20d1c86 | 2013-11-29 15:40:29 +0100 | [diff] [blame] | 1925 | userpg->time_mult = data->cyc2ns_mul; |
| 1926 | userpg->time_shift = data->cyc2ns_shift; |
| 1927 | userpg->time_offset = data->cyc2ns_offset - now; |
Adrian Hunter | c73deb6 | 2013-06-28 16:22:18 +0300 | [diff] [blame] | 1928 | |
Peter Zijlstra | d8b11a0 | 2013-10-03 16:00:14 +0200 | [diff] [blame] | 1929 | userpg->cap_user_time_zero = 1; |
Peter Zijlstra | 20d1c86 | 2013-11-29 15:40:29 +0100 | [diff] [blame] | 1930 | userpg->time_zero = data->cyc2ns_offset; |
| 1931 | |
| 1932 | cyc2ns_read_end(data); |
Peter Zijlstra | e3f3541 | 2011-11-21 11:43:53 +0100 | [diff] [blame] | 1933 | } |
| 1934 | |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 1935 | /* |
| 1936 | * callchain support |
| 1937 | */ |
| 1938 | |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 1939 | static int backtrace_stack(void *data, char *name) |
| 1940 | { |
Ingo Molnar | 038e836 | 2009-06-15 09:57:59 +0200 | [diff] [blame] | 1941 | return 0; |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 1942 | } |
| 1943 | |
| 1944 | static void backtrace_address(void *data, unsigned long addr, int reliable) |
| 1945 | { |
| 1946 | struct perf_callchain_entry *entry = data; |
| 1947 | |
Frederic Weisbecker | 70791ce | 2010-06-29 19:34:05 +0200 | [diff] [blame] | 1948 | perf_callchain_store(entry, addr); |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 1949 | } |
| 1950 | |
| 1951 | static const struct stacktrace_ops backtrace_ops = { |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 1952 | .stack = backtrace_stack, |
| 1953 | .address = backtrace_address, |
Frederic Weisbecker | 06d65bd | 2009-12-17 05:40:34 +0100 | [diff] [blame] | 1954 | .walk_stack = print_context_stack_bp, |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 1955 | }; |
| 1956 | |
Frederic Weisbecker | 56962b4 | 2010-06-30 23:03:51 +0200 | [diff] [blame] | 1957 | void |
| 1958 | perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs) |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 1959 | { |
Frederic Weisbecker | 927c7a9 | 2010-07-01 16:20:36 +0200 | [diff] [blame] | 1960 | if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { |
| 1961 | /* TODO: We don't support guest os callchain now */ |
Peter Zijlstra | ed80526 | 2010-08-20 14:30:41 +0200 | [diff] [blame] | 1962 | return; |
Frederic Weisbecker | 927c7a9 | 2010-07-01 16:20:36 +0200 | [diff] [blame] | 1963 | } |
| 1964 | |
Frederic Weisbecker | 70791ce | 2010-06-29 19:34:05 +0200 | [diff] [blame] | 1965 | perf_callchain_store(entry, regs->ip); |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 1966 | |
Namhyung Kim | e8e999cf | 2011-03-18 11:40:06 +0900 | [diff] [blame] | 1967 | dump_trace(NULL, regs, NULL, 0, &backtrace_ops, entry); |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 1968 | } |
| 1969 | |
Arun Sharma | bc6ca7b | 2012-04-20 15:41:35 -0700 | [diff] [blame] | 1970 | static inline int |
| 1971 | valid_user_frame(const void __user *fp, unsigned long size) |
| 1972 | { |
| 1973 | return (__range_not_ok(fp, size, TASK_SIZE) == 0); |
| 1974 | } |
| 1975 | |
Peter Zijlstra | d07bdfd | 2012-07-10 09:42:15 +0200 | [diff] [blame] | 1976 | static unsigned long get_segment_base(unsigned int segment) |
| 1977 | { |
| 1978 | struct desc_struct *desc; |
| 1979 | int idx = segment >> 3; |
| 1980 | |
| 1981 | if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) { |
| 1982 | if (idx > LDT_ENTRIES) |
| 1983 | return 0; |
| 1984 | |
| 1985 | if (idx > current->active_mm->context.size) |
| 1986 | return 0; |
| 1987 | |
| 1988 | desc = current->active_mm->context.ldt; |
| 1989 | } else { |
| 1990 | if (idx > GDT_ENTRIES) |
| 1991 | return 0; |
| 1992 | |
| 1993 | desc = __this_cpu_ptr(&gdt_page.gdt[0]); |
| 1994 | } |
| 1995 | |
| 1996 | return get_desc_base(desc + idx); |
| 1997 | } |
| 1998 | |
Torok Edwin | 257ef9d | 2010-03-17 12:07:16 +0200 | [diff] [blame] | 1999 | #ifdef CONFIG_COMPAT |
H. Peter Anvin | d1a797f | 2012-02-19 10:06:34 -0800 | [diff] [blame] | 2000 | |
| 2001 | #include <asm/compat.h> |
| 2002 | |
Torok Edwin | 257ef9d | 2010-03-17 12:07:16 +0200 | [diff] [blame] | 2003 | static inline int |
| 2004 | perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry) |
Peter Zijlstra | 74193ef | 2009-06-15 13:07:24 +0200 | [diff] [blame] | 2005 | { |
Torok Edwin | 257ef9d | 2010-03-17 12:07:16 +0200 | [diff] [blame] | 2006 | /* 32-bit process in 64-bit kernel. */ |
Peter Zijlstra | d07bdfd | 2012-07-10 09:42:15 +0200 | [diff] [blame] | 2007 | unsigned long ss_base, cs_base; |
Torok Edwin | 257ef9d | 2010-03-17 12:07:16 +0200 | [diff] [blame] | 2008 | struct stack_frame_ia32 frame; |
| 2009 | const void __user *fp; |
Peter Zijlstra | 74193ef | 2009-06-15 13:07:24 +0200 | [diff] [blame] | 2010 | |
Torok Edwin | 257ef9d | 2010-03-17 12:07:16 +0200 | [diff] [blame] | 2011 | if (!test_thread_flag(TIF_IA32)) |
| 2012 | return 0; |
Peter Zijlstra | 74193ef | 2009-06-15 13:07:24 +0200 | [diff] [blame] | 2013 | |
Peter Zijlstra | d07bdfd | 2012-07-10 09:42:15 +0200 | [diff] [blame] | 2014 | cs_base = get_segment_base(regs->cs); |
| 2015 | ss_base = get_segment_base(regs->ss); |
| 2016 | |
| 2017 | fp = compat_ptr(ss_base + regs->bp); |
Torok Edwin | 257ef9d | 2010-03-17 12:07:16 +0200 | [diff] [blame] | 2018 | while (entry->nr < PERF_MAX_STACK_DEPTH) { |
| 2019 | unsigned long bytes; |
| 2020 | frame.next_frame = 0; |
| 2021 | frame.return_address = 0; |
| 2022 | |
| 2023 | bytes = copy_from_user_nmi(&frame, fp, sizeof(frame)); |
Peter Zijlstra | 0a19684 | 2013-10-30 21:16:22 +0100 | [diff] [blame] | 2024 | if (bytes != 0) |
Torok Edwin | 257ef9d | 2010-03-17 12:07:16 +0200 | [diff] [blame] | 2025 | break; |
| 2026 | |
Arun Sharma | bc6ca7b | 2012-04-20 15:41:35 -0700 | [diff] [blame] | 2027 | if (!valid_user_frame(fp, sizeof(frame))) |
| 2028 | break; |
| 2029 | |
Peter Zijlstra | d07bdfd | 2012-07-10 09:42:15 +0200 | [diff] [blame] | 2030 | perf_callchain_store(entry, cs_base + frame.return_address); |
| 2031 | fp = compat_ptr(ss_base + frame.next_frame); |
Torok Edwin | 257ef9d | 2010-03-17 12:07:16 +0200 | [diff] [blame] | 2032 | } |
| 2033 | return 1; |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 2034 | } |
Torok Edwin | 257ef9d | 2010-03-17 12:07:16 +0200 | [diff] [blame] | 2035 | #else |
| 2036 | static inline int |
| 2037 | perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry) |
| 2038 | { |
| 2039 | return 0; |
| 2040 | } |
| 2041 | #endif |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 2042 | |
Frederic Weisbecker | 56962b4 | 2010-06-30 23:03:51 +0200 | [diff] [blame] | 2043 | void |
| 2044 | perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs) |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 2045 | { |
| 2046 | struct stack_frame frame; |
| 2047 | const void __user *fp; |
| 2048 | |
Frederic Weisbecker | 927c7a9 | 2010-07-01 16:20:36 +0200 | [diff] [blame] | 2049 | if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { |
| 2050 | /* TODO: We don't support guest os callchain now */ |
Peter Zijlstra | ed80526 | 2010-08-20 14:30:41 +0200 | [diff] [blame] | 2051 | return; |
Frederic Weisbecker | 927c7a9 | 2010-07-01 16:20:36 +0200 | [diff] [blame] | 2052 | } |
Ingo Molnar | 5a6cec3 | 2009-05-29 11:25:09 +0200 | [diff] [blame] | 2053 | |
Peter Zijlstra | d07bdfd | 2012-07-10 09:42:15 +0200 | [diff] [blame] | 2054 | /* |
| 2055 | * We don't know what to do with VM86 stacks.. ignore them for now. |
| 2056 | */ |
| 2057 | if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM)) |
| 2058 | return; |
| 2059 | |
Peter Zijlstra | 74193ef | 2009-06-15 13:07:24 +0200 | [diff] [blame] | 2060 | fp = (void __user *)regs->bp; |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 2061 | |
Frederic Weisbecker | 70791ce | 2010-06-29 19:34:05 +0200 | [diff] [blame] | 2062 | perf_callchain_store(entry, regs->ip); |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 2063 | |
Andrey Vagin | 20afc60 | 2011-08-30 12:32:36 +0400 | [diff] [blame] | 2064 | if (!current->mm) |
| 2065 | return; |
| 2066 | |
Torok Edwin | 257ef9d | 2010-03-17 12:07:16 +0200 | [diff] [blame] | 2067 | if (perf_callchain_user32(regs, entry)) |
| 2068 | return; |
| 2069 | |
Peter Zijlstra | f9188e0 | 2009-06-18 22:20:52 +0200 | [diff] [blame] | 2070 | while (entry->nr < PERF_MAX_STACK_DEPTH) { |
Torok Edwin | 257ef9d | 2010-03-17 12:07:16 +0200 | [diff] [blame] | 2071 | unsigned long bytes; |
Ingo Molnar | 038e836 | 2009-06-15 09:57:59 +0200 | [diff] [blame] | 2072 | frame.next_frame = NULL; |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 2073 | frame.return_address = 0; |
| 2074 | |
Torok Edwin | 257ef9d | 2010-03-17 12:07:16 +0200 | [diff] [blame] | 2075 | bytes = copy_from_user_nmi(&frame, fp, sizeof(frame)); |
Peter Zijlstra | 0a19684 | 2013-10-30 21:16:22 +0100 | [diff] [blame] | 2076 | if (bytes != 0) |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 2077 | break; |
| 2078 | |
Arun Sharma | bc6ca7b | 2012-04-20 15:41:35 -0700 | [diff] [blame] | 2079 | if (!valid_user_frame(fp, sizeof(frame))) |
| 2080 | break; |
| 2081 | |
Frederic Weisbecker | 70791ce | 2010-06-29 19:34:05 +0200 | [diff] [blame] | 2082 | perf_callchain_store(entry, frame.return_address); |
Ingo Molnar | 038e836 | 2009-06-15 09:57:59 +0200 | [diff] [blame] | 2083 | fp = frame.next_frame; |
Peter Zijlstra | d7d59fb | 2009-03-30 19:07:15 +0200 | [diff] [blame] | 2084 | } |
| 2085 | } |
| 2086 | |
Peter Zijlstra | d07bdfd | 2012-07-10 09:42:15 +0200 | [diff] [blame] | 2087 | /* |
| 2088 | * Deal with code segment offsets for the various execution modes: |
| 2089 | * |
| 2090 | * VM86 - the good olde 16 bit days, where the linear address is |
| 2091 | * 20 bits and we use regs->ip + 0x10 * regs->cs. |
| 2092 | * |
| 2093 | * IA32 - Where we need to look at GDT/LDT segment descriptor tables |
| 2094 | * to figure out what the 32bit base address is. |
| 2095 | * |
| 2096 | * X32 - has TIF_X32 set, but is running in x86_64 |
| 2097 | * |
| 2098 | * X86_64 - CS,DS,SS,ES are all zero based. |
| 2099 | */ |
| 2100 | static unsigned long code_segment_base(struct pt_regs *regs) |
| 2101 | { |
| 2102 | /* |
| 2103 | * If we are in VM86 mode, add the segment offset to convert to a |
| 2104 | * linear address. |
| 2105 | */ |
| 2106 | if (regs->flags & X86_VM_MASK) |
| 2107 | return 0x10 * regs->cs; |
| 2108 | |
| 2109 | /* |
| 2110 | * For IA32 we look at the GDT/LDT segment base to convert the |
| 2111 | * effective IP to a linear address. |
| 2112 | */ |
| 2113 | #ifdef CONFIG_X86_32 |
| 2114 | if (user_mode(regs) && regs->cs != __USER_CS) |
| 2115 | return get_segment_base(regs->cs); |
| 2116 | #else |
| 2117 | if (test_thread_flag(TIF_IA32)) { |
| 2118 | if (user_mode(regs) && regs->cs != __USER32_CS) |
| 2119 | return get_segment_base(regs->cs); |
| 2120 | } |
| 2121 | #endif |
| 2122 | return 0; |
| 2123 | } |
| 2124 | |
Zhang, Yanmin | 39447b3 | 2010-04-19 13:32:41 +0800 | [diff] [blame] | 2125 | unsigned long perf_instruction_pointer(struct pt_regs *regs) |
| 2126 | { |
Zhang, Yanmin | 39447b3 | 2010-04-19 13:32:41 +0800 | [diff] [blame] | 2127 | if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) |
Peter Zijlstra | d07bdfd | 2012-07-10 09:42:15 +0200 | [diff] [blame] | 2128 | return perf_guest_cbs->get_guest_ip(); |
Zhang, Yanmin | dcf46b9 | 2010-04-20 10:13:58 +0800 | [diff] [blame] | 2129 | |
Peter Zijlstra | d07bdfd | 2012-07-10 09:42:15 +0200 | [diff] [blame] | 2130 | return regs->ip + code_segment_base(regs); |
Zhang, Yanmin | 39447b3 | 2010-04-19 13:32:41 +0800 | [diff] [blame] | 2131 | } |
| 2132 | |
| 2133 | unsigned long perf_misc_flags(struct pt_regs *regs) |
| 2134 | { |
| 2135 | int misc = 0; |
Zhang, Yanmin | dcf46b9 | 2010-04-20 10:13:58 +0800 | [diff] [blame] | 2136 | |
Zhang, Yanmin | 39447b3 | 2010-04-19 13:32:41 +0800 | [diff] [blame] | 2137 | if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { |
Zhang, Yanmin | dcf46b9 | 2010-04-20 10:13:58 +0800 | [diff] [blame] | 2138 | if (perf_guest_cbs->is_user_mode()) |
| 2139 | misc |= PERF_RECORD_MISC_GUEST_USER; |
| 2140 | else |
| 2141 | misc |= PERF_RECORD_MISC_GUEST_KERNEL; |
| 2142 | } else { |
Peter Zijlstra | d07bdfd | 2012-07-10 09:42:15 +0200 | [diff] [blame] | 2143 | if (user_mode(regs)) |
Zhang, Yanmin | dcf46b9 | 2010-04-20 10:13:58 +0800 | [diff] [blame] | 2144 | misc |= PERF_RECORD_MISC_USER; |
| 2145 | else |
| 2146 | misc |= PERF_RECORD_MISC_KERNEL; |
| 2147 | } |
| 2148 | |
Zhang, Yanmin | 39447b3 | 2010-04-19 13:32:41 +0800 | [diff] [blame] | 2149 | if (regs->flags & PERF_EFLAGS_EXACT) |
Peter Zijlstra | ab60834 | 2010-04-08 23:03:20 +0200 | [diff] [blame] | 2150 | misc |= PERF_RECORD_MISC_EXACT_IP; |
Zhang, Yanmin | 39447b3 | 2010-04-19 13:32:41 +0800 | [diff] [blame] | 2151 | |
| 2152 | return misc; |
| 2153 | } |
Gleb Natapov | b3d9468 | 2011-11-10 14:57:27 +0200 | [diff] [blame] | 2154 | |
| 2155 | void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) |
| 2156 | { |
| 2157 | cap->version = x86_pmu.version; |
| 2158 | cap->num_counters_gp = x86_pmu.num_counters; |
| 2159 | cap->num_counters_fixed = x86_pmu.num_counters_fixed; |
| 2160 | cap->bit_width_gp = x86_pmu.cntval_bits; |
| 2161 | cap->bit_width_fixed = x86_pmu.cntval_bits; |
| 2162 | cap->events_mask = (unsigned int)x86_pmu.events_maskl; |
| 2163 | cap->events_mask_len = x86_pmu.events_mask_len; |
| 2164 | } |
| 2165 | EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability); |