blob: 8944062f46e284faaa65ec043403352ea7c5f14c [file] [log] [blame]
Kevin Winchesterde0428a2011-08-30 20:41:05 -03001/*
2 * Performance events x86 architecture header
3 *
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2009 Jaswinder Singh Rajput
7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
10 * Copyright (C) 2009 Google, Inc., Stephane Eranian
11 *
12 * For licencing details see kernel-base/COPYING
13 */
14
15#include <linux/perf_event.h>
16
17/*
18 * | NHM/WSM | SNB |
19 * register -------------------------------
20 * | HT | no HT | HT | no HT |
21 *-----------------------------------------
22 * offcore | core | core | cpu | core |
23 * lbr_sel | core | core | cpu | core |
24 * ld_lat | cpu | core | cpu | core |
25 *-----------------------------------------
26 *
27 * Given that there is a small number of shared regs,
28 * we can pre-allocate their slot in the per-cpu
29 * per-core reg tables.
30 */
31enum extra_reg_type {
32 EXTRA_REG_NONE = -1, /* not used */
33
34 EXTRA_REG_RSP_0 = 0, /* offcore_response_0 */
35 EXTRA_REG_RSP_1 = 1, /* offcore_response_1 */
36
37 EXTRA_REG_MAX /* number of entries needed */
38};
39
40struct event_constraint {
41 union {
42 unsigned long idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
43 u64 idxmsk64;
44 };
45 u64 code;
46 u64 cmask;
47 int weight;
Robert Richterbc1738f2011-11-18 12:35:22 +010048 int overlap;
Kevin Winchesterde0428a2011-08-30 20:41:05 -030049};
50
51struct amd_nb {
52 int nb_id; /* NorthBridge id */
53 int refcnt; /* reference count */
54 struct perf_event *owners[X86_PMC_IDX_MAX];
55 struct event_constraint event_constraints[X86_PMC_IDX_MAX];
56};
57
58/* The maximal number of PEBS events: */
59#define MAX_PEBS_EVENTS 4
60
61/*
62 * A debug store configuration.
63 *
64 * We only support architectures that use 64bit fields.
65 */
66struct debug_store {
67 u64 bts_buffer_base;
68 u64 bts_index;
69 u64 bts_absolute_maximum;
70 u64 bts_interrupt_threshold;
71 u64 pebs_buffer_base;
72 u64 pebs_index;
73 u64 pebs_absolute_maximum;
74 u64 pebs_interrupt_threshold;
75 u64 pebs_event_reset[MAX_PEBS_EVENTS];
76};
77
78/*
79 * Per register state.
80 */
81struct er_account {
82 raw_spinlock_t lock; /* per-core: protect structure */
83 u64 config; /* extra MSR config */
84 u64 reg; /* extra MSR number */
85 atomic_t ref; /* reference count */
86};
87
88/*
89 * Per core/cpu state
90 *
91 * Used to coordinate shared registers between HT threads or
92 * among events on a single PMU.
93 */
94struct intel_shared_regs {
95 struct er_account regs[EXTRA_REG_MAX];
96 int refcnt; /* per-core: #HT threads */
97 unsigned core_id; /* per-core: core id */
98};
99
100#define MAX_LBR_ENTRIES 16
101
102struct cpu_hw_events {
103 /*
104 * Generic x86 PMC bits
105 */
106 struct perf_event *events[X86_PMC_IDX_MAX]; /* in counter order */
107 unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
108 unsigned long running[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
109 int enabled;
110
111 int n_events;
112 int n_added;
113 int n_txn;
114 int assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
115 u64 tags[X86_PMC_IDX_MAX];
116 struct perf_event *event_list[X86_PMC_IDX_MAX]; /* in enabled order */
117
118 unsigned int group_flag;
119
120 /*
121 * Intel DebugStore bits
122 */
123 struct debug_store *ds;
124 u64 pebs_enabled;
125
126 /*
127 * Intel LBR bits
128 */
129 int lbr_users;
130 void *lbr_context;
131 struct perf_branch_stack lbr_stack;
132 struct perf_branch_entry lbr_entries[MAX_LBR_ENTRIES];
133
134 /*
Gleb Natapov144d31e2011-10-05 14:01:21 +0200135 * Intel host/guest exclude bits
136 */
137 u64 intel_ctrl_guest_mask;
138 u64 intel_ctrl_host_mask;
139 struct perf_guest_switch_msr guest_switch_msrs[X86_PMC_IDX_MAX];
140
141 /*
Kevin Winchesterde0428a2011-08-30 20:41:05 -0300142 * manage shared (per-core, per-cpu) registers
143 * used on Intel NHM/WSM/SNB
144 */
145 struct intel_shared_regs *shared_regs;
146
147 /*
148 * AMD specific bits
149 */
150 struct amd_nb *amd_nb;
151
152 void *kfree_on_online;
153};
154
Robert Richterbc1738f2011-11-18 12:35:22 +0100155#define __EVENT_CONSTRAINT(c, n, m, w, o) {\
Kevin Winchesterde0428a2011-08-30 20:41:05 -0300156 { .idxmsk64 = (n) }, \
157 .code = (c), \
158 .cmask = (m), \
159 .weight = (w), \
Robert Richterbc1738f2011-11-18 12:35:22 +0100160 .overlap = (o), \
Kevin Winchesterde0428a2011-08-30 20:41:05 -0300161}
162
163#define EVENT_CONSTRAINT(c, n, m) \
Robert Richterbc1738f2011-11-18 12:35:22 +0100164 __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 0)
165
166/*
167 * The overlap flag marks event constraints with overlapping counter
168 * masks. This is the case if the counter mask of such an event is not
169 * a subset of any other counter mask of a constraint with an equal or
170 * higher weight, e.g.:
171 *
172 * c_overlaps = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
173 * c_another1 = EVENT_CONSTRAINT(0, 0x07, 0);
174 * c_another2 = EVENT_CONSTRAINT(0, 0x38, 0);
175 *
176 * The event scheduler may not select the correct counter in the first
177 * cycle because it needs to know which subsequent events will be
178 * scheduled. It may fail to schedule the events then. So we set the
179 * overlap flag for such constraints to give the scheduler a hint which
180 * events to select for counter rescheduling.
181 *
182 * Care must be taken as the rescheduling algorithm is O(n!) which
183 * will increase scheduling cycles for an over-commited system
184 * dramatically. The number of such EVENT_CONSTRAINT_OVERLAP() macros
185 * and its counter masks must be kept at a minimum.
186 */
187#define EVENT_CONSTRAINT_OVERLAP(c, n, m) \
188 __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 1)
Kevin Winchesterde0428a2011-08-30 20:41:05 -0300189
190/*
191 * Constraint on the Event code.
192 */
193#define INTEL_EVENT_CONSTRAINT(c, n) \
194 EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT)
195
196/*
197 * Constraint on the Event code + UMask + fixed-mask
198 *
199 * filter mask to validate fixed counter events.
200 * the following filters disqualify for fixed counters:
201 * - inv
202 * - edge
203 * - cnt-mask
204 * The other filters are supported by fixed counters.
205 * The any-thread option is supported starting with v3.
206 */
207#define FIXED_EVENT_CONSTRAINT(c, n) \
208 EVENT_CONSTRAINT(c, (1ULL << (32+n)), X86_RAW_EVENT_MASK)
209
210/*
211 * Constraint on the Event code + UMask
212 */
213#define INTEL_UEVENT_CONSTRAINT(c, n) \
214 EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
215
216#define EVENT_CONSTRAINT_END \
217 EVENT_CONSTRAINT(0, 0, 0)
218
219#define for_each_event_constraint(e, c) \
220 for ((e) = (c); (e)->weight; (e)++)
221
222/*
223 * Extra registers for specific events.
224 *
225 * Some events need large masks and require external MSRs.
226 * Those extra MSRs end up being shared for all events on
227 * a PMU and sometimes between PMU of sibling HT threads.
228 * In either case, the kernel needs to handle conflicting
229 * accesses to those extra, shared, regs. The data structure
230 * to manage those registers is stored in cpu_hw_event.
231 */
232struct extra_reg {
233 unsigned int event;
234 unsigned int msr;
235 u64 config_mask;
236 u64 valid_mask;
237 int idx; /* per_xxx->regs[] reg index */
238};
239
240#define EVENT_EXTRA_REG(e, ms, m, vm, i) { \
241 .event = (e), \
242 .msr = (ms), \
243 .config_mask = (m), \
244 .valid_mask = (vm), \
245 .idx = EXTRA_REG_##i \
246 }
247
248#define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx) \
249 EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx)
250
251#define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0)
252
253union perf_capabilities {
254 struct {
255 u64 lbr_format:6;
256 u64 pebs_trap:1;
257 u64 pebs_arch_reg:1;
258 u64 pebs_format:4;
259 u64 smm_freeze:1;
260 };
261 u64 capabilities;
262};
263
Peter Zijlstrac1d6f422011-12-06 14:07:15 +0100264struct x86_pmu_quirk {
265 struct x86_pmu_quirk *next;
266 void (*func)(void);
267};
268
Kevin Winchesterde0428a2011-08-30 20:41:05 -0300269/*
270 * struct x86_pmu - generic x86 pmu
271 */
272struct x86_pmu {
273 /*
274 * Generic x86 PMC bits
275 */
276 const char *name;
277 int version;
278 int (*handle_irq)(struct pt_regs *);
279 void (*disable_all)(void);
280 void (*enable_all)(int added);
281 void (*enable)(struct perf_event *);
282 void (*disable)(struct perf_event *);
283 int (*hw_config)(struct perf_event *event);
284 int (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
285 unsigned eventsel;
286 unsigned perfctr;
287 u64 (*event_map)(int);
288 int max_events;
289 int num_counters;
290 int num_counters_fixed;
291 int cntval_bits;
292 u64 cntval_mask;
Gleb Natapovffb871b2011-11-10 14:57:26 +0200293 union {
294 unsigned long events_maskl;
295 unsigned long events_mask[BITS_TO_LONGS(ARCH_PERFMON_EVENTS_COUNT)];
296 };
297 int events_mask_len;
Kevin Winchesterde0428a2011-08-30 20:41:05 -0300298 int apic;
299 u64 max_period;
300 struct event_constraint *
301 (*get_event_constraints)(struct cpu_hw_events *cpuc,
302 struct perf_event *event);
303
304 void (*put_event_constraints)(struct cpu_hw_events *cpuc,
305 struct perf_event *event);
306 struct event_constraint *event_constraints;
Peter Zijlstrac1d6f422011-12-06 14:07:15 +0100307 struct x86_pmu_quirk *quirks;
Kevin Winchesterde0428a2011-08-30 20:41:05 -0300308 int perfctr_second_write;
309
310 int (*cpu_prepare)(int cpu);
311 void (*cpu_starting)(int cpu);
312 void (*cpu_dying)(int cpu);
313 void (*cpu_dead)(int cpu);
314
315 /*
316 * Intel Arch Perfmon v2+
317 */
318 u64 intel_ctrl;
319 union perf_capabilities intel_cap;
320
321 /*
322 * Intel DebugStore bits
323 */
324 int bts, pebs;
325 int bts_active, pebs_active;
326 int pebs_record_size;
327 void (*drain_pebs)(struct pt_regs *regs);
328 struct event_constraint *pebs_constraints;
329
330 /*
331 * Intel LBR
332 */
333 unsigned long lbr_tos, lbr_from, lbr_to; /* MSR base regs */
334 int lbr_nr; /* hardware stack size */
335
336 /*
337 * Extra registers for events
338 */
339 struct extra_reg *extra_regs;
340 unsigned int er_flags;
Gleb Natapov144d31e2011-10-05 14:01:21 +0200341
342 /*
343 * Intel host/guest support (KVM)
344 */
345 struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr);
Kevin Winchesterde0428a2011-08-30 20:41:05 -0300346};
347
Peter Zijlstrac1d6f422011-12-06 14:07:15 +0100348#define x86_add_quirk(func_) \
349do { \
350 static struct x86_pmu_quirk __quirk __initdata = { \
351 .func = func_, \
352 }; \
353 __quirk.next = x86_pmu.quirks; \
354 x86_pmu.quirks = &__quirk; \
355} while (0)
356
Kevin Winchesterde0428a2011-08-30 20:41:05 -0300357#define ERF_NO_HT_SHARING 1
358#define ERF_HAS_RSP_1 2
359
360extern struct x86_pmu x86_pmu __read_mostly;
361
362DECLARE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
363
364int x86_perf_event_set_period(struct perf_event *event);
365
366/*
367 * Generalized hw caching related hw_event table, filled
368 * in on a per model basis. A value of 0 means
369 * 'not supported', -1 means 'hw_event makes no sense on
370 * this CPU', any other value means the raw hw_event
371 * ID.
372 */
373
374#define C(x) PERF_COUNT_HW_CACHE_##x
375
376extern u64 __read_mostly hw_cache_event_ids
377 [PERF_COUNT_HW_CACHE_MAX]
378 [PERF_COUNT_HW_CACHE_OP_MAX]
379 [PERF_COUNT_HW_CACHE_RESULT_MAX];
380extern u64 __read_mostly hw_cache_extra_regs
381 [PERF_COUNT_HW_CACHE_MAX]
382 [PERF_COUNT_HW_CACHE_OP_MAX]
383 [PERF_COUNT_HW_CACHE_RESULT_MAX];
384
385u64 x86_perf_event_update(struct perf_event *event);
386
387static inline int x86_pmu_addr_offset(int index)
388{
389 int offset;
390
391 /* offset = X86_FEATURE_PERFCTR_CORE ? index << 1 : index */
392 alternative_io(ASM_NOP2,
393 "shll $1, %%eax",
394 X86_FEATURE_PERFCTR_CORE,
395 "=a" (offset),
396 "a" (index));
397
398 return offset;
399}
400
401static inline unsigned int x86_pmu_config_addr(int index)
402{
403 return x86_pmu.eventsel + x86_pmu_addr_offset(index);
404}
405
406static inline unsigned int x86_pmu_event_addr(int index)
407{
408 return x86_pmu.perfctr + x86_pmu_addr_offset(index);
409}
410
411int x86_setup_perfctr(struct perf_event *event);
412
413int x86_pmu_hw_config(struct perf_event *event);
414
415void x86_pmu_disable_all(void);
416
417static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
418 u64 enable_mask)
419{
420 if (hwc->extra_reg.reg)
421 wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config);
422 wrmsrl(hwc->config_base, hwc->config | enable_mask);
423}
424
425void x86_pmu_enable_all(int added);
426
427int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign);
428
429void x86_pmu_stop(struct perf_event *event, int flags);
430
431static inline void x86_pmu_disable_event(struct perf_event *event)
432{
433 struct hw_perf_event *hwc = &event->hw;
434
435 wrmsrl(hwc->config_base, hwc->config);
436}
437
438void x86_pmu_enable_event(struct perf_event *event);
439
440int x86_pmu_handle_irq(struct pt_regs *regs);
441
442extern struct event_constraint emptyconstraint;
443
444extern struct event_constraint unconstrained;
445
446#ifdef CONFIG_CPU_SUP_AMD
447
448int amd_pmu_init(void);
449
450#else /* CONFIG_CPU_SUP_AMD */
451
452static inline int amd_pmu_init(void)
453{
454 return 0;
455}
456
457#endif /* CONFIG_CPU_SUP_AMD */
458
459#ifdef CONFIG_CPU_SUP_INTEL
460
461int intel_pmu_save_and_restart(struct perf_event *event);
462
463struct event_constraint *
464x86_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event);
465
466struct intel_shared_regs *allocate_shared_regs(int cpu);
467
468int intel_pmu_init(void);
469
470void init_debug_store_on_cpu(int cpu);
471
472void fini_debug_store_on_cpu(int cpu);
473
474void release_ds_buffers(void);
475
476void reserve_ds_buffers(void);
477
478extern struct event_constraint bts_constraint;
479
480void intel_pmu_enable_bts(u64 config);
481
482void intel_pmu_disable_bts(void);
483
484int intel_pmu_drain_bts_buffer(void);
485
486extern struct event_constraint intel_core2_pebs_event_constraints[];
487
488extern struct event_constraint intel_atom_pebs_event_constraints[];
489
490extern struct event_constraint intel_nehalem_pebs_event_constraints[];
491
492extern struct event_constraint intel_westmere_pebs_event_constraints[];
493
494extern struct event_constraint intel_snb_pebs_event_constraints[];
495
496struct event_constraint *intel_pebs_constraints(struct perf_event *event);
497
498void intel_pmu_pebs_enable(struct perf_event *event);
499
500void intel_pmu_pebs_disable(struct perf_event *event);
501
502void intel_pmu_pebs_enable_all(void);
503
504void intel_pmu_pebs_disable_all(void);
505
506void intel_ds_init(void);
507
508void intel_pmu_lbr_reset(void);
509
510void intel_pmu_lbr_enable(struct perf_event *event);
511
512void intel_pmu_lbr_disable(struct perf_event *event);
513
514void intel_pmu_lbr_enable_all(void);
515
516void intel_pmu_lbr_disable_all(void);
517
518void intel_pmu_lbr_read(void);
519
520void intel_pmu_lbr_init_core(void);
521
522void intel_pmu_lbr_init_nhm(void);
523
524void intel_pmu_lbr_init_atom(void);
525
526int p4_pmu_init(void);
527
528int p6_pmu_init(void);
529
530#else /* CONFIG_CPU_SUP_INTEL */
531
532static inline void reserve_ds_buffers(void)
533{
534}
535
536static inline void release_ds_buffers(void)
537{
538}
539
540static inline int intel_pmu_init(void)
541{
542 return 0;
543}
544
545static inline struct intel_shared_regs *allocate_shared_regs(int cpu)
546{
547 return NULL;
548}
549
550#endif /* CONFIG_CPU_SUP_INTEL */