Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Performance event support for s390x - CPU-measurement Counter Facility |
| 3 | * |
| 4 | * Copyright IBM Corp. 2012 |
| 5 | * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> |
| 6 | * |
| 7 | * This program is free software; you can redistribute it and/or modify |
| 8 | * it under the terms of the GNU General Public License (version 2 only) |
| 9 | * as published by the Free Software Foundation. |
| 10 | */ |
| 11 | #define KMSG_COMPONENT "cpum_cf" |
| 12 | #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt |
| 13 | |
| 14 | #include <linux/kernel.h> |
| 15 | #include <linux/kernel_stat.h> |
| 16 | #include <linux/perf_event.h> |
| 17 | #include <linux/percpu.h> |
| 18 | #include <linux/notifier.h> |
| 19 | #include <linux/init.h> |
| 20 | #include <linux/export.h> |
Heiko Carstens | 1e3cab2 | 2012-03-30 09:40:55 +0200 | [diff] [blame] | 21 | #include <asm/ctl_reg.h> |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 22 | #include <asm/irq.h> |
| 23 | #include <asm/cpu_mf.h> |
| 24 | |
| 25 | /* CPU-measurement counter facility supports these CPU counter sets: |
| 26 | * For CPU counter sets: |
| 27 | * Basic counter set: 0-31 |
| 28 | * Problem-state counter set: 32-63 |
| 29 | * Crypto-activity counter set: 64-127 |
| 30 | * Extented counter set: 128-159 |
| 31 | */ |
| 32 | enum cpumf_ctr_set { |
| 33 | /* CPU counter sets */ |
| 34 | CPUMF_CTR_SET_BASIC = 0, |
| 35 | CPUMF_CTR_SET_USER = 1, |
| 36 | CPUMF_CTR_SET_CRYPTO = 2, |
| 37 | CPUMF_CTR_SET_EXT = 3, |
| 38 | |
| 39 | /* Maximum number of counter sets */ |
| 40 | CPUMF_CTR_SET_MAX, |
| 41 | }; |
| 42 | |
| 43 | #define CPUMF_LCCTL_ENABLE_SHIFT 16 |
| 44 | #define CPUMF_LCCTL_ACTCTL_SHIFT 0 |
| 45 | static const u64 cpumf_state_ctl[CPUMF_CTR_SET_MAX] = { |
| 46 | [CPUMF_CTR_SET_BASIC] = 0x02, |
| 47 | [CPUMF_CTR_SET_USER] = 0x04, |
| 48 | [CPUMF_CTR_SET_CRYPTO] = 0x08, |
| 49 | [CPUMF_CTR_SET_EXT] = 0x01, |
| 50 | }; |
| 51 | |
| 52 | static void ctr_set_enable(u64 *state, int ctr_set) |
| 53 | { |
| 54 | *state |= cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ENABLE_SHIFT; |
| 55 | } |
| 56 | static void ctr_set_disable(u64 *state, int ctr_set) |
| 57 | { |
| 58 | *state &= ~(cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ENABLE_SHIFT); |
| 59 | } |
| 60 | static void ctr_set_start(u64 *state, int ctr_set) |
| 61 | { |
| 62 | *state |= cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ACTCTL_SHIFT; |
| 63 | } |
| 64 | static void ctr_set_stop(u64 *state, int ctr_set) |
| 65 | { |
| 66 | *state &= ~(cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ACTCTL_SHIFT); |
| 67 | } |
| 68 | |
| 69 | /* Local CPUMF event structure */ |
| 70 | struct cpu_hw_events { |
| 71 | struct cpumf_ctr_info info; |
| 72 | atomic_t ctr_set[CPUMF_CTR_SET_MAX]; |
| 73 | u64 state, tx_state; |
| 74 | unsigned int flags; |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 75 | unsigned int txn_flags; |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 76 | }; |
| 77 | static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { |
| 78 | .ctr_set = { |
| 79 | [CPUMF_CTR_SET_BASIC] = ATOMIC_INIT(0), |
| 80 | [CPUMF_CTR_SET_USER] = ATOMIC_INIT(0), |
| 81 | [CPUMF_CTR_SET_CRYPTO] = ATOMIC_INIT(0), |
| 82 | [CPUMF_CTR_SET_EXT] = ATOMIC_INIT(0), |
| 83 | }, |
| 84 | .state = 0, |
| 85 | .flags = 0, |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 86 | .txn_flags = 0, |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 87 | }; |
| 88 | |
| 89 | static int get_counter_set(u64 event) |
| 90 | { |
| 91 | int set = -1; |
| 92 | |
| 93 | if (event < 32) |
| 94 | set = CPUMF_CTR_SET_BASIC; |
| 95 | else if (event < 64) |
| 96 | set = CPUMF_CTR_SET_USER; |
| 97 | else if (event < 128) |
| 98 | set = CPUMF_CTR_SET_CRYPTO; |
Hendrik Brueckner | f47586b2 | 2012-10-15 14:31:29 +0200 | [diff] [blame] | 99 | else if (event < 256) |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 100 | set = CPUMF_CTR_SET_EXT; |
| 101 | |
| 102 | return set; |
| 103 | } |
| 104 | |
| 105 | static int validate_event(const struct hw_perf_event *hwc) |
| 106 | { |
| 107 | switch (hwc->config_base) { |
| 108 | case CPUMF_CTR_SET_BASIC: |
| 109 | case CPUMF_CTR_SET_USER: |
| 110 | case CPUMF_CTR_SET_CRYPTO: |
| 111 | case CPUMF_CTR_SET_EXT: |
| 112 | /* check for reserved counters */ |
| 113 | if ((hwc->config >= 6 && hwc->config <= 31) || |
| 114 | (hwc->config >= 38 && hwc->config <= 63) || |
| 115 | (hwc->config >= 80 && hwc->config <= 127)) |
| 116 | return -EOPNOTSUPP; |
| 117 | break; |
| 118 | default: |
| 119 | return -EINVAL; |
| 120 | } |
| 121 | |
| 122 | return 0; |
| 123 | } |
| 124 | |
| 125 | static int validate_ctr_version(const struct hw_perf_event *hwc) |
| 126 | { |
| 127 | struct cpu_hw_events *cpuhw; |
| 128 | int err = 0; |
| 129 | |
| 130 | cpuhw = &get_cpu_var(cpu_hw_events); |
| 131 | |
| 132 | /* check required version for counter sets */ |
| 133 | switch (hwc->config_base) { |
| 134 | case CPUMF_CTR_SET_BASIC: |
| 135 | case CPUMF_CTR_SET_USER: |
| 136 | if (cpuhw->info.cfvn < 1) |
| 137 | err = -EOPNOTSUPP; |
| 138 | break; |
| 139 | case CPUMF_CTR_SET_CRYPTO: |
| 140 | case CPUMF_CTR_SET_EXT: |
| 141 | if (cpuhw->info.csvn < 1) |
| 142 | err = -EOPNOTSUPP; |
Hendrik Brueckner | f47586b2 | 2012-10-15 14:31:29 +0200 | [diff] [blame] | 143 | if ((cpuhw->info.csvn == 1 && hwc->config > 159) || |
| 144 | (cpuhw->info.csvn == 2 && hwc->config > 175) || |
| 145 | (cpuhw->info.csvn > 2 && hwc->config > 255)) |
| 146 | err = -EOPNOTSUPP; |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 147 | break; |
| 148 | } |
| 149 | |
| 150 | put_cpu_var(cpu_hw_events); |
| 151 | return err; |
| 152 | } |
| 153 | |
| 154 | static int validate_ctr_auth(const struct hw_perf_event *hwc) |
| 155 | { |
| 156 | struct cpu_hw_events *cpuhw; |
| 157 | u64 ctrs_state; |
| 158 | int err = 0; |
| 159 | |
| 160 | cpuhw = &get_cpu_var(cpu_hw_events); |
| 161 | |
Hendrik Brueckner | 58f8e9d | 2015-09-07 10:52:42 +0200 | [diff] [blame] | 162 | /* Check authorization for cpu counter sets. |
| 163 | * If the particular CPU counter set is not authorized, |
| 164 | * return with -ENOENT in order to fall back to other |
| 165 | * PMUs that might suffice the event request. |
| 166 | */ |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 167 | ctrs_state = cpumf_state_ctl[hwc->config_base]; |
| 168 | if (!(ctrs_state & cpuhw->info.auth_ctl)) |
Hendrik Brueckner | 58f8e9d | 2015-09-07 10:52:42 +0200 | [diff] [blame] | 169 | err = -ENOENT; |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 170 | |
| 171 | put_cpu_var(cpu_hw_events); |
| 172 | return err; |
| 173 | } |
| 174 | |
| 175 | /* |
| 176 | * Change the CPUMF state to active. |
| 177 | * Enable and activate the CPU-counter sets according |
| 178 | * to the per-cpu control state. |
| 179 | */ |
| 180 | static void cpumf_pmu_enable(struct pmu *pmu) |
| 181 | { |
Christoph Lameter | eb7e7d7 | 2014-08-17 12:30:45 -0500 | [diff] [blame] | 182 | struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 183 | int err; |
| 184 | |
| 185 | if (cpuhw->flags & PMU_F_ENABLED) |
| 186 | return; |
| 187 | |
| 188 | err = lcctl(cpuhw->state); |
| 189 | if (err) { |
| 190 | pr_err("Enabling the performance measuring unit " |
Heiko Carstens | af0ee94 | 2012-04-11 14:28:10 +0200 | [diff] [blame] | 191 | "failed with rc=%x\n", err); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 192 | return; |
| 193 | } |
| 194 | |
| 195 | cpuhw->flags |= PMU_F_ENABLED; |
| 196 | } |
| 197 | |
| 198 | /* |
| 199 | * Change the CPUMF state to inactive. |
| 200 | * Disable and enable (inactive) the CPU-counter sets according |
| 201 | * to the per-cpu control state. |
| 202 | */ |
| 203 | static void cpumf_pmu_disable(struct pmu *pmu) |
| 204 | { |
Christoph Lameter | eb7e7d7 | 2014-08-17 12:30:45 -0500 | [diff] [blame] | 205 | struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 206 | int err; |
| 207 | u64 inactive; |
| 208 | |
| 209 | if (!(cpuhw->flags & PMU_F_ENABLED)) |
| 210 | return; |
| 211 | |
| 212 | inactive = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1); |
| 213 | err = lcctl(inactive); |
| 214 | if (err) { |
| 215 | pr_err("Disabling the performance measuring unit " |
Heiko Carstens | af0ee94 | 2012-04-11 14:28:10 +0200 | [diff] [blame] | 216 | "failed with rc=%x\n", err); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 217 | return; |
| 218 | } |
| 219 | |
| 220 | cpuhw->flags &= ~PMU_F_ENABLED; |
| 221 | } |
| 222 | |
| 223 | |
| 224 | /* Number of perf events counting hardware events */ |
| 225 | static atomic_t num_events = ATOMIC_INIT(0); |
| 226 | /* Used to avoid races in calling reserve/release_cpumf_hardware */ |
| 227 | static DEFINE_MUTEX(pmc_reserve_mutex); |
| 228 | |
| 229 | /* CPU-measurement alerts for the counter facility */ |
| 230 | static void cpumf_measurement_alert(struct ext_code ext_code, |
| 231 | unsigned int alert, unsigned long unused) |
| 232 | { |
| 233 | struct cpu_hw_events *cpuhw; |
| 234 | |
| 235 | if (!(alert & CPU_MF_INT_CF_MASK)) |
| 236 | return; |
| 237 | |
Heiko Carstens | 420f42e | 2013-01-02 15:18:18 +0100 | [diff] [blame] | 238 | inc_irq_stat(IRQEXT_CMC); |
Christoph Lameter | eb7e7d7 | 2014-08-17 12:30:45 -0500 | [diff] [blame] | 239 | cpuhw = this_cpu_ptr(&cpu_hw_events); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 240 | |
| 241 | /* Measurement alerts are shared and might happen when the PMU |
| 242 | * is not reserved. Ignore these alerts in this case. */ |
| 243 | if (!(cpuhw->flags & PMU_F_RESERVED)) |
| 244 | return; |
| 245 | |
| 246 | /* counter authorization change alert */ |
| 247 | if (alert & CPU_MF_INT_CF_CACA) |
| 248 | qctri(&cpuhw->info); |
| 249 | |
| 250 | /* loss of counter data alert */ |
| 251 | if (alert & CPU_MF_INT_CF_LCDA) |
| 252 | pr_err("CPU[%i] Counter data was lost\n", smp_processor_id()); |
| 253 | } |
| 254 | |
| 255 | #define PMC_INIT 0 |
| 256 | #define PMC_RELEASE 1 |
| 257 | static void setup_pmc_cpu(void *flags) |
| 258 | { |
Christoph Lameter | eb7e7d7 | 2014-08-17 12:30:45 -0500 | [diff] [blame] | 259 | struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 260 | |
| 261 | switch (*((int *) flags)) { |
| 262 | case PMC_INIT: |
| 263 | memset(&cpuhw->info, 0, sizeof(cpuhw->info)); |
| 264 | qctri(&cpuhw->info); |
| 265 | cpuhw->flags |= PMU_F_RESERVED; |
| 266 | break; |
| 267 | |
| 268 | case PMC_RELEASE: |
| 269 | cpuhw->flags &= ~PMU_F_RESERVED; |
| 270 | break; |
| 271 | } |
| 272 | |
| 273 | /* Disable CPU counter sets */ |
| 274 | lcctl(0); |
| 275 | } |
| 276 | |
| 277 | /* Initialize the CPU-measurement facility */ |
| 278 | static int reserve_pmc_hardware(void) |
| 279 | { |
| 280 | int flags = PMC_INIT; |
| 281 | |
| 282 | on_each_cpu(setup_pmc_cpu, &flags, 1); |
Heiko Carstens | 82003c3 | 2013-09-04 13:35:45 +0200 | [diff] [blame] | 283 | irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 284 | |
| 285 | return 0; |
| 286 | } |
| 287 | |
| 288 | /* Release the CPU-measurement facility */ |
| 289 | static void release_pmc_hardware(void) |
| 290 | { |
| 291 | int flags = PMC_RELEASE; |
| 292 | |
| 293 | on_each_cpu(setup_pmc_cpu, &flags, 1); |
Heiko Carstens | 82003c3 | 2013-09-04 13:35:45 +0200 | [diff] [blame] | 294 | irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 295 | } |
| 296 | |
| 297 | /* Release the PMU if event is the last perf event */ |
| 298 | static void hw_perf_event_destroy(struct perf_event *event) |
| 299 | { |
| 300 | if (!atomic_add_unless(&num_events, -1, 1)) { |
| 301 | mutex_lock(&pmc_reserve_mutex); |
| 302 | if (atomic_dec_return(&num_events) == 0) |
| 303 | release_pmc_hardware(); |
| 304 | mutex_unlock(&pmc_reserve_mutex); |
| 305 | } |
| 306 | } |
| 307 | |
| 308 | /* CPUMF <-> perf event mappings for kernel+userspace (basic set) */ |
| 309 | static const int cpumf_generic_events_basic[] = { |
| 310 | [PERF_COUNT_HW_CPU_CYCLES] = 0, |
| 311 | [PERF_COUNT_HW_INSTRUCTIONS] = 1, |
| 312 | [PERF_COUNT_HW_CACHE_REFERENCES] = -1, |
| 313 | [PERF_COUNT_HW_CACHE_MISSES] = -1, |
| 314 | [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = -1, |
| 315 | [PERF_COUNT_HW_BRANCH_MISSES] = -1, |
| 316 | [PERF_COUNT_HW_BUS_CYCLES] = -1, |
| 317 | }; |
| 318 | /* CPUMF <-> perf event mappings for userspace (problem-state set) */ |
| 319 | static const int cpumf_generic_events_user[] = { |
| 320 | [PERF_COUNT_HW_CPU_CYCLES] = 32, |
| 321 | [PERF_COUNT_HW_INSTRUCTIONS] = 33, |
| 322 | [PERF_COUNT_HW_CACHE_REFERENCES] = -1, |
| 323 | [PERF_COUNT_HW_CACHE_MISSES] = -1, |
| 324 | [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = -1, |
| 325 | [PERF_COUNT_HW_BRANCH_MISSES] = -1, |
| 326 | [PERF_COUNT_HW_BUS_CYCLES] = -1, |
| 327 | }; |
| 328 | |
| 329 | static int __hw_perf_event_init(struct perf_event *event) |
| 330 | { |
| 331 | struct perf_event_attr *attr = &event->attr; |
| 332 | struct hw_perf_event *hwc = &event->hw; |
| 333 | int err; |
| 334 | u64 ev; |
| 335 | |
| 336 | switch (attr->type) { |
| 337 | case PERF_TYPE_RAW: |
| 338 | /* Raw events are used to access counters directly, |
| 339 | * hence do not permit excludes */ |
| 340 | if (attr->exclude_kernel || attr->exclude_user || |
| 341 | attr->exclude_hv) |
| 342 | return -EOPNOTSUPP; |
| 343 | ev = attr->config; |
| 344 | break; |
| 345 | |
| 346 | case PERF_TYPE_HARDWARE: |
| 347 | ev = attr->config; |
| 348 | /* Count user space (problem-state) only */ |
| 349 | if (!attr->exclude_user && attr->exclude_kernel) { |
| 350 | if (ev >= ARRAY_SIZE(cpumf_generic_events_user)) |
| 351 | return -EOPNOTSUPP; |
| 352 | ev = cpumf_generic_events_user[ev]; |
| 353 | |
| 354 | /* No support for kernel space counters only */ |
| 355 | } else if (!attr->exclude_kernel && attr->exclude_user) { |
| 356 | return -EOPNOTSUPP; |
| 357 | |
| 358 | /* Count user and kernel space */ |
| 359 | } else { |
| 360 | if (ev >= ARRAY_SIZE(cpumf_generic_events_basic)) |
| 361 | return -EOPNOTSUPP; |
| 362 | ev = cpumf_generic_events_basic[ev]; |
| 363 | } |
| 364 | break; |
| 365 | |
| 366 | default: |
| 367 | return -ENOENT; |
| 368 | } |
| 369 | |
| 370 | if (ev == -1) |
| 371 | return -ENOENT; |
| 372 | |
| 373 | if (ev >= PERF_CPUM_CF_MAX_CTR) |
| 374 | return -EINVAL; |
| 375 | |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 376 | /* Use the hardware perf event structure to store the counter number |
| 377 | * in 'config' member and the counter set to which the counter belongs |
| 378 | * in the 'config_base'. The counter set (config_base) is then used |
| 379 | * to enable/disable the counters. |
| 380 | */ |
| 381 | hwc->config = ev; |
| 382 | hwc->config_base = get_counter_set(ev); |
| 383 | |
| 384 | /* Validate the counter that is assigned to this event. |
| 385 | * Because the counter facility can use numerous counters at the |
Adam Buchbinder | 7eb792b | 2016-03-04 11:20:04 -0800 | [diff] [blame] | 386 | * same time without constraints, it is not necessary to explicitly |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 387 | * validate event groups (event->group_leader != event). |
| 388 | */ |
| 389 | err = validate_event(hwc); |
| 390 | if (err) |
| 391 | return err; |
| 392 | |
| 393 | /* Initialize for using the CPU-measurement counter facility */ |
| 394 | if (!atomic_inc_not_zero(&num_events)) { |
| 395 | mutex_lock(&pmc_reserve_mutex); |
| 396 | if (atomic_read(&num_events) == 0 && reserve_pmc_hardware()) |
| 397 | err = -EBUSY; |
| 398 | else |
| 399 | atomic_inc(&num_events); |
| 400 | mutex_unlock(&pmc_reserve_mutex); |
| 401 | } |
| 402 | event->destroy = hw_perf_event_destroy; |
| 403 | |
| 404 | /* Finally, validate version and authorization of the counter set */ |
| 405 | err = validate_ctr_auth(hwc); |
| 406 | if (!err) |
| 407 | err = validate_ctr_version(hwc); |
| 408 | |
| 409 | return err; |
| 410 | } |
| 411 | |
| 412 | static int cpumf_pmu_event_init(struct perf_event *event) |
| 413 | { |
| 414 | int err; |
| 415 | |
| 416 | switch (event->attr.type) { |
| 417 | case PERF_TYPE_HARDWARE: |
| 418 | case PERF_TYPE_HW_CACHE: |
| 419 | case PERF_TYPE_RAW: |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 420 | err = __hw_perf_event_init(event); |
| 421 | break; |
| 422 | default: |
| 423 | return -ENOENT; |
| 424 | } |
| 425 | |
| 426 | if (unlikely(err) && event->destroy) |
| 427 | event->destroy(event); |
| 428 | |
| 429 | return err; |
| 430 | } |
| 431 | |
| 432 | static int hw_perf_event_reset(struct perf_event *event) |
| 433 | { |
| 434 | u64 prev, new; |
| 435 | int err; |
| 436 | |
| 437 | do { |
| 438 | prev = local64_read(&event->hw.prev_count); |
| 439 | err = ecctr(event->hw.config, &new); |
| 440 | if (err) { |
| 441 | if (err != 3) |
| 442 | break; |
| 443 | /* The counter is not (yet) available. This |
| 444 | * might happen if the counter set to which |
| 445 | * this counter belongs is in the disabled |
| 446 | * state. |
| 447 | */ |
| 448 | new = 0; |
| 449 | } |
| 450 | } while (local64_cmpxchg(&event->hw.prev_count, prev, new) != prev); |
| 451 | |
| 452 | return err; |
| 453 | } |
| 454 | |
| 455 | static int hw_perf_event_update(struct perf_event *event) |
| 456 | { |
| 457 | u64 prev, new, delta; |
| 458 | int err; |
| 459 | |
| 460 | do { |
| 461 | prev = local64_read(&event->hw.prev_count); |
| 462 | err = ecctr(event->hw.config, &new); |
| 463 | if (err) |
| 464 | goto out; |
| 465 | } while (local64_cmpxchg(&event->hw.prev_count, prev, new) != prev); |
| 466 | |
| 467 | delta = (prev <= new) ? new - prev |
| 468 | : (-1ULL - prev) + new + 1; /* overflow */ |
| 469 | local64_add(delta, &event->count); |
| 470 | out: |
| 471 | return err; |
| 472 | } |
| 473 | |
| 474 | static void cpumf_pmu_read(struct perf_event *event) |
| 475 | { |
| 476 | if (event->hw.state & PERF_HES_STOPPED) |
| 477 | return; |
| 478 | |
| 479 | hw_perf_event_update(event); |
| 480 | } |
| 481 | |
| 482 | static void cpumf_pmu_start(struct perf_event *event, int flags) |
| 483 | { |
Christoph Lameter | eb7e7d7 | 2014-08-17 12:30:45 -0500 | [diff] [blame] | 484 | struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 485 | struct hw_perf_event *hwc = &event->hw; |
| 486 | |
| 487 | if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) |
| 488 | return; |
| 489 | |
| 490 | if (WARN_ON_ONCE(hwc->config == -1)) |
| 491 | return; |
| 492 | |
| 493 | if (flags & PERF_EF_RELOAD) |
| 494 | WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); |
| 495 | |
| 496 | hwc->state = 0; |
| 497 | |
| 498 | /* (Re-)enable and activate the counter set */ |
| 499 | ctr_set_enable(&cpuhw->state, hwc->config_base); |
| 500 | ctr_set_start(&cpuhw->state, hwc->config_base); |
| 501 | |
| 502 | /* The counter set to which this counter belongs can be already active. |
| 503 | * Because all counters in a set are active, the event->hw.prev_count |
| 504 | * needs to be synchronized. At this point, the counter set can be in |
| 505 | * the inactive or disabled state. |
| 506 | */ |
| 507 | hw_perf_event_reset(event); |
| 508 | |
| 509 | /* increment refcount for this counter set */ |
| 510 | atomic_inc(&cpuhw->ctr_set[hwc->config_base]); |
| 511 | } |
| 512 | |
| 513 | static void cpumf_pmu_stop(struct perf_event *event, int flags) |
| 514 | { |
Christoph Lameter | eb7e7d7 | 2014-08-17 12:30:45 -0500 | [diff] [blame] | 515 | struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 516 | struct hw_perf_event *hwc = &event->hw; |
| 517 | |
| 518 | if (!(hwc->state & PERF_HES_STOPPED)) { |
| 519 | /* Decrement reference count for this counter set and if this |
| 520 | * is the last used counter in the set, clear activation |
| 521 | * control and set the counter set state to inactive. |
| 522 | */ |
| 523 | if (!atomic_dec_return(&cpuhw->ctr_set[hwc->config_base])) |
| 524 | ctr_set_stop(&cpuhw->state, hwc->config_base); |
| 525 | event->hw.state |= PERF_HES_STOPPED; |
| 526 | } |
| 527 | |
| 528 | if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { |
| 529 | hw_perf_event_update(event); |
| 530 | event->hw.state |= PERF_HES_UPTODATE; |
| 531 | } |
| 532 | } |
| 533 | |
| 534 | static int cpumf_pmu_add(struct perf_event *event, int flags) |
| 535 | { |
Christoph Lameter | eb7e7d7 | 2014-08-17 12:30:45 -0500 | [diff] [blame] | 536 | struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 537 | |
| 538 | /* Check authorization for the counter set to which this |
| 539 | * counter belongs. |
| 540 | * For group events transaction, the authorization check is |
| 541 | * done in cpumf_pmu_commit_txn(). |
| 542 | */ |
Sukadev Bhattiprolu | 8f3e568 | 2015-09-03 20:07:53 -0700 | [diff] [blame] | 543 | if (!(cpuhw->txn_flags & PERF_PMU_TXN_ADD)) |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 544 | if (validate_ctr_auth(&event->hw)) |
Hendrik Brueckner | 58f8e9d | 2015-09-07 10:52:42 +0200 | [diff] [blame] | 545 | return -ENOENT; |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 546 | |
| 547 | ctr_set_enable(&cpuhw->state, event->hw.config_base); |
| 548 | event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; |
| 549 | |
| 550 | if (flags & PERF_EF_START) |
| 551 | cpumf_pmu_start(event, PERF_EF_RELOAD); |
| 552 | |
| 553 | perf_event_update_userpage(event); |
| 554 | |
| 555 | return 0; |
| 556 | } |
| 557 | |
| 558 | static void cpumf_pmu_del(struct perf_event *event, int flags) |
| 559 | { |
Christoph Lameter | eb7e7d7 | 2014-08-17 12:30:45 -0500 | [diff] [blame] | 560 | struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 561 | |
| 562 | cpumf_pmu_stop(event, PERF_EF_UPDATE); |
| 563 | |
| 564 | /* Check if any counter in the counter set is still used. If not used, |
| 565 | * change the counter set to the disabled state. This also clears the |
| 566 | * content of all counters in the set. |
| 567 | * |
| 568 | * When a new perf event has been added but not yet started, this can |
| 569 | * clear enable control and resets all counters in a set. Therefore, |
| 570 | * cpumf_pmu_start() always has to reenable a counter set. |
| 571 | */ |
| 572 | if (!atomic_read(&cpuhw->ctr_set[event->hw.config_base])) |
| 573 | ctr_set_disable(&cpuhw->state, event->hw.config_base); |
| 574 | |
| 575 | perf_event_update_userpage(event); |
| 576 | } |
| 577 | |
| 578 | /* |
| 579 | * Start group events scheduling transaction. |
| 580 | * Set flags to perform a single test at commit time. |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 581 | * |
| 582 | * We only support PERF_PMU_TXN_ADD transactions. Save the |
| 583 | * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD |
| 584 | * transactions. |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 585 | */ |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 586 | static void cpumf_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags) |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 587 | { |
Christoph Lameter | eb7e7d7 | 2014-08-17 12:30:45 -0500 | [diff] [blame] | 588 | struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 589 | |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 590 | WARN_ON_ONCE(cpuhw->txn_flags); /* txn already in flight */ |
| 591 | |
| 592 | cpuhw->txn_flags = txn_flags; |
| 593 | if (txn_flags & ~PERF_PMU_TXN_ADD) |
| 594 | return; |
| 595 | |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 596 | perf_pmu_disable(pmu); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 597 | cpuhw->tx_state = cpuhw->state; |
| 598 | } |
| 599 | |
| 600 | /* |
| 601 | * Stop and cancel a group events scheduling tranctions. |
| 602 | * Assumes cpumf_pmu_del() is called for each successful added |
| 603 | * cpumf_pmu_add() during the transaction. |
| 604 | */ |
| 605 | static void cpumf_pmu_cancel_txn(struct pmu *pmu) |
| 606 | { |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 607 | unsigned int txn_flags; |
Christoph Lameter | eb7e7d7 | 2014-08-17 12:30:45 -0500 | [diff] [blame] | 608 | struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 609 | |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 610 | WARN_ON_ONCE(!cpuhw->txn_flags); /* no txn in flight */ |
| 611 | |
| 612 | txn_flags = cpuhw->txn_flags; |
| 613 | cpuhw->txn_flags = 0; |
| 614 | if (txn_flags & ~PERF_PMU_TXN_ADD) |
| 615 | return; |
| 616 | |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 617 | WARN_ON(cpuhw->tx_state != cpuhw->state); |
| 618 | |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 619 | perf_pmu_enable(pmu); |
| 620 | } |
| 621 | |
| 622 | /* |
| 623 | * Commit the group events scheduling transaction. On success, the |
| 624 | * transaction is closed. On error, the transaction is kept open |
| 625 | * until cpumf_pmu_cancel_txn() is called. |
| 626 | */ |
| 627 | static int cpumf_pmu_commit_txn(struct pmu *pmu) |
| 628 | { |
Christoph Lameter | eb7e7d7 | 2014-08-17 12:30:45 -0500 | [diff] [blame] | 629 | struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 630 | u64 state; |
| 631 | |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 632 | WARN_ON_ONCE(!cpuhw->txn_flags); /* no txn in flight */ |
| 633 | |
| 634 | if (cpuhw->txn_flags & ~PERF_PMU_TXN_ADD) { |
| 635 | cpuhw->txn_flags = 0; |
| 636 | return 0; |
| 637 | } |
| 638 | |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 639 | /* check if the updated state can be scheduled */ |
| 640 | state = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1); |
| 641 | state >>= CPUMF_LCCTL_ENABLE_SHIFT; |
| 642 | if ((state & cpuhw->info.auth_ctl) != state) |
Hendrik Brueckner | 58f8e9d | 2015-09-07 10:52:42 +0200 | [diff] [blame] | 643 | return -ENOENT; |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 644 | |
Sukadev Bhattiprolu | fbbe070 | 2015-09-03 20:07:45 -0700 | [diff] [blame] | 645 | cpuhw->txn_flags = 0; |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 646 | perf_pmu_enable(pmu); |
| 647 | return 0; |
| 648 | } |
| 649 | |
| 650 | /* Performance monitoring unit for s390x */ |
| 651 | static struct pmu cpumf_pmu = { |
Hendrik Brueckner | 9254e70 | 2016-06-09 12:28:13 +0200 | [diff] [blame] | 652 | .task_ctx_nr = perf_sw_context, |
| 653 | .capabilities = PERF_PMU_CAP_NO_INTERRUPT, |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 654 | .pmu_enable = cpumf_pmu_enable, |
| 655 | .pmu_disable = cpumf_pmu_disable, |
| 656 | .event_init = cpumf_pmu_event_init, |
| 657 | .add = cpumf_pmu_add, |
| 658 | .del = cpumf_pmu_del, |
| 659 | .start = cpumf_pmu_start, |
| 660 | .stop = cpumf_pmu_stop, |
| 661 | .read = cpumf_pmu_read, |
| 662 | .start_txn = cpumf_pmu_start_txn, |
| 663 | .commit_txn = cpumf_pmu_commit_txn, |
| 664 | .cancel_txn = cpumf_pmu_cancel_txn, |
| 665 | }; |
| 666 | |
Thomas Gleixner | 4f0f821 | 2016-07-13 17:16:21 +0000 | [diff] [blame] | 667 | static int cpumf_pmf_setup(unsigned int cpu, int flags) |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 668 | { |
Thomas Gleixner | 4f0f821 | 2016-07-13 17:16:21 +0000 | [diff] [blame] | 669 | local_irq_disable(); |
| 670 | setup_pmc_cpu(&flags); |
| 671 | local_irq_enable(); |
| 672 | return 0; |
| 673 | } |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 674 | |
Thomas Gleixner | 4f0f821 | 2016-07-13 17:16:21 +0000 | [diff] [blame] | 675 | static int s390_pmu_online_cpu(unsigned int cpu) |
| 676 | { |
| 677 | return cpumf_pmf_setup(cpu, PMC_INIT); |
| 678 | } |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 679 | |
Thomas Gleixner | 4f0f821 | 2016-07-13 17:16:21 +0000 | [diff] [blame] | 680 | static int s390_pmu_offline_cpu(unsigned int cpu) |
| 681 | { |
| 682 | return cpumf_pmf_setup(cpu, PMC_RELEASE); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 683 | } |
| 684 | |
| 685 | static int __init cpumf_pmu_init(void) |
| 686 | { |
| 687 | int rc; |
| 688 | |
| 689 | if (!cpum_cf_avail()) |
| 690 | return -ENODEV; |
| 691 | |
| 692 | /* clear bit 15 of cr0 to unauthorize problem-state to |
| 693 | * extract measurement counters */ |
| 694 | ctl_clear_bit(0, 48); |
| 695 | |
| 696 | /* register handler for measurement-alert interruptions */ |
Thomas Huth | 1dad093 | 2014-03-31 15:24:08 +0200 | [diff] [blame] | 697 | rc = register_external_irq(EXT_IRQ_MEASURE_ALERT, |
| 698 | cpumf_measurement_alert); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 699 | if (rc) { |
| 700 | pr_err("Registering for CPU-measurement alerts " |
| 701 | "failed with rc=%i\n", rc); |
Thomas Gleixner | 4f0f821 | 2016-07-13 17:16:21 +0000 | [diff] [blame] | 702 | return rc; |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 703 | } |
| 704 | |
Hendrik Brueckner | c716832 | 2013-12-11 12:44:40 +0100 | [diff] [blame] | 705 | cpumf_pmu.attr_groups = cpumf_cf_event_group(); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 706 | rc = perf_pmu_register(&cpumf_pmu, "cpum_cf", PERF_TYPE_RAW); |
| 707 | if (rc) { |
| 708 | pr_err("Registering the cpum_cf PMU failed with rc=%i\n", rc); |
Thomas Huth | 1dad093 | 2014-03-31 15:24:08 +0200 | [diff] [blame] | 709 | unregister_external_irq(EXT_IRQ_MEASURE_ALERT, |
| 710 | cpumf_measurement_alert); |
Thomas Gleixner | 4f0f821 | 2016-07-13 17:16:21 +0000 | [diff] [blame] | 711 | return rc; |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 712 | } |
Thomas Gleixner | 4f0f821 | 2016-07-13 17:16:21 +0000 | [diff] [blame] | 713 | return cpuhp_setup_state(CPUHP_AP_PERF_S390_CF_ONLINE, |
| 714 | "AP_PERF_S390_CF_ONLINE", |
| 715 | s390_pmu_online_cpu, s390_pmu_offline_cpu); |
Hendrik Brueckner | 212188a | 2012-03-23 11:13:06 +0100 | [diff] [blame] | 716 | } |
| 717 | early_initcall(cpumf_pmu_init); |