blob: c839713c37cac7fb75d1dcbd3139fb93c9f66d3c [file] [log] [blame]
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +02001/*
2 * CPUFreq governor based on scheduler-provided CPU utilization data.
3 *
4 * Copyright (C) 2016, Intel Corporation
5 * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
Viresh Kumar60f05e82016-05-18 17:55:28 +053012#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
13
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +020014#include <linux/cpufreq.h>
Viresh Kumar02a7b1e2016-11-15 13:53:22 +053015#include <linux/kthread.h>
Ingo Molnarae7e81c2017-02-01 18:07:51 +010016#include <uapi/linux/sched/types.h>
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +020017#include <linux/slab.h>
18#include <trace/events/power.h>
19
20#include "sched.h"
21
Patrick Bellasi07d1bac2019-02-15 11:49:36 +000022unsigned long boosted_cpu_util(int cpu);
Patrick Bellasi159c14f2017-10-21 18:07:35 +010023
Viresh Kumar02a7b1e2016-11-15 13:53:22 +053024#define SUGOV_KTHREAD_PRIORITY 50
25
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +020026struct sugov_tunables {
27 struct gov_attr_set attr_set;
Steve Muckle50c26fd2016-11-17 10:48:45 +053028 unsigned int up_rate_limit_us;
29 unsigned int down_rate_limit_us;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +020030};
31
32struct sugov_policy {
33 struct cpufreq_policy *policy;
34
35 struct sugov_tunables *tunables;
36 struct list_head tunables_hook;
37
38 raw_spinlock_t update_lock; /* For shared policies */
39 u64 last_freq_update_time;
Steve Muckle50c26fd2016-11-17 10:48:45 +053040 s64 min_rate_limit_ns;
41 s64 up_rate_delay_ns;
42 s64 down_rate_delay_ns;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +020043 unsigned int next_freq;
Viresh Kumar6c4f0fa2017-03-02 14:03:20 +053044 unsigned int cached_raw_freq;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +020045
46 /* The next fields are only needed if fast switch cannot be used. */
47 struct irq_work irq_work;
Viresh Kumar02a7b1e2016-11-15 13:53:22 +053048 struct kthread_work work;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +020049 struct mutex work_lock;
Viresh Kumar02a7b1e2016-11-15 13:53:22 +053050 struct kthread_worker worker;
51 struct task_struct *thread;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +020052 bool work_in_progress;
53
54 bool need_freq_update;
55};
56
57struct sugov_cpu {
58 struct update_util_data update_util;
59 struct sugov_policy *sg_policy;
Viresh Kumar674e7542017-07-28 12:16:38 +053060 unsigned int cpu;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +020061
Joel Fernandesa5a08092017-07-23 08:54:25 -070062 bool iowait_boost_pending;
Joel Fernandes251accf2017-07-23 08:54:26 -070063 unsigned int iowait_boost;
64 unsigned int iowait_boost_max;
Rafael J. Wysocki21ca6d22016-09-10 00:00:31 +020065 u64 last_update;
Steve Muckle5cbea462016-07-13 13:25:26 -070066
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +020067 /* The fields below are only needed when sharing a policy. */
68 unsigned long util;
69 unsigned long max;
Rafael J. Wysocki58919e82016-08-16 22:14:55 +020070 unsigned int flags;
Rafael J. Wysockib7eaf1a2017-03-22 00:08:50 +010071
72 /* The field below is for single-CPU policies only. */
73#ifdef CONFIG_NO_HZ_COMMON
74 unsigned long saved_idle_calls;
75#endif
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +020076};
77
78static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu);
79
80/************************ Governor internals ***********************/
81
82static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time)
83{
84 s64 delta_ns;
85
Viresh Kumar674e7542017-07-28 12:16:38 +053086 /*
87 * Since cpufreq_update_util() is called with rq->lock held for
88 * the @target_cpu, our per-cpu data is fully serialized.
89 *
90 * However, drivers cannot in general deal with cross-cpu
91 * requests, so while get_next_freq() will work, our
Viresh Kumarc49cbc12017-08-14 14:50:16 +053092 * sugov_update_commit() call may not for the fast switching platforms.
Viresh Kumar674e7542017-07-28 12:16:38 +053093 *
94 * Hence stop here for remote requests if they aren't supported
95 * by the hardware, as calculating the frequency is pointless if
96 * we cannot in fact act on it.
Viresh Kumarc49cbc12017-08-14 14:50:16 +053097 *
98 * For the slow switching platforms, the kthread is always scheduled on
99 * the right set of CPUs and any CPU can find the next frequency and
100 * schedule the kthread.
Viresh Kumar674e7542017-07-28 12:16:38 +0530101 */
Viresh Kumarc49cbc12017-08-14 14:50:16 +0530102 if (sg_policy->policy->fast_switch_enabled &&
103 !cpufreq_can_do_remote_dvfs(sg_policy->policy))
Viresh Kumar674e7542017-07-28 12:16:38 +0530104 return false;
105
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200106 if (sg_policy->work_in_progress)
107 return false;
108
109 if (unlikely(sg_policy->need_freq_update)) {
110 sg_policy->need_freq_update = false;
111 /*
112 * This happens when limits change, so forget the previous
113 * next_freq value and force an update.
114 */
115 sg_policy->next_freq = UINT_MAX;
116 return true;
117 }
118
Steve Muckle50c26fd2016-11-17 10:48:45 +0530119 /* No need to recalculate next freq for min_rate_limit_us
120 * at least. However we might still decide to further rate
121 * limit once frequency change direction is decided, according
122 * to the separate rate limits.
123 */
124
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200125 delta_ns = time - sg_policy->last_freq_update_time;
Steve Muckle50c26fd2016-11-17 10:48:45 +0530126 return delta_ns >= sg_policy->min_rate_limit_ns;
127}
128
129static bool sugov_up_down_rate_limit(struct sugov_policy *sg_policy, u64 time,
130 unsigned int next_freq)
131{
132 s64 delta_ns;
133
134 delta_ns = time - sg_policy->last_freq_update_time;
135
136 if (next_freq > sg_policy->next_freq &&
137 delta_ns < sg_policy->up_rate_delay_ns)
138 return true;
139
140 if (next_freq < sg_policy->next_freq &&
141 delta_ns < sg_policy->down_rate_delay_ns)
142 return true;
143
144 return false;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200145}
146
147static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time,
148 unsigned int next_freq)
149{
150 struct cpufreq_policy *policy = sg_policy->policy;
151
Rafael J. Wysocki38d4ea22017-03-22 18:32:47 +0100152 if (sg_policy->next_freq == next_freq)
153 return;
154
Steve Muckle50c26fd2016-11-17 10:48:45 +0530155 if (sugov_up_down_rate_limit(sg_policy, time, next_freq))
156 return;
157
Rafael J. Wysocki38d4ea22017-03-22 18:32:47 +0100158 sg_policy->next_freq = next_freq;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200159 sg_policy->last_freq_update_time = time;
160
161 if (policy->fast_switch_enabled) {
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200162 next_freq = cpufreq_driver_fast_switch(policy, next_freq);
Viresh Kumar209887e2017-08-09 10:21:46 +0530163 if (!next_freq)
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200164 return;
165
166 policy->cur = next_freq;
167 trace_cpu_frequency(next_freq, smp_processor_id());
Rafael J. Wysocki38d4ea22017-03-22 18:32:47 +0100168 } else {
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200169 sg_policy->work_in_progress = true;
170 irq_work_queue(&sg_policy->irq_work);
171 }
172}
173
174/**
175 * get_next_freq - Compute a new frequency for a given cpufreq policy.
Viresh Kumar655cb1e2017-03-02 14:03:21 +0530176 * @sg_policy: schedutil policy object to compute the new frequency for.
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200177 * @util: Current CPU utilization.
178 * @max: CPU capacity.
179 *
180 * If the utilization is frequency-invariant, choose the new frequency to be
181 * proportional to it, that is
182 *
183 * next_freq = C * max_freq * util / max
184 *
185 * Otherwise, approximate the would-be frequency-invariant utilization by
186 * util_raw * (curr_freq / max_freq) which leads to
187 *
188 * next_freq = C * curr_freq * util_raw / max
189 *
190 * Take C = 1.25 for the frequency tipping point at (util / max) = 0.8.
Steve Muckle5cbea462016-07-13 13:25:26 -0700191 *
192 * The lowest driver-supported frequency which is equal or greater than the raw
193 * next_freq (as calculated above) is returned, subject to policy min/max and
194 * cpufreq driver limitations.
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200195 */
Viresh Kumar655cb1e2017-03-02 14:03:21 +0530196static unsigned int get_next_freq(struct sugov_policy *sg_policy,
197 unsigned long util, unsigned long max)
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200198{
Steve Muckle5cbea462016-07-13 13:25:26 -0700199 struct cpufreq_policy *policy = sg_policy->policy;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200200 unsigned int freq = arch_scale_freq_invariant() ?
201 policy->cpuinfo.max_freq : policy->cur;
202
Steve Muckle5cbea462016-07-13 13:25:26 -0700203 freq = (freq + (freq >> 2)) * util / max;
204
Viresh Kumar6c4f0fa2017-03-02 14:03:20 +0530205 if (freq == sg_policy->cached_raw_freq && sg_policy->next_freq != UINT_MAX)
Steve Muckle5cbea462016-07-13 13:25:26 -0700206 return sg_policy->next_freq;
Viresh Kumar6c4f0fa2017-03-02 14:03:20 +0530207 sg_policy->cached_raw_freq = freq;
Steve Muckle5cbea462016-07-13 13:25:26 -0700208 return cpufreq_driver_resolve_freq(policy, freq);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200209}
210
Viresh Kumar674e7542017-07-28 12:16:38 +0530211static void sugov_get_util(unsigned long *util, unsigned long *max, int cpu)
Rafael J. Wysocki58919e82016-08-16 22:14:55 +0200212{
Patrick Bellasi07d1bac2019-02-15 11:49:36 +0000213 unsigned long max_cap;
Steve Muckle8314bc82016-08-26 11:40:47 -0700214
Steve Mucklef1f71252016-08-25 15:59:17 -0700215 max_cap = arch_scale_cpu_capacity(NULL, cpu);
Rafael J. Wysocki58919e82016-08-16 22:14:55 +0200216
Patrick Bellasi07d1bac2019-02-15 11:49:36 +0000217 *util = boosted_cpu_util(cpu);
Patrick Bellasi159c14f2017-10-21 18:07:35 +0100218 *util = min(*util, max_cap);
Steve Mucklef1f71252016-08-25 15:59:17 -0700219 *max = max_cap;
Rafael J. Wysocki58919e82016-08-16 22:14:55 +0200220}
221
Rafael J. Wysocki21ca6d22016-09-10 00:00:31 +0200222static void sugov_set_iowait_boost(struct sugov_cpu *sg_cpu, u64 time,
223 unsigned int flags)
224{
225 if (flags & SCHED_CPUFREQ_IOWAIT) {
Joel Fernandesa5a08092017-07-23 08:54:25 -0700226 if (sg_cpu->iowait_boost_pending)
227 return;
228
229 sg_cpu->iowait_boost_pending = true;
230
231 if (sg_cpu->iowait_boost) {
232 sg_cpu->iowait_boost <<= 1;
233 if (sg_cpu->iowait_boost > sg_cpu->iowait_boost_max)
234 sg_cpu->iowait_boost = sg_cpu->iowait_boost_max;
235 } else {
236 sg_cpu->iowait_boost = sg_cpu->sg_policy->policy->min;
237 }
Rafael J. Wysocki21ca6d22016-09-10 00:00:31 +0200238 } else if (sg_cpu->iowait_boost) {
239 s64 delta_ns = time - sg_cpu->last_update;
240
241 /* Clear iowait_boost if the CPU apprears to have been idle. */
Joel Fernandesa5a08092017-07-23 08:54:25 -0700242 if (delta_ns > TICK_NSEC) {
Rafael J. Wysocki21ca6d22016-09-10 00:00:31 +0200243 sg_cpu->iowait_boost = 0;
Joel Fernandesa5a08092017-07-23 08:54:25 -0700244 sg_cpu->iowait_boost_pending = false;
245 }
Rafael J. Wysocki21ca6d22016-09-10 00:00:31 +0200246 }
247}
248
249static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, unsigned long *util,
250 unsigned long *max)
251{
Joel Fernandes251accf2017-07-23 08:54:26 -0700252 unsigned int boost_util, boost_max;
Rafael J. Wysocki21ca6d22016-09-10 00:00:31 +0200253
Joel Fernandesa5a08092017-07-23 08:54:25 -0700254 if (!sg_cpu->iowait_boost)
Rafael J. Wysocki21ca6d22016-09-10 00:00:31 +0200255 return;
256
Joel Fernandesa5a08092017-07-23 08:54:25 -0700257 if (sg_cpu->iowait_boost_pending) {
258 sg_cpu->iowait_boost_pending = false;
259 } else {
260 sg_cpu->iowait_boost >>= 1;
261 if (sg_cpu->iowait_boost < sg_cpu->sg_policy->policy->min) {
262 sg_cpu->iowait_boost = 0;
263 return;
264 }
265 }
266
267 boost_util = sg_cpu->iowait_boost;
268 boost_max = sg_cpu->iowait_boost_max;
269
Rafael J. Wysocki21ca6d22016-09-10 00:00:31 +0200270 if (*util * boost_max < *max * boost_util) {
271 *util = boost_util;
272 *max = boost_max;
273 }
Rafael J. Wysocki21ca6d22016-09-10 00:00:31 +0200274}
275
Rafael J. Wysockib7eaf1a2017-03-22 00:08:50 +0100276#ifdef CONFIG_NO_HZ_COMMON
277static bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu)
278{
Joel Fernandes0c688c22017-12-21 02:22:45 +0100279 unsigned long idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu);
Rafael J. Wysockib7eaf1a2017-03-22 00:08:50 +0100280 bool ret = idle_calls == sg_cpu->saved_idle_calls;
281
282 sg_cpu->saved_idle_calls = idle_calls;
283 return ret;
284}
285#else
286static inline bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu) { return false; }
287#endif /* CONFIG_NO_HZ_COMMON */
288
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200289static void sugov_update_single(struct update_util_data *hook, u64 time,
Rafael J. Wysocki58919e82016-08-16 22:14:55 +0200290 unsigned int flags)
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200291{
292 struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
293 struct sugov_policy *sg_policy = sg_cpu->sg_policy;
294 struct cpufreq_policy *policy = sg_policy->policy;
Rafael J. Wysocki58919e82016-08-16 22:14:55 +0200295 unsigned long util, max;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200296 unsigned int next_f;
Rafael J. Wysockib7eaf1a2017-03-22 00:08:50 +0100297 bool busy;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200298
Rafael J. Wysocki21ca6d22016-09-10 00:00:31 +0200299 sugov_set_iowait_boost(sg_cpu, time, flags);
300 sg_cpu->last_update = time;
301
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200302 if (!sugov_should_update_freq(sg_policy, time))
303 return;
304
Rafael J. Wysockib7eaf1a2017-03-22 00:08:50 +0100305 busy = sugov_cpu_is_busy(sg_cpu);
306
Steve Mucklef1f71252016-08-25 15:59:17 -0700307 if (flags & SCHED_CPUFREQ_DL) {
Rafael J. Wysocki58919e82016-08-16 22:14:55 +0200308 next_f = policy->cpuinfo.max_freq;
309 } else {
Viresh Kumar674e7542017-07-28 12:16:38 +0530310 sugov_get_util(&util, &max, sg_cpu->cpu);
Rafael J. Wysocki21ca6d22016-09-10 00:00:31 +0200311 sugov_iowait_boost(sg_cpu, &util, &max);
Viresh Kumar655cb1e2017-03-02 14:03:21 +0530312 next_f = get_next_freq(sg_policy, util, max);
Rafael J. Wysockib7eaf1a2017-03-22 00:08:50 +0100313 /*
314 * Do not reduce the frequency if the CPU has not been idle
315 * recently, as the reduction is likely to be premature then.
316 */
Rafael J. Wysocki99e9acc2018-05-09 11:44:56 +0200317 if (busy && next_f < sg_policy->next_freq &&
318 sg_policy->next_freq != UINT_MAX) {
Rafael J. Wysockib7eaf1a2017-03-22 00:08:50 +0100319 next_f = sg_policy->next_freq;
Viresh Kumarb7997492017-11-08 20:23:55 +0530320
321 /* Reset cached freq as next_freq has changed */
322 sg_policy->cached_raw_freq = 0;
323 }
Rafael J. Wysocki58919e82016-08-16 22:14:55 +0200324 }
Steve Muckle50c26fd2016-11-17 10:48:45 +0530325
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200326 sugov_update_commit(sg_policy, time, next_f);
327}
328
Juri Lellid86ab9c2017-05-03 14:30:48 +0100329static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time)
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200330{
Steve Muckle5cbea462016-07-13 13:25:26 -0700331 struct sugov_policy *sg_policy = sg_cpu->sg_policy;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200332 struct cpufreq_policy *policy = sg_policy->policy;
Viresh Kumarcba1dfb2017-03-09 09:34:54 +0530333 unsigned long util = 0, max = 1;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200334 unsigned int j;
335
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200336 for_each_cpu(j, policy->cpus) {
Viresh Kumarcba1dfb2017-03-09 09:34:54 +0530337 struct sugov_cpu *j_sg_cpu = &per_cpu(sugov_cpu, j);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200338 unsigned long j_util, j_max;
339 s64 delta_ns;
340
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200341 /*
342 * If the CPU utilization was last updated before the previous
343 * frequency update and the time elapsed between the last update
344 * of the CPU utilization and the last frequency update is long
345 * enough, don't take the CPU into account as it probably is
Rafael J. Wysocki21ca6d22016-09-10 00:00:31 +0200346 * idle now (and clear iowait_boost for it).
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200347 */
Juri Lellid86ab9c2017-05-03 14:30:48 +0100348 delta_ns = time - j_sg_cpu->last_update;
Rafael J. Wysocki21ca6d22016-09-10 00:00:31 +0200349 if (delta_ns > TICK_NSEC) {
350 j_sg_cpu->iowait_boost = 0;
Joel Fernandesa5a08092017-07-23 08:54:25 -0700351 j_sg_cpu->iowait_boost_pending = false;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200352 continue;
Rafael J. Wysocki21ca6d22016-09-10 00:00:31 +0200353 }
Steve Mucklef1f71252016-08-25 15:59:17 -0700354 if (j_sg_cpu->flags & SCHED_CPUFREQ_DL)
Viresh Kumarcba1dfb2017-03-09 09:34:54 +0530355 return policy->cpuinfo.max_freq;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200356
Rafael J. Wysocki58919e82016-08-16 22:14:55 +0200357 j_util = j_sg_cpu->util;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200358 j_max = j_sg_cpu->max;
359 if (j_util * max > j_max * util) {
360 util = j_util;
361 max = j_max;
362 }
Rafael J. Wysocki21ca6d22016-09-10 00:00:31 +0200363
364 sugov_iowait_boost(j_sg_cpu, &util, &max);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200365 }
366
Viresh Kumar655cb1e2017-03-02 14:03:21 +0530367 return get_next_freq(sg_policy, util, max);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200368}
369
370static void sugov_update_shared(struct update_util_data *hook, u64 time,
Rafael J. Wysocki58919e82016-08-16 22:14:55 +0200371 unsigned int flags)
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200372{
373 struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
374 struct sugov_policy *sg_policy = sg_cpu->sg_policy;
Rafael J. Wysocki58919e82016-08-16 22:14:55 +0200375 unsigned long util, max;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200376 unsigned int next_f;
377
Viresh Kumar674e7542017-07-28 12:16:38 +0530378 sugov_get_util(&util, &max, sg_cpu->cpu);
Rafael J. Wysocki58919e82016-08-16 22:14:55 +0200379
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200380 raw_spin_lock(&sg_policy->update_lock);
381
382 sg_cpu->util = util;
383 sg_cpu->max = max;
Rafael J. Wysocki58919e82016-08-16 22:14:55 +0200384 sg_cpu->flags = flags;
Rafael J. Wysocki21ca6d22016-09-10 00:00:31 +0200385
386 sugov_set_iowait_boost(sg_cpu, time, flags);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200387 sg_cpu->last_update = time;
388
389 if (sugov_should_update_freq(sg_policy, time)) {
Steve Mucklef1f71252016-08-25 15:59:17 -0700390 if (flags & SCHED_CPUFREQ_DL)
Viresh Kumarcba1dfb2017-03-09 09:34:54 +0530391 next_f = sg_policy->policy->cpuinfo.max_freq;
392 else
Juri Lellid86ab9c2017-05-03 14:30:48 +0100393 next_f = sugov_next_freq_shared(sg_cpu, time);
Viresh Kumarcba1dfb2017-03-09 09:34:54 +0530394
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200395 sugov_update_commit(sg_policy, time, next_f);
396 }
397
398 raw_spin_unlock(&sg_policy->update_lock);
399}
400
Viresh Kumar02a7b1e2016-11-15 13:53:22 +0530401static void sugov_work(struct kthread_work *work)
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200402{
403 struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work);
404
405 mutex_lock(&sg_policy->work_lock);
406 __cpufreq_driver_target(sg_policy->policy, sg_policy->next_freq,
407 CPUFREQ_RELATION_L);
408 mutex_unlock(&sg_policy->work_lock);
409
410 sg_policy->work_in_progress = false;
411}
412
413static void sugov_irq_work(struct irq_work *irq_work)
414{
415 struct sugov_policy *sg_policy;
416
417 sg_policy = container_of(irq_work, struct sugov_policy, irq_work);
Viresh Kumar02a7b1e2016-11-15 13:53:22 +0530418
419 /*
Viresh Kumard06e6222016-11-24 13:51:11 +0530420 * For RT and deadline tasks, the schedutil governor shoots the
421 * frequency to maximum. Special care must be taken to ensure that this
422 * kthread doesn't result in the same behavior.
Viresh Kumar02a7b1e2016-11-15 13:53:22 +0530423 *
424 * This is (mostly) guaranteed by the work_in_progress flag. The flag is
Viresh Kumard06e6222016-11-24 13:51:11 +0530425 * updated only at the end of the sugov_work() function and before that
426 * the schedutil governor rejects all other frequency scaling requests.
Viresh Kumar02a7b1e2016-11-15 13:53:22 +0530427 *
Viresh Kumard06e6222016-11-24 13:51:11 +0530428 * There is a very rare case though, where the RT thread yields right
Viresh Kumar02a7b1e2016-11-15 13:53:22 +0530429 * after the work_in_progress flag is cleared. The effects of that are
430 * neglected for now.
431 */
432 kthread_queue_work(&sg_policy->worker, &sg_policy->work);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200433}
434
435/************************** sysfs interface ************************/
436
437static struct sugov_tunables *global_tunables;
438static DEFINE_MUTEX(global_tunables_lock);
439
440static inline struct sugov_tunables *to_sugov_tunables(struct gov_attr_set *attr_set)
441{
442 return container_of(attr_set, struct sugov_tunables, attr_set);
443}
444
Steve Muckle50c26fd2016-11-17 10:48:45 +0530445static DEFINE_MUTEX(min_rate_lock);
446
447static void update_min_rate_limit_ns(struct sugov_policy *sg_policy)
448{
449 mutex_lock(&min_rate_lock);
450 sg_policy->min_rate_limit_ns = min(sg_policy->up_rate_delay_ns,
451 sg_policy->down_rate_delay_ns);
452 mutex_unlock(&min_rate_lock);
453}
454
455static ssize_t up_rate_limit_us_show(struct gov_attr_set *attr_set, char *buf)
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200456{
457 struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
458
Steve Muckle50c26fd2016-11-17 10:48:45 +0530459 return sprintf(buf, "%u\n", tunables->up_rate_limit_us);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200460}
461
Steve Muckle50c26fd2016-11-17 10:48:45 +0530462static ssize_t down_rate_limit_us_show(struct gov_attr_set *attr_set, char *buf)
463{
464 struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
465
466 return sprintf(buf, "%u\n", tunables->down_rate_limit_us);
467}
468
469static ssize_t up_rate_limit_us_store(struct gov_attr_set *attr_set,
470 const char *buf, size_t count)
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200471{
472 struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
473 struct sugov_policy *sg_policy;
474 unsigned int rate_limit_us;
475
476 if (kstrtouint(buf, 10, &rate_limit_us))
477 return -EINVAL;
478
Steve Muckle50c26fd2016-11-17 10:48:45 +0530479 tunables->up_rate_limit_us = rate_limit_us;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200480
Steve Muckle50c26fd2016-11-17 10:48:45 +0530481 list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook) {
482 sg_policy->up_rate_delay_ns = rate_limit_us * NSEC_PER_USEC;
483 update_min_rate_limit_ns(sg_policy);
484 }
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200485
486 return count;
487}
488
Steve Muckle50c26fd2016-11-17 10:48:45 +0530489static ssize_t down_rate_limit_us_store(struct gov_attr_set *attr_set,
490 const char *buf, size_t count)
491{
492 struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
493 struct sugov_policy *sg_policy;
494 unsigned int rate_limit_us;
495
496 if (kstrtouint(buf, 10, &rate_limit_us))
497 return -EINVAL;
498
499 tunables->down_rate_limit_us = rate_limit_us;
500
501 list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook) {
502 sg_policy->down_rate_delay_ns = rate_limit_us * NSEC_PER_USEC;
503 update_min_rate_limit_ns(sg_policy);
504 }
505
506 return count;
507}
508
509static struct governor_attr up_rate_limit_us = __ATTR_RW(up_rate_limit_us);
510static struct governor_attr down_rate_limit_us = __ATTR_RW(down_rate_limit_us);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200511
512static struct attribute *sugov_attributes[] = {
Steve Muckle50c26fd2016-11-17 10:48:45 +0530513 &up_rate_limit_us.attr,
514 &down_rate_limit_us.attr,
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200515 NULL
516};
517
518static struct kobj_type sugov_tunables_ktype = {
519 .default_attrs = sugov_attributes,
520 .sysfs_ops = &governor_sysfs_ops,
521};
522
523/********************** cpufreq governor interface *********************/
524
525static struct cpufreq_governor schedutil_gov;
526
527static struct sugov_policy *sugov_policy_alloc(struct cpufreq_policy *policy)
528{
529 struct sugov_policy *sg_policy;
530
531 sg_policy = kzalloc(sizeof(*sg_policy), GFP_KERNEL);
532 if (!sg_policy)
533 return NULL;
534
535 sg_policy->policy = policy;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200536 raw_spin_lock_init(&sg_policy->update_lock);
537 return sg_policy;
538}
539
540static void sugov_policy_free(struct sugov_policy *sg_policy)
541{
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200542 kfree(sg_policy);
543}
544
Viresh Kumar02a7b1e2016-11-15 13:53:22 +0530545static int sugov_kthread_create(struct sugov_policy *sg_policy)
546{
547 struct task_struct *thread;
548 struct sched_param param = { .sched_priority = MAX_USER_RT_PRIO / 2 };
549 struct cpufreq_policy *policy = sg_policy->policy;
550 int ret;
551
552 /* kthread only required for slow path */
553 if (policy->fast_switch_enabled)
554 return 0;
555
556 kthread_init_work(&sg_policy->work, sugov_work);
557 kthread_init_worker(&sg_policy->worker);
558 thread = kthread_create(kthread_worker_fn, &sg_policy->worker,
559 "sugov:%d",
560 cpumask_first(policy->related_cpus));
561 if (IS_ERR(thread)) {
562 pr_err("failed to create sugov thread: %ld\n", PTR_ERR(thread));
563 return PTR_ERR(thread);
564 }
565
566 ret = sched_setscheduler_nocheck(thread, SCHED_FIFO, &param);
567 if (ret) {
568 kthread_stop(thread);
569 pr_warn("%s: failed to set SCHED_FIFO\n", __func__);
570 return ret;
571 }
572
573 sg_policy->thread = thread;
Viresh Kumare2cabe42017-08-10 09:50:55 +0530574
575 /* Kthread is bound to all CPUs by default */
576 if (!policy->dvfs_possible_from_any_cpu)
577 kthread_bind_mask(thread, policy->related_cpus);
578
Viresh Kumar21ef5722016-11-15 13:53:23 +0530579 init_irq_work(&sg_policy->irq_work, sugov_irq_work);
580 mutex_init(&sg_policy->work_lock);
581
Viresh Kumar02a7b1e2016-11-15 13:53:22 +0530582 wake_up_process(thread);
583
584 return 0;
585}
586
587static void sugov_kthread_stop(struct sugov_policy *sg_policy)
588{
589 /* kthread only required for slow path */
590 if (sg_policy->policy->fast_switch_enabled)
591 return;
592
593 kthread_flush_worker(&sg_policy->worker);
594 kthread_stop(sg_policy->thread);
Viresh Kumar21ef5722016-11-15 13:53:23 +0530595 mutex_destroy(&sg_policy->work_lock);
Viresh Kumar02a7b1e2016-11-15 13:53:22 +0530596}
597
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200598static struct sugov_tunables *sugov_tunables_alloc(struct sugov_policy *sg_policy)
599{
600 struct sugov_tunables *tunables;
601
602 tunables = kzalloc(sizeof(*tunables), GFP_KERNEL);
603 if (tunables) {
604 gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook);
605 if (!have_governor_per_policy())
606 global_tunables = tunables;
607 }
608 return tunables;
609}
610
611static void sugov_tunables_free(struct sugov_tunables *tunables)
612{
613 if (!have_governor_per_policy())
614 global_tunables = NULL;
615
616 kfree(tunables);
617}
618
619static int sugov_init(struct cpufreq_policy *policy)
620{
621 struct sugov_policy *sg_policy;
622 struct sugov_tunables *tunables;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200623 int ret = 0;
624
625 /* State should be equivalent to EXIT */
626 if (policy->governor_data)
627 return -EBUSY;
628
Viresh Kumar4a71ce42016-11-15 13:53:21 +0530629 cpufreq_enable_fast_switch(policy);
630
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200631 sg_policy = sugov_policy_alloc(policy);
Viresh Kumar4a71ce42016-11-15 13:53:21 +0530632 if (!sg_policy) {
633 ret = -ENOMEM;
634 goto disable_fast_switch;
635 }
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200636
Viresh Kumar02a7b1e2016-11-15 13:53:22 +0530637 ret = sugov_kthread_create(sg_policy);
638 if (ret)
639 goto free_sg_policy;
640
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200641 mutex_lock(&global_tunables_lock);
642
643 if (global_tunables) {
644 if (WARN_ON(have_governor_per_policy())) {
645 ret = -EINVAL;
Viresh Kumar02a7b1e2016-11-15 13:53:22 +0530646 goto stop_kthread;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200647 }
648 policy->governor_data = sg_policy;
649 sg_policy->tunables = global_tunables;
650
651 gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook);
652 goto out;
653 }
654
655 tunables = sugov_tunables_alloc(sg_policy);
656 if (!tunables) {
657 ret = -ENOMEM;
Viresh Kumar02a7b1e2016-11-15 13:53:22 +0530658 goto stop_kthread;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200659 }
660
Steve Muckle50c26fd2016-11-17 10:48:45 +0530661 tunables->up_rate_limit_us = cpufreq_policy_transition_delay_us(policy);
662 tunables->down_rate_limit_us = cpufreq_policy_transition_delay_us(policy);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200663
664 policy->governor_data = sg_policy;
665 sg_policy->tunables = tunables;
666
667 ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype,
668 get_governor_parent_kobj(policy), "%s",
669 schedutil_gov.name);
670 if (ret)
671 goto fail;
672
Viresh Kumar8e2ddb02016-11-15 13:53:20 +0530673out:
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200674 mutex_unlock(&global_tunables_lock);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200675 return 0;
676
Viresh Kumar8e2ddb02016-11-15 13:53:20 +0530677fail:
Tobin C. Hardingabda2932019-04-30 10:11:44 +1000678 kobject_put(&tunables->attr_set.kobj);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200679 policy->governor_data = NULL;
680 sugov_tunables_free(tunables);
681
Viresh Kumar02a7b1e2016-11-15 13:53:22 +0530682stop_kthread:
683 sugov_kthread_stop(sg_policy);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200684 mutex_unlock(&global_tunables_lock);
685
Jules Maselbas423f2c92018-03-29 15:43:01 +0100686free_sg_policy:
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200687 sugov_policy_free(sg_policy);
Viresh Kumar4a71ce42016-11-15 13:53:21 +0530688
689disable_fast_switch:
690 cpufreq_disable_fast_switch(policy);
691
Viresh Kumar60f05e82016-05-18 17:55:28 +0530692 pr_err("initialization failed (error %d)\n", ret);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200693 return ret;
694}
695
Rafael J. Wysockie7888922016-06-02 23:24:15 +0200696static void sugov_exit(struct cpufreq_policy *policy)
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200697{
698 struct sugov_policy *sg_policy = policy->governor_data;
699 struct sugov_tunables *tunables = sg_policy->tunables;
700 unsigned int count;
701
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200702 mutex_lock(&global_tunables_lock);
703
704 count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook);
705 policy->governor_data = NULL;
706 if (!count)
707 sugov_tunables_free(tunables);
708
709 mutex_unlock(&global_tunables_lock);
710
Viresh Kumar02a7b1e2016-11-15 13:53:22 +0530711 sugov_kthread_stop(sg_policy);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200712 sugov_policy_free(sg_policy);
Viresh Kumar4a71ce42016-11-15 13:53:21 +0530713 cpufreq_disable_fast_switch(policy);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200714}
715
716static int sugov_start(struct cpufreq_policy *policy)
717{
718 struct sugov_policy *sg_policy = policy->governor_data;
719 unsigned int cpu;
720
Steve Muckle50c26fd2016-11-17 10:48:45 +0530721 sg_policy->up_rate_delay_ns =
722 sg_policy->tunables->up_rate_limit_us * NSEC_PER_USEC;
723 sg_policy->down_rate_delay_ns =
724 sg_policy->tunables->down_rate_limit_us * NSEC_PER_USEC;
725 update_min_rate_limit_ns(sg_policy);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200726 sg_policy->last_freq_update_time = 0;
727 sg_policy->next_freq = UINT_MAX;
728 sg_policy->work_in_progress = false;
729 sg_policy->need_freq_update = false;
Viresh Kumar6c4f0fa2017-03-02 14:03:20 +0530730 sg_policy->cached_raw_freq = 0;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200731
732 for_each_cpu(cpu, policy->cpus) {
733 struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);
734
Rafael J. Wysocki4296f232017-03-19 14:30:02 +0100735 memset(sg_cpu, 0, sizeof(*sg_cpu));
Chris Redpathd62d8132017-11-03 13:36:42 +0000736 sg_cpu->cpu = cpu;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200737 sg_cpu->sg_policy = sg_policy;
Steve Mucklef1f71252016-08-25 15:59:17 -0700738 sg_cpu->flags = SCHED_CPUFREQ_DL;
Rafael J. Wysocki4296f232017-03-19 14:30:02 +0100739 sg_cpu->iowait_boost_max = policy->cpuinfo.max_freq;
Vikram Mulukutlaab2f7cf2017-07-06 10:53:20 -0700740 }
741
742 for_each_cpu(cpu, policy->cpus) {
743 struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);
744
Rafael J. Wysocki4296f232017-03-19 14:30:02 +0100745 cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util,
746 policy_is_shared(policy) ?
747 sugov_update_shared :
748 sugov_update_single);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200749 }
750 return 0;
751}
752
Rafael J. Wysockie7888922016-06-02 23:24:15 +0200753static void sugov_stop(struct cpufreq_policy *policy)
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200754{
755 struct sugov_policy *sg_policy = policy->governor_data;
756 unsigned int cpu;
757
758 for_each_cpu(cpu, policy->cpus)
759 cpufreq_remove_update_util_hook(cpu);
760
761 synchronize_sched();
762
Viresh Kumar21ef5722016-11-15 13:53:23 +0530763 if (!policy->fast_switch_enabled) {
764 irq_work_sync(&sg_policy->irq_work);
765 kthread_cancel_work_sync(&sg_policy->work);
766 }
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200767}
768
Rafael J. Wysockie7888922016-06-02 23:24:15 +0200769static void sugov_limits(struct cpufreq_policy *policy)
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200770{
771 struct sugov_policy *sg_policy = policy->governor_data;
772
773 if (!policy->fast_switch_enabled) {
774 mutex_lock(&sg_policy->work_lock);
Viresh Kumarbf2be2d2016-05-18 17:55:31 +0530775 cpufreq_policy_apply_limits(policy);
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200776 mutex_unlock(&sg_policy->work_lock);
777 }
778
779 sg_policy->need_freq_update = true;
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200780}
781
782static struct cpufreq_governor schedutil_gov = {
783 .name = "schedutil",
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200784 .owner = THIS_MODULE,
Viresh Kumar560c6e42017-07-19 15:42:47 +0530785 .dynamic_switching = true,
Rafael J. Wysockie7888922016-06-02 23:24:15 +0200786 .init = sugov_init,
787 .exit = sugov_exit,
788 .start = sugov_start,
789 .stop = sugov_stop,
790 .limits = sugov_limits,
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200791};
792
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200793#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL
794struct cpufreq_governor *cpufreq_default_governor(void)
795{
796 return &schedutil_gov;
797}
Rafael J. Wysocki9bdcb442016-04-02 01:09:12 +0200798#endif
Rafael J. Wysocki58919e82016-08-16 22:14:55 +0200799
800static int __init sugov_register(void)
801{
802 return cpufreq_register_governor(&schedutil_gov);
803}
804fs_initcall(sugov_register);