diff options
Diffstat (limited to 'drivers/cpuidle')
| -rw-r--r-- | drivers/cpuidle/cpuidle.c | 12 | ||||
| -rw-r--r-- | drivers/cpuidle/driver.c | 10 | ||||
| -rw-r--r-- | drivers/cpuidle/governor.c | 4 | ||||
| -rw-r--r-- | drivers/cpuidle/governors/menu.c | 9 | ||||
| -rw-r--r-- | drivers/cpuidle/governors/teo.c | 159 | ||||
| -rw-r--r-- | drivers/cpuidle/poll_state.c | 4 |
6 files changed, 102 insertions, 96 deletions
diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index 56132e843c99..c7876e9e024f 100644 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -184,20 +184,22 @@ static noinstr void enter_s2idle_proper(struct cpuidle_driver *drv, * cpuidle_enter_s2idle - Enter an idle state suitable for suspend-to-idle. * @drv: cpuidle driver for the given CPU. * @dev: cpuidle device for the given CPU. + * @latency_limit_ns: Idle state exit latency limit * * If there are states with the ->enter_s2idle callback, find the deepest of * them and enter it with frozen tick. */ -int cpuidle_enter_s2idle(struct cpuidle_driver *drv, struct cpuidle_device *dev) +int cpuidle_enter_s2idle(struct cpuidle_driver *drv, struct cpuidle_device *dev, + u64 latency_limit_ns) { int index; /* - * Find the deepest state with ->enter_s2idle present, which guarantees - * that interrupts won't be enabled when it exits and allows the tick to - * be frozen safely. + * Find the deepest state with ->enter_s2idle present that meets the + * specified latency limit, which guarantees that interrupts won't be + * enabled when it exits and allows the tick to be frozen safely. */ - index = find_deepest_state(drv, dev, U64_MAX, 0, true); + index = find_deepest_state(drv, dev, latency_limit_ns, 0, true); if (index > 0) { enter_s2idle_proper(drv, dev, index); local_irq_enable(); diff --git a/drivers/cpuidle/driver.c b/drivers/cpuidle/driver.c index 9bbfa594c442..370664c47e65 100644 --- a/drivers/cpuidle/driver.c +++ b/drivers/cpuidle/driver.c @@ -8,6 +8,8 @@ * This code is licenced under the GPL. */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/mutex.h> #include <linux/module.h> #include <linux/sched.h> @@ -193,6 +195,14 @@ static void __cpuidle_driver_init(struct cpuidle_driver *drv) s->exit_latency_ns = 0; else s->exit_latency = div_u64(s->exit_latency_ns, NSEC_PER_USEC); + + /* + * Warn if the exit latency of a CPU idle state exceeds its + * target residency which is assumed to never happen in cpuidle + * in multiple places. + */ + if (s->exit_latency_ns > s->target_residency_ns) + pr_warn("Idle state %d target residency too low\n", i); } } diff --git a/drivers/cpuidle/governor.c b/drivers/cpuidle/governor.c index 0d0f9751ff8f..5d0e7f78c6c5 100644 --- a/drivers/cpuidle/governor.c +++ b/drivers/cpuidle/governor.c @@ -111,6 +111,10 @@ s64 cpuidle_governor_latency_req(unsigned int cpu) struct device *device = get_cpu_device(cpu); int device_req = dev_pm_qos_raw_resume_latency(device); int global_req = cpu_latency_qos_limit(); + int global_wake_req = cpu_wakeup_latency_qos_limit(); + + if (global_req > global_wake_req) + global_req = global_wake_req; if (device_req > global_req) device_req = global_req; diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c index 23239b0c04f9..64d6f7a1c776 100644 --- a/drivers/cpuidle/governors/menu.c +++ b/drivers/cpuidle/governors/menu.c @@ -317,12 +317,13 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, } /* - * Use a physical idle state, not busy polling, unless a timer - * is going to trigger soon enough or the exit latency of the - * idle state in question is greater than the predicted idle - * duration. + * Use a physical idle state instead of busy polling so long as + * its target residency is below the residency threshold, its + * exit latency is not greater than the predicted idle duration, + * and the next timer doesn't expire soon. */ if ((drv->states[idx].flags & CPUIDLE_FLAG_POLLING) && + s->target_residency_ns < RESIDENCY_THRESHOLD_NS && s->target_residency_ns <= data->next_timer_ns && s->exit_latency_ns <= predicted_ns) { predicted_ns = s->target_residency_ns; diff --git a/drivers/cpuidle/governors/teo.c b/drivers/cpuidle/governors/teo.c index bfa55c1eab5b..81ac5fd58a1c 100644 --- a/drivers/cpuidle/governors/teo.c +++ b/drivers/cpuidle/governors/teo.c @@ -76,7 +76,7 @@ * likely woken up by a non-timer wakeup source). * * 2. If the second sum computed in step 1 is greater than a half of the sum of - * both metrics for the candidate state bin and all subsequent bins(if any), + * both metrics for the candidate state bin and all subsequent bins (if any), * a shallower idle state is likely to be more suitable, so look for it. * * - Traverse the enabled idle states shallower than the candidate one in the @@ -133,21 +133,33 @@ struct teo_bin { * @sleep_length_ns: Time till the closest timer event (at the selection time). * @state_bins: Idle state data bins for this CPU. * @total: Grand total of the "intercepts" and "hits" metrics for all bins. + * @total_tick: Wakeups by the scheduler tick. * @tick_intercepts: "Intercepts" before TICK_NSEC. * @short_idles: Wakeups after short idle periods. - * @artificial_wakeup: Set if the wakeup has been triggered by a safety net. + * @tick_wakeup: Set if the last wakeup was by the scheduler tick. */ struct teo_cpu { s64 sleep_length_ns; struct teo_bin state_bins[CPUIDLE_STATE_MAX]; unsigned int total; + unsigned int total_tick; unsigned int tick_intercepts; unsigned int short_idles; - bool artificial_wakeup; + bool tick_wakeup; }; static DEFINE_PER_CPU(struct teo_cpu, teo_cpus); +static void teo_decay(unsigned int *metric) +{ + unsigned int delta = *metric >> DECAY_SHIFT; + + if (delta) + *metric -= delta; + else + *metric = 0; +} + /** * teo_update - Update CPU metrics after wakeup. * @drv: cpuidle driver containing state data. @@ -155,21 +167,22 @@ static DEFINE_PER_CPU(struct teo_cpu, teo_cpus); */ static void teo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev) { - struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu); + struct teo_cpu *cpu_data = this_cpu_ptr(&teo_cpus); int i, idx_timer = 0, idx_duration = 0; - s64 target_residency_ns; - u64 measured_ns; + s64 target_residency_ns, measured_ns; + unsigned int total = 0; - cpu_data->short_idles -= cpu_data->short_idles >> DECAY_SHIFT; + teo_decay(&cpu_data->short_idles); - if (cpu_data->artificial_wakeup) { + if (dev->poll_time_limit) { + dev->poll_time_limit = false; /* - * If one of the safety nets has triggered, assume that this + * Polling state timeout has triggered, so assume that this * might have been a long sleep. */ - measured_ns = U64_MAX; + measured_ns = S64_MAX; } else { - u64 lat_ns = drv->states[dev->last_state_idx].exit_latency_ns; + s64 lat_ns = drv->states[dev->last_state_idx].exit_latency_ns; measured_ns = dev->last_residency_ns; /* @@ -196,8 +209,10 @@ static void teo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev) for (i = 0; i < drv->state_count; i++) { struct teo_bin *bin = &cpu_data->state_bins[i]; - bin->hits -= bin->hits >> DECAY_SHIFT; - bin->intercepts -= bin->intercepts >> DECAY_SHIFT; + teo_decay(&bin->hits); + total += bin->hits; + teo_decay(&bin->intercepts); + total += bin->intercepts; target_residency_ns = drv->states[i].target_residency_ns; @@ -208,7 +223,24 @@ static void teo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev) } } - cpu_data->tick_intercepts -= cpu_data->tick_intercepts >> DECAY_SHIFT; + cpu_data->total = total + PULSE; + + teo_decay(&cpu_data->tick_intercepts); + + teo_decay(&cpu_data->total_tick); + if (cpu_data->tick_wakeup) { + cpu_data->total_tick += PULSE; + /* + * If tick wakeups dominate the wakeup pattern, count this one + * as a hit on the deepest available idle state to increase the + * likelihood of stopping the tick. + */ + if (3 * cpu_data->total_tick > 2 * cpu_data->total) { + cpu_data->state_bins[drv->state_count-1].hits += PULSE; + return; + } + } + /* * If the measured idle duration falls into the same bin as the sleep * length, this is a "hit", so update the "hits" metric for that bin. @@ -219,18 +251,9 @@ static void teo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev) cpu_data->state_bins[idx_timer].hits += PULSE; } else { cpu_data->state_bins[idx_duration].intercepts += PULSE; - if (TICK_NSEC <= measured_ns) + if (measured_ns <= TICK_NSEC) cpu_data->tick_intercepts += PULSE; } - - cpu_data->total -= cpu_data->total >> DECAY_SHIFT; - cpu_data->total += PULSE; -} - -static bool teo_state_ok(int i, struct cpuidle_driver *drv) -{ - return !tick_nohz_tick_stopped() || - drv->states[i].target_residency_ns >= TICK_NSEC; } /** @@ -239,17 +262,15 @@ static bool teo_state_ok(int i, struct cpuidle_driver *drv) * @dev: Target CPU. * @state_idx: Index of the capping idle state. * @duration_ns: Idle duration value to match. - * @no_poll: Don't consider polling states. */ static int teo_find_shallower_state(struct cpuidle_driver *drv, struct cpuidle_device *dev, int state_idx, - s64 duration_ns, bool no_poll) + s64 duration_ns) { int i; for (i = state_idx - 1; i >= 0; i--) { - if (dev->states_usage[i].disable || - (no_poll && drv->states[i].flags & CPUIDLE_FLAG_POLLING)) + if (dev->states_usage[i].disable) continue; state_idx = i; @@ -268,7 +289,7 @@ static int teo_find_shallower_state(struct cpuidle_driver *drv, static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, bool *stop_tick) { - struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu); + struct teo_cpu *cpu_data = this_cpu_ptr(&teo_cpus); s64 latency_req = cpuidle_governor_latency_req(dev->cpu); ktime_t delta_tick = TICK_NSEC / 2; unsigned int idx_intercept_sum = 0; @@ -356,7 +377,18 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, * better choice. */ if (2 * idx_intercept_sum > cpu_data->total - idx_hit_sum) { - int first_suitable_idx = idx; + int min_idx = idx0; + + if (tick_nohz_tick_stopped()) { + /* + * Look for the shallowest idle state below the current + * candidate one whose target residency is at least + * equal to the tick period length. + */ + while (min_idx < idx && + drv->states[min_idx].target_residency_ns < TICK_NSEC) + min_idx++; + } /* * Look for the deepest idle state whose target residency had @@ -366,49 +398,14 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, * Take the possible duration limitation present if the tick * has been stopped already into account. */ - intercept_sum = 0; - - for (i = idx - 1; i >= 0; i--) { - struct teo_bin *bin = &cpu_data->state_bins[i]; - - intercept_sum += bin->intercepts; - - if (2 * intercept_sum > idx_intercept_sum) { - /* - * Use the current state unless it is too - * shallow or disabled, in which case take the - * first enabled state that is deep enough. - */ - if (teo_state_ok(i, drv) && - !dev->states_usage[i].disable) { - idx = i; - break; - } - idx = first_suitable_idx; - break; - } + for (i = idx - 1, intercept_sum = 0; i >= min_idx; i--) { + intercept_sum += cpu_data->state_bins[i].intercepts; if (dev->states_usage[i].disable) continue; - if (teo_state_ok(i, drv)) { - /* - * The current state is deep enough, but still - * there may be a better one. - */ - first_suitable_idx = i; - continue; - } - - /* - * The current state is too shallow, so if no suitable - * states other than the initial candidate have been - * found, give up (the remaining states to check are - * shallower still), but otherwise the first suitable - * state other than the initial candidate may turn out - * to be preferable. - */ - if (first_suitable_idx == idx) + idx = i; + if (2 * intercept_sum > idx_intercept_sum) break; } } @@ -458,11 +455,8 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, * If the closest expected timer is before the target residency of the * candidate state, a shallower one needs to be found. */ - if (drv->states[idx].target_residency_ns > duration_ns) { - i = teo_find_shallower_state(drv, dev, idx, duration_ns, false); - if (teo_state_ok(i, drv)) - idx = i; - } + if (drv->states[idx].target_residency_ns > duration_ns) + idx = teo_find_shallower_state(drv, dev, idx, duration_ns); /* * If the selected state's target residency is below the tick length @@ -490,7 +484,7 @@ end: */ if (idx > idx0 && drv->states[idx].target_residency_ns > delta_tick) - idx = teo_find_shallower_state(drv, dev, idx, delta_tick, false); + idx = teo_find_shallower_state(drv, dev, idx, delta_tick); out_tick: *stop_tick = false; @@ -504,20 +498,11 @@ out_tick: */ static void teo_reflect(struct cpuidle_device *dev, int state) { - struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu); + struct teo_cpu *cpu_data = this_cpu_ptr(&teo_cpus); + + cpu_data->tick_wakeup = tick_nohz_idle_got_tick(); dev->last_state_idx = state; - if (dev->poll_time_limit || - (tick_nohz_idle_got_tick() && cpu_data->sleep_length_ns > TICK_NSEC)) { - /* - * The wakeup was not "genuine", but triggered by one of the - * safety nets. - */ - dev->poll_time_limit = false; - cpu_data->artificial_wakeup = true; - } else { - cpu_data->artificial_wakeup = false; - } } /** diff --git a/drivers/cpuidle/poll_state.c b/drivers/cpuidle/poll_state.c index 9b6d90a72601..c7524e4c522a 100644 --- a/drivers/cpuidle/poll_state.c +++ b/drivers/cpuidle/poll_state.c @@ -4,9 +4,13 @@ */ #include <linux/cpuidle.h> +#include <linux/export.h> +#include <linux/irqflags.h> #include <linux/sched.h> #include <linux/sched/clock.h> #include <linux/sched/idle.h> +#include <linux/sprintf.h> +#include <linux/types.h> #define POLL_IDLE_RELAX_COUNT 200 |