From 56a7b9f8b05981e929becb45a826558779bca6f6 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 13 Mar 2025 15:31:50 -0700 Subject: ratelimit: Create functions to handle ratelimit_state internals A number of ratelimit use cases do open-coded access to the ratelimit_state structure's ->missed field. This works, but is a bit messy and makes it more annoying to make changes to this field. Therefore, provide a ratelimit_state_inc_miss() function that increments the ->missed field, a ratelimit_state_get_miss() function that reads out the ->missed field, and a ratelimit_state_reset_miss() function that reads out that field, but that also resets its value to zero. These functions will replace client-code open-coded uses of ->missed. In addition, a new ratelimit_state_reset_interval() function encapsulates what was previously open-coded lock acquisition and direct field updates. [ paulmck: Apply kernel test robot feedback. ] Link: https://lore.kernel.org/all/fbe93a52-365e-47fe-93a4-44a44547d601@paulmck-laptop/ Link: https://lore.kernel.org/all/20250423115409.3425-1-spasswolf@web.de/ Signed-off-by: Paul E. McKenney Reviewed-by: Petr Mladek Cc: Andrew Morton Cc: Kuniyuki Iwashima Cc: Mateusz Guzik Cc: Steven Rostedt Cc: John Ogness Cc: Sergey Senozhatsky --- include/linux/ratelimit.h | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ratelimit.h b/include/linux/ratelimit.h index b17e0cd0a30c..8400c5356c18 100644 --- a/include/linux/ratelimit.h +++ b/include/linux/ratelimit.h @@ -22,16 +22,46 @@ static inline void ratelimit_default_init(struct ratelimit_state *rs) DEFAULT_RATELIMIT_BURST); } +static inline void ratelimit_state_inc_miss(struct ratelimit_state *rs) +{ + rs->missed++; +} + +static inline int ratelimit_state_get_miss(struct ratelimit_state *rs) +{ + return rs->missed; +} + +static inline int ratelimit_state_reset_miss(struct ratelimit_state *rs) +{ + int ret = rs->missed; + + rs->missed = 0; + return ret; +} + +static inline void ratelimit_state_reset_interval(struct ratelimit_state *rs, int interval_init) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&rs->lock, flags); + rs->interval = interval_init; + rs->begin = 0; + rs->printed = 0; + ratelimit_state_reset_miss(rs); + raw_spin_unlock_irqrestore(&rs->lock, flags); +} + static inline void ratelimit_state_exit(struct ratelimit_state *rs) { + int m; + if (!(rs->flags & RATELIMIT_MSG_ON_RELEASE)) return; - if (rs->missed) { - pr_warn("%s: %d output lines suppressed due to ratelimiting\n", - current->comm, rs->missed); - rs->missed = 0; - } + m = ratelimit_state_reset_miss(rs); + if (m) + pr_warn("%s: %d output lines suppressed due to ratelimiting\n", current->comm, m); } static inline void -- cgit v1.2.3 From 78bf44de47b3cec72ee8ddc14161d5b3212f25e0 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 14 Mar 2025 09:07:13 -0700 Subject: ratelimit: Convert the ->missed field to atomic_t The ratelimit_state structure's ->missed field is sometimes incremented locklessly, and it would be good to avoid lost counts. This is also needed to count the number of misses due to trylock failure. Therefore, convert the ratelimit_state structure's ->missed field to atomic_t. Link: https://lore.kernel.org/all/fbe93a52-365e-47fe-93a4-44a44547d601@paulmck-laptop/ Link: https://lore.kernel.org/all/20250423115409.3425-1-spasswolf@web.de/ Signed-off-by: Paul E. McKenney Reviewed-by: Petr Mladek Cc: Andrew Morton Cc: Kuniyuki Iwashima Cc: Mateusz Guzik Cc: Steven Rostedt Cc: John Ogness Cc: Sergey Senozhatsky --- include/linux/ratelimit.h | 9 +++------ include/linux/ratelimit_types.h | 2 +- lib/ratelimit.c | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ratelimit.h b/include/linux/ratelimit.h index 8400c5356c18..c78b92b3e5cd 100644 --- a/include/linux/ratelimit.h +++ b/include/linux/ratelimit.h @@ -24,20 +24,17 @@ static inline void ratelimit_default_init(struct ratelimit_state *rs) static inline void ratelimit_state_inc_miss(struct ratelimit_state *rs) { - rs->missed++; + atomic_inc(&rs->missed); } static inline int ratelimit_state_get_miss(struct ratelimit_state *rs) { - return rs->missed; + return atomic_read(&rs->missed); } static inline int ratelimit_state_reset_miss(struct ratelimit_state *rs) { - int ret = rs->missed; - - rs->missed = 0; - return ret; + return atomic_xchg_relaxed(&rs->missed, 0); } static inline void ratelimit_state_reset_interval(struct ratelimit_state *rs, int interval_init) diff --git a/include/linux/ratelimit_types.h b/include/linux/ratelimit_types.h index 765232ce0b5e..d21fe82b67f6 100644 --- a/include/linux/ratelimit_types.h +++ b/include/linux/ratelimit_types.h @@ -18,7 +18,7 @@ struct ratelimit_state { int interval; int burst; int printed; - int missed; + atomic_t missed; unsigned int flags; unsigned long begin; }; diff --git a/lib/ratelimit.c b/lib/ratelimit.c index 85e22f00180c..18703f92d73e 100644 --- a/lib/ratelimit.c +++ b/lib/ratelimit.c @@ -66,7 +66,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) rs->printed++; ret = 1; } else { - rs->missed++; + ratelimit_state_inc_miss(rs); ret = 0; } raw_spin_unlock_irqrestore(&rs->lock, flags); -- cgit v1.2.3 From e64a348dc148af41cec266b2143305a2d0228ee7 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 25 Mar 2025 15:32:54 -0700 Subject: ratelimit: Avoid jiffies=0 special case The ___ratelimit() function special-cases the jiffies-counter value of zero as "uninitialized". This works well on 64-bit systems, where the jiffies counter is not going to return to zero for more than half a billion years on systems with HZ=1000, but similar 32-bit systems take less than 50 days to wrap the jiffies counter. And although the consequences of wrapping the jiffies counter seem to be limited to minor confusion on the duration of the rate-limiting interval that happens to end at time zero, it is almost no work to avoid this confusion. Therefore, introduce a RATELIMIT_INITIALIZED bit to the ratelimit_state structure's ->flags field so that a ->begin value of zero is no longer special. Link: https://lore.kernel.org/all/fbe93a52-365e-47fe-93a4-44a44547d601@paulmck-laptop/ Link: https://lore.kernel.org/all/20250423115409.3425-1-spasswolf@web.de/ Signed-off-by: Paul E. McKenney Reviewed-by: Petr Mladek Cc: Andrew Morton Cc: Kuniyuki Iwashima Cc: Mateusz Guzik Cc: Steven Rostedt Cc: John Ogness Cc: Sergey Senozhatsky --- include/linux/ratelimit.h | 2 +- include/linux/ratelimit_types.h | 1 + lib/ratelimit.c | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ratelimit.h b/include/linux/ratelimit.h index c78b92b3e5cd..adfec24061d1 100644 --- a/include/linux/ratelimit.h +++ b/include/linux/ratelimit.h @@ -43,7 +43,7 @@ static inline void ratelimit_state_reset_interval(struct ratelimit_state *rs, in raw_spin_lock_irqsave(&rs->lock, flags); rs->interval = interval_init; - rs->begin = 0; + rs->flags &= ~RATELIMIT_INITIALIZED; rs->printed = 0; ratelimit_state_reset_miss(rs); raw_spin_unlock_irqrestore(&rs->lock, flags); diff --git a/include/linux/ratelimit_types.h b/include/linux/ratelimit_types.h index d21fe82b67f6..ef6711b6b229 100644 --- a/include/linux/ratelimit_types.h +++ b/include/linux/ratelimit_types.h @@ -11,6 +11,7 @@ /* issue num suppressed message on exit */ #define RATELIMIT_MSG_ON_RELEASE BIT(0) +#define RATELIMIT_INITIALIZED BIT(1) struct ratelimit_state { raw_spinlock_t lock; /* protect the state */ diff --git a/lib/ratelimit.c b/lib/ratelimit.c index 19ad3cdbd171..bd6e3b429e33 100644 --- a/lib/ratelimit.c +++ b/lib/ratelimit.c @@ -49,8 +49,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) return 0; } - if (!rs->begin) + if (!(rs->flags & RATELIMIT_INITIALIZED)) { rs->begin = jiffies; + rs->flags |= RATELIMIT_INITIALIZED; + } if (time_is_before_jiffies(rs->begin + interval)) { int m = ratelimit_state_reset_miss(rs); -- cgit v1.2.3 From cf8cfa8a9978bfe77f2648a04824a46d58258e68 Mon Sep 17 00:00:00 2001 From: Petr Mladek Date: Wed, 9 Apr 2025 16:54:33 -0700 Subject: ratelimit: Reduce ___ratelimit() false-positive rate limiting Retain the locked design, but check rate-limiting even when the lock could not be acquired. Link: https://lore.kernel.org/all/Z_VRo63o2UsVoxLG@pathway.suse.cz/ Link: https://lore.kernel.org/all/fbe93a52-365e-47fe-93a4-44a44547d601@paulmck-laptop/ Link: https://lore.kernel.org/all/20250423115409.3425-1-spasswolf@web.de/ Signed-off-by: Petr Mladek Signed-off-by: Paul E. McKenney Cc: Petr Mladek Cc: Andrew Morton Cc: Kuniyuki Iwashima Cc: Mateusz Guzik Cc: Steven Rostedt Cc: John Ogness Cc: Sergey Senozhatsky --- include/linux/ratelimit.h | 2 +- include/linux/ratelimit_types.h | 2 +- lib/ratelimit.c | 51 ++++++++++++++++++++++++++++++----------- 3 files changed, 40 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ratelimit.h b/include/linux/ratelimit.h index adfec24061d1..7aaad158ee37 100644 --- a/include/linux/ratelimit.h +++ b/include/linux/ratelimit.h @@ -44,7 +44,7 @@ static inline void ratelimit_state_reset_interval(struct ratelimit_state *rs, in raw_spin_lock_irqsave(&rs->lock, flags); rs->interval = interval_init; rs->flags &= ~RATELIMIT_INITIALIZED; - rs->printed = 0; + atomic_set(&rs->rs_n_left, rs->burst); ratelimit_state_reset_miss(rs); raw_spin_unlock_irqrestore(&rs->lock, flags); } diff --git a/include/linux/ratelimit_types.h b/include/linux/ratelimit_types.h index ef6711b6b229..b19c4354540a 100644 --- a/include/linux/ratelimit_types.h +++ b/include/linux/ratelimit_types.h @@ -18,7 +18,7 @@ struct ratelimit_state { int interval; int burst; - int printed; + atomic_t rs_n_left; atomic_t missed; unsigned int flags; unsigned long begin; diff --git a/lib/ratelimit.c b/lib/ratelimit.c index bd6e3b429e33..90c9fe57eb42 100644 --- a/lib/ratelimit.c +++ b/lib/ratelimit.c @@ -39,12 +39,22 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) return 1; /* - * If we contend on this state's lock then almost - * by definition we are too busy to print a message, - * in addition to the one that will be printed by - * the entity that is holding the lock already: + * If we contend on this state's lock then just check if + * the current burst is used or not. It might cause + * false positive when we are past the interval and + * the current lock owner is just about to reset it. */ if (!raw_spin_trylock_irqsave(&rs->lock, flags)) { + unsigned int rs_flags = READ_ONCE(rs->flags); + + if (rs_flags & RATELIMIT_INITIALIZED && burst) { + int n_left; + + n_left = atomic_dec_return(&rs->rs_n_left); + if (n_left >= 0) + return 1; + } + ratelimit_state_inc_miss(rs); return 0; } @@ -52,27 +62,42 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) if (!(rs->flags & RATELIMIT_INITIALIZED)) { rs->begin = jiffies; rs->flags |= RATELIMIT_INITIALIZED; + atomic_set(&rs->rs_n_left, rs->burst); } if (time_is_before_jiffies(rs->begin + interval)) { - int m = ratelimit_state_reset_miss(rs); + int m; + + /* + * Reset rs_n_left ASAP to reduce false positives + * in parallel calls, see above. + */ + atomic_set(&rs->rs_n_left, rs->burst); + rs->begin = jiffies; + m = ratelimit_state_reset_miss(rs); if (m) { if (!(rs->flags & RATELIMIT_MSG_ON_RELEASE)) { printk_deferred(KERN_WARNING "%s: %d callbacks suppressed\n", func, m); } } - rs->begin = jiffies; - rs->printed = 0; } - if (burst && burst > rs->printed) { - rs->printed++; - ret = 1; - } else { - ratelimit_state_inc_miss(rs); - ret = 0; + if (burst) { + int n_left; + + /* The burst might have been taken by a parallel call. */ + n_left = atomic_dec_return(&rs->rs_n_left); + if (n_left >= 0) { + ret = 1; + goto unlock_ret; + } } + + ratelimit_state_inc_miss(rs); + ret = 0; + +unlock_ret: raw_spin_unlock_irqrestore(&rs->lock, flags); return ret; -- cgit v1.2.3