From 4725344462362e2ce2645f354737a8ea4280fa57 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Sat, 29 Nov 2025 09:57:40 +0000 Subject: time/timecounter: Inline timecounter_cyc2time() New network transport protocols want NIC drivers to get hardware timestamps of all incoming packets, and possibly all outgoing packets. One example is the upcoming 'Swift congestion control' which is used by TCP transport and is the primary need for timecounter_cyc2time(). This means timecounter_cyc2time() can be called more than 100 million times per second on a busy server. Inlining timecounter_cyc2time() brings a 12% improvement on a UDP receive stress test on a 100Gbit NIC. Note that FDO, LTO, PGO are unable to magically help for this case, presumably because NIC drivers are almost exclusively shipped as modules. Add an unlikely() around the cc_cyc2ns_backwards() case, even if FDO (when used) is able to take care of this optimization. Signed-off-by: Eric Dumazet Signed-off-by: Thomas Gleixner Link: https://research.google/pubs/swift-delay-is-simple-and-effective-for-congestion-control-in-the-datacenter/ Link: https://patch.msgid.link/20251129095740.3338476-1-edumazet@google.com --- include/linux/timecounter.h | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/timecounter.h b/include/linux/timecounter.h index dce03a5cafb7..7de6b350e559 100644 --- a/include/linux/timecounter.h +++ b/include/linux/timecounter.h @@ -115,6 +115,15 @@ extern void timecounter_init(struct timecounter *tc, */ extern u64 timecounter_read(struct timecounter *tc); +/* + * This is like cyclecounter_cyc2ns(), but it is used for computing a + * time previous to the time stored in the cycle counter. + */ +static inline u64 cc_cyc2ns_backwards(const struct cyclecounter *cc, u64 cycles, u64 frac) +{ + return ((cycles * cc->mult) - frac) >> cc->shift; +} + /** * timecounter_cyc2time - convert a cycle counter to same * time base as values returned by @@ -131,7 +140,25 @@ extern u64 timecounter_read(struct timecounter *tc); * * Returns: cycle counter converted to nanoseconds since the initial time stamp */ -extern u64 timecounter_cyc2time(const struct timecounter *tc, - u64 cycle_tstamp); +static inline u64 timecounter_cyc2time(const struct timecounter *tc, u64 cycle_tstamp) +{ + const struct cyclecounter *cc = tc->cc; + u64 delta = (cycle_tstamp - tc->cycle_last) & cc->mask; + u64 nsec = tc->nsec, frac = tc->frac; + + /* + * Instead of always treating cycle_tstamp as more recent than + * tc->cycle_last, detect when it is too far in the future and + * treat it as old time stamp instead. + */ + if (unlikely(delta > cc->mask / 2)) { + delta = (tc->cycle_last - cycle_tstamp) & cc->mask; + nsec -= cc_cyc2ns_backwards(cc, delta, frac); + } else { + nsec += cyclecounter_cyc2ns(cc, delta, tc->mask, &frac); + } + + return nsec; +} #endif -- cgit v1.2.3 From 0483e5e1dc78f53c540b19231e9cf5571ce01d50 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Wed, 7 Jan 2026 11:36:56 +0100 Subject: hrtimer: Remove unused resolution constants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These constants are never used, remove them. Signed-off-by: Thomas Weißschuh Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260107-hrtimer-header-cleanup-v1-1-1a698ef0ddae@linutronix.de --- include/linux/hrtimer_defs.h | 8 -------- 1 file changed, 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hrtimer_defs.h b/include/linux/hrtimer_defs.h index aa49ffa130e5..e0280d851455 100644 --- a/include/linux/hrtimer_defs.h +++ b/include/linux/hrtimer_defs.h @@ -15,14 +15,6 @@ * this resolution values. */ # define HIGH_RES_NSEC 1 -# define KTIME_HIGH_RES (HIGH_RES_NSEC) -# define MONOTONIC_RES_NSEC HIGH_RES_NSEC -# define KTIME_MONOTONIC_RES KTIME_HIGH_RES - -#else - -# define MONOTONIC_RES_NSEC LOW_RES_NSEC -# define KTIME_MONOTONIC_RES KTIME_LOW_RES #endif -- cgit v1.2.3 From 84663a5ad6333e8dcb57be9bb113f592e05b33c6 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Wed, 7 Jan 2026 11:36:57 +0100 Subject: hrtimer: Remove public definition of HIGH_RES_NSEC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This constant is only used in a single place and is has a very generic name polluting the global namespace. Move the constant closer to its only user. Signed-off-by: Thomas Weißschuh Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260107-hrtimer-header-cleanup-v1-2-1a698ef0ddae@linutronix.de --- include/linux/hrtimer_defs.h | 12 ------------ kernel/time/hrtimer.c | 8 ++++++++ 2 files changed, 8 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hrtimer_defs.h b/include/linux/hrtimer_defs.h index e0280d851455..02b010df6570 100644 --- a/include/linux/hrtimer_defs.h +++ b/include/linux/hrtimer_defs.h @@ -6,18 +6,6 @@ #include #include -#ifdef CONFIG_HIGH_RES_TIMERS - -/* - * The resolution of the clocks. The resolution value is returned in - * the clock_getres() system call to give application programmers an - * idea of the (in)accuracy of timers. Timer values are rounded up to - * this resolution values. - */ -# define HIGH_RES_NSEC 1 - -#endif - #ifdef CONFIG_64BIT # define __hrtimer_clock_base_align ____cacheline_aligned #else diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index f8ea8c8fc895..2d319f8aff14 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -49,6 +49,14 @@ #include "tick-internal.h" +/* + * The resolution of the clocks. The resolution value is returned in + * the clock_getres() system call to give application programmers an + * idea of the (in)accuracy of timers. Timer values are rounded up to + * this resolution values. + */ +#define HIGH_RES_NSEC 1 + /* * Masks for selecting the soft and hard context timers from * cpu_base->active -- cgit v1.2.3 From ae4535b0d9372ca90a24f2d9970310ee48eb3cc2 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Wed, 7 Jan 2026 11:36:58 +0100 Subject: hrtimer: Drop _tv64() helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since ktime_t has become an alias to s64, these helpers are unnecessary. Migrate the few remaining users to the regular helpers and remove the now dead code. Signed-off-by: Thomas Weißschuh Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260107-hrtimer-header-cleanup-v1-3-1a698ef0ddae@linutronix.de --- include/linux/hrtimer.h | 15 --------------- kernel/time/hrtimer.c | 6 +++--- 2 files changed, 3 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 2cf1bf65b225..eb2d3d9dd064 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -112,12 +112,6 @@ static inline void hrtimer_set_expires_range_ns(struct hrtimer *timer, ktime_t t timer->node.expires = ktime_add_safe(time, ns_to_ktime(delta)); } -static inline void hrtimer_set_expires_tv64(struct hrtimer *timer, s64 tv64) -{ - timer->node.expires = tv64; - timer->_softexpires = tv64; -} - static inline void hrtimer_add_expires(struct hrtimer *timer, ktime_t time) { timer->node.expires = ktime_add_safe(timer->node.expires, time); @@ -140,15 +134,6 @@ static inline ktime_t hrtimer_get_softexpires(const struct hrtimer *timer) return timer->_softexpires; } -static inline s64 hrtimer_get_expires_tv64(const struct hrtimer *timer) -{ - return timer->node.expires; -} -static inline s64 hrtimer_get_softexpires_tv64(const struct hrtimer *timer) -{ - return timer->_softexpires; -} - static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer) { return ktime_to_ns(timer->node.expires); diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 2d319f8aff14..d0ab2e9e3f30 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -814,7 +814,7 @@ static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram) struct hrtimer_clock_base *base = timer->base; ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset); - WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0); + WARN_ON_ONCE(hrtimer_get_expires(timer) < 0); /* * CLOCK_REALTIME timer might be requested with an absolute @@ -1061,7 +1061,7 @@ u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval) orun = ktime_divns(delta, incr); hrtimer_add_expires_ns(timer, incr * orun); - if (hrtimer_get_expires_tv64(timer) > now) + if (hrtimer_get_expires(timer) > now) return orun; /* * This (and the ktime_add() below) is the @@ -1843,7 +1843,7 @@ static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now, * are right-of a not yet expired timer, because that * timer will have to trigger a wakeup anyway. */ - if (basenow < hrtimer_get_softexpires_tv64(timer)) + if (basenow < hrtimer_get_softexpires(timer)) break; __run_hrtimer(cpu_base, base, timer, &basenow, flags); -- cgit v1.2.3