From df7fcbefa71090a276fb841ffe19b8e5f12b4767 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Tue, 4 Feb 2025 13:05:37 +0100 Subject: vdso: Add generic time data storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Historically each architecture defined their own way to store the vDSO data page. Add a generic mechanism to provide storage for that page. Furthermore this generic storage will be extended to also provide uniform storage for *non*-time-related data, like the random state or architecture-specific data. These will have their own pages and data structures, so rename 'vdso_data' into 'vdso_time_data' to make that split clear from the name. Also introduce a new consistent naming scheme for the symbols related to the vDSO, which makes it clear if the symbol is accessible from userspace or kernel space and the type of data behind the symbol. The generic fault handler contains an optimization to prefault the vvar page when the timens page is accessed. This was lifted from s390 and x86. Co-developed-by: Nam Cao Signed-off-by: Nam Cao Signed-off-by: Thomas Weißschuh Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20250204-vdso-store-rng-v3-5-13a4669dfc8c@linutronix.de --- kernel/time/vsyscall.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/time/vsyscall.c b/kernel/time/vsyscall.c index 05d383143165..09c1e39a6dd8 100644 --- a/kernel/time/vsyscall.c +++ b/kernel/time/vsyscall.c @@ -77,7 +77,7 @@ static inline void update_vdso_data(struct vdso_data *vdata, void update_vsyscall(struct timekeeper *tk) { - struct vdso_data *vdata = __arch_get_k_vdso_data(); + struct vdso_data *vdata = vdso_k_time_data; struct vdso_timestamp *vdso_ts; s32 clock_mode; u64 nsec; @@ -128,7 +128,7 @@ void update_vsyscall(struct timekeeper *tk) void update_vsyscall_tz(void) { - struct vdso_data *vdata = __arch_get_k_vdso_data(); + struct vdso_data *vdata = vdso_k_time_data; vdata[CS_HRES_COARSE].tz_minuteswest = sys_tz.tz_minuteswest; vdata[CS_HRES_COARSE].tz_dsttime = sys_tz.tz_dsttime; @@ -150,7 +150,7 @@ void update_vsyscall_tz(void) */ unsigned long vdso_update_begin(void) { - struct vdso_data *vdata = __arch_get_k_vdso_data(); + struct vdso_data *vdata = vdso_k_time_data; unsigned long flags = timekeeper_lock_irqsave(); vdso_write_begin(vdata); @@ -167,7 +167,7 @@ unsigned long vdso_update_begin(void) */ void vdso_update_end(unsigned long flags) { - struct vdso_data *vdata = __arch_get_k_vdso_data(); + struct vdso_data *vdata = vdso_k_time_data; vdso_write_end(vdata); __arch_sync_vdso_data(vdata); -- cgit v1.2.3 From ac1a42f4e4e296b5ba5fdb39444f65d6e5196240 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Tue, 4 Feb 2025 13:05:50 +0100 Subject: vdso: Remove remnants of architecture-specific time storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All users of the time releated parts of the vDSO are now using the generic storage implementation. Remove the therefore unnecessary compatibility accessor functions and symbols. Co-developed-by: Nam Cao Signed-off-by: Nam Cao Signed-off-by: Thomas Weißschuh Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20250204-vdso-store-rng-v3-18-13a4669dfc8c@linutronix.de --- include/asm-generic/vdso/vsyscall.h | 20 +++------------ include/linux/time_namespace.h | 3 --- include/vdso/datapage.h | 19 +------------- include/vdso/helpers.h | 8 +++--- kernel/time/namespace.c | 12 ++++----- kernel/time/vsyscall.c | 19 +++++++------- lib/vdso/datastore.c | 10 +++----- lib/vdso/gettimeofday.c | 51 ++++++++++++++++++------------------- 8 files changed, 53 insertions(+), 89 deletions(-) (limited to 'kernel') diff --git a/include/asm-generic/vdso/vsyscall.h b/include/asm-generic/vdso/vsyscall.h index 13e2ac3736ee..1fb3000f5036 100644 --- a/include/asm-generic/vdso/vsyscall.h +++ b/include/asm-generic/vdso/vsyscall.h @@ -20,31 +20,19 @@ static __always_inline const struct vdso_rng_data *__arch_get_vdso_u_rng_data(vo } #endif -#else /* !CONFIG_GENERIC_VDSO_DATA_STORE */ - -#ifndef __arch_get_k_vdso_data -static __always_inline struct vdso_data *__arch_get_k_vdso_data(void) -{ - return NULL; -} -#endif /* __arch_get_k_vdso_data */ -#define vdso_k_time_data __arch_get_k_vdso_data() - -#define __arch_get_vdso_u_time_data __arch_get_vdso_data - #endif /* CONFIG_GENERIC_VDSO_DATA_STORE */ #ifndef __arch_update_vsyscall -static __always_inline void __arch_update_vsyscall(struct vdso_data *vdata) +static __always_inline void __arch_update_vsyscall(struct vdso_time_data *vdata) { } #endif /* __arch_update_vsyscall */ -#ifndef __arch_sync_vdso_data -static __always_inline void __arch_sync_vdso_data(struct vdso_data *vdata) +#ifndef __arch_sync_vdso_time_data +static __always_inline void __arch_sync_vdso_time_data(struct vdso_time_data *vdata) { } -#endif /* __arch_sync_vdso_data */ +#endif /* __arch_sync_vdso_time_data */ #endif /* !__ASSEMBLY__ */ diff --git a/include/linux/time_namespace.h b/include/linux/time_namespace.h index 4b81db223f54..0b8b32bf0655 100644 --- a/include/linux/time_namespace.h +++ b/include/linux/time_namespace.h @@ -8,7 +8,6 @@ #include #include #include -#include struct user_namespace; extern struct user_namespace init_user_ns; @@ -166,6 +165,4 @@ static inline ktime_t timens_ktime_to_host(clockid_t clockid, ktime_t tim) } #endif -struct vdso_data *arch_get_vdso_data(void *vvar_page); - #endif /* _LINUX_TIMENS_H */ diff --git a/include/vdso/datapage.h b/include/vdso/datapage.h index 497907c3aa11..ed4fb4c06e3e 100644 --- a/include/vdso/datapage.h +++ b/include/vdso/datapage.h @@ -128,8 +128,6 @@ struct vdso_time_data { struct arch_vdso_time_data arch_data; }; -#define vdso_data vdso_time_data - /** * struct vdso_rng_data - vdso RNG state information * @generation: counter representing the number of RNG reseeds @@ -149,10 +147,7 @@ struct vdso_rng_data { * With the hidden visibility, the compiler simply generates a PC-relative * relocation, and this is what we need. */ -#ifndef CONFIG_GENERIC_VDSO_DATA_STORE -extern struct vdso_time_data _vdso_data[CS_BASES] __attribute__((visibility("hidden"))); -extern struct vdso_time_data _timens_data[CS_BASES] __attribute__((visibility("hidden"))); -#else +#ifdef CONFIG_GENERIC_VDSO_DATA_STORE extern struct vdso_time_data vdso_u_time_data[CS_BASES] __attribute__((visibility("hidden"))); extern struct vdso_rng_data vdso_u_rng_data __attribute__((visibility("hidden"))); extern struct vdso_arch_data vdso_u_arch_data __attribute__((visibility("hidden"))); @@ -160,17 +155,6 @@ extern struct vdso_arch_data vdso_u_arch_data __attribute__((visibility("hidden" extern struct vdso_time_data *vdso_k_time_data; extern struct vdso_rng_data *vdso_k_rng_data; extern struct vdso_arch_data *vdso_k_arch_data; -#endif - -/** - * union vdso_data_store - Generic vDSO data page - */ -union vdso_data_store { - struct vdso_time_data data[CS_BASES]; - u8 page[1U << CONFIG_PAGE_SHIFT]; -}; - -#ifdef CONFIG_GENERIC_VDSO_DATA_STORE #define VDSO_ARCH_DATA_SIZE ALIGN(sizeof(struct vdso_arch_data), PAGE_SIZE) #define VDSO_ARCH_DATA_PAGES (VDSO_ARCH_DATA_SIZE >> PAGE_SHIFT) @@ -189,7 +173,6 @@ enum vdso_pages { /* * The generic vDSO implementation requires that gettimeofday.h * provides: - * - __arch_get_vdso_data(): to get the vdso datapage. * - __arch_get_hw_counter(): to get the hw counter based on the * clock_mode. * - gettimeofday_fallback(): fallback for gettimeofday. diff --git a/include/vdso/helpers.h b/include/vdso/helpers.h index 3ddb03bb05cb..41c3087070c7 100644 --- a/include/vdso/helpers.h +++ b/include/vdso/helpers.h @@ -7,7 +7,7 @@ #include #include -static __always_inline u32 vdso_read_begin(const struct vdso_data *vd) +static __always_inline u32 vdso_read_begin(const struct vdso_time_data *vd) { u32 seq; @@ -18,7 +18,7 @@ static __always_inline u32 vdso_read_begin(const struct vdso_data *vd) return seq; } -static __always_inline u32 vdso_read_retry(const struct vdso_data *vd, +static __always_inline u32 vdso_read_retry(const struct vdso_time_data *vd, u32 start) { u32 seq; @@ -28,7 +28,7 @@ static __always_inline u32 vdso_read_retry(const struct vdso_data *vd, return seq != start; } -static __always_inline void vdso_write_begin(struct vdso_data *vd) +static __always_inline void vdso_write_begin(struct vdso_time_data *vd) { /* * WRITE_ONCE() is required otherwise the compiler can validly tear @@ -40,7 +40,7 @@ static __always_inline void vdso_write_begin(struct vdso_data *vd) smp_wmb(); } -static __always_inline void vdso_write_end(struct vdso_data *vd) +static __always_inline void vdso_write_end(struct vdso_time_data *vd) { smp_wmb(); /* diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c index 0775b9ec952a..12f55aa539ad 100644 --- a/kernel/time/namespace.c +++ b/kernel/time/namespace.c @@ -165,18 +165,18 @@ static struct timens_offset offset_from_ts(struct timespec64 off) * HVCLOCK * VVAR * - * The check for vdso_data->clock_mode is in the unlikely path of + * The check for vdso_time_data->clock_mode is in the unlikely path of * the seq begin magic. So for the non-timens case most of the time * 'seq' is even, so the branch is not taken. * * If 'seq' is odd, i.e. a concurrent update is in progress, the extra check - * for vdso_data->clock_mode is a non-issue. The task is spin waiting for the + * for vdso_time_data->clock_mode is a non-issue. The task is spin waiting for the * update to finish and for 'seq' to become even anyway. * - * Timens page has vdso_data->clock_mode set to VDSO_CLOCKMODE_TIMENS which + * Timens page has vdso_time_data->clock_mode set to VDSO_CLOCKMODE_TIMENS which * enforces the time namespace handling path. */ -static void timens_setup_vdso_data(struct vdso_data *vdata, +static void timens_setup_vdso_data(struct vdso_time_data *vdata, struct time_namespace *ns) { struct timens_offset *offset = vdata->offset; @@ -219,7 +219,7 @@ static DEFINE_MUTEX(offset_lock); static void timens_set_vvar_page(struct task_struct *task, struct time_namespace *ns) { - struct vdso_data *vdata; + struct vdso_time_data *vdata; unsigned int i; if (ns == &init_time_ns) @@ -235,7 +235,7 @@ static void timens_set_vvar_page(struct task_struct *task, goto out; ns->frozen_offsets = true; - vdata = arch_get_vdso_data(page_address(ns->vvar_page)); + vdata = page_address(ns->vvar_page); for (i = 0; i < CS_BASES; i++) timens_setup_vdso_data(&vdata[i], ns); diff --git a/kernel/time/vsyscall.c b/kernel/time/vsyscall.c index 09c1e39a6dd8..418192296ef7 100644 --- a/kernel/time/vsyscall.c +++ b/kernel/time/vsyscall.c @@ -15,8 +15,7 @@ #include "timekeeping_internal.h" -static inline void update_vdso_data(struct vdso_data *vdata, - struct timekeeper *tk) +static inline void update_vdso_time_data(struct vdso_time_data *vdata, struct timekeeper *tk) { struct vdso_timestamp *vdso_ts; u64 nsec, sec; @@ -77,7 +76,7 @@ static inline void update_vdso_data(struct vdso_data *vdata, void update_vsyscall(struct timekeeper *tk) { - struct vdso_data *vdata = vdso_k_time_data; + struct vdso_time_data *vdata = vdso_k_time_data; struct vdso_timestamp *vdso_ts; s32 clock_mode; u64 nsec; @@ -117,23 +116,23 @@ void update_vsyscall(struct timekeeper *tk) * update of the high resolution parts. */ if (clock_mode != VDSO_CLOCKMODE_NONE) - update_vdso_data(vdata, tk); + update_vdso_time_data(vdata, tk); __arch_update_vsyscall(vdata); vdso_write_end(vdata); - __arch_sync_vdso_data(vdata); + __arch_sync_vdso_time_data(vdata); } void update_vsyscall_tz(void) { - struct vdso_data *vdata = vdso_k_time_data; + struct vdso_time_data *vdata = vdso_k_time_data; vdata[CS_HRES_COARSE].tz_minuteswest = sys_tz.tz_minuteswest; vdata[CS_HRES_COARSE].tz_dsttime = sys_tz.tz_dsttime; - __arch_sync_vdso_data(vdata); + __arch_sync_vdso_time_data(vdata); } /** @@ -150,7 +149,7 @@ void update_vsyscall_tz(void) */ unsigned long vdso_update_begin(void) { - struct vdso_data *vdata = vdso_k_time_data; + struct vdso_time_data *vdata = vdso_k_time_data; unsigned long flags = timekeeper_lock_irqsave(); vdso_write_begin(vdata); @@ -167,9 +166,9 @@ unsigned long vdso_update_begin(void) */ void vdso_update_end(unsigned long flags) { - struct vdso_data *vdata = vdso_k_time_data; + struct vdso_time_data *vdata = vdso_k_time_data; vdso_write_end(vdata); - __arch_sync_vdso_data(vdata); + __arch_sync_vdso_time_data(vdata); timekeeper_unlock_irqrestore(flags); } diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c index 0959d62d7858..e227fbbcb796 100644 --- a/lib/vdso/datastore.c +++ b/lib/vdso/datastore.c @@ -12,7 +12,10 @@ * The vDSO data page. */ #ifdef CONFIG_HAVE_GENERIC_VDSO -static union vdso_data_store vdso_time_data_store __page_aligned_data; +static union { + struct vdso_time_data data[CS_BASES]; + u8 page[PAGE_SIZE]; +} vdso_time_data_store __page_aligned_data; struct vdso_time_data *vdso_k_time_data = vdso_time_data_store.data; static_assert(sizeof(vdso_time_data_store) == PAGE_SIZE); #endif /* CONFIG_HAVE_GENERIC_VDSO */ @@ -123,9 +126,4 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) return 0; } - -struct vdso_time_data *arch_get_vdso_data(void *vvar_page) -{ - return (struct vdso_time_data *)vvar_page; -} #endif diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index 20c5b8752fcc..299f027116ee 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -17,12 +17,12 @@ #endif #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT -static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta) +static __always_inline bool vdso_delta_ok(const struct vdso_time_data *vd, u64 delta) { return delta < vd->max_cycles; } #else -static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta) +static __always_inline bool vdso_delta_ok(const struct vdso_time_data *vd, u64 delta) { return true; } @@ -39,7 +39,7 @@ static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) * Default implementation which works for all sane clocksources. That * obviously excludes x86/TSC. */ -static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base) +static __always_inline u64 vdso_calc_ns(const struct vdso_time_data *vd, u64 cycles, u64 base) { u64 delta = (cycles - vd->cycle_last) & VDSO_DELTA_MASK(vd); @@ -58,7 +58,7 @@ static inline bool __arch_vdso_hres_capable(void) #endif #ifndef vdso_clocksource_ok -static inline bool vdso_clocksource_ok(const struct vdso_data *vd) +static inline bool vdso_clocksource_ok(const struct vdso_time_data *vd) { return vd->clock_mode != VDSO_CLOCKMODE_NONE; } @@ -79,21 +79,20 @@ const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_tim { return (void *)vd + PAGE_SIZE; } -#define __arch_get_timens_vdso_data(vd) __arch_get_vdso_u_timens_data(vd) #endif /* CONFIG_GENERIC_VDSO_DATA_STORE */ -static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, +static __always_inline int do_hres_timens(const struct vdso_time_data *vdns, clockid_t clk, struct __kernel_timespec *ts) { const struct timens_offset *offs = &vdns->offset[clk]; const struct vdso_timestamp *vdso_ts; - const struct vdso_data *vd; + const struct vdso_time_data *vd; u64 cycles, ns; u32 seq; s64 sec; vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE); - vd = __arch_get_timens_vdso_data(vd); + vd = __arch_get_vdso_u_timens_data(vd); if (clk != CLOCK_MONOTONIC_RAW) vd = &vd[CS_HRES_COARSE]; else @@ -128,19 +127,19 @@ static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_ } #else static __always_inline -const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) +const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd) { return NULL; } -static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, +static __always_inline int do_hres_timens(const struct vdso_time_data *vdns, clockid_t clk, struct __kernel_timespec *ts) { return -EINVAL; } #endif -static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, +static __always_inline int do_hres(const struct vdso_time_data *vd, clockid_t clk, struct __kernel_timespec *ts) { const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; @@ -192,10 +191,10 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, } #ifdef CONFIG_TIME_NS -static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, +static __always_inline int do_coarse_timens(const struct vdso_time_data *vdns, clockid_t clk, struct __kernel_timespec *ts) { - const struct vdso_data *vd = __arch_get_timens_vdso_data(vdns); + const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; const struct timens_offset *offs = &vdns->offset[clk]; u64 nsec; @@ -221,14 +220,14 @@ static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clocki return 0; } #else -static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, +static __always_inline int do_coarse_timens(const struct vdso_time_data *vdns, clockid_t clk, struct __kernel_timespec *ts) { return -1; } #endif -static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, +static __always_inline int do_coarse(const struct vdso_time_data *vd, clockid_t clk, struct __kernel_timespec *ts) { const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; @@ -255,7 +254,7 @@ static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, } static __always_inline int -__cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock, +__cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *ts) { u32 msk; @@ -282,7 +281,7 @@ __cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock, } static __maybe_unused int -__cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock, +__cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *ts) { int ret = __cvdso_clock_gettime_common(vd, clock, ts); @@ -300,7 +299,7 @@ __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) #ifdef BUILD_VDSO32 static __maybe_unused int -__cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock, +__cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock, struct old_timespec32 *res) { struct __kernel_timespec ts; @@ -326,7 +325,7 @@ __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) #endif /* BUILD_VDSO32 */ static __maybe_unused int -__cvdso_gettimeofday_data(const struct vdso_data *vd, +__cvdso_gettimeofday_data(const struct vdso_time_data *vd, struct __kernel_old_timeval *tv, struct timezone *tz) { @@ -343,7 +342,7 @@ __cvdso_gettimeofday_data(const struct vdso_data *vd, if (unlikely(tz != NULL)) { if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VDSO_CLOCKMODE_TIMENS) - vd = __arch_get_timens_vdso_data(vd); + vd = __arch_get_vdso_u_timens_data(vd); tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; @@ -360,13 +359,13 @@ __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) #ifdef VDSO_HAS_TIME static __maybe_unused __kernel_old_time_t -__cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time) +__cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time) { __kernel_old_time_t t; if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VDSO_CLOCKMODE_TIMENS) - vd = __arch_get_timens_vdso_data(vd); + vd = __arch_get_vdso_u_timens_data(vd); t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); @@ -384,7 +383,7 @@ static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time #ifdef VDSO_HAS_CLOCK_GETRES static __maybe_unused -int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock, +int __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *res) { u32 msk; @@ -396,7 +395,7 @@ int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock, if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VDSO_CLOCKMODE_TIMENS) - vd = __arch_get_timens_vdso_data(vd); + vd = __arch_get_vdso_u_timens_data(vd); /* * Convert the clockid to a bitmask and use it to check which @@ -425,7 +424,7 @@ int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock, } static __maybe_unused -int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock, +int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *res) { int ret = __cvdso_clock_getres_common(vd, clock, res); @@ -443,7 +442,7 @@ int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) #ifdef BUILD_VDSO32 static __maybe_unused int -__cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock, +__cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock, struct old_timespec32 *res) { struct __kernel_timespec ts; -- cgit v1.2.3 From b5afbc106d7cbfa1ffbe7c8da2693e5effbba95e Mon Sep 17 00:00:00 2001 From: Anna-Maria Behnsen Date: Mon, 3 Mar 2025 12:11:14 +0100 Subject: vdso/vsyscall: Prepare introduction of struct vdso_clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To support multiple PTP clocks, the VDSO data structure needs to be reworked. All clock specific data will end up in struct vdso_clock and in struct vdso_time_data there will be array of VDSO clocks. At the moment, vdso_clock is simply a define which maps vdso_clock to vdso_time_data. To prepare for the rework of the data structures, replace the struct vdso_time_data pointer with a struct vdso_clock pointer where applicable. No functional change. Signed-off-by: Anna-Maria Behnsen Signed-off-by: Nam Cao Signed-off-by: Thomas Weißschuh Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20250303-vdso-clock-v1-12-c1b5c69a166f@linutronix.de --- kernel/time/vsyscall.c | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) (limited to 'kernel') diff --git a/kernel/time/vsyscall.c b/kernel/time/vsyscall.c index 418192296ef7..dd85b41a70be 100644 --- a/kernel/time/vsyscall.c +++ b/kernel/time/vsyscall.c @@ -18,25 +18,26 @@ static inline void update_vdso_time_data(struct vdso_time_data *vdata, struct timekeeper *tk) { struct vdso_timestamp *vdso_ts; + struct vdso_clock *vc = vdata; u64 nsec, sec; - vdata[CS_HRES_COARSE].cycle_last = tk->tkr_mono.cycle_last; + vc[CS_HRES_COARSE].cycle_last = tk->tkr_mono.cycle_last; #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT - vdata[CS_HRES_COARSE].max_cycles = tk->tkr_mono.clock->max_cycles; + vc[CS_HRES_COARSE].max_cycles = tk->tkr_mono.clock->max_cycles; #endif - vdata[CS_HRES_COARSE].mask = tk->tkr_mono.mask; - vdata[CS_HRES_COARSE].mult = tk->tkr_mono.mult; - vdata[CS_HRES_COARSE].shift = tk->tkr_mono.shift; - vdata[CS_RAW].cycle_last = tk->tkr_raw.cycle_last; + vc[CS_HRES_COARSE].mask = tk->tkr_mono.mask; + vc[CS_HRES_COARSE].mult = tk->tkr_mono.mult; + vc[CS_HRES_COARSE].shift = tk->tkr_mono.shift; + vc[CS_RAW].cycle_last = tk->tkr_raw.cycle_last; #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT - vdata[CS_RAW].max_cycles = tk->tkr_raw.clock->max_cycles; + vc[CS_RAW].max_cycles = tk->tkr_raw.clock->max_cycles; #endif - vdata[CS_RAW].mask = tk->tkr_raw.mask; - vdata[CS_RAW].mult = tk->tkr_raw.mult; - vdata[CS_RAW].shift = tk->tkr_raw.shift; + vc[CS_RAW].mask = tk->tkr_raw.mask; + vc[CS_RAW].mult = tk->tkr_raw.mult; + vc[CS_RAW].shift = tk->tkr_raw.shift; /* CLOCK_MONOTONIC */ - vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_MONOTONIC]; + vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_MONOTONIC]; vdso_ts->sec = tk->xtime_sec + tk->wall_to_monotonic.tv_sec; nsec = tk->tkr_mono.xtime_nsec; @@ -54,7 +55,7 @@ static inline void update_vdso_time_data(struct vdso_time_data *vdata, struct ti nsec += (u64)tk->monotonic_to_boot.tv_nsec << tk->tkr_mono.shift; /* CLOCK_BOOTTIME */ - vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_BOOTTIME]; + vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_BOOTTIME]; vdso_ts->sec = sec; while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) { @@ -64,12 +65,12 @@ static inline void update_vdso_time_data(struct vdso_time_data *vdata, struct ti vdso_ts->nsec = nsec; /* CLOCK_MONOTONIC_RAW */ - vdso_ts = &vdata[CS_RAW].basetime[CLOCK_MONOTONIC_RAW]; + vdso_ts = &vc[CS_RAW].basetime[CLOCK_MONOTONIC_RAW]; vdso_ts->sec = tk->raw_sec; vdso_ts->nsec = tk->tkr_raw.xtime_nsec; /* CLOCK_TAI */ - vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_TAI]; + vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_TAI]; vdso_ts->sec = tk->xtime_sec + (s64)tk->tai_offset; vdso_ts->nsec = tk->tkr_mono.xtime_nsec; } @@ -78,6 +79,7 @@ void update_vsyscall(struct timekeeper *tk) { struct vdso_time_data *vdata = vdso_k_time_data; struct vdso_timestamp *vdso_ts; + struct vdso_clock *vc = vdata; s32 clock_mode; u64 nsec; @@ -85,21 +87,21 @@ void update_vsyscall(struct timekeeper *tk) vdso_write_begin(vdata); clock_mode = tk->tkr_mono.clock->vdso_clock_mode; - vdata[CS_HRES_COARSE].clock_mode = clock_mode; - vdata[CS_RAW].clock_mode = clock_mode; + vc[CS_HRES_COARSE].clock_mode = clock_mode; + vc[CS_RAW].clock_mode = clock_mode; /* CLOCK_REALTIME also required for time() */ - vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_REALTIME]; + vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME]; vdso_ts->sec = tk->xtime_sec; vdso_ts->nsec = tk->tkr_mono.xtime_nsec; /* CLOCK_REALTIME_COARSE */ - vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_REALTIME_COARSE]; + vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME_COARSE]; vdso_ts->sec = tk->xtime_sec; vdso_ts->nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift; /* CLOCK_MONOTONIC_COARSE */ - vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_MONOTONIC_COARSE]; + vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_MONOTONIC_COARSE]; vdso_ts->sec = tk->xtime_sec + tk->wall_to_monotonic.tv_sec; nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift; nsec = nsec + tk->wall_to_monotonic.tv_nsec; -- cgit v1.2.3 From 0235220807033c7a1dc2a96cc23b6eae0dd11c81 Mon Sep 17 00:00:00 2001 From: Anna-Maria Behnsen Date: Mon, 3 Mar 2025 12:11:15 +0100 Subject: vdso/namespace: Rename timens_setup_vdso_data() to reflect new vdso_clock struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To support multiple PTP clocks, the VDSO data structure needs to be reworked. All clock specific data will end up in struct vdso_clock and in struct vdso_time_data there will be array of VDSO clocks. At the moment, vdso_clock is simply a define which maps vdso_clock to vdso_time_data. For time namespaces, vdso_time_data needs to be set up. But only the clock related part of the vdso_data thats requires this setup. To reflect the future struct vdso_clock, rename timens_setup_vdso_data() to timns_setup_vdso_clock_data(). No functional change. Signed-off-by: Anna-Maria Behnsen Signed-off-by: Nam Cao Signed-off-by: Thomas Weißschuh Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20250303-vdso-clock-v1-13-c1b5c69a166f@linutronix.de --- kernel/time/namespace.c | 6 +++--- lib/vdso/datastore.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c index 12f55aa539ad..f02430a73be8 100644 --- a/kernel/time/namespace.c +++ b/kernel/time/namespace.c @@ -176,8 +176,8 @@ static struct timens_offset offset_from_ts(struct timespec64 off) * Timens page has vdso_time_data->clock_mode set to VDSO_CLOCKMODE_TIMENS which * enforces the time namespace handling path. */ -static void timens_setup_vdso_data(struct vdso_time_data *vdata, - struct time_namespace *ns) +static void timens_setup_vdso_clock_data(struct vdso_time_data *vdata, + struct time_namespace *ns) { struct timens_offset *offset = vdata->offset; struct timens_offset monotonic = offset_from_ts(ns->offsets.monotonic); @@ -238,7 +238,7 @@ static void timens_set_vvar_page(struct task_struct *task, vdata = page_address(ns->vvar_page); for (i = 0; i < CS_BASES; i++) - timens_setup_vdso_data(&vdata[i], ns); + timens_setup_vdso_clock_data(&vdata[i], ns); out: mutex_unlock(&offset_lock); diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c index e227fbbcb796..4e350f56ace3 100644 --- a/lib/vdso/datastore.c +++ b/lib/vdso/datastore.c @@ -109,7 +109,7 @@ struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned * non-root time namespace. Whenever a task changes its namespace, the VVAR * page tables are cleared and then they will be re-faulted with a * corresponding layout. - * See also the comment near timens_setup_vdso_data() for details. + * See also the comment near timens_setup_vdso_clock_data() for details. */ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) { -- cgit v1.2.3 From 5911e16cad61f1735c2f8c847dc43f03f8eaccd2 Mon Sep 17 00:00:00 2001 From: Anna-Maria Behnsen Date: Mon, 3 Mar 2025 12:11:16 +0100 Subject: time/namespace: Prepare introduction of struct vdso_clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To support multiple PTP clocks, the VDSO data structure needs to be reworked. All clock specific data will end up in struct vdso_clock and in struct vdso_time_data there will be array of VDSO clocks. At the moment, vdso_clock is simply a define which maps vdso_clock to vdso_time_data. To prepare for the rework of the data structures, replace the struct vdso_time_data pointer with a struct vdso_clock pointer where applicable. No functional change. Signed-off-by: Anna-Maria Behnsen Signed-off-by: Nam Cao Signed-off-by: Thomas Weißschuh Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20250303-vdso-clock-v1-14-c1b5c69a166f@linutronix.de --- kernel/time/namespace.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c index f02430a73be8..09bc4fb39f24 100644 --- a/kernel/time/namespace.c +++ b/kernel/time/namespace.c @@ -165,26 +165,26 @@ static struct timens_offset offset_from_ts(struct timespec64 off) * HVCLOCK * VVAR * - * The check for vdso_time_data->clock_mode is in the unlikely path of + * The check for vdso_clock->clock_mode is in the unlikely path of * the seq begin magic. So for the non-timens case most of the time * 'seq' is even, so the branch is not taken. * * If 'seq' is odd, i.e. a concurrent update is in progress, the extra check - * for vdso_time_data->clock_mode is a non-issue. The task is spin waiting for the + * for vdso_clock->clock_mode is a non-issue. The task is spin waiting for the * update to finish and for 'seq' to become even anyway. * - * Timens page has vdso_time_data->clock_mode set to VDSO_CLOCKMODE_TIMENS which + * Timens page has vdso_clock->clock_mode set to VDSO_CLOCKMODE_TIMENS which * enforces the time namespace handling path. */ -static void timens_setup_vdso_clock_data(struct vdso_time_data *vdata, +static void timens_setup_vdso_clock_data(struct vdso_clock *vc, struct time_namespace *ns) { - struct timens_offset *offset = vdata->offset; + struct timens_offset *offset = vc->offset; struct timens_offset monotonic = offset_from_ts(ns->offsets.monotonic); struct timens_offset boottime = offset_from_ts(ns->offsets.boottime); - vdata->seq = 1; - vdata->clock_mode = VDSO_CLOCKMODE_TIMENS; + vc->seq = 1; + vc->clock_mode = VDSO_CLOCKMODE_TIMENS; offset[CLOCK_MONOTONIC] = monotonic; offset[CLOCK_MONOTONIC_RAW] = monotonic; offset[CLOCK_MONOTONIC_COARSE] = monotonic; @@ -220,6 +220,7 @@ static void timens_set_vvar_page(struct task_struct *task, struct time_namespace *ns) { struct vdso_time_data *vdata; + struct vdso_clock *vc; unsigned int i; if (ns == &init_time_ns) @@ -236,9 +237,10 @@ static void timens_set_vvar_page(struct task_struct *task, ns->frozen_offsets = true; vdata = page_address(ns->vvar_page); + vc = vdata; for (i = 0; i < CS_BASES; i++) - timens_setup_vdso_clock_data(&vdata[i], ns); + timens_setup_vdso_clock_data(&vc[i], ns); out: mutex_unlock(&offset_lock); -- cgit v1.2.3 From 886653e36639177dd3ec2e7a4f0dc843d7def3f4 Mon Sep 17 00:00:00 2001 From: Anna-Maria Behnsen Date: Mon, 3 Mar 2025 12:11:21 +0100 Subject: vdso: Rework struct vdso_time_data and introduce struct vdso_clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To support multiple PTP clocks, the VDSO data structure needs to be reworked. All clock specific data will end up in struct vdso_clock and in struct vdso_time_data there will be an array of VDSO clocks. Now that all preparatory changes are in place: Split the clock related struct members into a separate struct vdso_clock. Make sure all users are aware, that vdso_time_data is no longer initialized as an array and vdso_clock is now the array inside vdso_data. Remove the vdso_clock define, which mapped it to vdso_time_data for the transition. Signed-off-by: Anna-Maria Behnsen Signed-off-by: Nam Cao Signed-off-by: Thomas Weißschuh Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/all/20250303-vdso-clock-v1-19-c1b5c69a166f@linutronix.de --- arch/arm64/include/asm/vdso/compat_gettimeofday.h | 2 +- arch/arm64/include/asm/vdso/vsyscall.h | 4 +- arch/s390/kernel/time.c | 11 ++--- include/asm-generic/vdso/vsyscall.h | 2 +- include/vdso/datapage.h | 55 +++++++++++++---------- include/vdso/helpers.h | 4 +- kernel/time/namespace.c | 2 +- kernel/time/vsyscall.c | 11 +++-- lib/vdso/datastore.c | 4 +- lib/vdso/gettimeofday.c | 16 +++---- 10 files changed, 57 insertions(+), 54 deletions(-) (limited to 'kernel') diff --git a/arch/arm64/include/asm/vdso/compat_gettimeofday.h b/arch/arm64/include/asm/vdso/compat_gettimeofday.h index 2c6b90d26bc8..d60ea7a72a9c 100644 --- a/arch/arm64/include/asm/vdso/compat_gettimeofday.h +++ b/arch/arm64/include/asm/vdso/compat_gettimeofday.h @@ -149,7 +149,7 @@ static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data( * where __aarch64_get_vdso_u_time_data() is called, and then keep the * result in a register. */ - asm volatile("mov %0, %1" : "=r"(ret) : "r"(vdso_u_time_data)); + asm volatile("mov %0, %1" : "=r"(ret) : "r"(&vdso_u_time_data)); return ret; } diff --git a/arch/arm64/include/asm/vdso/vsyscall.h b/arch/arm64/include/asm/vdso/vsyscall.h index 3f65cbd00635..de58951b8df6 100644 --- a/arch/arm64/include/asm/vdso/vsyscall.h +++ b/arch/arm64/include/asm/vdso/vsyscall.h @@ -15,8 +15,8 @@ static __always_inline void __arm64_update_vsyscall(struct vdso_time_data *vdata) { - vdata[CS_HRES_COARSE].mask = VDSO_PRECISION_MASK; - vdata[CS_RAW].mask = VDSO_PRECISION_MASK; + vdata->clock_data[CS_HRES_COARSE].mask = VDSO_PRECISION_MASK; + vdata->clock_data[CS_RAW].mask = VDSO_PRECISION_MASK; } #define __arch_update_vsyscall __arm64_update_vsyscall diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 41ca3586b19f..699a18f1c54e 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -79,12 +79,10 @@ void __init time_early_init(void) { struct ptff_qto qto; struct ptff_qui qui; - int cs; /* Initialize TOD steering parameters */ tod_steering_end = tod_clock_base.tod; - for (cs = 0; cs < CS_BASES; cs++) - vdso_k_time_data[cs].arch_data.tod_steering_end = tod_steering_end; + vdso_k_time_data->arch_data.tod_steering_end = tod_steering_end; if (!test_facility(28)) return; @@ -373,7 +371,6 @@ static void clock_sync_global(long delta) { unsigned long now, adj; struct ptff_qto qto; - int cs; /* Fixup the monotonic sched clock. */ tod_clock_base.eitod += delta; @@ -389,10 +386,8 @@ static void clock_sync_global(long delta) panic("TOD clock sync offset %li is too large to drift\n", tod_steering_delta); tod_steering_end = now + (abs(tod_steering_delta) << 15); - for (cs = 0; cs < CS_BASES; cs++) { - vdso_k_time_data[cs].arch_data.tod_steering_end = tod_steering_end; - vdso_k_time_data[cs].arch_data.tod_steering_delta = tod_steering_delta; - } + vdso_k_time_data->arch_data.tod_steering_end = tod_steering_end; + vdso_k_time_data->arch_data.tod_steering_delta = tod_steering_delta; /* Update LPAR offset. */ if (ptff_query(PTFF_QTO) && ptff(&qto, sizeof(qto), PTFF_QTO) == 0) diff --git a/include/asm-generic/vdso/vsyscall.h b/include/asm-generic/vdso/vsyscall.h index 1fb3000f5036..b550afa15ecd 100644 --- a/include/asm-generic/vdso/vsyscall.h +++ b/include/asm-generic/vdso/vsyscall.h @@ -9,7 +9,7 @@ #ifndef __arch_get_vdso_u_time_data static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data(void) { - return vdso_u_time_data; + return &vdso_u_time_data; } #endif diff --git a/include/vdso/datapage.h b/include/vdso/datapage.h index bcd19c223783..1864e76e8f69 100644 --- a/include/vdso/datapage.h +++ b/include/vdso/datapage.h @@ -69,9 +69,7 @@ struct vdso_timestamp { }; /** - * struct vdso_time_data - vdso datapage representation - * @arch_data: architecture specific data (optional, defaults - * to an empty struct) + * struct vdso_clock - vdso per clocksource datapage representation * @seq: timebase sequence counter * @clock_mode: clock mode * @cycle_last: timebase at clocksource init @@ -81,17 +79,9 @@ struct vdso_timestamp { * @shift: clocksource shift * @basetime[clock_id]: basetime per clock_id * @offset[clock_id]: time namespace offset per clock_id - * @tz_minuteswest: minutes west of Greenwich - * @tz_dsttime: type of DST correction - * @hrtimer_res: hrtimer resolution - * @__unused: unused * - * vdso_time_data will be accessed by 64 bit and compat code at the same time - * so we should be careful before modifying this structure. - * - * The ordering of the struct members is optimized to have fast access to the - * often required struct members which are related to CLOCK_REALTIME and - * CLOCK_MONOTONIC. This information is stored in the first cache lines. + * See also struct vdso_time_data for basic access and ordering information as + * struct vdso_clock is used there. * * @basetime is used to store the base time for the system wide time getter * VVAR page. @@ -104,9 +94,7 @@ struct vdso_timestamp { * For clocks which are not affected by time namespace adjustment the * offset must be zero. */ -struct vdso_time_data { - struct arch_vdso_time_data arch_data; - +struct vdso_clock { u32 seq; s32 clock_mode; @@ -122,14 +110,35 @@ struct vdso_time_data { struct vdso_timestamp basetime[VDSO_BASES]; struct timens_offset offset[VDSO_BASES]; }; +}; - s32 tz_minuteswest; - s32 tz_dsttime; - u32 hrtimer_res; - u32 __unused; -} ____cacheline_aligned; +/** + * struct vdso_time_data - vdso datapage representation + * @arch_data: architecture specific data (optional, defaults + * to an empty struct) + * @clock_data: clocksource related data (array) + * @tz_minuteswest: minutes west of Greenwich + * @tz_dsttime: type of DST correction + * @hrtimer_res: hrtimer resolution + * @__unused: unused + * + * vdso_time_data will be accessed by 64 bit and compat code at the same time + * so we should be careful before modifying this structure. + * + * The ordering of the struct members is optimized to have fast acces to the + * often required struct members which are related to CLOCK_REALTIME and + * CLOCK_MONOTONIC. This information is stored in the first cache lines. + */ +struct vdso_time_data { + struct arch_vdso_time_data arch_data; -#define vdso_clock vdso_time_data + struct vdso_clock clock_data[CS_BASES]; + + s32 tz_minuteswest; + s32 tz_dsttime; + u32 hrtimer_res; + u32 __unused; +} ____cacheline_aligned; /** * struct vdso_rng_data - vdso RNG state information @@ -151,7 +160,7 @@ struct vdso_rng_data { * relocation, and this is what we need. */ #ifdef CONFIG_GENERIC_VDSO_DATA_STORE -extern struct vdso_time_data vdso_u_time_data[CS_BASES] __attribute__((visibility("hidden"))); +extern struct vdso_time_data vdso_u_time_data __attribute__((visibility("hidden"))); extern struct vdso_rng_data vdso_u_rng_data __attribute__((visibility("hidden"))); extern struct vdso_arch_data vdso_u_arch_data __attribute__((visibility("hidden"))); diff --git a/include/vdso/helpers.h b/include/vdso/helpers.h index 28f0707a46c6..0a98fed550ba 100644 --- a/include/vdso/helpers.h +++ b/include/vdso/helpers.h @@ -30,7 +30,7 @@ static __always_inline u32 vdso_read_retry(const struct vdso_clock *vc, static __always_inline void vdso_write_begin(struct vdso_time_data *vd) { - struct vdso_clock *vc = vd; + struct vdso_clock *vc = vd->clock_data; /* * WRITE_ONCE() is required otherwise the compiler can validly tear @@ -44,7 +44,7 @@ static __always_inline void vdso_write_begin(struct vdso_time_data *vd) static __always_inline void vdso_write_end(struct vdso_time_data *vd) { - struct vdso_clock *vc = vd; + struct vdso_clock *vc = vd->clock_data; smp_wmb(); /* diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c index 09bc4fb39f24..e3642278df43 100644 --- a/kernel/time/namespace.c +++ b/kernel/time/namespace.c @@ -237,7 +237,7 @@ static void timens_set_vvar_page(struct task_struct *task, ns->frozen_offsets = true; vdata = page_address(ns->vvar_page); - vc = vdata; + vc = vdata->clock_data; for (i = 0; i < CS_BASES; i++) timens_setup_vdso_clock_data(&vc[i], ns); diff --git a/kernel/time/vsyscall.c b/kernel/time/vsyscall.c index dd85b41a70be..01c2ab1e8971 100644 --- a/kernel/time/vsyscall.c +++ b/kernel/time/vsyscall.c @@ -17,8 +17,8 @@ static inline void update_vdso_time_data(struct vdso_time_data *vdata, struct timekeeper *tk) { + struct vdso_clock *vc = vdata->clock_data; struct vdso_timestamp *vdso_ts; - struct vdso_clock *vc = vdata; u64 nsec, sec; vc[CS_HRES_COARSE].cycle_last = tk->tkr_mono.cycle_last; @@ -78,8 +78,8 @@ static inline void update_vdso_time_data(struct vdso_time_data *vdata, struct ti void update_vsyscall(struct timekeeper *tk) { struct vdso_time_data *vdata = vdso_k_time_data; + struct vdso_clock *vc = vdata->clock_data; struct vdso_timestamp *vdso_ts; - struct vdso_clock *vc = vdata; s32 clock_mode; u64 nsec; @@ -109,9 +109,8 @@ void update_vsyscall(struct timekeeper *tk) /* * Read without the seqlock held by clock_getres(). - * Note: No need to have a second copy. */ - WRITE_ONCE(vdata[CS_HRES_COARSE].hrtimer_res, hrtimer_resolution); + WRITE_ONCE(vdata->hrtimer_res, hrtimer_resolution); /* * If the current clocksource is not VDSO capable, then spare the @@ -131,8 +130,8 @@ void update_vsyscall_tz(void) { struct vdso_time_data *vdata = vdso_k_time_data; - vdata[CS_HRES_COARSE].tz_minuteswest = sys_tz.tz_minuteswest; - vdata[CS_HRES_COARSE].tz_dsttime = sys_tz.tz_dsttime; + vdata->tz_minuteswest = sys_tz.tz_minuteswest; + vdata->tz_dsttime = sys_tz.tz_dsttime; __arch_sync_vdso_time_data(vdata); } diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c index 4e350f56ace3..c715e217ec65 100644 --- a/lib/vdso/datastore.c +++ b/lib/vdso/datastore.c @@ -13,10 +13,10 @@ */ #ifdef CONFIG_HAVE_GENERIC_VDSO static union { - struct vdso_time_data data[CS_BASES]; + struct vdso_time_data data; u8 page[PAGE_SIZE]; } vdso_time_data_store __page_aligned_data; -struct vdso_time_data *vdso_k_time_data = vdso_time_data_store.data; +struct vdso_time_data *vdso_k_time_data = &vdso_time_data_store.data; static_assert(sizeof(vdso_time_data_store) == PAGE_SIZE); #endif /* CONFIG_HAVE_GENERIC_VDSO */ diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index c6ff69345586..93ef801a97ef 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -87,8 +87,8 @@ int do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *v { const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); const struct timens_offset *offs = &vcns->offset[clk]; + const struct vdso_clock *vc = vd->clock_data; const struct vdso_timestamp *vdso_ts; - const struct vdso_clock *vc = vd; u64 cycles, ns; u32 seq; s64 sec; @@ -199,8 +199,8 @@ int do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock { const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); const struct timens_offset *offs = &vcns->offset[clk]; + const struct vdso_clock *vc = vd->clock_data; const struct vdso_timestamp *vdso_ts; - const struct vdso_clock *vc = vd; u64 nsec; s64 sec; s32 seq; @@ -265,7 +265,7 @@ static __always_inline int __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *ts) { - const struct vdso_clock *vc = vd; + const struct vdso_clock *vc = vd->clock_data; u32 msk; /* Check for negative values or invalid clocks */ @@ -337,7 +337,7 @@ static __maybe_unused int __cvdso_gettimeofday_data(const struct vdso_time_data *vd, struct __kernel_old_timeval *tv, struct timezone *tz) { - const struct vdso_clock *vc = vd; + const struct vdso_clock *vc = vd->clock_data; if (likely(tv != NULL)) { struct __kernel_timespec ts; @@ -371,13 +371,13 @@ __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) static __maybe_unused __kernel_old_time_t __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time) { - const struct vdso_clock *vc = vd; + const struct vdso_clock *vc = vd->clock_data; __kernel_old_time_t t; if (IS_ENABLED(CONFIG_TIME_NS) && vc->clock_mode == VDSO_CLOCKMODE_TIMENS) { vd = __arch_get_vdso_u_timens_data(vd); - vc = vd; + vc = vd->clock_data; } t = READ_ONCE(vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); @@ -399,7 +399,7 @@ static __maybe_unused int __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *res) { - const struct vdso_clock *vc = vd; + const struct vdso_clock *vc = vd->clock_data; u32 msk; u64 ns; @@ -420,7 +420,7 @@ int __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock /* * Preserves the behaviour of posix_get_hrtimer_res(). */ - ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); + ns = READ_ONCE(vd->hrtimer_res); } else if (msk & VDSO_COARSE) { /* * Preserves the behaviour of posix_get_coarse_res(). -- cgit v1.2.3