diff options
| author | Stephen Hemminger <shemminger@osdl.org> | 2003-02-04 23:25:27 -0800 |
|---|---|---|
| committer | Linus Torvalds <torvalds@penguin.transmeta.com> | 2003-02-04 23:25:27 -0800 |
| commit | bb59cfa4c9113214f91fa0ce744fd92fe2745039 (patch) | |
| tree | 392951d646a403765ff56681fa3a6f5ae99815ed | |
| parent | 62672619d8b2203538f37c05ca167b9a8b3f94d4 (diff) | |
[PATCH] seqlock for xtime
Add "seqlock" infrastructure for doing low-overhead optimistic reader
locks (writer increments a sequence number, reader verifies that no
writers came in during the critical region, and lots of careful memory
barriers to take care of business).
Make xtime/get_jiffies_64() use this new locking.
40 files changed, 701 insertions, 496 deletions
diff --git a/arch/alpha/kernel/time.c b/arch/alpha/kernel/time.c index 1525d25de0e3..0d56b55e52e6 100644 --- a/arch/alpha/kernel/time.c +++ b/arch/alpha/kernel/time.c @@ -44,6 +44,7 @@ #include <asm/hwrpb.h> #include <linux/mc146818rtc.h> +#include <linux/time.h> #include <linux/timex.h> #include "proto.h" @@ -51,7 +52,6 @@ u64 jiffies_64; -extern rwlock_t xtime_lock; extern unsigned long wall_jiffies; /* kernel/timer.c */ static int set_rtc_mmss(unsigned long); @@ -106,7 +106,7 @@ void timer_interrupt(int irq, void *dev, struct pt_regs * regs) alpha_do_profile(regs->pc); #endif - write_lock(&xtime_lock); + write_seqlock(&xtime_lock); /* * Calculate how many ticks have passed since the last update, @@ -138,7 +138,7 @@ void timer_interrupt(int irq, void *dev, struct pt_regs * regs) state.last_rtc_update = xtime.tv_sec - (tmp ? 600 : 0); } - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); } void @@ -395,18 +395,20 @@ time_init(void) void do_gettimeofday(struct timeval *tv) { - unsigned long sec, usec, lost, flags; + unsigned long flags; + unsigned long sec, usec, lost, seq; unsigned long delta_cycles, delta_usec, partial_tick; - read_lock_irqsave(&xtime_lock, flags); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); - delta_cycles = rpcc() - state.last_time; - sec = xtime.tv_sec; - usec = (xtime.tv_nsec / 1000); - partial_tick = state.partial_tick; - lost = jiffies - wall_jiffies; + delta_cycles = rpcc() - state.last_time; + sec = xtime.tv_sec; + usec = (xtime.tv_nsec / 1000); + partial_tick = state.partial_tick; + lost = jiffies - wall_jiffies; - read_unlock_irqrestore(&xtime_lock, flags); + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); #ifdef CONFIG_SMP /* Until and unless we figure out how to get cpu cycle counters @@ -448,7 +450,7 @@ do_settimeofday(struct timeval *tv) unsigned long delta_usec; long sec, usec; - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); /* The offset that is added into time in do_gettimeofday above must be subtracted out here to keep a coherent view of the @@ -479,7 +481,7 @@ do_settimeofday(struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } diff --git a/arch/arm/kernel/time.c b/arch/arm/kernel/time.c index 685d59b08623..1cb54c5d5204 100644 --- a/arch/arm/kernel/time.c +++ b/arch/arm/kernel/time.c @@ -34,7 +34,6 @@ u64 jiffies_64; -extern rwlock_t xtime_lock; extern unsigned long wall_jiffies; /* this needs a better home */ @@ -148,18 +147,20 @@ static void do_leds(void) void do_gettimeofday(struct timeval *tv) { unsigned long flags; + unsigned long seq; unsigned long usec, sec, lost; - read_lock_irqsave(&xtime_lock, flags); - usec = gettimeoffset(); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); + usec = gettimeoffset(); - lost = jiffies - wall_jiffies; - if (lost) - usec += lost * USECS_PER_JIFFY; + lost = jiffies - wall_jiffies; + if (lost) + usec += lost * USECS_PER_JIFFY; - sec = xtime.tv_sec; - usec += xtime.tv_nsec / 1000; - read_unlock_irqrestore(&xtime_lock, flags); + sec = xtime.tv_sec; + usec += xtime.tv_nsec / 1000; + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); /* usec may have gone up a lot: be safe */ while (usec >= 1000000) { @@ -173,7 +174,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); /* * This is revolting. We need to set "xtime" correctly. However, the * value in this location is the value at the most recent update of @@ -194,7 +195,7 @@ void do_settimeofday(struct timeval *tv) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } static struct irqaction timer_irq = { diff --git a/arch/i386/kernel/apm.c b/arch/i386/kernel/apm.c index 2fc083a98278..1f5a243376b7 100644 --- a/arch/i386/kernel/apm.c +++ b/arch/i386/kernel/apm.c @@ -215,6 +215,7 @@ #include <linux/miscdevice.h> #include <linux/apm_bios.h> #include <linux/init.h> +#include <linux/time.h> #include <linux/sched.h> #include <linux/pm.h> #include <linux/kernel.h> @@ -227,7 +228,6 @@ #include <linux/sysrq.h> -extern rwlock_t xtime_lock; extern spinlock_t i8253_lock; extern unsigned long get_cmos_time(void); extern void machine_real_restart(unsigned char *, int); @@ -1264,7 +1264,7 @@ static int suspend(int vetoable) printk(KERN_CRIT "apm: suspend was vetoed, but suspending anyway.\n"); } /* serialize with the timer interrupt */ - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); /* protect against access to timer chip registers */ spin_lock(&i8253_lock); @@ -1276,7 +1276,7 @@ static int suspend(int vetoable) ignore_normal_resume = 1; spin_unlock(&i8253_lock); - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); if (err == APM_NO_ERROR) err = APM_SUCCESS; @@ -1301,10 +1301,10 @@ static void standby(void) int err; /* serialize with the timer interrupt */ - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); /* If needed, notify drivers here */ get_time_diff(); - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); err = set_system_power_state(APM_STATE_STANDBY); if ((err != APM_SUCCESS) && (err != APM_NO_ERROR)) @@ -1393,9 +1393,9 @@ static void check_events(void) ignore_bounce = 1; if ((event != APM_NORMAL_RESUME) || (ignore_normal_resume == 0)) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); set_time(); - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); pm_send_all(PM_RESUME, (void *)0); queue_event(event, NULL); } @@ -1410,9 +1410,9 @@ static void check_events(void) break; case APM_UPDATE_TIME: - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); set_time(); - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); break; case APM_CRITICAL_SUSPEND: diff --git a/arch/i386/kernel/i386_ksyms.c b/arch/i386/kernel/i386_ksyms.c index 22eeea87c47d..90f14e4c3b31 100644 --- a/arch/i386/kernel/i386_ksyms.c +++ b/arch/i386/kernel/i386_ksyms.c @@ -15,6 +15,7 @@ #include <linux/string.h> #include <linux/tty.h> #include <linux/highmem.h> +#include <linux/time.h> #include <asm/semaphore.h> #include <asm/processor.h> diff --git a/arch/i386/kernel/time.c b/arch/i386/kernel/time.c index 5a8183a31eea..82033a081870 100644 --- a/arch/i386/kernel/time.c +++ b/arch/i386/kernel/time.c @@ -70,7 +70,6 @@ u64 jiffies_64; unsigned long cpu_khz; /* Detected as we calibrate the TSC */ -extern rwlock_t xtime_lock; extern unsigned long wall_jiffies; spinlock_t rtc_lock = SPIN_LOCK_UNLOCKED; @@ -87,19 +86,21 @@ struct timer_opts* timer = &timer_none; */ void do_gettimeofday(struct timeval *tv) { - unsigned long flags; + unsigned long seq; unsigned long usec, sec; - read_lock_irqsave(&xtime_lock, flags); - usec = timer->get_offset(); - { - unsigned long lost = jiffies - wall_jiffies; - if (lost) - usec += lost * (1000000 / HZ); - } - sec = xtime.tv_sec; - usec += (xtime.tv_nsec / 1000); - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = read_seqbegin(&xtime_lock); + + usec = timer->get_offset(); + { + unsigned long lost = jiffies - wall_jiffies; + if (lost) + usec += lost * (1000000 / HZ); + } + sec = xtime.tv_sec; + usec += (xtime.tv_nsec / 1000); + } while (read_seqretry(&xtime_lock, seq)); while (usec >= 1000000) { usec -= 1000000; @@ -112,7 +113,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); /* * This is revolting. We need to set "xtime" correctly. However, the * value in this location is the value at the most recent update of @@ -133,7 +134,7 @@ void do_settimeofday(struct timeval *tv) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } /* @@ -314,14 +315,14 @@ void timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) * the irq version of write_lock because as just said we have irq * locally disabled. -arca */ - write_lock(&xtime_lock); + write_seqlock(&xtime_lock); detect_lost_tick(); timer->mark_offset(); do_timer_interrupt(irq, NULL, regs); - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); } diff --git a/arch/i386/kernel/timers/timer_pit.c b/arch/i386/kernel/timers/timer_pit.c index a9e75a0fa006..8b9507e70e05 100644 --- a/arch/i386/kernel/timers/timer_pit.c +++ b/arch/i386/kernel/timers/timer_pit.c @@ -76,7 +76,7 @@ static void delay_pit(unsigned long loops) static unsigned long get_offset_pit(void) { int count; - + unsigned long flags; static int count_p = LATCH; /* for the first call after boot */ static unsigned long jiffies_p = 0; @@ -85,8 +85,7 @@ static unsigned long get_offset_pit(void) */ unsigned long jiffies_t; - /* gets recalled with irq locally disabled */ - spin_lock(&i8253_lock); + spin_lock_irqsave(&i8253_lock, flags); /* timer count may underflow right here */ outb_p(0x00, 0x43); /* latch the count ASAP */ @@ -108,7 +107,7 @@ static unsigned long get_offset_pit(void) count = LATCH - 1; } - spin_unlock(&i8253_lock); + spin_unlock_irqrestore(&i8253_lock, flags); /* * avoiding timer inconsistencies (they are rare, but they happen)... diff --git a/arch/ia64/kernel/time.c b/arch/ia64/kernel/time.c index 4e6c6268703d..101e53cef7de 100644 --- a/arch/ia64/kernel/time.c +++ b/arch/ia64/kernel/time.c @@ -24,7 +24,6 @@ #include <asm/sal.h> #include <asm/system.h> -extern rwlock_t xtime_lock; extern unsigned long wall_jiffies; extern unsigned long last_time_offset; @@ -89,7 +88,7 @@ gettimeoffset (void) void do_settimeofday (struct timeval *tv) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); { /* * This is revolting. We need to set "xtime" correctly. However, the value @@ -112,21 +111,21 @@ do_settimeofday (struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; } - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } void do_gettimeofday (struct timeval *tv) { - unsigned long flags, usec, sec, old; + unsigned long seq, usec, sec, old; - read_lock_irqsave(&xtime_lock, flags); - { + do { + seq = read_seqbegin(&xtime_lock); usec = gettimeoffset(); /* - * Ensure time never goes backwards, even when ITC on different CPUs are - * not perfectly synchronized. + * Ensure time never goes backwards, even when ITC on + * different CPUs are not perfectly synchronized. */ do { old = last_time_offset; @@ -138,8 +137,8 @@ do_gettimeofday (struct timeval *tv) sec = xtime.tv_sec; usec += xtime.tv_nsec / 1000; - } - read_unlock_irqrestore(&xtime_lock, flags); + } while (read_seqend(&xtime_lock, seq)); + while (usec >= 1000000) { usec -= 1000000; @@ -182,10 +181,10 @@ timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) * another CPU. We need to avoid to SMP race by acquiring the * xtime_lock. */ - write_lock(&xtime_lock); + write_seqlock(&xtime_lock); do_timer(regs); local_cpu_data->itm_next = new_itm; - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); } else local_cpu_data->itm_next = new_itm; diff --git a/arch/m68k/kernel/time.c b/arch/m68k/kernel/time.c index 3365e229d40d..03a38781e59e 100644 --- a/arch/m68k/kernel/time.c +++ b/arch/m68k/kernel/time.c @@ -22,6 +22,7 @@ #include <asm/machdep.h> #include <asm/io.h> +#include <linux/time.h> #include <linux/timex.h> #include <linux/profile.h> @@ -129,25 +130,27 @@ void time_init(void) mach_sched_init(timer_interrupt); } -extern rwlock_t xtime_lock; - /* * This version of gettimeofday has near microsecond resolution. */ void do_gettimeofday(struct timeval *tv) { - extern unsigned long wall_jiffies; unsigned long flags; + extern unsigned long wall_jiffies; + unsigned long seq; unsigned long usec, sec, lost; - read_lock_irqsave(&xtime_lock, flags); - usec = mach_gettimeoffset(); - lost = jiffies - wall_jiffies; - if (lost) - usec += lost * (1000000/HZ); - sec = xtime.tv_sec; - usec += xtime.tv_nsec/1000; - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); + + usec = mach_gettimeoffset(); + lost = jiffies - wall_jiffies; + if (lost) + usec += lost * (1000000/HZ); + sec = xtime.tv_sec; + usec += xtime.tv_nsec/1000; + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); + while (usec >= 1000000) { usec -= 1000000; @@ -162,7 +165,7 @@ void do_settimeofday(struct timeval *tv) { extern unsigned long wall_jiffies; - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); /* This is revolting. We need to set the xtime.tv_nsec * correctly. However, the value in this location is * is value at the last tick. @@ -183,5 +186,5 @@ void do_settimeofday(struct timeval *tv) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } diff --git a/arch/m68knommu/kernel/time.c b/arch/m68knommu/kernel/time.c index 1be752c65014..c12fb4af61fc 100644 --- a/arch/m68knommu/kernel/time.c +++ b/arch/m68knommu/kernel/time.c @@ -18,6 +18,7 @@ #include <linux/string.h> #include <linux/mm.h> #include <linux/profile.h> +#include <linux/time.h> #include <linux/timex.h> #include <asm/machdep.h> @@ -126,21 +127,23 @@ void time_init(void) mach_sched_init(timer_interrupt); } -extern rwlock_t xtime_lock; - /* * This version of gettimeofday has near microsecond resolution. */ void do_gettimeofday(struct timeval *tv) { unsigned long flags; + unsigned long seq; unsigned long usec, sec; - read_lock_irqsave(&xtime_lock, flags); - usec = mach_gettimeoffset ? mach_gettimeoffset() : 0; - sec = xtime.tv_sec; - usec += (xtime.tv_nsec / 1000); - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); + + usec = mach_gettimeoffset ? mach_gettimeoffset() : 0; + sec = xtime.tv_sec; + usec += (xtime.tv_nsec / 1000); + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); + while (usec >= 1000000) { usec -= 1000000; @@ -153,7 +156,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); /* This is revolting. We need to set the xtime.tv_usec * correctly. However, the value in this location is * is value at the last tick. @@ -174,5 +177,5 @@ void do_settimeofday(struct timeval *tv) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } diff --git a/arch/mips/au1000/common/time.c b/arch/mips/au1000/common/time.c index e57c446769c2..cbd1d77b306e 100644 --- a/arch/mips/au1000/common/time.c +++ b/arch/mips/au1000/common/time.c @@ -27,6 +27,7 @@ #include <linux/config.h> #include <linux/init.h> #include <linux/kernel_stat.h> +#include <linux/time.h> #include <linux/sched.h> #include <linux/spinlock.h> @@ -44,7 +45,6 @@ unsigned long uart_baud_base; static unsigned long r4k_offset; /* Amount to increment compare reg each time */ static unsigned long r4k_cur; /* What counter should be at next timer irq */ -extern rwlock_t xtime_lock; #define ALLINTS (IE_IRQ0 | IE_IRQ1 | IE_IRQ2 | IE_IRQ3 | IE_IRQ4 | IE_IRQ5) @@ -150,10 +150,10 @@ void __init time_init(void) set_cp0_status(ALLINTS); /* Read time from the RTC chipset. */ - write_lock_irqsave (&xtime_lock, flags); + write_seqlock_irqsave (&xtime_lock, flags); xtime.tv_sec = get_mips_time(); xtime.tv_usec = 0; - write_unlock_irqrestore(&xtime_lock, flags); + write_sequnlock_irqrestore(&xtime_lock, flags); } /* This is for machines which generate the exact clock. */ @@ -229,20 +229,24 @@ static unsigned long do_fast_gettimeoffset(void) void do_gettimeofday(struct timeval *tv) { - unsigned int flags; + unsigned long flags; + unsigned long seq; - read_lock_irqsave (&xtime_lock, flags); - *tv = xtime; - tv->tv_usec += do_fast_gettimeoffset(); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); - /* - * xtime is atomically updated in timer_bh. jiffies - wall_jiffies - * is nonzero if the timer bottom half hasnt executed yet. - */ - if (jiffies - wall_jiffies) - tv->tv_usec += USECS_PER_JIFFY; + *tv = xtime; + tv->tv_usec += do_fast_gettimeoffset(); + + /* + * xtime is atomically updated in timer_bh. jiffies - wall_jiffies + * is nonzero if the timer bottom half hasnt executed yet. + */ + if (jiffies - wall_jiffies) + tv->tv_usec += USECS_PER_JIFFY; + + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); - read_unlock_irqrestore (&xtime_lock, flags); if (tv->tv_usec >= 1000000) { tv->tv_usec -= 1000000; @@ -252,7 +256,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq (&xtime_lock); + write_seqlock_irq (&xtime_lock); /* This is revolting. We need to set the xtime.tv_usec correctly. * However, the value in this location is is value at the last tick. @@ -272,7 +276,7 @@ void do_settimeofday(struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq (&xtime_lock); + write_sequnlock_irq (&xtime_lock); } /* diff --git a/arch/mips/baget/time.c b/arch/mips/baget/time.c index 1c1c9840fa57..f792d5c425d6 100644 --- a/arch/mips/baget/time.c +++ b/arch/mips/baget/time.c @@ -11,6 +11,7 @@ #include <linux/param.h> #include <linux/string.h> #include <linux/mm.h> +#include <linux/time.h> #include <linux/interrupt.h> #include <linux/timex.h> #include <linux/spinlock.h> @@ -23,8 +24,6 @@ #include <asm/baget/baget.h> -extern rwlock_t xtime_lock; - /* * To have precision clock, we need to fix available clock frequency */ @@ -79,20 +78,21 @@ void __init time_init(void) void do_gettimeofday(struct timeval *tv) { - unsigned long flags; + unsigned long seq; - read_lock_irqsave (&xtime_lock, flags); - *tv = xtime; - read_unlock_irqrestore (&xtime_lock, flags); + do { + seq = read_seqbegin(&xtime_lock); + *tv = xtime; + } while (read_seqretry(&xtime_lock, seq)); } void do_settimeofday(struct timeval *tv) { - write_lock_irq (&xtime_lock); + write_seqlock_irq (&xtime_lock); xtime = *tv; time_adjust = 0; /* stop active adjtime() */ time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq (&xtime_lock); + write_sequnlock_irq (&xtime_lock); } diff --git a/arch/mips/dec/time.c b/arch/mips/dec/time.c index 56f06e90f72f..1c9a80121e10 100644 --- a/arch/mips/dec/time.c +++ b/arch/mips/dec/time.c @@ -15,6 +15,7 @@ #include <linux/param.h> #include <linux/string.h> #include <linux/mm.h> +#include <linux/time.h> #include <linux/interrupt.h> #include <linux/bcd.h> @@ -35,7 +36,6 @@ extern void (*board_time_init)(struct irqaction *irq); extern volatile unsigned long wall_jiffies; -extern rwlock_t xtime_lock; /* * Change this if you have some constant time drift @@ -211,19 +211,22 @@ static unsigned long (*do_gettimeoffset) (void) = do_slow_gettimeoffset; void do_gettimeofday(struct timeval *tv) { unsigned long flags; + unsigned long seq; - read_lock_irqsave(&xtime_lock, flags); - *tv = xtime; - tv->tv_usec += do_gettimeoffset(); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); + *tv = xtime; + tv->tv_usec += do_gettimeoffset(); - /* - * xtime is atomically updated in timer_bh. jiffies - wall_jiffies - * is nonzero if the timer bottom half hasnt executed yet. - */ - if (jiffies - wall_jiffies) - tv->tv_usec += USECS_PER_JIFFY; + /* + * xtime is atomically updated in timer_bh. jiffies - wall_jiffies + * is nonzero if the timer bottom half hasnt executed yet. + */ + if (jiffies - wall_jiffies) + tv->tv_usec += USECS_PER_JIFFY; + + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); - read_unlock_irqrestore(&xtime_lock, flags); if (tv->tv_usec >= 1000000) { tv->tv_usec -= 1000000; @@ -233,7 +236,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); /* This is revolting. We need to set the xtime.tv_usec * correctly. However, the value in this location is @@ -254,7 +257,7 @@ void do_settimeofday(struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } /* @@ -330,6 +333,7 @@ static inline void timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) { volatile unsigned char dummy; + unsigned long seq; dummy = CMOS_READ(RTC_REG_C); /* ACK RTC Interrupt */ @@ -357,23 +361,27 @@ timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) * CMOS clock accordingly every ~11 minutes. Set_rtc_mmss() has to be * called as close as possible to 500 ms before the new second starts. */ - read_lock(&xtime_lock); - if ((time_status & STA_UNSYNC) == 0 - && xtime.tv_sec > last_rtc_update + 660 - && xtime.tv_usec >= 500000 - tick / 2 - && xtime.tv_usec <= 500000 + tick / 2) { - if (set_rtc_mmss(xtime.tv_sec) == 0) - last_rtc_update = xtime.tv_sec; - else - /* do it again in 60 s */ - last_rtc_update = xtime.tv_sec - 600; - } + do { + seq = read_seqbegin(&xtime_lock); + + if ((time_status & STA_UNSYNC) == 0 + && xtime.tv_sec > last_rtc_update + 660 + && xtime.tv_usec >= 500000 - tick / 2 + && xtime.tv_usec <= 500000 + tick / 2) { + if (set_rtc_mmss(xtime.tv_sec) == 0) + last_rtc_update = xtime.tv_sec; + else + /* do it again in 60 s */ + last_rtc_update = xtime.tv_sec - 600; + } + } while (read_seqretry(&xtime_lock, seq)); + /* As we return to user mode fire off the other CPU schedulers.. this is basically because we don't yet share IRQ's around. This message is rigged to be safe on the 386 - basically it's a hack, so don't look closely for now.. */ /*smp_message_pass(MSG_ALL_BUT_SELF, MSG_RESCHEDULE, 0L, 0); */ - read_unlock(&xtime_lock); + } static void r4k_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) @@ -470,10 +478,10 @@ void __init time_init(void) real_year = CMOS_READ(RTC_DEC_YEAR); year += real_year - 72 + 2000; - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); xtime.tv_sec = mktime(year, mon, day, hour, min, sec); xtime.tv_usec = 0; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); if (mips_cpu.options & MIPS_CPU_COUNTER) { write_32bit_cp0_register(CP0_COUNT, 0); diff --git a/arch/mips/ite-boards/generic/time.c b/arch/mips/ite-boards/generic/time.c index 600b4cef1afa..d6081f41fc01 100644 --- a/arch/mips/ite-boards/generic/time.c +++ b/arch/mips/ite-boards/generic/time.c @@ -27,6 +27,7 @@ #include <linux/init.h> #include <linux/kernel_stat.h> #include <linux/sched.h> +#include <linux/time.h> #include <linux/spinlock.h> #include <asm/mipsregs.h> @@ -38,7 +39,6 @@ extern void enable_cpu_timer(void); extern volatile unsigned long wall_jiffies; -extern rwlock_t xtime_lock; unsigned long missed_heart_beats = 0; static long last_rtc_update = 0; @@ -119,6 +119,8 @@ static int set_rtc_mmss(unsigned long nowtime) */ void mips_timer_interrupt(struct pt_regs *regs) { + unsigned long seq; + if (r4k_offset == 0) goto null; @@ -133,18 +135,22 @@ void mips_timer_interrupt(struct pt_regs *regs) * within 500ms before the * next second starts, * thus the following code. */ - read_lock(&xtime_lock); - if ((time_status & STA_UNSYNC) == 0 - && xtime.tv_sec > last_rtc_update + 660 - && xtime.tv_usec >= 500000 - (tick >> 1) - && xtime.tv_usec <= 500000 + (tick >> 1)) - if (set_rtc_mmss(xtime.tv_sec) == 0) - last_rtc_update = xtime.tv_sec; - else { - /* do it again in 60 s */ - last_rtc_update = xtime.tv_sec - 600; - } - read_unlock(&xtime_lock); + do { + seq = read_seqbegin(&xtime_lock); + + + if ((time_status & STA_UNSYNC) == 0 + && xtime.tv_sec > last_rtc_update + 660 + && xtime.tv_usec >= 500000 - (tick >> 1) + && xtime.tv_usec <= 500000 + (tick >> 1)) + if (set_rtc_mmss(xtime.tv_sec) == 0) + last_rtc_update = xtime.tv_sec; + else { + /* do it again in 60 s */ + last_rtc_update = xtime.tv_sec - 600; + } + + } while (read_seqretry(&xtime_lock, seq)); r4k_cur += r4k_offset; ack_r4ktimer(r4k_cur); @@ -247,10 +253,10 @@ void __init time_init(void) enable_cpu_timer(); /* Read time from the RTC chipset. */ - write_lock_irqsave (&xtime_lock, flags); + write_seqlock_irqsave (&xtime_lock, flags); xtime.tv_sec = get_mips_time(); xtime.tv_usec = 0; - write_unlock_irqrestore(&xtime_lock, flags); + write_sequnlock_irqrestore(&xtime_lock, flags); } /* This is for machines which generate the exact clock. */ @@ -332,20 +338,24 @@ static unsigned long do_fast_gettimeoffset(void) void do_gettimeofday(struct timeval *tv) { - unsigned int flags; + unsigned long flags; + unsigned int seq; + + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); - read_lock_irqsave (&xtime_lock, flags); - *tv = xtime; - tv->tv_usec += do_fast_gettimeoffset(); + *tv = xtime; + tv->tv_usec += do_fast_gettimeoffset(); - /* - * xtime is atomically updated in timer_bh. jiffies - wall_jiffies - * is nonzero if the timer bottom half hasnt executed yet. - */ - if (jiffies - wall_jiffies) - tv->tv_usec += USECS_PER_JIFFY; + /* + * xtime is atomically updated in timer_bh. + * jiffies - wall_jiffies + * is nonzero if the timer bottom half hasnt executed yet. + */ + if (jiffies - wall_jiffies) + tv->tv_usec += USECS_PER_JIFFY; - read_unlock_irqrestore (&xtime_lock, flags); + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); if (tv->tv_usec >= 1000000) { tv->tv_usec -= 1000000; @@ -355,7 +365,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq (&xtime_lock); + write_seqlock_irq (&xtime_lock); /* This is revolting. We need to set the xtime.tv_usec correctly. * However, the value in this location is is value at the last tick. @@ -375,5 +385,5 @@ void do_settimeofday(struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq (&xtime_lock); + write_sequnlock_irq (&xtime_lock); } diff --git a/arch/mips/kernel/sysirix.c b/arch/mips/kernel/sysirix.c index 4c319482e1b8..7bdf103d4284 100644 --- a/arch/mips/kernel/sysirix.c +++ b/arch/mips/kernel/sysirix.c @@ -13,6 +13,7 @@ #include <linux/slab.h> #include <linux/swap.h> #include <linux/errno.h> +#include <linux/time.h> #include <linux/timex.h> #include <linux/times.h> #include <linux/elf.h> @@ -615,19 +616,17 @@ asmlinkage int irix_getgid(struct pt_regs *regs) return current->gid; } -extern rwlock_t xtime_lock; - asmlinkage int irix_stime(int value) { if (!capable(CAP_SYS_TIME)) return -EPERM; - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); xtime.tv_sec = value; xtime.tv_usec = 0; time_maxerror = MAXPHASE; time_esterror = MAXPHASE; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); return 0; } diff --git a/arch/mips/kernel/time.c b/arch/mips/kernel/time.c index ed347b400ee6..6adf09fdf6ad 100644 --- a/arch/mips/kernel/time.c +++ b/arch/mips/kernel/time.c @@ -37,7 +37,6 @@ u64 jiffies_64; /* * forward reference */ -extern rwlock_t xtime_lock; extern volatile unsigned long wall_jiffies; /* @@ -63,19 +62,23 @@ int (*rtc_set_time)(unsigned long) = null_rtc_set_time; void do_gettimeofday(struct timeval *tv) { unsigned long flags; + unsigned long seq; - read_lock_irqsave (&xtime_lock, flags); - *tv = xtime; - tv->tv_usec += do_gettimeoffset(); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); - /* - * xtime is atomically updated in timer_bh. jiffies - wall_jiffies - * is nonzero if the timer bottom half hasnt executed yet. - */ - if (jiffies - wall_jiffies) - tv->tv_usec += USECS_PER_JIFFY; + *tv = xtime; + tv->tv_usec += do_gettimeoffset(); + + /* + * xtime is atomically updated in timer_bh. + * jiffies - wall_jiffies + * is nonzero if the timer bottom half hasnt executed yet. + */ + if (jiffies - wall_jiffies) + tv->tv_usec += USECS_PER_JIFFY; + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); - read_unlock_irqrestore (&xtime_lock, flags); if (tv->tv_usec >= 1000000) { tv->tv_usec -= 1000000; @@ -85,7 +88,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq (&xtime_lock); + write_seqlock_irq (&xtime_lock); /* This is revolting. We need to set the xtime.tv_usec * correctly. However, the value in this location is @@ -105,7 +108,7 @@ void do_settimeofday(struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq (&xtime_lock); + write_sequnlock_irq (&xtime_lock); } @@ -291,6 +294,8 @@ unsigned long calibrate_div64_gettimeoffset(void) */ void timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) { + unsigned long seq; + if (mips_cpu.options & MIPS_CPU_COUNTER) { unsigned int count; @@ -340,19 +345,21 @@ void timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) * CMOS clock accordingly every ~11 minutes. rtc_set_time() has to be * called as close as possible to 500 ms before the new second starts. */ - read_lock (&xtime_lock); - if ((time_status & STA_UNSYNC) == 0 && - xtime.tv_sec > last_rtc_update + 660 && - xtime.tv_usec >= 500000 - ((unsigned) tick) / 2 && - xtime.tv_usec <= 500000 + ((unsigned) tick) / 2) { - if (rtc_set_time(xtime.tv_sec) == 0) { - last_rtc_update = xtime.tv_sec; - } else { - last_rtc_update = xtime.tv_sec - 600; - /* do it again in 60 s */ + do { + seq = read_seqbegin(&xtime_lock); + + if ((time_status & STA_UNSYNC) == 0 && + xtime.tv_sec > last_rtc_update + 660 && + xtime.tv_usec >= 500000 - ((unsigned) tick) / 2 && + xtime.tv_usec <= 500000 + ((unsigned) tick) / 2) { + if (rtc_set_time(xtime.tv_sec) == 0) { + last_rtc_update = xtime.tv_sec; + } else { + last_rtc_update = xtime.tv_sec - 600; + /* do it again in 60 s */ + } } - } - read_unlock (&xtime_lock); + } while (read_seqretry(&xtime_lock, seq)); /* * If jiffies has overflowed in this timer_interrupt we must diff --git a/arch/mips/mips-boards/generic/time.c b/arch/mips/mips-boards/generic/time.c index 73031a7668dd..f21ee167a17d 100644 --- a/arch/mips/mips-boards/generic/time.c +++ b/arch/mips/mips-boards/generic/time.c @@ -34,6 +34,7 @@ #include <asm/div64.h> #include <linux/mc146818rtc.h> +#include <linux/time.h> #include <linux/timex.h> #include <asm/mips-boards/generic.h> @@ -45,7 +46,6 @@ unsigned long missed_heart_beats = 0; static unsigned long r4k_offset; /* Amount to increment compare reg each time */ static unsigned long r4k_cur; /* What counter should be at next timer irq */ -extern rwlock_t xtime_lock; #define ALLINTS (IE_IRQ0 | IE_IRQ1 | IE_IRQ2 | IE_IRQ3 | IE_IRQ4 | IE_IRQ5) @@ -133,7 +133,9 @@ static int set_rtc_mmss(unsigned long nowtime) */ void mips_timer_interrupt(struct pt_regs *regs) { + unsigned long flags; int irq = 7; + unsigned long seq; if (r4k_offset == 0) goto null; @@ -149,18 +151,21 @@ void mips_timer_interrupt(struct pt_regs *regs) * within 500ms before the * next second starts, * thus the following code. */ - read_lock(&xtime_lock); - if ((time_status & STA_UNSYNC) == 0 - && xtime.tv_sec > last_rtc_update + 660 - && xtime.tv_usec >= 500000 - (tick >> 1) - && xtime.tv_usec <= 500000 + (tick >> 1)) - if (set_rtc_mmss(xtime.tv_sec) == 0) - last_rtc_update = xtime.tv_sec; - else - /* do it again in 60 s */ - last_rtc_update = xtime.tv_sec - 600; - read_unlock(&xtime_lock); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); + + if ((time_status & STA_UNSYNC) == 0 + && xtime.tv_sec > last_rtc_update + 660 + && xtime.tv_usec >= 500000 - (tick >> 1) + && xtime.tv_usec <= 500000 + (tick >> 1)) + if (set_rtc_mmss(xtime.tv_sec) == 0) + last_rtc_update = xtime.tv_sec; + else + /* do it again in 60 s */ + last_rtc_update = xtime.tv_sec - 600; + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); + if ((timer_tick_count++ % HZ) == 0) { mips_display_message(&display_string[display_count++]); if (display_count == MAX_DISPLAY_COUNT) @@ -267,10 +272,10 @@ void __init time_init(void) change_cp0_status(ST0_IM, ALLINTS); /* Read time from the RTC chipset. */ - write_lock_irqsave (&xtime_lock, flags); + write_seqlock_irqsave (&xtime_lock, flags); xtime.tv_sec = get_mips_time(); xtime.tv_usec = 0; - write_unlock_irqrestore(&xtime_lock, flags); + write_sequnlock_irqrestore(&xtime_lock, flags); } /* This is for machines which generate the exact clock. */ @@ -363,20 +368,24 @@ static unsigned long do_fast_gettimeoffset(void) void do_gettimeofday(struct timeval *tv) { - unsigned int flags; + unsigned long flags; + unsigned long seq; - read_lock_irqsave (&xtime_lock, flags); - *tv = xtime; - tv->tv_usec += do_fast_gettimeoffset(); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); - /* - * xtime is atomically updated in timer_bh. jiffies - wall_jiffies - * is nonzero if the timer bottom half hasnt executed yet. - */ - if (jiffies - wall_jiffies) - tv->tv_usec += USECS_PER_JIFFY; + *tv = xtime; + tv->tv_usec += do_fast_gettimeoffset(); + + /* + * xtime is atomically updated in timer_bh. + * jiffies - wall_jiffies + * is nonzero if the timer bottom half hasnt executed yet. + */ + if (jiffies - wall_jiffies) + tv->tv_usec += USECS_PER_JIFFY; - read_unlock_irqrestore (&xtime_lock, flags); + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); if (tv->tv_usec >= 1000000) { tv->tv_usec -= 1000000; @@ -386,7 +395,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq (&xtime_lock); + write_seqlock_irq (&xtime_lock); /* This is revolting. We need to set the xtime.tv_usec correctly. * However, the value in this location is is value at the last tick. @@ -406,5 +415,5 @@ void do_settimeofday(struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq (&xtime_lock); + write_sequnlock_irq (&xtime_lock); } diff --git a/arch/mips/philips/nino/time.c b/arch/mips/philips/nino/time.c index 5a3c368c17cb..95cc58b413c3 100644 --- a/arch/mips/philips/nino/time.c +++ b/arch/mips/philips/nino/time.c @@ -19,12 +19,12 @@ #include <linux/string.h> #include <linux/mm.h> #include <linux/interrupt.h> +#include <linux/time.h> #include <linux/timex.h> #include <linux/delay.h> #include <asm/tx3912.h> extern volatile unsigned long wall_jiffies; -extern rwlock_t xtime_lock; static struct timeval xbase; @@ -62,29 +62,31 @@ void inline readRTC(unsigned long *high, unsigned long *low) void do_gettimeofday(struct timeval *tv) { unsigned long flags; + unsigned long seq; unsigned long high, low; - read_lock_irqsave(&xtime_lock, flags); - // 40 bit RTC, driven by 32khz source: - // +-----------+-----------------------------------------+ - // | HHHH.HHHH | LLLL.LLLL.LLLL.LLLL.LMMM.MMMM.MMMM.MMMM | - // +-----------+-----------------------------------------+ - readRTC(&high,&low); - tv->tv_sec = (high << 17) | (low >> 15); - tv->tv_usec = (low % 32768) * 1953 / 64; - tv->tv_sec += xbase.tv_sec; - tv->tv_usec += xbase.tv_usec; + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); - tv->tv_usec += do_gettimeoffset(); + // 40 bit RTC, driven by 32khz source: + // +-----------+-----------------------------------------+ + // | HHHH.HHHH | LLLL.LLLL.LLLL.LLLL.LMMM.MMMM.MMMM.MMMM | + // +-----------+-----------------------------------------+ + readRTC(&high,&low); + tv->tv_sec = (high << 17) | (low >> 15); + tv->tv_usec = (low % 32768) * 1953 / 64; + tv->tv_sec += xbase.tv_sec; + tv->tv_usec += xbase.tv_usec; - /* - * xtime is atomically updated in timer_bh. lost_ticks is - * nonzero if the timer bottom half hasnt executed yet. - */ - if (jiffies - wall_jiffies) - tv->tv_usec += USECS_PER_JIFFY; + tv->tv_usec += do_gettimeoffset(); - read_unlock_irqrestore(&xtime_lock, flags); + /* + * xtime is atomically updated in timer_bh. lost_ticks is + * nonzero if the timer bottom half hasnt executed yet. + */ + if (jiffies - wall_jiffies) + tv->tv_usec += USECS_PER_JIFFY; + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); if (tv->tv_usec >= 1000000) { tv->tv_usec -= 1000000; @@ -94,7 +96,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); /* This is revolting. We need to set the xtime.tv_usec * correctly. However, the value in this location is * is value at the last tick. @@ -118,7 +120,7 @@ void do_settimeofday(struct timeval *tv) time_state = TIME_BAD; time_maxerror = MAXPHASE; time_esterror = MAXPHASE; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } static int set_rtc_mmss(unsigned long nowtime) diff --git a/arch/mips64/mips-boards/generic/time.c b/arch/mips64/mips-boards/generic/time.c index ed1f105c090d..196de531501c 100644 --- a/arch/mips64/mips-boards/generic/time.c +++ b/arch/mips64/mips-boards/generic/time.c @@ -27,6 +27,7 @@ #include <linux/init.h> #include <linux/kernel_stat.h> #include <linux/sched.h> +#include <linux/time.h> #include <linux/spinlock.h> #include <asm/mipsregs.h> @@ -44,7 +45,6 @@ unsigned long missed_heart_beats = 0; static unsigned long r4k_offset; /* Amount to increment compare reg each time */ static unsigned long r4k_cur; /* What counter should be at next timer irq */ -extern rwlock_t xtime_lock; #define ALLINTS (IE_IRQ0 | IE_IRQ1 | IE_IRQ2 | IE_IRQ3 | IE_IRQ4 | IE_IRQ5) @@ -132,6 +132,8 @@ static int set_rtc_mmss(unsigned long nowtime) */ void mips_timer_interrupt(struct pt_regs *regs) { + unsigned long flags; + unsigned long seq; int irq = 7; if (r4k_offset == 0) @@ -148,17 +150,20 @@ void mips_timer_interrupt(struct pt_regs *regs) * within 500ms before the * next second starts, * thus the following code. */ - read_lock(&xtime_lock); - if ((time_status & STA_UNSYNC) == 0 - && xtime.tv_sec > last_rtc_update + 660 - && xtime.tv_usec >= 500000 - (tick >> 1) - && xtime.tv_usec <= 500000 + (tick >> 1)) - if (set_rtc_mmss(xtime.tv_sec) == 0) - last_rtc_update = xtime.tv_sec; - else - /* do it again in 60 s */ - last_rtc_update = xtime.tv_sec - 600; - read_unlock(&xtime_lock); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); + + if ((time_status & STA_UNSYNC) == 0 + && xtime.tv_sec > last_rtc_update + 660 + && xtime.tv_usec >= 500000 - (tick >> 1) + && xtime.tv_usec <= 500000 + (tick >> 1)) + if (set_rtc_mmss(xtime.tv_sec) == 0) + last_rtc_update = xtime.tv_sec; + else + /* do it again in 60 s */ + last_rtc_update = xtime.tv_sec - 600; + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); + if ((timer_tick_count++ % HZ) == 0) { mips_display_message(&display_string[display_count++]); @@ -266,10 +271,10 @@ void __init time_init(void) set_cp0_status(ST0_IM, ALLINTS); /* Read time from the RTC chipset. */ - write_lock_irqsave (&xtime_lock, flags); + write_seqlock_irqsave (&xtime_lock, flags); xtime.tv_sec = get_mips_time(); xtime.tv_usec = 0; - write_unlock_irqrestore(&xtime_lock, flags); + write_sequnlock_irqrestore(&xtime_lock, flags); } /* This is for machines which generate the exact clock. */ @@ -352,20 +357,25 @@ static unsigned long do_fast_gettimeoffset(void) void do_gettimeofday(struct timeval *tv) { - unsigned int flags; + unsigned long flags; + unsigned long seq; - read_lock_irqsave (&xtime_lock, flags); - *tv = xtime; - tv->tv_usec += do_fast_gettimeoffset(); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); - /* - * xtime is atomically updated in timer_bh. jiffies - wall_jiffies - * is nonzero if the timer bottom half hasnt executed yet. - */ - if (jiffies - wall_jiffies) - tv->tv_usec += USECS_PER_JIFFY; + *tv = xtime; + tv->tv_usec += do_fast_gettimeoffset(); + + /* + * xtime is atomically updated in timer_bh. + * jiffies - wall_jiffies + * is nonzero if the timer bottom half hasnt executed yet. + */ + if (jiffies - wall_jiffies) + tv->tv_usec += USECS_PER_JIFFY; + + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); - read_unlock_irqrestore (&xtime_lock, flags); if (tv->tv_usec >= 1000000) { tv->tv_usec -= 1000000; @@ -375,7 +385,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq (&xtime_lock); + write_seqlock_irq (&xtime_lock); /* This is revolting. We need to set the xtime.tv_usec correctly. * However, the value in this location is is value at the last tick. @@ -395,5 +405,5 @@ void do_settimeofday(struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq (&xtime_lock); + write_sequnlock_irq (&xtime_lock); } diff --git a/arch/mips64/sgi-ip22/ip22-timer.c b/arch/mips64/sgi-ip22/ip22-timer.c index e7a80c90b480..5959fb864dcb 100644 --- a/arch/mips64/sgi-ip22/ip22-timer.c +++ b/arch/mips64/sgi-ip22/ip22-timer.c @@ -11,6 +11,7 @@ #include <linux/param.h> #include <linux/string.h> #include <linux/mm.h> +#include <linux/time.h> #include <linux/interrupt.h> #include <linux/timex.h> #include <linux/kernel_stat.h> @@ -32,8 +33,6 @@ static unsigned long r4k_offset; /* Amount to increment compare reg each time */ static unsigned long r4k_cur; /* What counter should be at next timer irq */ -extern rwlock_t xtime_lock; - static inline void ack_r4ktimer(unsigned long newval) { write_32bit_cp0_register(CP0_COMPARE, newval); @@ -86,7 +85,7 @@ void indy_timer_interrupt(struct pt_regs *regs) unsigned long count; int irq = 7; - write_lock(&xtime_lock); + write_seqlock(&xtime_lock); /* Ack timer and compute new compare. */ count = read_32bit_cp0_register(CP0_COUNT); /* This has races. */ @@ -116,7 +115,7 @@ void indy_timer_interrupt(struct pt_regs *regs) /* do it again in 60s */ last_rtc_update = xtime.tv_sec - 600; } - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); } static unsigned long dosample(volatile unsigned char *tcwp, @@ -224,10 +223,10 @@ void __init indy_timer_init(void) set_cp0_status(ST0_IM, ALLINTS); sti(); - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); xtime.tv_sec = get_indy_time(); /* Read time from RTC. */ xtime.tv_usec = 0; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } void indy_8254timer_irq(void) @@ -243,20 +242,21 @@ void indy_8254timer_irq(void) void do_gettimeofday(struct timeval *tv) { - unsigned long flags; + unsigned long seq; - read_lock_irqsave(&xtime_lock, flags); - *tv = xtime; - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = read_seqbegin(&xtime_lock); + *tv = xtime; + } while (read_seqretry(&xtime_lock, seq)); } void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); xtime = *tv; time_adjust = 0; /* stop active adjtime() */ time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } diff --git a/arch/mips64/sgi-ip27/ip27-timer.c b/arch/mips64/sgi-ip27/ip27-timer.c index d4ed43d11d66..dedb306364bb 100644 --- a/arch/mips64/sgi-ip27/ip27-timer.c +++ b/arch/mips64/sgi-ip27/ip27-timer.c @@ -9,6 +9,7 @@ #include <linux/interrupt.h> #include <linux/kernel_stat.h> #include <linux/param.h> +#include <linux/time.h> #include <linux/timex.h> #include <linux/mm.h> #include <linux/bcd.h> @@ -40,7 +41,6 @@ static unsigned long ct_cur[NR_CPUS]; /* What counter should be at next timer irq */ static long last_rtc_update; /* Last time the rtc clock got updated */ -extern rwlock_t xtime_lock; extern volatile unsigned long wall_jiffies; @@ -94,7 +94,7 @@ void rt_timer_interrupt(struct pt_regs *regs) int cpuA = ((cputoslice(cpu)) == 0); int irq = 7; /* XXX Assign number */ - write_lock(&xtime_lock); + write_seqlock(&xtime_lock); again: LOCAL_HUB_S(cpuA ? PI_RT_PEND_A : PI_RT_PEND_B, 0); /* Ack */ @@ -145,7 +145,7 @@ again: } } - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); if (softirq_pending(cpu)) do_softirq(); @@ -162,17 +162,20 @@ void do_gettimeofday(struct timeval *tv) { unsigned long flags; unsigned long usec, sec; + unsigned long seq; - read_lock_irqsave(&xtime_lock, flags); - usec = do_gettimeoffset(); - { - unsigned long lost = jiffies - wall_jiffies; - if (lost) - usec += lost * (1000000 / HZ); - } - sec = xtime.tv_sec; - usec += xtime.tv_usec; - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); + + usec = do_gettimeoffset(); + { + unsigned long lost = jiffies - wall_jiffies; + if (lost) + usec += lost * (1000000 / HZ); + } + sec = xtime.tv_sec; + usec += xtime.tv_usec; + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); while (usec >= 1000000) { usec -= 1000000; @@ -185,7 +188,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); tv->tv_usec -= do_gettimeoffset(); tv->tv_usec -= (jiffies - wall_jiffies) * (1000000 / HZ); @@ -199,7 +202,7 @@ void do_settimeofday(struct timeval *tv) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } /* Includes for ioc3_init(). */ diff --git a/arch/parisc/kernel/sys_parisc32.c b/arch/parisc/kernel/sys_parisc32.c index de994c1e6d1f..3e2d0013d340 100644 --- a/arch/parisc/kernel/sys_parisc32.c +++ b/arch/parisc/kernel/sys_parisc32.c @@ -20,6 +20,7 @@ #include <linux/resource.h> #include <linux/times.h> #include <linux/utsname.h> +#include <linux/time.h> #include <linux/timex.h> #include <linux/smp.h> #include <linux/smp_lock.h> @@ -2361,21 +2362,23 @@ asmlinkage int sys32_sysinfo(struct sysinfo32 *info) { struct sysinfo val; int err; - extern rwlock_t xtime_lock; + unsigned long seq; /* We don't need a memset here because we copy the * struct to userspace once element at a time. */ - read_lock_irq(&xtime_lock); - val.uptime = jiffies / HZ; + do { + seq = read_seqbegin(&xtime_lock); + val.uptime = jiffies / HZ; - val.loads[0] = avenrun[0] << (SI_LOAD_SHIFT - FSHIFT); - val.loads[1] = avenrun[1] << (SI_LOAD_SHIFT - FSHIFT); - val.loads[2] = avenrun[2] << (SI_LOAD_SHIFT - FSHIFT); + val.loads[0] = avenrun[0] << (SI_LOAD_SHIFT - FSHIFT); + val.loads[1] = avenrun[1] << (SI_LOAD_SHIFT - FSHIFT); + val.loads[2] = avenrun[2] << (SI_LOAD_SHIFT - FSHIFT); + + val.procs = nr_threads; + } while (read_seqretry(&xtime_lock, seq)); - val.procs = nr_threads; - read_unlock_irq(&xtime_lock); si_meminfo(&val); si_swapinfo(&val); diff --git a/arch/parisc/kernel/time.c b/arch/parisc/kernel/time.c index 0ce0ce916c8d..96cefb43ada6 100644 --- a/arch/parisc/kernel/time.c +++ b/arch/parisc/kernel/time.c @@ -36,7 +36,6 @@ u64 jiffies_64; /* xtime and wall_jiffies keep wall-clock time */ extern unsigned long wall_jiffies; -extern rwlock_t xtime_lock; static long clocktick; /* timer cycles per tick */ static long halftick; @@ -115,9 +114,9 @@ void timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) smp_do_timer(regs); #endif if (cpu == 0) { - write_lock(&xtime_lock); + write_seqlock(&xtime_lock); do_timer(regs); - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); } } @@ -172,16 +171,14 @@ gettimeoffset (void) void do_gettimeofday (struct timeval *tv) { - unsigned long flags, usec, sec; + unsigned long flags, seq, usec, sec; - read_lock_irqsave(&xtime_lock, flags); - { + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); usec = gettimeoffset(); - sec = xtime.tv_sec; usec += (xtime.tv_nsec / 1000); - } - read_unlock_irqrestore(&xtime_lock, flags); + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); while (usec >= 1000000) { usec -= 1000000; @@ -195,7 +192,7 @@ do_gettimeofday (struct timeval *tv) void do_settimeofday (struct timeval *tv) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); { /* * This is revolting. We need to set "xtime" @@ -219,7 +216,7 @@ do_settimeofday (struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; } - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } @@ -241,10 +238,10 @@ void __init time_init(void) mtctl(next_tick, 16); if(pdc_tod_read(&tod_data) == 0) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); xtime.tv_sec = tod_data.tod_sec; xtime.tv_nsec = tod_data.tod_usec * 1000; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } else { printk(KERN_ERR "Error reading tod clock\n"); xtime.tv_sec = 0; diff --git a/arch/ppc/kernel/time.c b/arch/ppc/kernel/time.c index 14c583e45876..3fef611a7df8 100644 --- a/arch/ppc/kernel/time.c +++ b/arch/ppc/kernel/time.c @@ -76,7 +76,6 @@ extern struct timezone sys_tz; /* keep track of when we need to update the rtc */ time_t last_rtc_update; -extern rwlock_t xtime_lock; /* The decrementer counts down by 128 every 128ns on a 601. */ #define DECREMENTER_COUNT_601 (1000000000 / HZ) @@ -161,7 +160,7 @@ void timer_interrupt(struct pt_regs * regs) continue; /* We are in an interrupt, no need to save/restore flags */ - write_lock(&xtime_lock); + write_seqlock(&xtime_lock); tb_last_stamp = jiffy_stamp; do_timer(regs); @@ -191,7 +190,7 @@ void timer_interrupt(struct pt_regs * regs) /* Try again one minute later */ last_rtc_update += 60; } - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); } if ( !disarm_decr[smp_processor_id()] ) set_dec(next_dec); @@ -213,21 +212,23 @@ void timer_interrupt(struct pt_regs * regs) void do_gettimeofday(struct timeval *tv) { unsigned long flags; + unsigned long seq; unsigned delta, lost_ticks, usec, sec; - read_lock_irqsave(&xtime_lock, flags); - sec = xtime.tv_sec; - usec = (xtime.tv_nsec / 1000); - delta = tb_ticks_since(tb_last_stamp); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); + sec = xtime.tv_sec; + usec = (xtime.tv_nsec / 1000); + delta = tb_ticks_since(tb_last_stamp); #ifdef CONFIG_SMP - /* As long as timebases are not in sync, gettimeofday can only - * have jiffy resolution on SMP. - */ - if (!smp_tb_synchronized) - delta = 0; + /* As long as timebases are not in sync, gettimeofday can only + * have jiffy resolution on SMP. + */ + if (!smp_tb_synchronized) + delta = 0; #endif /* CONFIG_SMP */ - lost_ticks = jiffies - wall_jiffies; - read_unlock_irqrestore(&xtime_lock, flags); + lost_ticks = jiffies - wall_jiffies; + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); usec += mulhwu(tb_to_us, tb_ticks_per_jiffy * lost_ticks + delta); while (usec >= 1000000) { @@ -243,7 +244,7 @@ void do_settimeofday(struct timeval *tv) unsigned long flags; int tb_delta, new_usec, new_sec; - write_lock_irqsave(&xtime_lock, flags); + write_seqlock_irqsave(&xtime_lock, flags); /* Updating the RTC is not the job of this code. If the time is * stepped under NTP, the RTC will be update after STA_UNSYNC * is cleared. Tool like clock/hwclock either copy the RTC @@ -283,7 +284,7 @@ void do_settimeofday(struct timeval *tv) time_state = TIME_ERROR; /* p. 24, (a) */ time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irqrestore(&xtime_lock, flags); + write_sequnlock_irqrestore(&xtime_lock, flags); } /* This function is only called on the boot processor */ diff --git a/arch/ppc/platforms/pmac_time.c b/arch/ppc/platforms/pmac_time.c index 56cf32227467..078c554145e2 100644 --- a/arch/ppc/platforms/pmac_time.c +++ b/arch/ppc/platforms/pmac_time.c @@ -15,6 +15,7 @@ #include <linux/string.h> #include <linux/mm.h> #include <linux/init.h> +#include <linux/time.h> #include <linux/adb.h> #include <linux/cuda.h> #include <linux/pmu.h> @@ -29,8 +30,6 @@ #include <asm/time.h> #include <asm/nvram.h> -extern rwlock_t xtime_lock; - /* Apparently the RTC stores seconds since 1 Jan 1904 */ #define RTC_OFFSET 2082844800 @@ -215,19 +214,21 @@ time_sleep_notify(struct pmu_sleep_notifier *self, int when) { static unsigned long time_diff; unsigned long flags; + unsigned long seq; switch (when) { case PBOOK_SLEEP_NOW: - read_lock_irqsave(&xtime_lock, flags); - time_diff = xtime.tv_sec - pmac_get_rtc_time(); - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); + time_diff = xtime.tv_sec - pmac_get_rtc_time(); + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); break; case PBOOK_WAKE: - write_lock_irqsave(&xtime_lock, flags); + write_seqlock_irqsave(&xtime_lock, flags); xtime.tv_sec = pmac_get_rtc_time() + time_diff; xtime.tv_nsec = 0; last_rtc_update = xtime.tv_sec; - write_unlock_irqrestore(&xtime_lock, flags); + write_sequnlock_irqrestore(&xtime_lock, flags); break; } return PBOOK_SLEEP_OK; diff --git a/arch/ppc64/kernel/time.c b/arch/ppc64/kernel/time.c index 4ab5eb9916fd..b42410131796 100644 --- a/arch/ppc64/kernel/time.c +++ b/arch/ppc64/kernel/time.c @@ -69,7 +69,6 @@ u64 jiffies_64; /* keep track of when we need to update the rtc */ time_t last_rtc_update; -extern rwlock_t xtime_lock; extern int piranha_simulator; #ifdef CONFIG_PPC_ISERIES unsigned long iSeries_recal_titan = 0; @@ -284,12 +283,12 @@ int timer_interrupt(struct pt_regs * regs) smp_local_timer_interrupt(regs); #endif if (cpu == boot_cpuid) { - write_lock(&xtime_lock); + write_seqlock(&xtime_lock); tb_last_stamp = lpaca->next_jiffy_update_tb; do_timer(regs); timer_sync_xtime( cur_tb ); timer_check_rtc(); - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); if ( adjusting_time && (time_adjust == 0) ) ppc_adjtimex(); } @@ -348,7 +347,7 @@ void do_settimeofday(struct timeval *tv) long int tb_delta, new_usec, new_sec; unsigned long new_xsec; - write_lock_irqsave(&xtime_lock, flags); + write_seqlock_irqsave(&xtime_lock, flags); /* Updating the RTC is not the job of this code. If the time is * stepped under NTP, the RTC will be update after STA_UNSYNC * is cleared. Tool like clock/hwclock either copy the RTC @@ -399,7 +398,7 @@ void do_settimeofday(struct timeval *tv) do_gtod.tb_orig_stamp = tb_last_stamp; } - write_unlock_irqrestore(&xtime_lock, flags); + write_sequnlock_irqrestore(&xtime_lock, flags); } /* @@ -465,7 +464,7 @@ void __init time_init(void) #endif ppc_md.get_boot_time(&tm); - write_lock_irqsave(&xtime_lock, flags); + write_seqlock_irqsave(&xtime_lock, flags); xtime.tv_sec = mktime(tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); tb_last_stamp = get_tb(); @@ -484,7 +483,7 @@ void __init time_init(void) xtime.tv_nsec = 0; last_rtc_update = xtime.tv_sec; - write_unlock_irqrestore(&xtime_lock, flags); + write_sequnlock_irqrestore(&xtime_lock, flags); /* Not exact, but the timer interrupt takes care of this */ set_dec(tb_ticks_per_jiffy); @@ -587,7 +586,7 @@ void ppc_adjtimex(void) new_tb_to_xs = divres.result_low; new_xsec = mulhdu( tb_ticks, new_tb_to_xs ); - write_lock_irqsave( &xtime_lock, flags ); + write_seqlock_irqsave( &xtime_lock, flags ); old_xsec = mulhdu( tb_ticks, do_gtod.varp->tb_to_xs ); new_stamp_xsec = do_gtod.varp->stamp_xsec + old_xsec - new_xsec; @@ -609,7 +608,7 @@ void ppc_adjtimex(void) do_gtod.varp = temp_varp; do_gtod.var_idx = temp_idx; - write_unlock_irqrestore( &xtime_lock, flags ); + write_sequnlock_irqrestore( &xtime_lock, flags ); } diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index cb1ff2bec4af..800bbeb8016b 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -52,7 +52,6 @@ static ext_int_info_t ext_int_info_timer; static uint64_t xtime_cc; static uint64_t init_timer_cc; -extern rwlock_t xtime_lock; extern unsigned long wall_jiffies; void tod_to_timeval(__u64 todval, struct timespec *xtime) @@ -83,12 +82,15 @@ static inline unsigned long do_gettimeoffset(void) void do_gettimeofday(struct timeval *tv) { unsigned long flags; + unsigned long seq; unsigned long usec, sec; - read_lock_irqsave(&xtime_lock, flags); - sec = xtime.tv_sec; - usec = xtime.tv_nsec / 1000 + do_gettimeoffset(); - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); + + sec = xtime.tv_sec; + usec = xtime.tv_nsec / 1000 + do_gettimeoffset(); + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); while (usec >= 1000000) { usec -= 1000000; @@ -102,7 +104,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); /* This is revolting. We need to set the xtime.tv_nsec * correctly. However, the value in this location is * is value at the last tick. @@ -122,7 +124,7 @@ void do_settimeofday(struct timeval *tv) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } static inline __u32 div64_32(__u64 dividend, __u32 divisor) @@ -166,7 +168,7 @@ static void do_comparator_interrupt(struct pt_regs *regs, __u16 error_code) * Do not rely on the boot cpu to do the calls to do_timer. * Spread it over all cpus instead. */ - write_lock(&xtime_lock); + write_seqlock(&xtime_lock); if (S390_lowcore.jiffy_timer > xtime_cc) { __u32 xticks; @@ -181,7 +183,7 @@ static void do_comparator_interrupt(struct pt_regs *regs, __u16 error_code) while (xticks--) do_timer(regs); } - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); while (ticks--) update_process_times(user_mode(regs)); #else diff --git a/arch/s390x/kernel/time.c b/arch/s390x/kernel/time.c index bf0ac7a45dc0..66115fc37677 100644 --- a/arch/s390x/kernel/time.c +++ b/arch/s390x/kernel/time.c @@ -51,7 +51,6 @@ static ext_int_info_t ext_int_info_timer; static uint64_t xtime_cc; static uint64_t init_timer_cc; -extern rwlock_t xtime_lock; extern unsigned long wall_jiffies; void tod_to_timeval(__u64 todval, struct timespec *xtime) @@ -78,12 +77,14 @@ static inline unsigned long do_gettimeoffset(void) void do_gettimeofday(struct timeval *tv) { unsigned long flags; + unsigned long seq; unsigned long usec, sec; - read_lock_irqsave(&xtime_lock, flags); - sec = xtime.tv_sec; - usec = xtime.tv_nsec + do_gettimeoffset(); - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); + sec = xtime.tv_sec; + usec = xtime.tv_nsec + do_gettimeoffset(); + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); while (usec >= 1000000) { usec -= 1000000; @@ -97,7 +98,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); /* This is revolting. We need to set the xtime.tv_usec * correctly. However, the value in this location is * is value at the last tick. @@ -117,7 +118,7 @@ void do_settimeofday(struct timeval *tv) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } /* @@ -152,7 +153,7 @@ static void do_comparator_interrupt(struct pt_regs *regs, __u16 error_code) * Do not rely on the boot cpu to do the calls to do_timer. * Spread it over all cpus instead. */ - write_lock(&xtime_lock); + write_seqlock(&xtime_lock); if (S390_lowcore.jiffy_timer > xtime_cc) { __u32 xticks; @@ -167,7 +168,7 @@ static void do_comparator_interrupt(struct pt_regs *regs, __u16 error_code) while (xticks--) do_timer(regs); } - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); while (ticks--) update_process_times(user_mode(regs)); #else diff --git a/arch/sh/kernel/time.c b/arch/sh/kernel/time.c index e51e0eb001d6..5a17b0510284 100644 --- a/arch/sh/kernel/time.c +++ b/arch/sh/kernel/time.c @@ -72,7 +72,6 @@ u64 jiffies_64; -extern rwlock_t xtime_lock; extern unsigned long wall_jiffies; #define TICK_SIZE tick @@ -128,18 +127,20 @@ static unsigned long do_gettimeoffset(void) void do_gettimeofday(struct timeval *tv) { unsigned long flags; + unsigned long seq; unsigned long usec, sec; - read_lock_irqsave(&xtime_lock, flags); - usec = do_gettimeoffset(); - { - unsigned long lost = jiffies - wall_jiffies; - if (lost) - usec += lost * (1000000 / HZ); - } - sec = xtime.tv_sec; - usec += xtime.tv_usec; - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); + usec = do_gettimeoffset(); + { + unsigned long lost = jiffies - wall_jiffies; + if (lost) + usec += lost * (1000000 / HZ); + } + sec = xtime.tv_sec; + usec += xtime.tv_usec; + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); while (usec >= 1000000) { usec -= 1000000; @@ -152,7 +153,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); /* * This is revolting. We need to set "xtime" correctly. However, the * value in this location is the value at the most recent update of @@ -172,7 +173,7 @@ void do_settimeofday(struct timeval *tv) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } /* last time the RTC clock got updated */ @@ -231,9 +232,9 @@ static void timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) * the irq version of write_lock because as just said we have irq * locally disabled. -arca */ - write_lock(&xtime_lock); + write_seqlock(&xtime_lock); do_timer_interrupt(irq, NULL, regs); - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); } static unsigned int __init get_timer_frequency(void) diff --git a/arch/sparc/kernel/pcic.c b/arch/sparc/kernel/pcic.c index 403a7cc35da1..b7e3c63efd9f 100644 --- a/arch/sparc/kernel/pcic.c +++ b/arch/sparc/kernel/pcic.c @@ -25,6 +25,7 @@ #include <linux/ctype.h> #include <linux/pci.h> +#include <linux/time.h> #include <linux/timex.h> #include <linux/interrupt.h> @@ -34,8 +35,6 @@ #include <asm/timer.h> #include <asm/uaccess.h> -extern rwlock_t xtime_lock; - #ifndef CONFIG_PCI asmlinkage int sys_pciconfig_read(unsigned long bus, @@ -739,10 +738,10 @@ static void pcic_clear_clock_irq(void) static void pcic_timer_handler (int irq, void *h, struct pt_regs *regs) { - write_lock(&xtime_lock); /* Dummy, to show that we remember */ + write_seqlock(&xtime_lock); /* Dummy, to show that we remember */ pcic_clear_clock_irq(); do_timer(regs); - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); } #define USECS_PER_JIFFY 10000 /* We have 100HZ "standard" timer for sparc */ @@ -795,18 +794,20 @@ extern unsigned long wall_jiffies; static void pci_do_gettimeofday(struct timeval *tv) { unsigned long flags; + unsigned long seq; unsigned long usec, sec; - read_lock_irqsave(&xtime_lock, flags); - usec = do_gettimeoffset(); - { - unsigned long lost = jiffies - wall_jiffies; - if (lost) - usec += lost * (1000000 / HZ); - } - sec = xtime.tv_sec; - usec += (xtime.tv_nsec / 1000); - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); + usec = do_gettimeoffset(); + { + unsigned long lost = jiffies - wall_jiffies; + if (lost) + usec += lost * (1000000 / HZ); + } + sec = xtime.tv_sec; + usec += (xtime.tv_nsec / 1000); + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); while (usec >= 1000000) { usec -= 1000000; diff --git a/arch/sparc/kernel/time.c b/arch/sparc/kernel/time.c index 62dee2eae417..43130a381cb7 100644 --- a/arch/sparc/kernel/time.c +++ b/arch/sparc/kernel/time.c @@ -23,6 +23,7 @@ #include <linux/string.h> #include <linux/mm.h> #include <linux/interrupt.h> +#include <linux/time.h> #include <linux/timex.h> #include <linux/init.h> #include <linux/pci.h> @@ -42,8 +43,6 @@ #include <asm/page.h> #include <asm/pcic.h> -extern rwlock_t xtime_lock; - extern unsigned long wall_jiffies; u64 jiffies_64; @@ -131,7 +130,7 @@ void timer_interrupt(int irq, void *dev_id, struct pt_regs * regs) #endif /* Protect counter clear so that do_gettimeoffset works */ - write_lock(&xtime_lock); + write_seqlock(&xtime_lock); #ifdef CONFIG_SUN4 if((idprom->id_machtype == (SM_SUN4 | SM_4_260)) || (idprom->id_machtype == (SM_SUN4 | SM_4_110))) { @@ -155,7 +154,7 @@ void timer_interrupt(int irq, void *dev_id, struct pt_regs * regs) else last_rtc_update = xtime.tv_sec - 600; /* do it again in 60 s */ } - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); } /* Kick start a stopped clock (procedure from the Sun NVRAM/hostid FAQ). */ @@ -470,18 +469,20 @@ extern __inline__ unsigned long do_gettimeoffset(void) void do_gettimeofday(struct timeval *tv) { unsigned long flags; + unsigned long seq; unsigned long usec, sec; - read_lock_irqsave(&xtime_lock, flags); - usec = do_gettimeoffset(); - { - unsigned long lost = jiffies - wall_jiffies; - if (lost) - usec += lost * (1000000 / HZ); - } - sec = xtime.tv_sec; - usec += (xtime.tv_nsec / 1000); - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); + usec = do_gettimeoffset(); + { + unsigned long lost = jiffies - wall_jiffies; + if (lost) + usec += lost * (1000000 / HZ); + } + sec = xtime.tv_sec; + usec += (xtime.tv_nsec / 1000); + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); while (usec >= 1000000) { usec -= 1000000; @@ -494,9 +495,9 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); bus_do_settimeofday(tv); - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } static void sbus_do_settimeofday(struct timeval *tv) diff --git a/arch/sparc64/kernel/time.c b/arch/sparc64/kernel/time.c index 3c48ed49ea5a..a7163a393bb4 100644 --- a/arch/sparc64/kernel/time.c +++ b/arch/sparc64/kernel/time.c @@ -17,6 +17,7 @@ #include <linux/string.h> #include <linux/mm.h> #include <linux/interrupt.h> +#include <linux/time.h> #include <linux/timex.h> #include <linux/init.h> #include <linux/ioport.h> @@ -37,8 +38,6 @@ #include <asm/isa.h> #include <asm/starfire.h> -extern rwlock_t xtime_lock; - spinlock_t mostek_lock = SPIN_LOCK_UNLOCKED; spinlock_t rtc_lock = SPIN_LOCK_UNLOCKED; unsigned long mstk48t02_regs = 0UL; @@ -134,7 +133,7 @@ static void timer_interrupt(int irq, void *dev_id, struct pt_regs * regs) { unsigned long ticks, pstate; - write_lock(&xtime_lock); + write_seqlock(&xtime_lock); do { #ifndef CONFIG_SMP @@ -196,13 +195,13 @@ static void timer_interrupt(int irq, void *dev_id, struct pt_regs * regs) timer_check_rtc(); - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); } #ifdef CONFIG_SMP void timer_tick_interrupt(struct pt_regs *regs) { - write_lock(&xtime_lock); + write_seqlock(&xtime_lock); do_timer(regs); @@ -225,7 +224,7 @@ void timer_tick_interrupt(struct pt_regs *regs) timer_check_rtc(); - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); } #endif @@ -665,7 +664,7 @@ void do_settimeofday(struct timeval *tv) if (this_is_starfire) return; - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); /* * This is revolting. We need to set "xtime" correctly. However, the * value in this location is the value at the most recent update of @@ -686,7 +685,7 @@ void do_settimeofday(struct timeval *tv) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } /* Ok, my cute asm atomicity trick doesn't work anymore. @@ -696,18 +695,20 @@ void do_settimeofday(struct timeval *tv) void do_gettimeofday(struct timeval *tv) { unsigned long flags; + unsigned long seq; unsigned long usec, sec; - read_lock_irqsave(&xtime_lock, flags); - usec = do_gettimeoffset(); - { - unsigned long lost = jiffies - wall_jiffies; - if (lost) - usec += lost * (1000000 / HZ); - } - sec = xtime.tv_sec; - usec += (xtime.tv_nsec / 1000); - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); + usec = do_gettimeoffset(); + { + unsigned long lost = jiffies - wall_jiffies; + if (lost) + usec += lost * (1000000 / HZ); + } + sec = xtime.tv_sec; + usec += (xtime.tv_nsec / 1000); + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); while (usec >= 1000000) { usec -= 1000000; diff --git a/arch/um/kernel/time_kern.c b/arch/um/kernel/time_kern.c index 713d7af4a696..f59b31a1eb50 100644 --- a/arch/um/kernel/time_kern.c +++ b/arch/um/kernel/time_kern.c @@ -7,6 +7,7 @@ #include "linux/unistd.h" #include "linux/stddef.h" #include "linux/spinlock.h" +#include "linux/time.h" #include "linux/sched.h" #include "linux/interrupt.h" #include "linux/init.h" @@ -21,8 +22,6 @@ u64 jiffies_64; -extern rwlock_t xtime_lock; - int hz(void) { return(HZ); @@ -57,9 +56,9 @@ void boot_timer_handler(int sig) void um_timer(int irq, void *dev, struct pt_regs *regs) { do_timer(regs); - write_lock(&xtime_lock); + write_seqlock(&xtime_lock); timer(); - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); } long um_time(int * tloc) diff --git a/arch/v850/kernel/time.c b/arch/v850/kernel/time.c index 7d65183a95e9..a1ecf796407c 100644 --- a/arch/v850/kernel/time.c +++ b/arch/v850/kernel/time.c @@ -17,6 +17,7 @@ #include <linux/string.h> #include <linux/mm.h> #include <linux/interrupt.h> +#include <linux/time.h> #include <linux/timex.h> #include <linux/profile.h> @@ -107,8 +108,6 @@ static void timer_interrupt (int irq, void *dummy, struct pt_regs *regs) #endif /* 0 */ } -extern rwlock_t xtime_lock; - /* * This version of gettimeofday has near microsecond resolution. */ @@ -120,21 +119,24 @@ void do_gettimeofday (struct timeval *tv) #endif unsigned long flags; unsigned long usec, sec; + unsigned long seq; + + do { + seq = read_seqbegin_irqsave(&xtime_lock, flags); - read_lock_irqsave (&xtime_lock, flags); #if 0 - usec = mach_gettimeoffset ? mach_gettimeoffset () : 0; + usec = mach_gettimeoffset ? mach_gettimeoffset () : 0; #else - usec = 0; + usec = 0; #endif #if 0 /* DAVIDM later if possible */ - lost = lost_ticks; - if (lost) - usec += lost * (1000000/HZ); + lost = lost_ticks; + if (lost) + usec += lost * (1000000/HZ); #endif - sec = xtime.tv_sec; - usec += xtime.tv_nsec / 1000; - read_unlock_irqrestore (&xtime_lock, flags); + sec = xtime.tv_sec; + usec += xtime.tv_nsec / 1000; + } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); while (usec >= 1000000) { usec -= 1000000; @@ -147,7 +149,7 @@ void do_gettimeofday (struct timeval *tv) void do_settimeofday (struct timeval *tv) { - write_lock_irq (&xtime_lock); + write_seqlock_irq (&xtime_lock); /* This is revolting. We need to set the xtime.tv_nsec * correctly. However, the value in this location is @@ -172,7 +174,7 @@ void do_settimeofday (struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq (&xtime_lock); + write_sequnlock_irq (&xtime_lock); } static int timer_dev_id; diff --git a/arch/x86_64/kernel/time.c b/arch/x86_64/kernel/time.c index 77d0696d320c..8fe37d578416 100644 --- a/arch/x86_64/kernel/time.c +++ b/arch/x86_64/kernel/time.c @@ -18,6 +18,7 @@ #include <linux/init.h> #include <linux/mc146818rtc.h> #include <linux/irq.h> +#include <linux/time.h> #include <linux/ioport.h> #include <linux/module.h> #include <linux/device.h> @@ -27,7 +28,6 @@ u64 jiffies_64; -extern rwlock_t xtime_lock; spinlock_t rtc_lock = SPIN_LOCK_UNLOCKED; unsigned int cpu_khz; /* TSC clocks / usec, not used here */ @@ -70,21 +70,22 @@ inline unsigned int do_gettimeoffset(void) void do_gettimeofday(struct timeval *tv) { - unsigned long flags, t; + unsigned long flags, t, seq; unsigned int sec, usec; - read_lock_irqsave(&xtime_lock, flags); - spin_lock(&time_offset_lock); + spin_lock_irqsave(&time_offset_lock, flags); + do { + seq = read_seqbegin(&xtime_lock); - sec = xtime.tv_sec; - usec = xtime.tv_nsec / 1000; + sec = xtime.tv_sec; + usec = xtime.tv_nsec / 1000; - t = (jiffies - wall_jiffies) * (1000000L / HZ) + do_gettimeoffset(); - if (t > timeoffset) timeoffset = t; - usec += timeoffset; + t = (jiffies - wall_jiffies) * (1000000L / HZ) + do_gettimeoffset(); + if (t > timeoffset) timeoffset = t; + usec += timeoffset; - spin_unlock(&time_offset_lock); - read_unlock_irqrestore(&xtime_lock, flags); + } while (read_seqretry(&xtime_lock, seq)); + spin_unlock_irqrestore(&time_offset_lock, flags); tv->tv_sec = sec + usec / 1000000; tv->tv_usec = usec % 1000000; @@ -98,7 +99,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); vxtime_lock(); tv->tv_usec -= do_gettimeoffset() + @@ -118,7 +119,7 @@ void do_settimeofday(struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } /* @@ -201,7 +202,7 @@ static void timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) * variables, because both do_timer() and us change them -arca+vojtech */ - write_lock(&xtime_lock); + write_seqlock(&xtime_lock); vxtime_lock(); { @@ -251,7 +252,7 @@ static void timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) } vxtime_unlock(); - write_unlock(&xtime_lock); + write_sequnlock(&xtime_lock); } unsigned long get_cmos_time(void) diff --git a/include/linux/jiffies.h b/include/linux/jiffies.h index c81b51bab1e3..0a60a4f52077 100644 --- a/include/linux/jiffies.h +++ b/include/linux/jiffies.h @@ -3,6 +3,7 @@ #include <linux/types.h> #include <linux/spinlock.h> +#include <linux/seqlock.h> #include <asm/system.h> #include <asm/param.h> /* for HZ */ @@ -17,13 +18,15 @@ extern unsigned long volatile jiffies; static inline u64 get_jiffies_64(void) { #if BITS_PER_LONG < 64 - extern rwlock_t xtime_lock; - unsigned long flags; + extern seqlock_t xtime_lock; + unsigned long seq; u64 tmp; - read_lock_irqsave(&xtime_lock, flags); - tmp = jiffies_64; - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = read_seqbegin(&xtime_lock); + tmp = jiffies_64; + } while (read_seqretry(&xtime_lock, seq)); + return tmp; #else return (u64)jiffies; diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h new file mode 100644 index 000000000000..2660cf7634c1 --- /dev/null +++ b/include/linux/seqlock.h @@ -0,0 +1,123 @@ +#ifndef __LINUX_SEQLOCK_H +#define __LINUX_SEQLOCK_H +/* + * Reader/writer consistent mechanism without starving writers. This type of + * lock for data where the reader wants a consitent set of information + * and is willing to retry if the information changes. Readers never + * block but they may have to retry if a writer is in + * progress. Writers do not wait for readers. + * + * This is not as cache friendly as brlock. Also, this will not work + * for data that contains pointers, because any writer could + * invalidate a pointer that a reader was following. + * + * Expected reader usage: + * do { + * seq = read_seqbegin(&foo); + * ... + * } while (read_seqretry(&foo, seq)); + * + * + * On non-SMP the spin locks disappear but the writer still needs + * to increment the sequence variables because an interrupt routine could + * change the state of the data. + * + * Based on x86_64 vsyscall gettimeofday + * by Keith Owens and Andrea Arcangeli + */ + +#include <linux/config.h> +#include <linux/spinlock.h> +#include <linux/preempt.h> + +typedef struct { + unsigned sequence; + spinlock_t lock; +} seqlock_t; + +/* + * These macros triggered gcc-3.x compile-time problems. We think these are + * OK now. Be cautious. + */ +#define SEQLOCK_UNLOCKED { 0, SPIN_LOCK_UNLOCKED } +#define seqlock_init(x) do { *(x) = (seqlock_t) SEQLOCK_UNLOCKED; } while (0) + + +/* Lock out other writers and update the count. + * Acts like a normal spin_lock/unlock. + * Don't need preempt_disable() because that is in the spin_lock already. + */ +static inline void write_seqlock(seqlock_t *sl) +{ + spin_lock(&sl->lock); + ++sl->sequence; + smp_wmb(); +} + +static inline void write_sequnlock(seqlock_t *sl) +{ + smp_wmb(); + sl->sequence++; + spin_unlock(&sl->lock); +} + +static inline int write_tryseqlock(seqlock_t *sl) +{ + int ret = spin_trylock(&sl->lock); + + if (ret) { + ++sl->sequence; + smp_wmb(); + } + return ret; +} + +/* Start of read calculation -- fetch last complete writer token */ +static inline unsigned read_seqbegin(const seqlock_t *sl) +{ + unsigned ret = sl->sequence; + smp_rmb(); + return ret; +} + +/* Test if reader processed invalid data. + * If initial values is odd, + * then writer had already started when section was entered + * If sequence value changed + * then writer changed data while in section + * + * Using xor saves one conditional branch. + */ +static inline int read_seqretry(const seqlock_t *sl, unsigned iv) +{ + smp_rmb(); + return (iv & 1) | (sl->sequence ^ iv); +} + +/* + * Possible sw/hw IRQ protected versions of the interfaces. + */ +#define write_seqlock_irqsave(lock, flags) \ + do { local_irq_save(flags); write_seqlock(lock); } while (0) +#define write_seqlock_irq(lock) \ + do { local_irq_disable(); write_seqlock(lock); } while (0) +#define write_seqlock_bh(lock) \ + do { local_bh_disable(); write_seqlock(lock); } while (0) + +#define write_sequnlock_irqrestore(lock, flags) \ + do { write_sequnlock(lock); local_irq_restore(flags); } while(0) +#define write_sequnlock_irq(lock) \ + do { write_sequnlock(lock); local_irq_enable(); } while(0) +#define write_sequnlock_bh(lock) \ + do { write_sequnlock(lock); local_bh_enable(); } while(0) + +#define read_seqbegin_irqsave(lock, flags) \ + ({ local_irq_save(flags); read_seqbegin(lock); }) + +#define read_seqretry_irqrestore(lock, iv, flags) \ + ({int ret = read_seqretry(&(lock)->seq, iv); \ + local_irq_restore(flags); \ + ret; \ + }) + +#endif /* __LINUX_SEQLOCK_H */ diff --git a/include/linux/time.h b/include/linux/time.h index 52d60ec2b364..7355ae1f78ca 100644 --- a/include/linux/time.h +++ b/include/linux/time.h @@ -25,6 +25,7 @@ struct timezone { #ifdef __KERNEL__ #include <linux/spinlock.h> +#include <linux/seqlock.h> /* * Change timeval to jiffies, trying to avoid the @@ -120,7 +121,7 @@ mktime (unsigned int year, unsigned int mon, } extern struct timespec xtime; -extern rwlock_t xtime_lock; +extern seqlock_t xtime_lock; static inline unsigned long get_seconds(void) { diff --git a/kernel/ksyms.c b/kernel/ksyms.c index aec2b90e5fe5..4d7bfe2accde 100644 --- a/kernel/ksyms.c +++ b/kernel/ksyms.c @@ -55,6 +55,7 @@ #include <linux/dnotify.h> #include <linux/mount.h> #include <linux/ptrace.h> +#include <linux/time.h> #include <asm/checksum.h> #if defined(CONFIG_PROC_FS) @@ -485,6 +486,7 @@ EXPORT_SYMBOL(kernel_flag); EXPORT_SYMBOL(jiffies); EXPORT_SYMBOL(jiffies_64); EXPORT_SYMBOL(xtime); +EXPORT_SYMBOL(xtime_lock); EXPORT_SYMBOL(do_gettimeofday); EXPORT_SYMBOL(do_settimeofday); #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP diff --git a/kernel/time.c b/kernel/time.c index ead1e3c7fb29..c8c8a10eae1f 100644 --- a/kernel/time.c +++ b/kernel/time.c @@ -36,9 +36,6 @@ */ struct timezone sys_tz; -/* The xtime_lock is not only serializing the xtime read/writes but it's also - serializing all accesses to the global NTP variables now. */ -extern rwlock_t xtime_lock; extern unsigned long last_time_offset; #if !defined(__alpha__) && !defined(__ia64__) @@ -80,7 +77,7 @@ asmlinkage long sys_stime(int * tptr) return -EPERM; if (get_user(value, tptr)) return -EFAULT; - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); xtime.tv_sec = value; xtime.tv_nsec = 0; last_time_offset = 0; @@ -88,7 +85,7 @@ asmlinkage long sys_stime(int * tptr) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); return 0; } @@ -96,13 +93,13 @@ asmlinkage long sys_stime(int * tptr) asmlinkage long sys_gettimeofday(struct timeval *tv, struct timezone *tz) { - if (tv) { + if (likely(tv != NULL)) { struct timeval ktv; do_gettimeofday(&ktv); if (copy_to_user(tv, &ktv, sizeof(ktv))) return -EFAULT; } - if (tz) { + if (unlikely(tz != NULL)) { if (copy_to_user(tz, &sys_tz, sizeof(sys_tz))) return -EFAULT; } @@ -127,10 +124,10 @@ asmlinkage long sys_gettimeofday(struct timeval *tv, struct timezone *tz) */ inline static void warp_clock(void) { - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); xtime.tv_sec += sys_tz.tz_minuteswest * 60; last_time_offset = 0; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); } /* @@ -235,7 +232,7 @@ int do_adjtimex(struct timex *txc) txc->tick > 1100000/USER_HZ) return -EINVAL; - write_lock_irq(&xtime_lock); + write_seqlock_irq(&xtime_lock); result = time_state; /* mostly `TIME_OK' */ /* Save for later - semantics of adjtime is to return old value */ @@ -386,7 +383,7 @@ leave: if ((time_status & (STA_UNSYNC|STA_CLOCKERR)) != 0 txc->errcnt = pps_errcnt; txc->stbcnt = pps_stbcnt; last_time_offset = 0; - write_unlock_irq(&xtime_lock); + write_sequnlock_irq(&xtime_lock); do_gettimeofday(&txc->time); return(result); } @@ -409,9 +406,13 @@ asmlinkage long sys_adjtimex(struct timex *txc_p) struct timespec current_kernel_time(void) { struct timespec now; - unsigned long flags; - read_lock_irqsave(&xtime_lock,flags); - now = xtime; - read_unlock_irqrestore(&xtime_lock,flags); + unsigned long seq; + + do { + seq = read_seqbegin(&xtime_lock); + + now = xtime; + } while (read_seqretry(&xtime_lock, seq)); + return now; } diff --git a/kernel/timer.c b/kernel/timer.c index e78ef95d6b64..2c297124b6d9 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -26,6 +26,7 @@ #include <linux/mm.h> #include <linux/notifier.h> #include <linux/thread_info.h> +#include <linux/time.h> #include <linux/jiffies.h> #include <asm/uaccess.h> @@ -760,7 +761,7 @@ unsigned long wall_jiffies; * This read-write spinlock protects us from races in SMP while * playing with xtime and avenrun. */ -rwlock_t xtime_lock __cacheline_aligned_in_smp = RW_LOCK_UNLOCKED; +seqlock_t xtime_lock __cacheline_aligned_in_smp = SEQLOCK_UNLOCKED; unsigned long last_time_offset; /* @@ -801,7 +802,7 @@ static inline void update_times(void) /* * The 64-bit jiffies value is not atomic - you MUST NOT read it - * without holding read_lock_irq(&xtime_lock). + * without sampling the sequence number in xtime_lock. * jiffies is defined in the linker script... */ @@ -1090,20 +1091,23 @@ asmlinkage long sys_sysinfo(struct sysinfo *info) u64 uptime; unsigned long mem_total, sav_total; unsigned int mem_unit, bitcount; + unsigned long seq; memset((char *)&val, 0, sizeof(struct sysinfo)); - read_lock_irq(&xtime_lock); - uptime = jiffies_64; - do_div(uptime, HZ); - val.uptime = (unsigned long) uptime; + do { + seq = read_seqbegin(&xtime_lock); + + uptime = jiffies_64; + do_div(uptime, HZ); + val.uptime = (unsigned long) uptime; - val.loads[0] = avenrun[0] << (SI_LOAD_SHIFT - FSHIFT); - val.loads[1] = avenrun[1] << (SI_LOAD_SHIFT - FSHIFT); - val.loads[2] = avenrun[2] << (SI_LOAD_SHIFT - FSHIFT); + val.loads[0] = avenrun[0] << (SI_LOAD_SHIFT - FSHIFT); + val.loads[1] = avenrun[1] << (SI_LOAD_SHIFT - FSHIFT); + val.loads[2] = avenrun[2] << (SI_LOAD_SHIFT - FSHIFT); - val.procs = nr_threads; - read_unlock_irq(&xtime_lock); + val.procs = nr_threads; + } while (read_seqretry(&xtime_lock, seq)); si_meminfo(&val); si_swapinfo(&val); @@ -1148,7 +1152,7 @@ asmlinkage long sys_sysinfo(struct sysinfo *info) val.totalhigh <<= bitcount; val.freehigh <<= bitcount; -out: + out: if (copy_to_user(info, &val, sizeof(struct sysinfo))) return -EFAULT; |
