diff options
Diffstat (limited to 'include/linux')
| -rw-r--r-- | include/linux/rcupdate.h | 48 |
1 files changed, 34 insertions, 14 deletions
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 58048abd7446..f9981251d542 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -65,11 +65,18 @@ struct rcu_head { /* Control variables for rcupdate callback mechanism. */ struct rcu_ctrlblk { - spinlock_t mutex; /* Guard this struct */ - long curbatch; /* Current batch number. */ - long maxbatch; /* Max requested batch number. */ - cpumask_t rcu_cpu_mask; /* CPUs that need to switch in order */ - /* for current batch to proceed. */ + /* "const" members: only changed when starting/ending a grace period */ + struct { + long cur; /* Current batch number. */ + long completed; /* Number of the last completed batch */ + } batch ____cacheline_maxaligned_in_smp; + /* remaining members: bookkeeping of the progress of the grace period */ + struct { + spinlock_t mutex; /* Guard this struct */ + int next_pending; /* Is the next batch already waiting? */ + cpumask_t rcu_cpu_mask; /* CPUs that need to switch */ + /* in order for current batch to proceed. */ + } state ____cacheline_maxaligned_in_smp; }; /* Is batch a before batch b ? */ @@ -90,9 +97,14 @@ static inline int rcu_batch_after(long a, long b) * curlist - current batch for which quiescent cycle started if any */ struct rcu_data { + /* 1) quiescent state handling : */ + long quiescbatch; /* Batch # for grace period */ long qsctr; /* User-mode/idle loop etc. */ long last_qsctr; /* value of qsctr at beginning */ /* of rcu grace period */ + int qs_pending; /* core waits for quiesc state */ + + /* 2) batch handling */ long batch; /* Batch # for current RCU batch */ struct list_head nxtlist; struct list_head curlist; @@ -101,24 +113,31 @@ struct rcu_data { DECLARE_PER_CPU(struct rcu_data, rcu_data); extern struct rcu_ctrlblk rcu_ctrlblk; +#define RCU_quiescbatch(cpu) (per_cpu(rcu_data, (cpu)).quiescbatch) #define RCU_qsctr(cpu) (per_cpu(rcu_data, (cpu)).qsctr) #define RCU_last_qsctr(cpu) (per_cpu(rcu_data, (cpu)).last_qsctr) +#define RCU_qs_pending(cpu) (per_cpu(rcu_data, (cpu)).qs_pending) #define RCU_batch(cpu) (per_cpu(rcu_data, (cpu)).batch) #define RCU_nxtlist(cpu) (per_cpu(rcu_data, (cpu)).nxtlist) #define RCU_curlist(cpu) (per_cpu(rcu_data, (cpu)).curlist) -#define RCU_QSCTR_INVALID 0 - static inline int rcu_pending(int cpu) { - if ((!list_empty(&RCU_curlist(cpu)) && - rcu_batch_before(RCU_batch(cpu), rcu_ctrlblk.curbatch)) || - (list_empty(&RCU_curlist(cpu)) && - !list_empty(&RCU_nxtlist(cpu))) || - cpu_isset(cpu, rcu_ctrlblk.rcu_cpu_mask)) + /* This cpu has pending rcu entries and the grace period + * for them has completed. + */ + if (!list_empty(&RCU_curlist(cpu)) && + !rcu_batch_before(rcu_ctrlblk.batch.completed,RCU_batch(cpu))) + return 1; + /* This cpu has no pending entries, but there are new entries */ + if (list_empty(&RCU_curlist(cpu)) && + !list_empty(&RCU_nxtlist(cpu))) + return 1; + /* The rcu core waits for a quiescent state from the cpu */ + if (RCU_quiescbatch(cpu) != rcu_ctrlblk.batch.cur || RCU_qs_pending(cpu)) return 1; - else - return 0; + /* nothing to do */ + return 0; } #define rcu_read_lock() preempt_disable() @@ -126,6 +145,7 @@ static inline int rcu_pending(int cpu) extern void rcu_init(void); extern void rcu_check_callbacks(int cpu, int user); +extern void rcu_restart_cpu(int cpu); /* Exported interfaces */ extern void FASTCALL(call_rcu(struct rcu_head *head, |