archives/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-03-13 21:26:14 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'rcu-misc.20260111a'

Browse source 
* rcu-misc.20260111a:
  rcu: Reduce synchronize_rcu() latency by reporting GP kthread's CPU QS early
  srcu: Use suitable gfp_flags for the init_srcu_struct_nodes()
  rcu: Fix rcu_read_unlock() deadloop due to softirq
  rcutorture: Correctly compute probability to invoke ->exp_current()
  rcu: Make expedited RCU CPU stall warnings detect stall-end races
This commit is contained in:
Boqun Feng 2026-01-11 20:15:07 +08:00
commit fe1d482884
6 changed files with 31 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

4
kernel/rcu/rcutorture.c
View file

@ -1749,7 +1749,7 @@ rcu_torture_writer(void *arg)
ulo[i] = cur_ops->get_comp_state();
gp_snap = cur_ops->start_gp_poll();
rcu_torture_writer_state = RTWS_POLL_WAIT;
if (cur_ops->exp_current && !torture_random(&rand) % 0xff)
if (cur_ops->exp_current && !(torture_random(&rand) & 0xff))
cur_ops->exp_current();
while (!cur_ops->poll_gp_state(gp_snap)) {
gp_snap1 = cur_ops->get_gp_state();
@ -1771,7 +1771,7 @@ rcu_torture_writer(void *arg)
cur_ops->get_comp_state_full(&rgo[i]);
cur_ops->start_gp_poll_full(&gp_snap_full);
rcu_torture_writer_state = RTWS_POLL_WAIT_FULL;
if (cur_ops->exp_current && !torture_random(&rand) % 0xff)
if (cur_ops->exp_current && !(torture_random(&rand) & 0xff))
cur_ops->exp_current();
while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
cur_ops->get_gp_state_full(&gp_snap1_full);

2
kernel/rcu/srcutree.c
View file

@ -262,7 +262,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
ssp->srcu_sup->srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL;
ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns();
if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) {
if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC))
if (!init_srcu_struct_nodes(ssp, is_static ? GFP_ATOMIC : GFP_KERNEL))
goto err_free_sda;
WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG);
}

12
kernel/rcu/tree.c
View file

@ -160,6 +160,7 @@ static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
unsigned long gps, unsigned long flags);
static void invoke_rcu_core(void);
static void rcu_report_exp_rdp(struct rcu_data *rdp);
static void rcu_report_qs_rdp(struct rcu_data *rdp);
static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp);
static bool rcu_rdp_is_offloaded(struct rcu_data *rdp);
static bool rcu_rdp_cpu_online(struct rcu_data *rdp);
@ -1983,6 +1984,17 @@ static noinline_for_stack bool rcu_gp_init(void)
if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
on_each_cpu(rcu_strict_gp_boundary, NULL, 0);
/*
* Immediately report QS for the GP kthread's CPU. The GP kthread
* cannot be in an RCU read-side critical section while running
* the FQS scan. This eliminates the need for a second FQS wait
* when all CPUs are idle.
*/
preempt_disable();
rcu_qs();
rcu_report_qs_rdp(this_cpu_ptr(&rcu_data));
preempt_enable();
return true;
}

2
kernel/rcu/tree.h
View file

@ -203,7 +203,7 @@ struct rcu_data {
/* during and after the last grace */
/* period it is aware of. */
struct irq_work defer_qs_iw; /* Obtain later scheduler attention. */
int defer_qs_iw_pending; /* Scheduler attention pending? */
int defer_qs_pending; /* irqwork or softirq pending? */
struct work_struct strict_work; /* Schedule readers for strict GPs. */
/* 2) batch handling */

7
kernel/rcu/tree_exp.h
View file

@ -589,7 +589,12 @@ static void synchronize_rcu_expedited_stall(unsigned long jiffies_start, unsigne
pr_cont(" } %lu jiffies s: %lu root: %#lx/%c\n",
j - jiffies_start, rcu_state.expedited_sequence, data_race(rnp_root->expmask),
".T"[!!data_race(rnp_root->exp_tasks)]);
if (ndetected) {
if (!ndetected) {
// This is invoked from the grace-period worker, so
// a new grace period cannot have started. And if this
// worker were stalled, we would not get here. ;-)
pr_err("INFO: Expedited stall ended before state dump start\n");
} else {
pr_err("blocking rcu_node structures (internal RCU debug):");
rcu_for_each_node_breadth_first(rnp) {
if (rnp == rnp_root)

15
kernel/rcu/tree_plugin.h
View file

@ -487,8 +487,8 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags)
union rcu_special special;
rdp = this_cpu_ptr(&rcu_data);
if (rdp->defer_qs_iw_pending == DEFER_QS_PENDING)
rdp->defer_qs_iw_pending = DEFER_QS_IDLE;
if (rdp->defer_qs_pending == DEFER_QS_PENDING)
rdp->defer_qs_pending = DEFER_QS_IDLE;
/*
* If RCU core is waiting for this CPU to exit its critical section,
@ -645,7 +645,7 @@ static void rcu_preempt_deferred_qs_handler(struct irq_work *iwp)
* 5. Deferred QS reporting does not happen.
*/
if (rcu_preempt_depth() > 0)
WRITE_ONCE(rdp->defer_qs_iw_pending, DEFER_QS_IDLE);
WRITE_ONCE(rdp->defer_qs_pending, DEFER_QS_IDLE);
}
/*
@ -747,7 +747,10 @@ static void rcu_read_unlock_special(struct task_struct *t)
// Using softirq, safe to awaken, and either the
// wakeup is free or there is either an expedited
// GP in flight or a potential need to deboost.
raise_softirq_irqoff(RCU_SOFTIRQ);
if (rdp->defer_qs_pending != DEFER_QS_PENDING) {
rdp->defer_qs_pending = DEFER_QS_PENDING;
raise_softirq_irqoff(RCU_SOFTIRQ);
}
} else {
// Enabling BH or preempt does reschedule, so...
// Also if no expediting and no possible deboosting,
@ -755,11 +758,11 @@ static void rcu_read_unlock_special(struct task_struct *t)
// tick enabled.
set_need_resched_current();
if (IS_ENABLED(CONFIG_IRQ_WORK) && irqs_were_disabled &&
needs_exp && rdp->defer_qs_iw_pending != DEFER_QS_PENDING &&
needs_exp && rdp->defer_qs_pending != DEFER_QS_PENDING &&
cpu_online(rdp->cpu)) {
// Get scheduler to re-evaluate and call hooks.
// If !IRQ_WORK, FQS scan will eventually IPI.
rdp->defer_qs_iw_pending = DEFER_QS_PENDING;
rdp->defer_qs_pending = DEFER_QS_PENDING;
irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu);
}
}