Commit 4a8cc433 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu-tasks: Drive synchronous grace periods from calling task 



This commit causes synchronous grace periods to be driven from the task
invoking synchronize_rcu_*(), allowing these functions to be invoked from
the mid-boot dead zone extending from when the scheduler was initialized
to to point that the various RCU tasks grace-period kthreads are spawned.
This change will allow the self-tests to run in a consistent manner.

Reported-by: default avatarMatthew Wilcox <willy@infradead.org>
Reported-by: default avatarZhouyi Zhou <zhouzhouyi@gmail.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent 68cb4720

kernel/rcu/tasks.h

+16 −8
Original line number Diff line number Diff line
@@ -495,17 +495,21 @@ static void rcu_tasks_invoke_cbs_wq(struct work_struct *wp) 
}



// Wait for one grace period.

static void rcu_tasks_one_gp(struct rcu_tasks *rtp)

static void rcu_tasks_one_gp(struct rcu_tasks *rtp, bool midboot)

{

	int needgpcb;



	mutex_lock(&rtp->tasks_gp_mutex);

	set_tasks_gp_state(rtp, RTGS_WAIT_CBS);



	// If there were none, wait a bit and start over.

	if (unlikely(midboot)) {

		needgpcb = 0x2;

	} else {

		set_tasks_gp_state(rtp, RTGS_WAIT_CBS);

		rcuwait_wait_event(&rtp->cbs_wait,

				   (needgpcb = rcu_tasks_need_gpcb(rtp)),

				   TASK_IDLE);

	}



	if (needgpcb & 0x2) {

		// Wait for one grace period.
@@ -540,7 +544,7 @@ static int __noreturn rcu_tasks_kthread(void *arg) 
	for (;;) {

		// Wait for one grace period and invoke any callbacks

		// that are ready.

		rcu_tasks_one_gp(rtp);

		rcu_tasks_one_gp(rtp, false);



		// Paranoid sleep to keep this from entering a tight loop.

		schedule_timeout_idle(rtp->gp_sleep);
@@ -554,8 +558,12 @@ static void synchronize_rcu_tasks_generic(struct rcu_tasks *rtp) 
	RCU_LOCKDEP_WARN(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE,

			 "synchronize_rcu_tasks called too soon");



	/* Wait for the grace period. */

	// If the grace-period kthread is running, use it.

	if (READ_ONCE(rtp->kthread_ptr)) {

		wait_rcu_gp(rtp->call_func);

		return;

	}

	rcu_tasks_one_gp(rtp, true);

}



/* Spawn RCU-tasks grace-period kthread. */