Commit d96225fd authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu-tasks: Split rcu_tasks_one_gp() from rcu_tasks_kthread() 



This commit abstracts most of the rcu_tasks_kthread() function's loop
body into a new rcu_tasks_one_gp() function.  It also introduces
a new ->tasks_gp_mutex to synchronize concurrent calls to this new
rcu_tasks_one_gp() function.  This commit is preparation for allowing
RCU tasks grace periods to be driven by the calling task during the
mid-boot dead zone.

Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent 4cf0585c

kernel/rcu/tasks.h

+36 −22
Original line number Diff line number Diff line
@@ -48,6 +48,7 @@ struct rcu_tasks_percpu { 
 * struct rcu_tasks - Definition for a Tasks-RCU-like mechanism.

 * @cbs_wait: RCU wait allowing a new callback to get kthread's attention.

 * @cbs_gbl_lock: Lock protecting callback list.

 * @tasks_gp_mutex: Mutex protecting grace period, needed during mid-boot dead zone.

 * @kthread_ptr: This flavor's grace-period/callback-invocation kthread.

 * @gp_func: This flavor's grace-period-wait function.

 * @gp_state: Grace period's most recent state transition (debugging).
@@ -79,6 +80,7 @@ struct rcu_tasks_percpu { 
struct rcu_tasks {

	struct rcuwait cbs_wait;

	raw_spinlock_t cbs_gbl_lock;

	struct mutex tasks_gp_mutex;

	int gp_state;

	int gp_sleep;

	int init_fract;
@@ -119,6 +121,7 @@ static struct rcu_tasks rt_name = \ 
{											\

	.cbs_wait = __RCUWAIT_INITIALIZER(rt_name.wait),				\

	.cbs_gbl_lock = __RAW_SPIN_LOCK_UNLOCKED(rt_name.cbs_gbl_lock),			\

	.tasks_gp_mutex = __MUTEX_INITIALIZER(rt_name.tasks_gp_mutex),			\

	.gp_func = gp,									\

	.call_func = call,								\

	.rtpcpu = &rt_name ## __percpu,							\
@@ -502,26 +505,15 @@ static void rcu_tasks_invoke_cbs_wq(struct work_struct *wp) 
	rcu_tasks_invoke_cbs(rtp, rtpcp);

}



/* RCU-tasks kthread that detects grace periods and invokes callbacks. */

static int __noreturn rcu_tasks_kthread(void *arg)

// Wait for one grace period.

static void rcu_tasks_one_gp(struct rcu_tasks *rtp)

{

	int needgpcb;

	struct rcu_tasks *rtp = arg;



	/* Run on housekeeping CPUs by default.  Sysadm can move if desired. */

	housekeeping_affine(current, HK_TYPE_RCU);

	WRITE_ONCE(rtp->kthread_ptr, current); // Let GPs start!



	/*

	 * Each pass through the following loop makes one check for

	 * newly arrived callbacks, and, if there are some, waits for

	 * one RCU-tasks grace period and then invokes the callbacks.

	 * This loop is terminated by the system going down.  ;-)

	 */

	for (;;) {

	mutex_lock(&rtp->tasks_gp_mutex);

	set_tasks_gp_state(rtp, RTGS_WAIT_CBS);



		/* If there were none, wait a bit and start over. */

	// If there were none, wait a bit and start over.

	rcuwait_wait_event(&rtp->cbs_wait,

			   (needgpcb = rcu_tasks_need_gpcb(rtp)),

			   TASK_IDLE);
@@ -535,11 +527,33 @@ static int __noreturn rcu_tasks_kthread(void *arg) 
		rcu_seq_end(&rtp->tasks_gp_seq);

	}



		/* Invoke callbacks. */

	// Invoke callbacks.

	set_tasks_gp_state(rtp, RTGS_INVOKE_CBS);

	rcu_tasks_invoke_cbs(rtp, per_cpu_ptr(rtp->rtpcpu, 0));

	mutex_unlock(&rtp->tasks_gp_mutex);

}



// RCU-tasks kthread that detects grace periods and invokes callbacks.

static int __noreturn rcu_tasks_kthread(void *arg)

{

	struct rcu_tasks *rtp = arg;



	/* Run on housekeeping CPUs by default.  Sysadm can move if desired. */

	housekeeping_affine(current, HK_TYPE_RCU);

	WRITE_ONCE(rtp->kthread_ptr, current); // Let GPs start!



	/*

	 * Each pass through the following loop makes one check for

	 * newly arrived callbacks, and, if there are some, waits for

	 * one RCU-tasks grace period and then invokes the callbacks.

	 * This loop is terminated by the system going down.  ;-)

	 */

	for (;;) {

		// Wait for one grace period and invoke any callbacks

		// that are ready.

		rcu_tasks_one_gp(rtp);



		/* Paranoid sleep to keep this from entering a tight loop */

		// Paranoid sleep to keep this from entering a tight loop.

		schedule_timeout_idle(rtp->gp_sleep);

	}

}