Commit 88db792b authored by Sebastian Andrzej Siewior's avatar Sebastian Andrzej Siewior Committed by Paul E. McKenney
Browse files

rcu-tasks: Use rcuwait for the rcu_tasks_kthread() 



The waitqueue used by rcu_tasks_kthread() has always only one waiter.
With a guaranteed only one waiter, this can be replaced with rcuwait
which is smaller and simpler. With rcuwait based wake counterpart, the
irqwork function (call_rcu_tasks_iw_wakeup()) can be invoked hardirq
context because it is only a wake up and no sleeping locks are involved
(unlike the wait_queue_head).
As a side effect, this is also one piece of the puzzle to pass the RCU
selftest at early boot on PREEMPT_RT.

Replace wait_queue_head with rcuwait and let the irqwork run in hardirq
context on PREEMPT_RT.

Signed-off-by: default avatarSebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent f2539003

kernel/rcu/tasks.h

+8 −6
Original line number Diff line number Diff line
@@ -46,7 +46,7 @@ struct rcu_tasks_percpu { 


/**

 * struct rcu_tasks - Definition for a Tasks-RCU-like mechanism.

 * @cbs_wq: Wait queue allowing new callback to get kthread's attention.

 * @cbs_wait: RCU wait allowing a new callback to get kthread's attention.

 * @cbs_gbl_lock: Lock protecting callback list.

 * @kthread_ptr: This flavor's grace-period/callback-invocation kthread.

 * @gp_func: This flavor's grace-period-wait function.
@@ -77,7 +77,7 @@ struct rcu_tasks_percpu { 
 * @kname: This flavor's kthread name.

 */

struct rcu_tasks {

	struct wait_queue_head cbs_wq;

	struct rcuwait cbs_wait;

	raw_spinlock_t cbs_gbl_lock;

	int gp_state;

	int gp_sleep;
@@ -113,11 +113,11 @@ static void call_rcu_tasks_iw_wakeup(struct irq_work *iwp); 
#define DEFINE_RCU_TASKS(rt_name, gp, call, n)						\

static DEFINE_PER_CPU(struct rcu_tasks_percpu, rt_name ## __percpu) = {			\

	.lock = __RAW_SPIN_LOCK_UNLOCKED(rt_name ## __percpu.cbs_pcpu_lock),		\

	.rtp_irq_work = IRQ_WORK_INIT(call_rcu_tasks_iw_wakeup),			\

	.rtp_irq_work = IRQ_WORK_INIT_HARD(call_rcu_tasks_iw_wakeup),			\

};											\

static struct rcu_tasks rt_name =							\

{											\

	.cbs_wq = __WAIT_QUEUE_HEAD_INITIALIZER(rt_name.cbs_wq),			\

	.cbs_wait = __RCUWAIT_INITIALIZER(rt_name.wait),				\

	.cbs_gbl_lock = __RAW_SPIN_LOCK_UNLOCKED(rt_name.cbs_gbl_lock),			\

	.gp_func = gp,									\

	.call_func = call,								\
@@ -266,7 +266,7 @@ static void call_rcu_tasks_iw_wakeup(struct irq_work *iwp) 
	struct rcu_tasks_percpu *rtpcp = container_of(iwp, struct rcu_tasks_percpu, rtp_irq_work);



	rtp = rtpcp->rtpp;

	wake_up(&rtp->cbs_wq);

	rcuwait_wake_up(&rtp->cbs_wait);

}



// Enqueue a callback for the specified flavor of Tasks RCU.
@@ -514,7 +514,9 @@ static int __noreturn rcu_tasks_kthread(void *arg) 
		set_tasks_gp_state(rtp, RTGS_WAIT_CBS);



		/* If there were none, wait a bit and start over. */

		wait_event_idle(rtp->cbs_wq, (needgpcb = rcu_tasks_need_gpcb(rtp)));

		rcuwait_wait_event(&rtp->cbs_wait,

				   (needgpcb = rcu_tasks_need_gpcb(rtp)),

				   TASK_IDLE);



		if (needgpcb & 0x2) {

			// Wait for one grace period.