Commit 0bcb3868 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu-tasks: Untrack blocked RCU Tasks Trace at reader end 



This commit causes rcu_read_unlock_trace() to check for the current
task being on a per-CPU list within the rcu_tasks_percpu structure,
and removes it from that list if so.  This has the effect of curtailing
tracking of a task that blocked within an RCU Tasks Trace read-side
critical section once it exits that critical section.

Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
Cc: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Martin KaFai Lau <kafai@fb.com>
Cc: KP Singh <kpsingh@kernel.org>
parent 0356d4e6

kernel/rcu/tasks.h

+18 −6
Original line number Diff line number Diff line
@@ -1245,17 +1245,29 @@ EXPORT_SYMBOL_GPL(rcu_trc_cmpxchg_need_qs); 
/* If we are the last reader, wake up the grace-period kthread. */

void rcu_read_unlock_trace_special(struct task_struct *t)

{

	int nqs = (rcu_ld_need_qs(t) == (TRC_NEED_QS_CHECKED | TRC_NEED_QS));

	unsigned long flags;

	struct rcu_tasks_percpu *rtpcp;

	union rcu_special trs;



	// Open-coded full-word version of rcu_ld_need_qs().

	smp_mb(); // Enforce full grace-period ordering.

	trs = smp_load_acquire(&t->trc_reader_special);



	if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) && t->trc_reader_special.b.need_mb)

		smp_mb(); // Pairs with update-side barriers.

	// Update .need_qs before ->trc_reader_nesting for irq/NMI handlers.

	if (nqs) {

	if (trs.b.need_qs == (TRC_NEED_QS_CHECKED | TRC_NEED_QS)) {

		u8 result = rcu_trc_cmpxchg_need_qs(t, TRC_NEED_QS_CHECKED | TRC_NEED_QS,

						       TRC_NEED_QS_CHECKED);



		WARN_ONCE(result != (TRC_NEED_QS_CHECKED | TRC_NEED_QS),

			  "%s: result = %d", __func__, result);

		WARN_ONCE(result != trs.b.need_qs, "%s: result = %d", __func__, result);

	}

	if (trs.b.blocked) {

		rtpcp = per_cpu_ptr(rcu_tasks_trace.rtpcpu, t->trc_blkd_cpu);

		raw_spin_lock_irqsave_rcu_node(rtpcp, flags);

		list_del_init(&t->trc_blkd_node);

		WRITE_ONCE(t->trc_reader_special.b.blocked, false);

		raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);

	}

	WRITE_ONCE(t->trc_reader_nesting, 0);

}
@@ -1274,7 +1286,7 @@ void rcu_tasks_trace_qs_blkd(struct task_struct *t) 
	if (!rtpcp->rtp_blkd_tasks.next)

		INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks);

	list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks);

	t->trc_reader_special.b.blocked = true;

	WRITE_ONCE(t->trc_reader_special.b.blocked, true);

	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);

}

EXPORT_SYMBOL_GPL(rcu_tasks_trace_qs_blkd);
@@ -1608,7 +1620,7 @@ static void exit_tasks_rcu_finish_trace(struct task_struct *t) 


	rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED);

	WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting));

	if (WARN_ON_ONCE(rcu_ld_need_qs(t) & TRC_NEED_QS) || trs.b.blocked)

	if (WARN_ON_ONCE(rcu_ld_need_qs(t) & TRC_NEED_QS || trs.b.blocked))

		rcu_read_unlock_trace_special(t);

	else

		WRITE_ONCE(t->trc_reader_nesting, 0);