Commit 0356d4e6 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu-tasks: Track blocked RCU Tasks Trace readers 



This commit places any task that has ever blocked within its current
RCU Tasks Trace read-side critical section on a per-CPU list within the
rcu_tasks_percpu structure.  Tasks are removed from this list when they
exit by the exit_tasks_rcu_finish_trace() function.  The purpose of this
commit is to provide the information needed to eliminate the current
scan of the full task list.

This commit offsets the INT_MIN value for ->trc_reader_nesting with the
new nesting level in order to avoid queueing tasks that are exiting
their read-side critical sections.

[ paulmck: Apply kernel test robot feedback. ]
[ paulmck: Apply feedback from syzbot+9bb26e7c5e8e4fa7e641@syzkaller.appspotmail.com ]

Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
Tested-by: default avatarsyzbot <syzbot+9bb26e7c5e8e4fa7e641@syzkaller.appspotmail.com>
Tested-by: default avatar"Zhang, Qiang1" <qiang1.zhang@intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Martin KaFai Lau <kafai@fb.com>
Cc: KP Singh <kpsingh@kernel.org>
parent 434c9eef

include/linux/rcupdate.h

+9 −2
Original line number Diff line number Diff line
@@ -174,12 +174,19 @@ void synchronize_rcu_tasks(void); 
#define TRC_NEED_QS_CHECKED	0x2  // Task has been checked for needing quiescent state.



u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new);

void rcu_tasks_trace_qs_blkd(struct task_struct *t);



# define rcu_tasks_trace_qs(t)							\

	do {									\

		int ___rttq_nesting = READ_ONCE((t)->trc_reader_nesting);	\

										\

		if (likely(!READ_ONCE((t)->trc_reader_special.b.need_qs)) &&	\

		    likely(!READ_ONCE((t)->trc_reader_nesting)))		\

		    likely(!___rttq_nesting)) {					\

			rcu_trc_cmpxchg_need_qs((t), 0,	TRC_NEED_QS_CHECKED);	\

		} else if (___rttq_nesting && ___rttq_nesting != INT_MIN &&	\

			   !READ_ONCE((t)->trc_reader_special.b.blocked)) {	\

			rcu_tasks_trace_qs_blkd(t);				\

		}								\

	} while (0)

# else

# define rcu_tasks_trace_qs(t) do { } while (0)
@@ -188,7 +195,7 @@ u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new); 
#define rcu_tasks_qs(t, preempt)					\

do {									\

	rcu_tasks_classic_qs((t), (preempt));				\

	rcu_tasks_trace_qs((t));					\

	rcu_tasks_trace_qs(t);						\

} while (0)



# ifdef CONFIG_TASKS_RUDE_RCU

include/linux/rcupdate_trace.h

+1 −1
Original line number Diff line number Diff line
@@ -75,7 +75,7 @@ static inline void rcu_read_unlock_trace(void) 
	nesting = READ_ONCE(t->trc_reader_nesting) - 1;

	barrier(); // Critical section before disabling.

	// Disable IPI-based setting of .need_qs.

	WRITE_ONCE(t->trc_reader_nesting, INT_MIN);

	WRITE_ONCE(t->trc_reader_nesting, INT_MIN + nesting);

	if (likely(!READ_ONCE(t->trc_reader_special.s)) || nesting) {

		WRITE_ONCE(t->trc_reader_nesting, nesting);

		return;  // We assume shallow reader nesting.

kernel/rcu/tasks.h

+21 −1
Original line number Diff line number Diff line
@@ -1261,6 +1261,24 @@ void rcu_read_unlock_trace_special(struct task_struct *t) 
}

EXPORT_SYMBOL_GPL(rcu_read_unlock_trace_special);



/* Add a newly blocked reader task to its CPU's list. */

void rcu_tasks_trace_qs_blkd(struct task_struct *t)

{

	unsigned long flags;

	struct rcu_tasks_percpu *rtpcp;



	local_irq_save(flags);

	rtpcp = this_cpu_ptr(rcu_tasks_trace.rtpcpu);

	raw_spin_lock_rcu_node(rtpcp); // irqs already disabled

	t->trc_blkd_cpu = smp_processor_id();

	if (!rtpcp->rtp_blkd_tasks.next)

		INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks);

	list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks);

	t->trc_reader_special.b.blocked = true;

	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);

}

EXPORT_SYMBOL_GPL(rcu_tasks_trace_qs_blkd);



/* Add a task to the holdout list, if it is not already on the list. */

static void trc_add_holdout(struct task_struct *t, struct list_head *bhp)

{
@@ -1586,9 +1604,11 @@ static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp) 
/* Report any needed quiescent state for this exiting task. */

static void exit_tasks_rcu_finish_trace(struct task_struct *t)

{

	union rcu_special trs = READ_ONCE(t->trc_reader_special);



	rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED);

	WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting));

	if (WARN_ON_ONCE(rcu_ld_need_qs(t) & TRC_NEED_QS))

	if (WARN_ON_ONCE(rcu_ld_need_qs(t) & TRC_NEED_QS) || trs.b.blocked)

		rcu_read_unlock_trace_special(t);

	else

		WRITE_ONCE(t->trc_reader_nesting, 0);