core/cpu: make cpu idle states simpler

Rework the CPU idle state code:

* in_idle is true for any kind of idle including spinning. This is not
  used anywhere except for state assertions for now.

* in_sleep is true for idle that requires an IPI to wake up.

* in_job_sleep is true for in_sleep idle which is also cpu_wake_on_job.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Cédric Le Goater <clg@kaod.org>

2 files changed, 53 insertions, 27 deletions

diff --git a/core/cpu.c b/core/cpu.c
index 2a3f708..e851647 100644
--- a/core/cpu.c
+++ b/core/cpu.c
@@ -106,14 +106,6 @@ static void cpu_send_ipi(struct cpu_thread *cpu)
 	}
 }
 
-static void cpu_wake(struct cpu_thread *cpu)
-{
-	/* Is it idle ? If not, no need to wake */
-	sync();
-	if (cpu->in_idle)
-		cpu_send_ipi(cpu);
-}
-
 /*
  * If chip_id is >= 0, schedule the job on that node.
  * Otherwise schedule the job anywhere.
@@ -200,7 +192,10 @@ static void queue_job_on_cpu(struct cpu_thread *cpu, struct cpu_job *job)
 		cpu->job_count++;
 	unlock(&cpu->job_lock);
 
-	cpu_wake(cpu);
+	/* Is it idle waiting for jobs? If so, must send an IPI. */
+	sync();
+	if (cpu->in_job_sleep)
+		cpu_send_ipi(cpu);
 }
 
 struct cpu_job *__cpu_queue_job(struct cpu_thread *cpu,
@@ -394,13 +389,21 @@ static unsigned int cpu_idle_p8(enum cpu_wake_cause wake_on)
 	/* Clean up ICP, be ready for IPIs */
 	icp_prep_for_pm();
 
+	/* Mark outselves sleeping so wake ups know to send an IPI */
+	cpu->in_sleep = true;
+
 	/* Synchronize with wakers */
 	if (wake_on == cpu_wake_on_job) {
-		/* Mark ourselves in idle so other CPUs know to send an IPI */
-		cpu->in_idle = true;
+		/* Mark ourselves in job sleep so queueing jobs send an IPI */
+		cpu->in_job_sleep = true;
+
+		/*
+		 * make stores to in_sleep and in_job_sleep visible before
+		 * checking jobs and testing reconfigure_idle
+		 */
 		sync();
 
-		/* Check for jobs again */
+		/* Check for jobs again, of if PM got disabled */
 		if (cpu_check_jobs(cpu) || reconfigure_idle)
 			goto skip_sleep;
 
@@ -409,10 +412,10 @@ static unsigned int cpu_idle_p8(enum cpu_wake_cause wake_on)
 		mtspr(SPR_LPCR, lpcr);
 
 	} else {
-		/* Mark outselves sleeping so cpu_set_pm_enable knows to
-		 * send an IPI
+		/*
+		 * make store to in_sleep visible before testing
+		 * reconfigure_idle
 		 */
-		cpu->in_sleep = true;
 		sync();
 
 		/* Check if PM got disabled */
@@ -431,8 +434,8 @@ static unsigned int cpu_idle_p8(enum cpu_wake_cause wake_on)
 skip_sleep:
 	/* Restore */
 	sync();
-	cpu->in_idle = false;
 	cpu->in_sleep = false;
+	cpu->in_job_sleep = false;
 	reset_cpu_icp();
 
 	return vec;
@@ -450,23 +453,31 @@ static unsigned int cpu_idle_p9(enum cpu_wake_cause wake_on)
 		return vec;
 	}
 
+	/* Mark outselves sleeping so wake ups know to send an IPI */
+	cpu->in_sleep = true;
+
 	/* Synchronize with wakers */
 	if (wake_on == cpu_wake_on_job) {
-		/* Mark ourselves in idle so other CPUs know to send an IPI */
-		cpu->in_idle = true;
+		/* Mark ourselves in job sleep so queueing jobs send an IPI */
+		cpu->in_job_sleep = true;
+
+		/*
+		 * make stores to in_sleep and in_job_sleep visible before
+		 * checking jobs and testing reconfigure_idle
+		 */
 		sync();
 
-		/* Check for jobs again */
+		/* Check for jobs again, of if PM got disabled */
 		if (cpu_check_jobs(cpu) || reconfigure_idle)
 			goto skip_sleep;
 
 		/* HV DBELL for IPI */
 		lpcr |= SPR_LPCR_P9_PECEL1;
 	} else {
-		/* Mark outselves sleeping so cpu_set_pm_enable knows to
-		 * send an IPI
+		/*
+		 * make store to in_sleep visible before testing
+		 * reconfigure_idle
 		 */
-		cpu->in_sleep = true;
 		sync();
 
 		/* Check if PM got disabled */
@@ -499,8 +510,8 @@ static unsigned int cpu_idle_p9(enum cpu_wake_cause wake_on)
  skip_sleep:
 	/* Restore */
 	sync();
-	cpu->in_idle = false;
 	cpu->in_sleep = false;
+	cpu->in_job_sleep = false;
 
 	return vec;
 }
@@ -549,10 +560,17 @@ static int nr_cpus_idle = 0;
 
 static void enter_idle(void)
 {
+	struct cpu_thread *cpu = this_cpu();
+
+	assert(!cpu->in_idle);
+	assert(!cpu->in_sleep);
+	assert(!cpu->in_job_sleep);
+
 	for (;;) {
 		lock(&idle_lock);
 		if (!reconfigure_idle) {
 			nr_cpus_idle++;
+			cpu->in_idle = true;
 			break;
 		}
 		unlock(&idle_lock);
@@ -569,9 +587,16 @@ static void enter_idle(void)
 
 static void exit_idle(void)
 {
+	struct cpu_thread *cpu = this_cpu();
+
+	assert(cpu->in_idle);
+	assert(!cpu->in_sleep);
+	assert(!cpu->in_job_sleep);
+
 	lock(&idle_lock);
 	assert(nr_cpus_idle > 0);
 	nr_cpus_idle--;
+	cpu->in_idle = false;
 	unlock(&idle_lock);
 }
 
@@ -606,12 +631,12 @@ static void reconfigure_idle_start(void)
 
 	/*
 	 * Order earlier store to reconfigure_idle=true vs load from
-	 * cpu->in_sleep and cpu->in_idle.
+	 * cpu->in_sleep.
 	 */
 	sync();
 
 	for_each_available_cpu(cpu) {
-		if (cpu->in_sleep || cpu->in_idle)
+		if (cpu->in_sleep)
 			cpu_send_ipi(cpu);
 	}
 
diff --git a/include/cpu.h b/include/cpu.h
index b0c78ce..d0fc6cc 100644
--- a/include/cpu.h
+++ b/include/cpu.h
@@ -60,8 +60,9 @@ struct cpu_thread {
 	bool				in_poller;
 	bool				in_reinit;
 	bool				in_fast_sleep;
-	bool				in_sleep;
-	bool				in_idle;
+	bool				in_idle; /* any idle state, even busy wait */
+	bool				in_sleep; /* idle which requires IPI */
+	bool				in_job_sleep; /* requires IPI and cpu_wake_on_job */
 	uint32_t			hbrt_spec_wakeup; /* primary only */
 	uint64_t			save_l2_fir_action1;
 	uint64_t			current_token;