From 267f685b8b20784c97251618b515fcd17b42aad6 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Sun, 28 Aug 2016 03:45:14 +0200 Subject: cpus-common: move CPU list management to common code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a mutex for the CPU list to system emulation, as it will be used to manage safe work. Abstract manipulation of the CPU list in new functions cpu_list_add and cpu_list_remove. Reviewed-by: Richard Henderson Reviewed-by: Alex Bennée Signed-off-by: Paolo Bonzini --- cpus-common.c | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 cpus-common.c (limited to 'cpus-common.c') diff --git a/cpus-common.c b/cpus-common.c new file mode 100644 index 0000000..fda3848 --- /dev/null +++ b/cpus-common.c @@ -0,0 +1,83 @@ +/* + * CPU thread main loop - common bits for user and system mode emulation + * + * Copyright (c) 2003-2005 Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#include "qemu/osdep.h" +#include "exec/cpu-common.h" +#include "qom/cpu.h" +#include "sysemu/cpus.h" + +static QemuMutex qemu_cpu_list_lock; + +void qemu_init_cpu_list(void) +{ + qemu_mutex_init(&qemu_cpu_list_lock); +} + +void cpu_list_lock(void) +{ + qemu_mutex_lock(&qemu_cpu_list_lock); +} + +void cpu_list_unlock(void) +{ + qemu_mutex_unlock(&qemu_cpu_list_lock); +} + +static bool cpu_index_auto_assigned; + +static int cpu_get_free_index(void) +{ + CPUState *some_cpu; + int cpu_index = 0; + + cpu_index_auto_assigned = true; + CPU_FOREACH(some_cpu) { + cpu_index++; + } + return cpu_index; +} + +void cpu_list_add(CPUState *cpu) +{ + qemu_mutex_lock(&qemu_cpu_list_lock); + if (cpu->cpu_index == UNASSIGNED_CPU_INDEX) { + cpu->cpu_index = cpu_get_free_index(); + assert(cpu->cpu_index != UNASSIGNED_CPU_INDEX); + } else { + assert(!cpu_index_auto_assigned); + } + QTAILQ_INSERT_TAIL(&cpus, cpu, node); + qemu_mutex_unlock(&qemu_cpu_list_lock); +} + +void cpu_list_remove(CPUState *cpu) +{ + qemu_mutex_lock(&qemu_cpu_list_lock); + if (!QTAILQ_IN_USE(cpu, node)) { + /* there is nothing to undo since cpu_exec_init() hasn't been called */ + qemu_mutex_unlock(&qemu_cpu_list_lock); + return; + } + + assert(!(cpu_index_auto_assigned && cpu != QTAILQ_LAST(&cpus, CPUTailQ))); + + QTAILQ_REMOVE(&cpus, cpu, node); + cpu->cpu_index = UNASSIGNED_CPU_INDEX; + qemu_mutex_unlock(&qemu_cpu_list_lock); +} -- cgit v1.1 From d148d90ee83738d45a90dc0b2fb7b1712f164103 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Mon, 29 Aug 2016 09:51:00 +0200 Subject: cpus-common: move CPU work item management to common code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make CPU work core functions common between system and user-mode emulation. User-mode does not use run_on_cpu, so do not implement it. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Alex Bennée Signed-off-by: Alex Bennée Message-Id: <1470158864-17651-10-git-send-email-alex.bennee@linaro.org> Reviewed-by: Richard Henderson Signed-off-by: Paolo Bonzini --- cpus-common.c | 94 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) (limited to 'cpus-common.c') diff --git a/cpus-common.c b/cpus-common.c index fda3848..2005bfe 100644 --- a/cpus-common.c +++ b/cpus-common.c @@ -23,10 +23,12 @@ #include "sysemu/cpus.h" static QemuMutex qemu_cpu_list_lock; +static QemuCond qemu_work_cond; void qemu_init_cpu_list(void) { qemu_mutex_init(&qemu_cpu_list_lock); + qemu_cond_init(&qemu_work_cond); } void cpu_list_lock(void) @@ -81,3 +83,95 @@ void cpu_list_remove(CPUState *cpu) cpu->cpu_index = UNASSIGNED_CPU_INDEX; qemu_mutex_unlock(&qemu_cpu_list_lock); } + +struct qemu_work_item { + struct qemu_work_item *next; + run_on_cpu_func func; + void *data; + int done; + bool free; +}; + +static void queue_work_on_cpu(CPUState *cpu, struct qemu_work_item *wi) +{ + qemu_mutex_lock(&cpu->work_mutex); + if (cpu->queued_work_first == NULL) { + cpu->queued_work_first = wi; + } else { + cpu->queued_work_last->next = wi; + } + cpu->queued_work_last = wi; + wi->next = NULL; + wi->done = false; + qemu_mutex_unlock(&cpu->work_mutex); + + qemu_cpu_kick(cpu); +} + +void do_run_on_cpu(CPUState *cpu, run_on_cpu_func func, void *data, + QemuMutex *mutex) +{ + struct qemu_work_item wi; + + if (qemu_cpu_is_self(cpu)) { + func(cpu, data); + return; + } + + wi.func = func; + wi.data = data; + wi.free = false; + + queue_work_on_cpu(cpu, &wi); + while (!atomic_mb_read(&wi.done)) { + CPUState *self_cpu = current_cpu; + + qemu_cond_wait(&qemu_work_cond, mutex); + current_cpu = self_cpu; + } +} + +void async_run_on_cpu(CPUState *cpu, run_on_cpu_func func, void *data) +{ + struct qemu_work_item *wi; + + if (qemu_cpu_is_self(cpu)) { + func(cpu, data); + return; + } + + wi = g_malloc0(sizeof(struct qemu_work_item)); + wi->func = func; + wi->data = data; + wi->free = true; + + queue_work_on_cpu(cpu, wi); +} + +void process_queued_cpu_work(CPUState *cpu) +{ + struct qemu_work_item *wi; + + if (cpu->queued_work_first == NULL) { + return; + } + + qemu_mutex_lock(&cpu->work_mutex); + while (cpu->queued_work_first != NULL) { + wi = cpu->queued_work_first; + cpu->queued_work_first = wi->next; + if (!cpu->queued_work_first) { + cpu->queued_work_last = NULL; + } + qemu_mutex_unlock(&cpu->work_mutex); + wi->func(cpu, wi->data); + qemu_mutex_lock(&cpu->work_mutex); + if (wi->free) { + g_free(wi); + } else { + atomic_mb_set(&wi->done, true); + } + } + qemu_mutex_unlock(&cpu->work_mutex); + qemu_cond_broadcast(&qemu_work_cond); +} -- cgit v1.1 From 0e55539c076a61b0b10a1aea1158fc20fb159d99 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 6 Sep 2016 17:28:03 +0200 Subject: cpus-common: fix uninitialized variable use in run_on_cpu MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Alex Bennée Reviewed-by: Richard Henderson Signed-off-by: Paolo Bonzini --- cpus-common.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cpus-common.c') diff --git a/cpus-common.c b/cpus-common.c index 2005bfe..d6cd426 100644 --- a/cpus-common.c +++ b/cpus-common.c @@ -88,8 +88,7 @@ struct qemu_work_item { struct qemu_work_item *next; run_on_cpu_func func; void *data; - int done; - bool free; + bool free, done; }; static void queue_work_on_cpu(CPUState *cpu, struct qemu_work_item *wi) @@ -120,6 +119,7 @@ void do_run_on_cpu(CPUState *cpu, run_on_cpu_func func, void *data, wi.func = func; wi.data = data; + wi.done = false; wi.free = false; queue_work_on_cpu(cpu, &wi); -- cgit v1.1 From ab129972c8b41e15b0521895a46fd9c752b68a5e Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 31 Aug 2016 16:56:04 +0200 Subject: cpus-common: move exclusive work infrastructure from linux-user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will serve as the base for async_safe_run_on_cpu. Because start_exclusive uses CPU_FOREACH, merge exclusive_lock with qemu_cpu_list_lock: together with a call to exclusive_idle (via cpu_exec_start/end) in cpu_list_add, this protects exclusive work against concurrent CPU addition and removal. Reviewed-by: Alex Bennée Reviewed-by: Richard Henderson Signed-off-by: Paolo Bonzini --- cpus-common.c | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) (limited to 'cpus-common.c') diff --git a/cpus-common.c b/cpus-common.c index d6cd426..7d935fd 100644 --- a/cpus-common.c +++ b/cpus-common.c @@ -23,11 +23,21 @@ #include "sysemu/cpus.h" static QemuMutex qemu_cpu_list_lock; +static QemuCond exclusive_cond; +static QemuCond exclusive_resume; static QemuCond qemu_work_cond; +static int pending_cpus; + void qemu_init_cpu_list(void) { + /* This is needed because qemu_init_cpu_list is also called by the + * child process in a fork. */ + pending_cpus = 0; + qemu_mutex_init(&qemu_cpu_list_lock); + qemu_cond_init(&exclusive_cond); + qemu_cond_init(&exclusive_resume); qemu_cond_init(&qemu_work_cond); } @@ -55,6 +65,12 @@ static int cpu_get_free_index(void) return cpu_index; } +static void finish_safe_work(CPUState *cpu) +{ + cpu_exec_start(cpu); + cpu_exec_end(cpu); +} + void cpu_list_add(CPUState *cpu) { qemu_mutex_lock(&qemu_cpu_list_lock); @@ -66,6 +82,8 @@ void cpu_list_add(CPUState *cpu) } QTAILQ_INSERT_TAIL(&cpus, cpu, node); qemu_mutex_unlock(&qemu_cpu_list_lock); + + finish_safe_work(cpu); } void cpu_list_remove(CPUState *cpu) @@ -148,6 +166,70 @@ void async_run_on_cpu(CPUState *cpu, run_on_cpu_func func, void *data) queue_work_on_cpu(cpu, wi); } +/* Wait for pending exclusive operations to complete. The CPU list lock + must be held. */ +static inline void exclusive_idle(void) +{ + while (pending_cpus) { + qemu_cond_wait(&exclusive_resume, &qemu_cpu_list_lock); + } +} + +/* Start an exclusive operation. + Must only be called from outside cpu_exec, takes + qemu_cpu_list_lock. */ +void start_exclusive(void) +{ + CPUState *other_cpu; + + qemu_mutex_lock(&qemu_cpu_list_lock); + exclusive_idle(); + + /* Make all other cpus stop executing. */ + pending_cpus = 1; + CPU_FOREACH(other_cpu) { + if (other_cpu->running) { + pending_cpus++; + qemu_cpu_kick(other_cpu); + } + } + while (pending_cpus > 1) { + qemu_cond_wait(&exclusive_cond, &qemu_cpu_list_lock); + } +} + +/* Finish an exclusive operation. Releases qemu_cpu_list_lock. */ +void end_exclusive(void) +{ + pending_cpus = 0; + qemu_cond_broadcast(&exclusive_resume); + qemu_mutex_unlock(&qemu_cpu_list_lock); +} + +/* Wait for exclusive ops to finish, and begin cpu execution. */ +void cpu_exec_start(CPUState *cpu) +{ + qemu_mutex_lock(&qemu_cpu_list_lock); + exclusive_idle(); + cpu->running = true; + qemu_mutex_unlock(&qemu_cpu_list_lock); +} + +/* Mark cpu as not executing, and release pending exclusive ops. */ +void cpu_exec_end(CPUState *cpu) +{ + qemu_mutex_lock(&qemu_cpu_list_lock); + cpu->running = false; + if (pending_cpus > 1) { + pending_cpus--; + if (pending_cpus == 1) { + qemu_cond_signal(&exclusive_cond); + } + } + exclusive_idle(); + qemu_mutex_unlock(&qemu_cpu_list_lock); +} + void process_queued_cpu_work(CPUState *cpu) { struct qemu_work_item *wi; -- cgit v1.1 From c978b3168727d3a76ffcb18462ea972f50b53634 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 31 Aug 2016 18:03:39 +0200 Subject: cpus-common: always defer async_run_on_cpu work items MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit async_run_on_cpu is only called from the I/O thread, not from CPU threads, so it doesn't make any difference. It will make a difference however for async_safe_run_on_cpu. Reviewed-by: Alex Bennée Reviewed-by: Richard Henderson Signed-off-by: Paolo Bonzini --- cpus-common.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'cpus-common.c') diff --git a/cpus-common.c b/cpus-common.c index 7d935fd..115f3d4 100644 --- a/cpus-common.c +++ b/cpus-common.c @@ -153,11 +153,6 @@ void async_run_on_cpu(CPUState *cpu, run_on_cpu_func func, void *data) { struct qemu_work_item *wi; - if (qemu_cpu_is_self(cpu)) { - func(cpu, data); - return; - } - wi = g_malloc0(sizeof(struct qemu_work_item)); wi->func = func; wi->data = data; -- cgit v1.1 From cf07da65f335b9a74e62f5413078f67280572f36 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 2 Sep 2016 21:02:10 +0200 Subject: cpus-common: remove redundant call to exclusive_idle() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No need to call exclusive_idle() from cpu_exec_end since it is done immediately afterwards in cpu_exec_start. Any exclusive section could run as soon as cpu_exec_end leaves, because cpu->running is false and the mutex is not taken, so the call does not add any protection either. Reviewed-by: Richard Henderson Reviewed-by: Alex Bennée Signed-off-by: Paolo Bonzini --- cpus-common.c | 1 - 1 file changed, 1 deletion(-) (limited to 'cpus-common.c') diff --git a/cpus-common.c b/cpus-common.c index 115f3d4..80aaf9b 100644 --- a/cpus-common.c +++ b/cpus-common.c @@ -221,7 +221,6 @@ void cpu_exec_end(CPUState *cpu) qemu_cond_signal(&exclusive_cond); } } - exclusive_idle(); qemu_mutex_unlock(&qemu_cpu_list_lock); } -- cgit v1.1 From 758e1b2b622d7c177dc2d95e887a11aa069b7e68 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 2 Sep 2016 23:33:38 +0200 Subject: cpus-common: simplify locking for start_exclusive/end_exclusive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is not necessary to hold qemu_cpu_list_mutex throughout the exclusive section, because no other exclusive section can run while pending_cpus != 0. exclusive_idle() is called in cpu_exec_start(), and that prevents any CPUs created after start_exclusive() from entering cpu_exec() during an exclusive section. Reviewed-by: Richard Henderson Reviewed-by: Alex Bennée Signed-off-by: Paolo Bonzini --- cpus-common.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'cpus-common.c') diff --git a/cpus-common.c b/cpus-common.c index 80aaf9b..429652c 100644 --- a/cpus-common.c +++ b/cpus-common.c @@ -171,8 +171,7 @@ static inline void exclusive_idle(void) } /* Start an exclusive operation. - Must only be called from outside cpu_exec, takes - qemu_cpu_list_lock. */ + Must only be called from outside cpu_exec. */ void start_exclusive(void) { CPUState *other_cpu; @@ -191,11 +190,17 @@ void start_exclusive(void) while (pending_cpus > 1) { qemu_cond_wait(&exclusive_cond, &qemu_cpu_list_lock); } + + /* Can release mutex, no one will enter another exclusive + * section until end_exclusive resets pending_cpus to 0. + */ + qemu_mutex_unlock(&qemu_cpu_list_lock); } -/* Finish an exclusive operation. Releases qemu_cpu_list_lock. */ +/* Finish an exclusive operation. */ void end_exclusive(void) { + qemu_mutex_lock(&qemu_cpu_list_lock); pending_cpus = 0; qemu_cond_broadcast(&exclusive_resume); qemu_mutex_unlock(&qemu_cpu_list_lock); -- cgit v1.1 From 53f5ed95064fe6807890cd5535445a05d3361bd2 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Sun, 28 Aug 2016 05:38:24 +0200 Subject: cpus-common: Introduce async_safe_run_on_cpu() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Richard Henderson Reviewed-by: Alex Bennée Signed-off-by: Paolo Bonzini --- cpus-common.c | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) (limited to 'cpus-common.c') diff --git a/cpus-common.c b/cpus-common.c index 429652c..38b1d55 100644 --- a/cpus-common.c +++ b/cpus-common.c @@ -18,6 +18,7 @@ */ #include "qemu/osdep.h" +#include "qemu/main-loop.h" #include "exec/cpu-common.h" #include "qom/cpu.h" #include "sysemu/cpus.h" @@ -106,7 +107,7 @@ struct qemu_work_item { struct qemu_work_item *next; run_on_cpu_func func; void *data; - bool free, done; + bool free, exclusive, done; }; static void queue_work_on_cpu(CPUState *cpu, struct qemu_work_item *wi) @@ -139,6 +140,7 @@ void do_run_on_cpu(CPUState *cpu, run_on_cpu_func func, void *data, wi.data = data; wi.done = false; wi.free = false; + wi.exclusive = false; queue_work_on_cpu(cpu, &wi); while (!atomic_mb_read(&wi.done)) { @@ -229,6 +231,19 @@ void cpu_exec_end(CPUState *cpu) qemu_mutex_unlock(&qemu_cpu_list_lock); } +void async_safe_run_on_cpu(CPUState *cpu, run_on_cpu_func func, void *data) +{ + struct qemu_work_item *wi; + + wi = g_malloc0(sizeof(struct qemu_work_item)); + wi->func = func; + wi->data = data; + wi->free = true; + wi->exclusive = true; + + queue_work_on_cpu(cpu, wi); +} + void process_queued_cpu_work(CPUState *cpu) { struct qemu_work_item *wi; @@ -245,7 +260,21 @@ void process_queued_cpu_work(CPUState *cpu) cpu->queued_work_last = NULL; } qemu_mutex_unlock(&cpu->work_mutex); - wi->func(cpu, wi->data); + if (wi->exclusive) { + /* Running work items outside the BQL avoids the following deadlock: + * 1) start_exclusive() is called with the BQL taken while another + * CPU is running; 2) cpu_exec in the other CPU tries to takes the + * BQL, so it goes to sleep; start_exclusive() is sleeping too, so + * neither CPU can proceed. + */ + qemu_mutex_unlock_iothread(); + start_exclusive(); + wi->func(cpu, wi->data); + end_exclusive(); + qemu_mutex_lock_iothread(); + } else { + wi->func(cpu, wi->data); + } qemu_mutex_lock(&cpu->work_mutex); if (wi->free) { g_free(wi); -- cgit v1.1 From c265e976f4669fd65f5b47e6865f50d1cb66bd02 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 31 Aug 2016 21:33:58 +0200 Subject: cpus-common: lock-free fast path for cpu_exec_start/end Set cpu->running without taking the cpu_list lock, only requiring it if there is a concurrent exclusive section. This requires adding a new field to CPUState, which records whether a running CPU is being counted in pending_cpus. When an exclusive section is started concurrently with cpu_exec_start, cpu_exec_start can use the new field to determine if it has to wait for the end of the exclusive section. Likewise, cpu_exec_end can use it to see if start_exclusive is waiting for that CPU. This a separate patch for easier bisection of issues. Signed-off-by: Paolo Bonzini --- cpus-common.c | 95 +++++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 80 insertions(+), 15 deletions(-) (limited to 'cpus-common.c') diff --git a/cpus-common.c b/cpus-common.c index 38b1d55..3e11452 100644 --- a/cpus-common.c +++ b/cpus-common.c @@ -28,6 +28,9 @@ static QemuCond exclusive_cond; static QemuCond exclusive_resume; static QemuCond qemu_work_cond; +/* >= 1 if a thread is inside start_exclusive/end_exclusive. Written + * under qemu_cpu_list_lock, read with atomic operations. + */ static int pending_cpus; void qemu_init_cpu_list(void) @@ -177,18 +180,26 @@ static inline void exclusive_idle(void) void start_exclusive(void) { CPUState *other_cpu; + int running_cpus; qemu_mutex_lock(&qemu_cpu_list_lock); exclusive_idle(); /* Make all other cpus stop executing. */ - pending_cpus = 1; + atomic_set(&pending_cpus, 1); + + /* Write pending_cpus before reading other_cpu->running. */ + smp_mb(); + running_cpus = 0; CPU_FOREACH(other_cpu) { - if (other_cpu->running) { - pending_cpus++; + if (atomic_read(&other_cpu->running)) { + other_cpu->has_waiter = true; + running_cpus++; qemu_cpu_kick(other_cpu); } } + + atomic_set(&pending_cpus, running_cpus + 1); while (pending_cpus > 1) { qemu_cond_wait(&exclusive_cond, &qemu_cpu_list_lock); } @@ -203,7 +214,7 @@ void start_exclusive(void) void end_exclusive(void) { qemu_mutex_lock(&qemu_cpu_list_lock); - pending_cpus = 0; + atomic_set(&pending_cpus, 0); qemu_cond_broadcast(&exclusive_resume); qemu_mutex_unlock(&qemu_cpu_list_lock); } @@ -211,24 +222,78 @@ void end_exclusive(void) /* Wait for exclusive ops to finish, and begin cpu execution. */ void cpu_exec_start(CPUState *cpu) { - qemu_mutex_lock(&qemu_cpu_list_lock); - exclusive_idle(); - cpu->running = true; - qemu_mutex_unlock(&qemu_cpu_list_lock); + atomic_set(&cpu->running, true); + + /* Write cpu->running before reading pending_cpus. */ + smp_mb(); + + /* 1. start_exclusive saw cpu->running == true and pending_cpus >= 1. + * After taking the lock we'll see cpu->has_waiter == true and run---not + * for long because start_exclusive kicked us. cpu_exec_end will + * decrement pending_cpus and signal the waiter. + * + * 2. start_exclusive saw cpu->running == false but pending_cpus >= 1. + * This includes the case when an exclusive item is running now. + * Then we'll see cpu->has_waiter == false and wait for the item to + * complete. + * + * 3. pending_cpus == 0. Then start_exclusive is definitely going to + * see cpu->running == true, and it will kick the CPU. + */ + if (unlikely(atomic_read(&pending_cpus))) { + qemu_mutex_lock(&qemu_cpu_list_lock); + if (!cpu->has_waiter) { + /* Not counted in pending_cpus, let the exclusive item + * run. Since we have the lock, just set cpu->running to true + * while holding it; no need to check pending_cpus again. + */ + atomic_set(&cpu->running, false); + exclusive_idle(); + /* Now pending_cpus is zero. */ + atomic_set(&cpu->running, true); + } else { + /* Counted in pending_cpus, go ahead and release the + * waiter at cpu_exec_end. + */ + } + qemu_mutex_unlock(&qemu_cpu_list_lock); + } } /* Mark cpu as not executing, and release pending exclusive ops. */ void cpu_exec_end(CPUState *cpu) { - qemu_mutex_lock(&qemu_cpu_list_lock); - cpu->running = false; - if (pending_cpus > 1) { - pending_cpus--; - if (pending_cpus == 1) { - qemu_cond_signal(&exclusive_cond); + atomic_set(&cpu->running, false); + + /* Write cpu->running before reading pending_cpus. */ + smp_mb(); + + /* 1. start_exclusive saw cpu->running == true. Then it will increment + * pending_cpus and wait for exclusive_cond. After taking the lock + * we'll see cpu->has_waiter == true. + * + * 2. start_exclusive saw cpu->running == false but here pending_cpus >= 1. + * This includes the case when an exclusive item started after setting + * cpu->running to false and before we read pending_cpus. Then we'll see + * cpu->has_waiter == false and not touch pending_cpus. The next call to + * cpu_exec_start will run exclusive_idle if still necessary, thus waiting + * for the item to complete. + * + * 3. pending_cpus == 0. Then start_exclusive is definitely going to + * see cpu->running == false, and it can ignore this CPU until the + * next cpu_exec_start. + */ + if (unlikely(atomic_read(&pending_cpus))) { + qemu_mutex_lock(&qemu_cpu_list_lock); + if (cpu->has_waiter) { + cpu->has_waiter = false; + atomic_set(&pending_cpus, pending_cpus - 1); + if (pending_cpus == 1) { + qemu_cond_signal(&exclusive_cond); + } } + qemu_mutex_unlock(&qemu_cpu_list_lock); } - qemu_mutex_unlock(&qemu_cpu_list_lock); } void async_safe_run_on_cpu(CPUState *cpu, run_on_cpu_func func, void *data) -- cgit v1.1