You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

cpus-common.c 10.0KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345
  1. /*
  2. * CPU thread main loop - common bits for user and system mode emulation
  3. *
  4. * Copyright (c) 2003-2005 Fabrice Bellard
  5. *
  6. * This library is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU Lesser General Public
  8. * License as published by the Free Software Foundation; either
  9. * version 2 of the License, or (at your option) any later version.
  10. *
  11. * This library is distributed in the hope that it will be useful,
  12. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. * Lesser General Public License for more details.
  15. *
  16. * You should have received a copy of the GNU Lesser General Public
  17. * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18. */
  19. #include "qemu/osdep.h"
  20. #include "qemu/main-loop.h"
  21. #include "exec/cpu-common.h"
  22. #include "hw/core/cpu.h"
  23. #include "sysemu/cpus.h"
  24. static QemuMutex qemu_cpu_list_lock;
  25. static QemuCond exclusive_cond;
  26. static QemuCond exclusive_resume;
  27. static QemuCond qemu_work_cond;
  28. /* >= 1 if a thread is inside start_exclusive/end_exclusive. Written
  29. * under qemu_cpu_list_lock, read with atomic operations.
  30. */
  31. static int pending_cpus;
  32. void qemu_init_cpu_list(void)
  33. {
  34. /* This is needed because qemu_init_cpu_list is also called by the
  35. * child process in a fork. */
  36. pending_cpus = 0;
  37. qemu_mutex_init(&qemu_cpu_list_lock);
  38. qemu_cond_init(&exclusive_cond);
  39. qemu_cond_init(&exclusive_resume);
  40. qemu_cond_init(&qemu_work_cond);
  41. }
  42. void cpu_list_lock(void)
  43. {
  44. qemu_mutex_lock(&qemu_cpu_list_lock);
  45. }
  46. void cpu_list_unlock(void)
  47. {
  48. qemu_mutex_unlock(&qemu_cpu_list_lock);
  49. }
  50. static bool cpu_index_auto_assigned;
  51. static int cpu_get_free_index(void)
  52. {
  53. CPUState *some_cpu;
  54. int cpu_index = 0;
  55. cpu_index_auto_assigned = true;
  56. CPU_FOREACH(some_cpu) {
  57. cpu_index++;
  58. }
  59. return cpu_index;
  60. }
  61. void cpu_list_add(CPUState *cpu)
  62. {
  63. qemu_mutex_lock(&qemu_cpu_list_lock);
  64. if (cpu->cpu_index == UNASSIGNED_CPU_INDEX) {
  65. cpu->cpu_index = cpu_get_free_index();
  66. assert(cpu->cpu_index != UNASSIGNED_CPU_INDEX);
  67. } else {
  68. assert(!cpu_index_auto_assigned);
  69. }
  70. QTAILQ_INSERT_TAIL_RCU(&cpus, cpu, node);
  71. qemu_mutex_unlock(&qemu_cpu_list_lock);
  72. }
  73. void cpu_list_remove(CPUState *cpu)
  74. {
  75. qemu_mutex_lock(&qemu_cpu_list_lock);
  76. if (!QTAILQ_IN_USE(cpu, node)) {
  77. /* there is nothing to undo since cpu_exec_init() hasn't been called */
  78. qemu_mutex_unlock(&qemu_cpu_list_lock);
  79. return;
  80. }
  81. assert(!(cpu_index_auto_assigned && cpu != QTAILQ_LAST(&cpus)));
  82. QTAILQ_REMOVE_RCU(&cpus, cpu, node);
  83. cpu->cpu_index = UNASSIGNED_CPU_INDEX;
  84. qemu_mutex_unlock(&qemu_cpu_list_lock);
  85. }
  86. struct qemu_work_item {
  87. struct qemu_work_item *next;
  88. run_on_cpu_func func;
  89. run_on_cpu_data data;
  90. bool free, exclusive, done;
  91. };
  92. static void queue_work_on_cpu(CPUState *cpu, struct qemu_work_item *wi)
  93. {
  94. qemu_mutex_lock(&cpu->work_mutex);
  95. if (cpu->queued_work_first == NULL) {
  96. cpu->queued_work_first = wi;
  97. } else {
  98. cpu->queued_work_last->next = wi;
  99. }
  100. cpu->queued_work_last = wi;
  101. wi->next = NULL;
  102. wi->done = false;
  103. qemu_mutex_unlock(&cpu->work_mutex);
  104. qemu_cpu_kick(cpu);
  105. }
  106. void do_run_on_cpu(CPUState *cpu, run_on_cpu_func func, run_on_cpu_data data,
  107. QemuMutex *mutex)
  108. {
  109. struct qemu_work_item wi;
  110. if (qemu_cpu_is_self(cpu)) {
  111. func(cpu, data);
  112. return;
  113. }
  114. wi.func = func;
  115. wi.data = data;
  116. wi.done = false;
  117. wi.free = false;
  118. wi.exclusive = false;
  119. queue_work_on_cpu(cpu, &wi);
  120. while (!atomic_mb_read(&wi.done)) {
  121. CPUState *self_cpu = current_cpu;
  122. qemu_cond_wait(&qemu_work_cond, mutex);
  123. current_cpu = self_cpu;
  124. }
  125. }
  126. void async_run_on_cpu(CPUState *cpu, run_on_cpu_func func, run_on_cpu_data data)
  127. {
  128. struct qemu_work_item *wi;
  129. wi = g_malloc0(sizeof(struct qemu_work_item));
  130. wi->func = func;
  131. wi->data = data;
  132. wi->free = true;
  133. queue_work_on_cpu(cpu, wi);
  134. }
  135. /* Wait for pending exclusive operations to complete. The CPU list lock
  136. must be held. */
  137. static inline void exclusive_idle(void)
  138. {
  139. while (pending_cpus) {
  140. qemu_cond_wait(&exclusive_resume, &qemu_cpu_list_lock);
  141. }
  142. }
  143. /* Start an exclusive operation.
  144. Must only be called from outside cpu_exec. */
  145. void start_exclusive(void)
  146. {
  147. CPUState *other_cpu;
  148. int running_cpus;
  149. qemu_mutex_lock(&qemu_cpu_list_lock);
  150. exclusive_idle();
  151. /* Make all other cpus stop executing. */
  152. atomic_set(&pending_cpus, 1);
  153. /* Write pending_cpus before reading other_cpu->running. */
  154. smp_mb();
  155. running_cpus = 0;
  156. CPU_FOREACH(other_cpu) {
  157. if (atomic_read(&other_cpu->running)) {
  158. other_cpu->has_waiter = true;
  159. running_cpus++;
  160. qemu_cpu_kick(other_cpu);
  161. }
  162. }
  163. atomic_set(&pending_cpus, running_cpus + 1);
  164. while (pending_cpus > 1) {
  165. qemu_cond_wait(&exclusive_cond, &qemu_cpu_list_lock);
  166. }
  167. /* Can release mutex, no one will enter another exclusive
  168. * section until end_exclusive resets pending_cpus to 0.
  169. */
  170. qemu_mutex_unlock(&qemu_cpu_list_lock);
  171. }
  172. /* Finish an exclusive operation. */
  173. void end_exclusive(void)
  174. {
  175. qemu_mutex_lock(&qemu_cpu_list_lock);
  176. atomic_set(&pending_cpus, 0);
  177. qemu_cond_broadcast(&exclusive_resume);
  178. qemu_mutex_unlock(&qemu_cpu_list_lock);
  179. }
  180. /* Wait for exclusive ops to finish, and begin cpu execution. */
  181. void cpu_exec_start(CPUState *cpu)
  182. {
  183. atomic_set(&cpu->running, true);
  184. /* Write cpu->running before reading pending_cpus. */
  185. smp_mb();
  186. /* 1. start_exclusive saw cpu->running == true and pending_cpus >= 1.
  187. * After taking the lock we'll see cpu->has_waiter == true and run---not
  188. * for long because start_exclusive kicked us. cpu_exec_end will
  189. * decrement pending_cpus and signal the waiter.
  190. *
  191. * 2. start_exclusive saw cpu->running == false but pending_cpus >= 1.
  192. * This includes the case when an exclusive item is running now.
  193. * Then we'll see cpu->has_waiter == false and wait for the item to
  194. * complete.
  195. *
  196. * 3. pending_cpus == 0. Then start_exclusive is definitely going to
  197. * see cpu->running == true, and it will kick the CPU.
  198. */
  199. if (unlikely(atomic_read(&pending_cpus))) {
  200. qemu_mutex_lock(&qemu_cpu_list_lock);
  201. if (!cpu->has_waiter) {
  202. /* Not counted in pending_cpus, let the exclusive item
  203. * run. Since we have the lock, just set cpu->running to true
  204. * while holding it; no need to check pending_cpus again.
  205. */
  206. atomic_set(&cpu->running, false);
  207. exclusive_idle();
  208. /* Now pending_cpus is zero. */
  209. atomic_set(&cpu->running, true);
  210. } else {
  211. /* Counted in pending_cpus, go ahead and release the
  212. * waiter at cpu_exec_end.
  213. */
  214. }
  215. qemu_mutex_unlock(&qemu_cpu_list_lock);
  216. }
  217. }
  218. /* Mark cpu as not executing, and release pending exclusive ops. */
  219. void cpu_exec_end(CPUState *cpu)
  220. {
  221. atomic_set(&cpu->running, false);
  222. /* Write cpu->running before reading pending_cpus. */
  223. smp_mb();
  224. /* 1. start_exclusive saw cpu->running == true. Then it will increment
  225. * pending_cpus and wait for exclusive_cond. After taking the lock
  226. * we'll see cpu->has_waiter == true.
  227. *
  228. * 2. start_exclusive saw cpu->running == false but here pending_cpus >= 1.
  229. * This includes the case when an exclusive item started after setting
  230. * cpu->running to false and before we read pending_cpus. Then we'll see
  231. * cpu->has_waiter == false and not touch pending_cpus. The next call to
  232. * cpu_exec_start will run exclusive_idle if still necessary, thus waiting
  233. * for the item to complete.
  234. *
  235. * 3. pending_cpus == 0. Then start_exclusive is definitely going to
  236. * see cpu->running == false, and it can ignore this CPU until the
  237. * next cpu_exec_start.
  238. */
  239. if (unlikely(atomic_read(&pending_cpus))) {
  240. qemu_mutex_lock(&qemu_cpu_list_lock);
  241. if (cpu->has_waiter) {
  242. cpu->has_waiter = false;
  243. atomic_set(&pending_cpus, pending_cpus - 1);
  244. if (pending_cpus == 1) {
  245. qemu_cond_signal(&exclusive_cond);
  246. }
  247. }
  248. qemu_mutex_unlock(&qemu_cpu_list_lock);
  249. }
  250. }
  251. void async_safe_run_on_cpu(CPUState *cpu, run_on_cpu_func func,
  252. run_on_cpu_data data)
  253. {
  254. struct qemu_work_item *wi;
  255. wi = g_malloc0(sizeof(struct qemu_work_item));
  256. wi->func = func;
  257. wi->data = data;
  258. wi->free = true;
  259. wi->exclusive = true;
  260. queue_work_on_cpu(cpu, wi);
  261. }
  262. void process_queued_cpu_work(CPUState *cpu)
  263. {
  264. struct qemu_work_item *wi;
  265. if (cpu->queued_work_first == NULL) {
  266. return;
  267. }
  268. qemu_mutex_lock(&cpu->work_mutex);
  269. while (cpu->queued_work_first != NULL) {
  270. wi = cpu->queued_work_first;
  271. cpu->queued_work_first = wi->next;
  272. if (!cpu->queued_work_first) {
  273. cpu->queued_work_last = NULL;
  274. }
  275. qemu_mutex_unlock(&cpu->work_mutex);
  276. if (wi->exclusive) {
  277. /* Running work items outside the BQL avoids the following deadlock:
  278. * 1) start_exclusive() is called with the BQL taken while another
  279. * CPU is running; 2) cpu_exec in the other CPU tries to takes the
  280. * BQL, so it goes to sleep; start_exclusive() is sleeping too, so
  281. * neither CPU can proceed.
  282. */
  283. qemu_mutex_unlock_iothread();
  284. start_exclusive();
  285. wi->func(cpu, wi->data);
  286. end_exclusive();
  287. qemu_mutex_lock_iothread();
  288. } else {
  289. wi->func(cpu, wi->data);
  290. }
  291. qemu_mutex_lock(&cpu->work_mutex);
  292. if (wi->free) {
  293. g_free(wi);
  294. } else {
  295. atomic_mb_set(&wi->done, true);
  296. }
  297. }
  298. qemu_mutex_unlock(&cpu->work_mutex);
  299. qemu_cond_broadcast(&qemu_work_cond);
  300. }