You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

cpus-common.c 10KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353
  1. /*
  2. * CPU thread main loop - common bits for user and system mode emulation
  3. *
  4. * Copyright (c) 2003-2005 Fabrice Bellard
  5. *
  6. * This library is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU Lesser General Public
  8. * License as published by the Free Software Foundation; either
  9. * version 2 of the License, or (at your option) any later version.
  10. *
  11. * This library is distributed in the hope that it will be useful,
  12. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. * Lesser General Public License for more details.
  15. *
  16. * You should have received a copy of the GNU Lesser General Public
  17. * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18. */
  19. #include "qemu/osdep.h"
  20. #include "qemu/main-loop.h"
  21. #include "exec/cpu-common.h"
  22. #include "qom/cpu.h"
  23. #include "sysemu/cpus.h"
  24. static QemuMutex qemu_cpu_list_lock;
  25. static QemuCond exclusive_cond;
  26. static QemuCond exclusive_resume;
  27. static QemuCond qemu_work_cond;
  28. /* >= 1 if a thread is inside start_exclusive/end_exclusive. Written
  29. * under qemu_cpu_list_lock, read with atomic operations.
  30. */
  31. static int pending_cpus;
  32. void qemu_init_cpu_list(void)
  33. {
  34. /* This is needed because qemu_init_cpu_list is also called by the
  35. * child process in a fork. */
  36. pending_cpus = 0;
  37. qemu_mutex_init(&qemu_cpu_list_lock);
  38. qemu_cond_init(&exclusive_cond);
  39. qemu_cond_init(&exclusive_resume);
  40. qemu_cond_init(&qemu_work_cond);
  41. }
  42. void cpu_list_lock(void)
  43. {
  44. qemu_mutex_lock(&qemu_cpu_list_lock);
  45. }
  46. void cpu_list_unlock(void)
  47. {
  48. qemu_mutex_unlock(&qemu_cpu_list_lock);
  49. }
  50. static bool cpu_index_auto_assigned;
  51. static int cpu_get_free_index(void)
  52. {
  53. CPUState *some_cpu;
  54. int cpu_index = 0;
  55. cpu_index_auto_assigned = true;
  56. CPU_FOREACH(some_cpu) {
  57. cpu_index++;
  58. }
  59. return cpu_index;
  60. }
  61. static void finish_safe_work(CPUState *cpu)
  62. {
  63. cpu_exec_start(cpu);
  64. cpu_exec_end(cpu);
  65. }
  66. void cpu_list_add(CPUState *cpu)
  67. {
  68. qemu_mutex_lock(&qemu_cpu_list_lock);
  69. if (cpu->cpu_index == UNASSIGNED_CPU_INDEX) {
  70. cpu->cpu_index = cpu_get_free_index();
  71. assert(cpu->cpu_index != UNASSIGNED_CPU_INDEX);
  72. } else {
  73. assert(!cpu_index_auto_assigned);
  74. }
  75. QTAILQ_INSERT_TAIL_RCU(&cpus, cpu, node);
  76. qemu_mutex_unlock(&qemu_cpu_list_lock);
  77. finish_safe_work(cpu);
  78. }
  79. void cpu_list_remove(CPUState *cpu)
  80. {
  81. qemu_mutex_lock(&qemu_cpu_list_lock);
  82. if (!QTAILQ_IN_USE(cpu, node)) {
  83. /* there is nothing to undo since cpu_exec_init() hasn't been called */
  84. qemu_mutex_unlock(&qemu_cpu_list_lock);
  85. return;
  86. }
  87. assert(!(cpu_index_auto_assigned && cpu != QTAILQ_LAST(&cpus)));
  88. QTAILQ_REMOVE_RCU(&cpus, cpu, node);
  89. cpu->cpu_index = UNASSIGNED_CPU_INDEX;
  90. qemu_mutex_unlock(&qemu_cpu_list_lock);
  91. }
  92. struct qemu_work_item {
  93. struct qemu_work_item *next;
  94. run_on_cpu_func func;
  95. run_on_cpu_data data;
  96. bool free, exclusive, done;
  97. };
  98. static void queue_work_on_cpu(CPUState *cpu, struct qemu_work_item *wi)
  99. {
  100. qemu_mutex_lock(&cpu->work_mutex);
  101. if (cpu->queued_work_first == NULL) {
  102. cpu->queued_work_first = wi;
  103. } else {
  104. cpu->queued_work_last->next = wi;
  105. }
  106. cpu->queued_work_last = wi;
  107. wi->next = NULL;
  108. wi->done = false;
  109. qemu_mutex_unlock(&cpu->work_mutex);
  110. qemu_cpu_kick(cpu);
  111. }
  112. void do_run_on_cpu(CPUState *cpu, run_on_cpu_func func, run_on_cpu_data data,
  113. QemuMutex *mutex)
  114. {
  115. struct qemu_work_item wi;
  116. if (qemu_cpu_is_self(cpu)) {
  117. func(cpu, data);
  118. return;
  119. }
  120. wi.func = func;
  121. wi.data = data;
  122. wi.done = false;
  123. wi.free = false;
  124. wi.exclusive = false;
  125. queue_work_on_cpu(cpu, &wi);
  126. while (!atomic_mb_read(&wi.done)) {
  127. CPUState *self_cpu = current_cpu;
  128. qemu_cond_wait(&qemu_work_cond, mutex);
  129. current_cpu = self_cpu;
  130. }
  131. }
  132. void async_run_on_cpu(CPUState *cpu, run_on_cpu_func func, run_on_cpu_data data)
  133. {
  134. struct qemu_work_item *wi;
  135. wi = g_malloc0(sizeof(struct qemu_work_item));
  136. wi->func = func;
  137. wi->data = data;
  138. wi->free = true;
  139. queue_work_on_cpu(cpu, wi);
  140. }
  141. /* Wait for pending exclusive operations to complete. The CPU list lock
  142. must be held. */
  143. static inline void exclusive_idle(void)
  144. {
  145. while (pending_cpus) {
  146. qemu_cond_wait(&exclusive_resume, &qemu_cpu_list_lock);
  147. }
  148. }
  149. /* Start an exclusive operation.
  150. Must only be called from outside cpu_exec. */
  151. void start_exclusive(void)
  152. {
  153. CPUState *other_cpu;
  154. int running_cpus;
  155. qemu_mutex_lock(&qemu_cpu_list_lock);
  156. exclusive_idle();
  157. /* Make all other cpus stop executing. */
  158. atomic_set(&pending_cpus, 1);
  159. /* Write pending_cpus before reading other_cpu->running. */
  160. smp_mb();
  161. running_cpus = 0;
  162. CPU_FOREACH(other_cpu) {
  163. if (atomic_read(&other_cpu->running)) {
  164. other_cpu->has_waiter = true;
  165. running_cpus++;
  166. qemu_cpu_kick(other_cpu);
  167. }
  168. }
  169. atomic_set(&pending_cpus, running_cpus + 1);
  170. while (pending_cpus > 1) {
  171. qemu_cond_wait(&exclusive_cond, &qemu_cpu_list_lock);
  172. }
  173. /* Can release mutex, no one will enter another exclusive
  174. * section until end_exclusive resets pending_cpus to 0.
  175. */
  176. qemu_mutex_unlock(&qemu_cpu_list_lock);
  177. }
  178. /* Finish an exclusive operation. */
  179. void end_exclusive(void)
  180. {
  181. qemu_mutex_lock(&qemu_cpu_list_lock);
  182. atomic_set(&pending_cpus, 0);
  183. qemu_cond_broadcast(&exclusive_resume);
  184. qemu_mutex_unlock(&qemu_cpu_list_lock);
  185. }
  186. /* Wait for exclusive ops to finish, and begin cpu execution. */
  187. void cpu_exec_start(CPUState *cpu)
  188. {
  189. atomic_set(&cpu->running, true);
  190. /* Write cpu->running before reading pending_cpus. */
  191. smp_mb();
  192. /* 1. start_exclusive saw cpu->running == true and pending_cpus >= 1.
  193. * After taking the lock we'll see cpu->has_waiter == true and run---not
  194. * for long because start_exclusive kicked us. cpu_exec_end will
  195. * decrement pending_cpus and signal the waiter.
  196. *
  197. * 2. start_exclusive saw cpu->running == false but pending_cpus >= 1.
  198. * This includes the case when an exclusive item is running now.
  199. * Then we'll see cpu->has_waiter == false and wait for the item to
  200. * complete.
  201. *
  202. * 3. pending_cpus == 0. Then start_exclusive is definitely going to
  203. * see cpu->running == true, and it will kick the CPU.
  204. */
  205. if (unlikely(atomic_read(&pending_cpus))) {
  206. qemu_mutex_lock(&qemu_cpu_list_lock);
  207. if (!cpu->has_waiter) {
  208. /* Not counted in pending_cpus, let the exclusive item
  209. * run. Since we have the lock, just set cpu->running to true
  210. * while holding it; no need to check pending_cpus again.
  211. */
  212. atomic_set(&cpu->running, false);
  213. exclusive_idle();
  214. /* Now pending_cpus is zero. */
  215. atomic_set(&cpu->running, true);
  216. } else {
  217. /* Counted in pending_cpus, go ahead and release the
  218. * waiter at cpu_exec_end.
  219. */
  220. }
  221. qemu_mutex_unlock(&qemu_cpu_list_lock);
  222. }
  223. }
  224. /* Mark cpu as not executing, and release pending exclusive ops. */
  225. void cpu_exec_end(CPUState *cpu)
  226. {
  227. atomic_set(&cpu->running, false);
  228. /* Write cpu->running before reading pending_cpus. */
  229. smp_mb();
  230. /* 1. start_exclusive saw cpu->running == true. Then it will increment
  231. * pending_cpus and wait for exclusive_cond. After taking the lock
  232. * we'll see cpu->has_waiter == true.
  233. *
  234. * 2. start_exclusive saw cpu->running == false but here pending_cpus >= 1.
  235. * This includes the case when an exclusive item started after setting
  236. * cpu->running to false and before we read pending_cpus. Then we'll see
  237. * cpu->has_waiter == false and not touch pending_cpus. The next call to
  238. * cpu_exec_start will run exclusive_idle if still necessary, thus waiting
  239. * for the item to complete.
  240. *
  241. * 3. pending_cpus == 0. Then start_exclusive is definitely going to
  242. * see cpu->running == false, and it can ignore this CPU until the
  243. * next cpu_exec_start.
  244. */
  245. if (unlikely(atomic_read(&pending_cpus))) {
  246. qemu_mutex_lock(&qemu_cpu_list_lock);
  247. if (cpu->has_waiter) {
  248. cpu->has_waiter = false;
  249. atomic_set(&pending_cpus, pending_cpus - 1);
  250. if (pending_cpus == 1) {
  251. qemu_cond_signal(&exclusive_cond);
  252. }
  253. }
  254. qemu_mutex_unlock(&qemu_cpu_list_lock);
  255. }
  256. }
  257. void async_safe_run_on_cpu(CPUState *cpu, run_on_cpu_func func,
  258. run_on_cpu_data data)
  259. {
  260. struct qemu_work_item *wi;
  261. wi = g_malloc0(sizeof(struct qemu_work_item));
  262. wi->func = func;
  263. wi->data = data;
  264. wi->free = true;
  265. wi->exclusive = true;
  266. queue_work_on_cpu(cpu, wi);
  267. }
  268. void process_queued_cpu_work(CPUState *cpu)
  269. {
  270. struct qemu_work_item *wi;
  271. if (cpu->queued_work_first == NULL) {
  272. return;
  273. }
  274. qemu_mutex_lock(&cpu->work_mutex);
  275. while (cpu->queued_work_first != NULL) {
  276. wi = cpu->queued_work_first;
  277. cpu->queued_work_first = wi->next;
  278. if (!cpu->queued_work_first) {
  279. cpu->queued_work_last = NULL;
  280. }
  281. qemu_mutex_unlock(&cpu->work_mutex);
  282. if (wi->exclusive) {
  283. /* Running work items outside the BQL avoids the following deadlock:
  284. * 1) start_exclusive() is called with the BQL taken while another
  285. * CPU is running; 2) cpu_exec in the other CPU tries to takes the
  286. * BQL, so it goes to sleep; start_exclusive() is sleeping too, so
  287. * neither CPU can proceed.
  288. */
  289. qemu_mutex_unlock_iothread();
  290. start_exclusive();
  291. wi->func(cpu, wi->data);
  292. end_exclusive();
  293. qemu_mutex_lock_iothread();
  294. } else {
  295. wi->func(cpu, wi->data);
  296. }
  297. qemu_mutex_lock(&cpu->work_mutex);
  298. if (wi->free) {
  299. g_free(wi);
  300. } else {
  301. atomic_mb_set(&wi->done, true);
  302. }
  303. }
  304. qemu_mutex_unlock(&cpu->work_mutex);
  305. qemu_cond_broadcast(&qemu_work_cond);
  306. }