You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

4079 lines
120KB

  1. /*
  2. * Virtual page mapping
  3. *
  4. * Copyright (c) 2003 Fabrice Bellard
  5. *
  6. * This library is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU Lesser General Public
  8. * License as published by the Free Software Foundation; either
  9. * version 2 of the License, or (at your option) any later version.
  10. *
  11. * This library is distributed in the hope that it will be useful,
  12. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. * Lesser General Public License for more details.
  15. *
  16. * You should have received a copy of the GNU Lesser General Public
  17. * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18. */
  19. #include "qemu/osdep.h"
  20. #include "qemu-common.h"
  21. #include "qapi/error.h"
  22. #include "qemu/cutils.h"
  23. #include "cpu.h"
  24. #include "exec/exec-all.h"
  25. #include "exec/target_page.h"
  26. #include "tcg/tcg.h"
  27. #include "hw/qdev-core.h"
  28. #include "hw/qdev-properties.h"
  29. #if !defined(CONFIG_USER_ONLY)
  30. #include "hw/boards.h"
  31. #include "hw/xen/xen.h"
  32. #endif
  33. #include "sysemu/kvm.h"
  34. #include "sysemu/sysemu.h"
  35. #include "sysemu/tcg.h"
  36. #include "qemu/timer.h"
  37. #include "qemu/config-file.h"
  38. #include "qemu/error-report.h"
  39. #include "qemu/qemu-print.h"
  40. #if defined(CONFIG_USER_ONLY)
  41. #include "qemu.h"
  42. #else /* !CONFIG_USER_ONLY */
  43. #include "exec/memory.h"
  44. #include "exec/ioport.h"
  45. #include "sysemu/dma.h"
  46. #include "sysemu/hostmem.h"
  47. #include "sysemu/hw_accel.h"
  48. #include "exec/address-spaces.h"
  49. #include "sysemu/xen-mapcache.h"
  50. #include "trace-root.h"
  51. #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
  52. #include <linux/falloc.h>
  53. #endif
  54. #endif
  55. #include "qemu/rcu_queue.h"
  56. #include "qemu/main-loop.h"
  57. #include "translate-all.h"
  58. #include "sysemu/replay.h"
  59. #include "exec/memory-internal.h"
  60. #include "exec/ram_addr.h"
  61. #include "exec/log.h"
  62. #include "qemu/pmem.h"
  63. #include "migration/vmstate.h"
  64. #include "qemu/range.h"
  65. #ifndef _WIN32
  66. #include "qemu/mmap-alloc.h"
  67. #endif
  68. #include "monitor/monitor.h"
  69. //#define DEBUG_SUBPAGE
  70. #if !defined(CONFIG_USER_ONLY)
  71. /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
  72. * are protected by the ramlist lock.
  73. */
  74. RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
  75. static MemoryRegion *system_memory;
  76. static MemoryRegion *system_io;
  77. AddressSpace address_space_io;
  78. AddressSpace address_space_memory;
  79. static MemoryRegion io_mem_unassigned;
  80. #endif
  81. CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
  82. /* current CPU in the current thread. It is only valid inside
  83. cpu_exec() */
  84. __thread CPUState *current_cpu;
  85. uintptr_t qemu_host_page_size;
  86. intptr_t qemu_host_page_mask;
  87. #if !defined(CONFIG_USER_ONLY)
  88. /* 0 = Do not count executed instructions.
  89. 1 = Precise instruction counting.
  90. 2 = Adaptive rate instruction counting. */
  91. int use_icount;
  92. typedef struct PhysPageEntry PhysPageEntry;
  93. struct PhysPageEntry {
  94. /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
  95. uint32_t skip : 6;
  96. /* index into phys_sections (!skip) or phys_map_nodes (skip) */
  97. uint32_t ptr : 26;
  98. };
  99. #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
  100. /* Size of the L2 (and L3, etc) page tables. */
  101. #define ADDR_SPACE_BITS 64
  102. #define P_L2_BITS 9
  103. #define P_L2_SIZE (1 << P_L2_BITS)
  104. #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
  105. typedef PhysPageEntry Node[P_L2_SIZE];
  106. typedef struct PhysPageMap {
  107. struct rcu_head rcu;
  108. unsigned sections_nb;
  109. unsigned sections_nb_alloc;
  110. unsigned nodes_nb;
  111. unsigned nodes_nb_alloc;
  112. Node *nodes;
  113. MemoryRegionSection *sections;
  114. } PhysPageMap;
  115. struct AddressSpaceDispatch {
  116. MemoryRegionSection *mru_section;
  117. /* This is a multi-level map on the physical address space.
  118. * The bottom level has pointers to MemoryRegionSections.
  119. */
  120. PhysPageEntry phys_map;
  121. PhysPageMap map;
  122. };
  123. #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
  124. typedef struct subpage_t {
  125. MemoryRegion iomem;
  126. FlatView *fv;
  127. hwaddr base;
  128. uint16_t sub_section[];
  129. } subpage_t;
  130. #define PHYS_SECTION_UNASSIGNED 0
  131. static void io_mem_init(void);
  132. static void memory_map_init(void);
  133. static void tcg_log_global_after_sync(MemoryListener *listener);
  134. static void tcg_commit(MemoryListener *listener);
  135. /**
  136. * CPUAddressSpace: all the information a CPU needs about an AddressSpace
  137. * @cpu: the CPU whose AddressSpace this is
  138. * @as: the AddressSpace itself
  139. * @memory_dispatch: its dispatch pointer (cached, RCU protected)
  140. * @tcg_as_listener: listener for tracking changes to the AddressSpace
  141. */
  142. struct CPUAddressSpace {
  143. CPUState *cpu;
  144. AddressSpace *as;
  145. struct AddressSpaceDispatch *memory_dispatch;
  146. MemoryListener tcg_as_listener;
  147. };
  148. struct DirtyBitmapSnapshot {
  149. ram_addr_t start;
  150. ram_addr_t end;
  151. unsigned long dirty[];
  152. };
  153. #endif
  154. #if !defined(CONFIG_USER_ONLY)
  155. static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
  156. {
  157. static unsigned alloc_hint = 16;
  158. if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
  159. map->nodes_nb_alloc = MAX(alloc_hint, map->nodes_nb + nodes);
  160. map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
  161. alloc_hint = map->nodes_nb_alloc;
  162. }
  163. }
  164. static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
  165. {
  166. unsigned i;
  167. uint32_t ret;
  168. PhysPageEntry e;
  169. PhysPageEntry *p;
  170. ret = map->nodes_nb++;
  171. p = map->nodes[ret];
  172. assert(ret != PHYS_MAP_NODE_NIL);
  173. assert(ret != map->nodes_nb_alloc);
  174. e.skip = leaf ? 0 : 1;
  175. e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
  176. for (i = 0; i < P_L2_SIZE; ++i) {
  177. memcpy(&p[i], &e, sizeof(e));
  178. }
  179. return ret;
  180. }
  181. static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
  182. hwaddr *index, uint64_t *nb, uint16_t leaf,
  183. int level)
  184. {
  185. PhysPageEntry *p;
  186. hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
  187. if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
  188. lp->ptr = phys_map_node_alloc(map, level == 0);
  189. }
  190. p = map->nodes[lp->ptr];
  191. lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
  192. while (*nb && lp < &p[P_L2_SIZE]) {
  193. if ((*index & (step - 1)) == 0 && *nb >= step) {
  194. lp->skip = 0;
  195. lp->ptr = leaf;
  196. *index += step;
  197. *nb -= step;
  198. } else {
  199. phys_page_set_level(map, lp, index, nb, leaf, level - 1);
  200. }
  201. ++lp;
  202. }
  203. }
  204. static void phys_page_set(AddressSpaceDispatch *d,
  205. hwaddr index, uint64_t nb,
  206. uint16_t leaf)
  207. {
  208. /* Wildly overreserve - it doesn't matter much. */
  209. phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
  210. phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
  211. }
  212. /* Compact a non leaf page entry. Simply detect that the entry has a single child,
  213. * and update our entry so we can skip it and go directly to the destination.
  214. */
  215. static void phys_page_compact(PhysPageEntry *lp, Node *nodes)
  216. {
  217. unsigned valid_ptr = P_L2_SIZE;
  218. int valid = 0;
  219. PhysPageEntry *p;
  220. int i;
  221. if (lp->ptr == PHYS_MAP_NODE_NIL) {
  222. return;
  223. }
  224. p = nodes[lp->ptr];
  225. for (i = 0; i < P_L2_SIZE; i++) {
  226. if (p[i].ptr == PHYS_MAP_NODE_NIL) {
  227. continue;
  228. }
  229. valid_ptr = i;
  230. valid++;
  231. if (p[i].skip) {
  232. phys_page_compact(&p[i], nodes);
  233. }
  234. }
  235. /* We can only compress if there's only one child. */
  236. if (valid != 1) {
  237. return;
  238. }
  239. assert(valid_ptr < P_L2_SIZE);
  240. /* Don't compress if it won't fit in the # of bits we have. */
  241. if (P_L2_LEVELS >= (1 << 6) &&
  242. lp->skip + p[valid_ptr].skip >= (1 << 6)) {
  243. return;
  244. }
  245. lp->ptr = p[valid_ptr].ptr;
  246. if (!p[valid_ptr].skip) {
  247. /* If our only child is a leaf, make this a leaf. */
  248. /* By design, we should have made this node a leaf to begin with so we
  249. * should never reach here.
  250. * But since it's so simple to handle this, let's do it just in case we
  251. * change this rule.
  252. */
  253. lp->skip = 0;
  254. } else {
  255. lp->skip += p[valid_ptr].skip;
  256. }
  257. }
  258. void address_space_dispatch_compact(AddressSpaceDispatch *d)
  259. {
  260. if (d->phys_map.skip) {
  261. phys_page_compact(&d->phys_map, d->map.nodes);
  262. }
  263. }
  264. static inline bool section_covers_addr(const MemoryRegionSection *section,
  265. hwaddr addr)
  266. {
  267. /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
  268. * the section must cover the entire address space.
  269. */
  270. return int128_gethi(section->size) ||
  271. range_covers_byte(section->offset_within_address_space,
  272. int128_getlo(section->size), addr);
  273. }
  274. static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr addr)
  275. {
  276. PhysPageEntry lp = d->phys_map, *p;
  277. Node *nodes = d->map.nodes;
  278. MemoryRegionSection *sections = d->map.sections;
  279. hwaddr index = addr >> TARGET_PAGE_BITS;
  280. int i;
  281. for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
  282. if (lp.ptr == PHYS_MAP_NODE_NIL) {
  283. return &sections[PHYS_SECTION_UNASSIGNED];
  284. }
  285. p = nodes[lp.ptr];
  286. lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
  287. }
  288. if (section_covers_addr(&sections[lp.ptr], addr)) {
  289. return &sections[lp.ptr];
  290. } else {
  291. return &sections[PHYS_SECTION_UNASSIGNED];
  292. }
  293. }
  294. /* Called from RCU critical section */
  295. static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
  296. hwaddr addr,
  297. bool resolve_subpage)
  298. {
  299. MemoryRegionSection *section = atomic_read(&d->mru_section);
  300. subpage_t *subpage;
  301. if (!section || section == &d->map.sections[PHYS_SECTION_UNASSIGNED] ||
  302. !section_covers_addr(section, addr)) {
  303. section = phys_page_find(d, addr);
  304. atomic_set(&d->mru_section, section);
  305. }
  306. if (resolve_subpage && section->mr->subpage) {
  307. subpage = container_of(section->mr, subpage_t, iomem);
  308. section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
  309. }
  310. return section;
  311. }
  312. /* Called from RCU critical section */
  313. static MemoryRegionSection *
  314. address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
  315. hwaddr *plen, bool resolve_subpage)
  316. {
  317. MemoryRegionSection *section;
  318. MemoryRegion *mr;
  319. Int128 diff;
  320. section = address_space_lookup_region(d, addr, resolve_subpage);
  321. /* Compute offset within MemoryRegionSection */
  322. addr -= section->offset_within_address_space;
  323. /* Compute offset within MemoryRegion */
  324. *xlat = addr + section->offset_within_region;
  325. mr = section->mr;
  326. /* MMIO registers can be expected to perform full-width accesses based only
  327. * on their address, without considering adjacent registers that could
  328. * decode to completely different MemoryRegions. When such registers
  329. * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
  330. * regions overlap wildly. For this reason we cannot clamp the accesses
  331. * here.
  332. *
  333. * If the length is small (as is the case for address_space_ldl/stl),
  334. * everything works fine. If the incoming length is large, however,
  335. * the caller really has to do the clamping through memory_access_size.
  336. */
  337. if (memory_region_is_ram(mr)) {
  338. diff = int128_sub(section->size, int128_make64(addr));
  339. *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
  340. }
  341. return section;
  342. }
  343. /**
  344. * address_space_translate_iommu - translate an address through an IOMMU
  345. * memory region and then through the target address space.
  346. *
  347. * @iommu_mr: the IOMMU memory region that we start the translation from
  348. * @addr: the address to be translated through the MMU
  349. * @xlat: the translated address offset within the destination memory region.
  350. * It cannot be %NULL.
  351. * @plen_out: valid read/write length of the translated address. It
  352. * cannot be %NULL.
  353. * @page_mask_out: page mask for the translated address. This
  354. * should only be meaningful for IOMMU translated
  355. * addresses, since there may be huge pages that this bit
  356. * would tell. It can be %NULL if we don't care about it.
  357. * @is_write: whether the translation operation is for write
  358. * @is_mmio: whether this can be MMIO, set true if it can
  359. * @target_as: the address space targeted by the IOMMU
  360. * @attrs: transaction attributes
  361. *
  362. * This function is called from RCU critical section. It is the common
  363. * part of flatview_do_translate and address_space_translate_cached.
  364. */
  365. static MemoryRegionSection address_space_translate_iommu(IOMMUMemoryRegion *iommu_mr,
  366. hwaddr *xlat,
  367. hwaddr *plen_out,
  368. hwaddr *page_mask_out,
  369. bool is_write,
  370. bool is_mmio,
  371. AddressSpace **target_as,
  372. MemTxAttrs attrs)
  373. {
  374. MemoryRegionSection *section;
  375. hwaddr page_mask = (hwaddr)-1;
  376. do {
  377. hwaddr addr = *xlat;
  378. IOMMUMemoryRegionClass *imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
  379. int iommu_idx = 0;
  380. IOMMUTLBEntry iotlb;
  381. if (imrc->attrs_to_index) {
  382. iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
  383. }
  384. iotlb = imrc->translate(iommu_mr, addr, is_write ?
  385. IOMMU_WO : IOMMU_RO, iommu_idx);
  386. if (!(iotlb.perm & (1 << is_write))) {
  387. goto unassigned;
  388. }
  389. addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
  390. | (addr & iotlb.addr_mask));
  391. page_mask &= iotlb.addr_mask;
  392. *plen_out = MIN(*plen_out, (addr | iotlb.addr_mask) - addr + 1);
  393. *target_as = iotlb.target_as;
  394. section = address_space_translate_internal(
  395. address_space_to_dispatch(iotlb.target_as), addr, xlat,
  396. plen_out, is_mmio);
  397. iommu_mr = memory_region_get_iommu(section->mr);
  398. } while (unlikely(iommu_mr));
  399. if (page_mask_out) {
  400. *page_mask_out = page_mask;
  401. }
  402. return *section;
  403. unassigned:
  404. return (MemoryRegionSection) { .mr = &io_mem_unassigned };
  405. }
  406. /**
  407. * flatview_do_translate - translate an address in FlatView
  408. *
  409. * @fv: the flat view that we want to translate on
  410. * @addr: the address to be translated in above address space
  411. * @xlat: the translated address offset within memory region. It
  412. * cannot be @NULL.
  413. * @plen_out: valid read/write length of the translated address. It
  414. * can be @NULL when we don't care about it.
  415. * @page_mask_out: page mask for the translated address. This
  416. * should only be meaningful for IOMMU translated
  417. * addresses, since there may be huge pages that this bit
  418. * would tell. It can be @NULL if we don't care about it.
  419. * @is_write: whether the translation operation is for write
  420. * @is_mmio: whether this can be MMIO, set true if it can
  421. * @target_as: the address space targeted by the IOMMU
  422. * @attrs: memory transaction attributes
  423. *
  424. * This function is called from RCU critical section
  425. */
  426. static MemoryRegionSection flatview_do_translate(FlatView *fv,
  427. hwaddr addr,
  428. hwaddr *xlat,
  429. hwaddr *plen_out,
  430. hwaddr *page_mask_out,
  431. bool is_write,
  432. bool is_mmio,
  433. AddressSpace **target_as,
  434. MemTxAttrs attrs)
  435. {
  436. MemoryRegionSection *section;
  437. IOMMUMemoryRegion *iommu_mr;
  438. hwaddr plen = (hwaddr)(-1);
  439. if (!plen_out) {
  440. plen_out = &plen;
  441. }
  442. section = address_space_translate_internal(
  443. flatview_to_dispatch(fv), addr, xlat,
  444. plen_out, is_mmio);
  445. iommu_mr = memory_region_get_iommu(section->mr);
  446. if (unlikely(iommu_mr)) {
  447. return address_space_translate_iommu(iommu_mr, xlat,
  448. plen_out, page_mask_out,
  449. is_write, is_mmio,
  450. target_as, attrs);
  451. }
  452. if (page_mask_out) {
  453. /* Not behind an IOMMU, use default page size. */
  454. *page_mask_out = ~TARGET_PAGE_MASK;
  455. }
  456. return *section;
  457. }
  458. /* Called from RCU critical section */
  459. IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr,
  460. bool is_write, MemTxAttrs attrs)
  461. {
  462. MemoryRegionSection section;
  463. hwaddr xlat, page_mask;
  464. /*
  465. * This can never be MMIO, and we don't really care about plen,
  466. * but page mask.
  467. */
  468. section = flatview_do_translate(address_space_to_flatview(as), addr, &xlat,
  469. NULL, &page_mask, is_write, false, &as,
  470. attrs);
  471. /* Illegal translation */
  472. if (section.mr == &io_mem_unassigned) {
  473. goto iotlb_fail;
  474. }
  475. /* Convert memory region offset into address space offset */
  476. xlat += section.offset_within_address_space -
  477. section.offset_within_region;
  478. return (IOMMUTLBEntry) {
  479. .target_as = as,
  480. .iova = addr & ~page_mask,
  481. .translated_addr = xlat & ~page_mask,
  482. .addr_mask = page_mask,
  483. /* IOTLBs are for DMAs, and DMA only allows on RAMs. */
  484. .perm = IOMMU_RW,
  485. };
  486. iotlb_fail:
  487. return (IOMMUTLBEntry) {0};
  488. }
  489. /* Called from RCU critical section */
  490. MemoryRegion *flatview_translate(FlatView *fv, hwaddr addr, hwaddr *xlat,
  491. hwaddr *plen, bool is_write,
  492. MemTxAttrs attrs)
  493. {
  494. MemoryRegion *mr;
  495. MemoryRegionSection section;
  496. AddressSpace *as = NULL;
  497. /* This can be MMIO, so setup MMIO bit. */
  498. section = flatview_do_translate(fv, addr, xlat, plen, NULL,
  499. is_write, true, &as, attrs);
  500. mr = section.mr;
  501. if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
  502. hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
  503. *plen = MIN(page, *plen);
  504. }
  505. return mr;
  506. }
  507. typedef struct TCGIOMMUNotifier {
  508. IOMMUNotifier n;
  509. MemoryRegion *mr;
  510. CPUState *cpu;
  511. int iommu_idx;
  512. bool active;
  513. } TCGIOMMUNotifier;
  514. static void tcg_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
  515. {
  516. TCGIOMMUNotifier *notifier = container_of(n, TCGIOMMUNotifier, n);
  517. if (!notifier->active) {
  518. return;
  519. }
  520. tlb_flush(notifier->cpu);
  521. notifier->active = false;
  522. /* We leave the notifier struct on the list to avoid reallocating it later.
  523. * Generally the number of IOMMUs a CPU deals with will be small.
  524. * In any case we can't unregister the iommu notifier from a notify
  525. * callback.
  526. */
  527. }
  528. static void tcg_register_iommu_notifier(CPUState *cpu,
  529. IOMMUMemoryRegion *iommu_mr,
  530. int iommu_idx)
  531. {
  532. /* Make sure this CPU has an IOMMU notifier registered for this
  533. * IOMMU/IOMMU index combination, so that we can flush its TLB
  534. * when the IOMMU tells us the mappings we've cached have changed.
  535. */
  536. MemoryRegion *mr = MEMORY_REGION(iommu_mr);
  537. TCGIOMMUNotifier *notifier;
  538. Error *err = NULL;
  539. int i, ret;
  540. for (i = 0; i < cpu->iommu_notifiers->len; i++) {
  541. notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
  542. if (notifier->mr == mr && notifier->iommu_idx == iommu_idx) {
  543. break;
  544. }
  545. }
  546. if (i == cpu->iommu_notifiers->len) {
  547. /* Not found, add a new entry at the end of the array */
  548. cpu->iommu_notifiers = g_array_set_size(cpu->iommu_notifiers, i + 1);
  549. notifier = g_new0(TCGIOMMUNotifier, 1);
  550. g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i) = notifier;
  551. notifier->mr = mr;
  552. notifier->iommu_idx = iommu_idx;
  553. notifier->cpu = cpu;
  554. /* Rather than trying to register interest in the specific part
  555. * of the iommu's address space that we've accessed and then
  556. * expand it later as subsequent accesses touch more of it, we
  557. * just register interest in the whole thing, on the assumption
  558. * that iommu reconfiguration will be rare.
  559. */
  560. iommu_notifier_init(&notifier->n,
  561. tcg_iommu_unmap_notify,
  562. IOMMU_NOTIFIER_UNMAP,
  563. 0,
  564. HWADDR_MAX,
  565. iommu_idx);
  566. ret = memory_region_register_iommu_notifier(notifier->mr, &notifier->n,
  567. &err);
  568. if (ret) {
  569. error_report_err(err);
  570. exit(1);
  571. }
  572. }
  573. if (!notifier->active) {
  574. notifier->active = true;
  575. }
  576. }
  577. static void tcg_iommu_free_notifier_list(CPUState *cpu)
  578. {
  579. /* Destroy the CPU's notifier list */
  580. int i;
  581. TCGIOMMUNotifier *notifier;
  582. for (i = 0; i < cpu->iommu_notifiers->len; i++) {
  583. notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
  584. memory_region_unregister_iommu_notifier(notifier->mr, &notifier->n);
  585. g_free(notifier);
  586. }
  587. g_array_free(cpu->iommu_notifiers, true);
  588. }
  589. /* Called from RCU critical section */
  590. MemoryRegionSection *
  591. address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
  592. hwaddr *xlat, hwaddr *plen,
  593. MemTxAttrs attrs, int *prot)
  594. {
  595. MemoryRegionSection *section;
  596. IOMMUMemoryRegion *iommu_mr;
  597. IOMMUMemoryRegionClass *imrc;
  598. IOMMUTLBEntry iotlb;
  599. int iommu_idx;
  600. AddressSpaceDispatch *d = atomic_rcu_read(&cpu->cpu_ases[asidx].memory_dispatch);
  601. for (;;) {
  602. section = address_space_translate_internal(d, addr, &addr, plen, false);
  603. iommu_mr = memory_region_get_iommu(section->mr);
  604. if (!iommu_mr) {
  605. break;
  606. }
  607. imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
  608. iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
  609. tcg_register_iommu_notifier(cpu, iommu_mr, iommu_idx);
  610. /* We need all the permissions, so pass IOMMU_NONE so the IOMMU
  611. * doesn't short-cut its translation table walk.
  612. */
  613. iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, iommu_idx);
  614. addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
  615. | (addr & iotlb.addr_mask));
  616. /* Update the caller's prot bits to remove permissions the IOMMU
  617. * is giving us a failure response for. If we get down to no
  618. * permissions left at all we can give up now.
  619. */
  620. if (!(iotlb.perm & IOMMU_RO)) {
  621. *prot &= ~(PAGE_READ | PAGE_EXEC);
  622. }
  623. if (!(iotlb.perm & IOMMU_WO)) {
  624. *prot &= ~PAGE_WRITE;
  625. }
  626. if (!*prot) {
  627. goto translate_fail;
  628. }
  629. d = flatview_to_dispatch(address_space_to_flatview(iotlb.target_as));
  630. }
  631. assert(!memory_region_is_iommu(section->mr));
  632. *xlat = addr;
  633. return section;
  634. translate_fail:
  635. return &d->map.sections[PHYS_SECTION_UNASSIGNED];
  636. }
  637. #endif
  638. #if !defined(CONFIG_USER_ONLY)
  639. static int cpu_common_post_load(void *opaque, int version_id)
  640. {
  641. CPUState *cpu = opaque;
  642. /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
  643. version_id is increased. */
  644. cpu->interrupt_request &= ~0x01;
  645. tlb_flush(cpu);
  646. /* loadvm has just updated the content of RAM, bypassing the
  647. * usual mechanisms that ensure we flush TBs for writes to
  648. * memory we've translated code from. So we must flush all TBs,
  649. * which will now be stale.
  650. */
  651. tb_flush(cpu);
  652. return 0;
  653. }
  654. static int cpu_common_pre_load(void *opaque)
  655. {
  656. CPUState *cpu = opaque;
  657. cpu->exception_index = -1;
  658. return 0;
  659. }
  660. static bool cpu_common_exception_index_needed(void *opaque)
  661. {
  662. CPUState *cpu = opaque;
  663. return tcg_enabled() && cpu->exception_index != -1;
  664. }
  665. static const VMStateDescription vmstate_cpu_common_exception_index = {
  666. .name = "cpu_common/exception_index",
  667. .version_id = 1,
  668. .minimum_version_id = 1,
  669. .needed = cpu_common_exception_index_needed,
  670. .fields = (VMStateField[]) {
  671. VMSTATE_INT32(exception_index, CPUState),
  672. VMSTATE_END_OF_LIST()
  673. }
  674. };
  675. static bool cpu_common_crash_occurred_needed(void *opaque)
  676. {
  677. CPUState *cpu = opaque;
  678. return cpu->crash_occurred;
  679. }
  680. static const VMStateDescription vmstate_cpu_common_crash_occurred = {
  681. .name = "cpu_common/crash_occurred",
  682. .version_id = 1,
  683. .minimum_version_id = 1,
  684. .needed = cpu_common_crash_occurred_needed,
  685. .fields = (VMStateField[]) {
  686. VMSTATE_BOOL(crash_occurred, CPUState),
  687. VMSTATE_END_OF_LIST()
  688. }
  689. };
  690. const VMStateDescription vmstate_cpu_common = {
  691. .name = "cpu_common",
  692. .version_id = 1,
  693. .minimum_version_id = 1,
  694. .pre_load = cpu_common_pre_load,
  695. .post_load = cpu_common_post_load,
  696. .fields = (VMStateField[]) {
  697. VMSTATE_UINT32(halted, CPUState),
  698. VMSTATE_UINT32(interrupt_request, CPUState),
  699. VMSTATE_END_OF_LIST()
  700. },
  701. .subsections = (const VMStateDescription*[]) {
  702. &vmstate_cpu_common_exception_index,
  703. &vmstate_cpu_common_crash_occurred,
  704. NULL
  705. }
  706. };
  707. #endif
  708. CPUState *qemu_get_cpu(int index)
  709. {
  710. CPUState *cpu;
  711. CPU_FOREACH(cpu) {
  712. if (cpu->cpu_index == index) {
  713. return cpu;
  714. }
  715. }
  716. return NULL;
  717. }
  718. #if !defined(CONFIG_USER_ONLY)
  719. void cpu_address_space_init(CPUState *cpu, int asidx,
  720. const char *prefix, MemoryRegion *mr)
  721. {
  722. CPUAddressSpace *newas;
  723. AddressSpace *as = g_new0(AddressSpace, 1);
  724. char *as_name;
  725. assert(mr);
  726. as_name = g_strdup_printf("%s-%d", prefix, cpu->cpu_index);
  727. address_space_init(as, mr, as_name);
  728. g_free(as_name);
  729. /* Target code should have set num_ases before calling us */
  730. assert(asidx < cpu->num_ases);
  731. if (asidx == 0) {
  732. /* address space 0 gets the convenience alias */
  733. cpu->as = as;
  734. }
  735. /* KVM cannot currently support multiple address spaces. */
  736. assert(asidx == 0 || !kvm_enabled());
  737. if (!cpu->cpu_ases) {
  738. cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
  739. }
  740. newas = &cpu->cpu_ases[asidx];
  741. newas->cpu = cpu;
  742. newas->as = as;
  743. if (tcg_enabled()) {
  744. newas->tcg_as_listener.log_global_after_sync = tcg_log_global_after_sync;
  745. newas->tcg_as_listener.commit = tcg_commit;
  746. memory_listener_register(&newas->tcg_as_listener, as);
  747. }
  748. }
  749. AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
  750. {
  751. /* Return the AddressSpace corresponding to the specified index */
  752. return cpu->cpu_ases[asidx].as;
  753. }
  754. #endif
  755. void cpu_exec_unrealizefn(CPUState *cpu)
  756. {
  757. CPUClass *cc = CPU_GET_CLASS(cpu);
  758. cpu_list_remove(cpu);
  759. if (cc->vmsd != NULL) {
  760. vmstate_unregister(NULL, cc->vmsd, cpu);
  761. }
  762. if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
  763. vmstate_unregister(NULL, &vmstate_cpu_common, cpu);
  764. }
  765. #ifndef CONFIG_USER_ONLY
  766. tcg_iommu_free_notifier_list(cpu);
  767. #endif
  768. }
  769. Property cpu_common_props[] = {
  770. #ifndef CONFIG_USER_ONLY
  771. /* Create a memory property for softmmu CPU object,
  772. * so users can wire up its memory. (This can't go in hw/core/cpu.c
  773. * because that file is compiled only once for both user-mode
  774. * and system builds.) The default if no link is set up is to use
  775. * the system address space.
  776. */
  777. DEFINE_PROP_LINK("memory", CPUState, memory, TYPE_MEMORY_REGION,
  778. MemoryRegion *),
  779. #endif
  780. DEFINE_PROP_END_OF_LIST(),
  781. };
  782. void cpu_exec_initfn(CPUState *cpu)
  783. {
  784. cpu->as = NULL;
  785. cpu->num_ases = 0;
  786. #ifndef CONFIG_USER_ONLY
  787. cpu->thread_id = qemu_get_thread_id();
  788. cpu->memory = system_memory;
  789. object_ref(OBJECT(cpu->memory));
  790. #endif
  791. }
  792. void cpu_exec_realizefn(CPUState *cpu, Error **errp)
  793. {
  794. CPUClass *cc = CPU_GET_CLASS(cpu);
  795. static bool tcg_target_initialized;
  796. cpu_list_add(cpu);
  797. if (tcg_enabled() && !tcg_target_initialized) {
  798. tcg_target_initialized = true;
  799. cc->tcg_initialize();
  800. }
  801. tlb_init(cpu);
  802. qemu_plugin_vcpu_init_hook(cpu);
  803. #ifndef CONFIG_USER_ONLY
  804. if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
  805. vmstate_register(NULL, cpu->cpu_index, &vmstate_cpu_common, cpu);
  806. }
  807. if (cc->vmsd != NULL) {
  808. vmstate_register(NULL, cpu->cpu_index, cc->vmsd, cpu);
  809. }
  810. cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier *));
  811. #endif
  812. }
  813. const char *parse_cpu_option(const char *cpu_option)
  814. {
  815. ObjectClass *oc;
  816. CPUClass *cc;
  817. gchar **model_pieces;
  818. const char *cpu_type;
  819. model_pieces = g_strsplit(cpu_option, ",", 2);
  820. if (!model_pieces[0]) {
  821. error_report("-cpu option cannot be empty");
  822. exit(1);
  823. }
  824. oc = cpu_class_by_name(CPU_RESOLVING_TYPE, model_pieces[0]);
  825. if (oc == NULL) {
  826. error_report("unable to find CPU model '%s'", model_pieces[0]);
  827. g_strfreev(model_pieces);
  828. exit(EXIT_FAILURE);
  829. }
  830. cpu_type = object_class_get_name(oc);
  831. cc = CPU_CLASS(oc);
  832. cc->parse_features(cpu_type, model_pieces[1], &error_fatal);
  833. g_strfreev(model_pieces);
  834. return cpu_type;
  835. }
  836. #if defined(CONFIG_USER_ONLY)
  837. void tb_invalidate_phys_addr(target_ulong addr)
  838. {
  839. mmap_lock();
  840. tb_invalidate_phys_page_range(addr, addr + 1);
  841. mmap_unlock();
  842. }
  843. static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
  844. {
  845. tb_invalidate_phys_addr(pc);
  846. }
  847. #else
  848. void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs)
  849. {
  850. ram_addr_t ram_addr;
  851. MemoryRegion *mr;
  852. hwaddr l = 1;
  853. if (!tcg_enabled()) {
  854. return;
  855. }
  856. RCU_READ_LOCK_GUARD();
  857. mr = address_space_translate(as, addr, &addr, &l, false, attrs);
  858. if (!(memory_region_is_ram(mr)
  859. || memory_region_is_romd(mr))) {
  860. return;
  861. }
  862. ram_addr = memory_region_get_ram_addr(mr) + addr;
  863. tb_invalidate_phys_page_range(ram_addr, ram_addr + 1);
  864. }
  865. static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
  866. {
  867. /*
  868. * There may not be a virtual to physical translation for the pc
  869. * right now, but there may exist cached TB for this pc.
  870. * Flush the whole TB cache to force re-translation of such TBs.
  871. * This is heavyweight, but we're debugging anyway.
  872. */
  873. tb_flush(cpu);
  874. }
  875. #endif
  876. #ifndef CONFIG_USER_ONLY
  877. /* Add a watchpoint. */
  878. int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
  879. int flags, CPUWatchpoint **watchpoint)
  880. {
  881. CPUWatchpoint *wp;
  882. /* forbid ranges which are empty or run off the end of the address space */
  883. if (len == 0 || (addr + len - 1) < addr) {
  884. error_report("tried to set invalid watchpoint at %"
  885. VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
  886. return -EINVAL;
  887. }
  888. wp = g_malloc(sizeof(*wp));
  889. wp->vaddr = addr;
  890. wp->len = len;
  891. wp->flags = flags;
  892. /* keep all GDB-injected watchpoints in front */
  893. if (flags & BP_GDB) {
  894. QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
  895. } else {
  896. QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
  897. }
  898. tlb_flush_page(cpu, addr);
  899. if (watchpoint)
  900. *watchpoint = wp;
  901. return 0;
  902. }
  903. /* Remove a specific watchpoint. */
  904. int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
  905. int flags)
  906. {
  907. CPUWatchpoint *wp;
  908. QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
  909. if (addr == wp->vaddr && len == wp->len
  910. && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
  911. cpu_watchpoint_remove_by_ref(cpu, wp);
  912. return 0;
  913. }
  914. }
  915. return -ENOENT;
  916. }
  917. /* Remove a specific watchpoint by reference. */
  918. void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
  919. {
  920. QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
  921. tlb_flush_page(cpu, watchpoint->vaddr);
  922. g_free(watchpoint);
  923. }
  924. /* Remove all matching watchpoints. */
  925. void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
  926. {
  927. CPUWatchpoint *wp, *next;
  928. QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
  929. if (wp->flags & mask) {
  930. cpu_watchpoint_remove_by_ref(cpu, wp);
  931. }
  932. }
  933. }
  934. /* Return true if this watchpoint address matches the specified
  935. * access (ie the address range covered by the watchpoint overlaps
  936. * partially or completely with the address range covered by the
  937. * access).
  938. */
  939. static inline bool watchpoint_address_matches(CPUWatchpoint *wp,
  940. vaddr addr, vaddr len)
  941. {
  942. /* We know the lengths are non-zero, but a little caution is
  943. * required to avoid errors in the case where the range ends
  944. * exactly at the top of the address space and so addr + len
  945. * wraps round to zero.
  946. */
  947. vaddr wpend = wp->vaddr + wp->len - 1;
  948. vaddr addrend = addr + len - 1;
  949. return !(addr > wpend || wp->vaddr > addrend);
  950. }
  951. /* Return flags for watchpoints that match addr + prot. */
  952. int cpu_watchpoint_address_matches(CPUState *cpu, vaddr addr, vaddr len)
  953. {
  954. CPUWatchpoint *wp;
  955. int ret = 0;
  956. QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
  957. if (watchpoint_address_matches(wp, addr, TARGET_PAGE_SIZE)) {
  958. ret |= wp->flags;
  959. }
  960. }
  961. return ret;
  962. }
  963. #endif /* !CONFIG_USER_ONLY */
  964. /* Add a breakpoint. */
  965. int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
  966. CPUBreakpoint **breakpoint)
  967. {
  968. CPUBreakpoint *bp;
  969. bp = g_malloc(sizeof(*bp));
  970. bp->pc = pc;
  971. bp->flags = flags;
  972. /* keep all GDB-injected breakpoints in front */
  973. if (flags & BP_GDB) {
  974. QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
  975. } else {
  976. QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
  977. }
  978. breakpoint_invalidate(cpu, pc);
  979. if (breakpoint) {
  980. *breakpoint = bp;
  981. }
  982. return 0;
  983. }
  984. /* Remove a specific breakpoint. */
  985. int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
  986. {
  987. CPUBreakpoint *bp;
  988. QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
  989. if (bp->pc == pc && bp->flags == flags) {
  990. cpu_breakpoint_remove_by_ref(cpu, bp);
  991. return 0;
  992. }
  993. }
  994. return -ENOENT;
  995. }
  996. /* Remove a specific breakpoint by reference. */
  997. void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
  998. {
  999. QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
  1000. breakpoint_invalidate(cpu, breakpoint->pc);
  1001. g_free(breakpoint);
  1002. }
  1003. /* Remove all matching breakpoints. */
  1004. void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
  1005. {
  1006. CPUBreakpoint *bp, *next;
  1007. QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
  1008. if (bp->flags & mask) {
  1009. cpu_breakpoint_remove_by_ref(cpu, bp);
  1010. }
  1011. }
  1012. }
  1013. /* enable or disable single step mode. EXCP_DEBUG is returned by the
  1014. CPU loop after each instruction */
  1015. void cpu_single_step(CPUState *cpu, int enabled)
  1016. {
  1017. if (cpu->singlestep_enabled != enabled) {
  1018. cpu->singlestep_enabled = enabled;
  1019. if (kvm_enabled()) {
  1020. kvm_update_guest_debug(cpu, 0);
  1021. } else {
  1022. /* must flush all the translated code to avoid inconsistencies */
  1023. /* XXX: only flush what is necessary */
  1024. tb_flush(cpu);
  1025. }
  1026. }
  1027. }
  1028. void cpu_abort(CPUState *cpu, const char *fmt, ...)
  1029. {
  1030. va_list ap;
  1031. va_list ap2;
  1032. va_start(ap, fmt);
  1033. va_copy(ap2, ap);
  1034. fprintf(stderr, "qemu: fatal: ");
  1035. vfprintf(stderr, fmt, ap);
  1036. fprintf(stderr, "\n");
  1037. cpu_dump_state(cpu, stderr, CPU_DUMP_FPU | CPU_DUMP_CCOP);
  1038. if (qemu_log_separate()) {
  1039. FILE *logfile = qemu_log_lock();
  1040. qemu_log("qemu: fatal: ");
  1041. qemu_log_vprintf(fmt, ap2);
  1042. qemu_log("\n");
  1043. log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
  1044. qemu_log_flush();
  1045. qemu_log_unlock(logfile);
  1046. qemu_log_close();
  1047. }
  1048. va_end(ap2);
  1049. va_end(ap);
  1050. replay_finish();
  1051. #if defined(CONFIG_USER_ONLY)
  1052. {
  1053. struct sigaction act;
  1054. sigfillset(&act.sa_mask);
  1055. act.sa_handler = SIG_DFL;
  1056. act.sa_flags = 0;
  1057. sigaction(SIGABRT, &act, NULL);
  1058. }
  1059. #endif
  1060. abort();
  1061. }
  1062. #if !defined(CONFIG_USER_ONLY)
  1063. /* Called from RCU critical section */
  1064. static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
  1065. {
  1066. RAMBlock *block;
  1067. block = atomic_rcu_read(&ram_list.mru_block);
  1068. if (block && addr - block->offset < block->max_length) {
  1069. return block;
  1070. }
  1071. RAMBLOCK_FOREACH(block) {
  1072. if (addr - block->offset < block->max_length) {
  1073. goto found;
  1074. }
  1075. }
  1076. fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
  1077. abort();
  1078. found:
  1079. /* It is safe to write mru_block outside the iothread lock. This
  1080. * is what happens:
  1081. *
  1082. * mru_block = xxx
  1083. * rcu_read_unlock()
  1084. * xxx removed from list
  1085. * rcu_read_lock()
  1086. * read mru_block
  1087. * mru_block = NULL;
  1088. * call_rcu(reclaim_ramblock, xxx);
  1089. * rcu_read_unlock()
  1090. *
  1091. * atomic_rcu_set is not needed here. The block was already published
  1092. * when it was placed into the list. Here we're just making an extra
  1093. * copy of the pointer.
  1094. */
  1095. ram_list.mru_block = block;
  1096. return block;
  1097. }
  1098. static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
  1099. {
  1100. CPUState *cpu;
  1101. ram_addr_t start1;
  1102. RAMBlock *block;
  1103. ram_addr_t end;
  1104. assert(tcg_enabled());
  1105. end = TARGET_PAGE_ALIGN(start + length);
  1106. start &= TARGET_PAGE_MASK;
  1107. RCU_READ_LOCK_GUARD();
  1108. block = qemu_get_ram_block(start);
  1109. assert(block == qemu_get_ram_block(end - 1));
  1110. start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
  1111. CPU_FOREACH(cpu) {
  1112. tlb_reset_dirty(cpu, start1, length);
  1113. }
  1114. }
  1115. /* Note: start and end must be within the same ram block. */
  1116. bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
  1117. ram_addr_t length,
  1118. unsigned client)
  1119. {
  1120. DirtyMemoryBlocks *blocks;
  1121. unsigned long end, page;
  1122. bool dirty = false;
  1123. RAMBlock *ramblock;
  1124. uint64_t mr_offset, mr_size;
  1125. if (length == 0) {
  1126. return false;
  1127. }
  1128. end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
  1129. page = start >> TARGET_PAGE_BITS;
  1130. WITH_RCU_READ_LOCK_GUARD() {
  1131. blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
  1132. ramblock = qemu_get_ram_block(start);
  1133. /* Range sanity check on the ramblock */
  1134. assert(start >= ramblock->offset &&
  1135. start + length <= ramblock->offset + ramblock->used_length);
  1136. while (page < end) {
  1137. unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
  1138. unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
  1139. unsigned long num = MIN(end - page,
  1140. DIRTY_MEMORY_BLOCK_SIZE - offset);
  1141. dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
  1142. offset, num);
  1143. page += num;
  1144. }
  1145. mr_offset = (ram_addr_t)(page << TARGET_PAGE_BITS) - ramblock->offset;
  1146. mr_size = (end - page) << TARGET_PAGE_BITS;
  1147. memory_region_clear_dirty_bitmap(ramblock->mr, mr_offset, mr_size);
  1148. }
  1149. if (dirty && tcg_enabled()) {
  1150. tlb_reset_dirty_range_all(start, length);
  1151. }
  1152. return dirty;
  1153. }
  1154. DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
  1155. (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client)
  1156. {
  1157. DirtyMemoryBlocks *blocks;
  1158. ram_addr_t start = memory_region_get_ram_addr(mr) + offset;
  1159. unsigned long align = 1UL << (TARGET_PAGE_BITS + BITS_PER_LEVEL);
  1160. ram_addr_t first = QEMU_ALIGN_DOWN(start, align);
  1161. ram_addr_t last = QEMU_ALIGN_UP(start + length, align);
  1162. DirtyBitmapSnapshot *snap;
  1163. unsigned long page, end, dest;
  1164. snap = g_malloc0(sizeof(*snap) +
  1165. ((last - first) >> (TARGET_PAGE_BITS + 3)));
  1166. snap->start = first;
  1167. snap->end = last;
  1168. page = first >> TARGET_PAGE_BITS;
  1169. end = last >> TARGET_PAGE_BITS;
  1170. dest = 0;
  1171. WITH_RCU_READ_LOCK_GUARD() {
  1172. blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
  1173. while (page < end) {
  1174. unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
  1175. unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
  1176. unsigned long num = MIN(end - page,
  1177. DIRTY_MEMORY_BLOCK_SIZE - offset);
  1178. assert(QEMU_IS_ALIGNED(offset, (1 << BITS_PER_LEVEL)));
  1179. assert(QEMU_IS_ALIGNED(num, (1 << BITS_PER_LEVEL)));
  1180. offset >>= BITS_PER_LEVEL;
  1181. bitmap_copy_and_clear_atomic(snap->dirty + dest,
  1182. blocks->blocks[idx] + offset,
  1183. num);
  1184. page += num;
  1185. dest += num >> BITS_PER_LEVEL;
  1186. }
  1187. }
  1188. if (tcg_enabled()) {
  1189. tlb_reset_dirty_range_all(start, length);
  1190. }
  1191. memory_region_clear_dirty_bitmap(mr, offset, length);
  1192. return snap;
  1193. }
  1194. bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
  1195. ram_addr_t start,
  1196. ram_addr_t length)
  1197. {
  1198. unsigned long page, end;
  1199. assert(start >= snap->start);
  1200. assert(start + length <= snap->end);
  1201. end = TARGET_PAGE_ALIGN(start + length - snap->start) >> TARGET_PAGE_BITS;
  1202. page = (start - snap->start) >> TARGET_PAGE_BITS;
  1203. while (page < end) {
  1204. if (test_bit(page, snap->dirty)) {
  1205. return true;
  1206. }
  1207. page++;
  1208. }
  1209. return false;
  1210. }
  1211. /* Called from RCU critical section */
  1212. hwaddr memory_region_section_get_iotlb(CPUState *cpu,
  1213. MemoryRegionSection *section)
  1214. {
  1215. AddressSpaceDispatch *d = flatview_to_dispatch(section->fv);
  1216. return section - d->map.sections;
  1217. }
  1218. #endif /* defined(CONFIG_USER_ONLY) */
  1219. #if !defined(CONFIG_USER_ONLY)
  1220. static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end,
  1221. uint16_t section);
  1222. static subpage_t *subpage_init(FlatView *fv, hwaddr base);
  1223. static void *(*phys_mem_alloc)(size_t size, uint64_t *align, bool shared) =
  1224. qemu_anon_ram_alloc;
  1225. /*
  1226. * Set a custom physical guest memory alloator.
  1227. * Accelerators with unusual needs may need this. Hopefully, we can
  1228. * get rid of it eventually.
  1229. */
  1230. void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align, bool shared))
  1231. {
  1232. phys_mem_alloc = alloc;
  1233. }
  1234. static uint16_t phys_section_add(PhysPageMap *map,
  1235. MemoryRegionSection *section)
  1236. {
  1237. /* The physical section number is ORed with a page-aligned
  1238. * pointer to produce the iotlb entries. Thus it should
  1239. * never overflow into the page-aligned value.
  1240. */
  1241. assert(map->sections_nb < TARGET_PAGE_SIZE);
  1242. if (map->sections_nb == map->sections_nb_alloc) {
  1243. map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
  1244. map->sections = g_renew(MemoryRegionSection, map->sections,
  1245. map->sections_nb_alloc);
  1246. }
  1247. map->sections[map->sections_nb] = *section;
  1248. memory_region_ref(section->mr);
  1249. return map->sections_nb++;
  1250. }
  1251. static void phys_section_destroy(MemoryRegion *mr)
  1252. {
  1253. bool have_sub_page = mr->subpage;
  1254. memory_region_unref(mr);
  1255. if (have_sub_page) {
  1256. subpage_t *subpage = container_of(mr, subpage_t, iomem);
  1257. object_unref(OBJECT(&subpage->iomem));
  1258. g_free(subpage);
  1259. }
  1260. }
  1261. static void phys_sections_free(PhysPageMap *map)
  1262. {
  1263. while (map->sections_nb > 0) {
  1264. MemoryRegionSection *section = &map->sections[--map->sections_nb];
  1265. phys_section_destroy(section->mr);
  1266. }
  1267. g_free(map->sections);
  1268. g_free(map->nodes);
  1269. }
  1270. static void register_subpage(FlatView *fv, MemoryRegionSection *section)
  1271. {
  1272. AddressSpaceDispatch *d = flatview_to_dispatch(fv);
  1273. subpage_t *subpage;
  1274. hwaddr base = section->offset_within_address_space
  1275. & TARGET_PAGE_MASK;
  1276. MemoryRegionSection *existing = phys_page_find(d, base);
  1277. MemoryRegionSection subsection = {
  1278. .offset_within_address_space = base,
  1279. .size = int128_make64(TARGET_PAGE_SIZE),
  1280. };
  1281. hwaddr start, end;
  1282. assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
  1283. if (!(existing->mr->subpage)) {
  1284. subpage = subpage_init(fv, base);
  1285. subsection.fv = fv;
  1286. subsection.mr = &subpage->iomem;
  1287. phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
  1288. phys_section_add(&d->map, &subsection));
  1289. } else {
  1290. subpage = container_of(existing->mr, subpage_t, iomem);
  1291. }
  1292. start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
  1293. end = start + int128_get64(section->size) - 1;
  1294. subpage_register(subpage, start, end,
  1295. phys_section_add(&d->map, section));
  1296. }
  1297. static void register_multipage(FlatView *fv,
  1298. MemoryRegionSection *section)
  1299. {
  1300. AddressSpaceDispatch *d = flatview_to_dispatch(fv);
  1301. hwaddr start_addr = section->offset_within_address_space;
  1302. uint16_t section_index = phys_section_add(&d->map, section);
  1303. uint64_t num_pages = int128_get64(int128_rshift(section->size,
  1304. TARGET_PAGE_BITS));
  1305. assert(num_pages);
  1306. phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
  1307. }
  1308. /*
  1309. * The range in *section* may look like this:
  1310. *
  1311. * |s|PPPPPPP|s|
  1312. *
  1313. * where s stands for subpage and P for page.
  1314. */
  1315. void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section)
  1316. {
  1317. MemoryRegionSection remain = *section;
  1318. Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
  1319. /* register first subpage */
  1320. if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
  1321. uint64_t left = TARGET_PAGE_ALIGN(remain.offset_within_address_space)
  1322. - remain.offset_within_address_space;
  1323. MemoryRegionSection now = remain;
  1324. now.size = int128_min(int128_make64(left), now.size);
  1325. register_subpage(fv, &now);
  1326. if (int128_eq(remain.size, now.size)) {
  1327. return;
  1328. }
  1329. remain.size = int128_sub(remain.size, now.size);
  1330. remain.offset_within_address_space += int128_get64(now.size);
  1331. remain.offset_within_region += int128_get64(now.size);
  1332. }
  1333. /* register whole pages */
  1334. if (int128_ge(remain.size, page_size)) {
  1335. MemoryRegionSection now = remain;
  1336. now.size = int128_and(now.size, int128_neg(page_size));
  1337. register_multipage(fv, &now);
  1338. if (int128_eq(remain.size, now.size)) {
  1339. return;
  1340. }
  1341. remain.size = int128_sub(remain.size, now.size);
  1342. remain.offset_within_address_space += int128_get64(now.size);
  1343. remain.offset_within_region += int128_get64(now.size);
  1344. }
  1345. /* register last subpage */
  1346. register_subpage(fv, &remain);
  1347. }
  1348. void qemu_flush_coalesced_mmio_buffer(void)
  1349. {
  1350. if (kvm_enabled())
  1351. kvm_flush_coalesced_mmio_buffer();
  1352. }
  1353. void qemu_mutex_lock_ramlist(void)
  1354. {
  1355. qemu_mutex_lock(&ram_list.mutex);
  1356. }
  1357. void qemu_mutex_unlock_ramlist(void)
  1358. {
  1359. qemu_mutex_unlock(&ram_list.mutex);
  1360. }
  1361. void ram_block_dump(Monitor *mon)
  1362. {
  1363. RAMBlock *block;
  1364. char *psize;
  1365. RCU_READ_LOCK_GUARD();
  1366. monitor_printf(mon, "%24s %8s %18s %18s %18s\n",
  1367. "Block Name", "PSize", "Offset", "Used", "Total");
  1368. RAMBLOCK_FOREACH(block) {
  1369. psize = size_to_str(block->page_size);
  1370. monitor_printf(mon, "%24s %8s 0x%016" PRIx64 " 0x%016" PRIx64
  1371. " 0x%016" PRIx64 "\n", block->idstr, psize,
  1372. (uint64_t)block->offset,
  1373. (uint64_t)block->used_length,
  1374. (uint64_t)block->max_length);
  1375. g_free(psize);
  1376. }
  1377. }
  1378. #ifdef __linux__
  1379. /*
  1380. * FIXME TOCTTOU: this iterates over memory backends' mem-path, which
  1381. * may or may not name the same files / on the same filesystem now as
  1382. * when we actually open and map them. Iterate over the file
  1383. * descriptors instead, and use qemu_fd_getpagesize().
  1384. */
  1385. static int find_min_backend_pagesize(Object *obj, void *opaque)
  1386. {
  1387. long *hpsize_min = opaque;
  1388. if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
  1389. HostMemoryBackend *backend = MEMORY_BACKEND(obj);
  1390. long hpsize = host_memory_backend_pagesize(backend);
  1391. if (host_memory_backend_is_mapped(backend) && (hpsize < *hpsize_min)) {
  1392. *hpsize_min = hpsize;
  1393. }
  1394. }
  1395. return 0;
  1396. }
  1397. static int find_max_backend_pagesize(Object *obj, void *opaque)
  1398. {
  1399. long *hpsize_max = opaque;
  1400. if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
  1401. HostMemoryBackend *backend = MEMORY_BACKEND(obj);
  1402. long hpsize = host_memory_backend_pagesize(backend);
  1403. if (host_memory_backend_is_mapped(backend) && (hpsize > *hpsize_max)) {
  1404. *hpsize_max = hpsize;
  1405. }
  1406. }
  1407. return 0;
  1408. }
  1409. /*
  1410. * TODO: We assume right now that all mapped host memory backends are
  1411. * used as RAM, however some might be used for different purposes.
  1412. */
  1413. long qemu_minrampagesize(void)
  1414. {
  1415. long hpsize = LONG_MAX;
  1416. long mainrampagesize;
  1417. Object *memdev_root;
  1418. MachineState *ms = MACHINE(qdev_get_machine());
  1419. mainrampagesize = qemu_mempath_getpagesize(mem_path);
  1420. /* it's possible we have memory-backend objects with
  1421. * hugepage-backed RAM. these may get mapped into system
  1422. * address space via -numa parameters or memory hotplug
  1423. * hooks. we want to take these into account, but we
  1424. * also want to make sure these supported hugepage
  1425. * sizes are applicable across the entire range of memory
  1426. * we may boot from, so we take the min across all
  1427. * backends, and assume normal pages in cases where a
  1428. * backend isn't backed by hugepages.
  1429. */
  1430. memdev_root = object_resolve_path("/objects", NULL);
  1431. if (memdev_root) {
  1432. object_child_foreach(memdev_root, find_min_backend_pagesize, &hpsize);
  1433. }
  1434. if (hpsize == LONG_MAX) {
  1435. /* No additional memory regions found ==> Report main RAM page size */
  1436. return mainrampagesize;
  1437. }
  1438. /* If NUMA is disabled or the NUMA nodes are not backed with a
  1439. * memory-backend, then there is at least one node using "normal" RAM,
  1440. * so if its page size is smaller we have got to report that size instead.
  1441. */
  1442. if (hpsize > mainrampagesize &&
  1443. (ms->numa_state == NULL ||
  1444. ms->numa_state->num_nodes == 0 ||
  1445. ms->numa_state->nodes[0].node_memdev == NULL)) {
  1446. static bool warned;
  1447. if (!warned) {
  1448. error_report("Huge page support disabled (n/a for main memory).");
  1449. warned = true;
  1450. }
  1451. return mainrampagesize;
  1452. }
  1453. return hpsize;
  1454. }
  1455. long qemu_maxrampagesize(void)
  1456. {
  1457. long pagesize = qemu_mempath_getpagesize(mem_path);
  1458. Object *memdev_root = object_resolve_path("/objects", NULL);
  1459. if (memdev_root) {
  1460. object_child_foreach(memdev_root, find_max_backend_pagesize,
  1461. &pagesize);
  1462. }
  1463. return pagesize;
  1464. }
  1465. #else
  1466. long qemu_minrampagesize(void)
  1467. {
  1468. return qemu_real_host_page_size;
  1469. }
  1470. long qemu_maxrampagesize(void)
  1471. {
  1472. return qemu_real_host_page_size;
  1473. }
  1474. #endif
  1475. #ifdef CONFIG_POSIX
  1476. static int64_t get_file_size(int fd)
  1477. {
  1478. int64_t size;
  1479. #if defined(__linux__)
  1480. struct stat st;
  1481. if (fstat(fd, &st) < 0) {
  1482. return -errno;
  1483. }
  1484. /* Special handling for devdax character devices */
  1485. if (S_ISCHR(st.st_mode)) {
  1486. g_autofree char *subsystem_path = NULL;
  1487. g_autofree char *subsystem = NULL;
  1488. subsystem_path = g_strdup_printf("/sys/dev/char/%d:%d/subsystem",
  1489. major(st.st_rdev), minor(st.st_rdev));
  1490. subsystem = g_file_read_link(subsystem_path, NULL);
  1491. if (subsystem && g_str_has_suffix(subsystem, "/dax")) {
  1492. g_autofree char *size_path = NULL;
  1493. g_autofree char *size_str = NULL;
  1494. size_path = g_strdup_printf("/sys/dev/char/%d:%d/size",
  1495. major(st.st_rdev), minor(st.st_rdev));
  1496. if (g_file_get_contents(size_path, &size_str, NULL, NULL)) {
  1497. return g_ascii_strtoll(size_str, NULL, 0);
  1498. }
  1499. }
  1500. }
  1501. #endif /* defined(__linux__) */
  1502. /* st.st_size may be zero for special files yet lseek(2) works */
  1503. size = lseek(fd, 0, SEEK_END);
  1504. if (size < 0) {
  1505. return -errno;
  1506. }
  1507. return size;
  1508. }
  1509. static int file_ram_open(const char *path,
  1510. const char *region_name,
  1511. bool *created,
  1512. Error **errp)
  1513. {
  1514. char *filename;
  1515. char *sanitized_name;
  1516. char *c;
  1517. int fd = -1;
  1518. *created = false;
  1519. for (;;) {
  1520. fd = open(path, O_RDWR);
  1521. if (fd >= 0) {
  1522. /* @path names an existing file, use it */
  1523. break;
  1524. }
  1525. if (errno == ENOENT) {
  1526. /* @path names a file that doesn't exist, create it */
  1527. fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
  1528. if (fd >= 0) {
  1529. *created = true;
  1530. break;
  1531. }
  1532. } else if (errno == EISDIR) {
  1533. /* @path names a directory, create a file there */
  1534. /* Make name safe to use with mkstemp by replacing '/' with '_'. */
  1535. sanitized_name = g_strdup(region_name);
  1536. for (c = sanitized_name; *c != '\0'; c++) {
  1537. if (*c == '/') {
  1538. *c = '_';
  1539. }
  1540. }
  1541. filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
  1542. sanitized_name);
  1543. g_free(sanitized_name);
  1544. fd = mkstemp(filename);
  1545. if (fd >= 0) {
  1546. unlink(filename);
  1547. g_free(filename);
  1548. break;
  1549. }
  1550. g_free(filename);
  1551. }
  1552. if (errno != EEXIST && errno != EINTR) {
  1553. error_setg_errno(errp, errno,
  1554. "can't open backing store %s for guest RAM",
  1555. path);
  1556. return -1;
  1557. }
  1558. /*
  1559. * Try again on EINTR and EEXIST. The latter happens when
  1560. * something else creates the file between our two open().
  1561. */
  1562. }
  1563. return fd;
  1564. }
  1565. static void *file_ram_alloc(RAMBlock *block,
  1566. ram_addr_t memory,
  1567. int fd,
  1568. bool truncate,
  1569. Error **errp)
  1570. {
  1571. Error *err = NULL;
  1572. MachineState *ms = MACHINE(qdev_get_machine());
  1573. void *area;
  1574. block->page_size = qemu_fd_getpagesize(fd);
  1575. if (block->mr->align % block->page_size) {
  1576. error_setg(errp, "alignment 0x%" PRIx64
  1577. " must be multiples of page size 0x%zx",
  1578. block->mr->align, block->page_size);
  1579. return NULL;
  1580. } else if (block->mr->align && !is_power_of_2(block->mr->align)) {
  1581. error_setg(errp, "alignment 0x%" PRIx64
  1582. " must be a power of two", block->mr->align);
  1583. return NULL;
  1584. }
  1585. block->mr->align = MAX(block->page_size, block->mr->align);
  1586. #if defined(__s390x__)
  1587. if (kvm_enabled()) {
  1588. block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN);
  1589. }
  1590. #endif
  1591. if (memory < block->page_size) {
  1592. error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
  1593. "or larger than page size 0x%zx",
  1594. memory, block->page_size);
  1595. return NULL;
  1596. }
  1597. memory = ROUND_UP(memory, block->page_size);
  1598. /*
  1599. * ftruncate is not supported by hugetlbfs in older
  1600. * hosts, so don't bother bailing out on errors.
  1601. * If anything goes wrong with it under other filesystems,
  1602. * mmap will fail.
  1603. *
  1604. * Do not truncate the non-empty backend file to avoid corrupting
  1605. * the existing data in the file. Disabling shrinking is not
  1606. * enough. For example, the current vNVDIMM implementation stores
  1607. * the guest NVDIMM labels at the end of the backend file. If the
  1608. * backend file is later extended, QEMU will not be able to find
  1609. * those labels. Therefore, extending the non-empty backend file
  1610. * is disabled as well.
  1611. */
  1612. if (truncate && ftruncate(fd, memory)) {
  1613. perror("ftruncate");
  1614. }
  1615. area = qemu_ram_mmap(fd, memory, block->mr->align,
  1616. block->flags & RAM_SHARED, block->flags & RAM_PMEM);
  1617. if (area == MAP_FAILED) {
  1618. error_setg_errno(errp, errno,
  1619. "unable to map backing store for guest RAM");
  1620. return NULL;
  1621. }
  1622. if (mem_prealloc) {
  1623. os_mem_prealloc(fd, area, memory, ms->smp.cpus, &err);
  1624. if (err) {
  1625. error_propagate(errp, err);
  1626. qemu_ram_munmap(fd, area, memory);
  1627. return NULL;
  1628. }
  1629. }
  1630. block->fd = fd;
  1631. return area;
  1632. }
  1633. #endif
  1634. /* Allocate space within the ram_addr_t space that governs the
  1635. * dirty bitmaps.
  1636. * Called with the ramlist lock held.
  1637. */
  1638. static ram_addr_t find_ram_offset(ram_addr_t size)
  1639. {
  1640. RAMBlock *block, *next_block;
  1641. ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
  1642. assert(size != 0); /* it would hand out same offset multiple times */
  1643. if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
  1644. return 0;
  1645. }
  1646. RAMBLOCK_FOREACH(block) {
  1647. ram_addr_t candidate, next = RAM_ADDR_MAX;
  1648. /* Align blocks to start on a 'long' in the bitmap
  1649. * which makes the bitmap sync'ing take the fast path.
  1650. */
  1651. candidate = block->offset + block->max_length;
  1652. candidate = ROUND_UP(candidate, BITS_PER_LONG << TARGET_PAGE_BITS);
  1653. /* Search for the closest following block
  1654. * and find the gap.
  1655. */
  1656. RAMBLOCK_FOREACH(next_block) {
  1657. if (next_block->offset >= candidate) {
  1658. next = MIN(next, next_block->offset);
  1659. }
  1660. }
  1661. /* If it fits remember our place and remember the size
  1662. * of gap, but keep going so that we might find a smaller
  1663. * gap to fill so avoiding fragmentation.
  1664. */
  1665. if (next - candidate >= size && next - candidate < mingap) {
  1666. offset = candidate;
  1667. mingap = next - candidate;
  1668. }
  1669. trace_find_ram_offset_loop(size, candidate, offset, next, mingap);
  1670. }
  1671. if (offset == RAM_ADDR_MAX) {
  1672. fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
  1673. (uint64_t)size);
  1674. abort();
  1675. }
  1676. trace_find_ram_offset(size, offset);
  1677. return offset;
  1678. }
  1679. static unsigned long last_ram_page(void)
  1680. {
  1681. RAMBlock *block;
  1682. ram_addr_t last = 0;
  1683. RCU_READ_LOCK_GUARD();
  1684. RAMBLOCK_FOREACH(block) {
  1685. last = MAX(last, block->offset + block->max_length);
  1686. }
  1687. return last >> TARGET_PAGE_BITS;
  1688. }
  1689. static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
  1690. {
  1691. int ret;
  1692. /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
  1693. if (!machine_dump_guest_core(current_machine)) {
  1694. ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
  1695. if (ret) {
  1696. perror("qemu_madvise");
  1697. fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
  1698. "but dump_guest_core=off specified\n");
  1699. }
  1700. }
  1701. }
  1702. const char *qemu_ram_get_idstr(RAMBlock *rb)
  1703. {
  1704. return rb->idstr;
  1705. }
  1706. void *qemu_ram_get_host_addr(RAMBlock *rb)
  1707. {
  1708. return rb->host;
  1709. }
  1710. ram_addr_t qemu_ram_get_offset(RAMBlock *rb)
  1711. {
  1712. return rb->offset;
  1713. }
  1714. ram_addr_t qemu_ram_get_used_length(RAMBlock *rb)
  1715. {
  1716. return rb->used_length;
  1717. }
  1718. bool qemu_ram_is_shared(RAMBlock *rb)
  1719. {
  1720. return rb->flags & RAM_SHARED;
  1721. }
  1722. /* Note: Only set at the start of postcopy */
  1723. bool qemu_ram_is_uf_zeroable(RAMBlock *rb)
  1724. {
  1725. return rb->flags & RAM_UF_ZEROPAGE;
  1726. }
  1727. void qemu_ram_set_uf_zeroable(RAMBlock *rb)
  1728. {
  1729. rb->flags |= RAM_UF_ZEROPAGE;
  1730. }
  1731. bool qemu_ram_is_migratable(RAMBlock *rb)
  1732. {
  1733. return rb->flags & RAM_MIGRATABLE;
  1734. }
  1735. void qemu_ram_set_migratable(RAMBlock *rb)
  1736. {
  1737. rb->flags |= RAM_MIGRATABLE;
  1738. }
  1739. void qemu_ram_unset_migratable(RAMBlock *rb)
  1740. {
  1741. rb->flags &= ~RAM_MIGRATABLE;
  1742. }
  1743. /* Called with iothread lock held. */
  1744. void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
  1745. {
  1746. RAMBlock *block;
  1747. assert(new_block);
  1748. assert(!new_block->idstr[0]);
  1749. if (dev) {
  1750. char *id = qdev_get_dev_path(dev);
  1751. if (id) {
  1752. snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
  1753. g_free(id);
  1754. }
  1755. }
  1756. pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
  1757. RCU_READ_LOCK_GUARD();
  1758. RAMBLOCK_FOREACH(block) {
  1759. if (block != new_block &&
  1760. !strcmp(block->idstr, new_block->idstr)) {
  1761. fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
  1762. new_block->idstr);
  1763. abort();
  1764. }
  1765. }
  1766. }
  1767. /* Called with iothread lock held. */
  1768. void qemu_ram_unset_idstr(RAMBlock *block)
  1769. {
  1770. /* FIXME: arch_init.c assumes that this is not called throughout
  1771. * migration. Ignore the problem since hot-unplug during migration
  1772. * does not work anyway.
  1773. */
  1774. if (block) {
  1775. memset(block->idstr, 0, sizeof(block->idstr));
  1776. }
  1777. }
  1778. size_t qemu_ram_pagesize(RAMBlock *rb)
  1779. {
  1780. return rb->page_size;
  1781. }
  1782. /* Returns the largest size of page in use */
  1783. size_t qemu_ram_pagesize_largest(void)
  1784. {
  1785. RAMBlock *block;
  1786. size_t largest = 0;
  1787. RAMBLOCK_FOREACH(block) {
  1788. largest = MAX(largest, qemu_ram_pagesize(block));
  1789. }
  1790. return largest;
  1791. }
  1792. static int memory_try_enable_merging(void *addr, size_t len)
  1793. {
  1794. if (!machine_mem_merge(current_machine)) {
  1795. /* disabled by the user */
  1796. return 0;
  1797. }
  1798. return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
  1799. }
  1800. /* Only legal before guest might have detected the memory size: e.g. on
  1801. * incoming migration, or right after reset.
  1802. *
  1803. * As memory core doesn't know how is memory accessed, it is up to
  1804. * resize callback to update device state and/or add assertions to detect
  1805. * misuse, if necessary.
  1806. */
  1807. int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
  1808. {
  1809. assert(block);
  1810. newsize = HOST_PAGE_ALIGN(newsize);
  1811. if (block->used_length == newsize) {
  1812. return 0;
  1813. }
  1814. if (!(block->flags & RAM_RESIZEABLE)) {
  1815. error_setg_errno(errp, EINVAL,
  1816. "Length mismatch: %s: 0x" RAM_ADDR_FMT
  1817. " in != 0x" RAM_ADDR_FMT, block->idstr,
  1818. newsize, block->used_length);
  1819. return -EINVAL;
  1820. }
  1821. if (block->max_length < newsize) {
  1822. error_setg_errno(errp, EINVAL,
  1823. "Length too large: %s: 0x" RAM_ADDR_FMT
  1824. " > 0x" RAM_ADDR_FMT, block->idstr,
  1825. newsize, block->max_length);
  1826. return -EINVAL;
  1827. }
  1828. cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
  1829. block->used_length = newsize;
  1830. cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
  1831. DIRTY_CLIENTS_ALL);
  1832. memory_region_set_size(block->mr, newsize);
  1833. if (block->resized) {
  1834. block->resized(block->idstr, newsize, block->host);
  1835. }
  1836. return 0;
  1837. }
  1838. /*
  1839. * Trigger sync on the given ram block for range [start, start + length]
  1840. * with the backing store if one is available.
  1841. * Otherwise no-op.
  1842. * @Note: this is supposed to be a synchronous op.
  1843. */
  1844. void qemu_ram_writeback(RAMBlock *block, ram_addr_t start, ram_addr_t length)
  1845. {
  1846. void *addr = ramblock_ptr(block, start);
  1847. /* The requested range should fit in within the block range */
  1848. g_assert((start + length) <= block->used_length);
  1849. #ifdef CONFIG_LIBPMEM
  1850. /* The lack of support for pmem should not block the sync */
  1851. if (ramblock_is_pmem(block)) {
  1852. pmem_persist(addr, length);
  1853. return;
  1854. }
  1855. #endif
  1856. if (block->fd >= 0) {
  1857. /**
  1858. * Case there is no support for PMEM or the memory has not been
  1859. * specified as persistent (or is not one) - use the msync.
  1860. * Less optimal but still achieves the same goal
  1861. */
  1862. if (qemu_msync(addr, length, block->fd)) {
  1863. warn_report("%s: failed to sync memory range: start: "
  1864. RAM_ADDR_FMT " length: " RAM_ADDR_FMT,
  1865. __func__, start, length);
  1866. }
  1867. }
  1868. }
  1869. /* Called with ram_list.mutex held */
  1870. static void dirty_memory_extend(ram_addr_t old_ram_size,
  1871. ram_addr_t new_ram_size)
  1872. {
  1873. ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size,
  1874. DIRTY_MEMORY_BLOCK_SIZE);
  1875. ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size,
  1876. DIRTY_MEMORY_BLOCK_SIZE);
  1877. int i;
  1878. /* Only need to extend if block count increased */
  1879. if (new_num_blocks <= old_num_blocks) {
  1880. return;
  1881. }
  1882. for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
  1883. DirtyMemoryBlocks *old_blocks;
  1884. DirtyMemoryBlocks *new_blocks;
  1885. int j;
  1886. old_blocks = atomic_rcu_read(&ram_list.dirty_memory[i]);
  1887. new_blocks = g_malloc(sizeof(*new_blocks) +
  1888. sizeof(new_blocks->blocks[0]) * new_num_blocks);
  1889. if (old_num_blocks) {
  1890. memcpy(new_blocks->blocks, old_blocks->blocks,
  1891. old_num_blocks * sizeof(old_blocks->blocks[0]));
  1892. }
  1893. for (j = old_num_blocks; j < new_num_blocks; j++) {
  1894. new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
  1895. }
  1896. atomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
  1897. if (old_blocks) {
  1898. g_free_rcu(old_blocks, rcu);
  1899. }
  1900. }
  1901. }
  1902. static void ram_block_add(RAMBlock *new_block, Error **errp, bool shared)
  1903. {
  1904. RAMBlock *block;
  1905. RAMBlock *last_block = NULL;
  1906. ram_addr_t old_ram_size, new_ram_size;
  1907. Error *err = NULL;
  1908. old_ram_size = last_ram_page();
  1909. qemu_mutex_lock_ramlist();
  1910. new_block->offset = find_ram_offset(new_block->max_length);
  1911. if (!new_block->host) {
  1912. if (xen_enabled()) {
  1913. xen_ram_alloc(new_block->offset, new_block->max_length,
  1914. new_block->mr, &err);
  1915. if (err) {
  1916. error_propagate(errp, err);
  1917. qemu_mutex_unlock_ramlist();
  1918. return;
  1919. }
  1920. } else {
  1921. new_block->host = phys_mem_alloc(new_block->max_length,
  1922. &new_block->mr->align, shared);
  1923. if (!new_block->host) {
  1924. error_setg_errno(errp, errno,
  1925. "cannot set up guest memory '%s'",
  1926. memory_region_name(new_block->mr));
  1927. qemu_mutex_unlock_ramlist();
  1928. return;
  1929. }
  1930. memory_try_enable_merging(new_block->host, new_block->max_length);
  1931. }
  1932. }
  1933. new_ram_size = MAX(old_ram_size,
  1934. (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
  1935. if (new_ram_size > old_ram_size) {
  1936. dirty_memory_extend(old_ram_size, new_ram_size);
  1937. }
  1938. /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
  1939. * QLIST (which has an RCU-friendly variant) does not have insertion at
  1940. * tail, so save the last element in last_block.
  1941. */
  1942. RAMBLOCK_FOREACH(block) {
  1943. last_block = block;
  1944. if (block->max_length < new_block->max_length) {
  1945. break;
  1946. }
  1947. }
  1948. if (block) {
  1949. QLIST_INSERT_BEFORE_RCU(block, new_block, next);
  1950. } else if (last_block) {
  1951. QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
  1952. } else { /* list is empty */
  1953. QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
  1954. }
  1955. ram_list.mru_block = NULL;
  1956. /* Write list before version */
  1957. smp_wmb();
  1958. ram_list.version++;
  1959. qemu_mutex_unlock_ramlist();
  1960. cpu_physical_memory_set_dirty_range(new_block->offset,
  1961. new_block->used_length,
  1962. DIRTY_CLIENTS_ALL);
  1963. if (new_block->host) {
  1964. qemu_ram_setup_dump(new_block->host, new_block->max_length);
  1965. qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
  1966. /* MADV_DONTFORK is also needed by KVM in absence of synchronous MMU */
  1967. qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
  1968. ram_block_notify_add(new_block->host, new_block->max_length);
  1969. }
  1970. }
  1971. #ifdef CONFIG_POSIX
  1972. RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
  1973. uint32_t ram_flags, int fd,
  1974. Error **errp)
  1975. {
  1976. RAMBlock *new_block;
  1977. Error *local_err = NULL;
  1978. int64_t file_size;
  1979. /* Just support these ram flags by now. */
  1980. assert((ram_flags & ~(RAM_SHARED | RAM_PMEM)) == 0);
  1981. if (xen_enabled()) {
  1982. error_setg(errp, "-mem-path not supported with Xen");
  1983. return NULL;
  1984. }
  1985. if (kvm_enabled() && !kvm_has_sync_mmu()) {
  1986. error_setg(errp,
  1987. "host lacks kvm mmu notifiers, -mem-path unsupported");
  1988. return NULL;
  1989. }
  1990. if (phys_mem_alloc != qemu_anon_ram_alloc) {
  1991. /*
  1992. * file_ram_alloc() needs to allocate just like
  1993. * phys_mem_alloc, but we haven't bothered to provide
  1994. * a hook there.
  1995. */
  1996. error_setg(errp,
  1997. "-mem-path not supported with this accelerator");
  1998. return NULL;
  1999. }
  2000. size = HOST_PAGE_ALIGN(size);
  2001. file_size = get_file_size(fd);
  2002. if (file_size > 0 && file_size < size) {
  2003. error_setg(errp, "backing store %s size 0x%" PRIx64
  2004. " does not match 'size' option 0x" RAM_ADDR_FMT,
  2005. mem_path, file_size, size);
  2006. return NULL;
  2007. }
  2008. new_block = g_malloc0(sizeof(*new_block));
  2009. new_block->mr = mr;
  2010. new_block->used_length = size;
  2011. new_block->max_length = size;
  2012. new_block->flags = ram_flags;
  2013. new_block->host = file_ram_alloc(new_block, size, fd, !file_size, errp);
  2014. if (!new_block->host) {
  2015. g_free(new_block);
  2016. return NULL;
  2017. }
  2018. ram_block_add(new_block, &local_err, ram_flags & RAM_SHARED);
  2019. if (local_err) {
  2020. g_free(new_block);
  2021. error_propagate(errp, local_err);
  2022. return NULL;
  2023. }
  2024. return new_block;
  2025. }
  2026. RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
  2027. uint32_t ram_flags, const char *mem_path,
  2028. Error **errp)
  2029. {
  2030. int fd;
  2031. bool created;
  2032. RAMBlock *block;
  2033. fd = file_ram_open(mem_path, memory_region_name(mr), &created, errp);
  2034. if (fd < 0) {
  2035. return NULL;
  2036. }
  2037. block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, errp);
  2038. if (!block) {
  2039. if (created) {
  2040. unlink(mem_path);
  2041. }
  2042. close(fd);
  2043. return NULL;
  2044. }
  2045. return block;
  2046. }
  2047. #endif
  2048. static
  2049. RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
  2050. void (*resized)(const char*,
  2051. uint64_t length,
  2052. void *host),
  2053. void *host, bool resizeable, bool share,
  2054. MemoryRegion *mr, Error **errp)
  2055. {
  2056. RAMBlock *new_block;
  2057. Error *local_err = NULL;
  2058. size = HOST_PAGE_ALIGN(size);
  2059. max_size = HOST_PAGE_ALIGN(max_size);
  2060. new_block = g_malloc0(sizeof(*new_block));
  2061. new_block->mr = mr;
  2062. new_block->resized = resized;
  2063. new_block->used_length = size;
  2064. new_block->max_length = max_size;
  2065. assert(max_size >= size);
  2066. new_block->fd = -1;
  2067. new_block->page_size = qemu_real_host_page_size;
  2068. new_block->host = host;
  2069. if (host) {
  2070. new_block->flags |= RAM_PREALLOC;
  2071. }
  2072. if (resizeable) {
  2073. new_block->flags |= RAM_RESIZEABLE;
  2074. }
  2075. ram_block_add(new_block, &local_err, share);
  2076. if (local_err) {
  2077. g_free(new_block);
  2078. error_propagate(errp, local_err);
  2079. return NULL;
  2080. }
  2081. return new_block;
  2082. }
  2083. RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
  2084. MemoryRegion *mr, Error **errp)
  2085. {
  2086. return qemu_ram_alloc_internal(size, size, NULL, host, false,
  2087. false, mr, errp);
  2088. }
  2089. RAMBlock *qemu_ram_alloc(ram_addr_t size, bool share,
  2090. MemoryRegion *mr, Error **errp)
  2091. {
  2092. return qemu_ram_alloc_internal(size, size, NULL, NULL, false,
  2093. share, mr, errp);
  2094. }
  2095. RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
  2096. void (*resized)(const char*,
  2097. uint64_t length,
  2098. void *host),
  2099. MemoryRegion *mr, Error **errp)
  2100. {
  2101. return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true,
  2102. false, mr, errp);
  2103. }
  2104. static void reclaim_ramblock(RAMBlock *block)
  2105. {
  2106. if (block->flags & RAM_PREALLOC) {
  2107. ;
  2108. } else if (xen_enabled()) {
  2109. xen_invalidate_map_cache_entry(block->host);
  2110. #ifndef _WIN32
  2111. } else if (block->fd >= 0) {
  2112. qemu_ram_munmap(block->fd, block->host, block->max_length);
  2113. close(block->fd);
  2114. #endif
  2115. } else {
  2116. qemu_anon_ram_free(block->host, block->max_length);
  2117. }
  2118. g_free(block);
  2119. }
  2120. void qemu_ram_free(RAMBlock *block)
  2121. {
  2122. if (!block) {
  2123. return;
  2124. }
  2125. if (block->host) {
  2126. ram_block_notify_remove(block->host, block->max_length);
  2127. }
  2128. qemu_mutex_lock_ramlist();
  2129. QLIST_REMOVE_RCU(block, next);
  2130. ram_list.mru_block = NULL;
  2131. /* Write list before version */
  2132. smp_wmb();
  2133. ram_list.version++;
  2134. call_rcu(block, reclaim_ramblock, rcu);
  2135. qemu_mutex_unlock_ramlist();
  2136. }
  2137. #ifndef _WIN32
  2138. void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
  2139. {
  2140. RAMBlock *block;
  2141. ram_addr_t offset;
  2142. int flags;
  2143. void *area, *vaddr;
  2144. RAMBLOCK_FOREACH(block) {
  2145. offset = addr - block->offset;
  2146. if (offset < block->max_length) {
  2147. vaddr = ramblock_ptr(block, offset);
  2148. if (block->flags & RAM_PREALLOC) {
  2149. ;
  2150. } else if (xen_enabled()) {
  2151. abort();
  2152. } else {
  2153. flags = MAP_FIXED;
  2154. if (block->fd >= 0) {
  2155. flags |= (block->flags & RAM_SHARED ?
  2156. MAP_SHARED : MAP_PRIVATE);
  2157. area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
  2158. flags, block->fd, offset);
  2159. } else {
  2160. /*
  2161. * Remap needs to match alloc. Accelerators that
  2162. * set phys_mem_alloc never remap. If they did,
  2163. * we'd need a remap hook here.
  2164. */
  2165. assert(phys_mem_alloc == qemu_anon_ram_alloc);
  2166. flags |= MAP_PRIVATE | MAP_ANONYMOUS;
  2167. area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
  2168. flags, -1, 0);
  2169. }
  2170. if (area != vaddr) {
  2171. error_report("Could not remap addr: "
  2172. RAM_ADDR_FMT "@" RAM_ADDR_FMT "",
  2173. length, addr);
  2174. exit(1);
  2175. }
  2176. memory_try_enable_merging(vaddr, length);
  2177. qemu_ram_setup_dump(vaddr, length);
  2178. }
  2179. }
  2180. }
  2181. }
  2182. #endif /* !_WIN32 */
  2183. /* Return a host pointer to ram allocated with qemu_ram_alloc.
  2184. * This should not be used for general purpose DMA. Use address_space_map
  2185. * or address_space_rw instead. For local memory (e.g. video ram) that the
  2186. * device owns, use memory_region_get_ram_ptr.
  2187. *
  2188. * Called within RCU critical section.
  2189. */
  2190. void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
  2191. {
  2192. RAMBlock *block = ram_block;
  2193. if (block == NULL) {
  2194. block = qemu_get_ram_block(addr);
  2195. addr -= block->offset;
  2196. }
  2197. if (xen_enabled() && block->host == NULL) {
  2198. /* We need to check if the requested address is in the RAM
  2199. * because we don't want to map the entire memory in QEMU.
  2200. * In that case just map until the end of the page.
  2201. */
  2202. if (block->offset == 0) {
  2203. return xen_map_cache(addr, 0, 0, false);
  2204. }
  2205. block->host = xen_map_cache(block->offset, block->max_length, 1, false);
  2206. }
  2207. return ramblock_ptr(block, addr);
  2208. }
  2209. /* Return a host pointer to guest's ram. Similar to qemu_map_ram_ptr
  2210. * but takes a size argument.
  2211. *
  2212. * Called within RCU critical section.
  2213. */
  2214. static void *qemu_ram_ptr_length(RAMBlock *ram_block, ram_addr_t addr,
  2215. hwaddr *size, bool lock)
  2216. {
  2217. RAMBlock *block = ram_block;
  2218. if (*size == 0) {
  2219. return NULL;
  2220. }
  2221. if (block == NULL) {
  2222. block = qemu_get_ram_block(addr);
  2223. addr -= block->offset;
  2224. }
  2225. *size = MIN(*size, block->max_length - addr);
  2226. if (xen_enabled() && block->host == NULL) {
  2227. /* We need to check if the requested address is in the RAM
  2228. * because we don't want to map the entire memory in QEMU.
  2229. * In that case just map the requested area.
  2230. */
  2231. if (block->offset == 0) {
  2232. return xen_map_cache(addr, *size, lock, lock);
  2233. }
  2234. block->host = xen_map_cache(block->offset, block->max_length, 1, lock);
  2235. }
  2236. return ramblock_ptr(block, addr);
  2237. }
  2238. /* Return the offset of a hostpointer within a ramblock */
  2239. ram_addr_t qemu_ram_block_host_offset(RAMBlock *rb, void *host)
  2240. {
  2241. ram_addr_t res = (uint8_t *)host - (uint8_t *)rb->host;
  2242. assert((uintptr_t)host >= (uintptr_t)rb->host);
  2243. assert(res < rb->max_length);
  2244. return res;
  2245. }
  2246. /*
  2247. * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
  2248. * in that RAMBlock.
  2249. *
  2250. * ptr: Host pointer to look up
  2251. * round_offset: If true round the result offset down to a page boundary
  2252. * *ram_addr: set to result ram_addr
  2253. * *offset: set to result offset within the RAMBlock
  2254. *
  2255. * Returns: RAMBlock (or NULL if not found)
  2256. *
  2257. * By the time this function returns, the returned pointer is not protected
  2258. * by RCU anymore. If the caller is not within an RCU critical section and
  2259. * does not hold the iothread lock, it must have other means of protecting the
  2260. * pointer, such as a reference to the region that includes the incoming
  2261. * ram_addr_t.
  2262. */
  2263. RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
  2264. ram_addr_t *offset)
  2265. {
  2266. RAMBlock *block;
  2267. uint8_t *host = ptr;
  2268. if (xen_enabled()) {
  2269. ram_addr_t ram_addr;
  2270. RCU_READ_LOCK_GUARD();
  2271. ram_addr = xen_ram_addr_from_mapcache(ptr);
  2272. block = qemu_get_ram_block(ram_addr);
  2273. if (block) {
  2274. *offset = ram_addr - block->offset;
  2275. }
  2276. return block;
  2277. }
  2278. RCU_READ_LOCK_GUARD();
  2279. block = atomic_rcu_read(&ram_list.mru_block);
  2280. if (block && block->host && host - block->host < block->max_length) {
  2281. goto found;
  2282. }
  2283. RAMBLOCK_FOREACH(block) {
  2284. /* This case append when the block is not mapped. */
  2285. if (block->host == NULL) {
  2286. continue;
  2287. }
  2288. if (host - block->host < block->max_length) {
  2289. goto found;
  2290. }
  2291. }
  2292. return NULL;
  2293. found:
  2294. *offset = (host - block->host);
  2295. if (round_offset) {
  2296. *offset &= TARGET_PAGE_MASK;
  2297. }
  2298. return block;
  2299. }
  2300. /*
  2301. * Finds the named RAMBlock
  2302. *
  2303. * name: The name of RAMBlock to find
  2304. *
  2305. * Returns: RAMBlock (or NULL if not found)
  2306. */
  2307. RAMBlock *qemu_ram_block_by_name(const char *name)
  2308. {
  2309. RAMBlock *block;
  2310. RAMBLOCK_FOREACH(block) {
  2311. if (!strcmp(name, block->idstr)) {
  2312. return block;
  2313. }
  2314. }
  2315. return NULL;
  2316. }
  2317. /* Some of the softmmu routines need to translate from a host pointer
  2318. (typically a TLB entry) back to a ram offset. */
  2319. ram_addr_t qemu_ram_addr_from_host(void *ptr)
  2320. {
  2321. RAMBlock *block;
  2322. ram_addr_t offset;
  2323. block = qemu_ram_block_from_host(ptr, false, &offset);
  2324. if (!block) {
  2325. return RAM_ADDR_INVALID;
  2326. }
  2327. return block->offset + offset;
  2328. }
  2329. /* Generate a debug exception if a watchpoint has been hit. */
  2330. void cpu_check_watchpoint(CPUState *cpu, vaddr addr, vaddr len,
  2331. MemTxAttrs attrs, int flags, uintptr_t ra)
  2332. {
  2333. CPUClass *cc = CPU_GET_CLASS(cpu);
  2334. CPUWatchpoint *wp;
  2335. assert(tcg_enabled());
  2336. if (cpu->watchpoint_hit) {
  2337. /*
  2338. * We re-entered the check after replacing the TB.
  2339. * Now raise the debug interrupt so that it will
  2340. * trigger after the current instruction.
  2341. */
  2342. qemu_mutex_lock_iothread();
  2343. cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
  2344. qemu_mutex_unlock_iothread();
  2345. return;
  2346. }
  2347. addr = cc->adjust_watchpoint_address(cpu, addr, len);
  2348. QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
  2349. if (watchpoint_address_matches(wp, addr, len)
  2350. && (wp->flags & flags)) {
  2351. if (flags == BP_MEM_READ) {
  2352. wp->flags |= BP_WATCHPOINT_HIT_READ;
  2353. } else {
  2354. wp->flags |= BP_WATCHPOINT_HIT_WRITE;
  2355. }
  2356. wp->hitaddr = MAX(addr, wp->vaddr);
  2357. wp->hitattrs = attrs;
  2358. if (!cpu->watchpoint_hit) {
  2359. if (wp->flags & BP_CPU &&
  2360. !cc->debug_check_watchpoint(cpu, wp)) {
  2361. wp->flags &= ~BP_WATCHPOINT_HIT;
  2362. continue;
  2363. }
  2364. cpu->watchpoint_hit = wp;
  2365. mmap_lock();
  2366. tb_check_watchpoint(cpu, ra);
  2367. if (wp->flags & BP_STOP_BEFORE_ACCESS) {
  2368. cpu->exception_index = EXCP_DEBUG;
  2369. mmap_unlock();
  2370. cpu_loop_exit_restore(cpu, ra);
  2371. } else {
  2372. /* Force execution of one insn next time. */
  2373. cpu->cflags_next_tb = 1 | curr_cflags();
  2374. mmap_unlock();
  2375. if (ra) {
  2376. cpu_restore_state(cpu, ra, true);
  2377. }
  2378. cpu_loop_exit_noexc(cpu);
  2379. }
  2380. }
  2381. } else {
  2382. wp->flags &= ~BP_WATCHPOINT_HIT;
  2383. }
  2384. }
  2385. }
  2386. static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
  2387. MemTxAttrs attrs, uint8_t *buf, hwaddr len);
  2388. static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
  2389. const uint8_t *buf, hwaddr len);
  2390. static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
  2391. bool is_write, MemTxAttrs attrs);
  2392. static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
  2393. unsigned len, MemTxAttrs attrs)
  2394. {
  2395. subpage_t *subpage = opaque;
  2396. uint8_t buf[8];
  2397. MemTxResult res;
  2398. #if defined(DEBUG_SUBPAGE)
  2399. printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
  2400. subpage, len, addr);
  2401. #endif
  2402. res = flatview_read(subpage->fv, addr + subpage->base, attrs, buf, len);
  2403. if (res) {
  2404. return res;
  2405. }
  2406. *data = ldn_p(buf, len);
  2407. return MEMTX_OK;
  2408. }
  2409. static MemTxResult subpage_write(void *opaque, hwaddr addr,
  2410. uint64_t value, unsigned len, MemTxAttrs attrs)
  2411. {
  2412. subpage_t *subpage = opaque;
  2413. uint8_t buf[8];
  2414. #if defined(DEBUG_SUBPAGE)
  2415. printf("%s: subpage %p len %u addr " TARGET_FMT_plx
  2416. " value %"PRIx64"\n",
  2417. __func__, subpage, len, addr, value);
  2418. #endif
  2419. stn_p(buf, len, value);
  2420. return flatview_write(subpage->fv, addr + subpage->base, attrs, buf, len);
  2421. }
  2422. static bool subpage_accepts(void *opaque, hwaddr addr,
  2423. unsigned len, bool is_write,
  2424. MemTxAttrs attrs)
  2425. {
  2426. subpage_t *subpage = opaque;
  2427. #if defined(DEBUG_SUBPAGE)
  2428. printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
  2429. __func__, subpage, is_write ? 'w' : 'r', len, addr);
  2430. #endif
  2431. return flatview_access_valid(subpage->fv, addr + subpage->base,
  2432. len, is_write, attrs);
  2433. }
  2434. static const MemoryRegionOps subpage_ops = {
  2435. .read_with_attrs = subpage_read,
  2436. .write_with_attrs = subpage_write,
  2437. .impl.min_access_size = 1,
  2438. .impl.max_access_size = 8,
  2439. .valid.min_access_size = 1,
  2440. .valid.max_access_size = 8,
  2441. .valid.accepts = subpage_accepts,
  2442. .endianness = DEVICE_NATIVE_ENDIAN,
  2443. };
  2444. static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end,
  2445. uint16_t section)
  2446. {
  2447. int idx, eidx;
  2448. if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
  2449. return -1;
  2450. idx = SUBPAGE_IDX(start);
  2451. eidx = SUBPAGE_IDX(end);
  2452. #if defined(DEBUG_SUBPAGE)
  2453. printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
  2454. __func__, mmio, start, end, idx, eidx, section);
  2455. #endif
  2456. for (; idx <= eidx; idx++) {
  2457. mmio->sub_section[idx] = section;
  2458. }
  2459. return 0;
  2460. }
  2461. static subpage_t *subpage_init(FlatView *fv, hwaddr base)
  2462. {
  2463. subpage_t *mmio;
  2464. /* mmio->sub_section is set to PHYS_SECTION_UNASSIGNED with g_malloc0 */
  2465. mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t));
  2466. mmio->fv = fv;
  2467. mmio->base = base;
  2468. memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
  2469. NULL, TARGET_PAGE_SIZE);
  2470. mmio->iomem.subpage = true;
  2471. #if defined(DEBUG_SUBPAGE)
  2472. printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
  2473. mmio, base, TARGET_PAGE_SIZE);
  2474. #endif
  2475. return mmio;
  2476. }
  2477. static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr)
  2478. {
  2479. assert(fv);
  2480. MemoryRegionSection section = {
  2481. .fv = fv,
  2482. .mr = mr,
  2483. .offset_within_address_space = 0,
  2484. .offset_within_region = 0,
  2485. .size = int128_2_64(),
  2486. };
  2487. return phys_section_add(map, &section);
  2488. }
  2489. MemoryRegionSection *iotlb_to_section(CPUState *cpu,
  2490. hwaddr index, MemTxAttrs attrs)
  2491. {
  2492. int asidx = cpu_asidx_from_attrs(cpu, attrs);
  2493. CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
  2494. AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
  2495. MemoryRegionSection *sections = d->map.sections;
  2496. return &sections[index & ~TARGET_PAGE_MASK];
  2497. }
  2498. static void io_mem_init(void)
  2499. {
  2500. memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
  2501. NULL, UINT64_MAX);
  2502. }
  2503. AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv)
  2504. {
  2505. AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
  2506. uint16_t n;
  2507. n = dummy_section(&d->map, fv, &io_mem_unassigned);
  2508. assert(n == PHYS_SECTION_UNASSIGNED);
  2509. d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
  2510. return d;
  2511. }
  2512. void address_space_dispatch_free(AddressSpaceDispatch *d)
  2513. {
  2514. phys_sections_free(&d->map);
  2515. g_free(d);
  2516. }
  2517. static void do_nothing(CPUState *cpu, run_on_cpu_data d)
  2518. {
  2519. }
  2520. static void tcg_log_global_after_sync(MemoryListener *listener)
  2521. {
  2522. CPUAddressSpace *cpuas;
  2523. /* Wait for the CPU to end the current TB. This avoids the following
  2524. * incorrect race:
  2525. *
  2526. * vCPU migration
  2527. * ---------------------- -------------------------
  2528. * TLB check -> slow path
  2529. * notdirty_mem_write
  2530. * write to RAM
  2531. * mark dirty
  2532. * clear dirty flag
  2533. * TLB check -> fast path
  2534. * read memory
  2535. * write to RAM
  2536. *
  2537. * by pushing the migration thread's memory read after the vCPU thread has
  2538. * written the memory.
  2539. */
  2540. if (replay_mode == REPLAY_MODE_NONE) {
  2541. /*
  2542. * VGA can make calls to this function while updating the screen.
  2543. * In record/replay mode this causes a deadlock, because
  2544. * run_on_cpu waits for rr mutex. Therefore no races are possible
  2545. * in this case and no need for making run_on_cpu when
  2546. * record/replay is not enabled.
  2547. */
  2548. cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
  2549. run_on_cpu(cpuas->cpu, do_nothing, RUN_ON_CPU_NULL);
  2550. }
  2551. }
  2552. static void tcg_commit(MemoryListener *listener)
  2553. {
  2554. CPUAddressSpace *cpuas;
  2555. AddressSpaceDispatch *d;
  2556. assert(tcg_enabled());
  2557. /* since each CPU stores ram addresses in its TLB cache, we must
  2558. reset the modified entries */
  2559. cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
  2560. cpu_reloading_memory_map();
  2561. /* The CPU and TLB are protected by the iothread lock.
  2562. * We reload the dispatch pointer now because cpu_reloading_memory_map()
  2563. * may have split the RCU critical section.
  2564. */
  2565. d = address_space_to_dispatch(cpuas->as);
  2566. atomic_rcu_set(&cpuas->memory_dispatch, d);
  2567. tlb_flush(cpuas->cpu);
  2568. }
  2569. static void memory_map_init(void)
  2570. {
  2571. system_memory = g_malloc(sizeof(*system_memory));
  2572. memory_region_init(system_memory, NULL, "system", UINT64_MAX);
  2573. address_space_init(&address_space_memory, system_memory, "memory");
  2574. system_io = g_malloc(sizeof(*system_io));
  2575. memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
  2576. 65536);
  2577. address_space_init(&address_space_io, system_io, "I/O");
  2578. }
  2579. MemoryRegion *get_system_memory(void)
  2580. {
  2581. return system_memory;
  2582. }
  2583. MemoryRegion *get_system_io(void)
  2584. {
  2585. return system_io;
  2586. }
  2587. #endif /* !defined(CONFIG_USER_ONLY) */
  2588. /* physical memory access (slow version, mainly for debug) */
  2589. #if defined(CONFIG_USER_ONLY)
  2590. int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
  2591. uint8_t *buf, target_ulong len, int is_write)
  2592. {
  2593. int flags;
  2594. target_ulong l, page;
  2595. void * p;
  2596. while (len > 0) {
  2597. page = addr & TARGET_PAGE_MASK;
  2598. l = (page + TARGET_PAGE_SIZE) - addr;
  2599. if (l > len)
  2600. l = len;
  2601. flags = page_get_flags(page);
  2602. if (!(flags & PAGE_VALID))
  2603. return -1;
  2604. if (is_write) {
  2605. if (!(flags & PAGE_WRITE))
  2606. return -1;
  2607. /* XXX: this code should not depend on lock_user */
  2608. if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
  2609. return -1;
  2610. memcpy(p, buf, l);
  2611. unlock_user(p, addr, l);
  2612. } else {
  2613. if (!(flags & PAGE_READ))
  2614. return -1;
  2615. /* XXX: this code should not depend on lock_user */
  2616. if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
  2617. return -1;
  2618. memcpy(buf, p, l);
  2619. unlock_user(p, addr, 0);
  2620. }
  2621. len -= l;
  2622. buf += l;
  2623. addr += l;
  2624. }
  2625. return 0;
  2626. }
  2627. #else
  2628. static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
  2629. hwaddr length)
  2630. {
  2631. uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
  2632. addr += memory_region_get_ram_addr(mr);
  2633. /* No early return if dirty_log_mask is or becomes 0, because
  2634. * cpu_physical_memory_set_dirty_range will still call
  2635. * xen_modified_memory.
  2636. */
  2637. if (dirty_log_mask) {
  2638. dirty_log_mask =
  2639. cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
  2640. }
  2641. if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
  2642. assert(tcg_enabled());
  2643. tb_invalidate_phys_range(addr, addr + length);
  2644. dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
  2645. }
  2646. cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
  2647. }
  2648. void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size)
  2649. {
  2650. /*
  2651. * In principle this function would work on other memory region types too,
  2652. * but the ROM device use case is the only one where this operation is
  2653. * necessary. Other memory regions should use the
  2654. * address_space_read/write() APIs.
  2655. */
  2656. assert(memory_region_is_romd(mr));
  2657. invalidate_and_set_dirty(mr, addr, size);
  2658. }
  2659. static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
  2660. {
  2661. unsigned access_size_max = mr->ops->valid.max_access_size;
  2662. /* Regions are assumed to support 1-4 byte accesses unless
  2663. otherwise specified. */
  2664. if (access_size_max == 0) {
  2665. access_size_max = 4;
  2666. }
  2667. /* Bound the maximum access by the alignment of the address. */
  2668. if (!mr->ops->impl.unaligned) {
  2669. unsigned align_size_max = addr & -addr;
  2670. if (align_size_max != 0 && align_size_max < access_size_max) {
  2671. access_size_max = align_size_max;
  2672. }
  2673. }
  2674. /* Don't attempt accesses larger than the maximum. */
  2675. if (l > access_size_max) {
  2676. l = access_size_max;
  2677. }
  2678. l = pow2floor(l);
  2679. return l;
  2680. }
  2681. static bool prepare_mmio_access(MemoryRegion *mr)
  2682. {
  2683. bool unlocked = !qemu_mutex_iothread_locked();
  2684. bool release_lock = false;
  2685. if (unlocked && mr->global_locking) {
  2686. qemu_mutex_lock_iothread();
  2687. unlocked = false;
  2688. release_lock = true;
  2689. }
  2690. if (mr->flush_coalesced_mmio) {
  2691. if (unlocked) {
  2692. qemu_mutex_lock_iothread();
  2693. }
  2694. qemu_flush_coalesced_mmio_buffer();
  2695. if (unlocked) {
  2696. qemu_mutex_unlock_iothread();
  2697. }
  2698. }
  2699. return release_lock;
  2700. }
  2701. /* Called within RCU critical section. */
  2702. static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr,
  2703. MemTxAttrs attrs,
  2704. const uint8_t *buf,
  2705. hwaddr len, hwaddr addr1,
  2706. hwaddr l, MemoryRegion *mr)
  2707. {
  2708. uint8_t *ptr;
  2709. uint64_t val;
  2710. MemTxResult result = MEMTX_OK;
  2711. bool release_lock = false;
  2712. for (;;) {
  2713. if (!memory_access_is_direct(mr, true)) {
  2714. release_lock |= prepare_mmio_access(mr);
  2715. l = memory_access_size(mr, l, addr1);
  2716. /* XXX: could force current_cpu to NULL to avoid
  2717. potential bugs */
  2718. val = ldn_he_p(buf, l);
  2719. result |= memory_region_dispatch_write(mr, addr1, val,
  2720. size_memop(l), attrs);
  2721. } else {
  2722. /* RAM case */
  2723. ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false);
  2724. memcpy(ptr, buf, l);
  2725. invalidate_and_set_dirty(mr, addr1, l);
  2726. }
  2727. if (release_lock) {
  2728. qemu_mutex_unlock_iothread();
  2729. release_lock = false;
  2730. }
  2731. len -= l;
  2732. buf += l;
  2733. addr += l;
  2734. if (!len) {
  2735. break;
  2736. }
  2737. l = len;
  2738. mr = flatview_translate(fv, addr, &addr1, &l, true, attrs);
  2739. }
  2740. return result;
  2741. }
  2742. /* Called from RCU critical section. */
  2743. static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
  2744. const uint8_t *buf, hwaddr len)
  2745. {
  2746. hwaddr l;
  2747. hwaddr addr1;
  2748. MemoryRegion *mr;
  2749. MemTxResult result = MEMTX_OK;
  2750. l = len;
  2751. mr = flatview_translate(fv, addr, &addr1, &l, true, attrs);
  2752. result = flatview_write_continue(fv, addr, attrs, buf, len,
  2753. addr1, l, mr);
  2754. return result;
  2755. }
  2756. /* Called within RCU critical section. */
  2757. MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr,
  2758. MemTxAttrs attrs, uint8_t *buf,
  2759. hwaddr len, hwaddr addr1, hwaddr l,
  2760. MemoryRegion *mr)
  2761. {
  2762. uint8_t *ptr;
  2763. uint64_t val;
  2764. MemTxResult result = MEMTX_OK;
  2765. bool release_lock = false;
  2766. for (;;) {
  2767. if (!memory_access_is_direct(mr, false)) {
  2768. /* I/O case */
  2769. release_lock |= prepare_mmio_access(mr);
  2770. l = memory_access_size(mr, l, addr1);
  2771. result |= memory_region_dispatch_read(mr, addr1, &val,
  2772. size_memop(l), attrs);
  2773. stn_he_p(buf, l, val);
  2774. } else {
  2775. /* RAM case */
  2776. ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false);
  2777. memcpy(buf, ptr, l);
  2778. }
  2779. if (release_lock) {
  2780. qemu_mutex_unlock_iothread();
  2781. release_lock = false;
  2782. }
  2783. len -= l;
  2784. buf += l;
  2785. addr += l;
  2786. if (!len) {
  2787. break;
  2788. }
  2789. l = len;
  2790. mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
  2791. }
  2792. return result;
  2793. }
  2794. /* Called from RCU critical section. */
  2795. static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
  2796. MemTxAttrs attrs, uint8_t *buf, hwaddr len)
  2797. {
  2798. hwaddr l;
  2799. hwaddr addr1;
  2800. MemoryRegion *mr;
  2801. l = len;
  2802. mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
  2803. return flatview_read_continue(fv, addr, attrs, buf, len,
  2804. addr1, l, mr);
  2805. }
  2806. MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
  2807. MemTxAttrs attrs, uint8_t *buf, hwaddr len)
  2808. {
  2809. MemTxResult result = MEMTX_OK;
  2810. FlatView *fv;
  2811. if (len > 0) {
  2812. RCU_READ_LOCK_GUARD();
  2813. fv = address_space_to_flatview(as);
  2814. result = flatview_read(fv, addr, attrs, buf, len);
  2815. }
  2816. return result;
  2817. }
  2818. MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
  2819. MemTxAttrs attrs,
  2820. const uint8_t *buf, hwaddr len)
  2821. {
  2822. MemTxResult result = MEMTX_OK;
  2823. FlatView *fv;
  2824. if (len > 0) {
  2825. RCU_READ_LOCK_GUARD();
  2826. fv = address_space_to_flatview(as);
  2827. result = flatview_write(fv, addr, attrs, buf, len);
  2828. }
  2829. return result;
  2830. }
  2831. MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
  2832. uint8_t *buf, hwaddr len, bool is_write)
  2833. {
  2834. if (is_write) {
  2835. return address_space_write(as, addr, attrs, buf, len);
  2836. } else {
  2837. return address_space_read_full(as, addr, attrs, buf, len);
  2838. }
  2839. }
  2840. void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
  2841. hwaddr len, int is_write)
  2842. {
  2843. address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
  2844. buf, len, is_write);
  2845. }
  2846. enum write_rom_type {
  2847. WRITE_DATA,
  2848. FLUSH_CACHE,
  2849. };
  2850. static inline MemTxResult address_space_write_rom_internal(AddressSpace *as,
  2851. hwaddr addr,
  2852. MemTxAttrs attrs,
  2853. const uint8_t *buf,
  2854. hwaddr len,
  2855. enum write_rom_type type)
  2856. {
  2857. hwaddr l;
  2858. uint8_t *ptr;
  2859. hwaddr addr1;
  2860. MemoryRegion *mr;
  2861. RCU_READ_LOCK_GUARD();
  2862. while (len > 0) {
  2863. l = len;
  2864. mr = address_space_translate(as, addr, &addr1, &l, true, attrs);
  2865. if (!(memory_region_is_ram(mr) ||
  2866. memory_region_is_romd(mr))) {
  2867. l = memory_access_size(mr, l, addr1);
  2868. } else {
  2869. /* ROM/RAM case */
  2870. ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
  2871. switch (type) {
  2872. case WRITE_DATA:
  2873. memcpy(ptr, buf, l);
  2874. invalidate_and_set_dirty(mr, addr1, l);
  2875. break;
  2876. case FLUSH_CACHE:
  2877. flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
  2878. break;
  2879. }
  2880. }
  2881. len -= l;
  2882. buf += l;
  2883. addr += l;
  2884. }
  2885. return MEMTX_OK;
  2886. }
  2887. /* used for ROM loading : can write in RAM and ROM */
  2888. MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr,
  2889. MemTxAttrs attrs,
  2890. const uint8_t *buf, hwaddr len)
  2891. {
  2892. return address_space_write_rom_internal(as, addr, attrs,
  2893. buf, len, WRITE_DATA);
  2894. }
  2895. void cpu_flush_icache_range(hwaddr start, hwaddr len)
  2896. {
  2897. /*
  2898. * This function should do the same thing as an icache flush that was
  2899. * triggered from within the guest. For TCG we are always cache coherent,
  2900. * so there is no need to flush anything. For KVM / Xen we need to flush
  2901. * the host's instruction cache at least.
  2902. */
  2903. if (tcg_enabled()) {
  2904. return;
  2905. }
  2906. address_space_write_rom_internal(&address_space_memory,
  2907. start, MEMTXATTRS_UNSPECIFIED,
  2908. NULL, len, FLUSH_CACHE);
  2909. }
  2910. typedef struct {
  2911. MemoryRegion *mr;
  2912. void *buffer;
  2913. hwaddr addr;
  2914. hwaddr len;
  2915. bool in_use;
  2916. } BounceBuffer;
  2917. static BounceBuffer bounce;
  2918. typedef struct MapClient {
  2919. QEMUBH *bh;
  2920. QLIST_ENTRY(MapClient) link;
  2921. } MapClient;
  2922. QemuMutex map_client_list_lock;
  2923. static QLIST_HEAD(, MapClient) map_client_list
  2924. = QLIST_HEAD_INITIALIZER(map_client_list);
  2925. static void cpu_unregister_map_client_do(MapClient *client)
  2926. {
  2927. QLIST_REMOVE(client, link);
  2928. g_free(client);
  2929. }
  2930. static void cpu_notify_map_clients_locked(void)
  2931. {
  2932. MapClient *client;
  2933. while (!QLIST_EMPTY(&map_client_list)) {
  2934. client = QLIST_FIRST(&map_client_list);
  2935. qemu_bh_schedule(client->bh);
  2936. cpu_unregister_map_client_do(client);
  2937. }
  2938. }
  2939. void cpu_register_map_client(QEMUBH *bh)
  2940. {
  2941. MapClient *client = g_malloc(sizeof(*client));
  2942. qemu_mutex_lock(&map_client_list_lock);
  2943. client->bh = bh;
  2944. QLIST_INSERT_HEAD(&map_client_list, client, link);
  2945. if (!atomic_read(&bounce.in_use)) {
  2946. cpu_notify_map_clients_locked();
  2947. }
  2948. qemu_mutex_unlock(&map_client_list_lock);
  2949. }
  2950. void cpu_exec_init_all(void)
  2951. {
  2952. qemu_mutex_init(&ram_list.mutex);
  2953. /* The data structures we set up here depend on knowing the page size,
  2954. * so no more changes can be made after this point.
  2955. * In an ideal world, nothing we did before we had finished the
  2956. * machine setup would care about the target page size, and we could
  2957. * do this much later, rather than requiring board models to state
  2958. * up front what their requirements are.
  2959. */
  2960. finalize_target_page_bits();
  2961. io_mem_init();
  2962. memory_map_init();
  2963. qemu_mutex_init(&map_client_list_lock);
  2964. }
  2965. void cpu_unregister_map_client(QEMUBH *bh)
  2966. {
  2967. MapClient *client;
  2968. qemu_mutex_lock(&map_client_list_lock);
  2969. QLIST_FOREACH(client, &map_client_list, link) {
  2970. if (client->bh == bh) {
  2971. cpu_unregister_map_client_do(client);
  2972. break;
  2973. }
  2974. }
  2975. qemu_mutex_unlock(&map_client_list_lock);
  2976. }
  2977. static void cpu_notify_map_clients(void)
  2978. {
  2979. qemu_mutex_lock(&map_client_list_lock);
  2980. cpu_notify_map_clients_locked();
  2981. qemu_mutex_unlock(&map_client_list_lock);
  2982. }
  2983. static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
  2984. bool is_write, MemTxAttrs attrs)
  2985. {
  2986. MemoryRegion *mr;
  2987. hwaddr l, xlat;
  2988. while (len > 0) {
  2989. l = len;
  2990. mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
  2991. if (!memory_access_is_direct(mr, is_write)) {
  2992. l = memory_access_size(mr, l, addr);
  2993. if (!memory_region_access_valid(mr, xlat, l, is_write, attrs)) {
  2994. return false;
  2995. }
  2996. }
  2997. len -= l;
  2998. addr += l;
  2999. }
  3000. return true;
  3001. }
  3002. bool address_space_access_valid(AddressSpace *as, hwaddr addr,
  3003. hwaddr len, bool is_write,
  3004. MemTxAttrs attrs)
  3005. {
  3006. FlatView *fv;
  3007. bool result;
  3008. RCU_READ_LOCK_GUARD();
  3009. fv = address_space_to_flatview(as);
  3010. result = flatview_access_valid(fv, addr, len, is_write, attrs);
  3011. return result;
  3012. }
  3013. static hwaddr
  3014. flatview_extend_translation(FlatView *fv, hwaddr addr,
  3015. hwaddr target_len,
  3016. MemoryRegion *mr, hwaddr base, hwaddr len,
  3017. bool is_write, MemTxAttrs attrs)
  3018. {
  3019. hwaddr done = 0;
  3020. hwaddr xlat;
  3021. MemoryRegion *this_mr;
  3022. for (;;) {
  3023. target_len -= len;
  3024. addr += len;
  3025. done += len;
  3026. if (target_len == 0) {
  3027. return done;
  3028. }
  3029. len = target_len;
  3030. this_mr = flatview_translate(fv, addr, &xlat,
  3031. &len, is_write, attrs);
  3032. if (this_mr != mr || xlat != base + done) {
  3033. return done;
  3034. }
  3035. }
  3036. }
  3037. /* Map a physical memory region into a host virtual address.
  3038. * May map a subset of the requested range, given by and returned in *plen.
  3039. * May return NULL if resources needed to perform the mapping are exhausted.
  3040. * Use only for reads OR writes - not for read-modify-write operations.
  3041. * Use cpu_register_map_client() to know when retrying the map operation is
  3042. * likely to succeed.
  3043. */
  3044. void *address_space_map(AddressSpace *as,
  3045. hwaddr addr,
  3046. hwaddr *plen,
  3047. bool is_write,
  3048. MemTxAttrs attrs)
  3049. {
  3050. hwaddr len = *plen;
  3051. hwaddr l, xlat;
  3052. MemoryRegion *mr;
  3053. void *ptr;
  3054. FlatView *fv;
  3055. if (len == 0) {
  3056. return NULL;
  3057. }
  3058. l = len;
  3059. RCU_READ_LOCK_GUARD();
  3060. fv = address_space_to_flatview(as);
  3061. mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
  3062. if (!memory_access_is_direct(mr, is_write)) {
  3063. if (atomic_xchg(&bounce.in_use, true)) {
  3064. return NULL;
  3065. }
  3066. /* Avoid unbounded allocations */
  3067. l = MIN(l, TARGET_PAGE_SIZE);
  3068. bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
  3069. bounce.addr = addr;
  3070. bounce.len = l;
  3071. memory_region_ref(mr);
  3072. bounce.mr = mr;
  3073. if (!is_write) {
  3074. flatview_read(fv, addr, MEMTXATTRS_UNSPECIFIED,
  3075. bounce.buffer, l);
  3076. }
  3077. *plen = l;
  3078. return bounce.buffer;
  3079. }
  3080. memory_region_ref(mr);
  3081. *plen = flatview_extend_translation(fv, addr, len, mr, xlat,
  3082. l, is_write, attrs);
  3083. ptr = qemu_ram_ptr_length(mr->ram_block, xlat, plen, true);
  3084. return ptr;
  3085. }
  3086. /* Unmaps a memory region previously mapped by address_space_map().
  3087. * Will also mark the memory as dirty if is_write == 1. access_len gives
  3088. * the amount of memory that was actually read or written by the caller.
  3089. */
  3090. void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
  3091. int is_write, hwaddr access_len)
  3092. {
  3093. if (buffer != bounce.buffer) {
  3094. MemoryRegion *mr;
  3095. ram_addr_t addr1;
  3096. mr = memory_region_from_host(buffer, &addr1);
  3097. assert(mr != NULL);
  3098. if (is_write) {
  3099. invalidate_and_set_dirty(mr, addr1, access_len);
  3100. }
  3101. if (xen_enabled()) {
  3102. xen_invalidate_map_cache_entry(buffer);
  3103. }
  3104. memory_region_unref(mr);
  3105. return;
  3106. }
  3107. if (is_write) {
  3108. address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
  3109. bounce.buffer, access_len);
  3110. }
  3111. qemu_vfree(bounce.buffer);
  3112. bounce.buffer = NULL;
  3113. memory_region_unref(bounce.mr);
  3114. atomic_mb_set(&bounce.in_use, false);
  3115. cpu_notify_map_clients();
  3116. }
  3117. void *cpu_physical_memory_map(hwaddr addr,
  3118. hwaddr *plen,
  3119. int is_write)
  3120. {
  3121. return address_space_map(&address_space_memory, addr, plen, is_write,
  3122. MEMTXATTRS_UNSPECIFIED);
  3123. }
  3124. void cpu_physical_memory_unmap(void *buffer, hwaddr len,
  3125. int is_write, hwaddr access_len)
  3126. {
  3127. return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
  3128. }
  3129. #define ARG1_DECL AddressSpace *as
  3130. #define ARG1 as
  3131. #define SUFFIX
  3132. #define TRANSLATE(...) address_space_translate(as, __VA_ARGS__)
  3133. #define RCU_READ_LOCK(...) rcu_read_lock()
  3134. #define RCU_READ_UNLOCK(...) rcu_read_unlock()
  3135. #include "memory_ldst.inc.c"
  3136. int64_t address_space_cache_init(MemoryRegionCache *cache,
  3137. AddressSpace *as,
  3138. hwaddr addr,
  3139. hwaddr len,
  3140. bool is_write)
  3141. {
  3142. AddressSpaceDispatch *d;
  3143. hwaddr l;
  3144. MemoryRegion *mr;
  3145. assert(len > 0);
  3146. l = len;
  3147. cache->fv = address_space_get_flatview(as);
  3148. d = flatview_to_dispatch(cache->fv);
  3149. cache->mrs = *address_space_translate_internal(d, addr, &cache->xlat, &l, true);
  3150. mr = cache->mrs.mr;
  3151. memory_region_ref(mr);
  3152. if (memory_access_is_direct(mr, is_write)) {
  3153. /* We don't care about the memory attributes here as we're only
  3154. * doing this if we found actual RAM, which behaves the same
  3155. * regardless of attributes; so UNSPECIFIED is fine.
  3156. */
  3157. l = flatview_extend_translation(cache->fv, addr, len, mr,
  3158. cache->xlat, l, is_write,
  3159. MEMTXATTRS_UNSPECIFIED);
  3160. cache->ptr = qemu_ram_ptr_length(mr->ram_block, cache->xlat, &l, true);
  3161. } else {
  3162. cache->ptr = NULL;
  3163. }
  3164. cache->len = l;
  3165. cache->is_write = is_write;
  3166. return l;
  3167. }
  3168. void address_space_cache_invalidate(MemoryRegionCache *cache,
  3169. hwaddr addr,
  3170. hwaddr access_len)
  3171. {
  3172. assert(cache->is_write);
  3173. if (likely(cache->ptr)) {
  3174. invalidate_and_set_dirty(cache->mrs.mr, addr + cache->xlat, access_len);
  3175. }
  3176. }
  3177. void address_space_cache_destroy(MemoryRegionCache *cache)
  3178. {
  3179. if (!cache->mrs.mr) {
  3180. return;
  3181. }
  3182. if (xen_enabled()) {
  3183. xen_invalidate_map_cache_entry(cache->ptr);
  3184. }
  3185. memory_region_unref(cache->mrs.mr);
  3186. flatview_unref(cache->fv);
  3187. cache->mrs.mr = NULL;
  3188. cache->fv = NULL;
  3189. }
  3190. /* Called from RCU critical section. This function has the same
  3191. * semantics as address_space_translate, but it only works on a
  3192. * predefined range of a MemoryRegion that was mapped with
  3193. * address_space_cache_init.
  3194. */
  3195. static inline MemoryRegion *address_space_translate_cached(
  3196. MemoryRegionCache *cache, hwaddr addr, hwaddr *xlat,
  3197. hwaddr *plen, bool is_write, MemTxAttrs attrs)
  3198. {
  3199. MemoryRegionSection section;
  3200. MemoryRegion *mr;
  3201. IOMMUMemoryRegion *iommu_mr;
  3202. AddressSpace *target_as;
  3203. assert(!cache->ptr);
  3204. *xlat = addr + cache->xlat;
  3205. mr = cache->mrs.mr;
  3206. iommu_mr = memory_region_get_iommu(mr);
  3207. if (!iommu_mr) {
  3208. /* MMIO region. */
  3209. return mr;
  3210. }
  3211. section = address_space_translate_iommu(iommu_mr, xlat, plen,
  3212. NULL, is_write, true,
  3213. &target_as, attrs);
  3214. return section.mr;
  3215. }
  3216. /* Called from RCU critical section. address_space_read_cached uses this
  3217. * out of line function when the target is an MMIO or IOMMU region.
  3218. */
  3219. void
  3220. address_space_read_cached_slow(MemoryRegionCache *cache, hwaddr addr,
  3221. void *buf, hwaddr len)
  3222. {
  3223. hwaddr addr1, l;
  3224. MemoryRegion *mr;
  3225. l = len;
  3226. mr = address_space_translate_cached(cache, addr, &addr1, &l, false,
  3227. MEMTXATTRS_UNSPECIFIED);
  3228. flatview_read_continue(cache->fv,
  3229. addr, MEMTXATTRS_UNSPECIFIED, buf, len,
  3230. addr1, l, mr);
  3231. }
  3232. /* Called from RCU critical section. address_space_write_cached uses this
  3233. * out of line function when the target is an MMIO or IOMMU region.
  3234. */
  3235. void
  3236. address_space_write_cached_slow(MemoryRegionCache *cache, hwaddr addr,
  3237. const void *buf, hwaddr len)
  3238. {
  3239. hwaddr addr1, l;
  3240. MemoryRegion *mr;
  3241. l = len;
  3242. mr = address_space_translate_cached(cache, addr, &addr1, &l, true,
  3243. MEMTXATTRS_UNSPECIFIED);
  3244. flatview_write_continue(cache->fv,
  3245. addr, MEMTXATTRS_UNSPECIFIED, buf, len,
  3246. addr1, l, mr);
  3247. }
  3248. #define ARG1_DECL MemoryRegionCache *cache
  3249. #define ARG1 cache
  3250. #define SUFFIX _cached_slow
  3251. #define TRANSLATE(...) address_space_translate_cached(cache, __VA_ARGS__)
  3252. #define RCU_READ_LOCK() ((void)0)
  3253. #define RCU_READ_UNLOCK() ((void)0)
  3254. #include "memory_ldst.inc.c"
  3255. /* virtual memory access for debug (includes writing to ROM) */
  3256. int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
  3257. uint8_t *buf, target_ulong len, int is_write)
  3258. {
  3259. hwaddr phys_addr;
  3260. target_ulong l, page;
  3261. cpu_synchronize_state(cpu);
  3262. while (len > 0) {
  3263. int asidx;
  3264. MemTxAttrs attrs;
  3265. page = addr & TARGET_PAGE_MASK;
  3266. phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
  3267. asidx = cpu_asidx_from_attrs(cpu, attrs);
  3268. /* if no physical page mapped, return an error */
  3269. if (phys_addr == -1)
  3270. return -1;
  3271. l = (page + TARGET_PAGE_SIZE) - addr;
  3272. if (l > len)
  3273. l = len;
  3274. phys_addr += (addr & ~TARGET_PAGE_MASK);
  3275. if (is_write) {
  3276. address_space_write_rom(cpu->cpu_ases[asidx].as, phys_addr,
  3277. attrs, buf, l);
  3278. } else {
  3279. address_space_rw(cpu->cpu_ases[asidx].as, phys_addr,
  3280. attrs, buf, l, 0);
  3281. }
  3282. len -= l;
  3283. buf += l;
  3284. addr += l;
  3285. }
  3286. return 0;
  3287. }
  3288. /*
  3289. * Allows code that needs to deal with migration bitmaps etc to still be built
  3290. * target independent.
  3291. */
  3292. size_t qemu_target_page_size(void)
  3293. {
  3294. return TARGET_PAGE_SIZE;
  3295. }
  3296. int qemu_target_page_bits(void)
  3297. {
  3298. return TARGET_PAGE_BITS;
  3299. }
  3300. int qemu_target_page_bits_min(void)
  3301. {
  3302. return TARGET_PAGE_BITS_MIN;
  3303. }
  3304. #endif
  3305. bool target_words_bigendian(void)
  3306. {
  3307. #if defined(TARGET_WORDS_BIGENDIAN)
  3308. return true;
  3309. #else
  3310. return false;
  3311. #endif
  3312. }
  3313. #ifndef CONFIG_USER_ONLY
  3314. bool cpu_physical_memory_is_io(hwaddr phys_addr)
  3315. {
  3316. MemoryRegion*mr;
  3317. hwaddr l = 1;
  3318. bool res;
  3319. RCU_READ_LOCK_GUARD();
  3320. mr = address_space_translate(&address_space_memory,
  3321. phys_addr, &phys_addr, &l, false,
  3322. MEMTXATTRS_UNSPECIFIED);
  3323. res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
  3324. return res;
  3325. }
  3326. int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
  3327. {
  3328. RAMBlock *block;
  3329. int ret = 0;
  3330. RCU_READ_LOCK_GUARD();
  3331. RAMBLOCK_FOREACH(block) {
  3332. ret = func(block, opaque);
  3333. if (ret) {
  3334. break;
  3335. }
  3336. }
  3337. return ret;
  3338. }
  3339. /*
  3340. * Unmap pages of memory from start to start+length such that
  3341. * they a) read as 0, b) Trigger whatever fault mechanism
  3342. * the OS provides for postcopy.
  3343. * The pages must be unmapped by the end of the function.
  3344. * Returns: 0 on success, none-0 on failure
  3345. *
  3346. */
  3347. int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length)
  3348. {
  3349. int ret = -1;
  3350. uint8_t *host_startaddr = rb->host + start;
  3351. if (!QEMU_PTR_IS_ALIGNED(host_startaddr, rb->page_size)) {
  3352. error_report("ram_block_discard_range: Unaligned start address: %p",
  3353. host_startaddr);
  3354. goto err;
  3355. }
  3356. if ((start + length) <= rb->used_length) {
  3357. bool need_madvise, need_fallocate;
  3358. if (!QEMU_IS_ALIGNED(length, rb->page_size)) {
  3359. error_report("ram_block_discard_range: Unaligned length: %zx",
  3360. length);
  3361. goto err;
  3362. }
  3363. errno = ENOTSUP; /* If we are missing MADVISE etc */
  3364. /* The logic here is messy;
  3365. * madvise DONTNEED fails for hugepages
  3366. * fallocate works on hugepages and shmem
  3367. */
  3368. need_madvise = (rb->page_size == qemu_host_page_size);
  3369. need_fallocate = rb->fd != -1;
  3370. if (need_fallocate) {
  3371. /* For a file, this causes the area of the file to be zero'd
  3372. * if read, and for hugetlbfs also causes it to be unmapped
  3373. * so a userfault will trigger.
  3374. */
  3375. #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
  3376. ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
  3377. start, length);
  3378. if (ret) {
  3379. ret = -errno;
  3380. error_report("ram_block_discard_range: Failed to fallocate "
  3381. "%s:%" PRIx64 " +%zx (%d)",
  3382. rb->idstr, start, length, ret);
  3383. goto err;
  3384. }
  3385. #else
  3386. ret = -ENOSYS;
  3387. error_report("ram_block_discard_range: fallocate not available/file"
  3388. "%s:%" PRIx64 " +%zx (%d)",
  3389. rb->idstr, start, length, ret);
  3390. goto err;
  3391. #endif
  3392. }
  3393. if (need_madvise) {
  3394. /* For normal RAM this causes it to be unmapped,
  3395. * for shared memory it causes the local mapping to disappear
  3396. * and to fall back on the file contents (which we just
  3397. * fallocate'd away).
  3398. */
  3399. #if defined(CONFIG_MADVISE)
  3400. ret = madvise(host_startaddr, length, MADV_DONTNEED);
  3401. if (ret) {
  3402. ret = -errno;
  3403. error_report("ram_block_discard_range: Failed to discard range "
  3404. "%s:%" PRIx64 " +%zx (%d)",
  3405. rb->idstr, start, length, ret);
  3406. goto err;
  3407. }
  3408. #else
  3409. ret = -ENOSYS;
  3410. error_report("ram_block_discard_range: MADVISE not available"
  3411. "%s:%" PRIx64 " +%zx (%d)",
  3412. rb->idstr, start, length, ret);
  3413. goto err;
  3414. #endif
  3415. }
  3416. trace_ram_block_discard_range(rb->idstr, host_startaddr, length,
  3417. need_madvise, need_fallocate, ret);
  3418. } else {
  3419. error_report("ram_block_discard_range: Overrun block '%s' (%" PRIu64
  3420. "/%zx/" RAM_ADDR_FMT")",
  3421. rb->idstr, start, length, rb->used_length);
  3422. }
  3423. err:
  3424. return ret;
  3425. }
  3426. bool ramblock_is_pmem(RAMBlock *rb)
  3427. {
  3428. return rb->flags & RAM_PMEM;
  3429. }
  3430. #endif
  3431. void page_size_init(void)
  3432. {
  3433. /* NOTE: we can always suppose that qemu_host_page_size >=
  3434. TARGET_PAGE_SIZE */
  3435. if (qemu_host_page_size == 0) {
  3436. qemu_host_page_size = qemu_real_host_page_size;
  3437. }
  3438. if (qemu_host_page_size < TARGET_PAGE_SIZE) {
  3439. qemu_host_page_size = TARGET_PAGE_SIZE;
  3440. }
  3441. qemu_host_page_mask = -(intptr_t)qemu_host_page_size;
  3442. }
  3443. #if !defined(CONFIG_USER_ONLY)
  3444. static void mtree_print_phys_entries(int start, int end, int skip, int ptr)
  3445. {
  3446. if (start == end - 1) {
  3447. qemu_printf("\t%3d ", start);
  3448. } else {
  3449. qemu_printf("\t%3d..%-3d ", start, end - 1);
  3450. }
  3451. qemu_printf(" skip=%d ", skip);
  3452. if (ptr == PHYS_MAP_NODE_NIL) {
  3453. qemu_printf(" ptr=NIL");
  3454. } else if (!skip) {
  3455. qemu_printf(" ptr=#%d", ptr);
  3456. } else {
  3457. qemu_printf(" ptr=[%d]", ptr);
  3458. }
  3459. qemu_printf("\n");
  3460. }
  3461. #define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \
  3462. int128_sub((size), int128_one())) : 0)
  3463. void mtree_print_dispatch(AddressSpaceDispatch *d, MemoryRegion *root)
  3464. {
  3465. int i;
  3466. qemu_printf(" Dispatch\n");
  3467. qemu_printf(" Physical sections\n");
  3468. for (i = 0; i < d->map.sections_nb; ++i) {
  3469. MemoryRegionSection *s = d->map.sections + i;
  3470. const char *names[] = { " [unassigned]", " [not dirty]",
  3471. " [ROM]", " [watch]" };
  3472. qemu_printf(" #%d @" TARGET_FMT_plx ".." TARGET_FMT_plx
  3473. " %s%s%s%s%s",
  3474. i,
  3475. s->offset_within_address_space,
  3476. s->offset_within_address_space + MR_SIZE(s->mr->size),
  3477. s->mr->name ? s->mr->name : "(noname)",
  3478. i < ARRAY_SIZE(names) ? names[i] : "",
  3479. s->mr == root ? " [ROOT]" : "",
  3480. s == d->mru_section ? " [MRU]" : "",
  3481. s->mr->is_iommu ? " [iommu]" : "");
  3482. if (s->mr->alias) {
  3483. qemu_printf(" alias=%s", s->mr->alias->name ?
  3484. s->mr->alias->name : "noname");
  3485. }
  3486. qemu_printf("\n");
  3487. }
  3488. qemu_printf(" Nodes (%d bits per level, %d levels) ptr=[%d] skip=%d\n",
  3489. P_L2_BITS, P_L2_LEVELS, d->phys_map.ptr, d->phys_map.skip);
  3490. for (i = 0; i < d->map.nodes_nb; ++i) {
  3491. int j, jprev;
  3492. PhysPageEntry prev;
  3493. Node *n = d->map.nodes + i;
  3494. qemu_printf(" [%d]\n", i);
  3495. for (j = 0, jprev = 0, prev = *n[0]; j < ARRAY_SIZE(*n); ++j) {
  3496. PhysPageEntry *pe = *n + j;
  3497. if (pe->ptr == prev.ptr && pe->skip == prev.skip) {
  3498. continue;
  3499. }
  3500. mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
  3501. jprev = j;
  3502. prev = *pe;
  3503. }
  3504. if (jprev != ARRAY_SIZE(*n)) {
  3505. mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
  3506. }
  3507. }
  3508. }
  3509. #endif