You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

4241 lines
124KB

  1. /*
  2. * Virtual page mapping
  3. *
  4. * Copyright (c) 2003 Fabrice Bellard
  5. *
  6. * This library is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU Lesser General Public
  8. * License as published by the Free Software Foundation; either
  9. * version 2 of the License, or (at your option) any later version.
  10. *
  11. * This library is distributed in the hope that it will be useful,
  12. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. * Lesser General Public License for more details.
  15. *
  16. * You should have received a copy of the GNU Lesser General Public
  17. * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18. */
  19. #include "qemu/osdep.h"
  20. #include "qapi/error.h"
  21. #include "qemu/cutils.h"
  22. #include "cpu.h"
  23. #include "exec/exec-all.h"
  24. #include "exec/target_page.h"
  25. #include "tcg.h"
  26. #include "hw/qdev-core.h"
  27. #include "hw/qdev-properties.h"
  28. #if !defined(CONFIG_USER_ONLY)
  29. #include "hw/boards.h"
  30. #include "hw/xen/xen.h"
  31. #endif
  32. #include "sysemu/kvm.h"
  33. #include "sysemu/sysemu.h"
  34. #include "qemu/timer.h"
  35. #include "qemu/config-file.h"
  36. #include "qemu/error-report.h"
  37. #include "qemu/qemu-print.h"
  38. #if defined(CONFIG_USER_ONLY)
  39. #include "qemu.h"
  40. #else /* !CONFIG_USER_ONLY */
  41. #include "hw/hw.h"
  42. #include "exec/memory.h"
  43. #include "exec/ioport.h"
  44. #include "sysemu/dma.h"
  45. #include "sysemu/numa.h"
  46. #include "sysemu/hw_accel.h"
  47. #include "exec/address-spaces.h"
  48. #include "sysemu/xen-mapcache.h"
  49. #include "trace-root.h"
  50. #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
  51. #include <linux/falloc.h>
  52. #endif
  53. #endif
  54. #include "qemu/rcu_queue.h"
  55. #include "qemu/main-loop.h"
  56. #include "translate-all.h"
  57. #include "sysemu/replay.h"
  58. #include "exec/memory-internal.h"
  59. #include "exec/ram_addr.h"
  60. #include "exec/log.h"
  61. #include "migration/vmstate.h"
  62. #include "qemu/range.h"
  63. #ifndef _WIN32
  64. #include "qemu/mmap-alloc.h"
  65. #endif
  66. #include "monitor/monitor.h"
  67. //#define DEBUG_SUBPAGE
  68. #if !defined(CONFIG_USER_ONLY)
  69. /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
  70. * are protected by the ramlist lock.
  71. */
  72. RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
  73. static MemoryRegion *system_memory;
  74. static MemoryRegion *system_io;
  75. AddressSpace address_space_io;
  76. AddressSpace address_space_memory;
  77. MemoryRegion io_mem_rom, io_mem_notdirty;
  78. static MemoryRegion io_mem_unassigned;
  79. #endif
  80. #ifdef TARGET_PAGE_BITS_VARY
  81. int target_page_bits;
  82. bool target_page_bits_decided;
  83. #endif
  84. CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
  85. /* current CPU in the current thread. It is only valid inside
  86. cpu_exec() */
  87. __thread CPUState *current_cpu;
  88. /* 0 = Do not count executed instructions.
  89. 1 = Precise instruction counting.
  90. 2 = Adaptive rate instruction counting. */
  91. int use_icount;
  92. uintptr_t qemu_host_page_size;
  93. intptr_t qemu_host_page_mask;
  94. bool set_preferred_target_page_bits(int bits)
  95. {
  96. /* The target page size is the lowest common denominator for all
  97. * the CPUs in the system, so we can only make it smaller, never
  98. * larger. And we can't make it smaller once we've committed to
  99. * a particular size.
  100. */
  101. #ifdef TARGET_PAGE_BITS_VARY
  102. assert(bits >= TARGET_PAGE_BITS_MIN);
  103. if (target_page_bits == 0 || target_page_bits > bits) {
  104. if (target_page_bits_decided) {
  105. return false;
  106. }
  107. target_page_bits = bits;
  108. }
  109. #endif
  110. return true;
  111. }
  112. #if !defined(CONFIG_USER_ONLY)
  113. static void finalize_target_page_bits(void)
  114. {
  115. #ifdef TARGET_PAGE_BITS_VARY
  116. if (target_page_bits == 0) {
  117. target_page_bits = TARGET_PAGE_BITS_MIN;
  118. }
  119. target_page_bits_decided = true;
  120. #endif
  121. }
  122. typedef struct PhysPageEntry PhysPageEntry;
  123. struct PhysPageEntry {
  124. /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
  125. uint32_t skip : 6;
  126. /* index into phys_sections (!skip) or phys_map_nodes (skip) */
  127. uint32_t ptr : 26;
  128. };
  129. #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
  130. /* Size of the L2 (and L3, etc) page tables. */
  131. #define ADDR_SPACE_BITS 64
  132. #define P_L2_BITS 9
  133. #define P_L2_SIZE (1 << P_L2_BITS)
  134. #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
  135. typedef PhysPageEntry Node[P_L2_SIZE];
  136. typedef struct PhysPageMap {
  137. struct rcu_head rcu;
  138. unsigned sections_nb;
  139. unsigned sections_nb_alloc;
  140. unsigned nodes_nb;
  141. unsigned nodes_nb_alloc;
  142. Node *nodes;
  143. MemoryRegionSection *sections;
  144. } PhysPageMap;
  145. struct AddressSpaceDispatch {
  146. MemoryRegionSection *mru_section;
  147. /* This is a multi-level map on the physical address space.
  148. * The bottom level has pointers to MemoryRegionSections.
  149. */
  150. PhysPageEntry phys_map;
  151. PhysPageMap map;
  152. };
  153. #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
  154. typedef struct subpage_t {
  155. MemoryRegion iomem;
  156. FlatView *fv;
  157. hwaddr base;
  158. uint16_t sub_section[];
  159. } subpage_t;
  160. #define PHYS_SECTION_UNASSIGNED 0
  161. #define PHYS_SECTION_NOTDIRTY 1
  162. #define PHYS_SECTION_ROM 2
  163. #define PHYS_SECTION_WATCH 3
  164. static void io_mem_init(void);
  165. static void memory_map_init(void);
  166. static void tcg_commit(MemoryListener *listener);
  167. static MemoryRegion io_mem_watch;
  168. /**
  169. * CPUAddressSpace: all the information a CPU needs about an AddressSpace
  170. * @cpu: the CPU whose AddressSpace this is
  171. * @as: the AddressSpace itself
  172. * @memory_dispatch: its dispatch pointer (cached, RCU protected)
  173. * @tcg_as_listener: listener for tracking changes to the AddressSpace
  174. */
  175. struct CPUAddressSpace {
  176. CPUState *cpu;
  177. AddressSpace *as;
  178. struct AddressSpaceDispatch *memory_dispatch;
  179. MemoryListener tcg_as_listener;
  180. };
  181. struct DirtyBitmapSnapshot {
  182. ram_addr_t start;
  183. ram_addr_t end;
  184. unsigned long dirty[];
  185. };
  186. #endif
  187. #if !defined(CONFIG_USER_ONLY)
  188. static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
  189. {
  190. static unsigned alloc_hint = 16;
  191. if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
  192. map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, alloc_hint);
  193. map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
  194. map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
  195. alloc_hint = map->nodes_nb_alloc;
  196. }
  197. }
  198. static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
  199. {
  200. unsigned i;
  201. uint32_t ret;
  202. PhysPageEntry e;
  203. PhysPageEntry *p;
  204. ret = map->nodes_nb++;
  205. p = map->nodes[ret];
  206. assert(ret != PHYS_MAP_NODE_NIL);
  207. assert(ret != map->nodes_nb_alloc);
  208. e.skip = leaf ? 0 : 1;
  209. e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
  210. for (i = 0; i < P_L2_SIZE; ++i) {
  211. memcpy(&p[i], &e, sizeof(e));
  212. }
  213. return ret;
  214. }
  215. static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
  216. hwaddr *index, hwaddr *nb, uint16_t leaf,
  217. int level)
  218. {
  219. PhysPageEntry *p;
  220. hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
  221. if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
  222. lp->ptr = phys_map_node_alloc(map, level == 0);
  223. }
  224. p = map->nodes[lp->ptr];
  225. lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
  226. while (*nb && lp < &p[P_L2_SIZE]) {
  227. if ((*index & (step - 1)) == 0 && *nb >= step) {
  228. lp->skip = 0;
  229. lp->ptr = leaf;
  230. *index += step;
  231. *nb -= step;
  232. } else {
  233. phys_page_set_level(map, lp, index, nb, leaf, level - 1);
  234. }
  235. ++lp;
  236. }
  237. }
  238. static void phys_page_set(AddressSpaceDispatch *d,
  239. hwaddr index, hwaddr nb,
  240. uint16_t leaf)
  241. {
  242. /* Wildly overreserve - it doesn't matter much. */
  243. phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
  244. phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
  245. }
  246. /* Compact a non leaf page entry. Simply detect that the entry has a single child,
  247. * and update our entry so we can skip it and go directly to the destination.
  248. */
  249. static void phys_page_compact(PhysPageEntry *lp, Node *nodes)
  250. {
  251. unsigned valid_ptr = P_L2_SIZE;
  252. int valid = 0;
  253. PhysPageEntry *p;
  254. int i;
  255. if (lp->ptr == PHYS_MAP_NODE_NIL) {
  256. return;
  257. }
  258. p = nodes[lp->ptr];
  259. for (i = 0; i < P_L2_SIZE; i++) {
  260. if (p[i].ptr == PHYS_MAP_NODE_NIL) {
  261. continue;
  262. }
  263. valid_ptr = i;
  264. valid++;
  265. if (p[i].skip) {
  266. phys_page_compact(&p[i], nodes);
  267. }
  268. }
  269. /* We can only compress if there's only one child. */
  270. if (valid != 1) {
  271. return;
  272. }
  273. assert(valid_ptr < P_L2_SIZE);
  274. /* Don't compress if it won't fit in the # of bits we have. */
  275. if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
  276. return;
  277. }
  278. lp->ptr = p[valid_ptr].ptr;
  279. if (!p[valid_ptr].skip) {
  280. /* If our only child is a leaf, make this a leaf. */
  281. /* By design, we should have made this node a leaf to begin with so we
  282. * should never reach here.
  283. * But since it's so simple to handle this, let's do it just in case we
  284. * change this rule.
  285. */
  286. lp->skip = 0;
  287. } else {
  288. lp->skip += p[valid_ptr].skip;
  289. }
  290. }
  291. void address_space_dispatch_compact(AddressSpaceDispatch *d)
  292. {
  293. if (d->phys_map.skip) {
  294. phys_page_compact(&d->phys_map, d->map.nodes);
  295. }
  296. }
  297. static inline bool section_covers_addr(const MemoryRegionSection *section,
  298. hwaddr addr)
  299. {
  300. /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
  301. * the section must cover the entire address space.
  302. */
  303. return int128_gethi(section->size) ||
  304. range_covers_byte(section->offset_within_address_space,
  305. int128_getlo(section->size), addr);
  306. }
  307. static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr addr)
  308. {
  309. PhysPageEntry lp = d->phys_map, *p;
  310. Node *nodes = d->map.nodes;
  311. MemoryRegionSection *sections = d->map.sections;
  312. hwaddr index = addr >> TARGET_PAGE_BITS;
  313. int i;
  314. for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
  315. if (lp.ptr == PHYS_MAP_NODE_NIL) {
  316. return &sections[PHYS_SECTION_UNASSIGNED];
  317. }
  318. p = nodes[lp.ptr];
  319. lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
  320. }
  321. if (section_covers_addr(&sections[lp.ptr], addr)) {
  322. return &sections[lp.ptr];
  323. } else {
  324. return &sections[PHYS_SECTION_UNASSIGNED];
  325. }
  326. }
  327. /* Called from RCU critical section */
  328. static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
  329. hwaddr addr,
  330. bool resolve_subpage)
  331. {
  332. MemoryRegionSection *section = atomic_read(&d->mru_section);
  333. subpage_t *subpage;
  334. if (!section || section == &d->map.sections[PHYS_SECTION_UNASSIGNED] ||
  335. !section_covers_addr(section, addr)) {
  336. section = phys_page_find(d, addr);
  337. atomic_set(&d->mru_section, section);
  338. }
  339. if (resolve_subpage && section->mr->subpage) {
  340. subpage = container_of(section->mr, subpage_t, iomem);
  341. section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
  342. }
  343. return section;
  344. }
  345. /* Called from RCU critical section */
  346. static MemoryRegionSection *
  347. address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
  348. hwaddr *plen, bool resolve_subpage)
  349. {
  350. MemoryRegionSection *section;
  351. MemoryRegion *mr;
  352. Int128 diff;
  353. section = address_space_lookup_region(d, addr, resolve_subpage);
  354. /* Compute offset within MemoryRegionSection */
  355. addr -= section->offset_within_address_space;
  356. /* Compute offset within MemoryRegion */
  357. *xlat = addr + section->offset_within_region;
  358. mr = section->mr;
  359. /* MMIO registers can be expected to perform full-width accesses based only
  360. * on their address, without considering adjacent registers that could
  361. * decode to completely different MemoryRegions. When such registers
  362. * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
  363. * regions overlap wildly. For this reason we cannot clamp the accesses
  364. * here.
  365. *
  366. * If the length is small (as is the case for address_space_ldl/stl),
  367. * everything works fine. If the incoming length is large, however,
  368. * the caller really has to do the clamping through memory_access_size.
  369. */
  370. if (memory_region_is_ram(mr)) {
  371. diff = int128_sub(section->size, int128_make64(addr));
  372. *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
  373. }
  374. return section;
  375. }
  376. /**
  377. * address_space_translate_iommu - translate an address through an IOMMU
  378. * memory region and then through the target address space.
  379. *
  380. * @iommu_mr: the IOMMU memory region that we start the translation from
  381. * @addr: the address to be translated through the MMU
  382. * @xlat: the translated address offset within the destination memory region.
  383. * It cannot be %NULL.
  384. * @plen_out: valid read/write length of the translated address. It
  385. * cannot be %NULL.
  386. * @page_mask_out: page mask for the translated address. This
  387. * should only be meaningful for IOMMU translated
  388. * addresses, since there may be huge pages that this bit
  389. * would tell. It can be %NULL if we don't care about it.
  390. * @is_write: whether the translation operation is for write
  391. * @is_mmio: whether this can be MMIO, set true if it can
  392. * @target_as: the address space targeted by the IOMMU
  393. * @attrs: transaction attributes
  394. *
  395. * This function is called from RCU critical section. It is the common
  396. * part of flatview_do_translate and address_space_translate_cached.
  397. */
  398. static MemoryRegionSection address_space_translate_iommu(IOMMUMemoryRegion *iommu_mr,
  399. hwaddr *xlat,
  400. hwaddr *plen_out,
  401. hwaddr *page_mask_out,
  402. bool is_write,
  403. bool is_mmio,
  404. AddressSpace **target_as,
  405. MemTxAttrs attrs)
  406. {
  407. MemoryRegionSection *section;
  408. hwaddr page_mask = (hwaddr)-1;
  409. do {
  410. hwaddr addr = *xlat;
  411. IOMMUMemoryRegionClass *imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
  412. int iommu_idx = 0;
  413. IOMMUTLBEntry iotlb;
  414. if (imrc->attrs_to_index) {
  415. iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
  416. }
  417. iotlb = imrc->translate(iommu_mr, addr, is_write ?
  418. IOMMU_WO : IOMMU_RO, iommu_idx);
  419. if (!(iotlb.perm & (1 << is_write))) {
  420. goto unassigned;
  421. }
  422. addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
  423. | (addr & iotlb.addr_mask));
  424. page_mask &= iotlb.addr_mask;
  425. *plen_out = MIN(*plen_out, (addr | iotlb.addr_mask) - addr + 1);
  426. *target_as = iotlb.target_as;
  427. section = address_space_translate_internal(
  428. address_space_to_dispatch(iotlb.target_as), addr, xlat,
  429. plen_out, is_mmio);
  430. iommu_mr = memory_region_get_iommu(section->mr);
  431. } while (unlikely(iommu_mr));
  432. if (page_mask_out) {
  433. *page_mask_out = page_mask;
  434. }
  435. return *section;
  436. unassigned:
  437. return (MemoryRegionSection) { .mr = &io_mem_unassigned };
  438. }
  439. /**
  440. * flatview_do_translate - translate an address in FlatView
  441. *
  442. * @fv: the flat view that we want to translate on
  443. * @addr: the address to be translated in above address space
  444. * @xlat: the translated address offset within memory region. It
  445. * cannot be @NULL.
  446. * @plen_out: valid read/write length of the translated address. It
  447. * can be @NULL when we don't care about it.
  448. * @page_mask_out: page mask for the translated address. This
  449. * should only be meaningful for IOMMU translated
  450. * addresses, since there may be huge pages that this bit
  451. * would tell. It can be @NULL if we don't care about it.
  452. * @is_write: whether the translation operation is for write
  453. * @is_mmio: whether this can be MMIO, set true if it can
  454. * @target_as: the address space targeted by the IOMMU
  455. * @attrs: memory transaction attributes
  456. *
  457. * This function is called from RCU critical section
  458. */
  459. static MemoryRegionSection flatview_do_translate(FlatView *fv,
  460. hwaddr addr,
  461. hwaddr *xlat,
  462. hwaddr *plen_out,
  463. hwaddr *page_mask_out,
  464. bool is_write,
  465. bool is_mmio,
  466. AddressSpace **target_as,
  467. MemTxAttrs attrs)
  468. {
  469. MemoryRegionSection *section;
  470. IOMMUMemoryRegion *iommu_mr;
  471. hwaddr plen = (hwaddr)(-1);
  472. if (!plen_out) {
  473. plen_out = &plen;
  474. }
  475. section = address_space_translate_internal(
  476. flatview_to_dispatch(fv), addr, xlat,
  477. plen_out, is_mmio);
  478. iommu_mr = memory_region_get_iommu(section->mr);
  479. if (unlikely(iommu_mr)) {
  480. return address_space_translate_iommu(iommu_mr, xlat,
  481. plen_out, page_mask_out,
  482. is_write, is_mmio,
  483. target_as, attrs);
  484. }
  485. if (page_mask_out) {
  486. /* Not behind an IOMMU, use default page size. */
  487. *page_mask_out = ~TARGET_PAGE_MASK;
  488. }
  489. return *section;
  490. }
  491. /* Called from RCU critical section */
  492. IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr,
  493. bool is_write, MemTxAttrs attrs)
  494. {
  495. MemoryRegionSection section;
  496. hwaddr xlat, page_mask;
  497. /*
  498. * This can never be MMIO, and we don't really care about plen,
  499. * but page mask.
  500. */
  501. section = flatview_do_translate(address_space_to_flatview(as), addr, &xlat,
  502. NULL, &page_mask, is_write, false, &as,
  503. attrs);
  504. /* Illegal translation */
  505. if (section.mr == &io_mem_unassigned) {
  506. goto iotlb_fail;
  507. }
  508. /* Convert memory region offset into address space offset */
  509. xlat += section.offset_within_address_space -
  510. section.offset_within_region;
  511. return (IOMMUTLBEntry) {
  512. .target_as = as,
  513. .iova = addr & ~page_mask,
  514. .translated_addr = xlat & ~page_mask,
  515. .addr_mask = page_mask,
  516. /* IOTLBs are for DMAs, and DMA only allows on RAMs. */
  517. .perm = IOMMU_RW,
  518. };
  519. iotlb_fail:
  520. return (IOMMUTLBEntry) {0};
  521. }
  522. /* Called from RCU critical section */
  523. MemoryRegion *flatview_translate(FlatView *fv, hwaddr addr, hwaddr *xlat,
  524. hwaddr *plen, bool is_write,
  525. MemTxAttrs attrs)
  526. {
  527. MemoryRegion *mr;
  528. MemoryRegionSection section;
  529. AddressSpace *as = NULL;
  530. /* This can be MMIO, so setup MMIO bit. */
  531. section = flatview_do_translate(fv, addr, xlat, plen, NULL,
  532. is_write, true, &as, attrs);
  533. mr = section.mr;
  534. if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
  535. hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
  536. *plen = MIN(page, *plen);
  537. }
  538. return mr;
  539. }
  540. typedef struct TCGIOMMUNotifier {
  541. IOMMUNotifier n;
  542. MemoryRegion *mr;
  543. CPUState *cpu;
  544. int iommu_idx;
  545. bool active;
  546. } TCGIOMMUNotifier;
  547. static void tcg_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
  548. {
  549. TCGIOMMUNotifier *notifier = container_of(n, TCGIOMMUNotifier, n);
  550. if (!notifier->active) {
  551. return;
  552. }
  553. tlb_flush(notifier->cpu);
  554. notifier->active = false;
  555. /* We leave the notifier struct on the list to avoid reallocating it later.
  556. * Generally the number of IOMMUs a CPU deals with will be small.
  557. * In any case we can't unregister the iommu notifier from a notify
  558. * callback.
  559. */
  560. }
  561. static void tcg_register_iommu_notifier(CPUState *cpu,
  562. IOMMUMemoryRegion *iommu_mr,
  563. int iommu_idx)
  564. {
  565. /* Make sure this CPU has an IOMMU notifier registered for this
  566. * IOMMU/IOMMU index combination, so that we can flush its TLB
  567. * when the IOMMU tells us the mappings we've cached have changed.
  568. */
  569. MemoryRegion *mr = MEMORY_REGION(iommu_mr);
  570. TCGIOMMUNotifier *notifier;
  571. int i;
  572. for (i = 0; i < cpu->iommu_notifiers->len; i++) {
  573. notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
  574. if (notifier->mr == mr && notifier->iommu_idx == iommu_idx) {
  575. break;
  576. }
  577. }
  578. if (i == cpu->iommu_notifiers->len) {
  579. /* Not found, add a new entry at the end of the array */
  580. cpu->iommu_notifiers = g_array_set_size(cpu->iommu_notifiers, i + 1);
  581. notifier = g_new0(TCGIOMMUNotifier, 1);
  582. g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i) = notifier;
  583. notifier->mr = mr;
  584. notifier->iommu_idx = iommu_idx;
  585. notifier->cpu = cpu;
  586. /* Rather than trying to register interest in the specific part
  587. * of the iommu's address space that we've accessed and then
  588. * expand it later as subsequent accesses touch more of it, we
  589. * just register interest in the whole thing, on the assumption
  590. * that iommu reconfiguration will be rare.
  591. */
  592. iommu_notifier_init(&notifier->n,
  593. tcg_iommu_unmap_notify,
  594. IOMMU_NOTIFIER_UNMAP,
  595. 0,
  596. HWADDR_MAX,
  597. iommu_idx);
  598. memory_region_register_iommu_notifier(notifier->mr, &notifier->n);
  599. }
  600. if (!notifier->active) {
  601. notifier->active = true;
  602. }
  603. }
  604. static void tcg_iommu_free_notifier_list(CPUState *cpu)
  605. {
  606. /* Destroy the CPU's notifier list */
  607. int i;
  608. TCGIOMMUNotifier *notifier;
  609. for (i = 0; i < cpu->iommu_notifiers->len; i++) {
  610. notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
  611. memory_region_unregister_iommu_notifier(notifier->mr, &notifier->n);
  612. g_free(notifier);
  613. }
  614. g_array_free(cpu->iommu_notifiers, true);
  615. }
  616. /* Called from RCU critical section */
  617. MemoryRegionSection *
  618. address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
  619. hwaddr *xlat, hwaddr *plen,
  620. MemTxAttrs attrs, int *prot)
  621. {
  622. MemoryRegionSection *section;
  623. IOMMUMemoryRegion *iommu_mr;
  624. IOMMUMemoryRegionClass *imrc;
  625. IOMMUTLBEntry iotlb;
  626. int iommu_idx;
  627. AddressSpaceDispatch *d = atomic_rcu_read(&cpu->cpu_ases[asidx].memory_dispatch);
  628. for (;;) {
  629. section = address_space_translate_internal(d, addr, &addr, plen, false);
  630. iommu_mr = memory_region_get_iommu(section->mr);
  631. if (!iommu_mr) {
  632. break;
  633. }
  634. imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
  635. iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
  636. tcg_register_iommu_notifier(cpu, iommu_mr, iommu_idx);
  637. /* We need all the permissions, so pass IOMMU_NONE so the IOMMU
  638. * doesn't short-cut its translation table walk.
  639. */
  640. iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, iommu_idx);
  641. addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
  642. | (addr & iotlb.addr_mask));
  643. /* Update the caller's prot bits to remove permissions the IOMMU
  644. * is giving us a failure response for. If we get down to no
  645. * permissions left at all we can give up now.
  646. */
  647. if (!(iotlb.perm & IOMMU_RO)) {
  648. *prot &= ~(PAGE_READ | PAGE_EXEC);
  649. }
  650. if (!(iotlb.perm & IOMMU_WO)) {
  651. *prot &= ~PAGE_WRITE;
  652. }
  653. if (!*prot) {
  654. goto translate_fail;
  655. }
  656. d = flatview_to_dispatch(address_space_to_flatview(iotlb.target_as));
  657. }
  658. assert(!memory_region_is_iommu(section->mr));
  659. *xlat = addr;
  660. return section;
  661. translate_fail:
  662. return &d->map.sections[PHYS_SECTION_UNASSIGNED];
  663. }
  664. #endif
  665. #if !defined(CONFIG_USER_ONLY)
  666. static int cpu_common_post_load(void *opaque, int version_id)
  667. {
  668. CPUState *cpu = opaque;
  669. /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
  670. version_id is increased. */
  671. cpu->interrupt_request &= ~0x01;
  672. tlb_flush(cpu);
  673. /* loadvm has just updated the content of RAM, bypassing the
  674. * usual mechanisms that ensure we flush TBs for writes to
  675. * memory we've translated code from. So we must flush all TBs,
  676. * which will now be stale.
  677. */
  678. tb_flush(cpu);
  679. return 0;
  680. }
  681. static int cpu_common_pre_load(void *opaque)
  682. {
  683. CPUState *cpu = opaque;
  684. cpu->exception_index = -1;
  685. return 0;
  686. }
  687. static bool cpu_common_exception_index_needed(void *opaque)
  688. {
  689. CPUState *cpu = opaque;
  690. return tcg_enabled() && cpu->exception_index != -1;
  691. }
  692. static const VMStateDescription vmstate_cpu_common_exception_index = {
  693. .name = "cpu_common/exception_index",
  694. .version_id = 1,
  695. .minimum_version_id = 1,
  696. .needed = cpu_common_exception_index_needed,
  697. .fields = (VMStateField[]) {
  698. VMSTATE_INT32(exception_index, CPUState),
  699. VMSTATE_END_OF_LIST()
  700. }
  701. };
  702. static bool cpu_common_crash_occurred_needed(void *opaque)
  703. {
  704. CPUState *cpu = opaque;
  705. return cpu->crash_occurred;
  706. }
  707. static const VMStateDescription vmstate_cpu_common_crash_occurred = {
  708. .name = "cpu_common/crash_occurred",
  709. .version_id = 1,
  710. .minimum_version_id = 1,
  711. .needed = cpu_common_crash_occurred_needed,
  712. .fields = (VMStateField[]) {
  713. VMSTATE_BOOL(crash_occurred, CPUState),
  714. VMSTATE_END_OF_LIST()
  715. }
  716. };
  717. const VMStateDescription vmstate_cpu_common = {
  718. .name = "cpu_common",
  719. .version_id = 1,
  720. .minimum_version_id = 1,
  721. .pre_load = cpu_common_pre_load,
  722. .post_load = cpu_common_post_load,
  723. .fields = (VMStateField[]) {
  724. VMSTATE_UINT32(halted, CPUState),
  725. VMSTATE_UINT32(interrupt_request, CPUState),
  726. VMSTATE_END_OF_LIST()
  727. },
  728. .subsections = (const VMStateDescription*[]) {
  729. &vmstate_cpu_common_exception_index,
  730. &vmstate_cpu_common_crash_occurred,
  731. NULL
  732. }
  733. };
  734. #endif
  735. CPUState *qemu_get_cpu(int index)
  736. {
  737. CPUState *cpu;
  738. CPU_FOREACH(cpu) {
  739. if (cpu->cpu_index == index) {
  740. return cpu;
  741. }
  742. }
  743. return NULL;
  744. }
  745. #if !defined(CONFIG_USER_ONLY)
  746. void cpu_address_space_init(CPUState *cpu, int asidx,
  747. const char *prefix, MemoryRegion *mr)
  748. {
  749. CPUAddressSpace *newas;
  750. AddressSpace *as = g_new0(AddressSpace, 1);
  751. char *as_name;
  752. assert(mr);
  753. as_name = g_strdup_printf("%s-%d", prefix, cpu->cpu_index);
  754. address_space_init(as, mr, as_name);
  755. g_free(as_name);
  756. /* Target code should have set num_ases before calling us */
  757. assert(asidx < cpu->num_ases);
  758. if (asidx == 0) {
  759. /* address space 0 gets the convenience alias */
  760. cpu->as = as;
  761. }
  762. /* KVM cannot currently support multiple address spaces. */
  763. assert(asidx == 0 || !kvm_enabled());
  764. if (!cpu->cpu_ases) {
  765. cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
  766. }
  767. newas = &cpu->cpu_ases[asidx];
  768. newas->cpu = cpu;
  769. newas->as = as;
  770. if (tcg_enabled()) {
  771. newas->tcg_as_listener.commit = tcg_commit;
  772. memory_listener_register(&newas->tcg_as_listener, as);
  773. }
  774. }
  775. AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
  776. {
  777. /* Return the AddressSpace corresponding to the specified index */
  778. return cpu->cpu_ases[asidx].as;
  779. }
  780. #endif
  781. void cpu_exec_unrealizefn(CPUState *cpu)
  782. {
  783. CPUClass *cc = CPU_GET_CLASS(cpu);
  784. cpu_list_remove(cpu);
  785. if (cc->vmsd != NULL) {
  786. vmstate_unregister(NULL, cc->vmsd, cpu);
  787. }
  788. if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
  789. vmstate_unregister(NULL, &vmstate_cpu_common, cpu);
  790. }
  791. #ifndef CONFIG_USER_ONLY
  792. tcg_iommu_free_notifier_list(cpu);
  793. #endif
  794. }
  795. Property cpu_common_props[] = {
  796. #ifndef CONFIG_USER_ONLY
  797. /* Create a memory property for softmmu CPU object,
  798. * so users can wire up its memory. (This can't go in qom/cpu.c
  799. * because that file is compiled only once for both user-mode
  800. * and system builds.) The default if no link is set up is to use
  801. * the system address space.
  802. */
  803. DEFINE_PROP_LINK("memory", CPUState, memory, TYPE_MEMORY_REGION,
  804. MemoryRegion *),
  805. #endif
  806. DEFINE_PROP_END_OF_LIST(),
  807. };
  808. void cpu_exec_initfn(CPUState *cpu)
  809. {
  810. cpu->as = NULL;
  811. cpu->num_ases = 0;
  812. #ifndef CONFIG_USER_ONLY
  813. cpu->thread_id = qemu_get_thread_id();
  814. cpu->memory = system_memory;
  815. object_ref(OBJECT(cpu->memory));
  816. #endif
  817. }
  818. void cpu_exec_realizefn(CPUState *cpu, Error **errp)
  819. {
  820. CPUClass *cc = CPU_GET_CLASS(cpu);
  821. static bool tcg_target_initialized;
  822. cpu_list_add(cpu);
  823. if (tcg_enabled() && !tcg_target_initialized) {
  824. tcg_target_initialized = true;
  825. cc->tcg_initialize();
  826. }
  827. tlb_init(cpu);
  828. #ifndef CONFIG_USER_ONLY
  829. if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
  830. vmstate_register(NULL, cpu->cpu_index, &vmstate_cpu_common, cpu);
  831. }
  832. if (cc->vmsd != NULL) {
  833. vmstate_register(NULL, cpu->cpu_index, cc->vmsd, cpu);
  834. }
  835. cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier *));
  836. #endif
  837. }
  838. const char *parse_cpu_option(const char *cpu_option)
  839. {
  840. ObjectClass *oc;
  841. CPUClass *cc;
  842. gchar **model_pieces;
  843. const char *cpu_type;
  844. model_pieces = g_strsplit(cpu_option, ",", 2);
  845. if (!model_pieces[0]) {
  846. error_report("-cpu option cannot be empty");
  847. exit(1);
  848. }
  849. oc = cpu_class_by_name(CPU_RESOLVING_TYPE, model_pieces[0]);
  850. if (oc == NULL) {
  851. error_report("unable to find CPU model '%s'", model_pieces[0]);
  852. g_strfreev(model_pieces);
  853. exit(EXIT_FAILURE);
  854. }
  855. cpu_type = object_class_get_name(oc);
  856. cc = CPU_CLASS(oc);
  857. cc->parse_features(cpu_type, model_pieces[1], &error_fatal);
  858. g_strfreev(model_pieces);
  859. return cpu_type;
  860. }
  861. #if defined(CONFIG_USER_ONLY)
  862. void tb_invalidate_phys_addr(target_ulong addr)
  863. {
  864. mmap_lock();
  865. tb_invalidate_phys_page_range(addr, addr + 1, 0);
  866. mmap_unlock();
  867. }
  868. static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
  869. {
  870. tb_invalidate_phys_addr(pc);
  871. }
  872. #else
  873. void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs)
  874. {
  875. ram_addr_t ram_addr;
  876. MemoryRegion *mr;
  877. hwaddr l = 1;
  878. if (!tcg_enabled()) {
  879. return;
  880. }
  881. rcu_read_lock();
  882. mr = address_space_translate(as, addr, &addr, &l, false, attrs);
  883. if (!(memory_region_is_ram(mr)
  884. || memory_region_is_romd(mr))) {
  885. rcu_read_unlock();
  886. return;
  887. }
  888. ram_addr = memory_region_get_ram_addr(mr) + addr;
  889. tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
  890. rcu_read_unlock();
  891. }
  892. static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
  893. {
  894. MemTxAttrs attrs;
  895. hwaddr phys = cpu_get_phys_page_attrs_debug(cpu, pc, &attrs);
  896. int asidx = cpu_asidx_from_attrs(cpu, attrs);
  897. if (phys != -1) {
  898. /* Locks grabbed by tb_invalidate_phys_addr */
  899. tb_invalidate_phys_addr(cpu->cpu_ases[asidx].as,
  900. phys | (pc & ~TARGET_PAGE_MASK), attrs);
  901. }
  902. }
  903. #endif
  904. #if defined(CONFIG_USER_ONLY)
  905. void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
  906. {
  907. }
  908. int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
  909. int flags)
  910. {
  911. return -ENOSYS;
  912. }
  913. void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
  914. {
  915. }
  916. int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
  917. int flags, CPUWatchpoint **watchpoint)
  918. {
  919. return -ENOSYS;
  920. }
  921. #else
  922. /* Add a watchpoint. */
  923. int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
  924. int flags, CPUWatchpoint **watchpoint)
  925. {
  926. CPUWatchpoint *wp;
  927. /* forbid ranges which are empty or run off the end of the address space */
  928. if (len == 0 || (addr + len - 1) < addr) {
  929. error_report("tried to set invalid watchpoint at %"
  930. VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
  931. return -EINVAL;
  932. }
  933. wp = g_malloc(sizeof(*wp));
  934. wp->vaddr = addr;
  935. wp->len = len;
  936. wp->flags = flags;
  937. /* keep all GDB-injected watchpoints in front */
  938. if (flags & BP_GDB) {
  939. QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
  940. } else {
  941. QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
  942. }
  943. tlb_flush_page(cpu, addr);
  944. if (watchpoint)
  945. *watchpoint = wp;
  946. return 0;
  947. }
  948. /* Remove a specific watchpoint. */
  949. int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
  950. int flags)
  951. {
  952. CPUWatchpoint *wp;
  953. QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
  954. if (addr == wp->vaddr && len == wp->len
  955. && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
  956. cpu_watchpoint_remove_by_ref(cpu, wp);
  957. return 0;
  958. }
  959. }
  960. return -ENOENT;
  961. }
  962. /* Remove a specific watchpoint by reference. */
  963. void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
  964. {
  965. QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
  966. tlb_flush_page(cpu, watchpoint->vaddr);
  967. g_free(watchpoint);
  968. }
  969. /* Remove all matching watchpoints. */
  970. void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
  971. {
  972. CPUWatchpoint *wp, *next;
  973. QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
  974. if (wp->flags & mask) {
  975. cpu_watchpoint_remove_by_ref(cpu, wp);
  976. }
  977. }
  978. }
  979. /* Return true if this watchpoint address matches the specified
  980. * access (ie the address range covered by the watchpoint overlaps
  981. * partially or completely with the address range covered by the
  982. * access).
  983. */
  984. static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
  985. vaddr addr,
  986. vaddr len)
  987. {
  988. /* We know the lengths are non-zero, but a little caution is
  989. * required to avoid errors in the case where the range ends
  990. * exactly at the top of the address space and so addr + len
  991. * wraps round to zero.
  992. */
  993. vaddr wpend = wp->vaddr + wp->len - 1;
  994. vaddr addrend = addr + len - 1;
  995. return !(addr > wpend || wp->vaddr > addrend);
  996. }
  997. #endif
  998. /* Add a breakpoint. */
  999. int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
  1000. CPUBreakpoint **breakpoint)
  1001. {
  1002. CPUBreakpoint *bp;
  1003. bp = g_malloc(sizeof(*bp));
  1004. bp->pc = pc;
  1005. bp->flags = flags;
  1006. /* keep all GDB-injected breakpoints in front */
  1007. if (flags & BP_GDB) {
  1008. QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
  1009. } else {
  1010. QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
  1011. }
  1012. breakpoint_invalidate(cpu, pc);
  1013. if (breakpoint) {
  1014. *breakpoint = bp;
  1015. }
  1016. return 0;
  1017. }
  1018. /* Remove a specific breakpoint. */
  1019. int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
  1020. {
  1021. CPUBreakpoint *bp;
  1022. QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
  1023. if (bp->pc == pc && bp->flags == flags) {
  1024. cpu_breakpoint_remove_by_ref(cpu, bp);
  1025. return 0;
  1026. }
  1027. }
  1028. return -ENOENT;
  1029. }
  1030. /* Remove a specific breakpoint by reference. */
  1031. void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
  1032. {
  1033. QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
  1034. breakpoint_invalidate(cpu, breakpoint->pc);
  1035. g_free(breakpoint);
  1036. }
  1037. /* Remove all matching breakpoints. */
  1038. void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
  1039. {
  1040. CPUBreakpoint *bp, *next;
  1041. QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
  1042. if (bp->flags & mask) {
  1043. cpu_breakpoint_remove_by_ref(cpu, bp);
  1044. }
  1045. }
  1046. }
  1047. /* enable or disable single step mode. EXCP_DEBUG is returned by the
  1048. CPU loop after each instruction */
  1049. void cpu_single_step(CPUState *cpu, int enabled)
  1050. {
  1051. if (cpu->singlestep_enabled != enabled) {
  1052. cpu->singlestep_enabled = enabled;
  1053. if (kvm_enabled()) {
  1054. kvm_update_guest_debug(cpu, 0);
  1055. } else {
  1056. /* must flush all the translated code to avoid inconsistencies */
  1057. /* XXX: only flush what is necessary */
  1058. tb_flush(cpu);
  1059. }
  1060. }
  1061. }
  1062. void cpu_abort(CPUState *cpu, const char *fmt, ...)
  1063. {
  1064. va_list ap;
  1065. va_list ap2;
  1066. va_start(ap, fmt);
  1067. va_copy(ap2, ap);
  1068. fprintf(stderr, "qemu: fatal: ");
  1069. vfprintf(stderr, fmt, ap);
  1070. fprintf(stderr, "\n");
  1071. cpu_dump_state(cpu, stderr, CPU_DUMP_FPU | CPU_DUMP_CCOP);
  1072. if (qemu_log_separate()) {
  1073. qemu_log_lock();
  1074. qemu_log("qemu: fatal: ");
  1075. qemu_log_vprintf(fmt, ap2);
  1076. qemu_log("\n");
  1077. log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
  1078. qemu_log_flush();
  1079. qemu_log_unlock();
  1080. qemu_log_close();
  1081. }
  1082. va_end(ap2);
  1083. va_end(ap);
  1084. replay_finish();
  1085. #if defined(CONFIG_USER_ONLY)
  1086. {
  1087. struct sigaction act;
  1088. sigfillset(&act.sa_mask);
  1089. act.sa_handler = SIG_DFL;
  1090. act.sa_flags = 0;
  1091. sigaction(SIGABRT, &act, NULL);
  1092. }
  1093. #endif
  1094. abort();
  1095. }
  1096. #if !defined(CONFIG_USER_ONLY)
  1097. /* Called from RCU critical section */
  1098. static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
  1099. {
  1100. RAMBlock *block;
  1101. block = atomic_rcu_read(&ram_list.mru_block);
  1102. if (block && addr - block->offset < block->max_length) {
  1103. return block;
  1104. }
  1105. RAMBLOCK_FOREACH(block) {
  1106. if (addr - block->offset < block->max_length) {
  1107. goto found;
  1108. }
  1109. }
  1110. fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
  1111. abort();
  1112. found:
  1113. /* It is safe to write mru_block outside the iothread lock. This
  1114. * is what happens:
  1115. *
  1116. * mru_block = xxx
  1117. * rcu_read_unlock()
  1118. * xxx removed from list
  1119. * rcu_read_lock()
  1120. * read mru_block
  1121. * mru_block = NULL;
  1122. * call_rcu(reclaim_ramblock, xxx);
  1123. * rcu_read_unlock()
  1124. *
  1125. * atomic_rcu_set is not needed here. The block was already published
  1126. * when it was placed into the list. Here we're just making an extra
  1127. * copy of the pointer.
  1128. */
  1129. ram_list.mru_block = block;
  1130. return block;
  1131. }
  1132. static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
  1133. {
  1134. CPUState *cpu;
  1135. ram_addr_t start1;
  1136. RAMBlock *block;
  1137. ram_addr_t end;
  1138. assert(tcg_enabled());
  1139. end = TARGET_PAGE_ALIGN(start + length);
  1140. start &= TARGET_PAGE_MASK;
  1141. rcu_read_lock();
  1142. block = qemu_get_ram_block(start);
  1143. assert(block == qemu_get_ram_block(end - 1));
  1144. start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
  1145. CPU_FOREACH(cpu) {
  1146. tlb_reset_dirty(cpu, start1, length);
  1147. }
  1148. rcu_read_unlock();
  1149. }
  1150. /* Note: start and end must be within the same ram block. */
  1151. bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
  1152. ram_addr_t length,
  1153. unsigned client)
  1154. {
  1155. DirtyMemoryBlocks *blocks;
  1156. unsigned long end, page;
  1157. bool dirty = false;
  1158. if (length == 0) {
  1159. return false;
  1160. }
  1161. end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
  1162. page = start >> TARGET_PAGE_BITS;
  1163. rcu_read_lock();
  1164. blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
  1165. while (page < end) {
  1166. unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
  1167. unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
  1168. unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
  1169. dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
  1170. offset, num);
  1171. page += num;
  1172. }
  1173. rcu_read_unlock();
  1174. if (dirty && tcg_enabled()) {
  1175. tlb_reset_dirty_range_all(start, length);
  1176. }
  1177. return dirty;
  1178. }
  1179. DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
  1180. (ram_addr_t start, ram_addr_t length, unsigned client)
  1181. {
  1182. DirtyMemoryBlocks *blocks;
  1183. unsigned long align = 1UL << (TARGET_PAGE_BITS + BITS_PER_LEVEL);
  1184. ram_addr_t first = QEMU_ALIGN_DOWN(start, align);
  1185. ram_addr_t last = QEMU_ALIGN_UP(start + length, align);
  1186. DirtyBitmapSnapshot *snap;
  1187. unsigned long page, end, dest;
  1188. snap = g_malloc0(sizeof(*snap) +
  1189. ((last - first) >> (TARGET_PAGE_BITS + 3)));
  1190. snap->start = first;
  1191. snap->end = last;
  1192. page = first >> TARGET_PAGE_BITS;
  1193. end = last >> TARGET_PAGE_BITS;
  1194. dest = 0;
  1195. rcu_read_lock();
  1196. blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
  1197. while (page < end) {
  1198. unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
  1199. unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
  1200. unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
  1201. assert(QEMU_IS_ALIGNED(offset, (1 << BITS_PER_LEVEL)));
  1202. assert(QEMU_IS_ALIGNED(num, (1 << BITS_PER_LEVEL)));
  1203. offset >>= BITS_PER_LEVEL;
  1204. bitmap_copy_and_clear_atomic(snap->dirty + dest,
  1205. blocks->blocks[idx] + offset,
  1206. num);
  1207. page += num;
  1208. dest += num >> BITS_PER_LEVEL;
  1209. }
  1210. rcu_read_unlock();
  1211. if (tcg_enabled()) {
  1212. tlb_reset_dirty_range_all(start, length);
  1213. }
  1214. return snap;
  1215. }
  1216. bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
  1217. ram_addr_t start,
  1218. ram_addr_t length)
  1219. {
  1220. unsigned long page, end;
  1221. assert(start >= snap->start);
  1222. assert(start + length <= snap->end);
  1223. end = TARGET_PAGE_ALIGN(start + length - snap->start) >> TARGET_PAGE_BITS;
  1224. page = (start - snap->start) >> TARGET_PAGE_BITS;
  1225. while (page < end) {
  1226. if (test_bit(page, snap->dirty)) {
  1227. return true;
  1228. }
  1229. page++;
  1230. }
  1231. return false;
  1232. }
  1233. /* Called from RCU critical section */
  1234. hwaddr memory_region_section_get_iotlb(CPUState *cpu,
  1235. MemoryRegionSection *section,
  1236. target_ulong vaddr,
  1237. hwaddr paddr, hwaddr xlat,
  1238. int prot,
  1239. target_ulong *address)
  1240. {
  1241. hwaddr iotlb;
  1242. CPUWatchpoint *wp;
  1243. if (memory_region_is_ram(section->mr)) {
  1244. /* Normal RAM. */
  1245. iotlb = memory_region_get_ram_addr(section->mr) + xlat;
  1246. if (!section->readonly) {
  1247. iotlb |= PHYS_SECTION_NOTDIRTY;
  1248. } else {
  1249. iotlb |= PHYS_SECTION_ROM;
  1250. }
  1251. } else {
  1252. AddressSpaceDispatch *d;
  1253. d = flatview_to_dispatch(section->fv);
  1254. iotlb = section - d->map.sections;
  1255. iotlb += xlat;
  1256. }
  1257. /* Make accesses to pages with watchpoints go via the
  1258. watchpoint trap routines. */
  1259. QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
  1260. if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
  1261. /* Avoid trapping reads of pages with a write breakpoint. */
  1262. if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
  1263. iotlb = PHYS_SECTION_WATCH + paddr;
  1264. *address |= TLB_MMIO;
  1265. break;
  1266. }
  1267. }
  1268. }
  1269. return iotlb;
  1270. }
  1271. #endif /* defined(CONFIG_USER_ONLY) */
  1272. #if !defined(CONFIG_USER_ONLY)
  1273. static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
  1274. uint16_t section);
  1275. static subpage_t *subpage_init(FlatView *fv, hwaddr base);
  1276. static void *(*phys_mem_alloc)(size_t size, uint64_t *align, bool shared) =
  1277. qemu_anon_ram_alloc;
  1278. /*
  1279. * Set a custom physical guest memory alloator.
  1280. * Accelerators with unusual needs may need this. Hopefully, we can
  1281. * get rid of it eventually.
  1282. */
  1283. void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align, bool shared))
  1284. {
  1285. phys_mem_alloc = alloc;
  1286. }
  1287. static uint16_t phys_section_add(PhysPageMap *map,
  1288. MemoryRegionSection *section)
  1289. {
  1290. /* The physical section number is ORed with a page-aligned
  1291. * pointer to produce the iotlb entries. Thus it should
  1292. * never overflow into the page-aligned value.
  1293. */
  1294. assert(map->sections_nb < TARGET_PAGE_SIZE);
  1295. if (map->sections_nb == map->sections_nb_alloc) {
  1296. map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
  1297. map->sections = g_renew(MemoryRegionSection, map->sections,
  1298. map->sections_nb_alloc);
  1299. }
  1300. map->sections[map->sections_nb] = *section;
  1301. memory_region_ref(section->mr);
  1302. return map->sections_nb++;
  1303. }
  1304. static void phys_section_destroy(MemoryRegion *mr)
  1305. {
  1306. bool have_sub_page = mr->subpage;
  1307. memory_region_unref(mr);
  1308. if (have_sub_page) {
  1309. subpage_t *subpage = container_of(mr, subpage_t, iomem);
  1310. object_unref(OBJECT(&subpage->iomem));
  1311. g_free(subpage);
  1312. }
  1313. }
  1314. static void phys_sections_free(PhysPageMap *map)
  1315. {
  1316. while (map->sections_nb > 0) {
  1317. MemoryRegionSection *section = &map->sections[--map->sections_nb];
  1318. phys_section_destroy(section->mr);
  1319. }
  1320. g_free(map->sections);
  1321. g_free(map->nodes);
  1322. }
  1323. static void register_subpage(FlatView *fv, MemoryRegionSection *section)
  1324. {
  1325. AddressSpaceDispatch *d = flatview_to_dispatch(fv);
  1326. subpage_t *subpage;
  1327. hwaddr base = section->offset_within_address_space
  1328. & TARGET_PAGE_MASK;
  1329. MemoryRegionSection *existing = phys_page_find(d, base);
  1330. MemoryRegionSection subsection = {
  1331. .offset_within_address_space = base,
  1332. .size = int128_make64(TARGET_PAGE_SIZE),
  1333. };
  1334. hwaddr start, end;
  1335. assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
  1336. if (!(existing->mr->subpage)) {
  1337. subpage = subpage_init(fv, base);
  1338. subsection.fv = fv;
  1339. subsection.mr = &subpage->iomem;
  1340. phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
  1341. phys_section_add(&d->map, &subsection));
  1342. } else {
  1343. subpage = container_of(existing->mr, subpage_t, iomem);
  1344. }
  1345. start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
  1346. end = start + int128_get64(section->size) - 1;
  1347. subpage_register(subpage, start, end,
  1348. phys_section_add(&d->map, section));
  1349. }
  1350. static void register_multipage(FlatView *fv,
  1351. MemoryRegionSection *section)
  1352. {
  1353. AddressSpaceDispatch *d = flatview_to_dispatch(fv);
  1354. hwaddr start_addr = section->offset_within_address_space;
  1355. uint16_t section_index = phys_section_add(&d->map, section);
  1356. uint64_t num_pages = int128_get64(int128_rshift(section->size,
  1357. TARGET_PAGE_BITS));
  1358. assert(num_pages);
  1359. phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
  1360. }
  1361. /*
  1362. * The range in *section* may look like this:
  1363. *
  1364. * |s|PPPPPPP|s|
  1365. *
  1366. * where s stands for subpage and P for page.
  1367. */
  1368. void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section)
  1369. {
  1370. MemoryRegionSection remain = *section;
  1371. Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
  1372. /* register first subpage */
  1373. if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
  1374. uint64_t left = TARGET_PAGE_ALIGN(remain.offset_within_address_space)
  1375. - remain.offset_within_address_space;
  1376. MemoryRegionSection now = remain;
  1377. now.size = int128_min(int128_make64(left), now.size);
  1378. register_subpage(fv, &now);
  1379. if (int128_eq(remain.size, now.size)) {
  1380. return;
  1381. }
  1382. remain.size = int128_sub(remain.size, now.size);
  1383. remain.offset_within_address_space += int128_get64(now.size);
  1384. remain.offset_within_region += int128_get64(now.size);
  1385. }
  1386. /* register whole pages */
  1387. if (int128_ge(remain.size, page_size)) {
  1388. MemoryRegionSection now = remain;
  1389. now.size = int128_and(now.size, int128_neg(page_size));
  1390. register_multipage(fv, &now);
  1391. if (int128_eq(remain.size, now.size)) {
  1392. return;
  1393. }
  1394. remain.size = int128_sub(remain.size, now.size);
  1395. remain.offset_within_address_space += int128_get64(now.size);
  1396. remain.offset_within_region += int128_get64(now.size);
  1397. }
  1398. /* register last subpage */
  1399. register_subpage(fv, &remain);
  1400. }
  1401. void qemu_flush_coalesced_mmio_buffer(void)
  1402. {
  1403. if (kvm_enabled())
  1404. kvm_flush_coalesced_mmio_buffer();
  1405. }
  1406. void qemu_mutex_lock_ramlist(void)
  1407. {
  1408. qemu_mutex_lock(&ram_list.mutex);
  1409. }
  1410. void qemu_mutex_unlock_ramlist(void)
  1411. {
  1412. qemu_mutex_unlock(&ram_list.mutex);
  1413. }
  1414. void ram_block_dump(Monitor *mon)
  1415. {
  1416. RAMBlock *block;
  1417. char *psize;
  1418. rcu_read_lock();
  1419. monitor_printf(mon, "%24s %8s %18s %18s %18s\n",
  1420. "Block Name", "PSize", "Offset", "Used", "Total");
  1421. RAMBLOCK_FOREACH(block) {
  1422. psize = size_to_str(block->page_size);
  1423. monitor_printf(mon, "%24s %8s 0x%016" PRIx64 " 0x%016" PRIx64
  1424. " 0x%016" PRIx64 "\n", block->idstr, psize,
  1425. (uint64_t)block->offset,
  1426. (uint64_t)block->used_length,
  1427. (uint64_t)block->max_length);
  1428. g_free(psize);
  1429. }
  1430. rcu_read_unlock();
  1431. }
  1432. #ifdef __linux__
  1433. /*
  1434. * FIXME TOCTTOU: this iterates over memory backends' mem-path, which
  1435. * may or may not name the same files / on the same filesystem now as
  1436. * when we actually open and map them. Iterate over the file
  1437. * descriptors instead, and use qemu_fd_getpagesize().
  1438. */
  1439. static int find_min_backend_pagesize(Object *obj, void *opaque)
  1440. {
  1441. long *hpsize_min = opaque;
  1442. if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
  1443. HostMemoryBackend *backend = MEMORY_BACKEND(obj);
  1444. long hpsize = host_memory_backend_pagesize(backend);
  1445. if (host_memory_backend_is_mapped(backend) && (hpsize < *hpsize_min)) {
  1446. *hpsize_min = hpsize;
  1447. }
  1448. }
  1449. return 0;
  1450. }
  1451. static int find_max_backend_pagesize(Object *obj, void *opaque)
  1452. {
  1453. long *hpsize_max = opaque;
  1454. if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
  1455. HostMemoryBackend *backend = MEMORY_BACKEND(obj);
  1456. long hpsize = host_memory_backend_pagesize(backend);
  1457. if (host_memory_backend_is_mapped(backend) && (hpsize > *hpsize_max)) {
  1458. *hpsize_max = hpsize;
  1459. }
  1460. }
  1461. return 0;
  1462. }
  1463. /*
  1464. * TODO: We assume right now that all mapped host memory backends are
  1465. * used as RAM, however some might be used for different purposes.
  1466. */
  1467. long qemu_minrampagesize(void)
  1468. {
  1469. long hpsize = LONG_MAX;
  1470. long mainrampagesize;
  1471. Object *memdev_root;
  1472. mainrampagesize = qemu_mempath_getpagesize(mem_path);
  1473. /* it's possible we have memory-backend objects with
  1474. * hugepage-backed RAM. these may get mapped into system
  1475. * address space via -numa parameters or memory hotplug
  1476. * hooks. we want to take these into account, but we
  1477. * also want to make sure these supported hugepage
  1478. * sizes are applicable across the entire range of memory
  1479. * we may boot from, so we take the min across all
  1480. * backends, and assume normal pages in cases where a
  1481. * backend isn't backed by hugepages.
  1482. */
  1483. memdev_root = object_resolve_path("/objects", NULL);
  1484. if (memdev_root) {
  1485. object_child_foreach(memdev_root, find_min_backend_pagesize, &hpsize);
  1486. }
  1487. if (hpsize == LONG_MAX) {
  1488. /* No additional memory regions found ==> Report main RAM page size */
  1489. return mainrampagesize;
  1490. }
  1491. /* If NUMA is disabled or the NUMA nodes are not backed with a
  1492. * memory-backend, then there is at least one node using "normal" RAM,
  1493. * so if its page size is smaller we have got to report that size instead.
  1494. */
  1495. if (hpsize > mainrampagesize &&
  1496. (nb_numa_nodes == 0 || numa_info[0].node_memdev == NULL)) {
  1497. static bool warned;
  1498. if (!warned) {
  1499. error_report("Huge page support disabled (n/a for main memory).");
  1500. warned = true;
  1501. }
  1502. return mainrampagesize;
  1503. }
  1504. return hpsize;
  1505. }
  1506. long qemu_maxrampagesize(void)
  1507. {
  1508. long pagesize = qemu_mempath_getpagesize(mem_path);
  1509. Object *memdev_root = object_resolve_path("/objects", NULL);
  1510. if (memdev_root) {
  1511. object_child_foreach(memdev_root, find_max_backend_pagesize,
  1512. &pagesize);
  1513. }
  1514. return pagesize;
  1515. }
  1516. #else
  1517. long qemu_minrampagesize(void)
  1518. {
  1519. return getpagesize();
  1520. }
  1521. long qemu_maxrampagesize(void)
  1522. {
  1523. return getpagesize();
  1524. }
  1525. #endif
  1526. #ifdef CONFIG_POSIX
  1527. static int64_t get_file_size(int fd)
  1528. {
  1529. int64_t size = lseek(fd, 0, SEEK_END);
  1530. if (size < 0) {
  1531. return -errno;
  1532. }
  1533. return size;
  1534. }
  1535. static int file_ram_open(const char *path,
  1536. const char *region_name,
  1537. bool *created,
  1538. Error **errp)
  1539. {
  1540. char *filename;
  1541. char *sanitized_name;
  1542. char *c;
  1543. int fd = -1;
  1544. *created = false;
  1545. for (;;) {
  1546. fd = open(path, O_RDWR);
  1547. if (fd >= 0) {
  1548. /* @path names an existing file, use it */
  1549. break;
  1550. }
  1551. if (errno == ENOENT) {
  1552. /* @path names a file that doesn't exist, create it */
  1553. fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
  1554. if (fd >= 0) {
  1555. *created = true;
  1556. break;
  1557. }
  1558. } else if (errno == EISDIR) {
  1559. /* @path names a directory, create a file there */
  1560. /* Make name safe to use with mkstemp by replacing '/' with '_'. */
  1561. sanitized_name = g_strdup(region_name);
  1562. for (c = sanitized_name; *c != '\0'; c++) {
  1563. if (*c == '/') {
  1564. *c = '_';
  1565. }
  1566. }
  1567. filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
  1568. sanitized_name);
  1569. g_free(sanitized_name);
  1570. fd = mkstemp(filename);
  1571. if (fd >= 0) {
  1572. unlink(filename);
  1573. g_free(filename);
  1574. break;
  1575. }
  1576. g_free(filename);
  1577. }
  1578. if (errno != EEXIST && errno != EINTR) {
  1579. error_setg_errno(errp, errno,
  1580. "can't open backing store %s for guest RAM",
  1581. path);
  1582. return -1;
  1583. }
  1584. /*
  1585. * Try again on EINTR and EEXIST. The latter happens when
  1586. * something else creates the file between our two open().
  1587. */
  1588. }
  1589. return fd;
  1590. }
  1591. static void *file_ram_alloc(RAMBlock *block,
  1592. ram_addr_t memory,
  1593. int fd,
  1594. bool truncate,
  1595. Error **errp)
  1596. {
  1597. void *area;
  1598. block->page_size = qemu_fd_getpagesize(fd);
  1599. if (block->mr->align % block->page_size) {
  1600. error_setg(errp, "alignment 0x%" PRIx64
  1601. " must be multiples of page size 0x%zx",
  1602. block->mr->align, block->page_size);
  1603. return NULL;
  1604. } else if (block->mr->align && !is_power_of_2(block->mr->align)) {
  1605. error_setg(errp, "alignment 0x%" PRIx64
  1606. " must be a power of two", block->mr->align);
  1607. return NULL;
  1608. }
  1609. block->mr->align = MAX(block->page_size, block->mr->align);
  1610. #if defined(__s390x__)
  1611. if (kvm_enabled()) {
  1612. block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN);
  1613. }
  1614. #endif
  1615. if (memory < block->page_size) {
  1616. error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
  1617. "or larger than page size 0x%zx",
  1618. memory, block->page_size);
  1619. return NULL;
  1620. }
  1621. memory = ROUND_UP(memory, block->page_size);
  1622. /*
  1623. * ftruncate is not supported by hugetlbfs in older
  1624. * hosts, so don't bother bailing out on errors.
  1625. * If anything goes wrong with it under other filesystems,
  1626. * mmap will fail.
  1627. *
  1628. * Do not truncate the non-empty backend file to avoid corrupting
  1629. * the existing data in the file. Disabling shrinking is not
  1630. * enough. For example, the current vNVDIMM implementation stores
  1631. * the guest NVDIMM labels at the end of the backend file. If the
  1632. * backend file is later extended, QEMU will not be able to find
  1633. * those labels. Therefore, extending the non-empty backend file
  1634. * is disabled as well.
  1635. */
  1636. if (truncate && ftruncate(fd, memory)) {
  1637. perror("ftruncate");
  1638. }
  1639. area = qemu_ram_mmap(fd, memory, block->mr->align,
  1640. block->flags & RAM_SHARED, block->flags & RAM_PMEM);
  1641. if (area == MAP_FAILED) {
  1642. error_setg_errno(errp, errno,
  1643. "unable to map backing store for guest RAM");
  1644. return NULL;
  1645. }
  1646. if (mem_prealloc) {
  1647. os_mem_prealloc(fd, area, memory, smp_cpus, errp);
  1648. if (errp && *errp) {
  1649. qemu_ram_munmap(fd, area, memory);
  1650. return NULL;
  1651. }
  1652. }
  1653. block->fd = fd;
  1654. return area;
  1655. }
  1656. #endif
  1657. /* Allocate space within the ram_addr_t space that governs the
  1658. * dirty bitmaps.
  1659. * Called with the ramlist lock held.
  1660. */
  1661. static ram_addr_t find_ram_offset(ram_addr_t size)
  1662. {
  1663. RAMBlock *block, *next_block;
  1664. ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
  1665. assert(size != 0); /* it would hand out same offset multiple times */
  1666. if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
  1667. return 0;
  1668. }
  1669. RAMBLOCK_FOREACH(block) {
  1670. ram_addr_t candidate, next = RAM_ADDR_MAX;
  1671. /* Align blocks to start on a 'long' in the bitmap
  1672. * which makes the bitmap sync'ing take the fast path.
  1673. */
  1674. candidate = block->offset + block->max_length;
  1675. candidate = ROUND_UP(candidate, BITS_PER_LONG << TARGET_PAGE_BITS);
  1676. /* Search for the closest following block
  1677. * and find the gap.
  1678. */
  1679. RAMBLOCK_FOREACH(next_block) {
  1680. if (next_block->offset >= candidate) {
  1681. next = MIN(next, next_block->offset);
  1682. }
  1683. }
  1684. /* If it fits remember our place and remember the size
  1685. * of gap, but keep going so that we might find a smaller
  1686. * gap to fill so avoiding fragmentation.
  1687. */
  1688. if (next - candidate >= size && next - candidate < mingap) {
  1689. offset = candidate;
  1690. mingap = next - candidate;
  1691. }
  1692. trace_find_ram_offset_loop(size, candidate, offset, next, mingap);
  1693. }
  1694. if (offset == RAM_ADDR_MAX) {
  1695. fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
  1696. (uint64_t)size);
  1697. abort();
  1698. }
  1699. trace_find_ram_offset(size, offset);
  1700. return offset;
  1701. }
  1702. static unsigned long last_ram_page(void)
  1703. {
  1704. RAMBlock *block;
  1705. ram_addr_t last = 0;
  1706. rcu_read_lock();
  1707. RAMBLOCK_FOREACH(block) {
  1708. last = MAX(last, block->offset + block->max_length);
  1709. }
  1710. rcu_read_unlock();
  1711. return last >> TARGET_PAGE_BITS;
  1712. }
  1713. static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
  1714. {
  1715. int ret;
  1716. /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
  1717. if (!machine_dump_guest_core(current_machine)) {
  1718. ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
  1719. if (ret) {
  1720. perror("qemu_madvise");
  1721. fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
  1722. "but dump_guest_core=off specified\n");
  1723. }
  1724. }
  1725. }
  1726. const char *qemu_ram_get_idstr(RAMBlock *rb)
  1727. {
  1728. return rb->idstr;
  1729. }
  1730. void *qemu_ram_get_host_addr(RAMBlock *rb)
  1731. {
  1732. return rb->host;
  1733. }
  1734. ram_addr_t qemu_ram_get_offset(RAMBlock *rb)
  1735. {
  1736. return rb->offset;
  1737. }
  1738. ram_addr_t qemu_ram_get_used_length(RAMBlock *rb)
  1739. {
  1740. return rb->used_length;
  1741. }
  1742. bool qemu_ram_is_shared(RAMBlock *rb)
  1743. {
  1744. return rb->flags & RAM_SHARED;
  1745. }
  1746. /* Note: Only set at the start of postcopy */
  1747. bool qemu_ram_is_uf_zeroable(RAMBlock *rb)
  1748. {
  1749. return rb->flags & RAM_UF_ZEROPAGE;
  1750. }
  1751. void qemu_ram_set_uf_zeroable(RAMBlock *rb)
  1752. {
  1753. rb->flags |= RAM_UF_ZEROPAGE;
  1754. }
  1755. bool qemu_ram_is_migratable(RAMBlock *rb)
  1756. {
  1757. return rb->flags & RAM_MIGRATABLE;
  1758. }
  1759. void qemu_ram_set_migratable(RAMBlock *rb)
  1760. {
  1761. rb->flags |= RAM_MIGRATABLE;
  1762. }
  1763. void qemu_ram_unset_migratable(RAMBlock *rb)
  1764. {
  1765. rb->flags &= ~RAM_MIGRATABLE;
  1766. }
  1767. /* Called with iothread lock held. */
  1768. void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
  1769. {
  1770. RAMBlock *block;
  1771. assert(new_block);
  1772. assert(!new_block->idstr[0]);
  1773. if (dev) {
  1774. char *id = qdev_get_dev_path(dev);
  1775. if (id) {
  1776. snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
  1777. g_free(id);
  1778. }
  1779. }
  1780. pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
  1781. rcu_read_lock();
  1782. RAMBLOCK_FOREACH(block) {
  1783. if (block != new_block &&
  1784. !strcmp(block->idstr, new_block->idstr)) {
  1785. fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
  1786. new_block->idstr);
  1787. abort();
  1788. }
  1789. }
  1790. rcu_read_unlock();
  1791. }
  1792. /* Called with iothread lock held. */
  1793. void qemu_ram_unset_idstr(RAMBlock *block)
  1794. {
  1795. /* FIXME: arch_init.c assumes that this is not called throughout
  1796. * migration. Ignore the problem since hot-unplug during migration
  1797. * does not work anyway.
  1798. */
  1799. if (block) {
  1800. memset(block->idstr, 0, sizeof(block->idstr));
  1801. }
  1802. }
  1803. size_t qemu_ram_pagesize(RAMBlock *rb)
  1804. {
  1805. return rb->page_size;
  1806. }
  1807. /* Returns the largest size of page in use */
  1808. size_t qemu_ram_pagesize_largest(void)
  1809. {
  1810. RAMBlock *block;
  1811. size_t largest = 0;
  1812. RAMBLOCK_FOREACH(block) {
  1813. largest = MAX(largest, qemu_ram_pagesize(block));
  1814. }
  1815. return largest;
  1816. }
  1817. static int memory_try_enable_merging(void *addr, size_t len)
  1818. {
  1819. if (!machine_mem_merge(current_machine)) {
  1820. /* disabled by the user */
  1821. return 0;
  1822. }
  1823. return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
  1824. }
  1825. /* Only legal before guest might have detected the memory size: e.g. on
  1826. * incoming migration, or right after reset.
  1827. *
  1828. * As memory core doesn't know how is memory accessed, it is up to
  1829. * resize callback to update device state and/or add assertions to detect
  1830. * misuse, if necessary.
  1831. */
  1832. int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
  1833. {
  1834. assert(block);
  1835. newsize = HOST_PAGE_ALIGN(newsize);
  1836. if (block->used_length == newsize) {
  1837. return 0;
  1838. }
  1839. if (!(block->flags & RAM_RESIZEABLE)) {
  1840. error_setg_errno(errp, EINVAL,
  1841. "Length mismatch: %s: 0x" RAM_ADDR_FMT
  1842. " in != 0x" RAM_ADDR_FMT, block->idstr,
  1843. newsize, block->used_length);
  1844. return -EINVAL;
  1845. }
  1846. if (block->max_length < newsize) {
  1847. error_setg_errno(errp, EINVAL,
  1848. "Length too large: %s: 0x" RAM_ADDR_FMT
  1849. " > 0x" RAM_ADDR_FMT, block->idstr,
  1850. newsize, block->max_length);
  1851. return -EINVAL;
  1852. }
  1853. cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
  1854. block->used_length = newsize;
  1855. cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
  1856. DIRTY_CLIENTS_ALL);
  1857. memory_region_set_size(block->mr, newsize);
  1858. if (block->resized) {
  1859. block->resized(block->idstr, newsize, block->host);
  1860. }
  1861. return 0;
  1862. }
  1863. /* Called with ram_list.mutex held */
  1864. static void dirty_memory_extend(ram_addr_t old_ram_size,
  1865. ram_addr_t new_ram_size)
  1866. {
  1867. ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size,
  1868. DIRTY_MEMORY_BLOCK_SIZE);
  1869. ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size,
  1870. DIRTY_MEMORY_BLOCK_SIZE);
  1871. int i;
  1872. /* Only need to extend if block count increased */
  1873. if (new_num_blocks <= old_num_blocks) {
  1874. return;
  1875. }
  1876. for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
  1877. DirtyMemoryBlocks *old_blocks;
  1878. DirtyMemoryBlocks *new_blocks;
  1879. int j;
  1880. old_blocks = atomic_rcu_read(&ram_list.dirty_memory[i]);
  1881. new_blocks = g_malloc(sizeof(*new_blocks) +
  1882. sizeof(new_blocks->blocks[0]) * new_num_blocks);
  1883. if (old_num_blocks) {
  1884. memcpy(new_blocks->blocks, old_blocks->blocks,
  1885. old_num_blocks * sizeof(old_blocks->blocks[0]));
  1886. }
  1887. for (j = old_num_blocks; j < new_num_blocks; j++) {
  1888. new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
  1889. }
  1890. atomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
  1891. if (old_blocks) {
  1892. g_free_rcu(old_blocks, rcu);
  1893. }
  1894. }
  1895. }
  1896. static void ram_block_add(RAMBlock *new_block, Error **errp, bool shared)
  1897. {
  1898. RAMBlock *block;
  1899. RAMBlock *last_block = NULL;
  1900. ram_addr_t old_ram_size, new_ram_size;
  1901. Error *err = NULL;
  1902. old_ram_size = last_ram_page();
  1903. qemu_mutex_lock_ramlist();
  1904. new_block->offset = find_ram_offset(new_block->max_length);
  1905. if (!new_block->host) {
  1906. if (xen_enabled()) {
  1907. xen_ram_alloc(new_block->offset, new_block->max_length,
  1908. new_block->mr, &err);
  1909. if (err) {
  1910. error_propagate(errp, err);
  1911. qemu_mutex_unlock_ramlist();
  1912. return;
  1913. }
  1914. } else {
  1915. new_block->host = phys_mem_alloc(new_block->max_length,
  1916. &new_block->mr->align, shared);
  1917. if (!new_block->host) {
  1918. error_setg_errno(errp, errno,
  1919. "cannot set up guest memory '%s'",
  1920. memory_region_name(new_block->mr));
  1921. qemu_mutex_unlock_ramlist();
  1922. return;
  1923. }
  1924. memory_try_enable_merging(new_block->host, new_block->max_length);
  1925. }
  1926. }
  1927. new_ram_size = MAX(old_ram_size,
  1928. (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
  1929. if (new_ram_size > old_ram_size) {
  1930. dirty_memory_extend(old_ram_size, new_ram_size);
  1931. }
  1932. /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
  1933. * QLIST (which has an RCU-friendly variant) does not have insertion at
  1934. * tail, so save the last element in last_block.
  1935. */
  1936. RAMBLOCK_FOREACH(block) {
  1937. last_block = block;
  1938. if (block->max_length < new_block->max_length) {
  1939. break;
  1940. }
  1941. }
  1942. if (block) {
  1943. QLIST_INSERT_BEFORE_RCU(block, new_block, next);
  1944. } else if (last_block) {
  1945. QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
  1946. } else { /* list is empty */
  1947. QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
  1948. }
  1949. ram_list.mru_block = NULL;
  1950. /* Write list before version */
  1951. smp_wmb();
  1952. ram_list.version++;
  1953. qemu_mutex_unlock_ramlist();
  1954. cpu_physical_memory_set_dirty_range(new_block->offset,
  1955. new_block->used_length,
  1956. DIRTY_CLIENTS_ALL);
  1957. if (new_block->host) {
  1958. qemu_ram_setup_dump(new_block->host, new_block->max_length);
  1959. qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
  1960. /* MADV_DONTFORK is also needed by KVM in absence of synchronous MMU */
  1961. qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
  1962. ram_block_notify_add(new_block->host, new_block->max_length);
  1963. }
  1964. }
  1965. #ifdef CONFIG_POSIX
  1966. RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
  1967. uint32_t ram_flags, int fd,
  1968. Error **errp)
  1969. {
  1970. RAMBlock *new_block;
  1971. Error *local_err = NULL;
  1972. int64_t file_size;
  1973. /* Just support these ram flags by now. */
  1974. assert((ram_flags & ~(RAM_SHARED | RAM_PMEM)) == 0);
  1975. if (xen_enabled()) {
  1976. error_setg(errp, "-mem-path not supported with Xen");
  1977. return NULL;
  1978. }
  1979. if (kvm_enabled() && !kvm_has_sync_mmu()) {
  1980. error_setg(errp,
  1981. "host lacks kvm mmu notifiers, -mem-path unsupported");
  1982. return NULL;
  1983. }
  1984. if (phys_mem_alloc != qemu_anon_ram_alloc) {
  1985. /*
  1986. * file_ram_alloc() needs to allocate just like
  1987. * phys_mem_alloc, but we haven't bothered to provide
  1988. * a hook there.
  1989. */
  1990. error_setg(errp,
  1991. "-mem-path not supported with this accelerator");
  1992. return NULL;
  1993. }
  1994. size = HOST_PAGE_ALIGN(size);
  1995. file_size = get_file_size(fd);
  1996. if (file_size > 0 && file_size < size) {
  1997. error_setg(errp, "backing store %s size 0x%" PRIx64
  1998. " does not match 'size' option 0x" RAM_ADDR_FMT,
  1999. mem_path, file_size, size);
  2000. return NULL;
  2001. }
  2002. new_block = g_malloc0(sizeof(*new_block));
  2003. new_block->mr = mr;
  2004. new_block->used_length = size;
  2005. new_block->max_length = size;
  2006. new_block->flags = ram_flags;
  2007. new_block->host = file_ram_alloc(new_block, size, fd, !file_size, errp);
  2008. if (!new_block->host) {
  2009. g_free(new_block);
  2010. return NULL;
  2011. }
  2012. ram_block_add(new_block, &local_err, ram_flags & RAM_SHARED);
  2013. if (local_err) {
  2014. g_free(new_block);
  2015. error_propagate(errp, local_err);
  2016. return NULL;
  2017. }
  2018. return new_block;
  2019. }
  2020. RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
  2021. uint32_t ram_flags, const char *mem_path,
  2022. Error **errp)
  2023. {
  2024. int fd;
  2025. bool created;
  2026. RAMBlock *block;
  2027. fd = file_ram_open(mem_path, memory_region_name(mr), &created, errp);
  2028. if (fd < 0) {
  2029. return NULL;
  2030. }
  2031. block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, errp);
  2032. if (!block) {
  2033. if (created) {
  2034. unlink(mem_path);
  2035. }
  2036. close(fd);
  2037. return NULL;
  2038. }
  2039. return block;
  2040. }
  2041. #endif
  2042. static
  2043. RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
  2044. void (*resized)(const char*,
  2045. uint64_t length,
  2046. void *host),
  2047. void *host, bool resizeable, bool share,
  2048. MemoryRegion *mr, Error **errp)
  2049. {
  2050. RAMBlock *new_block;
  2051. Error *local_err = NULL;
  2052. size = HOST_PAGE_ALIGN(size);
  2053. max_size = HOST_PAGE_ALIGN(max_size);
  2054. new_block = g_malloc0(sizeof(*new_block));
  2055. new_block->mr = mr;
  2056. new_block->resized = resized;
  2057. new_block->used_length = size;
  2058. new_block->max_length = max_size;
  2059. assert(max_size >= size);
  2060. new_block->fd = -1;
  2061. new_block->page_size = getpagesize();
  2062. new_block->host = host;
  2063. if (host) {
  2064. new_block->flags |= RAM_PREALLOC;
  2065. }
  2066. if (resizeable) {
  2067. new_block->flags |= RAM_RESIZEABLE;
  2068. }
  2069. ram_block_add(new_block, &local_err, share);
  2070. if (local_err) {
  2071. g_free(new_block);
  2072. error_propagate(errp, local_err);
  2073. return NULL;
  2074. }
  2075. return new_block;
  2076. }
  2077. RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
  2078. MemoryRegion *mr, Error **errp)
  2079. {
  2080. return qemu_ram_alloc_internal(size, size, NULL, host, false,
  2081. false, mr, errp);
  2082. }
  2083. RAMBlock *qemu_ram_alloc(ram_addr_t size, bool share,
  2084. MemoryRegion *mr, Error **errp)
  2085. {
  2086. return qemu_ram_alloc_internal(size, size, NULL, NULL, false,
  2087. share, mr, errp);
  2088. }
  2089. RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
  2090. void (*resized)(const char*,
  2091. uint64_t length,
  2092. void *host),
  2093. MemoryRegion *mr, Error **errp)
  2094. {
  2095. return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true,
  2096. false, mr, errp);
  2097. }
  2098. static void reclaim_ramblock(RAMBlock *block)
  2099. {
  2100. if (block->flags & RAM_PREALLOC) {
  2101. ;
  2102. } else if (xen_enabled()) {
  2103. xen_invalidate_map_cache_entry(block->host);
  2104. #ifndef _WIN32
  2105. } else if (block->fd >= 0) {
  2106. qemu_ram_munmap(block->fd, block->host, block->max_length);
  2107. close(block->fd);
  2108. #endif
  2109. } else {
  2110. qemu_anon_ram_free(block->host, block->max_length);
  2111. }
  2112. g_free(block);
  2113. }
  2114. void qemu_ram_free(RAMBlock *block)
  2115. {
  2116. if (!block) {
  2117. return;
  2118. }
  2119. if (block->host) {
  2120. ram_block_notify_remove(block->host, block->max_length);
  2121. }
  2122. qemu_mutex_lock_ramlist();
  2123. QLIST_REMOVE_RCU(block, next);
  2124. ram_list.mru_block = NULL;
  2125. /* Write list before version */
  2126. smp_wmb();
  2127. ram_list.version++;
  2128. call_rcu(block, reclaim_ramblock, rcu);
  2129. qemu_mutex_unlock_ramlist();
  2130. }
  2131. #ifndef _WIN32
  2132. void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
  2133. {
  2134. RAMBlock *block;
  2135. ram_addr_t offset;
  2136. int flags;
  2137. void *area, *vaddr;
  2138. RAMBLOCK_FOREACH(block) {
  2139. offset = addr - block->offset;
  2140. if (offset < block->max_length) {
  2141. vaddr = ramblock_ptr(block, offset);
  2142. if (block->flags & RAM_PREALLOC) {
  2143. ;
  2144. } else if (xen_enabled()) {
  2145. abort();
  2146. } else {
  2147. flags = MAP_FIXED;
  2148. if (block->fd >= 0) {
  2149. flags |= (block->flags & RAM_SHARED ?
  2150. MAP_SHARED : MAP_PRIVATE);
  2151. area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
  2152. flags, block->fd, offset);
  2153. } else {
  2154. /*
  2155. * Remap needs to match alloc. Accelerators that
  2156. * set phys_mem_alloc never remap. If they did,
  2157. * we'd need a remap hook here.
  2158. */
  2159. assert(phys_mem_alloc == qemu_anon_ram_alloc);
  2160. flags |= MAP_PRIVATE | MAP_ANONYMOUS;
  2161. area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
  2162. flags, -1, 0);
  2163. }
  2164. if (area != vaddr) {
  2165. error_report("Could not remap addr: "
  2166. RAM_ADDR_FMT "@" RAM_ADDR_FMT "",
  2167. length, addr);
  2168. exit(1);
  2169. }
  2170. memory_try_enable_merging(vaddr, length);
  2171. qemu_ram_setup_dump(vaddr, length);
  2172. }
  2173. }
  2174. }
  2175. }
  2176. #endif /* !_WIN32 */
  2177. /* Return a host pointer to ram allocated with qemu_ram_alloc.
  2178. * This should not be used for general purpose DMA. Use address_space_map
  2179. * or address_space_rw instead. For local memory (e.g. video ram) that the
  2180. * device owns, use memory_region_get_ram_ptr.
  2181. *
  2182. * Called within RCU critical section.
  2183. */
  2184. void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
  2185. {
  2186. RAMBlock *block = ram_block;
  2187. if (block == NULL) {
  2188. block = qemu_get_ram_block(addr);
  2189. addr -= block->offset;
  2190. }
  2191. if (xen_enabled() && block->host == NULL) {
  2192. /* We need to check if the requested address is in the RAM
  2193. * because we don't want to map the entire memory in QEMU.
  2194. * In that case just map until the end of the page.
  2195. */
  2196. if (block->offset == 0) {
  2197. return xen_map_cache(addr, 0, 0, false);
  2198. }
  2199. block->host = xen_map_cache(block->offset, block->max_length, 1, false);
  2200. }
  2201. return ramblock_ptr(block, addr);
  2202. }
  2203. /* Return a host pointer to guest's ram. Similar to qemu_map_ram_ptr
  2204. * but takes a size argument.
  2205. *
  2206. * Called within RCU critical section.
  2207. */
  2208. static void *qemu_ram_ptr_length(RAMBlock *ram_block, ram_addr_t addr,
  2209. hwaddr *size, bool lock)
  2210. {
  2211. RAMBlock *block = ram_block;
  2212. if (*size == 0) {
  2213. return NULL;
  2214. }
  2215. if (block == NULL) {
  2216. block = qemu_get_ram_block(addr);
  2217. addr -= block->offset;
  2218. }
  2219. *size = MIN(*size, block->max_length - addr);
  2220. if (xen_enabled() && block->host == NULL) {
  2221. /* We need to check if the requested address is in the RAM
  2222. * because we don't want to map the entire memory in QEMU.
  2223. * In that case just map the requested area.
  2224. */
  2225. if (block->offset == 0) {
  2226. return xen_map_cache(addr, *size, lock, lock);
  2227. }
  2228. block->host = xen_map_cache(block->offset, block->max_length, 1, lock);
  2229. }
  2230. return ramblock_ptr(block, addr);
  2231. }
  2232. /* Return the offset of a hostpointer within a ramblock */
  2233. ram_addr_t qemu_ram_block_host_offset(RAMBlock *rb, void *host)
  2234. {
  2235. ram_addr_t res = (uint8_t *)host - (uint8_t *)rb->host;
  2236. assert((uintptr_t)host >= (uintptr_t)rb->host);
  2237. assert(res < rb->max_length);
  2238. return res;
  2239. }
  2240. /*
  2241. * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
  2242. * in that RAMBlock.
  2243. *
  2244. * ptr: Host pointer to look up
  2245. * round_offset: If true round the result offset down to a page boundary
  2246. * *ram_addr: set to result ram_addr
  2247. * *offset: set to result offset within the RAMBlock
  2248. *
  2249. * Returns: RAMBlock (or NULL if not found)
  2250. *
  2251. * By the time this function returns, the returned pointer is not protected
  2252. * by RCU anymore. If the caller is not within an RCU critical section and
  2253. * does not hold the iothread lock, it must have other means of protecting the
  2254. * pointer, such as a reference to the region that includes the incoming
  2255. * ram_addr_t.
  2256. */
  2257. RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
  2258. ram_addr_t *offset)
  2259. {
  2260. RAMBlock *block;
  2261. uint8_t *host = ptr;
  2262. if (xen_enabled()) {
  2263. ram_addr_t ram_addr;
  2264. rcu_read_lock();
  2265. ram_addr = xen_ram_addr_from_mapcache(ptr);
  2266. block = qemu_get_ram_block(ram_addr);
  2267. if (block) {
  2268. *offset = ram_addr - block->offset;
  2269. }
  2270. rcu_read_unlock();
  2271. return block;
  2272. }
  2273. rcu_read_lock();
  2274. block = atomic_rcu_read(&ram_list.mru_block);
  2275. if (block && block->host && host - block->host < block->max_length) {
  2276. goto found;
  2277. }
  2278. RAMBLOCK_FOREACH(block) {
  2279. /* This case append when the block is not mapped. */
  2280. if (block->host == NULL) {
  2281. continue;
  2282. }
  2283. if (host - block->host < block->max_length) {
  2284. goto found;
  2285. }
  2286. }
  2287. rcu_read_unlock();
  2288. return NULL;
  2289. found:
  2290. *offset = (host - block->host);
  2291. if (round_offset) {
  2292. *offset &= TARGET_PAGE_MASK;
  2293. }
  2294. rcu_read_unlock();
  2295. return block;
  2296. }
  2297. /*
  2298. * Finds the named RAMBlock
  2299. *
  2300. * name: The name of RAMBlock to find
  2301. *
  2302. * Returns: RAMBlock (or NULL if not found)
  2303. */
  2304. RAMBlock *qemu_ram_block_by_name(const char *name)
  2305. {
  2306. RAMBlock *block;
  2307. RAMBLOCK_FOREACH(block) {
  2308. if (!strcmp(name, block->idstr)) {
  2309. return block;
  2310. }
  2311. }
  2312. return NULL;
  2313. }
  2314. /* Some of the softmmu routines need to translate from a host pointer
  2315. (typically a TLB entry) back to a ram offset. */
  2316. ram_addr_t qemu_ram_addr_from_host(void *ptr)
  2317. {
  2318. RAMBlock *block;
  2319. ram_addr_t offset;
  2320. block = qemu_ram_block_from_host(ptr, false, &offset);
  2321. if (!block) {
  2322. return RAM_ADDR_INVALID;
  2323. }
  2324. return block->offset + offset;
  2325. }
  2326. /* Called within RCU critical section. */
  2327. void memory_notdirty_write_prepare(NotDirtyInfo *ndi,
  2328. CPUState *cpu,
  2329. vaddr mem_vaddr,
  2330. ram_addr_t ram_addr,
  2331. unsigned size)
  2332. {
  2333. ndi->cpu = cpu;
  2334. ndi->ram_addr = ram_addr;
  2335. ndi->mem_vaddr = mem_vaddr;
  2336. ndi->size = size;
  2337. ndi->pages = NULL;
  2338. assert(tcg_enabled());
  2339. if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
  2340. ndi->pages = page_collection_lock(ram_addr, ram_addr + size);
  2341. tb_invalidate_phys_page_fast(ndi->pages, ram_addr, size);
  2342. }
  2343. }
  2344. /* Called within RCU critical section. */
  2345. void memory_notdirty_write_complete(NotDirtyInfo *ndi)
  2346. {
  2347. if (ndi->pages) {
  2348. assert(tcg_enabled());
  2349. page_collection_unlock(ndi->pages);
  2350. ndi->pages = NULL;
  2351. }
  2352. /* Set both VGA and migration bits for simplicity and to remove
  2353. * the notdirty callback faster.
  2354. */
  2355. cpu_physical_memory_set_dirty_range(ndi->ram_addr, ndi->size,
  2356. DIRTY_CLIENTS_NOCODE);
  2357. /* we remove the notdirty callback only if the code has been
  2358. flushed */
  2359. if (!cpu_physical_memory_is_clean(ndi->ram_addr)) {
  2360. tlb_set_dirty(ndi->cpu, ndi->mem_vaddr);
  2361. }
  2362. }
  2363. /* Called within RCU critical section. */
  2364. static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
  2365. uint64_t val, unsigned size)
  2366. {
  2367. NotDirtyInfo ndi;
  2368. memory_notdirty_write_prepare(&ndi, current_cpu, current_cpu->mem_io_vaddr,
  2369. ram_addr, size);
  2370. stn_p(qemu_map_ram_ptr(NULL, ram_addr), size, val);
  2371. memory_notdirty_write_complete(&ndi);
  2372. }
  2373. static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
  2374. unsigned size, bool is_write,
  2375. MemTxAttrs attrs)
  2376. {
  2377. return is_write;
  2378. }
  2379. static const MemoryRegionOps notdirty_mem_ops = {
  2380. .write = notdirty_mem_write,
  2381. .valid.accepts = notdirty_mem_accepts,
  2382. .endianness = DEVICE_NATIVE_ENDIAN,
  2383. .valid = {
  2384. .min_access_size = 1,
  2385. .max_access_size = 8,
  2386. .unaligned = false,
  2387. },
  2388. .impl = {
  2389. .min_access_size = 1,
  2390. .max_access_size = 8,
  2391. .unaligned = false,
  2392. },
  2393. };
  2394. /* Generate a debug exception if a watchpoint has been hit. */
  2395. static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
  2396. {
  2397. CPUState *cpu = current_cpu;
  2398. CPUClass *cc = CPU_GET_CLASS(cpu);
  2399. target_ulong vaddr;
  2400. CPUWatchpoint *wp;
  2401. assert(tcg_enabled());
  2402. if (cpu->watchpoint_hit) {
  2403. /* We re-entered the check after replacing the TB. Now raise
  2404. * the debug interrupt so that is will trigger after the
  2405. * current instruction. */
  2406. cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
  2407. return;
  2408. }
  2409. vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
  2410. vaddr = cc->adjust_watchpoint_address(cpu, vaddr, len);
  2411. QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
  2412. if (cpu_watchpoint_address_matches(wp, vaddr, len)
  2413. && (wp->flags & flags)) {
  2414. if (flags == BP_MEM_READ) {
  2415. wp->flags |= BP_WATCHPOINT_HIT_READ;
  2416. } else {
  2417. wp->flags |= BP_WATCHPOINT_HIT_WRITE;
  2418. }
  2419. wp->hitaddr = vaddr;
  2420. wp->hitattrs = attrs;
  2421. if (!cpu->watchpoint_hit) {
  2422. if (wp->flags & BP_CPU &&
  2423. !cc->debug_check_watchpoint(cpu, wp)) {
  2424. wp->flags &= ~BP_WATCHPOINT_HIT;
  2425. continue;
  2426. }
  2427. cpu->watchpoint_hit = wp;
  2428. mmap_lock();
  2429. tb_check_watchpoint(cpu);
  2430. if (wp->flags & BP_STOP_BEFORE_ACCESS) {
  2431. cpu->exception_index = EXCP_DEBUG;
  2432. mmap_unlock();
  2433. cpu_loop_exit(cpu);
  2434. } else {
  2435. /* Force execution of one insn next time. */
  2436. cpu->cflags_next_tb = 1 | curr_cflags();
  2437. mmap_unlock();
  2438. cpu_loop_exit_noexc(cpu);
  2439. }
  2440. }
  2441. } else {
  2442. wp->flags &= ~BP_WATCHPOINT_HIT;
  2443. }
  2444. }
  2445. }
  2446. /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
  2447. so these check for a hit then pass through to the normal out-of-line
  2448. phys routines. */
  2449. static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
  2450. unsigned size, MemTxAttrs attrs)
  2451. {
  2452. MemTxResult res;
  2453. uint64_t data;
  2454. int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
  2455. AddressSpace *as = current_cpu->cpu_ases[asidx].as;
  2456. check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
  2457. switch (size) {
  2458. case 1:
  2459. data = address_space_ldub(as, addr, attrs, &res);
  2460. break;
  2461. case 2:
  2462. data = address_space_lduw(as, addr, attrs, &res);
  2463. break;
  2464. case 4:
  2465. data = address_space_ldl(as, addr, attrs, &res);
  2466. break;
  2467. case 8:
  2468. data = address_space_ldq(as, addr, attrs, &res);
  2469. break;
  2470. default: abort();
  2471. }
  2472. *pdata = data;
  2473. return res;
  2474. }
  2475. static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
  2476. uint64_t val, unsigned size,
  2477. MemTxAttrs attrs)
  2478. {
  2479. MemTxResult res;
  2480. int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
  2481. AddressSpace *as = current_cpu->cpu_ases[asidx].as;
  2482. check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
  2483. switch (size) {
  2484. case 1:
  2485. address_space_stb(as, addr, val, attrs, &res);
  2486. break;
  2487. case 2:
  2488. address_space_stw(as, addr, val, attrs, &res);
  2489. break;
  2490. case 4:
  2491. address_space_stl(as, addr, val, attrs, &res);
  2492. break;
  2493. case 8:
  2494. address_space_stq(as, addr, val, attrs, &res);
  2495. break;
  2496. default: abort();
  2497. }
  2498. return res;
  2499. }
  2500. static const MemoryRegionOps watch_mem_ops = {
  2501. .read_with_attrs = watch_mem_read,
  2502. .write_with_attrs = watch_mem_write,
  2503. .endianness = DEVICE_NATIVE_ENDIAN,
  2504. .valid = {
  2505. .min_access_size = 1,
  2506. .max_access_size = 8,
  2507. .unaligned = false,
  2508. },
  2509. .impl = {
  2510. .min_access_size = 1,
  2511. .max_access_size = 8,
  2512. .unaligned = false,
  2513. },
  2514. };
  2515. static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
  2516. MemTxAttrs attrs, uint8_t *buf, hwaddr len);
  2517. static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
  2518. const uint8_t *buf, hwaddr len);
  2519. static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
  2520. bool is_write, MemTxAttrs attrs);
  2521. static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
  2522. unsigned len, MemTxAttrs attrs)
  2523. {
  2524. subpage_t *subpage = opaque;
  2525. uint8_t buf[8];
  2526. MemTxResult res;
  2527. #if defined(DEBUG_SUBPAGE)
  2528. printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
  2529. subpage, len, addr);
  2530. #endif
  2531. res = flatview_read(subpage->fv, addr + subpage->base, attrs, buf, len);
  2532. if (res) {
  2533. return res;
  2534. }
  2535. *data = ldn_p(buf, len);
  2536. return MEMTX_OK;
  2537. }
  2538. static MemTxResult subpage_write(void *opaque, hwaddr addr,
  2539. uint64_t value, unsigned len, MemTxAttrs attrs)
  2540. {
  2541. subpage_t *subpage = opaque;
  2542. uint8_t buf[8];
  2543. #if defined(DEBUG_SUBPAGE)
  2544. printf("%s: subpage %p len %u addr " TARGET_FMT_plx
  2545. " value %"PRIx64"\n",
  2546. __func__, subpage, len, addr, value);
  2547. #endif
  2548. stn_p(buf, len, value);
  2549. return flatview_write(subpage->fv, addr + subpage->base, attrs, buf, len);
  2550. }
  2551. static bool subpage_accepts(void *opaque, hwaddr addr,
  2552. unsigned len, bool is_write,
  2553. MemTxAttrs attrs)
  2554. {
  2555. subpage_t *subpage = opaque;
  2556. #if defined(DEBUG_SUBPAGE)
  2557. printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
  2558. __func__, subpage, is_write ? 'w' : 'r', len, addr);
  2559. #endif
  2560. return flatview_access_valid(subpage->fv, addr + subpage->base,
  2561. len, is_write, attrs);
  2562. }
  2563. static const MemoryRegionOps subpage_ops = {
  2564. .read_with_attrs = subpage_read,
  2565. .write_with_attrs = subpage_write,
  2566. .impl.min_access_size = 1,
  2567. .impl.max_access_size = 8,
  2568. .valid.min_access_size = 1,
  2569. .valid.max_access_size = 8,
  2570. .valid.accepts = subpage_accepts,
  2571. .endianness = DEVICE_NATIVE_ENDIAN,
  2572. };
  2573. static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
  2574. uint16_t section)
  2575. {
  2576. int idx, eidx;
  2577. if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
  2578. return -1;
  2579. idx = SUBPAGE_IDX(start);
  2580. eidx = SUBPAGE_IDX(end);
  2581. #if defined(DEBUG_SUBPAGE)
  2582. printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
  2583. __func__, mmio, start, end, idx, eidx, section);
  2584. #endif
  2585. for (; idx <= eidx; idx++) {
  2586. mmio->sub_section[idx] = section;
  2587. }
  2588. return 0;
  2589. }
  2590. static subpage_t *subpage_init(FlatView *fv, hwaddr base)
  2591. {
  2592. subpage_t *mmio;
  2593. mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t));
  2594. mmio->fv = fv;
  2595. mmio->base = base;
  2596. memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
  2597. NULL, TARGET_PAGE_SIZE);
  2598. mmio->iomem.subpage = true;
  2599. #if defined(DEBUG_SUBPAGE)
  2600. printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
  2601. mmio, base, TARGET_PAGE_SIZE);
  2602. #endif
  2603. subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
  2604. return mmio;
  2605. }
  2606. static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr)
  2607. {
  2608. assert(fv);
  2609. MemoryRegionSection section = {
  2610. .fv = fv,
  2611. .mr = mr,
  2612. .offset_within_address_space = 0,
  2613. .offset_within_region = 0,
  2614. .size = int128_2_64(),
  2615. };
  2616. return phys_section_add(map, &section);
  2617. }
  2618. static void readonly_mem_write(void *opaque, hwaddr addr,
  2619. uint64_t val, unsigned size)
  2620. {
  2621. /* Ignore any write to ROM. */
  2622. }
  2623. static bool readonly_mem_accepts(void *opaque, hwaddr addr,
  2624. unsigned size, bool is_write,
  2625. MemTxAttrs attrs)
  2626. {
  2627. return is_write;
  2628. }
  2629. /* This will only be used for writes, because reads are special cased
  2630. * to directly access the underlying host ram.
  2631. */
  2632. static const MemoryRegionOps readonly_mem_ops = {
  2633. .write = readonly_mem_write,
  2634. .valid.accepts = readonly_mem_accepts,
  2635. .endianness = DEVICE_NATIVE_ENDIAN,
  2636. .valid = {
  2637. .min_access_size = 1,
  2638. .max_access_size = 8,
  2639. .unaligned = false,
  2640. },
  2641. .impl = {
  2642. .min_access_size = 1,
  2643. .max_access_size = 8,
  2644. .unaligned = false,
  2645. },
  2646. };
  2647. MemoryRegionSection *iotlb_to_section(CPUState *cpu,
  2648. hwaddr index, MemTxAttrs attrs)
  2649. {
  2650. int asidx = cpu_asidx_from_attrs(cpu, attrs);
  2651. CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
  2652. AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
  2653. MemoryRegionSection *sections = d->map.sections;
  2654. return &sections[index & ~TARGET_PAGE_MASK];
  2655. }
  2656. static void io_mem_init(void)
  2657. {
  2658. memory_region_init_io(&io_mem_rom, NULL, &readonly_mem_ops,
  2659. NULL, NULL, UINT64_MAX);
  2660. memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
  2661. NULL, UINT64_MAX);
  2662. /* io_mem_notdirty calls tb_invalidate_phys_page_fast,
  2663. * which can be called without the iothread mutex.
  2664. */
  2665. memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
  2666. NULL, UINT64_MAX);
  2667. memory_region_clear_global_locking(&io_mem_notdirty);
  2668. memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
  2669. NULL, UINT64_MAX);
  2670. }
  2671. AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv)
  2672. {
  2673. AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
  2674. uint16_t n;
  2675. n = dummy_section(&d->map, fv, &io_mem_unassigned);
  2676. assert(n == PHYS_SECTION_UNASSIGNED);
  2677. n = dummy_section(&d->map, fv, &io_mem_notdirty);
  2678. assert(n == PHYS_SECTION_NOTDIRTY);
  2679. n = dummy_section(&d->map, fv, &io_mem_rom);
  2680. assert(n == PHYS_SECTION_ROM);
  2681. n = dummy_section(&d->map, fv, &io_mem_watch);
  2682. assert(n == PHYS_SECTION_WATCH);
  2683. d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
  2684. return d;
  2685. }
  2686. void address_space_dispatch_free(AddressSpaceDispatch *d)
  2687. {
  2688. phys_sections_free(&d->map);
  2689. g_free(d);
  2690. }
  2691. static void tcg_commit(MemoryListener *listener)
  2692. {
  2693. CPUAddressSpace *cpuas;
  2694. AddressSpaceDispatch *d;
  2695. assert(tcg_enabled());
  2696. /* since each CPU stores ram addresses in its TLB cache, we must
  2697. reset the modified entries */
  2698. cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
  2699. cpu_reloading_memory_map();
  2700. /* The CPU and TLB are protected by the iothread lock.
  2701. * We reload the dispatch pointer now because cpu_reloading_memory_map()
  2702. * may have split the RCU critical section.
  2703. */
  2704. d = address_space_to_dispatch(cpuas->as);
  2705. atomic_rcu_set(&cpuas->memory_dispatch, d);
  2706. tlb_flush(cpuas->cpu);
  2707. }
  2708. static void memory_map_init(void)
  2709. {
  2710. system_memory = g_malloc(sizeof(*system_memory));
  2711. memory_region_init(system_memory, NULL, "system", UINT64_MAX);
  2712. address_space_init(&address_space_memory, system_memory, "memory");
  2713. system_io = g_malloc(sizeof(*system_io));
  2714. memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
  2715. 65536);
  2716. address_space_init(&address_space_io, system_io, "I/O");
  2717. }
  2718. MemoryRegion *get_system_memory(void)
  2719. {
  2720. return system_memory;
  2721. }
  2722. MemoryRegion *get_system_io(void)
  2723. {
  2724. return system_io;
  2725. }
  2726. #endif /* !defined(CONFIG_USER_ONLY) */
  2727. /* physical memory access (slow version, mainly for debug) */
  2728. #if defined(CONFIG_USER_ONLY)
  2729. int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
  2730. uint8_t *buf, target_ulong len, int is_write)
  2731. {
  2732. int flags;
  2733. target_ulong l, page;
  2734. void * p;
  2735. while (len > 0) {
  2736. page = addr & TARGET_PAGE_MASK;
  2737. l = (page + TARGET_PAGE_SIZE) - addr;
  2738. if (l > len)
  2739. l = len;
  2740. flags = page_get_flags(page);
  2741. if (!(flags & PAGE_VALID))
  2742. return -1;
  2743. if (is_write) {
  2744. if (!(flags & PAGE_WRITE))
  2745. return -1;
  2746. /* XXX: this code should not depend on lock_user */
  2747. if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
  2748. return -1;
  2749. memcpy(p, buf, l);
  2750. unlock_user(p, addr, l);
  2751. } else {
  2752. if (!(flags & PAGE_READ))
  2753. return -1;
  2754. /* XXX: this code should not depend on lock_user */
  2755. if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
  2756. return -1;
  2757. memcpy(buf, p, l);
  2758. unlock_user(p, addr, 0);
  2759. }
  2760. len -= l;
  2761. buf += l;
  2762. addr += l;
  2763. }
  2764. return 0;
  2765. }
  2766. #else
  2767. static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
  2768. hwaddr length)
  2769. {
  2770. uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
  2771. addr += memory_region_get_ram_addr(mr);
  2772. /* No early return if dirty_log_mask is or becomes 0, because
  2773. * cpu_physical_memory_set_dirty_range will still call
  2774. * xen_modified_memory.
  2775. */
  2776. if (dirty_log_mask) {
  2777. dirty_log_mask =
  2778. cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
  2779. }
  2780. if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
  2781. assert(tcg_enabled());
  2782. tb_invalidate_phys_range(addr, addr + length);
  2783. dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
  2784. }
  2785. cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
  2786. }
  2787. void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size)
  2788. {
  2789. /*
  2790. * In principle this function would work on other memory region types too,
  2791. * but the ROM device use case is the only one where this operation is
  2792. * necessary. Other memory regions should use the
  2793. * address_space_read/write() APIs.
  2794. */
  2795. assert(memory_region_is_romd(mr));
  2796. invalidate_and_set_dirty(mr, addr, size);
  2797. }
  2798. static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
  2799. {
  2800. unsigned access_size_max = mr->ops->valid.max_access_size;
  2801. /* Regions are assumed to support 1-4 byte accesses unless
  2802. otherwise specified. */
  2803. if (access_size_max == 0) {
  2804. access_size_max = 4;
  2805. }
  2806. /* Bound the maximum access by the alignment of the address. */
  2807. if (!mr->ops->impl.unaligned) {
  2808. unsigned align_size_max = addr & -addr;
  2809. if (align_size_max != 0 && align_size_max < access_size_max) {
  2810. access_size_max = align_size_max;
  2811. }
  2812. }
  2813. /* Don't attempt accesses larger than the maximum. */
  2814. if (l > access_size_max) {
  2815. l = access_size_max;
  2816. }
  2817. l = pow2floor(l);
  2818. return l;
  2819. }
  2820. static bool prepare_mmio_access(MemoryRegion *mr)
  2821. {
  2822. bool unlocked = !qemu_mutex_iothread_locked();
  2823. bool release_lock = false;
  2824. if (unlocked && mr->global_locking) {
  2825. qemu_mutex_lock_iothread();
  2826. unlocked = false;
  2827. release_lock = true;
  2828. }
  2829. if (mr->flush_coalesced_mmio) {
  2830. if (unlocked) {
  2831. qemu_mutex_lock_iothread();
  2832. }
  2833. qemu_flush_coalesced_mmio_buffer();
  2834. if (unlocked) {
  2835. qemu_mutex_unlock_iothread();
  2836. }
  2837. }
  2838. return release_lock;
  2839. }
  2840. /* Called within RCU critical section. */
  2841. static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr,
  2842. MemTxAttrs attrs,
  2843. const uint8_t *buf,
  2844. hwaddr len, hwaddr addr1,
  2845. hwaddr l, MemoryRegion *mr)
  2846. {
  2847. uint8_t *ptr;
  2848. uint64_t val;
  2849. MemTxResult result = MEMTX_OK;
  2850. bool release_lock = false;
  2851. for (;;) {
  2852. if (!memory_access_is_direct(mr, true)) {
  2853. release_lock |= prepare_mmio_access(mr);
  2854. l = memory_access_size(mr, l, addr1);
  2855. /* XXX: could force current_cpu to NULL to avoid
  2856. potential bugs */
  2857. val = ldn_p(buf, l);
  2858. result |= memory_region_dispatch_write(mr, addr1, val, l, attrs);
  2859. } else {
  2860. /* RAM case */
  2861. ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false);
  2862. memcpy(ptr, buf, l);
  2863. invalidate_and_set_dirty(mr, addr1, l);
  2864. }
  2865. if (release_lock) {
  2866. qemu_mutex_unlock_iothread();
  2867. release_lock = false;
  2868. }
  2869. len -= l;
  2870. buf += l;
  2871. addr += l;
  2872. if (!len) {
  2873. break;
  2874. }
  2875. l = len;
  2876. mr = flatview_translate(fv, addr, &addr1, &l, true, attrs);
  2877. }
  2878. return result;
  2879. }
  2880. /* Called from RCU critical section. */
  2881. static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
  2882. const uint8_t *buf, hwaddr len)
  2883. {
  2884. hwaddr l;
  2885. hwaddr addr1;
  2886. MemoryRegion *mr;
  2887. MemTxResult result = MEMTX_OK;
  2888. l = len;
  2889. mr = flatview_translate(fv, addr, &addr1, &l, true, attrs);
  2890. result = flatview_write_continue(fv, addr, attrs, buf, len,
  2891. addr1, l, mr);
  2892. return result;
  2893. }
  2894. /* Called within RCU critical section. */
  2895. MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr,
  2896. MemTxAttrs attrs, uint8_t *buf,
  2897. hwaddr len, hwaddr addr1, hwaddr l,
  2898. MemoryRegion *mr)
  2899. {
  2900. uint8_t *ptr;
  2901. uint64_t val;
  2902. MemTxResult result = MEMTX_OK;
  2903. bool release_lock = false;
  2904. for (;;) {
  2905. if (!memory_access_is_direct(mr, false)) {
  2906. /* I/O case */
  2907. release_lock |= prepare_mmio_access(mr);
  2908. l = memory_access_size(mr, l, addr1);
  2909. result |= memory_region_dispatch_read(mr, addr1, &val, l, attrs);
  2910. stn_p(buf, l, val);
  2911. } else {
  2912. /* RAM case */
  2913. ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false);
  2914. memcpy(buf, ptr, l);
  2915. }
  2916. if (release_lock) {
  2917. qemu_mutex_unlock_iothread();
  2918. release_lock = false;
  2919. }
  2920. len -= l;
  2921. buf += l;
  2922. addr += l;
  2923. if (!len) {
  2924. break;
  2925. }
  2926. l = len;
  2927. mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
  2928. }
  2929. return result;
  2930. }
  2931. /* Called from RCU critical section. */
  2932. static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
  2933. MemTxAttrs attrs, uint8_t *buf, hwaddr len)
  2934. {
  2935. hwaddr l;
  2936. hwaddr addr1;
  2937. MemoryRegion *mr;
  2938. l = len;
  2939. mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
  2940. return flatview_read_continue(fv, addr, attrs, buf, len,
  2941. addr1, l, mr);
  2942. }
  2943. MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
  2944. MemTxAttrs attrs, uint8_t *buf, hwaddr len)
  2945. {
  2946. MemTxResult result = MEMTX_OK;
  2947. FlatView *fv;
  2948. if (len > 0) {
  2949. rcu_read_lock();
  2950. fv = address_space_to_flatview(as);
  2951. result = flatview_read(fv, addr, attrs, buf, len);
  2952. rcu_read_unlock();
  2953. }
  2954. return result;
  2955. }
  2956. MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
  2957. MemTxAttrs attrs,
  2958. const uint8_t *buf, hwaddr len)
  2959. {
  2960. MemTxResult result = MEMTX_OK;
  2961. FlatView *fv;
  2962. if (len > 0) {
  2963. rcu_read_lock();
  2964. fv = address_space_to_flatview(as);
  2965. result = flatview_write(fv, addr, attrs, buf, len);
  2966. rcu_read_unlock();
  2967. }
  2968. return result;
  2969. }
  2970. MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
  2971. uint8_t *buf, hwaddr len, bool is_write)
  2972. {
  2973. if (is_write) {
  2974. return address_space_write(as, addr, attrs, buf, len);
  2975. } else {
  2976. return address_space_read_full(as, addr, attrs, buf, len);
  2977. }
  2978. }
  2979. void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
  2980. hwaddr len, int is_write)
  2981. {
  2982. address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
  2983. buf, len, is_write);
  2984. }
  2985. enum write_rom_type {
  2986. WRITE_DATA,
  2987. FLUSH_CACHE,
  2988. };
  2989. static inline MemTxResult address_space_write_rom_internal(AddressSpace *as,
  2990. hwaddr addr,
  2991. MemTxAttrs attrs,
  2992. const uint8_t *buf,
  2993. hwaddr len,
  2994. enum write_rom_type type)
  2995. {
  2996. hwaddr l;
  2997. uint8_t *ptr;
  2998. hwaddr addr1;
  2999. MemoryRegion *mr;
  3000. rcu_read_lock();
  3001. while (len > 0) {
  3002. l = len;
  3003. mr = address_space_translate(as, addr, &addr1, &l, true, attrs);
  3004. if (!(memory_region_is_ram(mr) ||
  3005. memory_region_is_romd(mr))) {
  3006. l = memory_access_size(mr, l, addr1);
  3007. } else {
  3008. /* ROM/RAM case */
  3009. ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
  3010. switch (type) {
  3011. case WRITE_DATA:
  3012. memcpy(ptr, buf, l);
  3013. invalidate_and_set_dirty(mr, addr1, l);
  3014. break;
  3015. case FLUSH_CACHE:
  3016. flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
  3017. break;
  3018. }
  3019. }
  3020. len -= l;
  3021. buf += l;
  3022. addr += l;
  3023. }
  3024. rcu_read_unlock();
  3025. return MEMTX_OK;
  3026. }
  3027. /* used for ROM loading : can write in RAM and ROM */
  3028. MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr,
  3029. MemTxAttrs attrs,
  3030. const uint8_t *buf, hwaddr len)
  3031. {
  3032. return address_space_write_rom_internal(as, addr, attrs,
  3033. buf, len, WRITE_DATA);
  3034. }
  3035. void cpu_flush_icache_range(hwaddr start, hwaddr len)
  3036. {
  3037. /*
  3038. * This function should do the same thing as an icache flush that was
  3039. * triggered from within the guest. For TCG we are always cache coherent,
  3040. * so there is no need to flush anything. For KVM / Xen we need to flush
  3041. * the host's instruction cache at least.
  3042. */
  3043. if (tcg_enabled()) {
  3044. return;
  3045. }
  3046. address_space_write_rom_internal(&address_space_memory,
  3047. start, MEMTXATTRS_UNSPECIFIED,
  3048. NULL, len, FLUSH_CACHE);
  3049. }
  3050. typedef struct {
  3051. MemoryRegion *mr;
  3052. void *buffer;
  3053. hwaddr addr;
  3054. hwaddr len;
  3055. bool in_use;
  3056. } BounceBuffer;
  3057. static BounceBuffer bounce;
  3058. typedef struct MapClient {
  3059. QEMUBH *bh;
  3060. QLIST_ENTRY(MapClient) link;
  3061. } MapClient;
  3062. QemuMutex map_client_list_lock;
  3063. static QLIST_HEAD(, MapClient) map_client_list
  3064. = QLIST_HEAD_INITIALIZER(map_client_list);
  3065. static void cpu_unregister_map_client_do(MapClient *client)
  3066. {
  3067. QLIST_REMOVE(client, link);
  3068. g_free(client);
  3069. }
  3070. static void cpu_notify_map_clients_locked(void)
  3071. {
  3072. MapClient *client;
  3073. while (!QLIST_EMPTY(&map_client_list)) {
  3074. client = QLIST_FIRST(&map_client_list);
  3075. qemu_bh_schedule(client->bh);
  3076. cpu_unregister_map_client_do(client);
  3077. }
  3078. }
  3079. void cpu_register_map_client(QEMUBH *bh)
  3080. {
  3081. MapClient *client = g_malloc(sizeof(*client));
  3082. qemu_mutex_lock(&map_client_list_lock);
  3083. client->bh = bh;
  3084. QLIST_INSERT_HEAD(&map_client_list, client, link);
  3085. if (!atomic_read(&bounce.in_use)) {
  3086. cpu_notify_map_clients_locked();
  3087. }
  3088. qemu_mutex_unlock(&map_client_list_lock);
  3089. }
  3090. void cpu_exec_init_all(void)
  3091. {
  3092. qemu_mutex_init(&ram_list.mutex);
  3093. /* The data structures we set up here depend on knowing the page size,
  3094. * so no more changes can be made after this point.
  3095. * In an ideal world, nothing we did before we had finished the
  3096. * machine setup would care about the target page size, and we could
  3097. * do this much later, rather than requiring board models to state
  3098. * up front what their requirements are.
  3099. */
  3100. finalize_target_page_bits();
  3101. io_mem_init();
  3102. memory_map_init();
  3103. qemu_mutex_init(&map_client_list_lock);
  3104. }
  3105. void cpu_unregister_map_client(QEMUBH *bh)
  3106. {
  3107. MapClient *client;
  3108. qemu_mutex_lock(&map_client_list_lock);
  3109. QLIST_FOREACH(client, &map_client_list, link) {
  3110. if (client->bh == bh) {
  3111. cpu_unregister_map_client_do(client);
  3112. break;
  3113. }
  3114. }
  3115. qemu_mutex_unlock(&map_client_list_lock);
  3116. }
  3117. static void cpu_notify_map_clients(void)
  3118. {
  3119. qemu_mutex_lock(&map_client_list_lock);
  3120. cpu_notify_map_clients_locked();
  3121. qemu_mutex_unlock(&map_client_list_lock);
  3122. }
  3123. static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
  3124. bool is_write, MemTxAttrs attrs)
  3125. {
  3126. MemoryRegion *mr;
  3127. hwaddr l, xlat;
  3128. while (len > 0) {
  3129. l = len;
  3130. mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
  3131. if (!memory_access_is_direct(mr, is_write)) {
  3132. l = memory_access_size(mr, l, addr);
  3133. if (!memory_region_access_valid(mr, xlat, l, is_write, attrs)) {
  3134. return false;
  3135. }
  3136. }
  3137. len -= l;
  3138. addr += l;
  3139. }
  3140. return true;
  3141. }
  3142. bool address_space_access_valid(AddressSpace *as, hwaddr addr,
  3143. hwaddr len, bool is_write,
  3144. MemTxAttrs attrs)
  3145. {
  3146. FlatView *fv;
  3147. bool result;
  3148. rcu_read_lock();
  3149. fv = address_space_to_flatview(as);
  3150. result = flatview_access_valid(fv, addr, len, is_write, attrs);
  3151. rcu_read_unlock();
  3152. return result;
  3153. }
  3154. static hwaddr
  3155. flatview_extend_translation(FlatView *fv, hwaddr addr,
  3156. hwaddr target_len,
  3157. MemoryRegion *mr, hwaddr base, hwaddr len,
  3158. bool is_write, MemTxAttrs attrs)
  3159. {
  3160. hwaddr done = 0;
  3161. hwaddr xlat;
  3162. MemoryRegion *this_mr;
  3163. for (;;) {
  3164. target_len -= len;
  3165. addr += len;
  3166. done += len;
  3167. if (target_len == 0) {
  3168. return done;
  3169. }
  3170. len = target_len;
  3171. this_mr = flatview_translate(fv, addr, &xlat,
  3172. &len, is_write, attrs);
  3173. if (this_mr != mr || xlat != base + done) {
  3174. return done;
  3175. }
  3176. }
  3177. }
  3178. /* Map a physical memory region into a host virtual address.
  3179. * May map a subset of the requested range, given by and returned in *plen.
  3180. * May return NULL if resources needed to perform the mapping are exhausted.
  3181. * Use only for reads OR writes - not for read-modify-write operations.
  3182. * Use cpu_register_map_client() to know when retrying the map operation is
  3183. * likely to succeed.
  3184. */
  3185. void *address_space_map(AddressSpace *as,
  3186. hwaddr addr,
  3187. hwaddr *plen,
  3188. bool is_write,
  3189. MemTxAttrs attrs)
  3190. {
  3191. hwaddr len = *plen;
  3192. hwaddr l, xlat;
  3193. MemoryRegion *mr;
  3194. void *ptr;
  3195. FlatView *fv;
  3196. if (len == 0) {
  3197. return NULL;
  3198. }
  3199. l = len;
  3200. rcu_read_lock();
  3201. fv = address_space_to_flatview(as);
  3202. mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
  3203. if (!memory_access_is_direct(mr, is_write)) {
  3204. if (atomic_xchg(&bounce.in_use, true)) {
  3205. rcu_read_unlock();
  3206. return NULL;
  3207. }
  3208. /* Avoid unbounded allocations */
  3209. l = MIN(l, TARGET_PAGE_SIZE);
  3210. bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
  3211. bounce.addr = addr;
  3212. bounce.len = l;
  3213. memory_region_ref(mr);
  3214. bounce.mr = mr;
  3215. if (!is_write) {
  3216. flatview_read(fv, addr, MEMTXATTRS_UNSPECIFIED,
  3217. bounce.buffer, l);
  3218. }
  3219. rcu_read_unlock();
  3220. *plen = l;
  3221. return bounce.buffer;
  3222. }
  3223. memory_region_ref(mr);
  3224. *plen = flatview_extend_translation(fv, addr, len, mr, xlat,
  3225. l, is_write, attrs);
  3226. ptr = qemu_ram_ptr_length(mr->ram_block, xlat, plen, true);
  3227. rcu_read_unlock();
  3228. return ptr;
  3229. }
  3230. /* Unmaps a memory region previously mapped by address_space_map().
  3231. * Will also mark the memory as dirty if is_write == 1. access_len gives
  3232. * the amount of memory that was actually read or written by the caller.
  3233. */
  3234. void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
  3235. int is_write, hwaddr access_len)
  3236. {
  3237. if (buffer != bounce.buffer) {
  3238. MemoryRegion *mr;
  3239. ram_addr_t addr1;
  3240. mr = memory_region_from_host(buffer, &addr1);
  3241. assert(mr != NULL);
  3242. if (is_write) {
  3243. invalidate_and_set_dirty(mr, addr1, access_len);
  3244. }
  3245. if (xen_enabled()) {
  3246. xen_invalidate_map_cache_entry(buffer);
  3247. }
  3248. memory_region_unref(mr);
  3249. return;
  3250. }
  3251. if (is_write) {
  3252. address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
  3253. bounce.buffer, access_len);
  3254. }
  3255. qemu_vfree(bounce.buffer);
  3256. bounce.buffer = NULL;
  3257. memory_region_unref(bounce.mr);
  3258. atomic_mb_set(&bounce.in_use, false);
  3259. cpu_notify_map_clients();
  3260. }
  3261. void *cpu_physical_memory_map(hwaddr addr,
  3262. hwaddr *plen,
  3263. int is_write)
  3264. {
  3265. return address_space_map(&address_space_memory, addr, plen, is_write,
  3266. MEMTXATTRS_UNSPECIFIED);
  3267. }
  3268. void cpu_physical_memory_unmap(void *buffer, hwaddr len,
  3269. int is_write, hwaddr access_len)
  3270. {
  3271. return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
  3272. }
  3273. #define ARG1_DECL AddressSpace *as
  3274. #define ARG1 as
  3275. #define SUFFIX
  3276. #define TRANSLATE(...) address_space_translate(as, __VA_ARGS__)
  3277. #define RCU_READ_LOCK(...) rcu_read_lock()
  3278. #define RCU_READ_UNLOCK(...) rcu_read_unlock()
  3279. #include "memory_ldst.inc.c"
  3280. int64_t address_space_cache_init(MemoryRegionCache *cache,
  3281. AddressSpace *as,
  3282. hwaddr addr,
  3283. hwaddr len,
  3284. bool is_write)
  3285. {
  3286. AddressSpaceDispatch *d;
  3287. hwaddr l;
  3288. MemoryRegion *mr;
  3289. assert(len > 0);
  3290. l = len;
  3291. cache->fv = address_space_get_flatview(as);
  3292. d = flatview_to_dispatch(cache->fv);
  3293. cache->mrs = *address_space_translate_internal(d, addr, &cache->xlat, &l, true);
  3294. mr = cache->mrs.mr;
  3295. memory_region_ref(mr);
  3296. if (memory_access_is_direct(mr, is_write)) {
  3297. /* We don't care about the memory attributes here as we're only
  3298. * doing this if we found actual RAM, which behaves the same
  3299. * regardless of attributes; so UNSPECIFIED is fine.
  3300. */
  3301. l = flatview_extend_translation(cache->fv, addr, len, mr,
  3302. cache->xlat, l, is_write,
  3303. MEMTXATTRS_UNSPECIFIED);
  3304. cache->ptr = qemu_ram_ptr_length(mr->ram_block, cache->xlat, &l, true);
  3305. } else {
  3306. cache->ptr = NULL;
  3307. }
  3308. cache->len = l;
  3309. cache->is_write = is_write;
  3310. return l;
  3311. }
  3312. void address_space_cache_invalidate(MemoryRegionCache *cache,
  3313. hwaddr addr,
  3314. hwaddr access_len)
  3315. {
  3316. assert(cache->is_write);
  3317. if (likely(cache->ptr)) {
  3318. invalidate_and_set_dirty(cache->mrs.mr, addr + cache->xlat, access_len);
  3319. }
  3320. }
  3321. void address_space_cache_destroy(MemoryRegionCache *cache)
  3322. {
  3323. if (!cache->mrs.mr) {
  3324. return;
  3325. }
  3326. if (xen_enabled()) {
  3327. xen_invalidate_map_cache_entry(cache->ptr);
  3328. }
  3329. memory_region_unref(cache->mrs.mr);
  3330. flatview_unref(cache->fv);
  3331. cache->mrs.mr = NULL;
  3332. cache->fv = NULL;
  3333. }
  3334. /* Called from RCU critical section. This function has the same
  3335. * semantics as address_space_translate, but it only works on a
  3336. * predefined range of a MemoryRegion that was mapped with
  3337. * address_space_cache_init.
  3338. */
  3339. static inline MemoryRegion *address_space_translate_cached(
  3340. MemoryRegionCache *cache, hwaddr addr, hwaddr *xlat,
  3341. hwaddr *plen, bool is_write, MemTxAttrs attrs)
  3342. {
  3343. MemoryRegionSection section;
  3344. MemoryRegion *mr;
  3345. IOMMUMemoryRegion *iommu_mr;
  3346. AddressSpace *target_as;
  3347. assert(!cache->ptr);
  3348. *xlat = addr + cache->xlat;
  3349. mr = cache->mrs.mr;
  3350. iommu_mr = memory_region_get_iommu(mr);
  3351. if (!iommu_mr) {
  3352. /* MMIO region. */
  3353. return mr;
  3354. }
  3355. section = address_space_translate_iommu(iommu_mr, xlat, plen,
  3356. NULL, is_write, true,
  3357. &target_as, attrs);
  3358. return section.mr;
  3359. }
  3360. /* Called from RCU critical section. address_space_read_cached uses this
  3361. * out of line function when the target is an MMIO or IOMMU region.
  3362. */
  3363. void
  3364. address_space_read_cached_slow(MemoryRegionCache *cache, hwaddr addr,
  3365. void *buf, hwaddr len)
  3366. {
  3367. hwaddr addr1, l;
  3368. MemoryRegion *mr;
  3369. l = len;
  3370. mr = address_space_translate_cached(cache, addr, &addr1, &l, false,
  3371. MEMTXATTRS_UNSPECIFIED);
  3372. flatview_read_continue(cache->fv,
  3373. addr, MEMTXATTRS_UNSPECIFIED, buf, len,
  3374. addr1, l, mr);
  3375. }
  3376. /* Called from RCU critical section. address_space_write_cached uses this
  3377. * out of line function when the target is an MMIO or IOMMU region.
  3378. */
  3379. void
  3380. address_space_write_cached_slow(MemoryRegionCache *cache, hwaddr addr,
  3381. const void *buf, hwaddr len)
  3382. {
  3383. hwaddr addr1, l;
  3384. MemoryRegion *mr;
  3385. l = len;
  3386. mr = address_space_translate_cached(cache, addr, &addr1, &l, true,
  3387. MEMTXATTRS_UNSPECIFIED);
  3388. flatview_write_continue(cache->fv,
  3389. addr, MEMTXATTRS_UNSPECIFIED, buf, len,
  3390. addr1, l, mr);
  3391. }
  3392. #define ARG1_DECL MemoryRegionCache *cache
  3393. #define ARG1 cache
  3394. #define SUFFIX _cached_slow
  3395. #define TRANSLATE(...) address_space_translate_cached(cache, __VA_ARGS__)
  3396. #define RCU_READ_LOCK() ((void)0)
  3397. #define RCU_READ_UNLOCK() ((void)0)
  3398. #include "memory_ldst.inc.c"
  3399. /* virtual memory access for debug (includes writing to ROM) */
  3400. int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
  3401. uint8_t *buf, target_ulong len, int is_write)
  3402. {
  3403. hwaddr phys_addr;
  3404. target_ulong l, page;
  3405. cpu_synchronize_state(cpu);
  3406. while (len > 0) {
  3407. int asidx;
  3408. MemTxAttrs attrs;
  3409. page = addr & TARGET_PAGE_MASK;
  3410. phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
  3411. asidx = cpu_asidx_from_attrs(cpu, attrs);
  3412. /* if no physical page mapped, return an error */
  3413. if (phys_addr == -1)
  3414. return -1;
  3415. l = (page + TARGET_PAGE_SIZE) - addr;
  3416. if (l > len)
  3417. l = len;
  3418. phys_addr += (addr & ~TARGET_PAGE_MASK);
  3419. if (is_write) {
  3420. address_space_write_rom(cpu->cpu_ases[asidx].as, phys_addr,
  3421. attrs, buf, l);
  3422. } else {
  3423. address_space_rw(cpu->cpu_ases[asidx].as, phys_addr,
  3424. attrs, buf, l, 0);
  3425. }
  3426. len -= l;
  3427. buf += l;
  3428. addr += l;
  3429. }
  3430. return 0;
  3431. }
  3432. /*
  3433. * Allows code that needs to deal with migration bitmaps etc to still be built
  3434. * target independent.
  3435. */
  3436. size_t qemu_target_page_size(void)
  3437. {
  3438. return TARGET_PAGE_SIZE;
  3439. }
  3440. int qemu_target_page_bits(void)
  3441. {
  3442. return TARGET_PAGE_BITS;
  3443. }
  3444. int qemu_target_page_bits_min(void)
  3445. {
  3446. return TARGET_PAGE_BITS_MIN;
  3447. }
  3448. #endif
  3449. bool target_words_bigendian(void)
  3450. {
  3451. #if defined(TARGET_WORDS_BIGENDIAN)
  3452. return true;
  3453. #else
  3454. return false;
  3455. #endif
  3456. }
  3457. #ifndef CONFIG_USER_ONLY
  3458. bool cpu_physical_memory_is_io(hwaddr phys_addr)
  3459. {
  3460. MemoryRegion*mr;
  3461. hwaddr l = 1;
  3462. bool res;
  3463. rcu_read_lock();
  3464. mr = address_space_translate(&address_space_memory,
  3465. phys_addr, &phys_addr, &l, false,
  3466. MEMTXATTRS_UNSPECIFIED);
  3467. res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
  3468. rcu_read_unlock();
  3469. return res;
  3470. }
  3471. int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
  3472. {
  3473. RAMBlock *block;
  3474. int ret = 0;
  3475. rcu_read_lock();
  3476. RAMBLOCK_FOREACH(block) {
  3477. ret = func(block, opaque);
  3478. if (ret) {
  3479. break;
  3480. }
  3481. }
  3482. rcu_read_unlock();
  3483. return ret;
  3484. }
  3485. /*
  3486. * Unmap pages of memory from start to start+length such that
  3487. * they a) read as 0, b) Trigger whatever fault mechanism
  3488. * the OS provides for postcopy.
  3489. * The pages must be unmapped by the end of the function.
  3490. * Returns: 0 on success, none-0 on failure
  3491. *
  3492. */
  3493. int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length)
  3494. {
  3495. int ret = -1;
  3496. uint8_t *host_startaddr = rb->host + start;
  3497. if ((uintptr_t)host_startaddr & (rb->page_size - 1)) {
  3498. error_report("ram_block_discard_range: Unaligned start address: %p",
  3499. host_startaddr);
  3500. goto err;
  3501. }
  3502. if ((start + length) <= rb->used_length) {
  3503. bool need_madvise, need_fallocate;
  3504. uint8_t *host_endaddr = host_startaddr + length;
  3505. if ((uintptr_t)host_endaddr & (rb->page_size - 1)) {
  3506. error_report("ram_block_discard_range: Unaligned end address: %p",
  3507. host_endaddr);
  3508. goto err;
  3509. }
  3510. errno = ENOTSUP; /* If we are missing MADVISE etc */
  3511. /* The logic here is messy;
  3512. * madvise DONTNEED fails for hugepages
  3513. * fallocate works on hugepages and shmem
  3514. */
  3515. need_madvise = (rb->page_size == qemu_host_page_size);
  3516. need_fallocate = rb->fd != -1;
  3517. if (need_fallocate) {
  3518. /* For a file, this causes the area of the file to be zero'd
  3519. * if read, and for hugetlbfs also causes it to be unmapped
  3520. * so a userfault will trigger.
  3521. */
  3522. #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
  3523. ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
  3524. start, length);
  3525. if (ret) {
  3526. ret = -errno;
  3527. error_report("ram_block_discard_range: Failed to fallocate "
  3528. "%s:%" PRIx64 " +%zx (%d)",
  3529. rb->idstr, start, length, ret);
  3530. goto err;
  3531. }
  3532. #else
  3533. ret = -ENOSYS;
  3534. error_report("ram_block_discard_range: fallocate not available/file"
  3535. "%s:%" PRIx64 " +%zx (%d)",
  3536. rb->idstr, start, length, ret);
  3537. goto err;
  3538. #endif
  3539. }
  3540. if (need_madvise) {
  3541. /* For normal RAM this causes it to be unmapped,
  3542. * for shared memory it causes the local mapping to disappear
  3543. * and to fall back on the file contents (which we just
  3544. * fallocate'd away).
  3545. */
  3546. #if defined(CONFIG_MADVISE)
  3547. ret = madvise(host_startaddr, length, MADV_DONTNEED);
  3548. if (ret) {
  3549. ret = -errno;
  3550. error_report("ram_block_discard_range: Failed to discard range "
  3551. "%s:%" PRIx64 " +%zx (%d)",
  3552. rb->idstr, start, length, ret);
  3553. goto err;
  3554. }
  3555. #else
  3556. ret = -ENOSYS;
  3557. error_report("ram_block_discard_range: MADVISE not available"
  3558. "%s:%" PRIx64 " +%zx (%d)",
  3559. rb->idstr, start, length, ret);
  3560. goto err;
  3561. #endif
  3562. }
  3563. trace_ram_block_discard_range(rb->idstr, host_startaddr, length,
  3564. need_madvise, need_fallocate, ret);
  3565. } else {
  3566. error_report("ram_block_discard_range: Overrun block '%s' (%" PRIu64
  3567. "/%zx/" RAM_ADDR_FMT")",
  3568. rb->idstr, start, length, rb->used_length);
  3569. }
  3570. err:
  3571. return ret;
  3572. }
  3573. bool ramblock_is_pmem(RAMBlock *rb)
  3574. {
  3575. return rb->flags & RAM_PMEM;
  3576. }
  3577. #endif
  3578. void page_size_init(void)
  3579. {
  3580. /* NOTE: we can always suppose that qemu_host_page_size >=
  3581. TARGET_PAGE_SIZE */
  3582. if (qemu_host_page_size == 0) {
  3583. qemu_host_page_size = qemu_real_host_page_size;
  3584. }
  3585. if (qemu_host_page_size < TARGET_PAGE_SIZE) {
  3586. qemu_host_page_size = TARGET_PAGE_SIZE;
  3587. }
  3588. qemu_host_page_mask = -(intptr_t)qemu_host_page_size;
  3589. }
  3590. #if !defined(CONFIG_USER_ONLY)
  3591. static void mtree_print_phys_entries(int start, int end, int skip, int ptr)
  3592. {
  3593. if (start == end - 1) {
  3594. qemu_printf("\t%3d ", start);
  3595. } else {
  3596. qemu_printf("\t%3d..%-3d ", start, end - 1);
  3597. }
  3598. qemu_printf(" skip=%d ", skip);
  3599. if (ptr == PHYS_MAP_NODE_NIL) {
  3600. qemu_printf(" ptr=NIL");
  3601. } else if (!skip) {
  3602. qemu_printf(" ptr=#%d", ptr);
  3603. } else {
  3604. qemu_printf(" ptr=[%d]", ptr);
  3605. }
  3606. qemu_printf("\n");
  3607. }
  3608. #define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \
  3609. int128_sub((size), int128_one())) : 0)
  3610. void mtree_print_dispatch(AddressSpaceDispatch *d, MemoryRegion *root)
  3611. {
  3612. int i;
  3613. qemu_printf(" Dispatch\n");
  3614. qemu_printf(" Physical sections\n");
  3615. for (i = 0; i < d->map.sections_nb; ++i) {
  3616. MemoryRegionSection *s = d->map.sections + i;
  3617. const char *names[] = { " [unassigned]", " [not dirty]",
  3618. " [ROM]", " [watch]" };
  3619. qemu_printf(" #%d @" TARGET_FMT_plx ".." TARGET_FMT_plx
  3620. " %s%s%s%s%s",
  3621. i,
  3622. s->offset_within_address_space,
  3623. s->offset_within_address_space + MR_SIZE(s->mr->size),
  3624. s->mr->name ? s->mr->name : "(noname)",
  3625. i < ARRAY_SIZE(names) ? names[i] : "",
  3626. s->mr == root ? " [ROOT]" : "",
  3627. s == d->mru_section ? " [MRU]" : "",
  3628. s->mr->is_iommu ? " [iommu]" : "");
  3629. if (s->mr->alias) {
  3630. qemu_printf(" alias=%s", s->mr->alias->name ?
  3631. s->mr->alias->name : "noname");
  3632. }
  3633. qemu_printf("\n");
  3634. }
  3635. qemu_printf(" Nodes (%d bits per level, %d levels) ptr=[%d] skip=%d\n",
  3636. P_L2_BITS, P_L2_LEVELS, d->phys_map.ptr, d->phys_map.skip);
  3637. for (i = 0; i < d->map.nodes_nb; ++i) {
  3638. int j, jprev;
  3639. PhysPageEntry prev;
  3640. Node *n = d->map.nodes + i;
  3641. qemu_printf(" [%d]\n", i);
  3642. for (j = 0, jprev = 0, prev = *n[0]; j < ARRAY_SIZE(*n); ++j) {
  3643. PhysPageEntry *pe = *n + j;
  3644. if (pe->ptr == prev.ptr && pe->skip == prev.skip) {
  3645. continue;
  3646. }
  3647. mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
  3648. jprev = j;
  3649. prev = *pe;
  3650. }
  3651. if (jprev != ARRAY_SIZE(*n)) {
  3652. mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
  3653. }
  3654. }
  3655. }
  3656. #endif