You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

exec.c 124KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243
  1. /*
  2. * Virtual page mapping
  3. *
  4. * Copyright (c) 2003 Fabrice Bellard
  5. *
  6. * This library is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU Lesser General Public
  8. * License as published by the Free Software Foundation; either
  9. * version 2 of the License, or (at your option) any later version.
  10. *
  11. * This library is distributed in the hope that it will be useful,
  12. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. * Lesser General Public License for more details.
  15. *
  16. * You should have received a copy of the GNU Lesser General Public
  17. * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18. */
  19. #include "qemu/osdep.h"
  20. #include "qemu-common.h"
  21. #include "qapi/error.h"
  22. #include "qemu/cutils.h"
  23. #include "cpu.h"
  24. #include "exec/exec-all.h"
  25. #include "exec/target_page.h"
  26. #include "tcg.h"
  27. #include "hw/qdev-core.h"
  28. #include "hw/qdev-properties.h"
  29. #if !defined(CONFIG_USER_ONLY)
  30. #include "hw/boards.h"
  31. #include "hw/xen/xen.h"
  32. #endif
  33. #include "sysemu/kvm.h"
  34. #include "sysemu/sysemu.h"
  35. #include "sysemu/tcg.h"
  36. #include "qemu/timer.h"
  37. #include "qemu/config-file.h"
  38. #include "qemu/error-report.h"
  39. #include "qemu/qemu-print.h"
  40. #if defined(CONFIG_USER_ONLY)
  41. #include "qemu.h"
  42. #else /* !CONFIG_USER_ONLY */
  43. #include "hw/hw.h"
  44. #include "exec/memory.h"
  45. #include "exec/ioport.h"
  46. #include "sysemu/dma.h"
  47. #include "sysemu/numa.h"
  48. #include "sysemu/hw_accel.h"
  49. #include "exec/address-spaces.h"
  50. #include "sysemu/xen-mapcache.h"
  51. #include "trace-root.h"
  52. #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
  53. #include <linux/falloc.h>
  54. #endif
  55. #endif
  56. #include "qemu/rcu_queue.h"
  57. #include "qemu/main-loop.h"
  58. #include "translate-all.h"
  59. #include "sysemu/replay.h"
  60. #include "exec/memory-internal.h"
  61. #include "exec/ram_addr.h"
  62. #include "exec/log.h"
  63. #include "migration/vmstate.h"
  64. #include "qemu/range.h"
  65. #ifndef _WIN32
  66. #include "qemu/mmap-alloc.h"
  67. #endif
  68. #include "monitor/monitor.h"
  69. //#define DEBUG_SUBPAGE
  70. #if !defined(CONFIG_USER_ONLY)
  71. /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
  72. * are protected by the ramlist lock.
  73. */
  74. RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
  75. static MemoryRegion *system_memory;
  76. static MemoryRegion *system_io;
  77. AddressSpace address_space_io;
  78. AddressSpace address_space_memory;
  79. MemoryRegion io_mem_rom, io_mem_notdirty;
  80. static MemoryRegion io_mem_unassigned;
  81. #endif
  82. #ifdef TARGET_PAGE_BITS_VARY
  83. int target_page_bits;
  84. bool target_page_bits_decided;
  85. #endif
  86. CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
  87. /* current CPU in the current thread. It is only valid inside
  88. cpu_exec() */
  89. __thread CPUState *current_cpu;
  90. /* 0 = Do not count executed instructions.
  91. 1 = Precise instruction counting.
  92. 2 = Adaptive rate instruction counting. */
  93. int use_icount;
  94. uintptr_t qemu_host_page_size;
  95. intptr_t qemu_host_page_mask;
  96. bool set_preferred_target_page_bits(int bits)
  97. {
  98. /* The target page size is the lowest common denominator for all
  99. * the CPUs in the system, so we can only make it smaller, never
  100. * larger. And we can't make it smaller once we've committed to
  101. * a particular size.
  102. */
  103. #ifdef TARGET_PAGE_BITS_VARY
  104. assert(bits >= TARGET_PAGE_BITS_MIN);
  105. if (target_page_bits == 0 || target_page_bits > bits) {
  106. if (target_page_bits_decided) {
  107. return false;
  108. }
  109. target_page_bits = bits;
  110. }
  111. #endif
  112. return true;
  113. }
  114. #if !defined(CONFIG_USER_ONLY)
  115. static void finalize_target_page_bits(void)
  116. {
  117. #ifdef TARGET_PAGE_BITS_VARY
  118. if (target_page_bits == 0) {
  119. target_page_bits = TARGET_PAGE_BITS_MIN;
  120. }
  121. target_page_bits_decided = true;
  122. #endif
  123. }
  124. typedef struct PhysPageEntry PhysPageEntry;
  125. struct PhysPageEntry {
  126. /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
  127. uint32_t skip : 6;
  128. /* index into phys_sections (!skip) or phys_map_nodes (skip) */
  129. uint32_t ptr : 26;
  130. };
  131. #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
  132. /* Size of the L2 (and L3, etc) page tables. */
  133. #define ADDR_SPACE_BITS 64
  134. #define P_L2_BITS 9
  135. #define P_L2_SIZE (1 << P_L2_BITS)
  136. #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
  137. typedef PhysPageEntry Node[P_L2_SIZE];
  138. typedef struct PhysPageMap {
  139. struct rcu_head rcu;
  140. unsigned sections_nb;
  141. unsigned sections_nb_alloc;
  142. unsigned nodes_nb;
  143. unsigned nodes_nb_alloc;
  144. Node *nodes;
  145. MemoryRegionSection *sections;
  146. } PhysPageMap;
  147. struct AddressSpaceDispatch {
  148. MemoryRegionSection *mru_section;
  149. /* This is a multi-level map on the physical address space.
  150. * The bottom level has pointers to MemoryRegionSections.
  151. */
  152. PhysPageEntry phys_map;
  153. PhysPageMap map;
  154. };
  155. #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
  156. typedef struct subpage_t {
  157. MemoryRegion iomem;
  158. FlatView *fv;
  159. hwaddr base;
  160. uint16_t sub_section[];
  161. } subpage_t;
  162. #define PHYS_SECTION_UNASSIGNED 0
  163. #define PHYS_SECTION_NOTDIRTY 1
  164. #define PHYS_SECTION_ROM 2
  165. #define PHYS_SECTION_WATCH 3
  166. static void io_mem_init(void);
  167. static void memory_map_init(void);
  168. static void tcg_commit(MemoryListener *listener);
  169. static MemoryRegion io_mem_watch;
  170. /**
  171. * CPUAddressSpace: all the information a CPU needs about an AddressSpace
  172. * @cpu: the CPU whose AddressSpace this is
  173. * @as: the AddressSpace itself
  174. * @memory_dispatch: its dispatch pointer (cached, RCU protected)
  175. * @tcg_as_listener: listener for tracking changes to the AddressSpace
  176. */
  177. struct CPUAddressSpace {
  178. CPUState *cpu;
  179. AddressSpace *as;
  180. struct AddressSpaceDispatch *memory_dispatch;
  181. MemoryListener tcg_as_listener;
  182. };
  183. struct DirtyBitmapSnapshot {
  184. ram_addr_t start;
  185. ram_addr_t end;
  186. unsigned long dirty[];
  187. };
  188. #endif
  189. #if !defined(CONFIG_USER_ONLY)
  190. static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
  191. {
  192. static unsigned alloc_hint = 16;
  193. if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
  194. map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, alloc_hint);
  195. map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
  196. map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
  197. alloc_hint = map->nodes_nb_alloc;
  198. }
  199. }
  200. static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
  201. {
  202. unsigned i;
  203. uint32_t ret;
  204. PhysPageEntry e;
  205. PhysPageEntry *p;
  206. ret = map->nodes_nb++;
  207. p = map->nodes[ret];
  208. assert(ret != PHYS_MAP_NODE_NIL);
  209. assert(ret != map->nodes_nb_alloc);
  210. e.skip = leaf ? 0 : 1;
  211. e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
  212. for (i = 0; i < P_L2_SIZE; ++i) {
  213. memcpy(&p[i], &e, sizeof(e));
  214. }
  215. return ret;
  216. }
  217. static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
  218. hwaddr *index, hwaddr *nb, uint16_t leaf,
  219. int level)
  220. {
  221. PhysPageEntry *p;
  222. hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
  223. if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
  224. lp->ptr = phys_map_node_alloc(map, level == 0);
  225. }
  226. p = map->nodes[lp->ptr];
  227. lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
  228. while (*nb && lp < &p[P_L2_SIZE]) {
  229. if ((*index & (step - 1)) == 0 && *nb >= step) {
  230. lp->skip = 0;
  231. lp->ptr = leaf;
  232. *index += step;
  233. *nb -= step;
  234. } else {
  235. phys_page_set_level(map, lp, index, nb, leaf, level - 1);
  236. }
  237. ++lp;
  238. }
  239. }
  240. static void phys_page_set(AddressSpaceDispatch *d,
  241. hwaddr index, hwaddr nb,
  242. uint16_t leaf)
  243. {
  244. /* Wildly overreserve - it doesn't matter much. */
  245. phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
  246. phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
  247. }
  248. /* Compact a non leaf page entry. Simply detect that the entry has a single child,
  249. * and update our entry so we can skip it and go directly to the destination.
  250. */
  251. static void phys_page_compact(PhysPageEntry *lp, Node *nodes)
  252. {
  253. unsigned valid_ptr = P_L2_SIZE;
  254. int valid = 0;
  255. PhysPageEntry *p;
  256. int i;
  257. if (lp->ptr == PHYS_MAP_NODE_NIL) {
  258. return;
  259. }
  260. p = nodes[lp->ptr];
  261. for (i = 0; i < P_L2_SIZE; i++) {
  262. if (p[i].ptr == PHYS_MAP_NODE_NIL) {
  263. continue;
  264. }
  265. valid_ptr = i;
  266. valid++;
  267. if (p[i].skip) {
  268. phys_page_compact(&p[i], nodes);
  269. }
  270. }
  271. /* We can only compress if there's only one child. */
  272. if (valid != 1) {
  273. return;
  274. }
  275. assert(valid_ptr < P_L2_SIZE);
  276. /* Don't compress if it won't fit in the # of bits we have. */
  277. if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
  278. return;
  279. }
  280. lp->ptr = p[valid_ptr].ptr;
  281. if (!p[valid_ptr].skip) {
  282. /* If our only child is a leaf, make this a leaf. */
  283. /* By design, we should have made this node a leaf to begin with so we
  284. * should never reach here.
  285. * But since it's so simple to handle this, let's do it just in case we
  286. * change this rule.
  287. */
  288. lp->skip = 0;
  289. } else {
  290. lp->skip += p[valid_ptr].skip;
  291. }
  292. }
  293. void address_space_dispatch_compact(AddressSpaceDispatch *d)
  294. {
  295. if (d->phys_map.skip) {
  296. phys_page_compact(&d->phys_map, d->map.nodes);
  297. }
  298. }
  299. static inline bool section_covers_addr(const MemoryRegionSection *section,
  300. hwaddr addr)
  301. {
  302. /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
  303. * the section must cover the entire address space.
  304. */
  305. return int128_gethi(section->size) ||
  306. range_covers_byte(section->offset_within_address_space,
  307. int128_getlo(section->size), addr);
  308. }
  309. static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr addr)
  310. {
  311. PhysPageEntry lp = d->phys_map, *p;
  312. Node *nodes = d->map.nodes;
  313. MemoryRegionSection *sections = d->map.sections;
  314. hwaddr index = addr >> TARGET_PAGE_BITS;
  315. int i;
  316. for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
  317. if (lp.ptr == PHYS_MAP_NODE_NIL) {
  318. return &sections[PHYS_SECTION_UNASSIGNED];
  319. }
  320. p = nodes[lp.ptr];
  321. lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
  322. }
  323. if (section_covers_addr(&sections[lp.ptr], addr)) {
  324. return &sections[lp.ptr];
  325. } else {
  326. return &sections[PHYS_SECTION_UNASSIGNED];
  327. }
  328. }
  329. /* Called from RCU critical section */
  330. static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
  331. hwaddr addr,
  332. bool resolve_subpage)
  333. {
  334. MemoryRegionSection *section = atomic_read(&d->mru_section);
  335. subpage_t *subpage;
  336. if (!section || section == &d->map.sections[PHYS_SECTION_UNASSIGNED] ||
  337. !section_covers_addr(section, addr)) {
  338. section = phys_page_find(d, addr);
  339. atomic_set(&d->mru_section, section);
  340. }
  341. if (resolve_subpage && section->mr->subpage) {
  342. subpage = container_of(section->mr, subpage_t, iomem);
  343. section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
  344. }
  345. return section;
  346. }
  347. /* Called from RCU critical section */
  348. static MemoryRegionSection *
  349. address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
  350. hwaddr *plen, bool resolve_subpage)
  351. {
  352. MemoryRegionSection *section;
  353. MemoryRegion *mr;
  354. Int128 diff;
  355. section = address_space_lookup_region(d, addr, resolve_subpage);
  356. /* Compute offset within MemoryRegionSection */
  357. addr -= section->offset_within_address_space;
  358. /* Compute offset within MemoryRegion */
  359. *xlat = addr + section->offset_within_region;
  360. mr = section->mr;
  361. /* MMIO registers can be expected to perform full-width accesses based only
  362. * on their address, without considering adjacent registers that could
  363. * decode to completely different MemoryRegions. When such registers
  364. * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
  365. * regions overlap wildly. For this reason we cannot clamp the accesses
  366. * here.
  367. *
  368. * If the length is small (as is the case for address_space_ldl/stl),
  369. * everything works fine. If the incoming length is large, however,
  370. * the caller really has to do the clamping through memory_access_size.
  371. */
  372. if (memory_region_is_ram(mr)) {
  373. diff = int128_sub(section->size, int128_make64(addr));
  374. *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
  375. }
  376. return section;
  377. }
  378. /**
  379. * address_space_translate_iommu - translate an address through an IOMMU
  380. * memory region and then through the target address space.
  381. *
  382. * @iommu_mr: the IOMMU memory region that we start the translation from
  383. * @addr: the address to be translated through the MMU
  384. * @xlat: the translated address offset within the destination memory region.
  385. * It cannot be %NULL.
  386. * @plen_out: valid read/write length of the translated address. It
  387. * cannot be %NULL.
  388. * @page_mask_out: page mask for the translated address. This
  389. * should only be meaningful for IOMMU translated
  390. * addresses, since there may be huge pages that this bit
  391. * would tell. It can be %NULL if we don't care about it.
  392. * @is_write: whether the translation operation is for write
  393. * @is_mmio: whether this can be MMIO, set true if it can
  394. * @target_as: the address space targeted by the IOMMU
  395. * @attrs: transaction attributes
  396. *
  397. * This function is called from RCU critical section. It is the common
  398. * part of flatview_do_translate and address_space_translate_cached.
  399. */
  400. static MemoryRegionSection address_space_translate_iommu(IOMMUMemoryRegion *iommu_mr,
  401. hwaddr *xlat,
  402. hwaddr *plen_out,
  403. hwaddr *page_mask_out,
  404. bool is_write,
  405. bool is_mmio,
  406. AddressSpace **target_as,
  407. MemTxAttrs attrs)
  408. {
  409. MemoryRegionSection *section;
  410. hwaddr page_mask = (hwaddr)-1;
  411. do {
  412. hwaddr addr = *xlat;
  413. IOMMUMemoryRegionClass *imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
  414. int iommu_idx = 0;
  415. IOMMUTLBEntry iotlb;
  416. if (imrc->attrs_to_index) {
  417. iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
  418. }
  419. iotlb = imrc->translate(iommu_mr, addr, is_write ?
  420. IOMMU_WO : IOMMU_RO, iommu_idx);
  421. if (!(iotlb.perm & (1 << is_write))) {
  422. goto unassigned;
  423. }
  424. addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
  425. | (addr & iotlb.addr_mask));
  426. page_mask &= iotlb.addr_mask;
  427. *plen_out = MIN(*plen_out, (addr | iotlb.addr_mask) - addr + 1);
  428. *target_as = iotlb.target_as;
  429. section = address_space_translate_internal(
  430. address_space_to_dispatch(iotlb.target_as), addr, xlat,
  431. plen_out, is_mmio);
  432. iommu_mr = memory_region_get_iommu(section->mr);
  433. } while (unlikely(iommu_mr));
  434. if (page_mask_out) {
  435. *page_mask_out = page_mask;
  436. }
  437. return *section;
  438. unassigned:
  439. return (MemoryRegionSection) { .mr = &io_mem_unassigned };
  440. }
  441. /**
  442. * flatview_do_translate - translate an address in FlatView
  443. *
  444. * @fv: the flat view that we want to translate on
  445. * @addr: the address to be translated in above address space
  446. * @xlat: the translated address offset within memory region. It
  447. * cannot be @NULL.
  448. * @plen_out: valid read/write length of the translated address. It
  449. * can be @NULL when we don't care about it.
  450. * @page_mask_out: page mask for the translated address. This
  451. * should only be meaningful for IOMMU translated
  452. * addresses, since there may be huge pages that this bit
  453. * would tell. It can be @NULL if we don't care about it.
  454. * @is_write: whether the translation operation is for write
  455. * @is_mmio: whether this can be MMIO, set true if it can
  456. * @target_as: the address space targeted by the IOMMU
  457. * @attrs: memory transaction attributes
  458. *
  459. * This function is called from RCU critical section
  460. */
  461. static MemoryRegionSection flatview_do_translate(FlatView *fv,
  462. hwaddr addr,
  463. hwaddr *xlat,
  464. hwaddr *plen_out,
  465. hwaddr *page_mask_out,
  466. bool is_write,
  467. bool is_mmio,
  468. AddressSpace **target_as,
  469. MemTxAttrs attrs)
  470. {
  471. MemoryRegionSection *section;
  472. IOMMUMemoryRegion *iommu_mr;
  473. hwaddr plen = (hwaddr)(-1);
  474. if (!plen_out) {
  475. plen_out = &plen;
  476. }
  477. section = address_space_translate_internal(
  478. flatview_to_dispatch(fv), addr, xlat,
  479. plen_out, is_mmio);
  480. iommu_mr = memory_region_get_iommu(section->mr);
  481. if (unlikely(iommu_mr)) {
  482. return address_space_translate_iommu(iommu_mr, xlat,
  483. plen_out, page_mask_out,
  484. is_write, is_mmio,
  485. target_as, attrs);
  486. }
  487. if (page_mask_out) {
  488. /* Not behind an IOMMU, use default page size. */
  489. *page_mask_out = ~TARGET_PAGE_MASK;
  490. }
  491. return *section;
  492. }
  493. /* Called from RCU critical section */
  494. IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr,
  495. bool is_write, MemTxAttrs attrs)
  496. {
  497. MemoryRegionSection section;
  498. hwaddr xlat, page_mask;
  499. /*
  500. * This can never be MMIO, and we don't really care about plen,
  501. * but page mask.
  502. */
  503. section = flatview_do_translate(address_space_to_flatview(as), addr, &xlat,
  504. NULL, &page_mask, is_write, false, &as,
  505. attrs);
  506. /* Illegal translation */
  507. if (section.mr == &io_mem_unassigned) {
  508. goto iotlb_fail;
  509. }
  510. /* Convert memory region offset into address space offset */
  511. xlat += section.offset_within_address_space -
  512. section.offset_within_region;
  513. return (IOMMUTLBEntry) {
  514. .target_as = as,
  515. .iova = addr & ~page_mask,
  516. .translated_addr = xlat & ~page_mask,
  517. .addr_mask = page_mask,
  518. /* IOTLBs are for DMAs, and DMA only allows on RAMs. */
  519. .perm = IOMMU_RW,
  520. };
  521. iotlb_fail:
  522. return (IOMMUTLBEntry) {0};
  523. }
  524. /* Called from RCU critical section */
  525. MemoryRegion *flatview_translate(FlatView *fv, hwaddr addr, hwaddr *xlat,
  526. hwaddr *plen, bool is_write,
  527. MemTxAttrs attrs)
  528. {
  529. MemoryRegion *mr;
  530. MemoryRegionSection section;
  531. AddressSpace *as = NULL;
  532. /* This can be MMIO, so setup MMIO bit. */
  533. section = flatview_do_translate(fv, addr, xlat, plen, NULL,
  534. is_write, true, &as, attrs);
  535. mr = section.mr;
  536. if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
  537. hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
  538. *plen = MIN(page, *plen);
  539. }
  540. return mr;
  541. }
  542. typedef struct TCGIOMMUNotifier {
  543. IOMMUNotifier n;
  544. MemoryRegion *mr;
  545. CPUState *cpu;
  546. int iommu_idx;
  547. bool active;
  548. } TCGIOMMUNotifier;
  549. static void tcg_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
  550. {
  551. TCGIOMMUNotifier *notifier = container_of(n, TCGIOMMUNotifier, n);
  552. if (!notifier->active) {
  553. return;
  554. }
  555. tlb_flush(notifier->cpu);
  556. notifier->active = false;
  557. /* We leave the notifier struct on the list to avoid reallocating it later.
  558. * Generally the number of IOMMUs a CPU deals with will be small.
  559. * In any case we can't unregister the iommu notifier from a notify
  560. * callback.
  561. */
  562. }
  563. static void tcg_register_iommu_notifier(CPUState *cpu,
  564. IOMMUMemoryRegion *iommu_mr,
  565. int iommu_idx)
  566. {
  567. /* Make sure this CPU has an IOMMU notifier registered for this
  568. * IOMMU/IOMMU index combination, so that we can flush its TLB
  569. * when the IOMMU tells us the mappings we've cached have changed.
  570. */
  571. MemoryRegion *mr = MEMORY_REGION(iommu_mr);
  572. TCGIOMMUNotifier *notifier;
  573. int i;
  574. for (i = 0; i < cpu->iommu_notifiers->len; i++) {
  575. notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
  576. if (notifier->mr == mr && notifier->iommu_idx == iommu_idx) {
  577. break;
  578. }
  579. }
  580. if (i == cpu->iommu_notifiers->len) {
  581. /* Not found, add a new entry at the end of the array */
  582. cpu->iommu_notifiers = g_array_set_size(cpu->iommu_notifiers, i + 1);
  583. notifier = g_new0(TCGIOMMUNotifier, 1);
  584. g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i) = notifier;
  585. notifier->mr = mr;
  586. notifier->iommu_idx = iommu_idx;
  587. notifier->cpu = cpu;
  588. /* Rather than trying to register interest in the specific part
  589. * of the iommu's address space that we've accessed and then
  590. * expand it later as subsequent accesses touch more of it, we
  591. * just register interest in the whole thing, on the assumption
  592. * that iommu reconfiguration will be rare.
  593. */
  594. iommu_notifier_init(&notifier->n,
  595. tcg_iommu_unmap_notify,
  596. IOMMU_NOTIFIER_UNMAP,
  597. 0,
  598. HWADDR_MAX,
  599. iommu_idx);
  600. memory_region_register_iommu_notifier(notifier->mr, &notifier->n);
  601. }
  602. if (!notifier->active) {
  603. notifier->active = true;
  604. }
  605. }
  606. static void tcg_iommu_free_notifier_list(CPUState *cpu)
  607. {
  608. /* Destroy the CPU's notifier list */
  609. int i;
  610. TCGIOMMUNotifier *notifier;
  611. for (i = 0; i < cpu->iommu_notifiers->len; i++) {
  612. notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
  613. memory_region_unregister_iommu_notifier(notifier->mr, &notifier->n);
  614. g_free(notifier);
  615. }
  616. g_array_free(cpu->iommu_notifiers, true);
  617. }
  618. /* Called from RCU critical section */
  619. MemoryRegionSection *
  620. address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
  621. hwaddr *xlat, hwaddr *plen,
  622. MemTxAttrs attrs, int *prot)
  623. {
  624. MemoryRegionSection *section;
  625. IOMMUMemoryRegion *iommu_mr;
  626. IOMMUMemoryRegionClass *imrc;
  627. IOMMUTLBEntry iotlb;
  628. int iommu_idx;
  629. AddressSpaceDispatch *d = atomic_rcu_read(&cpu->cpu_ases[asidx].memory_dispatch);
  630. for (;;) {
  631. section = address_space_translate_internal(d, addr, &addr, plen, false);
  632. iommu_mr = memory_region_get_iommu(section->mr);
  633. if (!iommu_mr) {
  634. break;
  635. }
  636. imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
  637. iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
  638. tcg_register_iommu_notifier(cpu, iommu_mr, iommu_idx);
  639. /* We need all the permissions, so pass IOMMU_NONE so the IOMMU
  640. * doesn't short-cut its translation table walk.
  641. */
  642. iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, iommu_idx);
  643. addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
  644. | (addr & iotlb.addr_mask));
  645. /* Update the caller's prot bits to remove permissions the IOMMU
  646. * is giving us a failure response for. If we get down to no
  647. * permissions left at all we can give up now.
  648. */
  649. if (!(iotlb.perm & IOMMU_RO)) {
  650. *prot &= ~(PAGE_READ | PAGE_EXEC);
  651. }
  652. if (!(iotlb.perm & IOMMU_WO)) {
  653. *prot &= ~PAGE_WRITE;
  654. }
  655. if (!*prot) {
  656. goto translate_fail;
  657. }
  658. d = flatview_to_dispatch(address_space_to_flatview(iotlb.target_as));
  659. }
  660. assert(!memory_region_is_iommu(section->mr));
  661. *xlat = addr;
  662. return section;
  663. translate_fail:
  664. return &d->map.sections[PHYS_SECTION_UNASSIGNED];
  665. }
  666. #endif
  667. #if !defined(CONFIG_USER_ONLY)
  668. static int cpu_common_post_load(void *opaque, int version_id)
  669. {
  670. CPUState *cpu = opaque;
  671. /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
  672. version_id is increased. */
  673. cpu->interrupt_request &= ~0x01;
  674. tlb_flush(cpu);
  675. /* loadvm has just updated the content of RAM, bypassing the
  676. * usual mechanisms that ensure we flush TBs for writes to
  677. * memory we've translated code from. So we must flush all TBs,
  678. * which will now be stale.
  679. */
  680. tb_flush(cpu);
  681. return 0;
  682. }
  683. static int cpu_common_pre_load(void *opaque)
  684. {
  685. CPUState *cpu = opaque;
  686. cpu->exception_index = -1;
  687. return 0;
  688. }
  689. static bool cpu_common_exception_index_needed(void *opaque)
  690. {
  691. CPUState *cpu = opaque;
  692. return tcg_enabled() && cpu->exception_index != -1;
  693. }
  694. static const VMStateDescription vmstate_cpu_common_exception_index = {
  695. .name = "cpu_common/exception_index",
  696. .version_id = 1,
  697. .minimum_version_id = 1,
  698. .needed = cpu_common_exception_index_needed,
  699. .fields = (VMStateField[]) {
  700. VMSTATE_INT32(exception_index, CPUState),
  701. VMSTATE_END_OF_LIST()
  702. }
  703. };
  704. static bool cpu_common_crash_occurred_needed(void *opaque)
  705. {
  706. CPUState *cpu = opaque;
  707. return cpu->crash_occurred;
  708. }
  709. static const VMStateDescription vmstate_cpu_common_crash_occurred = {
  710. .name = "cpu_common/crash_occurred",
  711. .version_id = 1,
  712. .minimum_version_id = 1,
  713. .needed = cpu_common_crash_occurred_needed,
  714. .fields = (VMStateField[]) {
  715. VMSTATE_BOOL(crash_occurred, CPUState),
  716. VMSTATE_END_OF_LIST()
  717. }
  718. };
  719. const VMStateDescription vmstate_cpu_common = {
  720. .name = "cpu_common",
  721. .version_id = 1,
  722. .minimum_version_id = 1,
  723. .pre_load = cpu_common_pre_load,
  724. .post_load = cpu_common_post_load,
  725. .fields = (VMStateField[]) {
  726. VMSTATE_UINT32(halted, CPUState),
  727. VMSTATE_UINT32(interrupt_request, CPUState),
  728. VMSTATE_END_OF_LIST()
  729. },
  730. .subsections = (const VMStateDescription*[]) {
  731. &vmstate_cpu_common_exception_index,
  732. &vmstate_cpu_common_crash_occurred,
  733. NULL
  734. }
  735. };
  736. #endif
  737. CPUState *qemu_get_cpu(int index)
  738. {
  739. CPUState *cpu;
  740. CPU_FOREACH(cpu) {
  741. if (cpu->cpu_index == index) {
  742. return cpu;
  743. }
  744. }
  745. return NULL;
  746. }
  747. #if !defined(CONFIG_USER_ONLY)
  748. void cpu_address_space_init(CPUState *cpu, int asidx,
  749. const char *prefix, MemoryRegion *mr)
  750. {
  751. CPUAddressSpace *newas;
  752. AddressSpace *as = g_new0(AddressSpace, 1);
  753. char *as_name;
  754. assert(mr);
  755. as_name = g_strdup_printf("%s-%d", prefix, cpu->cpu_index);
  756. address_space_init(as, mr, as_name);
  757. g_free(as_name);
  758. /* Target code should have set num_ases before calling us */
  759. assert(asidx < cpu->num_ases);
  760. if (asidx == 0) {
  761. /* address space 0 gets the convenience alias */
  762. cpu->as = as;
  763. }
  764. /* KVM cannot currently support multiple address spaces. */
  765. assert(asidx == 0 || !kvm_enabled());
  766. if (!cpu->cpu_ases) {
  767. cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
  768. }
  769. newas = &cpu->cpu_ases[asidx];
  770. newas->cpu = cpu;
  771. newas->as = as;
  772. if (tcg_enabled()) {
  773. newas->tcg_as_listener.commit = tcg_commit;
  774. memory_listener_register(&newas->tcg_as_listener, as);
  775. }
  776. }
  777. AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
  778. {
  779. /* Return the AddressSpace corresponding to the specified index */
  780. return cpu->cpu_ases[asidx].as;
  781. }
  782. #endif
  783. void cpu_exec_unrealizefn(CPUState *cpu)
  784. {
  785. CPUClass *cc = CPU_GET_CLASS(cpu);
  786. cpu_list_remove(cpu);
  787. if (cc->vmsd != NULL) {
  788. vmstate_unregister(NULL, cc->vmsd, cpu);
  789. }
  790. if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
  791. vmstate_unregister(NULL, &vmstate_cpu_common, cpu);
  792. }
  793. #ifndef CONFIG_USER_ONLY
  794. tcg_iommu_free_notifier_list(cpu);
  795. #endif
  796. }
  797. Property cpu_common_props[] = {
  798. #ifndef CONFIG_USER_ONLY
  799. /* Create a memory property for softmmu CPU object,
  800. * so users can wire up its memory. (This can't go in qom/cpu.c
  801. * because that file is compiled only once for both user-mode
  802. * and system builds.) The default if no link is set up is to use
  803. * the system address space.
  804. */
  805. DEFINE_PROP_LINK("memory", CPUState, memory, TYPE_MEMORY_REGION,
  806. MemoryRegion *),
  807. #endif
  808. DEFINE_PROP_END_OF_LIST(),
  809. };
  810. void cpu_exec_initfn(CPUState *cpu)
  811. {
  812. cpu->as = NULL;
  813. cpu->num_ases = 0;
  814. #ifndef CONFIG_USER_ONLY
  815. cpu->thread_id = qemu_get_thread_id();
  816. cpu->memory = system_memory;
  817. object_ref(OBJECT(cpu->memory));
  818. #endif
  819. }
  820. void cpu_exec_realizefn(CPUState *cpu, Error **errp)
  821. {
  822. CPUClass *cc = CPU_GET_CLASS(cpu);
  823. static bool tcg_target_initialized;
  824. cpu_list_add(cpu);
  825. if (tcg_enabled() && !tcg_target_initialized) {
  826. tcg_target_initialized = true;
  827. cc->tcg_initialize();
  828. }
  829. tlb_init(cpu);
  830. #ifndef CONFIG_USER_ONLY
  831. if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
  832. vmstate_register(NULL, cpu->cpu_index, &vmstate_cpu_common, cpu);
  833. }
  834. if (cc->vmsd != NULL) {
  835. vmstate_register(NULL, cpu->cpu_index, cc->vmsd, cpu);
  836. }
  837. cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier *));
  838. #endif
  839. }
  840. const char *parse_cpu_option(const char *cpu_option)
  841. {
  842. ObjectClass *oc;
  843. CPUClass *cc;
  844. gchar **model_pieces;
  845. const char *cpu_type;
  846. model_pieces = g_strsplit(cpu_option, ",", 2);
  847. if (!model_pieces[0]) {
  848. error_report("-cpu option cannot be empty");
  849. exit(1);
  850. }
  851. oc = cpu_class_by_name(CPU_RESOLVING_TYPE, model_pieces[0]);
  852. if (oc == NULL) {
  853. error_report("unable to find CPU model '%s'", model_pieces[0]);
  854. g_strfreev(model_pieces);
  855. exit(EXIT_FAILURE);
  856. }
  857. cpu_type = object_class_get_name(oc);
  858. cc = CPU_CLASS(oc);
  859. cc->parse_features(cpu_type, model_pieces[1], &error_fatal);
  860. g_strfreev(model_pieces);
  861. return cpu_type;
  862. }
  863. #if defined(CONFIG_USER_ONLY)
  864. void tb_invalidate_phys_addr(target_ulong addr)
  865. {
  866. mmap_lock();
  867. tb_invalidate_phys_page_range(addr, addr + 1, 0);
  868. mmap_unlock();
  869. }
  870. static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
  871. {
  872. tb_invalidate_phys_addr(pc);
  873. }
  874. #else
  875. void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs)
  876. {
  877. ram_addr_t ram_addr;
  878. MemoryRegion *mr;
  879. hwaddr l = 1;
  880. if (!tcg_enabled()) {
  881. return;
  882. }
  883. rcu_read_lock();
  884. mr = address_space_translate(as, addr, &addr, &l, false, attrs);
  885. if (!(memory_region_is_ram(mr)
  886. || memory_region_is_romd(mr))) {
  887. rcu_read_unlock();
  888. return;
  889. }
  890. ram_addr = memory_region_get_ram_addr(mr) + addr;
  891. tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
  892. rcu_read_unlock();
  893. }
  894. static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
  895. {
  896. MemTxAttrs attrs;
  897. hwaddr phys = cpu_get_phys_page_attrs_debug(cpu, pc, &attrs);
  898. int asidx = cpu_asidx_from_attrs(cpu, attrs);
  899. if (phys != -1) {
  900. /* Locks grabbed by tb_invalidate_phys_addr */
  901. tb_invalidate_phys_addr(cpu->cpu_ases[asidx].as,
  902. phys | (pc & ~TARGET_PAGE_MASK), attrs);
  903. }
  904. }
  905. #endif
  906. #if defined(CONFIG_USER_ONLY)
  907. void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
  908. {
  909. }
  910. int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
  911. int flags)
  912. {
  913. return -ENOSYS;
  914. }
  915. void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
  916. {
  917. }
  918. int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
  919. int flags, CPUWatchpoint **watchpoint)
  920. {
  921. return -ENOSYS;
  922. }
  923. #else
  924. /* Add a watchpoint. */
  925. int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
  926. int flags, CPUWatchpoint **watchpoint)
  927. {
  928. CPUWatchpoint *wp;
  929. /* forbid ranges which are empty or run off the end of the address space */
  930. if (len == 0 || (addr + len - 1) < addr) {
  931. error_report("tried to set invalid watchpoint at %"
  932. VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
  933. return -EINVAL;
  934. }
  935. wp = g_malloc(sizeof(*wp));
  936. wp->vaddr = addr;
  937. wp->len = len;
  938. wp->flags = flags;
  939. /* keep all GDB-injected watchpoints in front */
  940. if (flags & BP_GDB) {
  941. QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
  942. } else {
  943. QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
  944. }
  945. tlb_flush_page(cpu, addr);
  946. if (watchpoint)
  947. *watchpoint = wp;
  948. return 0;
  949. }
  950. /* Remove a specific watchpoint. */
  951. int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
  952. int flags)
  953. {
  954. CPUWatchpoint *wp;
  955. QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
  956. if (addr == wp->vaddr && len == wp->len
  957. && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
  958. cpu_watchpoint_remove_by_ref(cpu, wp);
  959. return 0;
  960. }
  961. }
  962. return -ENOENT;
  963. }
  964. /* Remove a specific watchpoint by reference. */
  965. void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
  966. {
  967. QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
  968. tlb_flush_page(cpu, watchpoint->vaddr);
  969. g_free(watchpoint);
  970. }
  971. /* Remove all matching watchpoints. */
  972. void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
  973. {
  974. CPUWatchpoint *wp, *next;
  975. QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
  976. if (wp->flags & mask) {
  977. cpu_watchpoint_remove_by_ref(cpu, wp);
  978. }
  979. }
  980. }
  981. /* Return true if this watchpoint address matches the specified
  982. * access (ie the address range covered by the watchpoint overlaps
  983. * partially or completely with the address range covered by the
  984. * access).
  985. */
  986. static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
  987. vaddr addr,
  988. vaddr len)
  989. {
  990. /* We know the lengths are non-zero, but a little caution is
  991. * required to avoid errors in the case where the range ends
  992. * exactly at the top of the address space and so addr + len
  993. * wraps round to zero.
  994. */
  995. vaddr wpend = wp->vaddr + wp->len - 1;
  996. vaddr addrend = addr + len - 1;
  997. return !(addr > wpend || wp->vaddr > addrend);
  998. }
  999. #endif
  1000. /* Add a breakpoint. */
  1001. int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
  1002. CPUBreakpoint **breakpoint)
  1003. {
  1004. CPUBreakpoint *bp;
  1005. bp = g_malloc(sizeof(*bp));
  1006. bp->pc = pc;
  1007. bp->flags = flags;
  1008. /* keep all GDB-injected breakpoints in front */
  1009. if (flags & BP_GDB) {
  1010. QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
  1011. } else {
  1012. QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
  1013. }
  1014. breakpoint_invalidate(cpu, pc);
  1015. if (breakpoint) {
  1016. *breakpoint = bp;
  1017. }
  1018. return 0;
  1019. }
  1020. /* Remove a specific breakpoint. */
  1021. int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
  1022. {
  1023. CPUBreakpoint *bp;
  1024. QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
  1025. if (bp->pc == pc && bp->flags == flags) {
  1026. cpu_breakpoint_remove_by_ref(cpu, bp);
  1027. return 0;
  1028. }
  1029. }
  1030. return -ENOENT;
  1031. }
  1032. /* Remove a specific breakpoint by reference. */
  1033. void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
  1034. {
  1035. QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
  1036. breakpoint_invalidate(cpu, breakpoint->pc);
  1037. g_free(breakpoint);
  1038. }
  1039. /* Remove all matching breakpoints. */
  1040. void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
  1041. {
  1042. CPUBreakpoint *bp, *next;
  1043. QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
  1044. if (bp->flags & mask) {
  1045. cpu_breakpoint_remove_by_ref(cpu, bp);
  1046. }
  1047. }
  1048. }
  1049. /* enable or disable single step mode. EXCP_DEBUG is returned by the
  1050. CPU loop after each instruction */
  1051. void cpu_single_step(CPUState *cpu, int enabled)
  1052. {
  1053. if (cpu->singlestep_enabled != enabled) {
  1054. cpu->singlestep_enabled = enabled;
  1055. if (kvm_enabled()) {
  1056. kvm_update_guest_debug(cpu, 0);
  1057. } else {
  1058. /* must flush all the translated code to avoid inconsistencies */
  1059. /* XXX: only flush what is necessary */
  1060. tb_flush(cpu);
  1061. }
  1062. }
  1063. }
  1064. void cpu_abort(CPUState *cpu, const char *fmt, ...)
  1065. {
  1066. va_list ap;
  1067. va_list ap2;
  1068. va_start(ap, fmt);
  1069. va_copy(ap2, ap);
  1070. fprintf(stderr, "qemu: fatal: ");
  1071. vfprintf(stderr, fmt, ap);
  1072. fprintf(stderr, "\n");
  1073. cpu_dump_state(cpu, stderr, CPU_DUMP_FPU | CPU_DUMP_CCOP);
  1074. if (qemu_log_separate()) {
  1075. qemu_log_lock();
  1076. qemu_log("qemu: fatal: ");
  1077. qemu_log_vprintf(fmt, ap2);
  1078. qemu_log("\n");
  1079. log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
  1080. qemu_log_flush();
  1081. qemu_log_unlock();
  1082. qemu_log_close();
  1083. }
  1084. va_end(ap2);
  1085. va_end(ap);
  1086. replay_finish();
  1087. #if defined(CONFIG_USER_ONLY)
  1088. {
  1089. struct sigaction act;
  1090. sigfillset(&act.sa_mask);
  1091. act.sa_handler = SIG_DFL;
  1092. act.sa_flags = 0;
  1093. sigaction(SIGABRT, &act, NULL);
  1094. }
  1095. #endif
  1096. abort();
  1097. }
  1098. #if !defined(CONFIG_USER_ONLY)
  1099. /* Called from RCU critical section */
  1100. static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
  1101. {
  1102. RAMBlock *block;
  1103. block = atomic_rcu_read(&ram_list.mru_block);
  1104. if (block && addr - block->offset < block->max_length) {
  1105. return block;
  1106. }
  1107. RAMBLOCK_FOREACH(block) {
  1108. if (addr - block->offset < block->max_length) {
  1109. goto found;
  1110. }
  1111. }
  1112. fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
  1113. abort();
  1114. found:
  1115. /* It is safe to write mru_block outside the iothread lock. This
  1116. * is what happens:
  1117. *
  1118. * mru_block = xxx
  1119. * rcu_read_unlock()
  1120. * xxx removed from list
  1121. * rcu_read_lock()
  1122. * read mru_block
  1123. * mru_block = NULL;
  1124. * call_rcu(reclaim_ramblock, xxx);
  1125. * rcu_read_unlock()
  1126. *
  1127. * atomic_rcu_set is not needed here. The block was already published
  1128. * when it was placed into the list. Here we're just making an extra
  1129. * copy of the pointer.
  1130. */
  1131. ram_list.mru_block = block;
  1132. return block;
  1133. }
  1134. static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
  1135. {
  1136. CPUState *cpu;
  1137. ram_addr_t start1;
  1138. RAMBlock *block;
  1139. ram_addr_t end;
  1140. assert(tcg_enabled());
  1141. end = TARGET_PAGE_ALIGN(start + length);
  1142. start &= TARGET_PAGE_MASK;
  1143. rcu_read_lock();
  1144. block = qemu_get_ram_block(start);
  1145. assert(block == qemu_get_ram_block(end - 1));
  1146. start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
  1147. CPU_FOREACH(cpu) {
  1148. tlb_reset_dirty(cpu, start1, length);
  1149. }
  1150. rcu_read_unlock();
  1151. }
  1152. /* Note: start and end must be within the same ram block. */
  1153. bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
  1154. ram_addr_t length,
  1155. unsigned client)
  1156. {
  1157. DirtyMemoryBlocks *blocks;
  1158. unsigned long end, page;
  1159. bool dirty = false;
  1160. if (length == 0) {
  1161. return false;
  1162. }
  1163. end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
  1164. page = start >> TARGET_PAGE_BITS;
  1165. rcu_read_lock();
  1166. blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
  1167. while (page < end) {
  1168. unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
  1169. unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
  1170. unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
  1171. dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
  1172. offset, num);
  1173. page += num;
  1174. }
  1175. rcu_read_unlock();
  1176. if (dirty && tcg_enabled()) {
  1177. tlb_reset_dirty_range_all(start, length);
  1178. }
  1179. return dirty;
  1180. }
  1181. DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
  1182. (ram_addr_t start, ram_addr_t length, unsigned client)
  1183. {
  1184. DirtyMemoryBlocks *blocks;
  1185. unsigned long align = 1UL << (TARGET_PAGE_BITS + BITS_PER_LEVEL);
  1186. ram_addr_t first = QEMU_ALIGN_DOWN(start, align);
  1187. ram_addr_t last = QEMU_ALIGN_UP(start + length, align);
  1188. DirtyBitmapSnapshot *snap;
  1189. unsigned long page, end, dest;
  1190. snap = g_malloc0(sizeof(*snap) +
  1191. ((last - first) >> (TARGET_PAGE_BITS + 3)));
  1192. snap->start = first;
  1193. snap->end = last;
  1194. page = first >> TARGET_PAGE_BITS;
  1195. end = last >> TARGET_PAGE_BITS;
  1196. dest = 0;
  1197. rcu_read_lock();
  1198. blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
  1199. while (page < end) {
  1200. unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
  1201. unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
  1202. unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
  1203. assert(QEMU_IS_ALIGNED(offset, (1 << BITS_PER_LEVEL)));
  1204. assert(QEMU_IS_ALIGNED(num, (1 << BITS_PER_LEVEL)));
  1205. offset >>= BITS_PER_LEVEL;
  1206. bitmap_copy_and_clear_atomic(snap->dirty + dest,
  1207. blocks->blocks[idx] + offset,
  1208. num);
  1209. page += num;
  1210. dest += num >> BITS_PER_LEVEL;
  1211. }
  1212. rcu_read_unlock();
  1213. if (tcg_enabled()) {
  1214. tlb_reset_dirty_range_all(start, length);
  1215. }
  1216. return snap;
  1217. }
  1218. bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
  1219. ram_addr_t start,
  1220. ram_addr_t length)
  1221. {
  1222. unsigned long page, end;
  1223. assert(start >= snap->start);
  1224. assert(start + length <= snap->end);
  1225. end = TARGET_PAGE_ALIGN(start + length - snap->start) >> TARGET_PAGE_BITS;
  1226. page = (start - snap->start) >> TARGET_PAGE_BITS;
  1227. while (page < end) {
  1228. if (test_bit(page, snap->dirty)) {
  1229. return true;
  1230. }
  1231. page++;
  1232. }
  1233. return false;
  1234. }
  1235. /* Called from RCU critical section */
  1236. hwaddr memory_region_section_get_iotlb(CPUState *cpu,
  1237. MemoryRegionSection *section,
  1238. target_ulong vaddr,
  1239. hwaddr paddr, hwaddr xlat,
  1240. int prot,
  1241. target_ulong *address)
  1242. {
  1243. hwaddr iotlb;
  1244. CPUWatchpoint *wp;
  1245. if (memory_region_is_ram(section->mr)) {
  1246. /* Normal RAM. */
  1247. iotlb = memory_region_get_ram_addr(section->mr) + xlat;
  1248. if (!section->readonly) {
  1249. iotlb |= PHYS_SECTION_NOTDIRTY;
  1250. } else {
  1251. iotlb |= PHYS_SECTION_ROM;
  1252. }
  1253. } else {
  1254. AddressSpaceDispatch *d;
  1255. d = flatview_to_dispatch(section->fv);
  1256. iotlb = section - d->map.sections;
  1257. iotlb += xlat;
  1258. }
  1259. /* Make accesses to pages with watchpoints go via the
  1260. watchpoint trap routines. */
  1261. QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
  1262. if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
  1263. /* Avoid trapping reads of pages with a write breakpoint. */
  1264. if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
  1265. iotlb = PHYS_SECTION_WATCH + paddr;
  1266. *address |= TLB_MMIO;
  1267. break;
  1268. }
  1269. }
  1270. }
  1271. return iotlb;
  1272. }
  1273. #endif /* defined(CONFIG_USER_ONLY) */
  1274. #if !defined(CONFIG_USER_ONLY)
  1275. static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
  1276. uint16_t section);
  1277. static subpage_t *subpage_init(FlatView *fv, hwaddr base);
  1278. static void *(*phys_mem_alloc)(size_t size, uint64_t *align, bool shared) =
  1279. qemu_anon_ram_alloc;
  1280. /*
  1281. * Set a custom physical guest memory alloator.
  1282. * Accelerators with unusual needs may need this. Hopefully, we can
  1283. * get rid of it eventually.
  1284. */
  1285. void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align, bool shared))
  1286. {
  1287. phys_mem_alloc = alloc;
  1288. }
  1289. static uint16_t phys_section_add(PhysPageMap *map,
  1290. MemoryRegionSection *section)
  1291. {
  1292. /* The physical section number is ORed with a page-aligned
  1293. * pointer to produce the iotlb entries. Thus it should
  1294. * never overflow into the page-aligned value.
  1295. */
  1296. assert(map->sections_nb < TARGET_PAGE_SIZE);
  1297. if (map->sections_nb == map->sections_nb_alloc) {
  1298. map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
  1299. map->sections = g_renew(MemoryRegionSection, map->sections,
  1300. map->sections_nb_alloc);
  1301. }
  1302. map->sections[map->sections_nb] = *section;
  1303. memory_region_ref(section->mr);
  1304. return map->sections_nb++;
  1305. }
  1306. static void phys_section_destroy(MemoryRegion *mr)
  1307. {
  1308. bool have_sub_page = mr->subpage;
  1309. memory_region_unref(mr);
  1310. if (have_sub_page) {
  1311. subpage_t *subpage = container_of(mr, subpage_t, iomem);
  1312. object_unref(OBJECT(&subpage->iomem));
  1313. g_free(subpage);
  1314. }
  1315. }
  1316. static void phys_sections_free(PhysPageMap *map)
  1317. {
  1318. while (map->sections_nb > 0) {
  1319. MemoryRegionSection *section = &map->sections[--map->sections_nb];
  1320. phys_section_destroy(section->mr);
  1321. }
  1322. g_free(map->sections);
  1323. g_free(map->nodes);
  1324. }
  1325. static void register_subpage(FlatView *fv, MemoryRegionSection *section)
  1326. {
  1327. AddressSpaceDispatch *d = flatview_to_dispatch(fv);
  1328. subpage_t *subpage;
  1329. hwaddr base = section->offset_within_address_space
  1330. & TARGET_PAGE_MASK;
  1331. MemoryRegionSection *existing = phys_page_find(d, base);
  1332. MemoryRegionSection subsection = {
  1333. .offset_within_address_space = base,
  1334. .size = int128_make64(TARGET_PAGE_SIZE),
  1335. };
  1336. hwaddr start, end;
  1337. assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
  1338. if (!(existing->mr->subpage)) {
  1339. subpage = subpage_init(fv, base);
  1340. subsection.fv = fv;
  1341. subsection.mr = &subpage->iomem;
  1342. phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
  1343. phys_section_add(&d->map, &subsection));
  1344. } else {
  1345. subpage = container_of(existing->mr, subpage_t, iomem);
  1346. }
  1347. start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
  1348. end = start + int128_get64(section->size) - 1;
  1349. subpage_register(subpage, start, end,
  1350. phys_section_add(&d->map, section));
  1351. }
  1352. static void register_multipage(FlatView *fv,
  1353. MemoryRegionSection *section)
  1354. {
  1355. AddressSpaceDispatch *d = flatview_to_dispatch(fv);
  1356. hwaddr start_addr = section->offset_within_address_space;
  1357. uint16_t section_index = phys_section_add(&d->map, section);
  1358. uint64_t num_pages = int128_get64(int128_rshift(section->size,
  1359. TARGET_PAGE_BITS));
  1360. assert(num_pages);
  1361. phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
  1362. }
  1363. /*
  1364. * The range in *section* may look like this:
  1365. *
  1366. * |s|PPPPPPP|s|
  1367. *
  1368. * where s stands for subpage and P for page.
  1369. */
  1370. void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section)
  1371. {
  1372. MemoryRegionSection remain = *section;
  1373. Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
  1374. /* register first subpage */
  1375. if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
  1376. uint64_t left = TARGET_PAGE_ALIGN(remain.offset_within_address_space)
  1377. - remain.offset_within_address_space;
  1378. MemoryRegionSection now = remain;
  1379. now.size = int128_min(int128_make64(left), now.size);
  1380. register_subpage(fv, &now);
  1381. if (int128_eq(remain.size, now.size)) {
  1382. return;
  1383. }
  1384. remain.size = int128_sub(remain.size, now.size);
  1385. remain.offset_within_address_space += int128_get64(now.size);
  1386. remain.offset_within_region += int128_get64(now.size);
  1387. }
  1388. /* register whole pages */
  1389. if (int128_ge(remain.size, page_size)) {
  1390. MemoryRegionSection now = remain;
  1391. now.size = int128_and(now.size, int128_neg(page_size));
  1392. register_multipage(fv, &now);
  1393. if (int128_eq(remain.size, now.size)) {
  1394. return;
  1395. }
  1396. remain.size = int128_sub(remain.size, now.size);
  1397. remain.offset_within_address_space += int128_get64(now.size);
  1398. remain.offset_within_region += int128_get64(now.size);
  1399. }
  1400. /* register last subpage */
  1401. register_subpage(fv, &remain);
  1402. }
  1403. void qemu_flush_coalesced_mmio_buffer(void)
  1404. {
  1405. if (kvm_enabled())
  1406. kvm_flush_coalesced_mmio_buffer();
  1407. }
  1408. void qemu_mutex_lock_ramlist(void)
  1409. {
  1410. qemu_mutex_lock(&ram_list.mutex);
  1411. }
  1412. void qemu_mutex_unlock_ramlist(void)
  1413. {
  1414. qemu_mutex_unlock(&ram_list.mutex);
  1415. }
  1416. void ram_block_dump(Monitor *mon)
  1417. {
  1418. RAMBlock *block;
  1419. char *psize;
  1420. rcu_read_lock();
  1421. monitor_printf(mon, "%24s %8s %18s %18s %18s\n",
  1422. "Block Name", "PSize", "Offset", "Used", "Total");
  1423. RAMBLOCK_FOREACH(block) {
  1424. psize = size_to_str(block->page_size);
  1425. monitor_printf(mon, "%24s %8s 0x%016" PRIx64 " 0x%016" PRIx64
  1426. " 0x%016" PRIx64 "\n", block->idstr, psize,
  1427. (uint64_t)block->offset,
  1428. (uint64_t)block->used_length,
  1429. (uint64_t)block->max_length);
  1430. g_free(psize);
  1431. }
  1432. rcu_read_unlock();
  1433. }
  1434. #ifdef __linux__
  1435. /*
  1436. * FIXME TOCTTOU: this iterates over memory backends' mem-path, which
  1437. * may or may not name the same files / on the same filesystem now as
  1438. * when we actually open and map them. Iterate over the file
  1439. * descriptors instead, and use qemu_fd_getpagesize().
  1440. */
  1441. static int find_min_backend_pagesize(Object *obj, void *opaque)
  1442. {
  1443. long *hpsize_min = opaque;
  1444. if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
  1445. HostMemoryBackend *backend = MEMORY_BACKEND(obj);
  1446. long hpsize = host_memory_backend_pagesize(backend);
  1447. if (host_memory_backend_is_mapped(backend) && (hpsize < *hpsize_min)) {
  1448. *hpsize_min = hpsize;
  1449. }
  1450. }
  1451. return 0;
  1452. }
  1453. static int find_max_backend_pagesize(Object *obj, void *opaque)
  1454. {
  1455. long *hpsize_max = opaque;
  1456. if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
  1457. HostMemoryBackend *backend = MEMORY_BACKEND(obj);
  1458. long hpsize = host_memory_backend_pagesize(backend);
  1459. if (host_memory_backend_is_mapped(backend) && (hpsize > *hpsize_max)) {
  1460. *hpsize_max = hpsize;
  1461. }
  1462. }
  1463. return 0;
  1464. }
  1465. /*
  1466. * TODO: We assume right now that all mapped host memory backends are
  1467. * used as RAM, however some might be used for different purposes.
  1468. */
  1469. long qemu_minrampagesize(void)
  1470. {
  1471. long hpsize = LONG_MAX;
  1472. long mainrampagesize;
  1473. Object *memdev_root;
  1474. mainrampagesize = qemu_mempath_getpagesize(mem_path);
  1475. /* it's possible we have memory-backend objects with
  1476. * hugepage-backed RAM. these may get mapped into system
  1477. * address space via -numa parameters or memory hotplug
  1478. * hooks. we want to take these into account, but we
  1479. * also want to make sure these supported hugepage
  1480. * sizes are applicable across the entire range of memory
  1481. * we may boot from, so we take the min across all
  1482. * backends, and assume normal pages in cases where a
  1483. * backend isn't backed by hugepages.
  1484. */
  1485. memdev_root = object_resolve_path("/objects", NULL);
  1486. if (memdev_root) {
  1487. object_child_foreach(memdev_root, find_min_backend_pagesize, &hpsize);
  1488. }
  1489. if (hpsize == LONG_MAX) {
  1490. /* No additional memory regions found ==> Report main RAM page size */
  1491. return mainrampagesize;
  1492. }
  1493. /* If NUMA is disabled or the NUMA nodes are not backed with a
  1494. * memory-backend, then there is at least one node using "normal" RAM,
  1495. * so if its page size is smaller we have got to report that size instead.
  1496. */
  1497. if (hpsize > mainrampagesize &&
  1498. (nb_numa_nodes == 0 || numa_info[0].node_memdev == NULL)) {
  1499. static bool warned;
  1500. if (!warned) {
  1501. error_report("Huge page support disabled (n/a for main memory).");
  1502. warned = true;
  1503. }
  1504. return mainrampagesize;
  1505. }
  1506. return hpsize;
  1507. }
  1508. long qemu_maxrampagesize(void)
  1509. {
  1510. long pagesize = qemu_mempath_getpagesize(mem_path);
  1511. Object *memdev_root = object_resolve_path("/objects", NULL);
  1512. if (memdev_root) {
  1513. object_child_foreach(memdev_root, find_max_backend_pagesize,
  1514. &pagesize);
  1515. }
  1516. return pagesize;
  1517. }
  1518. #else
  1519. long qemu_minrampagesize(void)
  1520. {
  1521. return getpagesize();
  1522. }
  1523. long qemu_maxrampagesize(void)
  1524. {
  1525. return getpagesize();
  1526. }
  1527. #endif
  1528. #ifdef CONFIG_POSIX
  1529. static int64_t get_file_size(int fd)
  1530. {
  1531. int64_t size = lseek(fd, 0, SEEK_END);
  1532. if (size < 0) {
  1533. return -errno;
  1534. }
  1535. return size;
  1536. }
  1537. static int file_ram_open(const char *path,
  1538. const char *region_name,
  1539. bool *created,
  1540. Error **errp)
  1541. {
  1542. char *filename;
  1543. char *sanitized_name;
  1544. char *c;
  1545. int fd = -1;
  1546. *created = false;
  1547. for (;;) {
  1548. fd = open(path, O_RDWR);
  1549. if (fd >= 0) {
  1550. /* @path names an existing file, use it */
  1551. break;
  1552. }
  1553. if (errno == ENOENT) {
  1554. /* @path names a file that doesn't exist, create it */
  1555. fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
  1556. if (fd >= 0) {
  1557. *created = true;
  1558. break;
  1559. }
  1560. } else if (errno == EISDIR) {
  1561. /* @path names a directory, create a file there */
  1562. /* Make name safe to use with mkstemp by replacing '/' with '_'. */
  1563. sanitized_name = g_strdup(region_name);
  1564. for (c = sanitized_name; *c != '\0'; c++) {
  1565. if (*c == '/') {
  1566. *c = '_';
  1567. }
  1568. }
  1569. filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
  1570. sanitized_name);
  1571. g_free(sanitized_name);
  1572. fd = mkstemp(filename);
  1573. if (fd >= 0) {
  1574. unlink(filename);
  1575. g_free(filename);
  1576. break;
  1577. }
  1578. g_free(filename);
  1579. }
  1580. if (errno != EEXIST && errno != EINTR) {
  1581. error_setg_errno(errp, errno,
  1582. "can't open backing store %s for guest RAM",
  1583. path);
  1584. return -1;
  1585. }
  1586. /*
  1587. * Try again on EINTR and EEXIST. The latter happens when
  1588. * something else creates the file between our two open().
  1589. */
  1590. }
  1591. return fd;
  1592. }
  1593. static void *file_ram_alloc(RAMBlock *block,
  1594. ram_addr_t memory,
  1595. int fd,
  1596. bool truncate,
  1597. Error **errp)
  1598. {
  1599. void *area;
  1600. block->page_size = qemu_fd_getpagesize(fd);
  1601. if (block->mr->align % block->page_size) {
  1602. error_setg(errp, "alignment 0x%" PRIx64
  1603. " must be multiples of page size 0x%zx",
  1604. block->mr->align, block->page_size);
  1605. return NULL;
  1606. } else if (block->mr->align && !is_power_of_2(block->mr->align)) {
  1607. error_setg(errp, "alignment 0x%" PRIx64
  1608. " must be a power of two", block->mr->align);
  1609. return NULL;
  1610. }
  1611. block->mr->align = MAX(block->page_size, block->mr->align);
  1612. #if defined(__s390x__)
  1613. if (kvm_enabled()) {
  1614. block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN);
  1615. }
  1616. #endif
  1617. if (memory < block->page_size) {
  1618. error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
  1619. "or larger than page size 0x%zx",
  1620. memory, block->page_size);
  1621. return NULL;
  1622. }
  1623. memory = ROUND_UP(memory, block->page_size);
  1624. /*
  1625. * ftruncate is not supported by hugetlbfs in older
  1626. * hosts, so don't bother bailing out on errors.
  1627. * If anything goes wrong with it under other filesystems,
  1628. * mmap will fail.
  1629. *
  1630. * Do not truncate the non-empty backend file to avoid corrupting
  1631. * the existing data in the file. Disabling shrinking is not
  1632. * enough. For example, the current vNVDIMM implementation stores
  1633. * the guest NVDIMM labels at the end of the backend file. If the
  1634. * backend file is later extended, QEMU will not be able to find
  1635. * those labels. Therefore, extending the non-empty backend file
  1636. * is disabled as well.
  1637. */
  1638. if (truncate && ftruncate(fd, memory)) {
  1639. perror("ftruncate");
  1640. }
  1641. area = qemu_ram_mmap(fd, memory, block->mr->align,
  1642. block->flags & RAM_SHARED, block->flags & RAM_PMEM);
  1643. if (area == MAP_FAILED) {
  1644. error_setg_errno(errp, errno,
  1645. "unable to map backing store for guest RAM");
  1646. return NULL;
  1647. }
  1648. if (mem_prealloc) {
  1649. os_mem_prealloc(fd, area, memory, smp_cpus, errp);
  1650. if (errp && *errp) {
  1651. qemu_ram_munmap(fd, area, memory);
  1652. return NULL;
  1653. }
  1654. }
  1655. block->fd = fd;
  1656. return area;
  1657. }
  1658. #endif
  1659. /* Allocate space within the ram_addr_t space that governs the
  1660. * dirty bitmaps.
  1661. * Called with the ramlist lock held.
  1662. */
  1663. static ram_addr_t find_ram_offset(ram_addr_t size)
  1664. {
  1665. RAMBlock *block, *next_block;
  1666. ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
  1667. assert(size != 0); /* it would hand out same offset multiple times */
  1668. if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
  1669. return 0;
  1670. }
  1671. RAMBLOCK_FOREACH(block) {
  1672. ram_addr_t candidate, next = RAM_ADDR_MAX;
  1673. /* Align blocks to start on a 'long' in the bitmap
  1674. * which makes the bitmap sync'ing take the fast path.
  1675. */
  1676. candidate = block->offset + block->max_length;
  1677. candidate = ROUND_UP(candidate, BITS_PER_LONG << TARGET_PAGE_BITS);
  1678. /* Search for the closest following block
  1679. * and find the gap.
  1680. */
  1681. RAMBLOCK_FOREACH(next_block) {
  1682. if (next_block->offset >= candidate) {
  1683. next = MIN(next, next_block->offset);
  1684. }
  1685. }
  1686. /* If it fits remember our place and remember the size
  1687. * of gap, but keep going so that we might find a smaller
  1688. * gap to fill so avoiding fragmentation.
  1689. */
  1690. if (next - candidate >= size && next - candidate < mingap) {
  1691. offset = candidate;
  1692. mingap = next - candidate;
  1693. }
  1694. trace_find_ram_offset_loop(size, candidate, offset, next, mingap);
  1695. }
  1696. if (offset == RAM_ADDR_MAX) {
  1697. fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
  1698. (uint64_t)size);
  1699. abort();
  1700. }
  1701. trace_find_ram_offset(size, offset);
  1702. return offset;
  1703. }
  1704. static unsigned long last_ram_page(void)
  1705. {
  1706. RAMBlock *block;
  1707. ram_addr_t last = 0;
  1708. rcu_read_lock();
  1709. RAMBLOCK_FOREACH(block) {
  1710. last = MAX(last, block->offset + block->max_length);
  1711. }
  1712. rcu_read_unlock();
  1713. return last >> TARGET_PAGE_BITS;
  1714. }
  1715. static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
  1716. {
  1717. int ret;
  1718. /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
  1719. if (!machine_dump_guest_core(current_machine)) {
  1720. ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
  1721. if (ret) {
  1722. perror("qemu_madvise");
  1723. fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
  1724. "but dump_guest_core=off specified\n");
  1725. }
  1726. }
  1727. }
  1728. const char *qemu_ram_get_idstr(RAMBlock *rb)
  1729. {
  1730. return rb->idstr;
  1731. }
  1732. void *qemu_ram_get_host_addr(RAMBlock *rb)
  1733. {
  1734. return rb->host;
  1735. }
  1736. ram_addr_t qemu_ram_get_offset(RAMBlock *rb)
  1737. {
  1738. return rb->offset;
  1739. }
  1740. ram_addr_t qemu_ram_get_used_length(RAMBlock *rb)
  1741. {
  1742. return rb->used_length;
  1743. }
  1744. bool qemu_ram_is_shared(RAMBlock *rb)
  1745. {
  1746. return rb->flags & RAM_SHARED;
  1747. }
  1748. /* Note: Only set at the start of postcopy */
  1749. bool qemu_ram_is_uf_zeroable(RAMBlock *rb)
  1750. {
  1751. return rb->flags & RAM_UF_ZEROPAGE;
  1752. }
  1753. void qemu_ram_set_uf_zeroable(RAMBlock *rb)
  1754. {
  1755. rb->flags |= RAM_UF_ZEROPAGE;
  1756. }
  1757. bool qemu_ram_is_migratable(RAMBlock *rb)
  1758. {
  1759. return rb->flags & RAM_MIGRATABLE;
  1760. }
  1761. void qemu_ram_set_migratable(RAMBlock *rb)
  1762. {
  1763. rb->flags |= RAM_MIGRATABLE;
  1764. }
  1765. void qemu_ram_unset_migratable(RAMBlock *rb)
  1766. {
  1767. rb->flags &= ~RAM_MIGRATABLE;
  1768. }
  1769. /* Called with iothread lock held. */
  1770. void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
  1771. {
  1772. RAMBlock *block;
  1773. assert(new_block);
  1774. assert(!new_block->idstr[0]);
  1775. if (dev) {
  1776. char *id = qdev_get_dev_path(dev);
  1777. if (id) {
  1778. snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
  1779. g_free(id);
  1780. }
  1781. }
  1782. pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
  1783. rcu_read_lock();
  1784. RAMBLOCK_FOREACH(block) {
  1785. if (block != new_block &&
  1786. !strcmp(block->idstr, new_block->idstr)) {
  1787. fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
  1788. new_block->idstr);
  1789. abort();
  1790. }
  1791. }
  1792. rcu_read_unlock();
  1793. }
  1794. /* Called with iothread lock held. */
  1795. void qemu_ram_unset_idstr(RAMBlock *block)
  1796. {
  1797. /* FIXME: arch_init.c assumes that this is not called throughout
  1798. * migration. Ignore the problem since hot-unplug during migration
  1799. * does not work anyway.
  1800. */
  1801. if (block) {
  1802. memset(block->idstr, 0, sizeof(block->idstr));
  1803. }
  1804. }
  1805. size_t qemu_ram_pagesize(RAMBlock *rb)
  1806. {
  1807. return rb->page_size;
  1808. }
  1809. /* Returns the largest size of page in use */
  1810. size_t qemu_ram_pagesize_largest(void)
  1811. {
  1812. RAMBlock *block;
  1813. size_t largest = 0;
  1814. RAMBLOCK_FOREACH(block) {
  1815. largest = MAX(largest, qemu_ram_pagesize(block));
  1816. }
  1817. return largest;
  1818. }
  1819. static int memory_try_enable_merging(void *addr, size_t len)
  1820. {
  1821. if (!machine_mem_merge(current_machine)) {
  1822. /* disabled by the user */
  1823. return 0;
  1824. }
  1825. return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
  1826. }
  1827. /* Only legal before guest might have detected the memory size: e.g. on
  1828. * incoming migration, or right after reset.
  1829. *
  1830. * As memory core doesn't know how is memory accessed, it is up to
  1831. * resize callback to update device state and/or add assertions to detect
  1832. * misuse, if necessary.
  1833. */
  1834. int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
  1835. {
  1836. assert(block);
  1837. newsize = HOST_PAGE_ALIGN(newsize);
  1838. if (block->used_length == newsize) {
  1839. return 0;
  1840. }
  1841. if (!(block->flags & RAM_RESIZEABLE)) {
  1842. error_setg_errno(errp, EINVAL,
  1843. "Length mismatch: %s: 0x" RAM_ADDR_FMT
  1844. " in != 0x" RAM_ADDR_FMT, block->idstr,
  1845. newsize, block->used_length);
  1846. return -EINVAL;
  1847. }
  1848. if (block->max_length < newsize) {
  1849. error_setg_errno(errp, EINVAL,
  1850. "Length too large: %s: 0x" RAM_ADDR_FMT
  1851. " > 0x" RAM_ADDR_FMT, block->idstr,
  1852. newsize, block->max_length);
  1853. return -EINVAL;
  1854. }
  1855. cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
  1856. block->used_length = newsize;
  1857. cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
  1858. DIRTY_CLIENTS_ALL);
  1859. memory_region_set_size(block->mr, newsize);
  1860. if (block->resized) {
  1861. block->resized(block->idstr, newsize, block->host);
  1862. }
  1863. return 0;
  1864. }
  1865. /* Called with ram_list.mutex held */
  1866. static void dirty_memory_extend(ram_addr_t old_ram_size,
  1867. ram_addr_t new_ram_size)
  1868. {
  1869. ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size,
  1870. DIRTY_MEMORY_BLOCK_SIZE);
  1871. ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size,
  1872. DIRTY_MEMORY_BLOCK_SIZE);
  1873. int i;
  1874. /* Only need to extend if block count increased */
  1875. if (new_num_blocks <= old_num_blocks) {
  1876. return;
  1877. }
  1878. for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
  1879. DirtyMemoryBlocks *old_blocks;
  1880. DirtyMemoryBlocks *new_blocks;
  1881. int j;
  1882. old_blocks = atomic_rcu_read(&ram_list.dirty_memory[i]);
  1883. new_blocks = g_malloc(sizeof(*new_blocks) +
  1884. sizeof(new_blocks->blocks[0]) * new_num_blocks);
  1885. if (old_num_blocks) {
  1886. memcpy(new_blocks->blocks, old_blocks->blocks,
  1887. old_num_blocks * sizeof(old_blocks->blocks[0]));
  1888. }
  1889. for (j = old_num_blocks; j < new_num_blocks; j++) {
  1890. new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
  1891. }
  1892. atomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
  1893. if (old_blocks) {
  1894. g_free_rcu(old_blocks, rcu);
  1895. }
  1896. }
  1897. }
  1898. static void ram_block_add(RAMBlock *new_block, Error **errp, bool shared)
  1899. {
  1900. RAMBlock *block;
  1901. RAMBlock *last_block = NULL;
  1902. ram_addr_t old_ram_size, new_ram_size;
  1903. Error *err = NULL;
  1904. old_ram_size = last_ram_page();
  1905. qemu_mutex_lock_ramlist();
  1906. new_block->offset = find_ram_offset(new_block->max_length);
  1907. if (!new_block->host) {
  1908. if (xen_enabled()) {
  1909. xen_ram_alloc(new_block->offset, new_block->max_length,
  1910. new_block->mr, &err);
  1911. if (err) {
  1912. error_propagate(errp, err);
  1913. qemu_mutex_unlock_ramlist();
  1914. return;
  1915. }
  1916. } else {
  1917. new_block->host = phys_mem_alloc(new_block->max_length,
  1918. &new_block->mr->align, shared);
  1919. if (!new_block->host) {
  1920. error_setg_errno(errp, errno,
  1921. "cannot set up guest memory '%s'",
  1922. memory_region_name(new_block->mr));
  1923. qemu_mutex_unlock_ramlist();
  1924. return;
  1925. }
  1926. memory_try_enable_merging(new_block->host, new_block->max_length);
  1927. }
  1928. }
  1929. new_ram_size = MAX(old_ram_size,
  1930. (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
  1931. if (new_ram_size > old_ram_size) {
  1932. dirty_memory_extend(old_ram_size, new_ram_size);
  1933. }
  1934. /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
  1935. * QLIST (which has an RCU-friendly variant) does not have insertion at
  1936. * tail, so save the last element in last_block.
  1937. */
  1938. RAMBLOCK_FOREACH(block) {
  1939. last_block = block;
  1940. if (block->max_length < new_block->max_length) {
  1941. break;
  1942. }
  1943. }
  1944. if (block) {
  1945. QLIST_INSERT_BEFORE_RCU(block, new_block, next);
  1946. } else if (last_block) {
  1947. QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
  1948. } else { /* list is empty */
  1949. QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
  1950. }
  1951. ram_list.mru_block = NULL;
  1952. /* Write list before version */
  1953. smp_wmb();
  1954. ram_list.version++;
  1955. qemu_mutex_unlock_ramlist();
  1956. cpu_physical_memory_set_dirty_range(new_block->offset,
  1957. new_block->used_length,
  1958. DIRTY_CLIENTS_ALL);
  1959. if (new_block->host) {
  1960. qemu_ram_setup_dump(new_block->host, new_block->max_length);
  1961. qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
  1962. /* MADV_DONTFORK is also needed by KVM in absence of synchronous MMU */
  1963. qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
  1964. ram_block_notify_add(new_block->host, new_block->max_length);
  1965. }
  1966. }
  1967. #ifdef CONFIG_POSIX
  1968. RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
  1969. uint32_t ram_flags, int fd,
  1970. Error **errp)
  1971. {
  1972. RAMBlock *new_block;
  1973. Error *local_err = NULL;
  1974. int64_t file_size;
  1975. /* Just support these ram flags by now. */
  1976. assert((ram_flags & ~(RAM_SHARED | RAM_PMEM)) == 0);
  1977. if (xen_enabled()) {
  1978. error_setg(errp, "-mem-path not supported with Xen");
  1979. return NULL;
  1980. }
  1981. if (kvm_enabled() && !kvm_has_sync_mmu()) {
  1982. error_setg(errp,
  1983. "host lacks kvm mmu notifiers, -mem-path unsupported");
  1984. return NULL;
  1985. }
  1986. if (phys_mem_alloc != qemu_anon_ram_alloc) {
  1987. /*
  1988. * file_ram_alloc() needs to allocate just like
  1989. * phys_mem_alloc, but we haven't bothered to provide
  1990. * a hook there.
  1991. */
  1992. error_setg(errp,
  1993. "-mem-path not supported with this accelerator");
  1994. return NULL;
  1995. }
  1996. size = HOST_PAGE_ALIGN(size);
  1997. file_size = get_file_size(fd);
  1998. if (file_size > 0 && file_size < size) {
  1999. error_setg(errp, "backing store %s size 0x%" PRIx64
  2000. " does not match 'size' option 0x" RAM_ADDR_FMT,
  2001. mem_path, file_size, size);
  2002. return NULL;
  2003. }
  2004. new_block = g_malloc0(sizeof(*new_block));
  2005. new_block->mr = mr;
  2006. new_block->used_length = size;
  2007. new_block->max_length = size;
  2008. new_block->flags = ram_flags;
  2009. new_block->host = file_ram_alloc(new_block, size, fd, !file_size, errp);
  2010. if (!new_block->host) {
  2011. g_free(new_block);
  2012. return NULL;
  2013. }
  2014. ram_block_add(new_block, &local_err, ram_flags & RAM_SHARED);
  2015. if (local_err) {
  2016. g_free(new_block);
  2017. error_propagate(errp, local_err);
  2018. return NULL;
  2019. }
  2020. return new_block;
  2021. }
  2022. RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
  2023. uint32_t ram_flags, const char *mem_path,
  2024. Error **errp)
  2025. {
  2026. int fd;
  2027. bool created;
  2028. RAMBlock *block;
  2029. fd = file_ram_open(mem_path, memory_region_name(mr), &created, errp);
  2030. if (fd < 0) {
  2031. return NULL;
  2032. }
  2033. block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, errp);
  2034. if (!block) {
  2035. if (created) {
  2036. unlink(mem_path);
  2037. }
  2038. close(fd);
  2039. return NULL;
  2040. }
  2041. return block;
  2042. }
  2043. #endif
  2044. static
  2045. RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
  2046. void (*resized)(const char*,
  2047. uint64_t length,
  2048. void *host),
  2049. void *host, bool resizeable, bool share,
  2050. MemoryRegion *mr, Error **errp)
  2051. {
  2052. RAMBlock *new_block;
  2053. Error *local_err = NULL;
  2054. size = HOST_PAGE_ALIGN(size);
  2055. max_size = HOST_PAGE_ALIGN(max_size);
  2056. new_block = g_malloc0(sizeof(*new_block));
  2057. new_block->mr = mr;
  2058. new_block->resized = resized;
  2059. new_block->used_length = size;
  2060. new_block->max_length = max_size;
  2061. assert(max_size >= size);
  2062. new_block->fd = -1;
  2063. new_block->page_size = getpagesize();
  2064. new_block->host = host;
  2065. if (host) {
  2066. new_block->flags |= RAM_PREALLOC;
  2067. }
  2068. if (resizeable) {
  2069. new_block->flags |= RAM_RESIZEABLE;
  2070. }
  2071. ram_block_add(new_block, &local_err, share);
  2072. if (local_err) {
  2073. g_free(new_block);
  2074. error_propagate(errp, local_err);
  2075. return NULL;
  2076. }
  2077. return new_block;
  2078. }
  2079. RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
  2080. MemoryRegion *mr, Error **errp)
  2081. {
  2082. return qemu_ram_alloc_internal(size, size, NULL, host, false,
  2083. false, mr, errp);
  2084. }
  2085. RAMBlock *qemu_ram_alloc(ram_addr_t size, bool share,
  2086. MemoryRegion *mr, Error **errp)
  2087. {
  2088. return qemu_ram_alloc_internal(size, size, NULL, NULL, false,
  2089. share, mr, errp);
  2090. }
  2091. RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
  2092. void (*resized)(const char*,
  2093. uint64_t length,
  2094. void *host),
  2095. MemoryRegion *mr, Error **errp)
  2096. {
  2097. return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true,
  2098. false, mr, errp);
  2099. }
  2100. static void reclaim_ramblock(RAMBlock *block)
  2101. {
  2102. if (block->flags & RAM_PREALLOC) {
  2103. ;
  2104. } else if (xen_enabled()) {
  2105. xen_invalidate_map_cache_entry(block->host);
  2106. #ifndef _WIN32
  2107. } else if (block->fd >= 0) {
  2108. qemu_ram_munmap(block->fd, block->host, block->max_length);
  2109. close(block->fd);
  2110. #endif
  2111. } else {
  2112. qemu_anon_ram_free(block->host, block->max_length);
  2113. }
  2114. g_free(block);
  2115. }
  2116. void qemu_ram_free(RAMBlock *block)
  2117. {
  2118. if (!block) {
  2119. return;
  2120. }
  2121. if (block->host) {
  2122. ram_block_notify_remove(block->host, block->max_length);
  2123. }
  2124. qemu_mutex_lock_ramlist();
  2125. QLIST_REMOVE_RCU(block, next);
  2126. ram_list.mru_block = NULL;
  2127. /* Write list before version */
  2128. smp_wmb();
  2129. ram_list.version++;
  2130. call_rcu(block, reclaim_ramblock, rcu);
  2131. qemu_mutex_unlock_ramlist();
  2132. }
  2133. #ifndef _WIN32
  2134. void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
  2135. {
  2136. RAMBlock *block;
  2137. ram_addr_t offset;
  2138. int flags;
  2139. void *area, *vaddr;
  2140. RAMBLOCK_FOREACH(block) {
  2141. offset = addr - block->offset;
  2142. if (offset < block->max_length) {
  2143. vaddr = ramblock_ptr(block, offset);
  2144. if (block->flags & RAM_PREALLOC) {
  2145. ;
  2146. } else if (xen_enabled()) {
  2147. abort();
  2148. } else {
  2149. flags = MAP_FIXED;
  2150. if (block->fd >= 0) {
  2151. flags |= (block->flags & RAM_SHARED ?
  2152. MAP_SHARED : MAP_PRIVATE);
  2153. area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
  2154. flags, block->fd, offset);
  2155. } else {
  2156. /*
  2157. * Remap needs to match alloc. Accelerators that
  2158. * set phys_mem_alloc never remap. If they did,
  2159. * we'd need a remap hook here.
  2160. */
  2161. assert(phys_mem_alloc == qemu_anon_ram_alloc);
  2162. flags |= MAP_PRIVATE | MAP_ANONYMOUS;
  2163. area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
  2164. flags, -1, 0);
  2165. }
  2166. if (area != vaddr) {
  2167. error_report("Could not remap addr: "
  2168. RAM_ADDR_FMT "@" RAM_ADDR_FMT "",
  2169. length, addr);
  2170. exit(1);
  2171. }
  2172. memory_try_enable_merging(vaddr, length);
  2173. qemu_ram_setup_dump(vaddr, length);
  2174. }
  2175. }
  2176. }
  2177. }
  2178. #endif /* !_WIN32 */
  2179. /* Return a host pointer to ram allocated with qemu_ram_alloc.
  2180. * This should not be used for general purpose DMA. Use address_space_map
  2181. * or address_space_rw instead. For local memory (e.g. video ram) that the
  2182. * device owns, use memory_region_get_ram_ptr.
  2183. *
  2184. * Called within RCU critical section.
  2185. */
  2186. void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
  2187. {
  2188. RAMBlock *block = ram_block;
  2189. if (block == NULL) {
  2190. block = qemu_get_ram_block(addr);
  2191. addr -= block->offset;
  2192. }
  2193. if (xen_enabled() && block->host == NULL) {
  2194. /* We need to check if the requested address is in the RAM
  2195. * because we don't want to map the entire memory in QEMU.
  2196. * In that case just map until the end of the page.
  2197. */
  2198. if (block->offset == 0) {
  2199. return xen_map_cache(addr, 0, 0, false);
  2200. }
  2201. block->host = xen_map_cache(block->offset, block->max_length, 1, false);
  2202. }
  2203. return ramblock_ptr(block, addr);
  2204. }
  2205. /* Return a host pointer to guest's ram. Similar to qemu_map_ram_ptr
  2206. * but takes a size argument.
  2207. *
  2208. * Called within RCU critical section.
  2209. */
  2210. static void *qemu_ram_ptr_length(RAMBlock *ram_block, ram_addr_t addr,
  2211. hwaddr *size, bool lock)
  2212. {
  2213. RAMBlock *block = ram_block;
  2214. if (*size == 0) {
  2215. return NULL;
  2216. }
  2217. if (block == NULL) {
  2218. block = qemu_get_ram_block(addr);
  2219. addr -= block->offset;
  2220. }
  2221. *size = MIN(*size, block->max_length - addr);
  2222. if (xen_enabled() && block->host == NULL) {
  2223. /* We need to check if the requested address is in the RAM
  2224. * because we don't want to map the entire memory in QEMU.
  2225. * In that case just map the requested area.
  2226. */
  2227. if (block->offset == 0) {
  2228. return xen_map_cache(addr, *size, lock, lock);
  2229. }
  2230. block->host = xen_map_cache(block->offset, block->max_length, 1, lock);
  2231. }
  2232. return ramblock_ptr(block, addr);
  2233. }
  2234. /* Return the offset of a hostpointer within a ramblock */
  2235. ram_addr_t qemu_ram_block_host_offset(RAMBlock *rb, void *host)
  2236. {
  2237. ram_addr_t res = (uint8_t *)host - (uint8_t *)rb->host;
  2238. assert((uintptr_t)host >= (uintptr_t)rb->host);
  2239. assert(res < rb->max_length);
  2240. return res;
  2241. }
  2242. /*
  2243. * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
  2244. * in that RAMBlock.
  2245. *
  2246. * ptr: Host pointer to look up
  2247. * round_offset: If true round the result offset down to a page boundary
  2248. * *ram_addr: set to result ram_addr
  2249. * *offset: set to result offset within the RAMBlock
  2250. *
  2251. * Returns: RAMBlock (or NULL if not found)
  2252. *
  2253. * By the time this function returns, the returned pointer is not protected
  2254. * by RCU anymore. If the caller is not within an RCU critical section and
  2255. * does not hold the iothread lock, it must have other means of protecting the
  2256. * pointer, such as a reference to the region that includes the incoming
  2257. * ram_addr_t.
  2258. */
  2259. RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
  2260. ram_addr_t *offset)
  2261. {
  2262. RAMBlock *block;
  2263. uint8_t *host = ptr;
  2264. if (xen_enabled()) {
  2265. ram_addr_t ram_addr;
  2266. rcu_read_lock();
  2267. ram_addr = xen_ram_addr_from_mapcache(ptr);
  2268. block = qemu_get_ram_block(ram_addr);
  2269. if (block) {
  2270. *offset = ram_addr - block->offset;
  2271. }
  2272. rcu_read_unlock();
  2273. return block;
  2274. }
  2275. rcu_read_lock();
  2276. block = atomic_rcu_read(&ram_list.mru_block);
  2277. if (block && block->host && host - block->host < block->max_length) {
  2278. goto found;
  2279. }
  2280. RAMBLOCK_FOREACH(block) {
  2281. /* This case append when the block is not mapped. */
  2282. if (block->host == NULL) {
  2283. continue;
  2284. }
  2285. if (host - block->host < block->max_length) {
  2286. goto found;
  2287. }
  2288. }
  2289. rcu_read_unlock();
  2290. return NULL;
  2291. found:
  2292. *offset = (host - block->host);
  2293. if (round_offset) {
  2294. *offset &= TARGET_PAGE_MASK;
  2295. }
  2296. rcu_read_unlock();
  2297. return block;
  2298. }
  2299. /*
  2300. * Finds the named RAMBlock
  2301. *
  2302. * name: The name of RAMBlock to find
  2303. *
  2304. * Returns: RAMBlock (or NULL if not found)
  2305. */
  2306. RAMBlock *qemu_ram_block_by_name(const char *name)
  2307. {
  2308. RAMBlock *block;
  2309. RAMBLOCK_FOREACH(block) {
  2310. if (!strcmp(name, block->idstr)) {
  2311. return block;
  2312. }
  2313. }
  2314. return NULL;
  2315. }
  2316. /* Some of the softmmu routines need to translate from a host pointer
  2317. (typically a TLB entry) back to a ram offset. */
  2318. ram_addr_t qemu_ram_addr_from_host(void *ptr)
  2319. {
  2320. RAMBlock *block;
  2321. ram_addr_t offset;
  2322. block = qemu_ram_block_from_host(ptr, false, &offset);
  2323. if (!block) {
  2324. return RAM_ADDR_INVALID;
  2325. }
  2326. return block->offset + offset;
  2327. }
  2328. /* Called within RCU critical section. */
  2329. void memory_notdirty_write_prepare(NotDirtyInfo *ndi,
  2330. CPUState *cpu,
  2331. vaddr mem_vaddr,
  2332. ram_addr_t ram_addr,
  2333. unsigned size)
  2334. {
  2335. ndi->cpu = cpu;
  2336. ndi->ram_addr = ram_addr;
  2337. ndi->mem_vaddr = mem_vaddr;
  2338. ndi->size = size;
  2339. ndi->pages = NULL;
  2340. assert(tcg_enabled());
  2341. if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
  2342. ndi->pages = page_collection_lock(ram_addr, ram_addr + size);
  2343. tb_invalidate_phys_page_fast(ndi->pages, ram_addr, size);
  2344. }
  2345. }
  2346. /* Called within RCU critical section. */
  2347. void memory_notdirty_write_complete(NotDirtyInfo *ndi)
  2348. {
  2349. if (ndi->pages) {
  2350. assert(tcg_enabled());
  2351. page_collection_unlock(ndi->pages);
  2352. ndi->pages = NULL;
  2353. }
  2354. /* Set both VGA and migration bits for simplicity and to remove
  2355. * the notdirty callback faster.
  2356. */
  2357. cpu_physical_memory_set_dirty_range(ndi->ram_addr, ndi->size,
  2358. DIRTY_CLIENTS_NOCODE);
  2359. /* we remove the notdirty callback only if the code has been
  2360. flushed */
  2361. if (!cpu_physical_memory_is_clean(ndi->ram_addr)) {
  2362. tlb_set_dirty(ndi->cpu, ndi->mem_vaddr);
  2363. }
  2364. }
  2365. /* Called within RCU critical section. */
  2366. static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
  2367. uint64_t val, unsigned size)
  2368. {
  2369. NotDirtyInfo ndi;
  2370. memory_notdirty_write_prepare(&ndi, current_cpu, current_cpu->mem_io_vaddr,
  2371. ram_addr, size);
  2372. stn_p(qemu_map_ram_ptr(NULL, ram_addr), size, val);
  2373. memory_notdirty_write_complete(&ndi);
  2374. }
  2375. static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
  2376. unsigned size, bool is_write,
  2377. MemTxAttrs attrs)
  2378. {
  2379. return is_write;
  2380. }
  2381. static const MemoryRegionOps notdirty_mem_ops = {
  2382. .write = notdirty_mem_write,
  2383. .valid.accepts = notdirty_mem_accepts,
  2384. .endianness = DEVICE_NATIVE_ENDIAN,
  2385. .valid = {
  2386. .min_access_size = 1,
  2387. .max_access_size = 8,
  2388. .unaligned = false,
  2389. },
  2390. .impl = {
  2391. .min_access_size = 1,
  2392. .max_access_size = 8,
  2393. .unaligned = false,
  2394. },
  2395. };
  2396. /* Generate a debug exception if a watchpoint has been hit. */
  2397. static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
  2398. {
  2399. CPUState *cpu = current_cpu;
  2400. CPUClass *cc = CPU_GET_CLASS(cpu);
  2401. target_ulong vaddr;
  2402. CPUWatchpoint *wp;
  2403. assert(tcg_enabled());
  2404. if (cpu->watchpoint_hit) {
  2405. /* We re-entered the check after replacing the TB. Now raise
  2406. * the debug interrupt so that is will trigger after the
  2407. * current instruction. */
  2408. cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
  2409. return;
  2410. }
  2411. vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
  2412. vaddr = cc->adjust_watchpoint_address(cpu, vaddr, len);
  2413. QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
  2414. if (cpu_watchpoint_address_matches(wp, vaddr, len)
  2415. && (wp->flags & flags)) {
  2416. if (flags == BP_MEM_READ) {
  2417. wp->flags |= BP_WATCHPOINT_HIT_READ;
  2418. } else {
  2419. wp->flags |= BP_WATCHPOINT_HIT_WRITE;
  2420. }
  2421. wp->hitaddr = vaddr;
  2422. wp->hitattrs = attrs;
  2423. if (!cpu->watchpoint_hit) {
  2424. if (wp->flags & BP_CPU &&
  2425. !cc->debug_check_watchpoint(cpu, wp)) {
  2426. wp->flags &= ~BP_WATCHPOINT_HIT;
  2427. continue;
  2428. }
  2429. cpu->watchpoint_hit = wp;
  2430. mmap_lock();
  2431. tb_check_watchpoint(cpu);
  2432. if (wp->flags & BP_STOP_BEFORE_ACCESS) {
  2433. cpu->exception_index = EXCP_DEBUG;
  2434. mmap_unlock();
  2435. cpu_loop_exit(cpu);
  2436. } else {
  2437. /* Force execution of one insn next time. */
  2438. cpu->cflags_next_tb = 1 | curr_cflags();
  2439. mmap_unlock();
  2440. cpu_loop_exit_noexc(cpu);
  2441. }
  2442. }
  2443. } else {
  2444. wp->flags &= ~BP_WATCHPOINT_HIT;
  2445. }
  2446. }
  2447. }
  2448. /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
  2449. so these check for a hit then pass through to the normal out-of-line
  2450. phys routines. */
  2451. static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
  2452. unsigned size, MemTxAttrs attrs)
  2453. {
  2454. MemTxResult res;
  2455. uint64_t data;
  2456. int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
  2457. AddressSpace *as = current_cpu->cpu_ases[asidx].as;
  2458. check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
  2459. switch (size) {
  2460. case 1:
  2461. data = address_space_ldub(as, addr, attrs, &res);
  2462. break;
  2463. case 2:
  2464. data = address_space_lduw(as, addr, attrs, &res);
  2465. break;
  2466. case 4:
  2467. data = address_space_ldl(as, addr, attrs, &res);
  2468. break;
  2469. case 8:
  2470. data = address_space_ldq(as, addr, attrs, &res);
  2471. break;
  2472. default: abort();
  2473. }
  2474. *pdata = data;
  2475. return res;
  2476. }
  2477. static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
  2478. uint64_t val, unsigned size,
  2479. MemTxAttrs attrs)
  2480. {
  2481. MemTxResult res;
  2482. int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
  2483. AddressSpace *as = current_cpu->cpu_ases[asidx].as;
  2484. check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
  2485. switch (size) {
  2486. case 1:
  2487. address_space_stb(as, addr, val, attrs, &res);
  2488. break;
  2489. case 2:
  2490. address_space_stw(as, addr, val, attrs, &res);
  2491. break;
  2492. case 4:
  2493. address_space_stl(as, addr, val, attrs, &res);
  2494. break;
  2495. case 8:
  2496. address_space_stq(as, addr, val, attrs, &res);
  2497. break;
  2498. default: abort();
  2499. }
  2500. return res;
  2501. }
  2502. static const MemoryRegionOps watch_mem_ops = {
  2503. .read_with_attrs = watch_mem_read,
  2504. .write_with_attrs = watch_mem_write,
  2505. .endianness = DEVICE_NATIVE_ENDIAN,
  2506. .valid = {
  2507. .min_access_size = 1,
  2508. .max_access_size = 8,
  2509. .unaligned = false,
  2510. },
  2511. .impl = {
  2512. .min_access_size = 1,
  2513. .max_access_size = 8,
  2514. .unaligned = false,
  2515. },
  2516. };
  2517. static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
  2518. MemTxAttrs attrs, uint8_t *buf, hwaddr len);
  2519. static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
  2520. const uint8_t *buf, hwaddr len);
  2521. static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
  2522. bool is_write, MemTxAttrs attrs);
  2523. static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
  2524. unsigned len, MemTxAttrs attrs)
  2525. {
  2526. subpage_t *subpage = opaque;
  2527. uint8_t buf[8];
  2528. MemTxResult res;
  2529. #if defined(DEBUG_SUBPAGE)
  2530. printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
  2531. subpage, len, addr);
  2532. #endif
  2533. res = flatview_read(subpage->fv, addr + subpage->base, attrs, buf, len);
  2534. if (res) {
  2535. return res;
  2536. }
  2537. *data = ldn_p(buf, len);
  2538. return MEMTX_OK;
  2539. }
  2540. static MemTxResult subpage_write(void *opaque, hwaddr addr,
  2541. uint64_t value, unsigned len, MemTxAttrs attrs)
  2542. {
  2543. subpage_t *subpage = opaque;
  2544. uint8_t buf[8];
  2545. #if defined(DEBUG_SUBPAGE)
  2546. printf("%s: subpage %p len %u addr " TARGET_FMT_plx
  2547. " value %"PRIx64"\n",
  2548. __func__, subpage, len, addr, value);
  2549. #endif
  2550. stn_p(buf, len, value);
  2551. return flatview_write(subpage->fv, addr + subpage->base, attrs, buf, len);
  2552. }
  2553. static bool subpage_accepts(void *opaque, hwaddr addr,
  2554. unsigned len, bool is_write,
  2555. MemTxAttrs attrs)
  2556. {
  2557. subpage_t *subpage = opaque;
  2558. #if defined(DEBUG_SUBPAGE)
  2559. printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
  2560. __func__, subpage, is_write ? 'w' : 'r', len, addr);
  2561. #endif
  2562. return flatview_access_valid(subpage->fv, addr + subpage->base,
  2563. len, is_write, attrs);
  2564. }
  2565. static const MemoryRegionOps subpage_ops = {
  2566. .read_with_attrs = subpage_read,
  2567. .write_with_attrs = subpage_write,
  2568. .impl.min_access_size = 1,
  2569. .impl.max_access_size = 8,
  2570. .valid.min_access_size = 1,
  2571. .valid.max_access_size = 8,
  2572. .valid.accepts = subpage_accepts,
  2573. .endianness = DEVICE_NATIVE_ENDIAN,
  2574. };
  2575. static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
  2576. uint16_t section)
  2577. {
  2578. int idx, eidx;
  2579. if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
  2580. return -1;
  2581. idx = SUBPAGE_IDX(start);
  2582. eidx = SUBPAGE_IDX(end);
  2583. #if defined(DEBUG_SUBPAGE)
  2584. printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
  2585. __func__, mmio, start, end, idx, eidx, section);
  2586. #endif
  2587. for (; idx <= eidx; idx++) {
  2588. mmio->sub_section[idx] = section;
  2589. }
  2590. return 0;
  2591. }
  2592. static subpage_t *subpage_init(FlatView *fv, hwaddr base)
  2593. {
  2594. subpage_t *mmio;
  2595. mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t));
  2596. mmio->fv = fv;
  2597. mmio->base = base;
  2598. memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
  2599. NULL, TARGET_PAGE_SIZE);
  2600. mmio->iomem.subpage = true;
  2601. #if defined(DEBUG_SUBPAGE)
  2602. printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
  2603. mmio, base, TARGET_PAGE_SIZE);
  2604. #endif
  2605. subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
  2606. return mmio;
  2607. }
  2608. static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr)
  2609. {
  2610. assert(fv);
  2611. MemoryRegionSection section = {
  2612. .fv = fv,
  2613. .mr = mr,
  2614. .offset_within_address_space = 0,
  2615. .offset_within_region = 0,
  2616. .size = int128_2_64(),
  2617. };
  2618. return phys_section_add(map, &section);
  2619. }
  2620. static void readonly_mem_write(void *opaque, hwaddr addr,
  2621. uint64_t val, unsigned size)
  2622. {
  2623. /* Ignore any write to ROM. */
  2624. }
  2625. static bool readonly_mem_accepts(void *opaque, hwaddr addr,
  2626. unsigned size, bool is_write,
  2627. MemTxAttrs attrs)
  2628. {
  2629. return is_write;
  2630. }
  2631. /* This will only be used for writes, because reads are special cased
  2632. * to directly access the underlying host ram.
  2633. */
  2634. static const MemoryRegionOps readonly_mem_ops = {
  2635. .write = readonly_mem_write,
  2636. .valid.accepts = readonly_mem_accepts,
  2637. .endianness = DEVICE_NATIVE_ENDIAN,
  2638. .valid = {
  2639. .min_access_size = 1,
  2640. .max_access_size = 8,
  2641. .unaligned = false,
  2642. },
  2643. .impl = {
  2644. .min_access_size = 1,
  2645. .max_access_size = 8,
  2646. .unaligned = false,
  2647. },
  2648. };
  2649. MemoryRegionSection *iotlb_to_section(CPUState *cpu,
  2650. hwaddr index, MemTxAttrs attrs)
  2651. {
  2652. int asidx = cpu_asidx_from_attrs(cpu, attrs);
  2653. CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
  2654. AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
  2655. MemoryRegionSection *sections = d->map.sections;
  2656. return &sections[index & ~TARGET_PAGE_MASK];
  2657. }
  2658. static void io_mem_init(void)
  2659. {
  2660. memory_region_init_io(&io_mem_rom, NULL, &readonly_mem_ops,
  2661. NULL, NULL, UINT64_MAX);
  2662. memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
  2663. NULL, UINT64_MAX);
  2664. /* io_mem_notdirty calls tb_invalidate_phys_page_fast,
  2665. * which can be called without the iothread mutex.
  2666. */
  2667. memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
  2668. NULL, UINT64_MAX);
  2669. memory_region_clear_global_locking(&io_mem_notdirty);
  2670. memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
  2671. NULL, UINT64_MAX);
  2672. }
  2673. AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv)
  2674. {
  2675. AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
  2676. uint16_t n;
  2677. n = dummy_section(&d->map, fv, &io_mem_unassigned);
  2678. assert(n == PHYS_SECTION_UNASSIGNED);
  2679. n = dummy_section(&d->map, fv, &io_mem_notdirty);
  2680. assert(n == PHYS_SECTION_NOTDIRTY);
  2681. n = dummy_section(&d->map, fv, &io_mem_rom);
  2682. assert(n == PHYS_SECTION_ROM);
  2683. n = dummy_section(&d->map, fv, &io_mem_watch);
  2684. assert(n == PHYS_SECTION_WATCH);
  2685. d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
  2686. return d;
  2687. }
  2688. void address_space_dispatch_free(AddressSpaceDispatch *d)
  2689. {
  2690. phys_sections_free(&d->map);
  2691. g_free(d);
  2692. }
  2693. static void tcg_commit(MemoryListener *listener)
  2694. {
  2695. CPUAddressSpace *cpuas;
  2696. AddressSpaceDispatch *d;
  2697. assert(tcg_enabled());
  2698. /* since each CPU stores ram addresses in its TLB cache, we must
  2699. reset the modified entries */
  2700. cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
  2701. cpu_reloading_memory_map();
  2702. /* The CPU and TLB are protected by the iothread lock.
  2703. * We reload the dispatch pointer now because cpu_reloading_memory_map()
  2704. * may have split the RCU critical section.
  2705. */
  2706. d = address_space_to_dispatch(cpuas->as);
  2707. atomic_rcu_set(&cpuas->memory_dispatch, d);
  2708. tlb_flush(cpuas->cpu);
  2709. }
  2710. static void memory_map_init(void)
  2711. {
  2712. system_memory = g_malloc(sizeof(*system_memory));
  2713. memory_region_init(system_memory, NULL, "system", UINT64_MAX);
  2714. address_space_init(&address_space_memory, system_memory, "memory");
  2715. system_io = g_malloc(sizeof(*system_io));
  2716. memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
  2717. 65536);
  2718. address_space_init(&address_space_io, system_io, "I/O");
  2719. }
  2720. MemoryRegion *get_system_memory(void)
  2721. {
  2722. return system_memory;
  2723. }
  2724. MemoryRegion *get_system_io(void)
  2725. {
  2726. return system_io;
  2727. }
  2728. #endif /* !defined(CONFIG_USER_ONLY) */
  2729. /* physical memory access (slow version, mainly for debug) */
  2730. #if defined(CONFIG_USER_ONLY)
  2731. int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
  2732. uint8_t *buf, target_ulong len, int is_write)
  2733. {
  2734. int flags;
  2735. target_ulong l, page;
  2736. void * p;
  2737. while (len > 0) {
  2738. page = addr & TARGET_PAGE_MASK;
  2739. l = (page + TARGET_PAGE_SIZE) - addr;
  2740. if (l > len)
  2741. l = len;
  2742. flags = page_get_flags(page);
  2743. if (!(flags & PAGE_VALID))
  2744. return -1;
  2745. if (is_write) {
  2746. if (!(flags & PAGE_WRITE))
  2747. return -1;
  2748. /* XXX: this code should not depend on lock_user */
  2749. if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
  2750. return -1;
  2751. memcpy(p, buf, l);
  2752. unlock_user(p, addr, l);
  2753. } else {
  2754. if (!(flags & PAGE_READ))
  2755. return -1;
  2756. /* XXX: this code should not depend on lock_user */
  2757. if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
  2758. return -1;
  2759. memcpy(buf, p, l);
  2760. unlock_user(p, addr, 0);
  2761. }
  2762. len -= l;
  2763. buf += l;
  2764. addr += l;
  2765. }
  2766. return 0;
  2767. }
  2768. #else
  2769. static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
  2770. hwaddr length)
  2771. {
  2772. uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
  2773. addr += memory_region_get_ram_addr(mr);
  2774. /* No early return if dirty_log_mask is or becomes 0, because
  2775. * cpu_physical_memory_set_dirty_range will still call
  2776. * xen_modified_memory.
  2777. */
  2778. if (dirty_log_mask) {
  2779. dirty_log_mask =
  2780. cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
  2781. }
  2782. if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
  2783. assert(tcg_enabled());
  2784. tb_invalidate_phys_range(addr, addr + length);
  2785. dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
  2786. }
  2787. cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
  2788. }
  2789. void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size)
  2790. {
  2791. /*
  2792. * In principle this function would work on other memory region types too,
  2793. * but the ROM device use case is the only one where this operation is
  2794. * necessary. Other memory regions should use the
  2795. * address_space_read/write() APIs.
  2796. */
  2797. assert(memory_region_is_romd(mr));
  2798. invalidate_and_set_dirty(mr, addr, size);
  2799. }
  2800. static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
  2801. {
  2802. unsigned access_size_max = mr->ops->valid.max_access_size;
  2803. /* Regions are assumed to support 1-4 byte accesses unless
  2804. otherwise specified. */
  2805. if (access_size_max == 0) {
  2806. access_size_max = 4;
  2807. }
  2808. /* Bound the maximum access by the alignment of the address. */
  2809. if (!mr->ops->impl.unaligned) {
  2810. unsigned align_size_max = addr & -addr;
  2811. if (align_size_max != 0 && align_size_max < access_size_max) {
  2812. access_size_max = align_size_max;
  2813. }
  2814. }
  2815. /* Don't attempt accesses larger than the maximum. */
  2816. if (l > access_size_max) {
  2817. l = access_size_max;
  2818. }
  2819. l = pow2floor(l);
  2820. return l;
  2821. }
  2822. static bool prepare_mmio_access(MemoryRegion *mr)
  2823. {
  2824. bool unlocked = !qemu_mutex_iothread_locked();
  2825. bool release_lock = false;
  2826. if (unlocked && mr->global_locking) {
  2827. qemu_mutex_lock_iothread();
  2828. unlocked = false;
  2829. release_lock = true;
  2830. }
  2831. if (mr->flush_coalesced_mmio) {
  2832. if (unlocked) {
  2833. qemu_mutex_lock_iothread();
  2834. }
  2835. qemu_flush_coalesced_mmio_buffer();
  2836. if (unlocked) {
  2837. qemu_mutex_unlock_iothread();
  2838. }
  2839. }
  2840. return release_lock;
  2841. }
  2842. /* Called within RCU critical section. */
  2843. static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr,
  2844. MemTxAttrs attrs,
  2845. const uint8_t *buf,
  2846. hwaddr len, hwaddr addr1,
  2847. hwaddr l, MemoryRegion *mr)
  2848. {
  2849. uint8_t *ptr;
  2850. uint64_t val;
  2851. MemTxResult result = MEMTX_OK;
  2852. bool release_lock = false;
  2853. for (;;) {
  2854. if (!memory_access_is_direct(mr, true)) {
  2855. release_lock |= prepare_mmio_access(mr);
  2856. l = memory_access_size(mr, l, addr1);
  2857. /* XXX: could force current_cpu to NULL to avoid
  2858. potential bugs */
  2859. val = ldn_p(buf, l);
  2860. result |= memory_region_dispatch_write(mr, addr1, val, l, attrs);
  2861. } else {
  2862. /* RAM case */
  2863. ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false);
  2864. memcpy(ptr, buf, l);
  2865. invalidate_and_set_dirty(mr, addr1, l);
  2866. }
  2867. if (release_lock) {
  2868. qemu_mutex_unlock_iothread();
  2869. release_lock = false;
  2870. }
  2871. len -= l;
  2872. buf += l;
  2873. addr += l;
  2874. if (!len) {
  2875. break;
  2876. }
  2877. l = len;
  2878. mr = flatview_translate(fv, addr, &addr1, &l, true, attrs);
  2879. }
  2880. return result;
  2881. }
  2882. /* Called from RCU critical section. */
  2883. static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
  2884. const uint8_t *buf, hwaddr len)
  2885. {
  2886. hwaddr l;
  2887. hwaddr addr1;
  2888. MemoryRegion *mr;
  2889. MemTxResult result = MEMTX_OK;
  2890. l = len;
  2891. mr = flatview_translate(fv, addr, &addr1, &l, true, attrs);
  2892. result = flatview_write_continue(fv, addr, attrs, buf, len,
  2893. addr1, l, mr);
  2894. return result;
  2895. }
  2896. /* Called within RCU critical section. */
  2897. MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr,
  2898. MemTxAttrs attrs, uint8_t *buf,
  2899. hwaddr len, hwaddr addr1, hwaddr l,
  2900. MemoryRegion *mr)
  2901. {
  2902. uint8_t *ptr;
  2903. uint64_t val;
  2904. MemTxResult result = MEMTX_OK;
  2905. bool release_lock = false;
  2906. for (;;) {
  2907. if (!memory_access_is_direct(mr, false)) {
  2908. /* I/O case */
  2909. release_lock |= prepare_mmio_access(mr);
  2910. l = memory_access_size(mr, l, addr1);
  2911. result |= memory_region_dispatch_read(mr, addr1, &val, l, attrs);
  2912. stn_p(buf, l, val);
  2913. } else {
  2914. /* RAM case */
  2915. ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false);
  2916. memcpy(buf, ptr, l);
  2917. }
  2918. if (release_lock) {
  2919. qemu_mutex_unlock_iothread();
  2920. release_lock = false;
  2921. }
  2922. len -= l;
  2923. buf += l;
  2924. addr += l;
  2925. if (!len) {
  2926. break;
  2927. }
  2928. l = len;
  2929. mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
  2930. }
  2931. return result;
  2932. }
  2933. /* Called from RCU critical section. */
  2934. static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
  2935. MemTxAttrs attrs, uint8_t *buf, hwaddr len)
  2936. {
  2937. hwaddr l;
  2938. hwaddr addr1;
  2939. MemoryRegion *mr;
  2940. l = len;
  2941. mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
  2942. return flatview_read_continue(fv, addr, attrs, buf, len,
  2943. addr1, l, mr);
  2944. }
  2945. MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
  2946. MemTxAttrs attrs, uint8_t *buf, hwaddr len)
  2947. {
  2948. MemTxResult result = MEMTX_OK;
  2949. FlatView *fv;
  2950. if (len > 0) {
  2951. rcu_read_lock();
  2952. fv = address_space_to_flatview(as);
  2953. result = flatview_read(fv, addr, attrs, buf, len);
  2954. rcu_read_unlock();
  2955. }
  2956. return result;
  2957. }
  2958. MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
  2959. MemTxAttrs attrs,
  2960. const uint8_t *buf, hwaddr len)
  2961. {
  2962. MemTxResult result = MEMTX_OK;
  2963. FlatView *fv;
  2964. if (len > 0) {
  2965. rcu_read_lock();
  2966. fv = address_space_to_flatview(as);
  2967. result = flatview_write(fv, addr, attrs, buf, len);
  2968. rcu_read_unlock();
  2969. }
  2970. return result;
  2971. }
  2972. MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
  2973. uint8_t *buf, hwaddr len, bool is_write)
  2974. {
  2975. if (is_write) {
  2976. return address_space_write(as, addr, attrs, buf, len);
  2977. } else {
  2978. return address_space_read_full(as, addr, attrs, buf, len);
  2979. }
  2980. }
  2981. void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
  2982. hwaddr len, int is_write)
  2983. {
  2984. address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
  2985. buf, len, is_write);
  2986. }
  2987. enum write_rom_type {
  2988. WRITE_DATA,
  2989. FLUSH_CACHE,
  2990. };
  2991. static inline MemTxResult address_space_write_rom_internal(AddressSpace *as,
  2992. hwaddr addr,
  2993. MemTxAttrs attrs,
  2994. const uint8_t *buf,
  2995. hwaddr len,
  2996. enum write_rom_type type)
  2997. {
  2998. hwaddr l;
  2999. uint8_t *ptr;
  3000. hwaddr addr1;
  3001. MemoryRegion *mr;
  3002. rcu_read_lock();
  3003. while (len > 0) {
  3004. l = len;
  3005. mr = address_space_translate(as, addr, &addr1, &l, true, attrs);
  3006. if (!(memory_region_is_ram(mr) ||
  3007. memory_region_is_romd(mr))) {
  3008. l = memory_access_size(mr, l, addr1);
  3009. } else {
  3010. /* ROM/RAM case */
  3011. ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
  3012. switch (type) {
  3013. case WRITE_DATA:
  3014. memcpy(ptr, buf, l);
  3015. invalidate_and_set_dirty(mr, addr1, l);
  3016. break;
  3017. case FLUSH_CACHE:
  3018. flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
  3019. break;
  3020. }
  3021. }
  3022. len -= l;
  3023. buf += l;
  3024. addr += l;
  3025. }
  3026. rcu_read_unlock();
  3027. return MEMTX_OK;
  3028. }
  3029. /* used for ROM loading : can write in RAM and ROM */
  3030. MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr,
  3031. MemTxAttrs attrs,
  3032. const uint8_t *buf, hwaddr len)
  3033. {
  3034. return address_space_write_rom_internal(as, addr, attrs,
  3035. buf, len, WRITE_DATA);
  3036. }
  3037. void cpu_flush_icache_range(hwaddr start, hwaddr len)
  3038. {
  3039. /*
  3040. * This function should do the same thing as an icache flush that was
  3041. * triggered from within the guest. For TCG we are always cache coherent,
  3042. * so there is no need to flush anything. For KVM / Xen we need to flush
  3043. * the host's instruction cache at least.
  3044. */
  3045. if (tcg_enabled()) {
  3046. return;
  3047. }
  3048. address_space_write_rom_internal(&address_space_memory,
  3049. start, MEMTXATTRS_UNSPECIFIED,
  3050. NULL, len, FLUSH_CACHE);
  3051. }
  3052. typedef struct {
  3053. MemoryRegion *mr;
  3054. void *buffer;
  3055. hwaddr addr;
  3056. hwaddr len;
  3057. bool in_use;
  3058. } BounceBuffer;
  3059. static BounceBuffer bounce;
  3060. typedef struct MapClient {
  3061. QEMUBH *bh;
  3062. QLIST_ENTRY(MapClient) link;
  3063. } MapClient;
  3064. QemuMutex map_client_list_lock;
  3065. static QLIST_HEAD(, MapClient) map_client_list
  3066. = QLIST_HEAD_INITIALIZER(map_client_list);
  3067. static void cpu_unregister_map_client_do(MapClient *client)
  3068. {
  3069. QLIST_REMOVE(client, link);
  3070. g_free(client);
  3071. }
  3072. static void cpu_notify_map_clients_locked(void)
  3073. {
  3074. MapClient *client;
  3075. while (!QLIST_EMPTY(&map_client_list)) {
  3076. client = QLIST_FIRST(&map_client_list);
  3077. qemu_bh_schedule(client->bh);
  3078. cpu_unregister_map_client_do(client);
  3079. }
  3080. }
  3081. void cpu_register_map_client(QEMUBH *bh)
  3082. {
  3083. MapClient *client = g_malloc(sizeof(*client));
  3084. qemu_mutex_lock(&map_client_list_lock);
  3085. client->bh = bh;
  3086. QLIST_INSERT_HEAD(&map_client_list, client, link);
  3087. if (!atomic_read(&bounce.in_use)) {
  3088. cpu_notify_map_clients_locked();
  3089. }
  3090. qemu_mutex_unlock(&map_client_list_lock);
  3091. }
  3092. void cpu_exec_init_all(void)
  3093. {
  3094. qemu_mutex_init(&ram_list.mutex);
  3095. /* The data structures we set up here depend on knowing the page size,
  3096. * so no more changes can be made after this point.
  3097. * In an ideal world, nothing we did before we had finished the
  3098. * machine setup would care about the target page size, and we could
  3099. * do this much later, rather than requiring board models to state
  3100. * up front what their requirements are.
  3101. */
  3102. finalize_target_page_bits();
  3103. io_mem_init();
  3104. memory_map_init();
  3105. qemu_mutex_init(&map_client_list_lock);
  3106. }
  3107. void cpu_unregister_map_client(QEMUBH *bh)
  3108. {
  3109. MapClient *client;
  3110. qemu_mutex_lock(&map_client_list_lock);
  3111. QLIST_FOREACH(client, &map_client_list, link) {
  3112. if (client->bh == bh) {
  3113. cpu_unregister_map_client_do(client);
  3114. break;
  3115. }
  3116. }
  3117. qemu_mutex_unlock(&map_client_list_lock);
  3118. }
  3119. static void cpu_notify_map_clients(void)
  3120. {
  3121. qemu_mutex_lock(&map_client_list_lock);
  3122. cpu_notify_map_clients_locked();
  3123. qemu_mutex_unlock(&map_client_list_lock);
  3124. }
  3125. static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
  3126. bool is_write, MemTxAttrs attrs)
  3127. {
  3128. MemoryRegion *mr;
  3129. hwaddr l, xlat;
  3130. while (len > 0) {
  3131. l = len;
  3132. mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
  3133. if (!memory_access_is_direct(mr, is_write)) {
  3134. l = memory_access_size(mr, l, addr);
  3135. if (!memory_region_access_valid(mr, xlat, l, is_write, attrs)) {
  3136. return false;
  3137. }
  3138. }
  3139. len -= l;
  3140. addr += l;
  3141. }
  3142. return true;
  3143. }
  3144. bool address_space_access_valid(AddressSpace *as, hwaddr addr,
  3145. hwaddr len, bool is_write,
  3146. MemTxAttrs attrs)
  3147. {
  3148. FlatView *fv;
  3149. bool result;
  3150. rcu_read_lock();
  3151. fv = address_space_to_flatview(as);
  3152. result = flatview_access_valid(fv, addr, len, is_write, attrs);
  3153. rcu_read_unlock();
  3154. return result;
  3155. }
  3156. static hwaddr
  3157. flatview_extend_translation(FlatView *fv, hwaddr addr,
  3158. hwaddr target_len,
  3159. MemoryRegion *mr, hwaddr base, hwaddr len,
  3160. bool is_write, MemTxAttrs attrs)
  3161. {
  3162. hwaddr done = 0;
  3163. hwaddr xlat;
  3164. MemoryRegion *this_mr;
  3165. for (;;) {
  3166. target_len -= len;
  3167. addr += len;
  3168. done += len;
  3169. if (target_len == 0) {
  3170. return done;
  3171. }
  3172. len = target_len;
  3173. this_mr = flatview_translate(fv, addr, &xlat,
  3174. &len, is_write, attrs);
  3175. if (this_mr != mr || xlat != base + done) {
  3176. return done;
  3177. }
  3178. }
  3179. }
  3180. /* Map a physical memory region into a host virtual address.
  3181. * May map a subset of the requested range, given by and returned in *plen.
  3182. * May return NULL if resources needed to perform the mapping are exhausted.
  3183. * Use only for reads OR writes - not for read-modify-write operations.
  3184. * Use cpu_register_map_client() to know when retrying the map operation is
  3185. * likely to succeed.
  3186. */
  3187. void *address_space_map(AddressSpace *as,
  3188. hwaddr addr,
  3189. hwaddr *plen,
  3190. bool is_write,
  3191. MemTxAttrs attrs)
  3192. {
  3193. hwaddr len = *plen;
  3194. hwaddr l, xlat;
  3195. MemoryRegion *mr;
  3196. void *ptr;
  3197. FlatView *fv;
  3198. if (len == 0) {
  3199. return NULL;
  3200. }
  3201. l = len;
  3202. rcu_read_lock();
  3203. fv = address_space_to_flatview(as);
  3204. mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
  3205. if (!memory_access_is_direct(mr, is_write)) {
  3206. if (atomic_xchg(&bounce.in_use, true)) {
  3207. rcu_read_unlock();
  3208. return NULL;
  3209. }
  3210. /* Avoid unbounded allocations */
  3211. l = MIN(l, TARGET_PAGE_SIZE);
  3212. bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
  3213. bounce.addr = addr;
  3214. bounce.len = l;
  3215. memory_region_ref(mr);
  3216. bounce.mr = mr;
  3217. if (!is_write) {
  3218. flatview_read(fv, addr, MEMTXATTRS_UNSPECIFIED,
  3219. bounce.buffer, l);
  3220. }
  3221. rcu_read_unlock();
  3222. *plen = l;
  3223. return bounce.buffer;
  3224. }
  3225. memory_region_ref(mr);
  3226. *plen = flatview_extend_translation(fv, addr, len, mr, xlat,
  3227. l, is_write, attrs);
  3228. ptr = qemu_ram_ptr_length(mr->ram_block, xlat, plen, true);
  3229. rcu_read_unlock();
  3230. return ptr;
  3231. }
  3232. /* Unmaps a memory region previously mapped by address_space_map().
  3233. * Will also mark the memory as dirty if is_write == 1. access_len gives
  3234. * the amount of memory that was actually read or written by the caller.
  3235. */
  3236. void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
  3237. int is_write, hwaddr access_len)
  3238. {
  3239. if (buffer != bounce.buffer) {
  3240. MemoryRegion *mr;
  3241. ram_addr_t addr1;
  3242. mr = memory_region_from_host(buffer, &addr1);
  3243. assert(mr != NULL);
  3244. if (is_write) {
  3245. invalidate_and_set_dirty(mr, addr1, access_len);
  3246. }
  3247. if (xen_enabled()) {
  3248. xen_invalidate_map_cache_entry(buffer);
  3249. }
  3250. memory_region_unref(mr);
  3251. return;
  3252. }
  3253. if (is_write) {
  3254. address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
  3255. bounce.buffer, access_len);
  3256. }
  3257. qemu_vfree(bounce.buffer);
  3258. bounce.buffer = NULL;
  3259. memory_region_unref(bounce.mr);
  3260. atomic_mb_set(&bounce.in_use, false);
  3261. cpu_notify_map_clients();
  3262. }
  3263. void *cpu_physical_memory_map(hwaddr addr,
  3264. hwaddr *plen,
  3265. int is_write)
  3266. {
  3267. return address_space_map(&address_space_memory, addr, plen, is_write,
  3268. MEMTXATTRS_UNSPECIFIED);
  3269. }
  3270. void cpu_physical_memory_unmap(void *buffer, hwaddr len,
  3271. int is_write, hwaddr access_len)
  3272. {
  3273. return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
  3274. }
  3275. #define ARG1_DECL AddressSpace *as
  3276. #define ARG1 as
  3277. #define SUFFIX
  3278. #define TRANSLATE(...) address_space_translate(as, __VA_ARGS__)
  3279. #define RCU_READ_LOCK(...) rcu_read_lock()
  3280. #define RCU_READ_UNLOCK(...) rcu_read_unlock()
  3281. #include "memory_ldst.inc.c"
  3282. int64_t address_space_cache_init(MemoryRegionCache *cache,
  3283. AddressSpace *as,
  3284. hwaddr addr,
  3285. hwaddr len,
  3286. bool is_write)
  3287. {
  3288. AddressSpaceDispatch *d;
  3289. hwaddr l;
  3290. MemoryRegion *mr;
  3291. assert(len > 0);
  3292. l = len;
  3293. cache->fv = address_space_get_flatview(as);
  3294. d = flatview_to_dispatch(cache->fv);
  3295. cache->mrs = *address_space_translate_internal(d, addr, &cache->xlat, &l, true);
  3296. mr = cache->mrs.mr;
  3297. memory_region_ref(mr);
  3298. if (memory_access_is_direct(mr, is_write)) {
  3299. /* We don't care about the memory attributes here as we're only
  3300. * doing this if we found actual RAM, which behaves the same
  3301. * regardless of attributes; so UNSPECIFIED is fine.
  3302. */
  3303. l = flatview_extend_translation(cache->fv, addr, len, mr,
  3304. cache->xlat, l, is_write,
  3305. MEMTXATTRS_UNSPECIFIED);
  3306. cache->ptr = qemu_ram_ptr_length(mr->ram_block, cache->xlat, &l, true);
  3307. } else {
  3308. cache->ptr = NULL;
  3309. }
  3310. cache->len = l;
  3311. cache->is_write = is_write;
  3312. return l;
  3313. }
  3314. void address_space_cache_invalidate(MemoryRegionCache *cache,
  3315. hwaddr addr,
  3316. hwaddr access_len)
  3317. {
  3318. assert(cache->is_write);
  3319. if (likely(cache->ptr)) {
  3320. invalidate_and_set_dirty(cache->mrs.mr, addr + cache->xlat, access_len);
  3321. }
  3322. }
  3323. void address_space_cache_destroy(MemoryRegionCache *cache)
  3324. {
  3325. if (!cache->mrs.mr) {
  3326. return;
  3327. }
  3328. if (xen_enabled()) {
  3329. xen_invalidate_map_cache_entry(cache->ptr);
  3330. }
  3331. memory_region_unref(cache->mrs.mr);
  3332. flatview_unref(cache->fv);
  3333. cache->mrs.mr = NULL;
  3334. cache->fv = NULL;
  3335. }
  3336. /* Called from RCU critical section. This function has the same
  3337. * semantics as address_space_translate, but it only works on a
  3338. * predefined range of a MemoryRegion that was mapped with
  3339. * address_space_cache_init.
  3340. */
  3341. static inline MemoryRegion *address_space_translate_cached(
  3342. MemoryRegionCache *cache, hwaddr addr, hwaddr *xlat,
  3343. hwaddr *plen, bool is_write, MemTxAttrs attrs)
  3344. {
  3345. MemoryRegionSection section;
  3346. MemoryRegion *mr;
  3347. IOMMUMemoryRegion *iommu_mr;
  3348. AddressSpace *target_as;
  3349. assert(!cache->ptr);
  3350. *xlat = addr + cache->xlat;
  3351. mr = cache->mrs.mr;
  3352. iommu_mr = memory_region_get_iommu(mr);
  3353. if (!iommu_mr) {
  3354. /* MMIO region. */
  3355. return mr;
  3356. }
  3357. section = address_space_translate_iommu(iommu_mr, xlat, plen,
  3358. NULL, is_write, true,
  3359. &target_as, attrs);
  3360. return section.mr;
  3361. }
  3362. /* Called from RCU critical section. address_space_read_cached uses this
  3363. * out of line function when the target is an MMIO or IOMMU region.
  3364. */
  3365. void
  3366. address_space_read_cached_slow(MemoryRegionCache *cache, hwaddr addr,
  3367. void *buf, hwaddr len)
  3368. {
  3369. hwaddr addr1, l;
  3370. MemoryRegion *mr;
  3371. l = len;
  3372. mr = address_space_translate_cached(cache, addr, &addr1, &l, false,
  3373. MEMTXATTRS_UNSPECIFIED);
  3374. flatview_read_continue(cache->fv,
  3375. addr, MEMTXATTRS_UNSPECIFIED, buf, len,
  3376. addr1, l, mr);
  3377. }
  3378. /* Called from RCU critical section. address_space_write_cached uses this
  3379. * out of line function when the target is an MMIO or IOMMU region.
  3380. */
  3381. void
  3382. address_space_write_cached_slow(MemoryRegionCache *cache, hwaddr addr,
  3383. const void *buf, hwaddr len)
  3384. {
  3385. hwaddr addr1, l;
  3386. MemoryRegion *mr;
  3387. l = len;
  3388. mr = address_space_translate_cached(cache, addr, &addr1, &l, true,
  3389. MEMTXATTRS_UNSPECIFIED);
  3390. flatview_write_continue(cache->fv,
  3391. addr, MEMTXATTRS_UNSPECIFIED, buf, len,
  3392. addr1, l, mr);
  3393. }
  3394. #define ARG1_DECL MemoryRegionCache *cache
  3395. #define ARG1 cache
  3396. #define SUFFIX _cached_slow
  3397. #define TRANSLATE(...) address_space_translate_cached(cache, __VA_ARGS__)
  3398. #define RCU_READ_LOCK() ((void)0)
  3399. #define RCU_READ_UNLOCK() ((void)0)
  3400. #include "memory_ldst.inc.c"
  3401. /* virtual memory access for debug (includes writing to ROM) */
  3402. int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
  3403. uint8_t *buf, target_ulong len, int is_write)
  3404. {
  3405. hwaddr phys_addr;
  3406. target_ulong l, page;
  3407. cpu_synchronize_state(cpu);
  3408. while (len > 0) {
  3409. int asidx;
  3410. MemTxAttrs attrs;
  3411. page = addr & TARGET_PAGE_MASK;
  3412. phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
  3413. asidx = cpu_asidx_from_attrs(cpu, attrs);
  3414. /* if no physical page mapped, return an error */
  3415. if (phys_addr == -1)
  3416. return -1;
  3417. l = (page + TARGET_PAGE_SIZE) - addr;
  3418. if (l > len)
  3419. l = len;
  3420. phys_addr += (addr & ~TARGET_PAGE_MASK);
  3421. if (is_write) {
  3422. address_space_write_rom(cpu->cpu_ases[asidx].as, phys_addr,
  3423. attrs, buf, l);
  3424. } else {
  3425. address_space_rw(cpu->cpu_ases[asidx].as, phys_addr,
  3426. attrs, buf, l, 0);
  3427. }
  3428. len -= l;
  3429. buf += l;
  3430. addr += l;
  3431. }
  3432. return 0;
  3433. }
  3434. /*
  3435. * Allows code that needs to deal with migration bitmaps etc to still be built
  3436. * target independent.
  3437. */
  3438. size_t qemu_target_page_size(void)
  3439. {
  3440. return TARGET_PAGE_SIZE;
  3441. }
  3442. int qemu_target_page_bits(void)
  3443. {
  3444. return TARGET_PAGE_BITS;
  3445. }
  3446. int qemu_target_page_bits_min(void)
  3447. {
  3448. return TARGET_PAGE_BITS_MIN;
  3449. }
  3450. #endif
  3451. bool target_words_bigendian(void)
  3452. {
  3453. #if defined(TARGET_WORDS_BIGENDIAN)
  3454. return true;
  3455. #else
  3456. return false;
  3457. #endif
  3458. }
  3459. #ifndef CONFIG_USER_ONLY
  3460. bool cpu_physical_memory_is_io(hwaddr phys_addr)
  3461. {
  3462. MemoryRegion*mr;
  3463. hwaddr l = 1;
  3464. bool res;
  3465. rcu_read_lock();
  3466. mr = address_space_translate(&address_space_memory,
  3467. phys_addr, &phys_addr, &l, false,
  3468. MEMTXATTRS_UNSPECIFIED);
  3469. res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
  3470. rcu_read_unlock();
  3471. return res;
  3472. }
  3473. int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
  3474. {
  3475. RAMBlock *block;
  3476. int ret = 0;
  3477. rcu_read_lock();
  3478. RAMBLOCK_FOREACH(block) {
  3479. ret = func(block, opaque);
  3480. if (ret) {
  3481. break;
  3482. }
  3483. }
  3484. rcu_read_unlock();
  3485. return ret;
  3486. }
  3487. /*
  3488. * Unmap pages of memory from start to start+length such that
  3489. * they a) read as 0, b) Trigger whatever fault mechanism
  3490. * the OS provides for postcopy.
  3491. * The pages must be unmapped by the end of the function.
  3492. * Returns: 0 on success, none-0 on failure
  3493. *
  3494. */
  3495. int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length)
  3496. {
  3497. int ret = -1;
  3498. uint8_t *host_startaddr = rb->host + start;
  3499. if ((uintptr_t)host_startaddr & (rb->page_size - 1)) {
  3500. error_report("ram_block_discard_range: Unaligned start address: %p",
  3501. host_startaddr);
  3502. goto err;
  3503. }
  3504. if ((start + length) <= rb->used_length) {
  3505. bool need_madvise, need_fallocate;
  3506. uint8_t *host_endaddr = host_startaddr + length;
  3507. if ((uintptr_t)host_endaddr & (rb->page_size - 1)) {
  3508. error_report("ram_block_discard_range: Unaligned end address: %p",
  3509. host_endaddr);
  3510. goto err;
  3511. }
  3512. errno = ENOTSUP; /* If we are missing MADVISE etc */
  3513. /* The logic here is messy;
  3514. * madvise DONTNEED fails for hugepages
  3515. * fallocate works on hugepages and shmem
  3516. */
  3517. need_madvise = (rb->page_size == qemu_host_page_size);
  3518. need_fallocate = rb->fd != -1;
  3519. if (need_fallocate) {
  3520. /* For a file, this causes the area of the file to be zero'd
  3521. * if read, and for hugetlbfs also causes it to be unmapped
  3522. * so a userfault will trigger.
  3523. */
  3524. #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
  3525. ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
  3526. start, length);
  3527. if (ret) {
  3528. ret = -errno;
  3529. error_report("ram_block_discard_range: Failed to fallocate "
  3530. "%s:%" PRIx64 " +%zx (%d)",
  3531. rb->idstr, start, length, ret);
  3532. goto err;
  3533. }
  3534. #else
  3535. ret = -ENOSYS;
  3536. error_report("ram_block_discard_range: fallocate not available/file"
  3537. "%s:%" PRIx64 " +%zx (%d)",
  3538. rb->idstr, start, length, ret);
  3539. goto err;
  3540. #endif
  3541. }
  3542. if (need_madvise) {
  3543. /* For normal RAM this causes it to be unmapped,
  3544. * for shared memory it causes the local mapping to disappear
  3545. * and to fall back on the file contents (which we just
  3546. * fallocate'd away).
  3547. */
  3548. #if defined(CONFIG_MADVISE)
  3549. ret = madvise(host_startaddr, length, MADV_DONTNEED);
  3550. if (ret) {
  3551. ret = -errno;
  3552. error_report("ram_block_discard_range: Failed to discard range "
  3553. "%s:%" PRIx64 " +%zx (%d)",
  3554. rb->idstr, start, length, ret);
  3555. goto err;
  3556. }
  3557. #else
  3558. ret = -ENOSYS;
  3559. error_report("ram_block_discard_range: MADVISE not available"
  3560. "%s:%" PRIx64 " +%zx (%d)",
  3561. rb->idstr, start, length, ret);
  3562. goto err;
  3563. #endif
  3564. }
  3565. trace_ram_block_discard_range(rb->idstr, host_startaddr, length,
  3566. need_madvise, need_fallocate, ret);
  3567. } else {
  3568. error_report("ram_block_discard_range: Overrun block '%s' (%" PRIu64
  3569. "/%zx/" RAM_ADDR_FMT")",
  3570. rb->idstr, start, length, rb->used_length);
  3571. }
  3572. err:
  3573. return ret;
  3574. }
  3575. bool ramblock_is_pmem(RAMBlock *rb)
  3576. {
  3577. return rb->flags & RAM_PMEM;
  3578. }
  3579. #endif
  3580. void page_size_init(void)
  3581. {
  3582. /* NOTE: we can always suppose that qemu_host_page_size >=
  3583. TARGET_PAGE_SIZE */
  3584. if (qemu_host_page_size == 0) {
  3585. qemu_host_page_size = qemu_real_host_page_size;
  3586. }
  3587. if (qemu_host_page_size < TARGET_PAGE_SIZE) {
  3588. qemu_host_page_size = TARGET_PAGE_SIZE;
  3589. }
  3590. qemu_host_page_mask = -(intptr_t)qemu_host_page_size;
  3591. }
  3592. #if !defined(CONFIG_USER_ONLY)
  3593. static void mtree_print_phys_entries(int start, int end, int skip, int ptr)
  3594. {
  3595. if (start == end - 1) {
  3596. qemu_printf("\t%3d ", start);
  3597. } else {
  3598. qemu_printf("\t%3d..%-3d ", start, end - 1);
  3599. }
  3600. qemu_printf(" skip=%d ", skip);
  3601. if (ptr == PHYS_MAP_NODE_NIL) {
  3602. qemu_printf(" ptr=NIL");
  3603. } else if (!skip) {
  3604. qemu_printf(" ptr=#%d", ptr);
  3605. } else {
  3606. qemu_printf(" ptr=[%d]", ptr);
  3607. }
  3608. qemu_printf("\n");
  3609. }
  3610. #define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \
  3611. int128_sub((size), int128_one())) : 0)
  3612. void mtree_print_dispatch(AddressSpaceDispatch *d, MemoryRegion *root)
  3613. {
  3614. int i;
  3615. qemu_printf(" Dispatch\n");
  3616. qemu_printf(" Physical sections\n");
  3617. for (i = 0; i < d->map.sections_nb; ++i) {
  3618. MemoryRegionSection *s = d->map.sections + i;
  3619. const char *names[] = { " [unassigned]", " [not dirty]",
  3620. " [ROM]", " [watch]" };
  3621. qemu_printf(" #%d @" TARGET_FMT_plx ".." TARGET_FMT_plx
  3622. " %s%s%s%s%s",
  3623. i,
  3624. s->offset_within_address_space,
  3625. s->offset_within_address_space + MR_SIZE(s->mr->size),
  3626. s->mr->name ? s->mr->name : "(noname)",
  3627. i < ARRAY_SIZE(names) ? names[i] : "",
  3628. s->mr == root ? " [ROOT]" : "",
  3629. s == d->mru_section ? " [MRU]" : "",
  3630. s->mr->is_iommu ? " [iommu]" : "");
  3631. if (s->mr->alias) {
  3632. qemu_printf(" alias=%s", s->mr->alias->name ?
  3633. s->mr->alias->name : "noname");
  3634. }
  3635. qemu_printf("\n");
  3636. }
  3637. qemu_printf(" Nodes (%d bits per level, %d levels) ptr=[%d] skip=%d\n",
  3638. P_L2_BITS, P_L2_LEVELS, d->phys_map.ptr, d->phys_map.skip);
  3639. for (i = 0; i < d->map.nodes_nb; ++i) {
  3640. int j, jprev;
  3641. PhysPageEntry prev;
  3642. Node *n = d->map.nodes + i;
  3643. qemu_printf(" [%d]\n", i);
  3644. for (j = 0, jprev = 0, prev = *n[0]; j < ARRAY_SIZE(*n); ++j) {
  3645. PhysPageEntry *pe = *n + j;
  3646. if (pe->ptr == prev.ptr && pe->skip == prev.skip) {
  3647. continue;
  3648. }
  3649. mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
  3650. jprev = j;
  3651. prev = *pe;
  3652. }
  3653. if (jprev != ARRAY_SIZE(*n)) {
  3654. mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
  3655. }
  3656. }
  3657. }
  3658. #endif