Vous ne pouvez pas sélectionner plus de 25 sujets Les noms de sujets doivent commencer par une lettre ou un nombre, peuvent contenir des tirets ('-') et peuvent comporter jusqu'à 35 caractères.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074
  1. /*
  2. * Virtual page mapping
  3. *
  4. * Copyright (c) 2003 Fabrice Bellard
  5. *
  6. * This library is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU Lesser General Public
  8. * License as published by the Free Software Foundation; either
  9. * version 2 of the License, or (at your option) any later version.
  10. *
  11. * This library is distributed in the hope that it will be useful,
  12. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. * Lesser General Public License for more details.
  15. *
  16. * You should have received a copy of the GNU Lesser General Public
  17. * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18. */
  19. #include "qemu/osdep.h"
  20. #include "qemu-common.h"
  21. #include "qapi/error.h"
  22. #include "qemu/cutils.h"
  23. #include "cpu.h"
  24. #include "exec/exec-all.h"
  25. #include "exec/target_page.h"
  26. #include "tcg.h"
  27. #include "hw/qdev-core.h"
  28. #include "hw/qdev-properties.h"
  29. #if !defined(CONFIG_USER_ONLY)
  30. #include "hw/boards.h"
  31. #include "hw/xen/xen.h"
  32. #endif
  33. #include "sysemu/kvm.h"
  34. #include "sysemu/sysemu.h"
  35. #include "sysemu/tcg.h"
  36. #include "qemu/timer.h"
  37. #include "qemu/config-file.h"
  38. #include "qemu/error-report.h"
  39. #include "qemu/qemu-print.h"
  40. #if defined(CONFIG_USER_ONLY)
  41. #include "qemu.h"
  42. #else /* !CONFIG_USER_ONLY */
  43. #include "exec/memory.h"
  44. #include "exec/ioport.h"
  45. #include "sysemu/dma.h"
  46. #include "sysemu/hostmem.h"
  47. #include "sysemu/hw_accel.h"
  48. #include "exec/address-spaces.h"
  49. #include "sysemu/xen-mapcache.h"
  50. #include "trace-root.h"
  51. #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
  52. #include <linux/falloc.h>
  53. #endif
  54. #endif
  55. #include "qemu/rcu_queue.h"
  56. #include "qemu/main-loop.h"
  57. #include "translate-all.h"
  58. #include "sysemu/replay.h"
  59. #include "exec/memory-internal.h"
  60. #include "exec/ram_addr.h"
  61. #include "exec/log.h"
  62. #include "migration/vmstate.h"
  63. #include "qemu/range.h"
  64. #ifndef _WIN32
  65. #include "qemu/mmap-alloc.h"
  66. #endif
  67. #include "monitor/monitor.h"
  68. //#define DEBUG_SUBPAGE
  69. #if !defined(CONFIG_USER_ONLY)
  70. /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
  71. * are protected by the ramlist lock.
  72. */
  73. RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
  74. static MemoryRegion *system_memory;
  75. static MemoryRegion *system_io;
  76. AddressSpace address_space_io;
  77. AddressSpace address_space_memory;
  78. static MemoryRegion io_mem_unassigned;
  79. #endif
  80. #ifdef TARGET_PAGE_BITS_VARY
  81. int target_page_bits;
  82. bool target_page_bits_decided;
  83. #endif
  84. CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
  85. /* current CPU in the current thread. It is only valid inside
  86. cpu_exec() */
  87. __thread CPUState *current_cpu;
  88. /* 0 = Do not count executed instructions.
  89. 1 = Precise instruction counting.
  90. 2 = Adaptive rate instruction counting. */
  91. int use_icount;
  92. uintptr_t qemu_host_page_size;
  93. intptr_t qemu_host_page_mask;
  94. bool set_preferred_target_page_bits(int bits)
  95. {
  96. /* The target page size is the lowest common denominator for all
  97. * the CPUs in the system, so we can only make it smaller, never
  98. * larger. And we can't make it smaller once we've committed to
  99. * a particular size.
  100. */
  101. #ifdef TARGET_PAGE_BITS_VARY
  102. assert(bits >= TARGET_PAGE_BITS_MIN);
  103. if (target_page_bits == 0 || target_page_bits > bits) {
  104. if (target_page_bits_decided) {
  105. return false;
  106. }
  107. target_page_bits = bits;
  108. }
  109. #endif
  110. return true;
  111. }
  112. #if !defined(CONFIG_USER_ONLY)
  113. static void finalize_target_page_bits(void)
  114. {
  115. #ifdef TARGET_PAGE_BITS_VARY
  116. if (target_page_bits == 0) {
  117. target_page_bits = TARGET_PAGE_BITS_MIN;
  118. }
  119. target_page_bits_decided = true;
  120. #endif
  121. }
  122. typedef struct PhysPageEntry PhysPageEntry;
  123. struct PhysPageEntry {
  124. /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
  125. uint32_t skip : 6;
  126. /* index into phys_sections (!skip) or phys_map_nodes (skip) */
  127. uint32_t ptr : 26;
  128. };
  129. #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
  130. /* Size of the L2 (and L3, etc) page tables. */
  131. #define ADDR_SPACE_BITS 64
  132. #define P_L2_BITS 9
  133. #define P_L2_SIZE (1 << P_L2_BITS)
  134. #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
  135. typedef PhysPageEntry Node[P_L2_SIZE];
  136. typedef struct PhysPageMap {
  137. struct rcu_head rcu;
  138. unsigned sections_nb;
  139. unsigned sections_nb_alloc;
  140. unsigned nodes_nb;
  141. unsigned nodes_nb_alloc;
  142. Node *nodes;
  143. MemoryRegionSection *sections;
  144. } PhysPageMap;
  145. struct AddressSpaceDispatch {
  146. MemoryRegionSection *mru_section;
  147. /* This is a multi-level map on the physical address space.
  148. * The bottom level has pointers to MemoryRegionSections.
  149. */
  150. PhysPageEntry phys_map;
  151. PhysPageMap map;
  152. };
  153. #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
  154. typedef struct subpage_t {
  155. MemoryRegion iomem;
  156. FlatView *fv;
  157. hwaddr base;
  158. uint16_t sub_section[];
  159. } subpage_t;
  160. #define PHYS_SECTION_UNASSIGNED 0
  161. static void io_mem_init(void);
  162. static void memory_map_init(void);
  163. static void tcg_log_global_after_sync(MemoryListener *listener);
  164. static void tcg_commit(MemoryListener *listener);
  165. /**
  166. * CPUAddressSpace: all the information a CPU needs about an AddressSpace
  167. * @cpu: the CPU whose AddressSpace this is
  168. * @as: the AddressSpace itself
  169. * @memory_dispatch: its dispatch pointer (cached, RCU protected)
  170. * @tcg_as_listener: listener for tracking changes to the AddressSpace
  171. */
  172. struct CPUAddressSpace {
  173. CPUState *cpu;
  174. AddressSpace *as;
  175. struct AddressSpaceDispatch *memory_dispatch;
  176. MemoryListener tcg_as_listener;
  177. };
  178. struct DirtyBitmapSnapshot {
  179. ram_addr_t start;
  180. ram_addr_t end;
  181. unsigned long dirty[];
  182. };
  183. #endif
  184. #if !defined(CONFIG_USER_ONLY)
  185. static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
  186. {
  187. static unsigned alloc_hint = 16;
  188. if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
  189. map->nodes_nb_alloc = MAX(alloc_hint, map->nodes_nb + nodes);
  190. map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
  191. alloc_hint = map->nodes_nb_alloc;
  192. }
  193. }
  194. static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
  195. {
  196. unsigned i;
  197. uint32_t ret;
  198. PhysPageEntry e;
  199. PhysPageEntry *p;
  200. ret = map->nodes_nb++;
  201. p = map->nodes[ret];
  202. assert(ret != PHYS_MAP_NODE_NIL);
  203. assert(ret != map->nodes_nb_alloc);
  204. e.skip = leaf ? 0 : 1;
  205. e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
  206. for (i = 0; i < P_L2_SIZE; ++i) {
  207. memcpy(&p[i], &e, sizeof(e));
  208. }
  209. return ret;
  210. }
  211. static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
  212. hwaddr *index, uint64_t *nb, uint16_t leaf,
  213. int level)
  214. {
  215. PhysPageEntry *p;
  216. hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
  217. if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
  218. lp->ptr = phys_map_node_alloc(map, level == 0);
  219. }
  220. p = map->nodes[lp->ptr];
  221. lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
  222. while (*nb && lp < &p[P_L2_SIZE]) {
  223. if ((*index & (step - 1)) == 0 && *nb >= step) {
  224. lp->skip = 0;
  225. lp->ptr = leaf;
  226. *index += step;
  227. *nb -= step;
  228. } else {
  229. phys_page_set_level(map, lp, index, nb, leaf, level - 1);
  230. }
  231. ++lp;
  232. }
  233. }
  234. static void phys_page_set(AddressSpaceDispatch *d,
  235. hwaddr index, uint64_t nb,
  236. uint16_t leaf)
  237. {
  238. /* Wildly overreserve - it doesn't matter much. */
  239. phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
  240. phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
  241. }
  242. /* Compact a non leaf page entry. Simply detect that the entry has a single child,
  243. * and update our entry so we can skip it and go directly to the destination.
  244. */
  245. static void phys_page_compact(PhysPageEntry *lp, Node *nodes)
  246. {
  247. unsigned valid_ptr = P_L2_SIZE;
  248. int valid = 0;
  249. PhysPageEntry *p;
  250. int i;
  251. if (lp->ptr == PHYS_MAP_NODE_NIL) {
  252. return;
  253. }
  254. p = nodes[lp->ptr];
  255. for (i = 0; i < P_L2_SIZE; i++) {
  256. if (p[i].ptr == PHYS_MAP_NODE_NIL) {
  257. continue;
  258. }
  259. valid_ptr = i;
  260. valid++;
  261. if (p[i].skip) {
  262. phys_page_compact(&p[i], nodes);
  263. }
  264. }
  265. /* We can only compress if there's only one child. */
  266. if (valid != 1) {
  267. return;
  268. }
  269. assert(valid_ptr < P_L2_SIZE);
  270. /* Don't compress if it won't fit in the # of bits we have. */
  271. if (P_L2_LEVELS >= (1 << 6) &&
  272. lp->skip + p[valid_ptr].skip >= (1 << 6)) {
  273. return;
  274. }
  275. lp->ptr = p[valid_ptr].ptr;
  276. if (!p[valid_ptr].skip) {
  277. /* If our only child is a leaf, make this a leaf. */
  278. /* By design, we should have made this node a leaf to begin with so we
  279. * should never reach here.
  280. * But since it's so simple to handle this, let's do it just in case we
  281. * change this rule.
  282. */
  283. lp->skip = 0;
  284. } else {
  285. lp->skip += p[valid_ptr].skip;
  286. }
  287. }
  288. void address_space_dispatch_compact(AddressSpaceDispatch *d)
  289. {
  290. if (d->phys_map.skip) {
  291. phys_page_compact(&d->phys_map, d->map.nodes);
  292. }
  293. }
  294. static inline bool section_covers_addr(const MemoryRegionSection *section,
  295. hwaddr addr)
  296. {
  297. /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
  298. * the section must cover the entire address space.
  299. */
  300. return int128_gethi(section->size) ||
  301. range_covers_byte(section->offset_within_address_space,
  302. int128_getlo(section->size), addr);
  303. }
  304. static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr addr)
  305. {
  306. PhysPageEntry lp = d->phys_map, *p;
  307. Node *nodes = d->map.nodes;
  308. MemoryRegionSection *sections = d->map.sections;
  309. hwaddr index = addr >> TARGET_PAGE_BITS;
  310. int i;
  311. for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
  312. if (lp.ptr == PHYS_MAP_NODE_NIL) {
  313. return &sections[PHYS_SECTION_UNASSIGNED];
  314. }
  315. p = nodes[lp.ptr];
  316. lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
  317. }
  318. if (section_covers_addr(&sections[lp.ptr], addr)) {
  319. return &sections[lp.ptr];
  320. } else {
  321. return &sections[PHYS_SECTION_UNASSIGNED];
  322. }
  323. }
  324. /* Called from RCU critical section */
  325. static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
  326. hwaddr addr,
  327. bool resolve_subpage)
  328. {
  329. MemoryRegionSection *section = atomic_read(&d->mru_section);
  330. subpage_t *subpage;
  331. if (!section || section == &d->map.sections[PHYS_SECTION_UNASSIGNED] ||
  332. !section_covers_addr(section, addr)) {
  333. section = phys_page_find(d, addr);
  334. atomic_set(&d->mru_section, section);
  335. }
  336. if (resolve_subpage && section->mr->subpage) {
  337. subpage = container_of(section->mr, subpage_t, iomem);
  338. section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
  339. }
  340. return section;
  341. }
  342. /* Called from RCU critical section */
  343. static MemoryRegionSection *
  344. address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
  345. hwaddr *plen, bool resolve_subpage)
  346. {
  347. MemoryRegionSection *section;
  348. MemoryRegion *mr;
  349. Int128 diff;
  350. section = address_space_lookup_region(d, addr, resolve_subpage);
  351. /* Compute offset within MemoryRegionSection */
  352. addr -= section->offset_within_address_space;
  353. /* Compute offset within MemoryRegion */
  354. *xlat = addr + section->offset_within_region;
  355. mr = section->mr;
  356. /* MMIO registers can be expected to perform full-width accesses based only
  357. * on their address, without considering adjacent registers that could
  358. * decode to completely different MemoryRegions. When such registers
  359. * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
  360. * regions overlap wildly. For this reason we cannot clamp the accesses
  361. * here.
  362. *
  363. * If the length is small (as is the case for address_space_ldl/stl),
  364. * everything works fine. If the incoming length is large, however,
  365. * the caller really has to do the clamping through memory_access_size.
  366. */
  367. if (memory_region_is_ram(mr)) {
  368. diff = int128_sub(section->size, int128_make64(addr));
  369. *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
  370. }
  371. return section;
  372. }
  373. /**
  374. * address_space_translate_iommu - translate an address through an IOMMU
  375. * memory region and then through the target address space.
  376. *
  377. * @iommu_mr: the IOMMU memory region that we start the translation from
  378. * @addr: the address to be translated through the MMU
  379. * @xlat: the translated address offset within the destination memory region.
  380. * It cannot be %NULL.
  381. * @plen_out: valid read/write length of the translated address. It
  382. * cannot be %NULL.
  383. * @page_mask_out: page mask for the translated address. This
  384. * should only be meaningful for IOMMU translated
  385. * addresses, since there may be huge pages that this bit
  386. * would tell. It can be %NULL if we don't care about it.
  387. * @is_write: whether the translation operation is for write
  388. * @is_mmio: whether this can be MMIO, set true if it can
  389. * @target_as: the address space targeted by the IOMMU
  390. * @attrs: transaction attributes
  391. *
  392. * This function is called from RCU critical section. It is the common
  393. * part of flatview_do_translate and address_space_translate_cached.
  394. */
  395. static MemoryRegionSection address_space_translate_iommu(IOMMUMemoryRegion *iommu_mr,
  396. hwaddr *xlat,
  397. hwaddr *plen_out,
  398. hwaddr *page_mask_out,
  399. bool is_write,
  400. bool is_mmio,
  401. AddressSpace **target_as,
  402. MemTxAttrs attrs)
  403. {
  404. MemoryRegionSection *section;
  405. hwaddr page_mask = (hwaddr)-1;
  406. do {
  407. hwaddr addr = *xlat;
  408. IOMMUMemoryRegionClass *imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
  409. int iommu_idx = 0;
  410. IOMMUTLBEntry iotlb;
  411. if (imrc->attrs_to_index) {
  412. iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
  413. }
  414. iotlb = imrc->translate(iommu_mr, addr, is_write ?
  415. IOMMU_WO : IOMMU_RO, iommu_idx);
  416. if (!(iotlb.perm & (1 << is_write))) {
  417. goto unassigned;
  418. }
  419. addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
  420. | (addr & iotlb.addr_mask));
  421. page_mask &= iotlb.addr_mask;
  422. *plen_out = MIN(*plen_out, (addr | iotlb.addr_mask) - addr + 1);
  423. *target_as = iotlb.target_as;
  424. section = address_space_translate_internal(
  425. address_space_to_dispatch(iotlb.target_as), addr, xlat,
  426. plen_out, is_mmio);
  427. iommu_mr = memory_region_get_iommu(section->mr);
  428. } while (unlikely(iommu_mr));
  429. if (page_mask_out) {
  430. *page_mask_out = page_mask;
  431. }
  432. return *section;
  433. unassigned:
  434. return (MemoryRegionSection) { .mr = &io_mem_unassigned };
  435. }
  436. /**
  437. * flatview_do_translate - translate an address in FlatView
  438. *
  439. * @fv: the flat view that we want to translate on
  440. * @addr: the address to be translated in above address space
  441. * @xlat: the translated address offset within memory region. It
  442. * cannot be @NULL.
  443. * @plen_out: valid read/write length of the translated address. It
  444. * can be @NULL when we don't care about it.
  445. * @page_mask_out: page mask for the translated address. This
  446. * should only be meaningful for IOMMU translated
  447. * addresses, since there may be huge pages that this bit
  448. * would tell. It can be @NULL if we don't care about it.
  449. * @is_write: whether the translation operation is for write
  450. * @is_mmio: whether this can be MMIO, set true if it can
  451. * @target_as: the address space targeted by the IOMMU
  452. * @attrs: memory transaction attributes
  453. *
  454. * This function is called from RCU critical section
  455. */
  456. static MemoryRegionSection flatview_do_translate(FlatView *fv,
  457. hwaddr addr,
  458. hwaddr *xlat,
  459. hwaddr *plen_out,
  460. hwaddr *page_mask_out,
  461. bool is_write,
  462. bool is_mmio,
  463. AddressSpace **target_as,
  464. MemTxAttrs attrs)
  465. {
  466. MemoryRegionSection *section;
  467. IOMMUMemoryRegion *iommu_mr;
  468. hwaddr plen = (hwaddr)(-1);
  469. if (!plen_out) {
  470. plen_out = &plen;
  471. }
  472. section = address_space_translate_internal(
  473. flatview_to_dispatch(fv), addr, xlat,
  474. plen_out, is_mmio);
  475. iommu_mr = memory_region_get_iommu(section->mr);
  476. if (unlikely(iommu_mr)) {
  477. return address_space_translate_iommu(iommu_mr, xlat,
  478. plen_out, page_mask_out,
  479. is_write, is_mmio,
  480. target_as, attrs);
  481. }
  482. if (page_mask_out) {
  483. /* Not behind an IOMMU, use default page size. */
  484. *page_mask_out = ~TARGET_PAGE_MASK;
  485. }
  486. return *section;
  487. }
  488. /* Called from RCU critical section */
  489. IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr,
  490. bool is_write, MemTxAttrs attrs)
  491. {
  492. MemoryRegionSection section;
  493. hwaddr xlat, page_mask;
  494. /*
  495. * This can never be MMIO, and we don't really care about plen,
  496. * but page mask.
  497. */
  498. section = flatview_do_translate(address_space_to_flatview(as), addr, &xlat,
  499. NULL, &page_mask, is_write, false, &as,
  500. attrs);
  501. /* Illegal translation */
  502. if (section.mr == &io_mem_unassigned) {
  503. goto iotlb_fail;
  504. }
  505. /* Convert memory region offset into address space offset */
  506. xlat += section.offset_within_address_space -
  507. section.offset_within_region;
  508. return (IOMMUTLBEntry) {
  509. .target_as = as,
  510. .iova = addr & ~page_mask,
  511. .translated_addr = xlat & ~page_mask,
  512. .addr_mask = page_mask,
  513. /* IOTLBs are for DMAs, and DMA only allows on RAMs. */
  514. .perm = IOMMU_RW,
  515. };
  516. iotlb_fail:
  517. return (IOMMUTLBEntry) {0};
  518. }
  519. /* Called from RCU critical section */
  520. MemoryRegion *flatview_translate(FlatView *fv, hwaddr addr, hwaddr *xlat,
  521. hwaddr *plen, bool is_write,
  522. MemTxAttrs attrs)
  523. {
  524. MemoryRegion *mr;
  525. MemoryRegionSection section;
  526. AddressSpace *as = NULL;
  527. /* This can be MMIO, so setup MMIO bit. */
  528. section = flatview_do_translate(fv, addr, xlat, plen, NULL,
  529. is_write, true, &as, attrs);
  530. mr = section.mr;
  531. if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
  532. hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
  533. *plen = MIN(page, *plen);
  534. }
  535. return mr;
  536. }
  537. typedef struct TCGIOMMUNotifier {
  538. IOMMUNotifier n;
  539. MemoryRegion *mr;
  540. CPUState *cpu;
  541. int iommu_idx;
  542. bool active;
  543. } TCGIOMMUNotifier;
  544. static void tcg_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
  545. {
  546. TCGIOMMUNotifier *notifier = container_of(n, TCGIOMMUNotifier, n);
  547. if (!notifier->active) {
  548. return;
  549. }
  550. tlb_flush(notifier->cpu);
  551. notifier->active = false;
  552. /* We leave the notifier struct on the list to avoid reallocating it later.
  553. * Generally the number of IOMMUs a CPU deals with will be small.
  554. * In any case we can't unregister the iommu notifier from a notify
  555. * callback.
  556. */
  557. }
  558. static void tcg_register_iommu_notifier(CPUState *cpu,
  559. IOMMUMemoryRegion *iommu_mr,
  560. int iommu_idx)
  561. {
  562. /* Make sure this CPU has an IOMMU notifier registered for this
  563. * IOMMU/IOMMU index combination, so that we can flush its TLB
  564. * when the IOMMU tells us the mappings we've cached have changed.
  565. */
  566. MemoryRegion *mr = MEMORY_REGION(iommu_mr);
  567. TCGIOMMUNotifier *notifier;
  568. Error *err = NULL;
  569. int i, ret;
  570. for (i = 0; i < cpu->iommu_notifiers->len; i++) {
  571. notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
  572. if (notifier->mr == mr && notifier->iommu_idx == iommu_idx) {
  573. break;
  574. }
  575. }
  576. if (i == cpu->iommu_notifiers->len) {
  577. /* Not found, add a new entry at the end of the array */
  578. cpu->iommu_notifiers = g_array_set_size(cpu->iommu_notifiers, i + 1);
  579. notifier = g_new0(TCGIOMMUNotifier, 1);
  580. g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i) = notifier;
  581. notifier->mr = mr;
  582. notifier->iommu_idx = iommu_idx;
  583. notifier->cpu = cpu;
  584. /* Rather than trying to register interest in the specific part
  585. * of the iommu's address space that we've accessed and then
  586. * expand it later as subsequent accesses touch more of it, we
  587. * just register interest in the whole thing, on the assumption
  588. * that iommu reconfiguration will be rare.
  589. */
  590. iommu_notifier_init(&notifier->n,
  591. tcg_iommu_unmap_notify,
  592. IOMMU_NOTIFIER_UNMAP,
  593. 0,
  594. HWADDR_MAX,
  595. iommu_idx);
  596. ret = memory_region_register_iommu_notifier(notifier->mr, &notifier->n,
  597. &err);
  598. if (ret) {
  599. error_report_err(err);
  600. exit(1);
  601. }
  602. }
  603. if (!notifier->active) {
  604. notifier->active = true;
  605. }
  606. }
  607. static void tcg_iommu_free_notifier_list(CPUState *cpu)
  608. {
  609. /* Destroy the CPU's notifier list */
  610. int i;
  611. TCGIOMMUNotifier *notifier;
  612. for (i = 0; i < cpu->iommu_notifiers->len; i++) {
  613. notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
  614. memory_region_unregister_iommu_notifier(notifier->mr, &notifier->n);
  615. g_free(notifier);
  616. }
  617. g_array_free(cpu->iommu_notifiers, true);
  618. }
  619. /* Called from RCU critical section */
  620. MemoryRegionSection *
  621. address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
  622. hwaddr *xlat, hwaddr *plen,
  623. MemTxAttrs attrs, int *prot)
  624. {
  625. MemoryRegionSection *section;
  626. IOMMUMemoryRegion *iommu_mr;
  627. IOMMUMemoryRegionClass *imrc;
  628. IOMMUTLBEntry iotlb;
  629. int iommu_idx;
  630. AddressSpaceDispatch *d = atomic_rcu_read(&cpu->cpu_ases[asidx].memory_dispatch);
  631. for (;;) {
  632. section = address_space_translate_internal(d, addr, &addr, plen, false);
  633. iommu_mr = memory_region_get_iommu(section->mr);
  634. if (!iommu_mr) {
  635. break;
  636. }
  637. imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
  638. iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
  639. tcg_register_iommu_notifier(cpu, iommu_mr, iommu_idx);
  640. /* We need all the permissions, so pass IOMMU_NONE so the IOMMU
  641. * doesn't short-cut its translation table walk.
  642. */
  643. iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, iommu_idx);
  644. addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
  645. | (addr & iotlb.addr_mask));
  646. /* Update the caller's prot bits to remove permissions the IOMMU
  647. * is giving us a failure response for. If we get down to no
  648. * permissions left at all we can give up now.
  649. */
  650. if (!(iotlb.perm & IOMMU_RO)) {
  651. *prot &= ~(PAGE_READ | PAGE_EXEC);
  652. }
  653. if (!(iotlb.perm & IOMMU_WO)) {
  654. *prot &= ~PAGE_WRITE;
  655. }
  656. if (!*prot) {
  657. goto translate_fail;
  658. }
  659. d = flatview_to_dispatch(address_space_to_flatview(iotlb.target_as));
  660. }
  661. assert(!memory_region_is_iommu(section->mr));
  662. *xlat = addr;
  663. return section;
  664. translate_fail:
  665. return &d->map.sections[PHYS_SECTION_UNASSIGNED];
  666. }
  667. #endif
  668. #if !defined(CONFIG_USER_ONLY)
  669. static int cpu_common_post_load(void *opaque, int version_id)
  670. {
  671. CPUState *cpu = opaque;
  672. /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
  673. version_id is increased. */
  674. cpu->interrupt_request &= ~0x01;
  675. tlb_flush(cpu);
  676. /* loadvm has just updated the content of RAM, bypassing the
  677. * usual mechanisms that ensure we flush TBs for writes to
  678. * memory we've translated code from. So we must flush all TBs,
  679. * which will now be stale.
  680. */
  681. tb_flush(cpu);
  682. return 0;
  683. }
  684. static int cpu_common_pre_load(void *opaque)
  685. {
  686. CPUState *cpu = opaque;
  687. cpu->exception_index = -1;
  688. return 0;
  689. }
  690. static bool cpu_common_exception_index_needed(void *opaque)
  691. {
  692. CPUState *cpu = opaque;
  693. return tcg_enabled() && cpu->exception_index != -1;
  694. }
  695. static const VMStateDescription vmstate_cpu_common_exception_index = {
  696. .name = "cpu_common/exception_index",
  697. .version_id = 1,
  698. .minimum_version_id = 1,
  699. .needed = cpu_common_exception_index_needed,
  700. .fields = (VMStateField[]) {
  701. VMSTATE_INT32(exception_index, CPUState),
  702. VMSTATE_END_OF_LIST()
  703. }
  704. };
  705. static bool cpu_common_crash_occurred_needed(void *opaque)
  706. {
  707. CPUState *cpu = opaque;
  708. return cpu->crash_occurred;
  709. }
  710. static const VMStateDescription vmstate_cpu_common_crash_occurred = {
  711. .name = "cpu_common/crash_occurred",
  712. .version_id = 1,
  713. .minimum_version_id = 1,
  714. .needed = cpu_common_crash_occurred_needed,
  715. .fields = (VMStateField[]) {
  716. VMSTATE_BOOL(crash_occurred, CPUState),
  717. VMSTATE_END_OF_LIST()
  718. }
  719. };
  720. const VMStateDescription vmstate_cpu_common = {
  721. .name = "cpu_common",
  722. .version_id = 1,
  723. .minimum_version_id = 1,
  724. .pre_load = cpu_common_pre_load,
  725. .post_load = cpu_common_post_load,
  726. .fields = (VMStateField[]) {
  727. VMSTATE_UINT32(halted, CPUState),
  728. VMSTATE_UINT32(interrupt_request, CPUState),
  729. VMSTATE_END_OF_LIST()
  730. },
  731. .subsections = (const VMStateDescription*[]) {
  732. &vmstate_cpu_common_exception_index,
  733. &vmstate_cpu_common_crash_occurred,
  734. NULL
  735. }
  736. };
  737. #endif
  738. CPUState *qemu_get_cpu(int index)
  739. {
  740. CPUState *cpu;
  741. CPU_FOREACH(cpu) {
  742. if (cpu->cpu_index == index) {
  743. return cpu;
  744. }
  745. }
  746. return NULL;
  747. }
  748. #if !defined(CONFIG_USER_ONLY)
  749. void cpu_address_space_init(CPUState *cpu, int asidx,
  750. const char *prefix, MemoryRegion *mr)
  751. {
  752. CPUAddressSpace *newas;
  753. AddressSpace *as = g_new0(AddressSpace, 1);
  754. char *as_name;
  755. assert(mr);
  756. as_name = g_strdup_printf("%s-%d", prefix, cpu->cpu_index);
  757. address_space_init(as, mr, as_name);
  758. g_free(as_name);
  759. /* Target code should have set num_ases before calling us */
  760. assert(asidx < cpu->num_ases);
  761. if (asidx == 0) {
  762. /* address space 0 gets the convenience alias */
  763. cpu->as = as;
  764. }
  765. /* KVM cannot currently support multiple address spaces. */
  766. assert(asidx == 0 || !kvm_enabled());
  767. if (!cpu->cpu_ases) {
  768. cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
  769. }
  770. newas = &cpu->cpu_ases[asidx];
  771. newas->cpu = cpu;
  772. newas->as = as;
  773. if (tcg_enabled()) {
  774. newas->tcg_as_listener.log_global_after_sync = tcg_log_global_after_sync;
  775. newas->tcg_as_listener.commit = tcg_commit;
  776. memory_listener_register(&newas->tcg_as_listener, as);
  777. }
  778. }
  779. AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
  780. {
  781. /* Return the AddressSpace corresponding to the specified index */
  782. return cpu->cpu_ases[asidx].as;
  783. }
  784. #endif
  785. void cpu_exec_unrealizefn(CPUState *cpu)
  786. {
  787. CPUClass *cc = CPU_GET_CLASS(cpu);
  788. cpu_list_remove(cpu);
  789. if (cc->vmsd != NULL) {
  790. vmstate_unregister(NULL, cc->vmsd, cpu);
  791. }
  792. if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
  793. vmstate_unregister(NULL, &vmstate_cpu_common, cpu);
  794. }
  795. #ifndef CONFIG_USER_ONLY
  796. tcg_iommu_free_notifier_list(cpu);
  797. #endif
  798. }
  799. Property cpu_common_props[] = {
  800. #ifndef CONFIG_USER_ONLY
  801. /* Create a memory property for softmmu CPU object,
  802. * so users can wire up its memory. (This can't go in hw/core/cpu.c
  803. * because that file is compiled only once for both user-mode
  804. * and system builds.) The default if no link is set up is to use
  805. * the system address space.
  806. */
  807. DEFINE_PROP_LINK("memory", CPUState, memory, TYPE_MEMORY_REGION,
  808. MemoryRegion *),
  809. #endif
  810. DEFINE_PROP_END_OF_LIST(),
  811. };
  812. void cpu_exec_initfn(CPUState *cpu)
  813. {
  814. cpu->as = NULL;
  815. cpu->num_ases = 0;
  816. #ifndef CONFIG_USER_ONLY
  817. cpu->thread_id = qemu_get_thread_id();
  818. cpu->memory = system_memory;
  819. object_ref(OBJECT(cpu->memory));
  820. #endif
  821. }
  822. void cpu_exec_realizefn(CPUState *cpu, Error **errp)
  823. {
  824. CPUClass *cc = CPU_GET_CLASS(cpu);
  825. static bool tcg_target_initialized;
  826. cpu_list_add(cpu);
  827. if (tcg_enabled() && !tcg_target_initialized) {
  828. tcg_target_initialized = true;
  829. cc->tcg_initialize();
  830. }
  831. tlb_init(cpu);
  832. #ifndef CONFIG_USER_ONLY
  833. if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
  834. vmstate_register(NULL, cpu->cpu_index, &vmstate_cpu_common, cpu);
  835. }
  836. if (cc->vmsd != NULL) {
  837. vmstate_register(NULL, cpu->cpu_index, cc->vmsd, cpu);
  838. }
  839. cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier *));
  840. #endif
  841. }
  842. const char *parse_cpu_option(const char *cpu_option)
  843. {
  844. ObjectClass *oc;
  845. CPUClass *cc;
  846. gchar **model_pieces;
  847. const char *cpu_type;
  848. model_pieces = g_strsplit(cpu_option, ",", 2);
  849. if (!model_pieces[0]) {
  850. error_report("-cpu option cannot be empty");
  851. exit(1);
  852. }
  853. oc = cpu_class_by_name(CPU_RESOLVING_TYPE, model_pieces[0]);
  854. if (oc == NULL) {
  855. error_report("unable to find CPU model '%s'", model_pieces[0]);
  856. g_strfreev(model_pieces);
  857. exit(EXIT_FAILURE);
  858. }
  859. cpu_type = object_class_get_name(oc);
  860. cc = CPU_CLASS(oc);
  861. cc->parse_features(cpu_type, model_pieces[1], &error_fatal);
  862. g_strfreev(model_pieces);
  863. return cpu_type;
  864. }
  865. #if defined(CONFIG_USER_ONLY)
  866. void tb_invalidate_phys_addr(target_ulong addr)
  867. {
  868. mmap_lock();
  869. tb_invalidate_phys_page_range(addr, addr + 1);
  870. mmap_unlock();
  871. }
  872. static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
  873. {
  874. tb_invalidate_phys_addr(pc);
  875. }
  876. #else
  877. void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs)
  878. {
  879. ram_addr_t ram_addr;
  880. MemoryRegion *mr;
  881. hwaddr l = 1;
  882. if (!tcg_enabled()) {
  883. return;
  884. }
  885. RCU_READ_LOCK_GUARD();
  886. mr = address_space_translate(as, addr, &addr, &l, false, attrs);
  887. if (!(memory_region_is_ram(mr)
  888. || memory_region_is_romd(mr))) {
  889. return;
  890. }
  891. ram_addr = memory_region_get_ram_addr(mr) + addr;
  892. tb_invalidate_phys_page_range(ram_addr, ram_addr + 1);
  893. }
  894. static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
  895. {
  896. MemTxAttrs attrs;
  897. hwaddr phys = cpu_get_phys_page_attrs_debug(cpu, pc, &attrs);
  898. int asidx = cpu_asidx_from_attrs(cpu, attrs);
  899. if (phys != -1) {
  900. /* Locks grabbed by tb_invalidate_phys_addr */
  901. tb_invalidate_phys_addr(cpu->cpu_ases[asidx].as,
  902. phys | (pc & ~TARGET_PAGE_MASK), attrs);
  903. }
  904. }
  905. #endif
  906. #ifndef CONFIG_USER_ONLY
  907. /* Add a watchpoint. */
  908. int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
  909. int flags, CPUWatchpoint **watchpoint)
  910. {
  911. CPUWatchpoint *wp;
  912. /* forbid ranges which are empty or run off the end of the address space */
  913. if (len == 0 || (addr + len - 1) < addr) {
  914. error_report("tried to set invalid watchpoint at %"
  915. VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
  916. return -EINVAL;
  917. }
  918. wp = g_malloc(sizeof(*wp));
  919. wp->vaddr = addr;
  920. wp->len = len;
  921. wp->flags = flags;
  922. /* keep all GDB-injected watchpoints in front */
  923. if (flags & BP_GDB) {
  924. QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
  925. } else {
  926. QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
  927. }
  928. tlb_flush_page(cpu, addr);
  929. if (watchpoint)
  930. *watchpoint = wp;
  931. return 0;
  932. }
  933. /* Remove a specific watchpoint. */
  934. int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
  935. int flags)
  936. {
  937. CPUWatchpoint *wp;
  938. QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
  939. if (addr == wp->vaddr && len == wp->len
  940. && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
  941. cpu_watchpoint_remove_by_ref(cpu, wp);
  942. return 0;
  943. }
  944. }
  945. return -ENOENT;
  946. }
  947. /* Remove a specific watchpoint by reference. */
  948. void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
  949. {
  950. QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
  951. tlb_flush_page(cpu, watchpoint->vaddr);
  952. g_free(watchpoint);
  953. }
  954. /* Remove all matching watchpoints. */
  955. void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
  956. {
  957. CPUWatchpoint *wp, *next;
  958. QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
  959. if (wp->flags & mask) {
  960. cpu_watchpoint_remove_by_ref(cpu, wp);
  961. }
  962. }
  963. }
  964. /* Return true if this watchpoint address matches the specified
  965. * access (ie the address range covered by the watchpoint overlaps
  966. * partially or completely with the address range covered by the
  967. * access).
  968. */
  969. static inline bool watchpoint_address_matches(CPUWatchpoint *wp,
  970. vaddr addr, vaddr len)
  971. {
  972. /* We know the lengths are non-zero, but a little caution is
  973. * required to avoid errors in the case where the range ends
  974. * exactly at the top of the address space and so addr + len
  975. * wraps round to zero.
  976. */
  977. vaddr wpend = wp->vaddr + wp->len - 1;
  978. vaddr addrend = addr + len - 1;
  979. return !(addr > wpend || wp->vaddr > addrend);
  980. }
  981. /* Return flags for watchpoints that match addr + prot. */
  982. int cpu_watchpoint_address_matches(CPUState *cpu, vaddr addr, vaddr len)
  983. {
  984. CPUWatchpoint *wp;
  985. int ret = 0;
  986. QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
  987. if (watchpoint_address_matches(wp, addr, TARGET_PAGE_SIZE)) {
  988. ret |= wp->flags;
  989. }
  990. }
  991. return ret;
  992. }
  993. #endif /* !CONFIG_USER_ONLY */
  994. /* Add a breakpoint. */
  995. int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
  996. CPUBreakpoint **breakpoint)
  997. {
  998. CPUBreakpoint *bp;
  999. bp = g_malloc(sizeof(*bp));
  1000. bp->pc = pc;
  1001. bp->flags = flags;
  1002. /* keep all GDB-injected breakpoints in front */
  1003. if (flags & BP_GDB) {
  1004. QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
  1005. } else {
  1006. QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
  1007. }
  1008. breakpoint_invalidate(cpu, pc);
  1009. if (breakpoint) {
  1010. *breakpoint = bp;
  1011. }
  1012. return 0;
  1013. }
  1014. /* Remove a specific breakpoint. */
  1015. int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
  1016. {
  1017. CPUBreakpoint *bp;
  1018. QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
  1019. if (bp->pc == pc && bp->flags == flags) {
  1020. cpu_breakpoint_remove_by_ref(cpu, bp);
  1021. return 0;
  1022. }
  1023. }
  1024. return -ENOENT;
  1025. }
  1026. /* Remove a specific breakpoint by reference. */
  1027. void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
  1028. {
  1029. QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
  1030. breakpoint_invalidate(cpu, breakpoint->pc);
  1031. g_free(breakpoint);
  1032. }
  1033. /* Remove all matching breakpoints. */
  1034. void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
  1035. {
  1036. CPUBreakpoint *bp, *next;
  1037. QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
  1038. if (bp->flags & mask) {
  1039. cpu_breakpoint_remove_by_ref(cpu, bp);
  1040. }
  1041. }
  1042. }
  1043. /* enable or disable single step mode. EXCP_DEBUG is returned by the
  1044. CPU loop after each instruction */
  1045. void cpu_single_step(CPUState *cpu, int enabled)
  1046. {
  1047. if (cpu->singlestep_enabled != enabled) {
  1048. cpu->singlestep_enabled = enabled;
  1049. if (kvm_enabled()) {
  1050. kvm_update_guest_debug(cpu, 0);
  1051. } else {
  1052. /* must flush all the translated code to avoid inconsistencies */
  1053. /* XXX: only flush what is necessary */
  1054. tb_flush(cpu);
  1055. }
  1056. }
  1057. }
  1058. void cpu_abort(CPUState *cpu, const char *fmt, ...)
  1059. {
  1060. va_list ap;
  1061. va_list ap2;
  1062. va_start(ap, fmt);
  1063. va_copy(ap2, ap);
  1064. fprintf(stderr, "qemu: fatal: ");
  1065. vfprintf(stderr, fmt, ap);
  1066. fprintf(stderr, "\n");
  1067. cpu_dump_state(cpu, stderr, CPU_DUMP_FPU | CPU_DUMP_CCOP);
  1068. if (qemu_log_separate()) {
  1069. qemu_log_lock();
  1070. qemu_log("qemu: fatal: ");
  1071. qemu_log_vprintf(fmt, ap2);
  1072. qemu_log("\n");
  1073. log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
  1074. qemu_log_flush();
  1075. qemu_log_unlock();
  1076. qemu_log_close();
  1077. }
  1078. va_end(ap2);
  1079. va_end(ap);
  1080. replay_finish();
  1081. #if defined(CONFIG_USER_ONLY)
  1082. {
  1083. struct sigaction act;
  1084. sigfillset(&act.sa_mask);
  1085. act.sa_handler = SIG_DFL;
  1086. act.sa_flags = 0;
  1087. sigaction(SIGABRT, &act, NULL);
  1088. }
  1089. #endif
  1090. abort();
  1091. }
  1092. #if !defined(CONFIG_USER_ONLY)
  1093. /* Called from RCU critical section */
  1094. static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
  1095. {
  1096. RAMBlock *block;
  1097. block = atomic_rcu_read(&ram_list.mru_block);
  1098. if (block && addr - block->offset < block->max_length) {
  1099. return block;
  1100. }
  1101. RAMBLOCK_FOREACH(block) {
  1102. if (addr - block->offset < block->max_length) {
  1103. goto found;
  1104. }
  1105. }
  1106. fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
  1107. abort();
  1108. found:
  1109. /* It is safe to write mru_block outside the iothread lock. This
  1110. * is what happens:
  1111. *
  1112. * mru_block = xxx
  1113. * rcu_read_unlock()
  1114. * xxx removed from list
  1115. * rcu_read_lock()
  1116. * read mru_block
  1117. * mru_block = NULL;
  1118. * call_rcu(reclaim_ramblock, xxx);
  1119. * rcu_read_unlock()
  1120. *
  1121. * atomic_rcu_set is not needed here. The block was already published
  1122. * when it was placed into the list. Here we're just making an extra
  1123. * copy of the pointer.
  1124. */
  1125. ram_list.mru_block = block;
  1126. return block;
  1127. }
  1128. static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
  1129. {
  1130. CPUState *cpu;
  1131. ram_addr_t start1;
  1132. RAMBlock *block;
  1133. ram_addr_t end;
  1134. assert(tcg_enabled());
  1135. end = TARGET_PAGE_ALIGN(start + length);
  1136. start &= TARGET_PAGE_MASK;
  1137. RCU_READ_LOCK_GUARD();
  1138. block = qemu_get_ram_block(start);
  1139. assert(block == qemu_get_ram_block(end - 1));
  1140. start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
  1141. CPU_FOREACH(cpu) {
  1142. tlb_reset_dirty(cpu, start1, length);
  1143. }
  1144. }
  1145. /* Note: start and end must be within the same ram block. */
  1146. bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
  1147. ram_addr_t length,
  1148. unsigned client)
  1149. {
  1150. DirtyMemoryBlocks *blocks;
  1151. unsigned long end, page;
  1152. bool dirty = false;
  1153. RAMBlock *ramblock;
  1154. uint64_t mr_offset, mr_size;
  1155. if (length == 0) {
  1156. return false;
  1157. }
  1158. end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
  1159. page = start >> TARGET_PAGE_BITS;
  1160. WITH_RCU_READ_LOCK_GUARD() {
  1161. blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
  1162. ramblock = qemu_get_ram_block(start);
  1163. /* Range sanity check on the ramblock */
  1164. assert(start >= ramblock->offset &&
  1165. start + length <= ramblock->offset + ramblock->used_length);
  1166. while (page < end) {
  1167. unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
  1168. unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
  1169. unsigned long num = MIN(end - page,
  1170. DIRTY_MEMORY_BLOCK_SIZE - offset);
  1171. dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
  1172. offset, num);
  1173. page += num;
  1174. }
  1175. mr_offset = (ram_addr_t)(page << TARGET_PAGE_BITS) - ramblock->offset;
  1176. mr_size = (end - page) << TARGET_PAGE_BITS;
  1177. memory_region_clear_dirty_bitmap(ramblock->mr, mr_offset, mr_size);
  1178. }
  1179. if (dirty && tcg_enabled()) {
  1180. tlb_reset_dirty_range_all(start, length);
  1181. }
  1182. return dirty;
  1183. }
  1184. DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
  1185. (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client)
  1186. {
  1187. DirtyMemoryBlocks *blocks;
  1188. ram_addr_t start = memory_region_get_ram_addr(mr) + offset;
  1189. unsigned long align = 1UL << (TARGET_PAGE_BITS + BITS_PER_LEVEL);
  1190. ram_addr_t first = QEMU_ALIGN_DOWN(start, align);
  1191. ram_addr_t last = QEMU_ALIGN_UP(start + length, align);
  1192. DirtyBitmapSnapshot *snap;
  1193. unsigned long page, end, dest;
  1194. snap = g_malloc0(sizeof(*snap) +
  1195. ((last - first) >> (TARGET_PAGE_BITS + 3)));
  1196. snap->start = first;
  1197. snap->end = last;
  1198. page = first >> TARGET_PAGE_BITS;
  1199. end = last >> TARGET_PAGE_BITS;
  1200. dest = 0;
  1201. WITH_RCU_READ_LOCK_GUARD() {
  1202. blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
  1203. while (page < end) {
  1204. unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
  1205. unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
  1206. unsigned long num = MIN(end - page,
  1207. DIRTY_MEMORY_BLOCK_SIZE - offset);
  1208. assert(QEMU_IS_ALIGNED(offset, (1 << BITS_PER_LEVEL)));
  1209. assert(QEMU_IS_ALIGNED(num, (1 << BITS_PER_LEVEL)));
  1210. offset >>= BITS_PER_LEVEL;
  1211. bitmap_copy_and_clear_atomic(snap->dirty + dest,
  1212. blocks->blocks[idx] + offset,
  1213. num);
  1214. page += num;
  1215. dest += num >> BITS_PER_LEVEL;
  1216. }
  1217. }
  1218. if (tcg_enabled()) {
  1219. tlb_reset_dirty_range_all(start, length);
  1220. }
  1221. memory_region_clear_dirty_bitmap(mr, offset, length);
  1222. return snap;
  1223. }
  1224. bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
  1225. ram_addr_t start,
  1226. ram_addr_t length)
  1227. {
  1228. unsigned long page, end;
  1229. assert(start >= snap->start);
  1230. assert(start + length <= snap->end);
  1231. end = TARGET_PAGE_ALIGN(start + length - snap->start) >> TARGET_PAGE_BITS;
  1232. page = (start - snap->start) >> TARGET_PAGE_BITS;
  1233. while (page < end) {
  1234. if (test_bit(page, snap->dirty)) {
  1235. return true;
  1236. }
  1237. page++;
  1238. }
  1239. return false;
  1240. }
  1241. /* Called from RCU critical section */
  1242. hwaddr memory_region_section_get_iotlb(CPUState *cpu,
  1243. MemoryRegionSection *section)
  1244. {
  1245. AddressSpaceDispatch *d = flatview_to_dispatch(section->fv);
  1246. return section - d->map.sections;
  1247. }
  1248. #endif /* defined(CONFIG_USER_ONLY) */
  1249. #if !defined(CONFIG_USER_ONLY)
  1250. static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end,
  1251. uint16_t section);
  1252. static subpage_t *subpage_init(FlatView *fv, hwaddr base);
  1253. static void *(*phys_mem_alloc)(size_t size, uint64_t *align, bool shared) =
  1254. qemu_anon_ram_alloc;
  1255. /*
  1256. * Set a custom physical guest memory alloator.
  1257. * Accelerators with unusual needs may need this. Hopefully, we can
  1258. * get rid of it eventually.
  1259. */
  1260. void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align, bool shared))
  1261. {
  1262. phys_mem_alloc = alloc;
  1263. }
  1264. static uint16_t phys_section_add(PhysPageMap *map,
  1265. MemoryRegionSection *section)
  1266. {
  1267. /* The physical section number is ORed with a page-aligned
  1268. * pointer to produce the iotlb entries. Thus it should
  1269. * never overflow into the page-aligned value.
  1270. */
  1271. assert(map->sections_nb < TARGET_PAGE_SIZE);
  1272. if (map->sections_nb == map->sections_nb_alloc) {
  1273. map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
  1274. map->sections = g_renew(MemoryRegionSection, map->sections,
  1275. map->sections_nb_alloc);
  1276. }
  1277. map->sections[map->sections_nb] = *section;
  1278. memory_region_ref(section->mr);
  1279. return map->sections_nb++;
  1280. }
  1281. static void phys_section_destroy(MemoryRegion *mr)
  1282. {
  1283. bool have_sub_page = mr->subpage;
  1284. memory_region_unref(mr);
  1285. if (have_sub_page) {
  1286. subpage_t *subpage = container_of(mr, subpage_t, iomem);
  1287. object_unref(OBJECT(&subpage->iomem));
  1288. g_free(subpage);
  1289. }
  1290. }
  1291. static void phys_sections_free(PhysPageMap *map)
  1292. {
  1293. while (map->sections_nb > 0) {
  1294. MemoryRegionSection *section = &map->sections[--map->sections_nb];
  1295. phys_section_destroy(section->mr);
  1296. }
  1297. g_free(map->sections);
  1298. g_free(map->nodes);
  1299. }
  1300. static void register_subpage(FlatView *fv, MemoryRegionSection *section)
  1301. {
  1302. AddressSpaceDispatch *d = flatview_to_dispatch(fv);
  1303. subpage_t *subpage;
  1304. hwaddr base = section->offset_within_address_space
  1305. & TARGET_PAGE_MASK;
  1306. MemoryRegionSection *existing = phys_page_find(d, base);
  1307. MemoryRegionSection subsection = {
  1308. .offset_within_address_space = base,
  1309. .size = int128_make64(TARGET_PAGE_SIZE),
  1310. };
  1311. hwaddr start, end;
  1312. assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
  1313. if (!(existing->mr->subpage)) {
  1314. subpage = subpage_init(fv, base);
  1315. subsection.fv = fv;
  1316. subsection.mr = &subpage->iomem;
  1317. phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
  1318. phys_section_add(&d->map, &subsection));
  1319. } else {
  1320. subpage = container_of(existing->mr, subpage_t, iomem);
  1321. }
  1322. start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
  1323. end = start + int128_get64(section->size) - 1;
  1324. subpage_register(subpage, start, end,
  1325. phys_section_add(&d->map, section));
  1326. }
  1327. static void register_multipage(FlatView *fv,
  1328. MemoryRegionSection *section)
  1329. {
  1330. AddressSpaceDispatch *d = flatview_to_dispatch(fv);
  1331. hwaddr start_addr = section->offset_within_address_space;
  1332. uint16_t section_index = phys_section_add(&d->map, section);
  1333. uint64_t num_pages = int128_get64(int128_rshift(section->size,
  1334. TARGET_PAGE_BITS));
  1335. assert(num_pages);
  1336. phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
  1337. }
  1338. /*
  1339. * The range in *section* may look like this:
  1340. *
  1341. * |s|PPPPPPP|s|
  1342. *
  1343. * where s stands for subpage and P for page.
  1344. */
  1345. void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section)
  1346. {
  1347. MemoryRegionSection remain = *section;
  1348. Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
  1349. /* register first subpage */
  1350. if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
  1351. uint64_t left = TARGET_PAGE_ALIGN(remain.offset_within_address_space)
  1352. - remain.offset_within_address_space;
  1353. MemoryRegionSection now = remain;
  1354. now.size = int128_min(int128_make64(left), now.size);
  1355. register_subpage(fv, &now);
  1356. if (int128_eq(remain.size, now.size)) {
  1357. return;
  1358. }
  1359. remain.size = int128_sub(remain.size, now.size);
  1360. remain.offset_within_address_space += int128_get64(now.size);
  1361. remain.offset_within_region += int128_get64(now.size);
  1362. }
  1363. /* register whole pages */
  1364. if (int128_ge(remain.size, page_size)) {
  1365. MemoryRegionSection now = remain;
  1366. now.size = int128_and(now.size, int128_neg(page_size));
  1367. register_multipage(fv, &now);
  1368. if (int128_eq(remain.size, now.size)) {
  1369. return;
  1370. }
  1371. remain.size = int128_sub(remain.size, now.size);
  1372. remain.offset_within_address_space += int128_get64(now.size);
  1373. remain.offset_within_region += int128_get64(now.size);
  1374. }
  1375. /* register last subpage */
  1376. register_subpage(fv, &remain);
  1377. }
  1378. void qemu_flush_coalesced_mmio_buffer(void)
  1379. {
  1380. if (kvm_enabled())
  1381. kvm_flush_coalesced_mmio_buffer();
  1382. }
  1383. void qemu_mutex_lock_ramlist(void)
  1384. {
  1385. qemu_mutex_lock(&ram_list.mutex);
  1386. }
  1387. void qemu_mutex_unlock_ramlist(void)
  1388. {
  1389. qemu_mutex_unlock(&ram_list.mutex);
  1390. }
  1391. void ram_block_dump(Monitor *mon)
  1392. {
  1393. RAMBlock *block;
  1394. char *psize;
  1395. RCU_READ_LOCK_GUARD();
  1396. monitor_printf(mon, "%24s %8s %18s %18s %18s\n",
  1397. "Block Name", "PSize", "Offset", "Used", "Total");
  1398. RAMBLOCK_FOREACH(block) {
  1399. psize = size_to_str(block->page_size);
  1400. monitor_printf(mon, "%24s %8s 0x%016" PRIx64 " 0x%016" PRIx64
  1401. " 0x%016" PRIx64 "\n", block->idstr, psize,
  1402. (uint64_t)block->offset,
  1403. (uint64_t)block->used_length,
  1404. (uint64_t)block->max_length);
  1405. g_free(psize);
  1406. }
  1407. }
  1408. #ifdef __linux__
  1409. /*
  1410. * FIXME TOCTTOU: this iterates over memory backends' mem-path, which
  1411. * may or may not name the same files / on the same filesystem now as
  1412. * when we actually open and map them. Iterate over the file
  1413. * descriptors instead, and use qemu_fd_getpagesize().
  1414. */
  1415. static int find_min_backend_pagesize(Object *obj, void *opaque)
  1416. {
  1417. long *hpsize_min = opaque;
  1418. if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
  1419. HostMemoryBackend *backend = MEMORY_BACKEND(obj);
  1420. long hpsize = host_memory_backend_pagesize(backend);
  1421. if (host_memory_backend_is_mapped(backend) && (hpsize < *hpsize_min)) {
  1422. *hpsize_min = hpsize;
  1423. }
  1424. }
  1425. return 0;
  1426. }
  1427. static int find_max_backend_pagesize(Object *obj, void *opaque)
  1428. {
  1429. long *hpsize_max = opaque;
  1430. if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
  1431. HostMemoryBackend *backend = MEMORY_BACKEND(obj);
  1432. long hpsize = host_memory_backend_pagesize(backend);
  1433. if (host_memory_backend_is_mapped(backend) && (hpsize > *hpsize_max)) {
  1434. *hpsize_max = hpsize;
  1435. }
  1436. }
  1437. return 0;
  1438. }
  1439. /*
  1440. * TODO: We assume right now that all mapped host memory backends are
  1441. * used as RAM, however some might be used for different purposes.
  1442. */
  1443. long qemu_minrampagesize(void)
  1444. {
  1445. long hpsize = LONG_MAX;
  1446. long mainrampagesize;
  1447. Object *memdev_root;
  1448. MachineState *ms = MACHINE(qdev_get_machine());
  1449. mainrampagesize = qemu_mempath_getpagesize(mem_path);
  1450. /* it's possible we have memory-backend objects with
  1451. * hugepage-backed RAM. these may get mapped into system
  1452. * address space via -numa parameters or memory hotplug
  1453. * hooks. we want to take these into account, but we
  1454. * also want to make sure these supported hugepage
  1455. * sizes are applicable across the entire range of memory
  1456. * we may boot from, so we take the min across all
  1457. * backends, and assume normal pages in cases where a
  1458. * backend isn't backed by hugepages.
  1459. */
  1460. memdev_root = object_resolve_path("/objects", NULL);
  1461. if (memdev_root) {
  1462. object_child_foreach(memdev_root, find_min_backend_pagesize, &hpsize);
  1463. }
  1464. if (hpsize == LONG_MAX) {
  1465. /* No additional memory regions found ==> Report main RAM page size */
  1466. return mainrampagesize;
  1467. }
  1468. /* If NUMA is disabled or the NUMA nodes are not backed with a
  1469. * memory-backend, then there is at least one node using "normal" RAM,
  1470. * so if its page size is smaller we have got to report that size instead.
  1471. */
  1472. if (hpsize > mainrampagesize &&
  1473. (ms->numa_state == NULL ||
  1474. ms->numa_state->num_nodes == 0 ||
  1475. ms->numa_state->nodes[0].node_memdev == NULL)) {
  1476. static bool warned;
  1477. if (!warned) {
  1478. error_report("Huge page support disabled (n/a for main memory).");
  1479. warned = true;
  1480. }
  1481. return mainrampagesize;
  1482. }
  1483. return hpsize;
  1484. }
  1485. long qemu_maxrampagesize(void)
  1486. {
  1487. long pagesize = qemu_mempath_getpagesize(mem_path);
  1488. Object *memdev_root = object_resolve_path("/objects", NULL);
  1489. if (memdev_root) {
  1490. object_child_foreach(memdev_root, find_max_backend_pagesize,
  1491. &pagesize);
  1492. }
  1493. return pagesize;
  1494. }
  1495. #else
  1496. long qemu_minrampagesize(void)
  1497. {
  1498. return getpagesize();
  1499. }
  1500. long qemu_maxrampagesize(void)
  1501. {
  1502. return getpagesize();
  1503. }
  1504. #endif
  1505. #ifdef CONFIG_POSIX
  1506. static int64_t get_file_size(int fd)
  1507. {
  1508. int64_t size;
  1509. #if defined(__linux__)
  1510. struct stat st;
  1511. if (fstat(fd, &st) < 0) {
  1512. return -errno;
  1513. }
  1514. /* Special handling for devdax character devices */
  1515. if (S_ISCHR(st.st_mode)) {
  1516. g_autofree char *subsystem_path = NULL;
  1517. g_autofree char *subsystem = NULL;
  1518. subsystem_path = g_strdup_printf("/sys/dev/char/%d:%d/subsystem",
  1519. major(st.st_rdev), minor(st.st_rdev));
  1520. subsystem = g_file_read_link(subsystem_path, NULL);
  1521. if (subsystem && g_str_has_suffix(subsystem, "/dax")) {
  1522. g_autofree char *size_path = NULL;
  1523. g_autofree char *size_str = NULL;
  1524. size_path = g_strdup_printf("/sys/dev/char/%d:%d/size",
  1525. major(st.st_rdev), minor(st.st_rdev));
  1526. if (g_file_get_contents(size_path, &size_str, NULL, NULL)) {
  1527. return g_ascii_strtoll(size_str, NULL, 0);
  1528. }
  1529. }
  1530. }
  1531. #endif /* defined(__linux__) */
  1532. /* st.st_size may be zero for special files yet lseek(2) works */
  1533. size = lseek(fd, 0, SEEK_END);
  1534. if (size < 0) {
  1535. return -errno;
  1536. }
  1537. return size;
  1538. }
  1539. static int file_ram_open(const char *path,
  1540. const char *region_name,
  1541. bool *created,
  1542. Error **errp)
  1543. {
  1544. char *filename;
  1545. char *sanitized_name;
  1546. char *c;
  1547. int fd = -1;
  1548. *created = false;
  1549. for (;;) {
  1550. fd = open(path, O_RDWR);
  1551. if (fd >= 0) {
  1552. /* @path names an existing file, use it */
  1553. break;
  1554. }
  1555. if (errno == ENOENT) {
  1556. /* @path names a file that doesn't exist, create it */
  1557. fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
  1558. if (fd >= 0) {
  1559. *created = true;
  1560. break;
  1561. }
  1562. } else if (errno == EISDIR) {
  1563. /* @path names a directory, create a file there */
  1564. /* Make name safe to use with mkstemp by replacing '/' with '_'. */
  1565. sanitized_name = g_strdup(region_name);
  1566. for (c = sanitized_name; *c != '\0'; c++) {
  1567. if (*c == '/') {
  1568. *c = '_';
  1569. }
  1570. }
  1571. filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
  1572. sanitized_name);
  1573. g_free(sanitized_name);
  1574. fd = mkstemp(filename);
  1575. if (fd >= 0) {
  1576. unlink(filename);
  1577. g_free(filename);
  1578. break;
  1579. }
  1580. g_free(filename);
  1581. }
  1582. if (errno != EEXIST && errno != EINTR) {
  1583. error_setg_errno(errp, errno,
  1584. "can't open backing store %s for guest RAM",
  1585. path);
  1586. return -1;
  1587. }
  1588. /*
  1589. * Try again on EINTR and EEXIST. The latter happens when
  1590. * something else creates the file between our two open().
  1591. */
  1592. }
  1593. return fd;
  1594. }
  1595. static void *file_ram_alloc(RAMBlock *block,
  1596. ram_addr_t memory,
  1597. int fd,
  1598. bool truncate,
  1599. Error **errp)
  1600. {
  1601. MachineState *ms = MACHINE(qdev_get_machine());
  1602. void *area;
  1603. block->page_size = qemu_fd_getpagesize(fd);
  1604. if (block->mr->align % block->page_size) {
  1605. error_setg(errp, "alignment 0x%" PRIx64
  1606. " must be multiples of page size 0x%zx",
  1607. block->mr->align, block->page_size);
  1608. return NULL;
  1609. } else if (block->mr->align && !is_power_of_2(block->mr->align)) {
  1610. error_setg(errp, "alignment 0x%" PRIx64
  1611. " must be a power of two", block->mr->align);
  1612. return NULL;
  1613. }
  1614. block->mr->align = MAX(block->page_size, block->mr->align);
  1615. #if defined(__s390x__)
  1616. if (kvm_enabled()) {
  1617. block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN);
  1618. }
  1619. #endif
  1620. if (memory < block->page_size) {
  1621. error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
  1622. "or larger than page size 0x%zx",
  1623. memory, block->page_size);
  1624. return NULL;
  1625. }
  1626. memory = ROUND_UP(memory, block->page_size);
  1627. /*
  1628. * ftruncate is not supported by hugetlbfs in older
  1629. * hosts, so don't bother bailing out on errors.
  1630. * If anything goes wrong with it under other filesystems,
  1631. * mmap will fail.
  1632. *
  1633. * Do not truncate the non-empty backend file to avoid corrupting
  1634. * the existing data in the file. Disabling shrinking is not
  1635. * enough. For example, the current vNVDIMM implementation stores
  1636. * the guest NVDIMM labels at the end of the backend file. If the
  1637. * backend file is later extended, QEMU will not be able to find
  1638. * those labels. Therefore, extending the non-empty backend file
  1639. * is disabled as well.
  1640. */
  1641. if (truncate && ftruncate(fd, memory)) {
  1642. perror("ftruncate");
  1643. }
  1644. area = qemu_ram_mmap(fd, memory, block->mr->align,
  1645. block->flags & RAM_SHARED, block->flags & RAM_PMEM);
  1646. if (area == MAP_FAILED) {
  1647. error_setg_errno(errp, errno,
  1648. "unable to map backing store for guest RAM");
  1649. return NULL;
  1650. }
  1651. if (mem_prealloc) {
  1652. os_mem_prealloc(fd, area, memory, ms->smp.cpus, errp);
  1653. if (errp && *errp) {
  1654. qemu_ram_munmap(fd, area, memory);
  1655. return NULL;
  1656. }
  1657. }
  1658. block->fd = fd;
  1659. return area;
  1660. }
  1661. #endif
  1662. /* Allocate space within the ram_addr_t space that governs the
  1663. * dirty bitmaps.
  1664. * Called with the ramlist lock held.
  1665. */
  1666. static ram_addr_t find_ram_offset(ram_addr_t size)
  1667. {
  1668. RAMBlock *block, *next_block;
  1669. ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
  1670. assert(size != 0); /* it would hand out same offset multiple times */
  1671. if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
  1672. return 0;
  1673. }
  1674. RAMBLOCK_FOREACH(block) {
  1675. ram_addr_t candidate, next = RAM_ADDR_MAX;
  1676. /* Align blocks to start on a 'long' in the bitmap
  1677. * which makes the bitmap sync'ing take the fast path.
  1678. */
  1679. candidate = block->offset + block->max_length;
  1680. candidate = ROUND_UP(candidate, BITS_PER_LONG << TARGET_PAGE_BITS);
  1681. /* Search for the closest following block
  1682. * and find the gap.
  1683. */
  1684. RAMBLOCK_FOREACH(next_block) {
  1685. if (next_block->offset >= candidate) {
  1686. next = MIN(next, next_block->offset);
  1687. }
  1688. }
  1689. /* If it fits remember our place and remember the size
  1690. * of gap, but keep going so that we might find a smaller
  1691. * gap to fill so avoiding fragmentation.
  1692. */
  1693. if (next - candidate >= size && next - candidate < mingap) {
  1694. offset = candidate;
  1695. mingap = next - candidate;
  1696. }
  1697. trace_find_ram_offset_loop(size, candidate, offset, next, mingap);
  1698. }
  1699. if (offset == RAM_ADDR_MAX) {
  1700. fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
  1701. (uint64_t)size);
  1702. abort();
  1703. }
  1704. trace_find_ram_offset(size, offset);
  1705. return offset;
  1706. }
  1707. static unsigned long last_ram_page(void)
  1708. {
  1709. RAMBlock *block;
  1710. ram_addr_t last = 0;
  1711. RCU_READ_LOCK_GUARD();
  1712. RAMBLOCK_FOREACH(block) {
  1713. last = MAX(last, block->offset + block->max_length);
  1714. }
  1715. return last >> TARGET_PAGE_BITS;
  1716. }
  1717. static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
  1718. {
  1719. int ret;
  1720. /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
  1721. if (!machine_dump_guest_core(current_machine)) {
  1722. ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
  1723. if (ret) {
  1724. perror("qemu_madvise");
  1725. fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
  1726. "but dump_guest_core=off specified\n");
  1727. }
  1728. }
  1729. }
  1730. const char *qemu_ram_get_idstr(RAMBlock *rb)
  1731. {
  1732. return rb->idstr;
  1733. }
  1734. void *qemu_ram_get_host_addr(RAMBlock *rb)
  1735. {
  1736. return rb->host;
  1737. }
  1738. ram_addr_t qemu_ram_get_offset(RAMBlock *rb)
  1739. {
  1740. return rb->offset;
  1741. }
  1742. ram_addr_t qemu_ram_get_used_length(RAMBlock *rb)
  1743. {
  1744. return rb->used_length;
  1745. }
  1746. bool qemu_ram_is_shared(RAMBlock *rb)
  1747. {
  1748. return rb->flags & RAM_SHARED;
  1749. }
  1750. /* Note: Only set at the start of postcopy */
  1751. bool qemu_ram_is_uf_zeroable(RAMBlock *rb)
  1752. {
  1753. return rb->flags & RAM_UF_ZEROPAGE;
  1754. }
  1755. void qemu_ram_set_uf_zeroable(RAMBlock *rb)
  1756. {
  1757. rb->flags |= RAM_UF_ZEROPAGE;
  1758. }
  1759. bool qemu_ram_is_migratable(RAMBlock *rb)
  1760. {
  1761. return rb->flags & RAM_MIGRATABLE;
  1762. }
  1763. void qemu_ram_set_migratable(RAMBlock *rb)
  1764. {
  1765. rb->flags |= RAM_MIGRATABLE;
  1766. }
  1767. void qemu_ram_unset_migratable(RAMBlock *rb)
  1768. {
  1769. rb->flags &= ~RAM_MIGRATABLE;
  1770. }
  1771. /* Called with iothread lock held. */
  1772. void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
  1773. {
  1774. RAMBlock *block;
  1775. assert(new_block);
  1776. assert(!new_block->idstr[0]);
  1777. if (dev) {
  1778. char *id = qdev_get_dev_path(dev);
  1779. if (id) {
  1780. snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
  1781. g_free(id);
  1782. }
  1783. }
  1784. pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
  1785. RCU_READ_LOCK_GUARD();
  1786. RAMBLOCK_FOREACH(block) {
  1787. if (block != new_block &&
  1788. !strcmp(block->idstr, new_block->idstr)) {
  1789. fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
  1790. new_block->idstr);
  1791. abort();
  1792. }
  1793. }
  1794. }
  1795. /* Called with iothread lock held. */
  1796. void qemu_ram_unset_idstr(RAMBlock *block)
  1797. {
  1798. /* FIXME: arch_init.c assumes that this is not called throughout
  1799. * migration. Ignore the problem since hot-unplug during migration
  1800. * does not work anyway.
  1801. */
  1802. if (block) {
  1803. memset(block->idstr, 0, sizeof(block->idstr));
  1804. }
  1805. }
  1806. size_t qemu_ram_pagesize(RAMBlock *rb)
  1807. {
  1808. return rb->page_size;
  1809. }
  1810. /* Returns the largest size of page in use */
  1811. size_t qemu_ram_pagesize_largest(void)
  1812. {
  1813. RAMBlock *block;
  1814. size_t largest = 0;
  1815. RAMBLOCK_FOREACH(block) {
  1816. largest = MAX(largest, qemu_ram_pagesize(block));
  1817. }
  1818. return largest;
  1819. }
  1820. static int memory_try_enable_merging(void *addr, size_t len)
  1821. {
  1822. if (!machine_mem_merge(current_machine)) {
  1823. /* disabled by the user */
  1824. return 0;
  1825. }
  1826. return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
  1827. }
  1828. /* Only legal before guest might have detected the memory size: e.g. on
  1829. * incoming migration, or right after reset.
  1830. *
  1831. * As memory core doesn't know how is memory accessed, it is up to
  1832. * resize callback to update device state and/or add assertions to detect
  1833. * misuse, if necessary.
  1834. */
  1835. int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
  1836. {
  1837. assert(block);
  1838. newsize = HOST_PAGE_ALIGN(newsize);
  1839. if (block->used_length == newsize) {
  1840. return 0;
  1841. }
  1842. if (!(block->flags & RAM_RESIZEABLE)) {
  1843. error_setg_errno(errp, EINVAL,
  1844. "Length mismatch: %s: 0x" RAM_ADDR_FMT
  1845. " in != 0x" RAM_ADDR_FMT, block->idstr,
  1846. newsize, block->used_length);
  1847. return -EINVAL;
  1848. }
  1849. if (block->max_length < newsize) {
  1850. error_setg_errno(errp, EINVAL,
  1851. "Length too large: %s: 0x" RAM_ADDR_FMT
  1852. " > 0x" RAM_ADDR_FMT, block->idstr,
  1853. newsize, block->max_length);
  1854. return -EINVAL;
  1855. }
  1856. cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
  1857. block->used_length = newsize;
  1858. cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
  1859. DIRTY_CLIENTS_ALL);
  1860. memory_region_set_size(block->mr, newsize);
  1861. if (block->resized) {
  1862. block->resized(block->idstr, newsize, block->host);
  1863. }
  1864. return 0;
  1865. }
  1866. /* Called with ram_list.mutex held */
  1867. static void dirty_memory_extend(ram_addr_t old_ram_size,
  1868. ram_addr_t new_ram_size)
  1869. {
  1870. ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size,
  1871. DIRTY_MEMORY_BLOCK_SIZE);
  1872. ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size,
  1873. DIRTY_MEMORY_BLOCK_SIZE);
  1874. int i;
  1875. /* Only need to extend if block count increased */
  1876. if (new_num_blocks <= old_num_blocks) {
  1877. return;
  1878. }
  1879. for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
  1880. DirtyMemoryBlocks *old_blocks;
  1881. DirtyMemoryBlocks *new_blocks;
  1882. int j;
  1883. old_blocks = atomic_rcu_read(&ram_list.dirty_memory[i]);
  1884. new_blocks = g_malloc(sizeof(*new_blocks) +
  1885. sizeof(new_blocks->blocks[0]) * new_num_blocks);
  1886. if (old_num_blocks) {
  1887. memcpy(new_blocks->blocks, old_blocks->blocks,
  1888. old_num_blocks * sizeof(old_blocks->blocks[0]));
  1889. }
  1890. for (j = old_num_blocks; j < new_num_blocks; j++) {
  1891. new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
  1892. }
  1893. atomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
  1894. if (old_blocks) {
  1895. g_free_rcu(old_blocks, rcu);
  1896. }
  1897. }
  1898. }
  1899. static void ram_block_add(RAMBlock *new_block, Error **errp, bool shared)
  1900. {
  1901. RAMBlock *block;
  1902. RAMBlock *last_block = NULL;
  1903. ram_addr_t old_ram_size, new_ram_size;
  1904. Error *err = NULL;
  1905. old_ram_size = last_ram_page();
  1906. qemu_mutex_lock_ramlist();
  1907. new_block->offset = find_ram_offset(new_block->max_length);
  1908. if (!new_block->host) {
  1909. if (xen_enabled()) {
  1910. xen_ram_alloc(new_block->offset, new_block->max_length,
  1911. new_block->mr, &err);
  1912. if (err) {
  1913. error_propagate(errp, err);
  1914. qemu_mutex_unlock_ramlist();
  1915. return;
  1916. }
  1917. } else {
  1918. new_block->host = phys_mem_alloc(new_block->max_length,
  1919. &new_block->mr->align, shared);
  1920. if (!new_block->host) {
  1921. error_setg_errno(errp, errno,
  1922. "cannot set up guest memory '%s'",
  1923. memory_region_name(new_block->mr));
  1924. qemu_mutex_unlock_ramlist();
  1925. return;
  1926. }
  1927. memory_try_enable_merging(new_block->host, new_block->max_length);
  1928. }
  1929. }
  1930. new_ram_size = MAX(old_ram_size,
  1931. (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
  1932. if (new_ram_size > old_ram_size) {
  1933. dirty_memory_extend(old_ram_size, new_ram_size);
  1934. }
  1935. /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
  1936. * QLIST (which has an RCU-friendly variant) does not have insertion at
  1937. * tail, so save the last element in last_block.
  1938. */
  1939. RAMBLOCK_FOREACH(block) {
  1940. last_block = block;
  1941. if (block->max_length < new_block->max_length) {
  1942. break;
  1943. }
  1944. }
  1945. if (block) {
  1946. QLIST_INSERT_BEFORE_RCU(block, new_block, next);
  1947. } else if (last_block) {
  1948. QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
  1949. } else { /* list is empty */
  1950. QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
  1951. }
  1952. ram_list.mru_block = NULL;
  1953. /* Write list before version */
  1954. smp_wmb();
  1955. ram_list.version++;
  1956. qemu_mutex_unlock_ramlist();
  1957. cpu_physical_memory_set_dirty_range(new_block->offset,
  1958. new_block->used_length,
  1959. DIRTY_CLIENTS_ALL);
  1960. if (new_block->host) {
  1961. qemu_ram_setup_dump(new_block->host, new_block->max_length);
  1962. qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
  1963. /* MADV_DONTFORK is also needed by KVM in absence of synchronous MMU */
  1964. qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
  1965. ram_block_notify_add(new_block->host, new_block->max_length);
  1966. }
  1967. }
  1968. #ifdef CONFIG_POSIX
  1969. RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
  1970. uint32_t ram_flags, int fd,
  1971. Error **errp)
  1972. {
  1973. RAMBlock *new_block;
  1974. Error *local_err = NULL;
  1975. int64_t file_size;
  1976. /* Just support these ram flags by now. */
  1977. assert((ram_flags & ~(RAM_SHARED | RAM_PMEM)) == 0);
  1978. if (xen_enabled()) {
  1979. error_setg(errp, "-mem-path not supported with Xen");
  1980. return NULL;
  1981. }
  1982. if (kvm_enabled() && !kvm_has_sync_mmu()) {
  1983. error_setg(errp,
  1984. "host lacks kvm mmu notifiers, -mem-path unsupported");
  1985. return NULL;
  1986. }
  1987. if (phys_mem_alloc != qemu_anon_ram_alloc) {
  1988. /*
  1989. * file_ram_alloc() needs to allocate just like
  1990. * phys_mem_alloc, but we haven't bothered to provide
  1991. * a hook there.
  1992. */
  1993. error_setg(errp,
  1994. "-mem-path not supported with this accelerator");
  1995. return NULL;
  1996. }
  1997. size = HOST_PAGE_ALIGN(size);
  1998. file_size = get_file_size(fd);
  1999. if (file_size > 0 && file_size < size) {
  2000. error_setg(errp, "backing store %s size 0x%" PRIx64
  2001. " does not match 'size' option 0x" RAM_ADDR_FMT,
  2002. mem_path, file_size, size);
  2003. return NULL;
  2004. }
  2005. new_block = g_malloc0(sizeof(*new_block));
  2006. new_block->mr = mr;
  2007. new_block->used_length = size;
  2008. new_block->max_length = size;
  2009. new_block->flags = ram_flags;
  2010. new_block->host = file_ram_alloc(new_block, size, fd, !file_size, errp);
  2011. if (!new_block->host) {
  2012. g_free(new_block);
  2013. return NULL;
  2014. }
  2015. ram_block_add(new_block, &local_err, ram_flags & RAM_SHARED);
  2016. if (local_err) {
  2017. g_free(new_block);
  2018. error_propagate(errp, local_err);
  2019. return NULL;
  2020. }
  2021. return new_block;
  2022. }
  2023. RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
  2024. uint32_t ram_flags, const char *mem_path,
  2025. Error **errp)
  2026. {
  2027. int fd;
  2028. bool created;
  2029. RAMBlock *block;
  2030. fd = file_ram_open(mem_path, memory_region_name(mr), &created, errp);
  2031. if (fd < 0) {
  2032. return NULL;
  2033. }
  2034. block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, errp);
  2035. if (!block) {
  2036. if (created) {
  2037. unlink(mem_path);
  2038. }
  2039. close(fd);
  2040. return NULL;
  2041. }
  2042. return block;
  2043. }
  2044. #endif
  2045. static
  2046. RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
  2047. void (*resized)(const char*,
  2048. uint64_t length,
  2049. void *host),
  2050. void *host, bool resizeable, bool share,
  2051. MemoryRegion *mr, Error **errp)
  2052. {
  2053. RAMBlock *new_block;
  2054. Error *local_err = NULL;
  2055. size = HOST_PAGE_ALIGN(size);
  2056. max_size = HOST_PAGE_ALIGN(max_size);
  2057. new_block = g_malloc0(sizeof(*new_block));
  2058. new_block->mr = mr;
  2059. new_block->resized = resized;
  2060. new_block->used_length = size;
  2061. new_block->max_length = max_size;
  2062. assert(max_size >= size);
  2063. new_block->fd = -1;
  2064. new_block->page_size = getpagesize();
  2065. new_block->host = host;
  2066. if (host) {
  2067. new_block->flags |= RAM_PREALLOC;
  2068. }
  2069. if (resizeable) {
  2070. new_block->flags |= RAM_RESIZEABLE;
  2071. }
  2072. ram_block_add(new_block, &local_err, share);
  2073. if (local_err) {
  2074. g_free(new_block);
  2075. error_propagate(errp, local_err);
  2076. return NULL;
  2077. }
  2078. return new_block;
  2079. }
  2080. RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
  2081. MemoryRegion *mr, Error **errp)
  2082. {
  2083. return qemu_ram_alloc_internal(size, size, NULL, host, false,
  2084. false, mr, errp);
  2085. }
  2086. RAMBlock *qemu_ram_alloc(ram_addr_t size, bool share,
  2087. MemoryRegion *mr, Error **errp)
  2088. {
  2089. return qemu_ram_alloc_internal(size, size, NULL, NULL, false,
  2090. share, mr, errp);
  2091. }
  2092. RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
  2093. void (*resized)(const char*,
  2094. uint64_t length,
  2095. void *host),
  2096. MemoryRegion *mr, Error **errp)
  2097. {
  2098. return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true,
  2099. false, mr, errp);
  2100. }
  2101. static void reclaim_ramblock(RAMBlock *block)
  2102. {
  2103. if (block->flags & RAM_PREALLOC) {
  2104. ;
  2105. } else if (xen_enabled()) {
  2106. xen_invalidate_map_cache_entry(block->host);
  2107. #ifndef _WIN32
  2108. } else if (block->fd >= 0) {
  2109. qemu_ram_munmap(block->fd, block->host, block->max_length);
  2110. close(block->fd);
  2111. #endif
  2112. } else {
  2113. qemu_anon_ram_free(block->host, block->max_length);
  2114. }
  2115. g_free(block);
  2116. }
  2117. void qemu_ram_free(RAMBlock *block)
  2118. {
  2119. if (!block) {
  2120. return;
  2121. }
  2122. if (block->host) {
  2123. ram_block_notify_remove(block->host, block->max_length);
  2124. }
  2125. qemu_mutex_lock_ramlist();
  2126. QLIST_REMOVE_RCU(block, next);
  2127. ram_list.mru_block = NULL;
  2128. /* Write list before version */
  2129. smp_wmb();
  2130. ram_list.version++;
  2131. call_rcu(block, reclaim_ramblock, rcu);
  2132. qemu_mutex_unlock_ramlist();
  2133. }
  2134. #ifndef _WIN32
  2135. void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
  2136. {
  2137. RAMBlock *block;
  2138. ram_addr_t offset;
  2139. int flags;
  2140. void *area, *vaddr;
  2141. RAMBLOCK_FOREACH(block) {
  2142. offset = addr - block->offset;
  2143. if (offset < block->max_length) {
  2144. vaddr = ramblock_ptr(block, offset);
  2145. if (block->flags & RAM_PREALLOC) {
  2146. ;
  2147. } else if (xen_enabled()) {
  2148. abort();
  2149. } else {
  2150. flags = MAP_FIXED;
  2151. if (block->fd >= 0) {
  2152. flags |= (block->flags & RAM_SHARED ?
  2153. MAP_SHARED : MAP_PRIVATE);
  2154. area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
  2155. flags, block->fd, offset);
  2156. } else {
  2157. /*
  2158. * Remap needs to match alloc. Accelerators that
  2159. * set phys_mem_alloc never remap. If they did,
  2160. * we'd need a remap hook here.
  2161. */
  2162. assert(phys_mem_alloc == qemu_anon_ram_alloc);
  2163. flags |= MAP_PRIVATE | MAP_ANONYMOUS;
  2164. area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
  2165. flags, -1, 0);
  2166. }
  2167. if (area != vaddr) {
  2168. error_report("Could not remap addr: "
  2169. RAM_ADDR_FMT "@" RAM_ADDR_FMT "",
  2170. length, addr);
  2171. exit(1);
  2172. }
  2173. memory_try_enable_merging(vaddr, length);
  2174. qemu_ram_setup_dump(vaddr, length);
  2175. }
  2176. }
  2177. }
  2178. }
  2179. #endif /* !_WIN32 */
  2180. /* Return a host pointer to ram allocated with qemu_ram_alloc.
  2181. * This should not be used for general purpose DMA. Use address_space_map
  2182. * or address_space_rw instead. For local memory (e.g. video ram) that the
  2183. * device owns, use memory_region_get_ram_ptr.
  2184. *
  2185. * Called within RCU critical section.
  2186. */
  2187. void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
  2188. {
  2189. RAMBlock *block = ram_block;
  2190. if (block == NULL) {
  2191. block = qemu_get_ram_block(addr);
  2192. addr -= block->offset;
  2193. }
  2194. if (xen_enabled() && block->host == NULL) {
  2195. /* We need to check if the requested address is in the RAM
  2196. * because we don't want to map the entire memory in QEMU.
  2197. * In that case just map until the end of the page.
  2198. */
  2199. if (block->offset == 0) {
  2200. return xen_map_cache(addr, 0, 0, false);
  2201. }
  2202. block->host = xen_map_cache(block->offset, block->max_length, 1, false);
  2203. }
  2204. return ramblock_ptr(block, addr);
  2205. }
  2206. /* Return a host pointer to guest's ram. Similar to qemu_map_ram_ptr
  2207. * but takes a size argument.
  2208. *
  2209. * Called within RCU critical section.
  2210. */
  2211. static void *qemu_ram_ptr_length(RAMBlock *ram_block, ram_addr_t addr,
  2212. hwaddr *size, bool lock)
  2213. {
  2214. RAMBlock *block = ram_block;
  2215. if (*size == 0) {
  2216. return NULL;
  2217. }
  2218. if (block == NULL) {
  2219. block = qemu_get_ram_block(addr);
  2220. addr -= block->offset;
  2221. }
  2222. *size = MIN(*size, block->max_length - addr);
  2223. if (xen_enabled() && block->host == NULL) {
  2224. /* We need to check if the requested address is in the RAM
  2225. * because we don't want to map the entire memory in QEMU.
  2226. * In that case just map the requested area.
  2227. */
  2228. if (block->offset == 0) {
  2229. return xen_map_cache(addr, *size, lock, lock);
  2230. }
  2231. block->host = xen_map_cache(block->offset, block->max_length, 1, lock);
  2232. }
  2233. return ramblock_ptr(block, addr);
  2234. }
  2235. /* Return the offset of a hostpointer within a ramblock */
  2236. ram_addr_t qemu_ram_block_host_offset(RAMBlock *rb, void *host)
  2237. {
  2238. ram_addr_t res = (uint8_t *)host - (uint8_t *)rb->host;
  2239. assert((uintptr_t)host >= (uintptr_t)rb->host);
  2240. assert(res < rb->max_length);
  2241. return res;
  2242. }
  2243. /*
  2244. * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
  2245. * in that RAMBlock.
  2246. *
  2247. * ptr: Host pointer to look up
  2248. * round_offset: If true round the result offset down to a page boundary
  2249. * *ram_addr: set to result ram_addr
  2250. * *offset: set to result offset within the RAMBlock
  2251. *
  2252. * Returns: RAMBlock (or NULL if not found)
  2253. *
  2254. * By the time this function returns, the returned pointer is not protected
  2255. * by RCU anymore. If the caller is not within an RCU critical section and
  2256. * does not hold the iothread lock, it must have other means of protecting the
  2257. * pointer, such as a reference to the region that includes the incoming
  2258. * ram_addr_t.
  2259. */
  2260. RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
  2261. ram_addr_t *offset)
  2262. {
  2263. RAMBlock *block;
  2264. uint8_t *host = ptr;
  2265. if (xen_enabled()) {
  2266. ram_addr_t ram_addr;
  2267. RCU_READ_LOCK_GUARD();
  2268. ram_addr = xen_ram_addr_from_mapcache(ptr);
  2269. block = qemu_get_ram_block(ram_addr);
  2270. if (block) {
  2271. *offset = ram_addr - block->offset;
  2272. }
  2273. return block;
  2274. }
  2275. RCU_READ_LOCK_GUARD();
  2276. block = atomic_rcu_read(&ram_list.mru_block);
  2277. if (block && block->host && host - block->host < block->max_length) {
  2278. goto found;
  2279. }
  2280. RAMBLOCK_FOREACH(block) {
  2281. /* This case append when the block is not mapped. */
  2282. if (block->host == NULL) {
  2283. continue;
  2284. }
  2285. if (host - block->host < block->max_length) {
  2286. goto found;
  2287. }
  2288. }
  2289. return NULL;
  2290. found:
  2291. *offset = (host - block->host);
  2292. if (round_offset) {
  2293. *offset &= TARGET_PAGE_MASK;
  2294. }
  2295. return block;
  2296. }
  2297. /*
  2298. * Finds the named RAMBlock
  2299. *
  2300. * name: The name of RAMBlock to find
  2301. *
  2302. * Returns: RAMBlock (or NULL if not found)
  2303. */
  2304. RAMBlock *qemu_ram_block_by_name(const char *name)
  2305. {
  2306. RAMBlock *block;
  2307. RAMBLOCK_FOREACH(block) {
  2308. if (!strcmp(name, block->idstr)) {
  2309. return block;
  2310. }
  2311. }
  2312. return NULL;
  2313. }
  2314. /* Some of the softmmu routines need to translate from a host pointer
  2315. (typically a TLB entry) back to a ram offset. */
  2316. ram_addr_t qemu_ram_addr_from_host(void *ptr)
  2317. {
  2318. RAMBlock *block;
  2319. ram_addr_t offset;
  2320. block = qemu_ram_block_from_host(ptr, false, &offset);
  2321. if (!block) {
  2322. return RAM_ADDR_INVALID;
  2323. }
  2324. return block->offset + offset;
  2325. }
  2326. /* Generate a debug exception if a watchpoint has been hit. */
  2327. void cpu_check_watchpoint(CPUState *cpu, vaddr addr, vaddr len,
  2328. MemTxAttrs attrs, int flags, uintptr_t ra)
  2329. {
  2330. CPUClass *cc = CPU_GET_CLASS(cpu);
  2331. CPUWatchpoint *wp;
  2332. assert(tcg_enabled());
  2333. if (cpu->watchpoint_hit) {
  2334. /*
  2335. * We re-entered the check after replacing the TB.
  2336. * Now raise the debug interrupt so that it will
  2337. * trigger after the current instruction.
  2338. */
  2339. qemu_mutex_lock_iothread();
  2340. cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
  2341. qemu_mutex_unlock_iothread();
  2342. return;
  2343. }
  2344. addr = cc->adjust_watchpoint_address(cpu, addr, len);
  2345. QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
  2346. if (watchpoint_address_matches(wp, addr, len)
  2347. && (wp->flags & flags)) {
  2348. if (flags == BP_MEM_READ) {
  2349. wp->flags |= BP_WATCHPOINT_HIT_READ;
  2350. } else {
  2351. wp->flags |= BP_WATCHPOINT_HIT_WRITE;
  2352. }
  2353. wp->hitaddr = MAX(addr, wp->vaddr);
  2354. wp->hitattrs = attrs;
  2355. if (!cpu->watchpoint_hit) {
  2356. if (wp->flags & BP_CPU &&
  2357. !cc->debug_check_watchpoint(cpu, wp)) {
  2358. wp->flags &= ~BP_WATCHPOINT_HIT;
  2359. continue;
  2360. }
  2361. cpu->watchpoint_hit = wp;
  2362. mmap_lock();
  2363. tb_check_watchpoint(cpu, ra);
  2364. if (wp->flags & BP_STOP_BEFORE_ACCESS) {
  2365. cpu->exception_index = EXCP_DEBUG;
  2366. mmap_unlock();
  2367. cpu_loop_exit_restore(cpu, ra);
  2368. } else {
  2369. /* Force execution of one insn next time. */
  2370. cpu->cflags_next_tb = 1 | curr_cflags();
  2371. mmap_unlock();
  2372. if (ra) {
  2373. cpu_restore_state(cpu, ra, true);
  2374. }
  2375. cpu_loop_exit_noexc(cpu);
  2376. }
  2377. }
  2378. } else {
  2379. wp->flags &= ~BP_WATCHPOINT_HIT;
  2380. }
  2381. }
  2382. }
  2383. static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
  2384. MemTxAttrs attrs, uint8_t *buf, hwaddr len);
  2385. static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
  2386. const uint8_t *buf, hwaddr len);
  2387. static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
  2388. bool is_write, MemTxAttrs attrs);
  2389. static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
  2390. unsigned len, MemTxAttrs attrs)
  2391. {
  2392. subpage_t *subpage = opaque;
  2393. uint8_t buf[8];
  2394. MemTxResult res;
  2395. #if defined(DEBUG_SUBPAGE)
  2396. printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
  2397. subpage, len, addr);
  2398. #endif
  2399. res = flatview_read(subpage->fv, addr + subpage->base, attrs, buf, len);
  2400. if (res) {
  2401. return res;
  2402. }
  2403. *data = ldn_p(buf, len);
  2404. return MEMTX_OK;
  2405. }
  2406. static MemTxResult subpage_write(void *opaque, hwaddr addr,
  2407. uint64_t value, unsigned len, MemTxAttrs attrs)
  2408. {
  2409. subpage_t *subpage = opaque;
  2410. uint8_t buf[8];
  2411. #if defined(DEBUG_SUBPAGE)
  2412. printf("%s: subpage %p len %u addr " TARGET_FMT_plx
  2413. " value %"PRIx64"\n",
  2414. __func__, subpage, len, addr, value);
  2415. #endif
  2416. stn_p(buf, len, value);
  2417. return flatview_write(subpage->fv, addr + subpage->base, attrs, buf, len);
  2418. }
  2419. static bool subpage_accepts(void *opaque, hwaddr addr,
  2420. unsigned len, bool is_write,
  2421. MemTxAttrs attrs)
  2422. {
  2423. subpage_t *subpage = opaque;
  2424. #if defined(DEBUG_SUBPAGE)
  2425. printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
  2426. __func__, subpage, is_write ? 'w' : 'r', len, addr);
  2427. #endif
  2428. return flatview_access_valid(subpage->fv, addr + subpage->base,
  2429. len, is_write, attrs);
  2430. }
  2431. static const MemoryRegionOps subpage_ops = {
  2432. .read_with_attrs = subpage_read,
  2433. .write_with_attrs = subpage_write,
  2434. .impl.min_access_size = 1,
  2435. .impl.max_access_size = 8,
  2436. .valid.min_access_size = 1,
  2437. .valid.max_access_size = 8,
  2438. .valid.accepts = subpage_accepts,
  2439. .endianness = DEVICE_NATIVE_ENDIAN,
  2440. };
  2441. static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end,
  2442. uint16_t section)
  2443. {
  2444. int idx, eidx;
  2445. if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
  2446. return -1;
  2447. idx = SUBPAGE_IDX(start);
  2448. eidx = SUBPAGE_IDX(end);
  2449. #if defined(DEBUG_SUBPAGE)
  2450. printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
  2451. __func__, mmio, start, end, idx, eidx, section);
  2452. #endif
  2453. for (; idx <= eidx; idx++) {
  2454. mmio->sub_section[idx] = section;
  2455. }
  2456. return 0;
  2457. }
  2458. static subpage_t *subpage_init(FlatView *fv, hwaddr base)
  2459. {
  2460. subpage_t *mmio;
  2461. /* mmio->sub_section is set to PHYS_SECTION_UNASSIGNED with g_malloc0 */
  2462. mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t));
  2463. mmio->fv = fv;
  2464. mmio->base = base;
  2465. memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
  2466. NULL, TARGET_PAGE_SIZE);
  2467. mmio->iomem.subpage = true;
  2468. #if defined(DEBUG_SUBPAGE)
  2469. printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
  2470. mmio, base, TARGET_PAGE_SIZE);
  2471. #endif
  2472. return mmio;
  2473. }
  2474. static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr)
  2475. {
  2476. assert(fv);
  2477. MemoryRegionSection section = {
  2478. .fv = fv,
  2479. .mr = mr,
  2480. .offset_within_address_space = 0,
  2481. .offset_within_region = 0,
  2482. .size = int128_2_64(),
  2483. };
  2484. return phys_section_add(map, &section);
  2485. }
  2486. MemoryRegionSection *iotlb_to_section(CPUState *cpu,
  2487. hwaddr index, MemTxAttrs attrs)
  2488. {
  2489. int asidx = cpu_asidx_from_attrs(cpu, attrs);
  2490. CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
  2491. AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
  2492. MemoryRegionSection *sections = d->map.sections;
  2493. return &sections[index & ~TARGET_PAGE_MASK];
  2494. }
  2495. static void io_mem_init(void)
  2496. {
  2497. memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
  2498. NULL, UINT64_MAX);
  2499. }
  2500. AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv)
  2501. {
  2502. AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
  2503. uint16_t n;
  2504. n = dummy_section(&d->map, fv, &io_mem_unassigned);
  2505. assert(n == PHYS_SECTION_UNASSIGNED);
  2506. d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
  2507. return d;
  2508. }
  2509. void address_space_dispatch_free(AddressSpaceDispatch *d)
  2510. {
  2511. phys_sections_free(&d->map);
  2512. g_free(d);
  2513. }
  2514. static void do_nothing(CPUState *cpu, run_on_cpu_data d)
  2515. {
  2516. }
  2517. static void tcg_log_global_after_sync(MemoryListener *listener)
  2518. {
  2519. CPUAddressSpace *cpuas;
  2520. /* Wait for the CPU to end the current TB. This avoids the following
  2521. * incorrect race:
  2522. *
  2523. * vCPU migration
  2524. * ---------------------- -------------------------
  2525. * TLB check -> slow path
  2526. * notdirty_mem_write
  2527. * write to RAM
  2528. * mark dirty
  2529. * clear dirty flag
  2530. * TLB check -> fast path
  2531. * read memory
  2532. * write to RAM
  2533. *
  2534. * by pushing the migration thread's memory read after the vCPU thread has
  2535. * written the memory.
  2536. */
  2537. if (replay_mode == REPLAY_MODE_NONE) {
  2538. /*
  2539. * VGA can make calls to this function while updating the screen.
  2540. * In record/replay mode this causes a deadlock, because
  2541. * run_on_cpu waits for rr mutex. Therefore no races are possible
  2542. * in this case and no need for making run_on_cpu when
  2543. * record/replay is not enabled.
  2544. */
  2545. cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
  2546. run_on_cpu(cpuas->cpu, do_nothing, RUN_ON_CPU_NULL);
  2547. }
  2548. }
  2549. static void tcg_commit(MemoryListener *listener)
  2550. {
  2551. CPUAddressSpace *cpuas;
  2552. AddressSpaceDispatch *d;
  2553. assert(tcg_enabled());
  2554. /* since each CPU stores ram addresses in its TLB cache, we must
  2555. reset the modified entries */
  2556. cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
  2557. cpu_reloading_memory_map();
  2558. /* The CPU and TLB are protected by the iothread lock.
  2559. * We reload the dispatch pointer now because cpu_reloading_memory_map()
  2560. * may have split the RCU critical section.
  2561. */
  2562. d = address_space_to_dispatch(cpuas->as);
  2563. atomic_rcu_set(&cpuas->memory_dispatch, d);
  2564. tlb_flush(cpuas->cpu);
  2565. }
  2566. static void memory_map_init(void)
  2567. {
  2568. system_memory = g_malloc(sizeof(*system_memory));
  2569. memory_region_init(system_memory, NULL, "system", UINT64_MAX);
  2570. address_space_init(&address_space_memory, system_memory, "memory");
  2571. system_io = g_malloc(sizeof(*system_io));
  2572. memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
  2573. 65536);
  2574. address_space_init(&address_space_io, system_io, "I/O");
  2575. }
  2576. MemoryRegion *get_system_memory(void)
  2577. {
  2578. return system_memory;
  2579. }
  2580. MemoryRegion *get_system_io(void)
  2581. {
  2582. return system_io;
  2583. }
  2584. #endif /* !defined(CONFIG_USER_ONLY) */
  2585. /* physical memory access (slow version, mainly for debug) */
  2586. #if defined(CONFIG_USER_ONLY)
  2587. int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
  2588. uint8_t *buf, target_ulong len, int is_write)
  2589. {
  2590. int flags;
  2591. target_ulong l, page;
  2592. void * p;
  2593. while (len > 0) {
  2594. page = addr & TARGET_PAGE_MASK;
  2595. l = (page + TARGET_PAGE_SIZE) - addr;
  2596. if (l > len)
  2597. l = len;
  2598. flags = page_get_flags(page);
  2599. if (!(flags & PAGE_VALID))
  2600. return -1;
  2601. if (is_write) {
  2602. if (!(flags & PAGE_WRITE))
  2603. return -1;
  2604. /* XXX: this code should not depend on lock_user */
  2605. if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
  2606. return -1;
  2607. memcpy(p, buf, l);
  2608. unlock_user(p, addr, l);
  2609. } else {
  2610. if (!(flags & PAGE_READ))
  2611. return -1;
  2612. /* XXX: this code should not depend on lock_user */
  2613. if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
  2614. return -1;
  2615. memcpy(buf, p, l);
  2616. unlock_user(p, addr, 0);
  2617. }
  2618. len -= l;
  2619. buf += l;
  2620. addr += l;
  2621. }
  2622. return 0;
  2623. }
  2624. #else
  2625. static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
  2626. hwaddr length)
  2627. {
  2628. uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
  2629. addr += memory_region_get_ram_addr(mr);
  2630. /* No early return if dirty_log_mask is or becomes 0, because
  2631. * cpu_physical_memory_set_dirty_range will still call
  2632. * xen_modified_memory.
  2633. */
  2634. if (dirty_log_mask) {
  2635. dirty_log_mask =
  2636. cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
  2637. }
  2638. if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
  2639. assert(tcg_enabled());
  2640. tb_invalidate_phys_range(addr, addr + length);
  2641. dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
  2642. }
  2643. cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
  2644. }
  2645. void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size)
  2646. {
  2647. /*
  2648. * In principle this function would work on other memory region types too,
  2649. * but the ROM device use case is the only one where this operation is
  2650. * necessary. Other memory regions should use the
  2651. * address_space_read/write() APIs.
  2652. */
  2653. assert(memory_region_is_romd(mr));
  2654. invalidate_and_set_dirty(mr, addr, size);
  2655. }
  2656. static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
  2657. {
  2658. unsigned access_size_max = mr->ops->valid.max_access_size;
  2659. /* Regions are assumed to support 1-4 byte accesses unless
  2660. otherwise specified. */
  2661. if (access_size_max == 0) {
  2662. access_size_max = 4;
  2663. }
  2664. /* Bound the maximum access by the alignment of the address. */
  2665. if (!mr->ops->impl.unaligned) {
  2666. unsigned align_size_max = addr & -addr;
  2667. if (align_size_max != 0 && align_size_max < access_size_max) {
  2668. access_size_max = align_size_max;
  2669. }
  2670. }
  2671. /* Don't attempt accesses larger than the maximum. */
  2672. if (l > access_size_max) {
  2673. l = access_size_max;
  2674. }
  2675. l = pow2floor(l);
  2676. return l;
  2677. }
  2678. static bool prepare_mmio_access(MemoryRegion *mr)
  2679. {
  2680. bool unlocked = !qemu_mutex_iothread_locked();
  2681. bool release_lock = false;
  2682. if (unlocked && mr->global_locking) {
  2683. qemu_mutex_lock_iothread();
  2684. unlocked = false;
  2685. release_lock = true;
  2686. }
  2687. if (mr->flush_coalesced_mmio) {
  2688. if (unlocked) {
  2689. qemu_mutex_lock_iothread();
  2690. }
  2691. qemu_flush_coalesced_mmio_buffer();
  2692. if (unlocked) {
  2693. qemu_mutex_unlock_iothread();
  2694. }
  2695. }
  2696. return release_lock;
  2697. }
  2698. /* Called within RCU critical section. */
  2699. static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr,
  2700. MemTxAttrs attrs,
  2701. const uint8_t *buf,
  2702. hwaddr len, hwaddr addr1,
  2703. hwaddr l, MemoryRegion *mr)
  2704. {
  2705. uint8_t *ptr;
  2706. uint64_t val;
  2707. MemTxResult result = MEMTX_OK;
  2708. bool release_lock = false;
  2709. for (;;) {
  2710. if (!memory_access_is_direct(mr, true)) {
  2711. release_lock |= prepare_mmio_access(mr);
  2712. l = memory_access_size(mr, l, addr1);
  2713. /* XXX: could force current_cpu to NULL to avoid
  2714. potential bugs */
  2715. val = ldn_he_p(buf, l);
  2716. result |= memory_region_dispatch_write(mr, addr1, val,
  2717. size_memop(l), attrs);
  2718. } else {
  2719. /* RAM case */
  2720. ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false);
  2721. memcpy(ptr, buf, l);
  2722. invalidate_and_set_dirty(mr, addr1, l);
  2723. }
  2724. if (release_lock) {
  2725. qemu_mutex_unlock_iothread();
  2726. release_lock = false;
  2727. }
  2728. len -= l;
  2729. buf += l;
  2730. addr += l;
  2731. if (!len) {
  2732. break;
  2733. }
  2734. l = len;
  2735. mr = flatview_translate(fv, addr, &addr1, &l, true, attrs);
  2736. }
  2737. return result;
  2738. }
  2739. /* Called from RCU critical section. */
  2740. static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
  2741. const uint8_t *buf, hwaddr len)
  2742. {
  2743. hwaddr l;
  2744. hwaddr addr1;
  2745. MemoryRegion *mr;
  2746. MemTxResult result = MEMTX_OK;
  2747. l = len;
  2748. mr = flatview_translate(fv, addr, &addr1, &l, true, attrs);
  2749. result = flatview_write_continue(fv, addr, attrs, buf, len,
  2750. addr1, l, mr);
  2751. return result;
  2752. }
  2753. /* Called within RCU critical section. */
  2754. MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr,
  2755. MemTxAttrs attrs, uint8_t *buf,
  2756. hwaddr len, hwaddr addr1, hwaddr l,
  2757. MemoryRegion *mr)
  2758. {
  2759. uint8_t *ptr;
  2760. uint64_t val;
  2761. MemTxResult result = MEMTX_OK;
  2762. bool release_lock = false;
  2763. for (;;) {
  2764. if (!memory_access_is_direct(mr, false)) {
  2765. /* I/O case */
  2766. release_lock |= prepare_mmio_access(mr);
  2767. l = memory_access_size(mr, l, addr1);
  2768. result |= memory_region_dispatch_read(mr, addr1, &val,
  2769. size_memop(l), attrs);
  2770. stn_he_p(buf, l, val);
  2771. } else {
  2772. /* RAM case */
  2773. ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false);
  2774. memcpy(buf, ptr, l);
  2775. }
  2776. if (release_lock) {
  2777. qemu_mutex_unlock_iothread();
  2778. release_lock = false;
  2779. }
  2780. len -= l;
  2781. buf += l;
  2782. addr += l;
  2783. if (!len) {
  2784. break;
  2785. }
  2786. l = len;
  2787. mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
  2788. }
  2789. return result;
  2790. }
  2791. /* Called from RCU critical section. */
  2792. static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
  2793. MemTxAttrs attrs, uint8_t *buf, hwaddr len)
  2794. {
  2795. hwaddr l;
  2796. hwaddr addr1;
  2797. MemoryRegion *mr;
  2798. l = len;
  2799. mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
  2800. return flatview_read_continue(fv, addr, attrs, buf, len,
  2801. addr1, l, mr);
  2802. }
  2803. MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
  2804. MemTxAttrs attrs, uint8_t *buf, hwaddr len)
  2805. {
  2806. MemTxResult result = MEMTX_OK;
  2807. FlatView *fv;
  2808. if (len > 0) {
  2809. RCU_READ_LOCK_GUARD();
  2810. fv = address_space_to_flatview(as);
  2811. result = flatview_read(fv, addr, attrs, buf, len);
  2812. }
  2813. return result;
  2814. }
  2815. MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
  2816. MemTxAttrs attrs,
  2817. const uint8_t *buf, hwaddr len)
  2818. {
  2819. MemTxResult result = MEMTX_OK;
  2820. FlatView *fv;
  2821. if (len > 0) {
  2822. RCU_READ_LOCK_GUARD();
  2823. fv = address_space_to_flatview(as);
  2824. result = flatview_write(fv, addr, attrs, buf, len);
  2825. }
  2826. return result;
  2827. }
  2828. MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
  2829. uint8_t *buf, hwaddr len, bool is_write)
  2830. {
  2831. if (is_write) {
  2832. return address_space_write(as, addr, attrs, buf, len);
  2833. } else {
  2834. return address_space_read_full(as, addr, attrs, buf, len);
  2835. }
  2836. }
  2837. void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
  2838. hwaddr len, int is_write)
  2839. {
  2840. address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
  2841. buf, len, is_write);
  2842. }
  2843. enum write_rom_type {
  2844. WRITE_DATA,
  2845. FLUSH_CACHE,
  2846. };
  2847. static inline MemTxResult address_space_write_rom_internal(AddressSpace *as,
  2848. hwaddr addr,
  2849. MemTxAttrs attrs,
  2850. const uint8_t *buf,
  2851. hwaddr len,
  2852. enum write_rom_type type)
  2853. {
  2854. hwaddr l;
  2855. uint8_t *ptr;
  2856. hwaddr addr1;
  2857. MemoryRegion *mr;
  2858. RCU_READ_LOCK_GUARD();
  2859. while (len > 0) {
  2860. l = len;
  2861. mr = address_space_translate(as, addr, &addr1, &l, true, attrs);
  2862. if (!(memory_region_is_ram(mr) ||
  2863. memory_region_is_romd(mr))) {
  2864. l = memory_access_size(mr, l, addr1);
  2865. } else {
  2866. /* ROM/RAM case */
  2867. ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
  2868. switch (type) {
  2869. case WRITE_DATA:
  2870. memcpy(ptr, buf, l);
  2871. invalidate_and_set_dirty(mr, addr1, l);
  2872. break;
  2873. case FLUSH_CACHE:
  2874. flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
  2875. break;
  2876. }
  2877. }
  2878. len -= l;
  2879. buf += l;
  2880. addr += l;
  2881. }
  2882. return MEMTX_OK;
  2883. }
  2884. /* used for ROM loading : can write in RAM and ROM */
  2885. MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr,
  2886. MemTxAttrs attrs,
  2887. const uint8_t *buf, hwaddr len)
  2888. {
  2889. return address_space_write_rom_internal(as, addr, attrs,
  2890. buf, len, WRITE_DATA);
  2891. }
  2892. void cpu_flush_icache_range(hwaddr start, hwaddr len)
  2893. {
  2894. /*
  2895. * This function should do the same thing as an icache flush that was
  2896. * triggered from within the guest. For TCG we are always cache coherent,
  2897. * so there is no need to flush anything. For KVM / Xen we need to flush
  2898. * the host's instruction cache at least.
  2899. */
  2900. if (tcg_enabled()) {
  2901. return;
  2902. }
  2903. address_space_write_rom_internal(&address_space_memory,
  2904. start, MEMTXATTRS_UNSPECIFIED,
  2905. NULL, len, FLUSH_CACHE);
  2906. }
  2907. typedef struct {
  2908. MemoryRegion *mr;
  2909. void *buffer;
  2910. hwaddr addr;
  2911. hwaddr len;
  2912. bool in_use;
  2913. } BounceBuffer;
  2914. static BounceBuffer bounce;
  2915. typedef struct MapClient {
  2916. QEMUBH *bh;
  2917. QLIST_ENTRY(MapClient) link;
  2918. } MapClient;
  2919. QemuMutex map_client_list_lock;
  2920. static QLIST_HEAD(, MapClient) map_client_list
  2921. = QLIST_HEAD_INITIALIZER(map_client_list);
  2922. static void cpu_unregister_map_client_do(MapClient *client)
  2923. {
  2924. QLIST_REMOVE(client, link);
  2925. g_free(client);
  2926. }
  2927. static void cpu_notify_map_clients_locked(void)
  2928. {
  2929. MapClient *client;
  2930. while (!QLIST_EMPTY(&map_client_list)) {
  2931. client = QLIST_FIRST(&map_client_list);
  2932. qemu_bh_schedule(client->bh);
  2933. cpu_unregister_map_client_do(client);
  2934. }
  2935. }
  2936. void cpu_register_map_client(QEMUBH *bh)
  2937. {
  2938. MapClient *client = g_malloc(sizeof(*client));
  2939. qemu_mutex_lock(&map_client_list_lock);
  2940. client->bh = bh;
  2941. QLIST_INSERT_HEAD(&map_client_list, client, link);
  2942. if (!atomic_read(&bounce.in_use)) {
  2943. cpu_notify_map_clients_locked();
  2944. }
  2945. qemu_mutex_unlock(&map_client_list_lock);
  2946. }
  2947. void cpu_exec_init_all(void)
  2948. {
  2949. qemu_mutex_init(&ram_list.mutex);
  2950. /* The data structures we set up here depend on knowing the page size,
  2951. * so no more changes can be made after this point.
  2952. * In an ideal world, nothing we did before we had finished the
  2953. * machine setup would care about the target page size, and we could
  2954. * do this much later, rather than requiring board models to state
  2955. * up front what their requirements are.
  2956. */
  2957. finalize_target_page_bits();
  2958. io_mem_init();
  2959. memory_map_init();
  2960. qemu_mutex_init(&map_client_list_lock);
  2961. }
  2962. void cpu_unregister_map_client(QEMUBH *bh)
  2963. {
  2964. MapClient *client;
  2965. qemu_mutex_lock(&map_client_list_lock);
  2966. QLIST_FOREACH(client, &map_client_list, link) {
  2967. if (client->bh == bh) {
  2968. cpu_unregister_map_client_do(client);
  2969. break;
  2970. }
  2971. }
  2972. qemu_mutex_unlock(&map_client_list_lock);
  2973. }
  2974. static void cpu_notify_map_clients(void)
  2975. {
  2976. qemu_mutex_lock(&map_client_list_lock);
  2977. cpu_notify_map_clients_locked();
  2978. qemu_mutex_unlock(&map_client_list_lock);
  2979. }
  2980. static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
  2981. bool is_write, MemTxAttrs attrs)
  2982. {
  2983. MemoryRegion *mr;
  2984. hwaddr l, xlat;
  2985. while (len > 0) {
  2986. l = len;
  2987. mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
  2988. if (!memory_access_is_direct(mr, is_write)) {
  2989. l = memory_access_size(mr, l, addr);
  2990. if (!memory_region_access_valid(mr, xlat, l, is_write, attrs)) {
  2991. return false;
  2992. }
  2993. }
  2994. len -= l;
  2995. addr += l;
  2996. }
  2997. return true;
  2998. }
  2999. bool address_space_access_valid(AddressSpace *as, hwaddr addr,
  3000. hwaddr len, bool is_write,
  3001. MemTxAttrs attrs)
  3002. {
  3003. FlatView *fv;
  3004. bool result;
  3005. RCU_READ_LOCK_GUARD();
  3006. fv = address_space_to_flatview(as);
  3007. result = flatview_access_valid(fv, addr, len, is_write, attrs);
  3008. return result;
  3009. }
  3010. static hwaddr
  3011. flatview_extend_translation(FlatView *fv, hwaddr addr,
  3012. hwaddr target_len,
  3013. MemoryRegion *mr, hwaddr base, hwaddr len,
  3014. bool is_write, MemTxAttrs attrs)
  3015. {
  3016. hwaddr done = 0;
  3017. hwaddr xlat;
  3018. MemoryRegion *this_mr;
  3019. for (;;) {
  3020. target_len -= len;
  3021. addr += len;
  3022. done += len;
  3023. if (target_len == 0) {
  3024. return done;
  3025. }
  3026. len = target_len;
  3027. this_mr = flatview_translate(fv, addr, &xlat,
  3028. &len, is_write, attrs);
  3029. if (this_mr != mr || xlat != base + done) {
  3030. return done;
  3031. }
  3032. }
  3033. }
  3034. /* Map a physical memory region into a host virtual address.
  3035. * May map a subset of the requested range, given by and returned in *plen.
  3036. * May return NULL if resources needed to perform the mapping are exhausted.
  3037. * Use only for reads OR writes - not for read-modify-write operations.
  3038. * Use cpu_register_map_client() to know when retrying the map operation is
  3039. * likely to succeed.
  3040. */
  3041. void *address_space_map(AddressSpace *as,
  3042. hwaddr addr,
  3043. hwaddr *plen,
  3044. bool is_write,
  3045. MemTxAttrs attrs)
  3046. {
  3047. hwaddr len = *plen;
  3048. hwaddr l, xlat;
  3049. MemoryRegion *mr;
  3050. void *ptr;
  3051. FlatView *fv;
  3052. if (len == 0) {
  3053. return NULL;
  3054. }
  3055. l = len;
  3056. RCU_READ_LOCK_GUARD();
  3057. fv = address_space_to_flatview(as);
  3058. mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
  3059. if (!memory_access_is_direct(mr, is_write)) {
  3060. if (atomic_xchg(&bounce.in_use, true)) {
  3061. return NULL;
  3062. }
  3063. /* Avoid unbounded allocations */
  3064. l = MIN(l, TARGET_PAGE_SIZE);
  3065. bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
  3066. bounce.addr = addr;
  3067. bounce.len = l;
  3068. memory_region_ref(mr);
  3069. bounce.mr = mr;
  3070. if (!is_write) {
  3071. flatview_read(fv, addr, MEMTXATTRS_UNSPECIFIED,
  3072. bounce.buffer, l);
  3073. }
  3074. *plen = l;
  3075. return bounce.buffer;
  3076. }
  3077. memory_region_ref(mr);
  3078. *plen = flatview_extend_translation(fv, addr, len, mr, xlat,
  3079. l, is_write, attrs);
  3080. ptr = qemu_ram_ptr_length(mr->ram_block, xlat, plen, true);
  3081. return ptr;
  3082. }
  3083. /* Unmaps a memory region previously mapped by address_space_map().
  3084. * Will also mark the memory as dirty if is_write == 1. access_len gives
  3085. * the amount of memory that was actually read or written by the caller.
  3086. */
  3087. void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
  3088. int is_write, hwaddr access_len)
  3089. {
  3090. if (buffer != bounce.buffer) {
  3091. MemoryRegion *mr;
  3092. ram_addr_t addr1;
  3093. mr = memory_region_from_host(buffer, &addr1);
  3094. assert(mr != NULL);
  3095. if (is_write) {
  3096. invalidate_and_set_dirty(mr, addr1, access_len);
  3097. }
  3098. if (xen_enabled()) {
  3099. xen_invalidate_map_cache_entry(buffer);
  3100. }
  3101. memory_region_unref(mr);
  3102. return;
  3103. }
  3104. if (is_write) {
  3105. address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
  3106. bounce.buffer, access_len);
  3107. }
  3108. qemu_vfree(bounce.buffer);
  3109. bounce.buffer = NULL;
  3110. memory_region_unref(bounce.mr);
  3111. atomic_mb_set(&bounce.in_use, false);
  3112. cpu_notify_map_clients();
  3113. }
  3114. void *cpu_physical_memory_map(hwaddr addr,
  3115. hwaddr *plen,
  3116. int is_write)
  3117. {
  3118. return address_space_map(&address_space_memory, addr, plen, is_write,
  3119. MEMTXATTRS_UNSPECIFIED);
  3120. }
  3121. void cpu_physical_memory_unmap(void *buffer, hwaddr len,
  3122. int is_write, hwaddr access_len)
  3123. {
  3124. return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
  3125. }
  3126. #define ARG1_DECL AddressSpace *as
  3127. #define ARG1 as
  3128. #define SUFFIX
  3129. #define TRANSLATE(...) address_space_translate(as, __VA_ARGS__)
  3130. #define RCU_READ_LOCK(...) rcu_read_lock()
  3131. #define RCU_READ_UNLOCK(...) rcu_read_unlock()
  3132. #include "memory_ldst.inc.c"
  3133. int64_t address_space_cache_init(MemoryRegionCache *cache,
  3134. AddressSpace *as,
  3135. hwaddr addr,
  3136. hwaddr len,
  3137. bool is_write)
  3138. {
  3139. AddressSpaceDispatch *d;
  3140. hwaddr l;
  3141. MemoryRegion *mr;
  3142. assert(len > 0);
  3143. l = len;
  3144. cache->fv = address_space_get_flatview(as);
  3145. d = flatview_to_dispatch(cache->fv);
  3146. cache->mrs = *address_space_translate_internal(d, addr, &cache->xlat, &l, true);
  3147. mr = cache->mrs.mr;
  3148. memory_region_ref(mr);
  3149. if (memory_access_is_direct(mr, is_write)) {
  3150. /* We don't care about the memory attributes here as we're only
  3151. * doing this if we found actual RAM, which behaves the same
  3152. * regardless of attributes; so UNSPECIFIED is fine.
  3153. */
  3154. l = flatview_extend_translation(cache->fv, addr, len, mr,
  3155. cache->xlat, l, is_write,
  3156. MEMTXATTRS_UNSPECIFIED);
  3157. cache->ptr = qemu_ram_ptr_length(mr->ram_block, cache->xlat, &l, true);
  3158. } else {
  3159. cache->ptr = NULL;
  3160. }
  3161. cache->len = l;
  3162. cache->is_write = is_write;
  3163. return l;
  3164. }
  3165. void address_space_cache_invalidate(MemoryRegionCache *cache,
  3166. hwaddr addr,
  3167. hwaddr access_len)
  3168. {
  3169. assert(cache->is_write);
  3170. if (likely(cache->ptr)) {
  3171. invalidate_and_set_dirty(cache->mrs.mr, addr + cache->xlat, access_len);
  3172. }
  3173. }
  3174. void address_space_cache_destroy(MemoryRegionCache *cache)
  3175. {
  3176. if (!cache->mrs.mr) {
  3177. return;
  3178. }
  3179. if (xen_enabled()) {
  3180. xen_invalidate_map_cache_entry(cache->ptr);
  3181. }
  3182. memory_region_unref(cache->mrs.mr);
  3183. flatview_unref(cache->fv);
  3184. cache->mrs.mr = NULL;
  3185. cache->fv = NULL;
  3186. }
  3187. /* Called from RCU critical section. This function has the same
  3188. * semantics as address_space_translate, but it only works on a
  3189. * predefined range of a MemoryRegion that was mapped with
  3190. * address_space_cache_init.
  3191. */
  3192. static inline MemoryRegion *address_space_translate_cached(
  3193. MemoryRegionCache *cache, hwaddr addr, hwaddr *xlat,
  3194. hwaddr *plen, bool is_write, MemTxAttrs attrs)
  3195. {
  3196. MemoryRegionSection section;
  3197. MemoryRegion *mr;
  3198. IOMMUMemoryRegion *iommu_mr;
  3199. AddressSpace *target_as;
  3200. assert(!cache->ptr);
  3201. *xlat = addr + cache->xlat;
  3202. mr = cache->mrs.mr;
  3203. iommu_mr = memory_region_get_iommu(mr);
  3204. if (!iommu_mr) {
  3205. /* MMIO region. */
  3206. return mr;
  3207. }
  3208. section = address_space_translate_iommu(iommu_mr, xlat, plen,
  3209. NULL, is_write, true,
  3210. &target_as, attrs);
  3211. return section.mr;
  3212. }
  3213. /* Called from RCU critical section. address_space_read_cached uses this
  3214. * out of line function when the target is an MMIO or IOMMU region.
  3215. */
  3216. void
  3217. address_space_read_cached_slow(MemoryRegionCache *cache, hwaddr addr,
  3218. void *buf, hwaddr len)
  3219. {
  3220. hwaddr addr1, l;
  3221. MemoryRegion *mr;
  3222. l = len;
  3223. mr = address_space_translate_cached(cache, addr, &addr1, &l, false,
  3224. MEMTXATTRS_UNSPECIFIED);
  3225. flatview_read_continue(cache->fv,
  3226. addr, MEMTXATTRS_UNSPECIFIED, buf, len,
  3227. addr1, l, mr);
  3228. }
  3229. /* Called from RCU critical section. address_space_write_cached uses this
  3230. * out of line function when the target is an MMIO or IOMMU region.
  3231. */
  3232. void
  3233. address_space_write_cached_slow(MemoryRegionCache *cache, hwaddr addr,
  3234. const void *buf, hwaddr len)
  3235. {
  3236. hwaddr addr1, l;
  3237. MemoryRegion *mr;
  3238. l = len;
  3239. mr = address_space_translate_cached(cache, addr, &addr1, &l, true,
  3240. MEMTXATTRS_UNSPECIFIED);
  3241. flatview_write_continue(cache->fv,
  3242. addr, MEMTXATTRS_UNSPECIFIED, buf, len,
  3243. addr1, l, mr);
  3244. }
  3245. #define ARG1_DECL MemoryRegionCache *cache
  3246. #define ARG1 cache
  3247. #define SUFFIX _cached_slow
  3248. #define TRANSLATE(...) address_space_translate_cached(cache, __VA_ARGS__)
  3249. #define RCU_READ_LOCK() ((void)0)
  3250. #define RCU_READ_UNLOCK() ((void)0)
  3251. #include "memory_ldst.inc.c"
  3252. /* virtual memory access for debug (includes writing to ROM) */
  3253. int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
  3254. uint8_t *buf, target_ulong len, int is_write)
  3255. {
  3256. hwaddr phys_addr;
  3257. target_ulong l, page;
  3258. cpu_synchronize_state(cpu);
  3259. while (len > 0) {
  3260. int asidx;
  3261. MemTxAttrs attrs;
  3262. page = addr & TARGET_PAGE_MASK;
  3263. phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
  3264. asidx = cpu_asidx_from_attrs(cpu, attrs);
  3265. /* if no physical page mapped, return an error */
  3266. if (phys_addr == -1)
  3267. return -1;
  3268. l = (page + TARGET_PAGE_SIZE) - addr;
  3269. if (l > len)
  3270. l = len;
  3271. phys_addr += (addr & ~TARGET_PAGE_MASK);
  3272. if (is_write) {
  3273. address_space_write_rom(cpu->cpu_ases[asidx].as, phys_addr,
  3274. attrs, buf, l);
  3275. } else {
  3276. address_space_rw(cpu->cpu_ases[asidx].as, phys_addr,
  3277. attrs, buf, l, 0);
  3278. }
  3279. len -= l;
  3280. buf += l;
  3281. addr += l;
  3282. }
  3283. return 0;
  3284. }
  3285. /*
  3286. * Allows code that needs to deal with migration bitmaps etc to still be built
  3287. * target independent.
  3288. */
  3289. size_t qemu_target_page_size(void)
  3290. {
  3291. return TARGET_PAGE_SIZE;
  3292. }
  3293. int qemu_target_page_bits(void)
  3294. {
  3295. return TARGET_PAGE_BITS;
  3296. }
  3297. int qemu_target_page_bits_min(void)
  3298. {
  3299. return TARGET_PAGE_BITS_MIN;
  3300. }
  3301. #endif
  3302. bool target_words_bigendian(void)
  3303. {
  3304. #if defined(TARGET_WORDS_BIGENDIAN)
  3305. return true;
  3306. #else
  3307. return false;
  3308. #endif
  3309. }
  3310. #ifndef CONFIG_USER_ONLY
  3311. bool cpu_physical_memory_is_io(hwaddr phys_addr)
  3312. {
  3313. MemoryRegion*mr;
  3314. hwaddr l = 1;
  3315. bool res;
  3316. RCU_READ_LOCK_GUARD();
  3317. mr = address_space_translate(&address_space_memory,
  3318. phys_addr, &phys_addr, &l, false,
  3319. MEMTXATTRS_UNSPECIFIED);
  3320. res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
  3321. return res;
  3322. }
  3323. int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
  3324. {
  3325. RAMBlock *block;
  3326. int ret = 0;
  3327. RCU_READ_LOCK_GUARD();
  3328. RAMBLOCK_FOREACH(block) {
  3329. ret = func(block, opaque);
  3330. if (ret) {
  3331. break;
  3332. }
  3333. }
  3334. return ret;
  3335. }
  3336. /*
  3337. * Unmap pages of memory from start to start+length such that
  3338. * they a) read as 0, b) Trigger whatever fault mechanism
  3339. * the OS provides for postcopy.
  3340. * The pages must be unmapped by the end of the function.
  3341. * Returns: 0 on success, none-0 on failure
  3342. *
  3343. */
  3344. int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length)
  3345. {
  3346. int ret = -1;
  3347. uint8_t *host_startaddr = rb->host + start;
  3348. if ((uintptr_t)host_startaddr & (rb->page_size - 1)) {
  3349. error_report("ram_block_discard_range: Unaligned start address: %p",
  3350. host_startaddr);
  3351. goto err;
  3352. }
  3353. if ((start + length) <= rb->used_length) {
  3354. bool need_madvise, need_fallocate;
  3355. uint8_t *host_endaddr = host_startaddr + length;
  3356. if ((uintptr_t)host_endaddr & (rb->page_size - 1)) {
  3357. error_report("ram_block_discard_range: Unaligned end address: %p",
  3358. host_endaddr);
  3359. goto err;
  3360. }
  3361. errno = ENOTSUP; /* If we are missing MADVISE etc */
  3362. /* The logic here is messy;
  3363. * madvise DONTNEED fails for hugepages
  3364. * fallocate works on hugepages and shmem
  3365. */
  3366. need_madvise = (rb->page_size == qemu_host_page_size);
  3367. need_fallocate = rb->fd != -1;
  3368. if (need_fallocate) {
  3369. /* For a file, this causes the area of the file to be zero'd
  3370. * if read, and for hugetlbfs also causes it to be unmapped
  3371. * so a userfault will trigger.
  3372. */
  3373. #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
  3374. ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
  3375. start, length);
  3376. if (ret) {
  3377. ret = -errno;
  3378. error_report("ram_block_discard_range: Failed to fallocate "
  3379. "%s:%" PRIx64 " +%zx (%d)",
  3380. rb->idstr, start, length, ret);
  3381. goto err;
  3382. }
  3383. #else
  3384. ret = -ENOSYS;
  3385. error_report("ram_block_discard_range: fallocate not available/file"
  3386. "%s:%" PRIx64 " +%zx (%d)",
  3387. rb->idstr, start, length, ret);
  3388. goto err;
  3389. #endif
  3390. }
  3391. if (need_madvise) {
  3392. /* For normal RAM this causes it to be unmapped,
  3393. * for shared memory it causes the local mapping to disappear
  3394. * and to fall back on the file contents (which we just
  3395. * fallocate'd away).
  3396. */
  3397. #if defined(CONFIG_MADVISE)
  3398. ret = madvise(host_startaddr, length, MADV_DONTNEED);
  3399. if (ret) {
  3400. ret = -errno;
  3401. error_report("ram_block_discard_range: Failed to discard range "
  3402. "%s:%" PRIx64 " +%zx (%d)",
  3403. rb->idstr, start, length, ret);
  3404. goto err;
  3405. }
  3406. #else
  3407. ret = -ENOSYS;
  3408. error_report("ram_block_discard_range: MADVISE not available"
  3409. "%s:%" PRIx64 " +%zx (%d)",
  3410. rb->idstr, start, length, ret);
  3411. goto err;
  3412. #endif
  3413. }
  3414. trace_ram_block_discard_range(rb->idstr, host_startaddr, length,
  3415. need_madvise, need_fallocate, ret);
  3416. } else {
  3417. error_report("ram_block_discard_range: Overrun block '%s' (%" PRIu64
  3418. "/%zx/" RAM_ADDR_FMT")",
  3419. rb->idstr, start, length, rb->used_length);
  3420. }
  3421. err:
  3422. return ret;
  3423. }
  3424. bool ramblock_is_pmem(RAMBlock *rb)
  3425. {
  3426. return rb->flags & RAM_PMEM;
  3427. }
  3428. #endif
  3429. void page_size_init(void)
  3430. {
  3431. /* NOTE: we can always suppose that qemu_host_page_size >=
  3432. TARGET_PAGE_SIZE */
  3433. if (qemu_host_page_size == 0) {
  3434. qemu_host_page_size = qemu_real_host_page_size;
  3435. }
  3436. if (qemu_host_page_size < TARGET_PAGE_SIZE) {
  3437. qemu_host_page_size = TARGET_PAGE_SIZE;
  3438. }
  3439. qemu_host_page_mask = -(intptr_t)qemu_host_page_size;
  3440. }
  3441. #if !defined(CONFIG_USER_ONLY)
  3442. static void mtree_print_phys_entries(int start, int end, int skip, int ptr)
  3443. {
  3444. if (start == end - 1) {
  3445. qemu_printf("\t%3d ", start);
  3446. } else {
  3447. qemu_printf("\t%3d..%-3d ", start, end - 1);
  3448. }
  3449. qemu_printf(" skip=%d ", skip);
  3450. if (ptr == PHYS_MAP_NODE_NIL) {
  3451. qemu_printf(" ptr=NIL");
  3452. } else if (!skip) {
  3453. qemu_printf(" ptr=#%d", ptr);
  3454. } else {
  3455. qemu_printf(" ptr=[%d]", ptr);
  3456. }
  3457. qemu_printf("\n");
  3458. }
  3459. #define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \
  3460. int128_sub((size), int128_one())) : 0)
  3461. void mtree_print_dispatch(AddressSpaceDispatch *d, MemoryRegion *root)
  3462. {
  3463. int i;
  3464. qemu_printf(" Dispatch\n");
  3465. qemu_printf(" Physical sections\n");
  3466. for (i = 0; i < d->map.sections_nb; ++i) {
  3467. MemoryRegionSection *s = d->map.sections + i;
  3468. const char *names[] = { " [unassigned]", " [not dirty]",
  3469. " [ROM]", " [watch]" };
  3470. qemu_printf(" #%d @" TARGET_FMT_plx ".." TARGET_FMT_plx
  3471. " %s%s%s%s%s",
  3472. i,
  3473. s->offset_within_address_space,
  3474. s->offset_within_address_space + MR_SIZE(s->mr->size),
  3475. s->mr->name ? s->mr->name : "(noname)",
  3476. i < ARRAY_SIZE(names) ? names[i] : "",
  3477. s->mr == root ? " [ROOT]" : "",
  3478. s == d->mru_section ? " [MRU]" : "",
  3479. s->mr->is_iommu ? " [iommu]" : "");
  3480. if (s->mr->alias) {
  3481. qemu_printf(" alias=%s", s->mr->alias->name ?
  3482. s->mr->alias->name : "noname");
  3483. }
  3484. qemu_printf("\n");
  3485. }
  3486. qemu_printf(" Nodes (%d bits per level, %d levels) ptr=[%d] skip=%d\n",
  3487. P_L2_BITS, P_L2_LEVELS, d->phys_map.ptr, d->phys_map.skip);
  3488. for (i = 0; i < d->map.nodes_nb; ++i) {
  3489. int j, jprev;
  3490. PhysPageEntry prev;
  3491. Node *n = d->map.nodes + i;
  3492. qemu_printf(" [%d]\n", i);
  3493. for (j = 0, jprev = 0, prev = *n[0]; j < ARRAY_SIZE(*n); ++j) {
  3494. PhysPageEntry *pe = *n + j;
  3495. if (pe->ptr == prev.ptr && pe->skip == prev.skip) {
  3496. continue;
  3497. }
  3498. mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
  3499. jprev = j;
  3500. prev = *pe;
  3501. }
  3502. if (jprev != ARRAY_SIZE(*n)) {
  3503. mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
  3504. }
  3505. }
  3506. }
  3507. #endif