You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

memory.c 98KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186
  1. /*
  2. * Physical memory management
  3. *
  4. * Copyright 2011 Red Hat, Inc. and/or its affiliates
  5. *
  6. * Authors:
  7. * Avi Kivity <avi@redhat.com>
  8. *
  9. * This work is licensed under the terms of the GNU GPL, version 2. See
  10. * the COPYING file in the top-level directory.
  11. *
  12. * Contributions after 2012-01-13 are licensed under the terms of the
  13. * GNU GPL, version 2 or (at your option) any later version.
  14. */
  15. #include "qemu/osdep.h"
  16. #include "qapi/error.h"
  17. #include "cpu.h"
  18. #include "exec/memory.h"
  19. #include "exec/address-spaces.h"
  20. #include "qapi/visitor.h"
  21. #include "qemu/bitops.h"
  22. #include "qemu/error-report.h"
  23. #include "qemu/qemu-print.h"
  24. #include "qom/object.h"
  25. #include "trace-root.h"
  26. #include "exec/memory-internal.h"
  27. #include "exec/ram_addr.h"
  28. #include "sysemu/kvm.h"
  29. #include "sysemu/sysemu.h"
  30. #include "sysemu/tcg.h"
  31. #include "hw/qdev-properties.h"
  32. #include "migration/vmstate.h"
  33. //#define DEBUG_UNASSIGNED
  34. static unsigned memory_region_transaction_depth;
  35. static bool memory_region_update_pending;
  36. static bool ioeventfd_update_pending;
  37. static bool global_dirty_log = false;
  38. static QTAILQ_HEAD(, MemoryListener) memory_listeners
  39. = QTAILQ_HEAD_INITIALIZER(memory_listeners);
  40. static QTAILQ_HEAD(, AddressSpace) address_spaces
  41. = QTAILQ_HEAD_INITIALIZER(address_spaces);
  42. static GHashTable *flat_views;
  43. typedef struct AddrRange AddrRange;
  44. /*
  45. * Note that signed integers are needed for negative offsetting in aliases
  46. * (large MemoryRegion::alias_offset).
  47. */
  48. struct AddrRange {
  49. Int128 start;
  50. Int128 size;
  51. };
  52. static AddrRange addrrange_make(Int128 start, Int128 size)
  53. {
  54. return (AddrRange) { start, size };
  55. }
  56. static bool addrrange_equal(AddrRange r1, AddrRange r2)
  57. {
  58. return int128_eq(r1.start, r2.start) && int128_eq(r1.size, r2.size);
  59. }
  60. static Int128 addrrange_end(AddrRange r)
  61. {
  62. return int128_add(r.start, r.size);
  63. }
  64. static AddrRange addrrange_shift(AddrRange range, Int128 delta)
  65. {
  66. int128_addto(&range.start, delta);
  67. return range;
  68. }
  69. static bool addrrange_contains(AddrRange range, Int128 addr)
  70. {
  71. return int128_ge(addr, range.start)
  72. && int128_lt(addr, addrrange_end(range));
  73. }
  74. static bool addrrange_intersects(AddrRange r1, AddrRange r2)
  75. {
  76. return addrrange_contains(r1, r2.start)
  77. || addrrange_contains(r2, r1.start);
  78. }
  79. static AddrRange addrrange_intersection(AddrRange r1, AddrRange r2)
  80. {
  81. Int128 start = int128_max(r1.start, r2.start);
  82. Int128 end = int128_min(addrrange_end(r1), addrrange_end(r2));
  83. return addrrange_make(start, int128_sub(end, start));
  84. }
  85. enum ListenerDirection { Forward, Reverse };
  86. #define MEMORY_LISTENER_CALL_GLOBAL(_callback, _direction, _args...) \
  87. do { \
  88. MemoryListener *_listener; \
  89. \
  90. switch (_direction) { \
  91. case Forward: \
  92. QTAILQ_FOREACH(_listener, &memory_listeners, link) { \
  93. if (_listener->_callback) { \
  94. _listener->_callback(_listener, ##_args); \
  95. } \
  96. } \
  97. break; \
  98. case Reverse: \
  99. QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, link) { \
  100. if (_listener->_callback) { \
  101. _listener->_callback(_listener, ##_args); \
  102. } \
  103. } \
  104. break; \
  105. default: \
  106. abort(); \
  107. } \
  108. } while (0)
  109. #define MEMORY_LISTENER_CALL(_as, _callback, _direction, _section, _args...) \
  110. do { \
  111. MemoryListener *_listener; \
  112. \
  113. switch (_direction) { \
  114. case Forward: \
  115. QTAILQ_FOREACH(_listener, &(_as)->listeners, link_as) { \
  116. if (_listener->_callback) { \
  117. _listener->_callback(_listener, _section, ##_args); \
  118. } \
  119. } \
  120. break; \
  121. case Reverse: \
  122. QTAILQ_FOREACH_REVERSE(_listener, &(_as)->listeners, link_as) { \
  123. if (_listener->_callback) { \
  124. _listener->_callback(_listener, _section, ##_args); \
  125. } \
  126. } \
  127. break; \
  128. default: \
  129. abort(); \
  130. } \
  131. } while (0)
  132. /* No need to ref/unref .mr, the FlatRange keeps it alive. */
  133. #define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback, _args...) \
  134. do { \
  135. MemoryRegionSection mrs = section_from_flat_range(fr, \
  136. address_space_to_flatview(as)); \
  137. MEMORY_LISTENER_CALL(as, callback, dir, &mrs, ##_args); \
  138. } while(0)
  139. struct CoalescedMemoryRange {
  140. AddrRange addr;
  141. QTAILQ_ENTRY(CoalescedMemoryRange) link;
  142. };
  143. struct MemoryRegionIoeventfd {
  144. AddrRange addr;
  145. bool match_data;
  146. uint64_t data;
  147. EventNotifier *e;
  148. };
  149. static bool memory_region_ioeventfd_before(MemoryRegionIoeventfd *a,
  150. MemoryRegionIoeventfd *b)
  151. {
  152. if (int128_lt(a->addr.start, b->addr.start)) {
  153. return true;
  154. } else if (int128_gt(a->addr.start, b->addr.start)) {
  155. return false;
  156. } else if (int128_lt(a->addr.size, b->addr.size)) {
  157. return true;
  158. } else if (int128_gt(a->addr.size, b->addr.size)) {
  159. return false;
  160. } else if (a->match_data < b->match_data) {
  161. return true;
  162. } else if (a->match_data > b->match_data) {
  163. return false;
  164. } else if (a->match_data) {
  165. if (a->data < b->data) {
  166. return true;
  167. } else if (a->data > b->data) {
  168. return false;
  169. }
  170. }
  171. if (a->e < b->e) {
  172. return true;
  173. } else if (a->e > b->e) {
  174. return false;
  175. }
  176. return false;
  177. }
  178. static bool memory_region_ioeventfd_equal(MemoryRegionIoeventfd *a,
  179. MemoryRegionIoeventfd *b)
  180. {
  181. return !memory_region_ioeventfd_before(a, b)
  182. && !memory_region_ioeventfd_before(b, a);
  183. }
  184. /* Range of memory in the global map. Addresses are absolute. */
  185. struct FlatRange {
  186. MemoryRegion *mr;
  187. hwaddr offset_in_region;
  188. AddrRange addr;
  189. uint8_t dirty_log_mask;
  190. bool romd_mode;
  191. bool readonly;
  192. bool nonvolatile;
  193. int has_coalesced_range;
  194. };
  195. #define FOR_EACH_FLAT_RANGE(var, view) \
  196. for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var)
  197. static inline MemoryRegionSection
  198. section_from_flat_range(FlatRange *fr, FlatView *fv)
  199. {
  200. return (MemoryRegionSection) {
  201. .mr = fr->mr,
  202. .fv = fv,
  203. .offset_within_region = fr->offset_in_region,
  204. .size = fr->addr.size,
  205. .offset_within_address_space = int128_get64(fr->addr.start),
  206. .readonly = fr->readonly,
  207. .nonvolatile = fr->nonvolatile,
  208. };
  209. }
  210. static bool flatrange_equal(FlatRange *a, FlatRange *b)
  211. {
  212. return a->mr == b->mr
  213. && addrrange_equal(a->addr, b->addr)
  214. && a->offset_in_region == b->offset_in_region
  215. && a->romd_mode == b->romd_mode
  216. && a->readonly == b->readonly
  217. && a->nonvolatile == b->nonvolatile;
  218. }
  219. static FlatView *flatview_new(MemoryRegion *mr_root)
  220. {
  221. FlatView *view;
  222. view = g_new0(FlatView, 1);
  223. view->ref = 1;
  224. view->root = mr_root;
  225. memory_region_ref(mr_root);
  226. trace_flatview_new(view, mr_root);
  227. return view;
  228. }
  229. /* Insert a range into a given position. Caller is responsible for maintaining
  230. * sorting order.
  231. */
  232. static void flatview_insert(FlatView *view, unsigned pos, FlatRange *range)
  233. {
  234. if (view->nr == view->nr_allocated) {
  235. view->nr_allocated = MAX(2 * view->nr, 10);
  236. view->ranges = g_realloc(view->ranges,
  237. view->nr_allocated * sizeof(*view->ranges));
  238. }
  239. memmove(view->ranges + pos + 1, view->ranges + pos,
  240. (view->nr - pos) * sizeof(FlatRange));
  241. view->ranges[pos] = *range;
  242. memory_region_ref(range->mr);
  243. ++view->nr;
  244. }
  245. static void flatview_destroy(FlatView *view)
  246. {
  247. int i;
  248. trace_flatview_destroy(view, view->root);
  249. if (view->dispatch) {
  250. address_space_dispatch_free(view->dispatch);
  251. }
  252. for (i = 0; i < view->nr; i++) {
  253. memory_region_unref(view->ranges[i].mr);
  254. }
  255. g_free(view->ranges);
  256. memory_region_unref(view->root);
  257. g_free(view);
  258. }
  259. static bool flatview_ref(FlatView *view)
  260. {
  261. return atomic_fetch_inc_nonzero(&view->ref) > 0;
  262. }
  263. void flatview_unref(FlatView *view)
  264. {
  265. if (atomic_fetch_dec(&view->ref) == 1) {
  266. trace_flatview_destroy_rcu(view, view->root);
  267. assert(view->root);
  268. call_rcu(view, flatview_destroy, rcu);
  269. }
  270. }
  271. static bool can_merge(FlatRange *r1, FlatRange *r2)
  272. {
  273. return int128_eq(addrrange_end(r1->addr), r2->addr.start)
  274. && r1->mr == r2->mr
  275. && int128_eq(int128_add(int128_make64(r1->offset_in_region),
  276. r1->addr.size),
  277. int128_make64(r2->offset_in_region))
  278. && r1->dirty_log_mask == r2->dirty_log_mask
  279. && r1->romd_mode == r2->romd_mode
  280. && r1->readonly == r2->readonly
  281. && r1->nonvolatile == r2->nonvolatile;
  282. }
  283. /* Attempt to simplify a view by merging adjacent ranges */
  284. static void flatview_simplify(FlatView *view)
  285. {
  286. unsigned i, j;
  287. i = 0;
  288. while (i < view->nr) {
  289. j = i + 1;
  290. while (j < view->nr
  291. && can_merge(&view->ranges[j-1], &view->ranges[j])) {
  292. int128_addto(&view->ranges[i].addr.size, view->ranges[j].addr.size);
  293. ++j;
  294. }
  295. ++i;
  296. memmove(&view->ranges[i], &view->ranges[j],
  297. (view->nr - j) * sizeof(view->ranges[j]));
  298. view->nr -= j - i;
  299. }
  300. }
  301. static bool memory_region_big_endian(MemoryRegion *mr)
  302. {
  303. #ifdef TARGET_WORDS_BIGENDIAN
  304. return mr->ops->endianness != DEVICE_LITTLE_ENDIAN;
  305. #else
  306. return mr->ops->endianness == DEVICE_BIG_ENDIAN;
  307. #endif
  308. }
  309. static bool memory_region_wrong_endianness(MemoryRegion *mr)
  310. {
  311. #ifdef TARGET_WORDS_BIGENDIAN
  312. return mr->ops->endianness == DEVICE_LITTLE_ENDIAN;
  313. #else
  314. return mr->ops->endianness == DEVICE_BIG_ENDIAN;
  315. #endif
  316. }
  317. static void adjust_endianness(MemoryRegion *mr, uint64_t *data, unsigned size)
  318. {
  319. if (memory_region_wrong_endianness(mr)) {
  320. switch (size) {
  321. case 1:
  322. break;
  323. case 2:
  324. *data = bswap16(*data);
  325. break;
  326. case 4:
  327. *data = bswap32(*data);
  328. break;
  329. case 8:
  330. *data = bswap64(*data);
  331. break;
  332. default:
  333. abort();
  334. }
  335. }
  336. }
  337. static inline void memory_region_shift_read_access(uint64_t *value,
  338. signed shift,
  339. uint64_t mask,
  340. uint64_t tmp)
  341. {
  342. if (shift >= 0) {
  343. *value |= (tmp & mask) << shift;
  344. } else {
  345. *value |= (tmp & mask) >> -shift;
  346. }
  347. }
  348. static inline uint64_t memory_region_shift_write_access(uint64_t *value,
  349. signed shift,
  350. uint64_t mask)
  351. {
  352. uint64_t tmp;
  353. if (shift >= 0) {
  354. tmp = (*value >> shift) & mask;
  355. } else {
  356. tmp = (*value << -shift) & mask;
  357. }
  358. return tmp;
  359. }
  360. static hwaddr memory_region_to_absolute_addr(MemoryRegion *mr, hwaddr offset)
  361. {
  362. MemoryRegion *root;
  363. hwaddr abs_addr = offset;
  364. abs_addr += mr->addr;
  365. for (root = mr; root->container; ) {
  366. root = root->container;
  367. abs_addr += root->addr;
  368. }
  369. return abs_addr;
  370. }
  371. static int get_cpu_index(void)
  372. {
  373. if (current_cpu) {
  374. return current_cpu->cpu_index;
  375. }
  376. return -1;
  377. }
  378. static MemTxResult memory_region_read_accessor(MemoryRegion *mr,
  379. hwaddr addr,
  380. uint64_t *value,
  381. unsigned size,
  382. signed shift,
  383. uint64_t mask,
  384. MemTxAttrs attrs)
  385. {
  386. uint64_t tmp;
  387. tmp = mr->ops->read(mr->opaque, addr, size);
  388. if (mr->subpage) {
  389. trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);
  390. } else if (mr == &io_mem_notdirty) {
  391. /* Accesses to code which has previously been translated into a TB show
  392. * up in the MMIO path, as accesses to the io_mem_notdirty
  393. * MemoryRegion. */
  394. trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);
  395. } else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {
  396. hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
  397. trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);
  398. }
  399. memory_region_shift_read_access(value, shift, mask, tmp);
  400. return MEMTX_OK;
  401. }
  402. static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr,
  403. hwaddr addr,
  404. uint64_t *value,
  405. unsigned size,
  406. signed shift,
  407. uint64_t mask,
  408. MemTxAttrs attrs)
  409. {
  410. uint64_t tmp = 0;
  411. MemTxResult r;
  412. r = mr->ops->read_with_attrs(mr->opaque, addr, &tmp, size, attrs);
  413. if (mr->subpage) {
  414. trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);
  415. } else if (mr == &io_mem_notdirty) {
  416. /* Accesses to code which has previously been translated into a TB show
  417. * up in the MMIO path, as accesses to the io_mem_notdirty
  418. * MemoryRegion. */
  419. trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);
  420. } else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {
  421. hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
  422. trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);
  423. }
  424. memory_region_shift_read_access(value, shift, mask, tmp);
  425. return r;
  426. }
  427. static MemTxResult memory_region_write_accessor(MemoryRegion *mr,
  428. hwaddr addr,
  429. uint64_t *value,
  430. unsigned size,
  431. signed shift,
  432. uint64_t mask,
  433. MemTxAttrs attrs)
  434. {
  435. uint64_t tmp = memory_region_shift_write_access(value, shift, mask);
  436. if (mr->subpage) {
  437. trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);
  438. } else if (mr == &io_mem_notdirty) {
  439. /* Accesses to code which has previously been translated into a TB show
  440. * up in the MMIO path, as accesses to the io_mem_notdirty
  441. * MemoryRegion. */
  442. trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);
  443. } else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {
  444. hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
  445. trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);
  446. }
  447. mr->ops->write(mr->opaque, addr, tmp, size);
  448. return MEMTX_OK;
  449. }
  450. static MemTxResult memory_region_write_with_attrs_accessor(MemoryRegion *mr,
  451. hwaddr addr,
  452. uint64_t *value,
  453. unsigned size,
  454. signed shift,
  455. uint64_t mask,
  456. MemTxAttrs attrs)
  457. {
  458. uint64_t tmp = memory_region_shift_write_access(value, shift, mask);
  459. if (mr->subpage) {
  460. trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);
  461. } else if (mr == &io_mem_notdirty) {
  462. /* Accesses to code which has previously been translated into a TB show
  463. * up in the MMIO path, as accesses to the io_mem_notdirty
  464. * MemoryRegion. */
  465. trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);
  466. } else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {
  467. hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
  468. trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);
  469. }
  470. return mr->ops->write_with_attrs(mr->opaque, addr, tmp, size, attrs);
  471. }
  472. static MemTxResult access_with_adjusted_size(hwaddr addr,
  473. uint64_t *value,
  474. unsigned size,
  475. unsigned access_size_min,
  476. unsigned access_size_max,
  477. MemTxResult (*access_fn)
  478. (MemoryRegion *mr,
  479. hwaddr addr,
  480. uint64_t *value,
  481. unsigned size,
  482. signed shift,
  483. uint64_t mask,
  484. MemTxAttrs attrs),
  485. MemoryRegion *mr,
  486. MemTxAttrs attrs)
  487. {
  488. uint64_t access_mask;
  489. unsigned access_size;
  490. unsigned i;
  491. MemTxResult r = MEMTX_OK;
  492. if (!access_size_min) {
  493. access_size_min = 1;
  494. }
  495. if (!access_size_max) {
  496. access_size_max = 4;
  497. }
  498. /* FIXME: support unaligned access? */
  499. access_size = MAX(MIN(size, access_size_max), access_size_min);
  500. access_mask = MAKE_64BIT_MASK(0, access_size * 8);
  501. if (memory_region_big_endian(mr)) {
  502. for (i = 0; i < size; i += access_size) {
  503. r |= access_fn(mr, addr + i, value, access_size,
  504. (size - access_size - i) * 8, access_mask, attrs);
  505. }
  506. } else {
  507. for (i = 0; i < size; i += access_size) {
  508. r |= access_fn(mr, addr + i, value, access_size, i * 8,
  509. access_mask, attrs);
  510. }
  511. }
  512. return r;
  513. }
  514. static AddressSpace *memory_region_to_address_space(MemoryRegion *mr)
  515. {
  516. AddressSpace *as;
  517. while (mr->container) {
  518. mr = mr->container;
  519. }
  520. QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
  521. if (mr == as->root) {
  522. return as;
  523. }
  524. }
  525. return NULL;
  526. }
  527. /* Render a memory region into the global view. Ranges in @view obscure
  528. * ranges in @mr.
  529. */
  530. static void render_memory_region(FlatView *view,
  531. MemoryRegion *mr,
  532. Int128 base,
  533. AddrRange clip,
  534. bool readonly,
  535. bool nonvolatile)
  536. {
  537. MemoryRegion *subregion;
  538. unsigned i;
  539. hwaddr offset_in_region;
  540. Int128 remain;
  541. Int128 now;
  542. FlatRange fr;
  543. AddrRange tmp;
  544. if (!mr->enabled) {
  545. return;
  546. }
  547. int128_addto(&base, int128_make64(mr->addr));
  548. readonly |= mr->readonly;
  549. nonvolatile |= mr->nonvolatile;
  550. tmp = addrrange_make(base, mr->size);
  551. if (!addrrange_intersects(tmp, clip)) {
  552. return;
  553. }
  554. clip = addrrange_intersection(tmp, clip);
  555. if (mr->alias) {
  556. int128_subfrom(&base, int128_make64(mr->alias->addr));
  557. int128_subfrom(&base, int128_make64(mr->alias_offset));
  558. render_memory_region(view, mr->alias, base, clip,
  559. readonly, nonvolatile);
  560. return;
  561. }
  562. /* Render subregions in priority order. */
  563. QTAILQ_FOREACH(subregion, &mr->subregions, subregions_link) {
  564. render_memory_region(view, subregion, base, clip,
  565. readonly, nonvolatile);
  566. }
  567. if (!mr->terminates) {
  568. return;
  569. }
  570. offset_in_region = int128_get64(int128_sub(clip.start, base));
  571. base = clip.start;
  572. remain = clip.size;
  573. fr.mr = mr;
  574. fr.dirty_log_mask = memory_region_get_dirty_log_mask(mr);
  575. fr.romd_mode = mr->romd_mode;
  576. fr.readonly = readonly;
  577. fr.nonvolatile = nonvolatile;
  578. fr.has_coalesced_range = 0;
  579. /* Render the region itself into any gaps left by the current view. */
  580. for (i = 0; i < view->nr && int128_nz(remain); ++i) {
  581. if (int128_ge(base, addrrange_end(view->ranges[i].addr))) {
  582. continue;
  583. }
  584. if (int128_lt(base, view->ranges[i].addr.start)) {
  585. now = int128_min(remain,
  586. int128_sub(view->ranges[i].addr.start, base));
  587. fr.offset_in_region = offset_in_region;
  588. fr.addr = addrrange_make(base, now);
  589. flatview_insert(view, i, &fr);
  590. ++i;
  591. int128_addto(&base, now);
  592. offset_in_region += int128_get64(now);
  593. int128_subfrom(&remain, now);
  594. }
  595. now = int128_sub(int128_min(int128_add(base, remain),
  596. addrrange_end(view->ranges[i].addr)),
  597. base);
  598. int128_addto(&base, now);
  599. offset_in_region += int128_get64(now);
  600. int128_subfrom(&remain, now);
  601. }
  602. if (int128_nz(remain)) {
  603. fr.offset_in_region = offset_in_region;
  604. fr.addr = addrrange_make(base, remain);
  605. flatview_insert(view, i, &fr);
  606. }
  607. }
  608. static MemoryRegion *memory_region_get_flatview_root(MemoryRegion *mr)
  609. {
  610. while (mr->enabled) {
  611. if (mr->alias) {
  612. if (!mr->alias_offset && int128_ge(mr->size, mr->alias->size)) {
  613. /* The alias is included in its entirety. Use it as
  614. * the "real" root, so that we can share more FlatViews.
  615. */
  616. mr = mr->alias;
  617. continue;
  618. }
  619. } else if (!mr->terminates) {
  620. unsigned int found = 0;
  621. MemoryRegion *child, *next = NULL;
  622. QTAILQ_FOREACH(child, &mr->subregions, subregions_link) {
  623. if (child->enabled) {
  624. if (++found > 1) {
  625. next = NULL;
  626. break;
  627. }
  628. if (!child->addr && int128_ge(mr->size, child->size)) {
  629. /* A child is included in its entirety. If it's the only
  630. * enabled one, use it in the hope of finding an alias down the
  631. * way. This will also let us share FlatViews.
  632. */
  633. next = child;
  634. }
  635. }
  636. }
  637. if (found == 0) {
  638. return NULL;
  639. }
  640. if (next) {
  641. mr = next;
  642. continue;
  643. }
  644. }
  645. return mr;
  646. }
  647. return NULL;
  648. }
  649. /* Render a memory topology into a list of disjoint absolute ranges. */
  650. static FlatView *generate_memory_topology(MemoryRegion *mr)
  651. {
  652. int i;
  653. FlatView *view;
  654. view = flatview_new(mr);
  655. if (mr) {
  656. render_memory_region(view, mr, int128_zero(),
  657. addrrange_make(int128_zero(), int128_2_64()),
  658. false, false);
  659. }
  660. flatview_simplify(view);
  661. view->dispatch = address_space_dispatch_new(view);
  662. for (i = 0; i < view->nr; i++) {
  663. MemoryRegionSection mrs =
  664. section_from_flat_range(&view->ranges[i], view);
  665. flatview_add_to_dispatch(view, &mrs);
  666. }
  667. address_space_dispatch_compact(view->dispatch);
  668. g_hash_table_replace(flat_views, mr, view);
  669. return view;
  670. }
  671. static void address_space_add_del_ioeventfds(AddressSpace *as,
  672. MemoryRegionIoeventfd *fds_new,
  673. unsigned fds_new_nb,
  674. MemoryRegionIoeventfd *fds_old,
  675. unsigned fds_old_nb)
  676. {
  677. unsigned iold, inew;
  678. MemoryRegionIoeventfd *fd;
  679. MemoryRegionSection section;
  680. /* Generate a symmetric difference of the old and new fd sets, adding
  681. * and deleting as necessary.
  682. */
  683. iold = inew = 0;
  684. while (iold < fds_old_nb || inew < fds_new_nb) {
  685. if (iold < fds_old_nb
  686. && (inew == fds_new_nb
  687. || memory_region_ioeventfd_before(&fds_old[iold],
  688. &fds_new[inew]))) {
  689. fd = &fds_old[iold];
  690. section = (MemoryRegionSection) {
  691. .fv = address_space_to_flatview(as),
  692. .offset_within_address_space = int128_get64(fd->addr.start),
  693. .size = fd->addr.size,
  694. };
  695. MEMORY_LISTENER_CALL(as, eventfd_del, Forward, &section,
  696. fd->match_data, fd->data, fd->e);
  697. ++iold;
  698. } else if (inew < fds_new_nb
  699. && (iold == fds_old_nb
  700. || memory_region_ioeventfd_before(&fds_new[inew],
  701. &fds_old[iold]))) {
  702. fd = &fds_new[inew];
  703. section = (MemoryRegionSection) {
  704. .fv = address_space_to_flatview(as),
  705. .offset_within_address_space = int128_get64(fd->addr.start),
  706. .size = fd->addr.size,
  707. };
  708. MEMORY_LISTENER_CALL(as, eventfd_add, Reverse, &section,
  709. fd->match_data, fd->data, fd->e);
  710. ++inew;
  711. } else {
  712. ++iold;
  713. ++inew;
  714. }
  715. }
  716. }
  717. FlatView *address_space_get_flatview(AddressSpace *as)
  718. {
  719. FlatView *view;
  720. rcu_read_lock();
  721. do {
  722. view = address_space_to_flatview(as);
  723. /* If somebody has replaced as->current_map concurrently,
  724. * flatview_ref returns false.
  725. */
  726. } while (!flatview_ref(view));
  727. rcu_read_unlock();
  728. return view;
  729. }
  730. static void address_space_update_ioeventfds(AddressSpace *as)
  731. {
  732. FlatView *view;
  733. FlatRange *fr;
  734. unsigned ioeventfd_nb = 0;
  735. MemoryRegionIoeventfd *ioeventfds = NULL;
  736. AddrRange tmp;
  737. unsigned i;
  738. view = address_space_get_flatview(as);
  739. FOR_EACH_FLAT_RANGE(fr, view) {
  740. for (i = 0; i < fr->mr->ioeventfd_nb; ++i) {
  741. tmp = addrrange_shift(fr->mr->ioeventfds[i].addr,
  742. int128_sub(fr->addr.start,
  743. int128_make64(fr->offset_in_region)));
  744. if (addrrange_intersects(fr->addr, tmp)) {
  745. ++ioeventfd_nb;
  746. ioeventfds = g_realloc(ioeventfds,
  747. ioeventfd_nb * sizeof(*ioeventfds));
  748. ioeventfds[ioeventfd_nb-1] = fr->mr->ioeventfds[i];
  749. ioeventfds[ioeventfd_nb-1].addr = tmp;
  750. }
  751. }
  752. }
  753. address_space_add_del_ioeventfds(as, ioeventfds, ioeventfd_nb,
  754. as->ioeventfds, as->ioeventfd_nb);
  755. g_free(as->ioeventfds);
  756. as->ioeventfds = ioeventfds;
  757. as->ioeventfd_nb = ioeventfd_nb;
  758. flatview_unref(view);
  759. }
  760. static void flat_range_coalesced_io_del(FlatRange *fr, AddressSpace *as)
  761. {
  762. if (!fr->has_coalesced_range) {
  763. return;
  764. }
  765. if (--fr->has_coalesced_range > 0) {
  766. return;
  767. }
  768. MEMORY_LISTENER_UPDATE_REGION(fr, as, Reverse, coalesced_io_del,
  769. int128_get64(fr->addr.start),
  770. int128_get64(fr->addr.size));
  771. }
  772. static void flat_range_coalesced_io_add(FlatRange *fr, AddressSpace *as)
  773. {
  774. MemoryRegion *mr = fr->mr;
  775. CoalescedMemoryRange *cmr;
  776. AddrRange tmp;
  777. if (QTAILQ_EMPTY(&mr->coalesced)) {
  778. return;
  779. }
  780. if (fr->has_coalesced_range++) {
  781. return;
  782. }
  783. QTAILQ_FOREACH(cmr, &mr->coalesced, link) {
  784. tmp = addrrange_shift(cmr->addr,
  785. int128_sub(fr->addr.start,
  786. int128_make64(fr->offset_in_region)));
  787. if (!addrrange_intersects(tmp, fr->addr)) {
  788. continue;
  789. }
  790. tmp = addrrange_intersection(tmp, fr->addr);
  791. MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, coalesced_io_add,
  792. int128_get64(tmp.start),
  793. int128_get64(tmp.size));
  794. }
  795. }
  796. static void address_space_update_topology_pass(AddressSpace *as,
  797. const FlatView *old_view,
  798. const FlatView *new_view,
  799. bool adding)
  800. {
  801. unsigned iold, inew;
  802. FlatRange *frold, *frnew;
  803. /* Generate a symmetric difference of the old and new memory maps.
  804. * Kill ranges in the old map, and instantiate ranges in the new map.
  805. */
  806. iold = inew = 0;
  807. while (iold < old_view->nr || inew < new_view->nr) {
  808. if (iold < old_view->nr) {
  809. frold = &old_view->ranges[iold];
  810. } else {
  811. frold = NULL;
  812. }
  813. if (inew < new_view->nr) {
  814. frnew = &new_view->ranges[inew];
  815. } else {
  816. frnew = NULL;
  817. }
  818. if (frold
  819. && (!frnew
  820. || int128_lt(frold->addr.start, frnew->addr.start)
  821. || (int128_eq(frold->addr.start, frnew->addr.start)
  822. && !flatrange_equal(frold, frnew)))) {
  823. /* In old but not in new, or in both but attributes changed. */
  824. if (!adding) {
  825. flat_range_coalesced_io_del(frold, as);
  826. MEMORY_LISTENER_UPDATE_REGION(frold, as, Reverse, region_del);
  827. }
  828. ++iold;
  829. } else if (frold && frnew && flatrange_equal(frold, frnew)) {
  830. /* In both and unchanged (except logging may have changed) */
  831. if (adding) {
  832. MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_nop);
  833. if (frnew->dirty_log_mask & ~frold->dirty_log_mask) {
  834. MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start,
  835. frold->dirty_log_mask,
  836. frnew->dirty_log_mask);
  837. }
  838. if (frold->dirty_log_mask & ~frnew->dirty_log_mask) {
  839. MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop,
  840. frold->dirty_log_mask,
  841. frnew->dirty_log_mask);
  842. }
  843. }
  844. ++iold;
  845. ++inew;
  846. } else {
  847. /* In new */
  848. if (adding) {
  849. MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_add);
  850. flat_range_coalesced_io_add(frnew, as);
  851. }
  852. ++inew;
  853. }
  854. }
  855. }
  856. static void flatviews_init(void)
  857. {
  858. static FlatView *empty_view;
  859. if (flat_views) {
  860. return;
  861. }
  862. flat_views = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL,
  863. (GDestroyNotify) flatview_unref);
  864. if (!empty_view) {
  865. empty_view = generate_memory_topology(NULL);
  866. /* We keep it alive forever in the global variable. */
  867. flatview_ref(empty_view);
  868. } else {
  869. g_hash_table_replace(flat_views, NULL, empty_view);
  870. flatview_ref(empty_view);
  871. }
  872. }
  873. static void flatviews_reset(void)
  874. {
  875. AddressSpace *as;
  876. if (flat_views) {
  877. g_hash_table_unref(flat_views);
  878. flat_views = NULL;
  879. }
  880. flatviews_init();
  881. /* Render unique FVs */
  882. QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
  883. MemoryRegion *physmr = memory_region_get_flatview_root(as->root);
  884. if (g_hash_table_lookup(flat_views, physmr)) {
  885. continue;
  886. }
  887. generate_memory_topology(physmr);
  888. }
  889. }
  890. static void address_space_set_flatview(AddressSpace *as)
  891. {
  892. FlatView *old_view = address_space_to_flatview(as);
  893. MemoryRegion *physmr = memory_region_get_flatview_root(as->root);
  894. FlatView *new_view = g_hash_table_lookup(flat_views, physmr);
  895. assert(new_view);
  896. if (old_view == new_view) {
  897. return;
  898. }
  899. if (old_view) {
  900. flatview_ref(old_view);
  901. }
  902. flatview_ref(new_view);
  903. if (!QTAILQ_EMPTY(&as->listeners)) {
  904. FlatView tmpview = { .nr = 0 }, *old_view2 = old_view;
  905. if (!old_view2) {
  906. old_view2 = &tmpview;
  907. }
  908. address_space_update_topology_pass(as, old_view2, new_view, false);
  909. address_space_update_topology_pass(as, old_view2, new_view, true);
  910. }
  911. /* Writes are protected by the BQL. */
  912. atomic_rcu_set(&as->current_map, new_view);
  913. if (old_view) {
  914. flatview_unref(old_view);
  915. }
  916. /* Note that all the old MemoryRegions are still alive up to this
  917. * point. This relieves most MemoryListeners from the need to
  918. * ref/unref the MemoryRegions they get---unless they use them
  919. * outside the iothread mutex, in which case precise reference
  920. * counting is necessary.
  921. */
  922. if (old_view) {
  923. flatview_unref(old_view);
  924. }
  925. }
  926. static void address_space_update_topology(AddressSpace *as)
  927. {
  928. MemoryRegion *physmr = memory_region_get_flatview_root(as->root);
  929. flatviews_init();
  930. if (!g_hash_table_lookup(flat_views, physmr)) {
  931. generate_memory_topology(physmr);
  932. }
  933. address_space_set_flatview(as);
  934. }
  935. void memory_region_transaction_begin(void)
  936. {
  937. qemu_flush_coalesced_mmio_buffer();
  938. ++memory_region_transaction_depth;
  939. }
  940. void memory_region_transaction_commit(void)
  941. {
  942. AddressSpace *as;
  943. assert(memory_region_transaction_depth);
  944. assert(qemu_mutex_iothread_locked());
  945. --memory_region_transaction_depth;
  946. if (!memory_region_transaction_depth) {
  947. if (memory_region_update_pending) {
  948. flatviews_reset();
  949. MEMORY_LISTENER_CALL_GLOBAL(begin, Forward);
  950. QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
  951. address_space_set_flatview(as);
  952. address_space_update_ioeventfds(as);
  953. }
  954. memory_region_update_pending = false;
  955. ioeventfd_update_pending = false;
  956. MEMORY_LISTENER_CALL_GLOBAL(commit, Forward);
  957. } else if (ioeventfd_update_pending) {
  958. QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
  959. address_space_update_ioeventfds(as);
  960. }
  961. ioeventfd_update_pending = false;
  962. }
  963. }
  964. }
  965. static void memory_region_destructor_none(MemoryRegion *mr)
  966. {
  967. }
  968. static void memory_region_destructor_ram(MemoryRegion *mr)
  969. {
  970. qemu_ram_free(mr->ram_block);
  971. }
  972. static bool memory_region_need_escape(char c)
  973. {
  974. return c == '/' || c == '[' || c == '\\' || c == ']';
  975. }
  976. static char *memory_region_escape_name(const char *name)
  977. {
  978. const char *p;
  979. char *escaped, *q;
  980. uint8_t c;
  981. size_t bytes = 0;
  982. for (p = name; *p; p++) {
  983. bytes += memory_region_need_escape(*p) ? 4 : 1;
  984. }
  985. if (bytes == p - name) {
  986. return g_memdup(name, bytes + 1);
  987. }
  988. escaped = g_malloc(bytes + 1);
  989. for (p = name, q = escaped; *p; p++) {
  990. c = *p;
  991. if (unlikely(memory_region_need_escape(c))) {
  992. *q++ = '\\';
  993. *q++ = 'x';
  994. *q++ = "0123456789abcdef"[c >> 4];
  995. c = "0123456789abcdef"[c & 15];
  996. }
  997. *q++ = c;
  998. }
  999. *q = 0;
  1000. return escaped;
  1001. }
  1002. static void memory_region_do_init(MemoryRegion *mr,
  1003. Object *owner,
  1004. const char *name,
  1005. uint64_t size)
  1006. {
  1007. mr->size = int128_make64(size);
  1008. if (size == UINT64_MAX) {
  1009. mr->size = int128_2_64();
  1010. }
  1011. mr->name = g_strdup(name);
  1012. mr->owner = owner;
  1013. mr->ram_block = NULL;
  1014. if (name) {
  1015. char *escaped_name = memory_region_escape_name(name);
  1016. char *name_array = g_strdup_printf("%s[*]", escaped_name);
  1017. if (!owner) {
  1018. owner = container_get(qdev_get_machine(), "/unattached");
  1019. }
  1020. object_property_add_child(owner, name_array, OBJECT(mr), &error_abort);
  1021. object_unref(OBJECT(mr));
  1022. g_free(name_array);
  1023. g_free(escaped_name);
  1024. }
  1025. }
  1026. void memory_region_init(MemoryRegion *mr,
  1027. Object *owner,
  1028. const char *name,
  1029. uint64_t size)
  1030. {
  1031. object_initialize(mr, sizeof(*mr), TYPE_MEMORY_REGION);
  1032. memory_region_do_init(mr, owner, name, size);
  1033. }
  1034. static void memory_region_get_addr(Object *obj, Visitor *v, const char *name,
  1035. void *opaque, Error **errp)
  1036. {
  1037. MemoryRegion *mr = MEMORY_REGION(obj);
  1038. uint64_t value = mr->addr;
  1039. visit_type_uint64(v, name, &value, errp);
  1040. }
  1041. static void memory_region_get_container(Object *obj, Visitor *v,
  1042. const char *name, void *opaque,
  1043. Error **errp)
  1044. {
  1045. MemoryRegion *mr = MEMORY_REGION(obj);
  1046. gchar *path = (gchar *)"";
  1047. if (mr->container) {
  1048. path = object_get_canonical_path(OBJECT(mr->container));
  1049. }
  1050. visit_type_str(v, name, &path, errp);
  1051. if (mr->container) {
  1052. g_free(path);
  1053. }
  1054. }
  1055. static Object *memory_region_resolve_container(Object *obj, void *opaque,
  1056. const char *part)
  1057. {
  1058. MemoryRegion *mr = MEMORY_REGION(obj);
  1059. return OBJECT(mr->container);
  1060. }
  1061. static void memory_region_get_priority(Object *obj, Visitor *v,
  1062. const char *name, void *opaque,
  1063. Error **errp)
  1064. {
  1065. MemoryRegion *mr = MEMORY_REGION(obj);
  1066. int32_t value = mr->priority;
  1067. visit_type_int32(v, name, &value, errp);
  1068. }
  1069. static void memory_region_get_size(Object *obj, Visitor *v, const char *name,
  1070. void *opaque, Error **errp)
  1071. {
  1072. MemoryRegion *mr = MEMORY_REGION(obj);
  1073. uint64_t value = memory_region_size(mr);
  1074. visit_type_uint64(v, name, &value, errp);
  1075. }
  1076. static void memory_region_initfn(Object *obj)
  1077. {
  1078. MemoryRegion *mr = MEMORY_REGION(obj);
  1079. ObjectProperty *op;
  1080. mr->ops = &unassigned_mem_ops;
  1081. mr->enabled = true;
  1082. mr->romd_mode = true;
  1083. mr->global_locking = true;
  1084. mr->destructor = memory_region_destructor_none;
  1085. QTAILQ_INIT(&mr->subregions);
  1086. QTAILQ_INIT(&mr->coalesced);
  1087. op = object_property_add(OBJECT(mr), "container",
  1088. "link<" TYPE_MEMORY_REGION ">",
  1089. memory_region_get_container,
  1090. NULL, /* memory_region_set_container */
  1091. NULL, NULL, &error_abort);
  1092. op->resolve = memory_region_resolve_container;
  1093. object_property_add(OBJECT(mr), "addr", "uint64",
  1094. memory_region_get_addr,
  1095. NULL, /* memory_region_set_addr */
  1096. NULL, NULL, &error_abort);
  1097. object_property_add(OBJECT(mr), "priority", "uint32",
  1098. memory_region_get_priority,
  1099. NULL, /* memory_region_set_priority */
  1100. NULL, NULL, &error_abort);
  1101. object_property_add(OBJECT(mr), "size", "uint64",
  1102. memory_region_get_size,
  1103. NULL, /* memory_region_set_size, */
  1104. NULL, NULL, &error_abort);
  1105. }
  1106. static void iommu_memory_region_initfn(Object *obj)
  1107. {
  1108. MemoryRegion *mr = MEMORY_REGION(obj);
  1109. mr->is_iommu = true;
  1110. }
  1111. static uint64_t unassigned_mem_read(void *opaque, hwaddr addr,
  1112. unsigned size)
  1113. {
  1114. #ifdef DEBUG_UNASSIGNED
  1115. printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
  1116. #endif
  1117. if (current_cpu != NULL) {
  1118. bool is_exec = current_cpu->mem_io_access_type == MMU_INST_FETCH;
  1119. cpu_unassigned_access(current_cpu, addr, false, is_exec, 0, size);
  1120. }
  1121. return 0;
  1122. }
  1123. static void unassigned_mem_write(void *opaque, hwaddr addr,
  1124. uint64_t val, unsigned size)
  1125. {
  1126. #ifdef DEBUG_UNASSIGNED
  1127. printf("Unassigned mem write " TARGET_FMT_plx " = 0x%"PRIx64"\n", addr, val);
  1128. #endif
  1129. if (current_cpu != NULL) {
  1130. cpu_unassigned_access(current_cpu, addr, true, false, 0, size);
  1131. }
  1132. }
  1133. static bool unassigned_mem_accepts(void *opaque, hwaddr addr,
  1134. unsigned size, bool is_write,
  1135. MemTxAttrs attrs)
  1136. {
  1137. return false;
  1138. }
  1139. const MemoryRegionOps unassigned_mem_ops = {
  1140. .valid.accepts = unassigned_mem_accepts,
  1141. .endianness = DEVICE_NATIVE_ENDIAN,
  1142. };
  1143. static uint64_t memory_region_ram_device_read(void *opaque,
  1144. hwaddr addr, unsigned size)
  1145. {
  1146. MemoryRegion *mr = opaque;
  1147. uint64_t data = (uint64_t)~0;
  1148. switch (size) {
  1149. case 1:
  1150. data = *(uint8_t *)(mr->ram_block->host + addr);
  1151. break;
  1152. case 2:
  1153. data = *(uint16_t *)(mr->ram_block->host + addr);
  1154. break;
  1155. case 4:
  1156. data = *(uint32_t *)(mr->ram_block->host + addr);
  1157. break;
  1158. case 8:
  1159. data = *(uint64_t *)(mr->ram_block->host + addr);
  1160. break;
  1161. }
  1162. trace_memory_region_ram_device_read(get_cpu_index(), mr, addr, data, size);
  1163. return data;
  1164. }
  1165. static void memory_region_ram_device_write(void *opaque, hwaddr addr,
  1166. uint64_t data, unsigned size)
  1167. {
  1168. MemoryRegion *mr = opaque;
  1169. trace_memory_region_ram_device_write(get_cpu_index(), mr, addr, data, size);
  1170. switch (size) {
  1171. case 1:
  1172. *(uint8_t *)(mr->ram_block->host + addr) = (uint8_t)data;
  1173. break;
  1174. case 2:
  1175. *(uint16_t *)(mr->ram_block->host + addr) = (uint16_t)data;
  1176. break;
  1177. case 4:
  1178. *(uint32_t *)(mr->ram_block->host + addr) = (uint32_t)data;
  1179. break;
  1180. case 8:
  1181. *(uint64_t *)(mr->ram_block->host + addr) = data;
  1182. break;
  1183. }
  1184. }
  1185. static const MemoryRegionOps ram_device_mem_ops = {
  1186. .read = memory_region_ram_device_read,
  1187. .write = memory_region_ram_device_write,
  1188. .endianness = DEVICE_HOST_ENDIAN,
  1189. .valid = {
  1190. .min_access_size = 1,
  1191. .max_access_size = 8,
  1192. .unaligned = true,
  1193. },
  1194. .impl = {
  1195. .min_access_size = 1,
  1196. .max_access_size = 8,
  1197. .unaligned = true,
  1198. },
  1199. };
  1200. bool memory_region_access_valid(MemoryRegion *mr,
  1201. hwaddr addr,
  1202. unsigned size,
  1203. bool is_write,
  1204. MemTxAttrs attrs)
  1205. {
  1206. int access_size_min, access_size_max;
  1207. int access_size, i;
  1208. if (!mr->ops->valid.unaligned && (addr & (size - 1))) {
  1209. return false;
  1210. }
  1211. if (!mr->ops->valid.accepts) {
  1212. return true;
  1213. }
  1214. access_size_min = mr->ops->valid.min_access_size;
  1215. if (!mr->ops->valid.min_access_size) {
  1216. access_size_min = 1;
  1217. }
  1218. access_size_max = mr->ops->valid.max_access_size;
  1219. if (!mr->ops->valid.max_access_size) {
  1220. access_size_max = 4;
  1221. }
  1222. access_size = MAX(MIN(size, access_size_max), access_size_min);
  1223. for (i = 0; i < size; i += access_size) {
  1224. if (!mr->ops->valid.accepts(mr->opaque, addr + i, access_size,
  1225. is_write, attrs)) {
  1226. return false;
  1227. }
  1228. }
  1229. return true;
  1230. }
  1231. static MemTxResult memory_region_dispatch_read1(MemoryRegion *mr,
  1232. hwaddr addr,
  1233. uint64_t *pval,
  1234. unsigned size,
  1235. MemTxAttrs attrs)
  1236. {
  1237. *pval = 0;
  1238. if (mr->ops->read) {
  1239. return access_with_adjusted_size(addr, pval, size,
  1240. mr->ops->impl.min_access_size,
  1241. mr->ops->impl.max_access_size,
  1242. memory_region_read_accessor,
  1243. mr, attrs);
  1244. } else {
  1245. return access_with_adjusted_size(addr, pval, size,
  1246. mr->ops->impl.min_access_size,
  1247. mr->ops->impl.max_access_size,
  1248. memory_region_read_with_attrs_accessor,
  1249. mr, attrs);
  1250. }
  1251. }
  1252. MemTxResult memory_region_dispatch_read(MemoryRegion *mr,
  1253. hwaddr addr,
  1254. uint64_t *pval,
  1255. unsigned size,
  1256. MemTxAttrs attrs)
  1257. {
  1258. MemTxResult r;
  1259. if (!memory_region_access_valid(mr, addr, size, false, attrs)) {
  1260. *pval = unassigned_mem_read(mr, addr, size);
  1261. return MEMTX_DECODE_ERROR;
  1262. }
  1263. r = memory_region_dispatch_read1(mr, addr, pval, size, attrs);
  1264. adjust_endianness(mr, pval, size);
  1265. return r;
  1266. }
  1267. /* Return true if an eventfd was signalled */
  1268. static bool memory_region_dispatch_write_eventfds(MemoryRegion *mr,
  1269. hwaddr addr,
  1270. uint64_t data,
  1271. unsigned size,
  1272. MemTxAttrs attrs)
  1273. {
  1274. MemoryRegionIoeventfd ioeventfd = {
  1275. .addr = addrrange_make(int128_make64(addr), int128_make64(size)),
  1276. .data = data,
  1277. };
  1278. unsigned i;
  1279. for (i = 0; i < mr->ioeventfd_nb; i++) {
  1280. ioeventfd.match_data = mr->ioeventfds[i].match_data;
  1281. ioeventfd.e = mr->ioeventfds[i].e;
  1282. if (memory_region_ioeventfd_equal(&ioeventfd, &mr->ioeventfds[i])) {
  1283. event_notifier_set(ioeventfd.e);
  1284. return true;
  1285. }
  1286. }
  1287. return false;
  1288. }
  1289. MemTxResult memory_region_dispatch_write(MemoryRegion *mr,
  1290. hwaddr addr,
  1291. uint64_t data,
  1292. unsigned size,
  1293. MemTxAttrs attrs)
  1294. {
  1295. if (!memory_region_access_valid(mr, addr, size, true, attrs)) {
  1296. unassigned_mem_write(mr, addr, data, size);
  1297. return MEMTX_DECODE_ERROR;
  1298. }
  1299. adjust_endianness(mr, &data, size);
  1300. if ((!kvm_eventfds_enabled()) &&
  1301. memory_region_dispatch_write_eventfds(mr, addr, data, size, attrs)) {
  1302. return MEMTX_OK;
  1303. }
  1304. if (mr->ops->write) {
  1305. return access_with_adjusted_size(addr, &data, size,
  1306. mr->ops->impl.min_access_size,
  1307. mr->ops->impl.max_access_size,
  1308. memory_region_write_accessor, mr,
  1309. attrs);
  1310. } else {
  1311. return
  1312. access_with_adjusted_size(addr, &data, size,
  1313. mr->ops->impl.min_access_size,
  1314. mr->ops->impl.max_access_size,
  1315. memory_region_write_with_attrs_accessor,
  1316. mr, attrs);
  1317. }
  1318. }
  1319. void memory_region_init_io(MemoryRegion *mr,
  1320. Object *owner,
  1321. const MemoryRegionOps *ops,
  1322. void *opaque,
  1323. const char *name,
  1324. uint64_t size)
  1325. {
  1326. memory_region_init(mr, owner, name, size);
  1327. mr->ops = ops ? ops : &unassigned_mem_ops;
  1328. mr->opaque = opaque;
  1329. mr->terminates = true;
  1330. }
  1331. void memory_region_init_ram_nomigrate(MemoryRegion *mr,
  1332. Object *owner,
  1333. const char *name,
  1334. uint64_t size,
  1335. Error **errp)
  1336. {
  1337. memory_region_init_ram_shared_nomigrate(mr, owner, name, size, false, errp);
  1338. }
  1339. void memory_region_init_ram_shared_nomigrate(MemoryRegion *mr,
  1340. Object *owner,
  1341. const char *name,
  1342. uint64_t size,
  1343. bool share,
  1344. Error **errp)
  1345. {
  1346. Error *err = NULL;
  1347. memory_region_init(mr, owner, name, size);
  1348. mr->ram = true;
  1349. mr->terminates = true;
  1350. mr->destructor = memory_region_destructor_ram;
  1351. mr->ram_block = qemu_ram_alloc(size, share, mr, &err);
  1352. mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
  1353. if (err) {
  1354. mr->size = int128_zero();
  1355. object_unparent(OBJECT(mr));
  1356. error_propagate(errp, err);
  1357. }
  1358. }
  1359. void memory_region_init_resizeable_ram(MemoryRegion *mr,
  1360. Object *owner,
  1361. const char *name,
  1362. uint64_t size,
  1363. uint64_t max_size,
  1364. void (*resized)(const char*,
  1365. uint64_t length,
  1366. void *host),
  1367. Error **errp)
  1368. {
  1369. Error *err = NULL;
  1370. memory_region_init(mr, owner, name, size);
  1371. mr->ram = true;
  1372. mr->terminates = true;
  1373. mr->destructor = memory_region_destructor_ram;
  1374. mr->ram_block = qemu_ram_alloc_resizeable(size, max_size, resized,
  1375. mr, &err);
  1376. mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
  1377. if (err) {
  1378. mr->size = int128_zero();
  1379. object_unparent(OBJECT(mr));
  1380. error_propagate(errp, err);
  1381. }
  1382. }
  1383. #ifdef CONFIG_POSIX
  1384. void memory_region_init_ram_from_file(MemoryRegion *mr,
  1385. struct Object *owner,
  1386. const char *name,
  1387. uint64_t size,
  1388. uint64_t align,
  1389. uint32_t ram_flags,
  1390. const char *path,
  1391. Error **errp)
  1392. {
  1393. Error *err = NULL;
  1394. memory_region_init(mr, owner, name, size);
  1395. mr->ram = true;
  1396. mr->terminates = true;
  1397. mr->destructor = memory_region_destructor_ram;
  1398. mr->align = align;
  1399. mr->ram_block = qemu_ram_alloc_from_file(size, mr, ram_flags, path, &err);
  1400. mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
  1401. if (err) {
  1402. mr->size = int128_zero();
  1403. object_unparent(OBJECT(mr));
  1404. error_propagate(errp, err);
  1405. }
  1406. }
  1407. void memory_region_init_ram_from_fd(MemoryRegion *mr,
  1408. struct Object *owner,
  1409. const char *name,
  1410. uint64_t size,
  1411. bool share,
  1412. int fd,
  1413. Error **errp)
  1414. {
  1415. Error *err = NULL;
  1416. memory_region_init(mr, owner, name, size);
  1417. mr->ram = true;
  1418. mr->terminates = true;
  1419. mr->destructor = memory_region_destructor_ram;
  1420. mr->ram_block = qemu_ram_alloc_from_fd(size, mr,
  1421. share ? RAM_SHARED : 0,
  1422. fd, &err);
  1423. mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
  1424. if (err) {
  1425. mr->size = int128_zero();
  1426. object_unparent(OBJECT(mr));
  1427. error_propagate(errp, err);
  1428. }
  1429. }
  1430. #endif
  1431. void memory_region_init_ram_ptr(MemoryRegion *mr,
  1432. Object *owner,
  1433. const char *name,
  1434. uint64_t size,
  1435. void *ptr)
  1436. {
  1437. memory_region_init(mr, owner, name, size);
  1438. mr->ram = true;
  1439. mr->terminates = true;
  1440. mr->destructor = memory_region_destructor_ram;
  1441. mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
  1442. /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL. */
  1443. assert(ptr != NULL);
  1444. mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_fatal);
  1445. }
  1446. void memory_region_init_ram_device_ptr(MemoryRegion *mr,
  1447. Object *owner,
  1448. const char *name,
  1449. uint64_t size,
  1450. void *ptr)
  1451. {
  1452. memory_region_init(mr, owner, name, size);
  1453. mr->ram = true;
  1454. mr->terminates = true;
  1455. mr->ram_device = true;
  1456. mr->ops = &ram_device_mem_ops;
  1457. mr->opaque = mr;
  1458. mr->destructor = memory_region_destructor_ram;
  1459. mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
  1460. /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL. */
  1461. assert(ptr != NULL);
  1462. mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_fatal);
  1463. }
  1464. void memory_region_init_alias(MemoryRegion *mr,
  1465. Object *owner,
  1466. const char *name,
  1467. MemoryRegion *orig,
  1468. hwaddr offset,
  1469. uint64_t size)
  1470. {
  1471. memory_region_init(mr, owner, name, size);
  1472. mr->alias = orig;
  1473. mr->alias_offset = offset;
  1474. }
  1475. void memory_region_init_rom_nomigrate(MemoryRegion *mr,
  1476. struct Object *owner,
  1477. const char *name,
  1478. uint64_t size,
  1479. Error **errp)
  1480. {
  1481. Error *err = NULL;
  1482. memory_region_init(mr, owner, name, size);
  1483. mr->ram = true;
  1484. mr->readonly = true;
  1485. mr->terminates = true;
  1486. mr->destructor = memory_region_destructor_ram;
  1487. mr->ram_block = qemu_ram_alloc(size, false, mr, &err);
  1488. mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
  1489. if (err) {
  1490. mr->size = int128_zero();
  1491. object_unparent(OBJECT(mr));
  1492. error_propagate(errp, err);
  1493. }
  1494. }
  1495. void memory_region_init_rom_device_nomigrate(MemoryRegion *mr,
  1496. Object *owner,
  1497. const MemoryRegionOps *ops,
  1498. void *opaque,
  1499. const char *name,
  1500. uint64_t size,
  1501. Error **errp)
  1502. {
  1503. Error *err = NULL;
  1504. assert(ops);
  1505. memory_region_init(mr, owner, name, size);
  1506. mr->ops = ops;
  1507. mr->opaque = opaque;
  1508. mr->terminates = true;
  1509. mr->rom_device = true;
  1510. mr->destructor = memory_region_destructor_ram;
  1511. mr->ram_block = qemu_ram_alloc(size, false, mr, &err);
  1512. if (err) {
  1513. mr->size = int128_zero();
  1514. object_unparent(OBJECT(mr));
  1515. error_propagate(errp, err);
  1516. }
  1517. }
  1518. void memory_region_init_iommu(void *_iommu_mr,
  1519. size_t instance_size,
  1520. const char *mrtypename,
  1521. Object *owner,
  1522. const char *name,
  1523. uint64_t size)
  1524. {
  1525. struct IOMMUMemoryRegion *iommu_mr;
  1526. struct MemoryRegion *mr;
  1527. object_initialize(_iommu_mr, instance_size, mrtypename);
  1528. mr = MEMORY_REGION(_iommu_mr);
  1529. memory_region_do_init(mr, owner, name, size);
  1530. iommu_mr = IOMMU_MEMORY_REGION(mr);
  1531. mr->terminates = true; /* then re-forwards */
  1532. QLIST_INIT(&iommu_mr->iommu_notify);
  1533. iommu_mr->iommu_notify_flags = IOMMU_NOTIFIER_NONE;
  1534. }
  1535. static void memory_region_finalize(Object *obj)
  1536. {
  1537. MemoryRegion *mr = MEMORY_REGION(obj);
  1538. assert(!mr->container);
  1539. /* We know the region is not visible in any address space (it
  1540. * does not have a container and cannot be a root either because
  1541. * it has no references, so we can blindly clear mr->enabled.
  1542. * memory_region_set_enabled instead could trigger a transaction
  1543. * and cause an infinite loop.
  1544. */
  1545. mr->enabled = false;
  1546. memory_region_transaction_begin();
  1547. while (!QTAILQ_EMPTY(&mr->subregions)) {
  1548. MemoryRegion *subregion = QTAILQ_FIRST(&mr->subregions);
  1549. memory_region_del_subregion(mr, subregion);
  1550. }
  1551. memory_region_transaction_commit();
  1552. mr->destructor(mr);
  1553. memory_region_clear_coalescing(mr);
  1554. g_free((char *)mr->name);
  1555. g_free(mr->ioeventfds);
  1556. }
  1557. Object *memory_region_owner(MemoryRegion *mr)
  1558. {
  1559. Object *obj = OBJECT(mr);
  1560. return obj->parent;
  1561. }
  1562. void memory_region_ref(MemoryRegion *mr)
  1563. {
  1564. /* MMIO callbacks most likely will access data that belongs
  1565. * to the owner, hence the need to ref/unref the owner whenever
  1566. * the memory region is in use.
  1567. *
  1568. * The memory region is a child of its owner. As long as the
  1569. * owner doesn't call unparent itself on the memory region,
  1570. * ref-ing the owner will also keep the memory region alive.
  1571. * Memory regions without an owner are supposed to never go away;
  1572. * we do not ref/unref them because it slows down DMA sensibly.
  1573. */
  1574. if (mr && mr->owner) {
  1575. object_ref(mr->owner);
  1576. }
  1577. }
  1578. void memory_region_unref(MemoryRegion *mr)
  1579. {
  1580. if (mr && mr->owner) {
  1581. object_unref(mr->owner);
  1582. }
  1583. }
  1584. uint64_t memory_region_size(MemoryRegion *mr)
  1585. {
  1586. if (int128_eq(mr->size, int128_2_64())) {
  1587. return UINT64_MAX;
  1588. }
  1589. return int128_get64(mr->size);
  1590. }
  1591. const char *memory_region_name(const MemoryRegion *mr)
  1592. {
  1593. if (!mr->name) {
  1594. ((MemoryRegion *)mr)->name =
  1595. object_get_canonical_path_component(OBJECT(mr));
  1596. }
  1597. return mr->name;
  1598. }
  1599. bool memory_region_is_ram_device(MemoryRegion *mr)
  1600. {
  1601. return mr->ram_device;
  1602. }
  1603. uint8_t memory_region_get_dirty_log_mask(MemoryRegion *mr)
  1604. {
  1605. uint8_t mask = mr->dirty_log_mask;
  1606. if (global_dirty_log && mr->ram_block) {
  1607. mask |= (1 << DIRTY_MEMORY_MIGRATION);
  1608. }
  1609. return mask;
  1610. }
  1611. bool memory_region_is_logging(MemoryRegion *mr, uint8_t client)
  1612. {
  1613. return memory_region_get_dirty_log_mask(mr) & (1 << client);
  1614. }
  1615. static void memory_region_update_iommu_notify_flags(IOMMUMemoryRegion *iommu_mr)
  1616. {
  1617. IOMMUNotifierFlag flags = IOMMU_NOTIFIER_NONE;
  1618. IOMMUNotifier *iommu_notifier;
  1619. IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr);
  1620. IOMMU_NOTIFIER_FOREACH(iommu_notifier, iommu_mr) {
  1621. flags |= iommu_notifier->notifier_flags;
  1622. }
  1623. if (flags != iommu_mr->iommu_notify_flags && imrc->notify_flag_changed) {
  1624. imrc->notify_flag_changed(iommu_mr,
  1625. iommu_mr->iommu_notify_flags,
  1626. flags);
  1627. }
  1628. iommu_mr->iommu_notify_flags = flags;
  1629. }
  1630. void memory_region_register_iommu_notifier(MemoryRegion *mr,
  1631. IOMMUNotifier *n)
  1632. {
  1633. IOMMUMemoryRegion *iommu_mr;
  1634. if (mr->alias) {
  1635. memory_region_register_iommu_notifier(mr->alias, n);
  1636. return;
  1637. }
  1638. /* We need to register for at least one bitfield */
  1639. iommu_mr = IOMMU_MEMORY_REGION(mr);
  1640. assert(n->notifier_flags != IOMMU_NOTIFIER_NONE);
  1641. assert(n->start <= n->end);
  1642. assert(n->iommu_idx >= 0 &&
  1643. n->iommu_idx < memory_region_iommu_num_indexes(iommu_mr));
  1644. QLIST_INSERT_HEAD(&iommu_mr->iommu_notify, n, node);
  1645. memory_region_update_iommu_notify_flags(iommu_mr);
  1646. }
  1647. uint64_t memory_region_iommu_get_min_page_size(IOMMUMemoryRegion *iommu_mr)
  1648. {
  1649. IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr);
  1650. if (imrc->get_min_page_size) {
  1651. return imrc->get_min_page_size(iommu_mr);
  1652. }
  1653. return TARGET_PAGE_SIZE;
  1654. }
  1655. void memory_region_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n)
  1656. {
  1657. MemoryRegion *mr = MEMORY_REGION(iommu_mr);
  1658. IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr);
  1659. hwaddr addr, granularity;
  1660. IOMMUTLBEntry iotlb;
  1661. /* If the IOMMU has its own replay callback, override */
  1662. if (imrc->replay) {
  1663. imrc->replay(iommu_mr, n);
  1664. return;
  1665. }
  1666. granularity = memory_region_iommu_get_min_page_size(iommu_mr);
  1667. for (addr = 0; addr < memory_region_size(mr); addr += granularity) {
  1668. iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, n->iommu_idx);
  1669. if (iotlb.perm != IOMMU_NONE) {
  1670. n->notify(n, &iotlb);
  1671. }
  1672. /* if (2^64 - MR size) < granularity, it's possible to get an
  1673. * infinite loop here. This should catch such a wraparound */
  1674. if ((addr + granularity) < addr) {
  1675. break;
  1676. }
  1677. }
  1678. }
  1679. void memory_region_iommu_replay_all(IOMMUMemoryRegion *iommu_mr)
  1680. {
  1681. IOMMUNotifier *notifier;
  1682. IOMMU_NOTIFIER_FOREACH(notifier, iommu_mr) {
  1683. memory_region_iommu_replay(iommu_mr, notifier);
  1684. }
  1685. }
  1686. void memory_region_unregister_iommu_notifier(MemoryRegion *mr,
  1687. IOMMUNotifier *n)
  1688. {
  1689. IOMMUMemoryRegion *iommu_mr;
  1690. if (mr->alias) {
  1691. memory_region_unregister_iommu_notifier(mr->alias, n);
  1692. return;
  1693. }
  1694. QLIST_REMOVE(n, node);
  1695. iommu_mr = IOMMU_MEMORY_REGION(mr);
  1696. memory_region_update_iommu_notify_flags(iommu_mr);
  1697. }
  1698. void memory_region_notify_one(IOMMUNotifier *notifier,
  1699. IOMMUTLBEntry *entry)
  1700. {
  1701. IOMMUNotifierFlag request_flags;
  1702. /*
  1703. * Skip the notification if the notification does not overlap
  1704. * with registered range.
  1705. */
  1706. if (notifier->start > entry->iova + entry->addr_mask ||
  1707. notifier->end < entry->iova) {
  1708. return;
  1709. }
  1710. if (entry->perm & IOMMU_RW) {
  1711. request_flags = IOMMU_NOTIFIER_MAP;
  1712. } else {
  1713. request_flags = IOMMU_NOTIFIER_UNMAP;
  1714. }
  1715. if (notifier->notifier_flags & request_flags) {
  1716. notifier->notify(notifier, entry);
  1717. }
  1718. }
  1719. void memory_region_notify_iommu(IOMMUMemoryRegion *iommu_mr,
  1720. int iommu_idx,
  1721. IOMMUTLBEntry entry)
  1722. {
  1723. IOMMUNotifier *iommu_notifier;
  1724. assert(memory_region_is_iommu(MEMORY_REGION(iommu_mr)));
  1725. IOMMU_NOTIFIER_FOREACH(iommu_notifier, iommu_mr) {
  1726. if (iommu_notifier->iommu_idx == iommu_idx) {
  1727. memory_region_notify_one(iommu_notifier, &entry);
  1728. }
  1729. }
  1730. }
  1731. int memory_region_iommu_get_attr(IOMMUMemoryRegion *iommu_mr,
  1732. enum IOMMUMemoryRegionAttr attr,
  1733. void *data)
  1734. {
  1735. IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr);
  1736. if (!imrc->get_attr) {
  1737. return -EINVAL;
  1738. }
  1739. return imrc->get_attr(iommu_mr, attr, data);
  1740. }
  1741. int memory_region_iommu_attrs_to_index(IOMMUMemoryRegion *iommu_mr,
  1742. MemTxAttrs attrs)
  1743. {
  1744. IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr);
  1745. if (!imrc->attrs_to_index) {
  1746. return 0;
  1747. }
  1748. return imrc->attrs_to_index(iommu_mr, attrs);
  1749. }
  1750. int memory_region_iommu_num_indexes(IOMMUMemoryRegion *iommu_mr)
  1751. {
  1752. IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr);
  1753. if (!imrc->num_indexes) {
  1754. return 1;
  1755. }
  1756. return imrc->num_indexes(iommu_mr);
  1757. }
  1758. void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client)
  1759. {
  1760. uint8_t mask = 1 << client;
  1761. uint8_t old_logging;
  1762. assert(client == DIRTY_MEMORY_VGA);
  1763. old_logging = mr->vga_logging_count;
  1764. mr->vga_logging_count += log ? 1 : -1;
  1765. if (!!old_logging == !!mr->vga_logging_count) {
  1766. return;
  1767. }
  1768. memory_region_transaction_begin();
  1769. mr->dirty_log_mask = (mr->dirty_log_mask & ~mask) | (log * mask);
  1770. memory_region_update_pending |= mr->enabled;
  1771. memory_region_transaction_commit();
  1772. }
  1773. void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr,
  1774. hwaddr size)
  1775. {
  1776. assert(mr->ram_block);
  1777. cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr,
  1778. size,
  1779. memory_region_get_dirty_log_mask(mr));
  1780. }
  1781. static void memory_region_sync_dirty_bitmap(MemoryRegion *mr)
  1782. {
  1783. MemoryListener *listener;
  1784. AddressSpace *as;
  1785. FlatView *view;
  1786. FlatRange *fr;
  1787. /* If the same address space has multiple log_sync listeners, we
  1788. * visit that address space's FlatView multiple times. But because
  1789. * log_sync listeners are rare, it's still cheaper than walking each
  1790. * address space once.
  1791. */
  1792. QTAILQ_FOREACH(listener, &memory_listeners, link) {
  1793. if (!listener->log_sync) {
  1794. continue;
  1795. }
  1796. as = listener->address_space;
  1797. view = address_space_get_flatview(as);
  1798. FOR_EACH_FLAT_RANGE(fr, view) {
  1799. if (fr->dirty_log_mask && (!mr || fr->mr == mr)) {
  1800. MemoryRegionSection mrs = section_from_flat_range(fr, view);
  1801. listener->log_sync(listener, &mrs);
  1802. }
  1803. }
  1804. flatview_unref(view);
  1805. }
  1806. }
  1807. DirtyBitmapSnapshot *memory_region_snapshot_and_clear_dirty(MemoryRegion *mr,
  1808. hwaddr addr,
  1809. hwaddr size,
  1810. unsigned client)
  1811. {
  1812. assert(mr->ram_block);
  1813. memory_region_sync_dirty_bitmap(mr);
  1814. return cpu_physical_memory_snapshot_and_clear_dirty(
  1815. memory_region_get_ram_addr(mr) + addr, size, client);
  1816. }
  1817. bool memory_region_snapshot_get_dirty(MemoryRegion *mr, DirtyBitmapSnapshot *snap,
  1818. hwaddr addr, hwaddr size)
  1819. {
  1820. assert(mr->ram_block);
  1821. return cpu_physical_memory_snapshot_get_dirty(snap,
  1822. memory_region_get_ram_addr(mr) + addr, size);
  1823. }
  1824. void memory_region_set_readonly(MemoryRegion *mr, bool readonly)
  1825. {
  1826. if (mr->readonly != readonly) {
  1827. memory_region_transaction_begin();
  1828. mr->readonly = readonly;
  1829. memory_region_update_pending |= mr->enabled;
  1830. memory_region_transaction_commit();
  1831. }
  1832. }
  1833. void memory_region_set_nonvolatile(MemoryRegion *mr, bool nonvolatile)
  1834. {
  1835. if (mr->nonvolatile != nonvolatile) {
  1836. memory_region_transaction_begin();
  1837. mr->nonvolatile = nonvolatile;
  1838. memory_region_update_pending |= mr->enabled;
  1839. memory_region_transaction_commit();
  1840. }
  1841. }
  1842. void memory_region_rom_device_set_romd(MemoryRegion *mr, bool romd_mode)
  1843. {
  1844. if (mr->romd_mode != romd_mode) {
  1845. memory_region_transaction_begin();
  1846. mr->romd_mode = romd_mode;
  1847. memory_region_update_pending |= mr->enabled;
  1848. memory_region_transaction_commit();
  1849. }
  1850. }
  1851. void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr,
  1852. hwaddr size, unsigned client)
  1853. {
  1854. assert(mr->ram_block);
  1855. cpu_physical_memory_test_and_clear_dirty(
  1856. memory_region_get_ram_addr(mr) + addr, size, client);
  1857. }
  1858. int memory_region_get_fd(MemoryRegion *mr)
  1859. {
  1860. int fd;
  1861. rcu_read_lock();
  1862. while (mr->alias) {
  1863. mr = mr->alias;
  1864. }
  1865. fd = mr->ram_block->fd;
  1866. rcu_read_unlock();
  1867. return fd;
  1868. }
  1869. void *memory_region_get_ram_ptr(MemoryRegion *mr)
  1870. {
  1871. void *ptr;
  1872. uint64_t offset = 0;
  1873. rcu_read_lock();
  1874. while (mr->alias) {
  1875. offset += mr->alias_offset;
  1876. mr = mr->alias;
  1877. }
  1878. assert(mr->ram_block);
  1879. ptr = qemu_map_ram_ptr(mr->ram_block, offset);
  1880. rcu_read_unlock();
  1881. return ptr;
  1882. }
  1883. MemoryRegion *memory_region_from_host(void *ptr, ram_addr_t *offset)
  1884. {
  1885. RAMBlock *block;
  1886. block = qemu_ram_block_from_host(ptr, false, offset);
  1887. if (!block) {
  1888. return NULL;
  1889. }
  1890. return block->mr;
  1891. }
  1892. ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr)
  1893. {
  1894. return mr->ram_block ? mr->ram_block->offset : RAM_ADDR_INVALID;
  1895. }
  1896. void memory_region_ram_resize(MemoryRegion *mr, ram_addr_t newsize, Error **errp)
  1897. {
  1898. assert(mr->ram_block);
  1899. qemu_ram_resize(mr->ram_block, newsize, errp);
  1900. }
  1901. static void memory_region_update_coalesced_range_as(MemoryRegion *mr, AddressSpace *as)
  1902. {
  1903. FlatView *view;
  1904. FlatRange *fr;
  1905. view = address_space_get_flatview(as);
  1906. FOR_EACH_FLAT_RANGE(fr, view) {
  1907. if (fr->mr == mr) {
  1908. flat_range_coalesced_io_del(fr, as);
  1909. flat_range_coalesced_io_add(fr, as);
  1910. }
  1911. }
  1912. flatview_unref(view);
  1913. }
  1914. static void memory_region_update_coalesced_range(MemoryRegion *mr)
  1915. {
  1916. AddressSpace *as;
  1917. QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
  1918. memory_region_update_coalesced_range_as(mr, as);
  1919. }
  1920. }
  1921. void memory_region_set_coalescing(MemoryRegion *mr)
  1922. {
  1923. memory_region_clear_coalescing(mr);
  1924. memory_region_add_coalescing(mr, 0, int128_get64(mr->size));
  1925. }
  1926. void memory_region_add_coalescing(MemoryRegion *mr,
  1927. hwaddr offset,
  1928. uint64_t size)
  1929. {
  1930. CoalescedMemoryRange *cmr = g_malloc(sizeof(*cmr));
  1931. cmr->addr = addrrange_make(int128_make64(offset), int128_make64(size));
  1932. QTAILQ_INSERT_TAIL(&mr->coalesced, cmr, link);
  1933. memory_region_update_coalesced_range(mr);
  1934. memory_region_set_flush_coalesced(mr);
  1935. }
  1936. void memory_region_clear_coalescing(MemoryRegion *mr)
  1937. {
  1938. CoalescedMemoryRange *cmr;
  1939. bool updated = false;
  1940. qemu_flush_coalesced_mmio_buffer();
  1941. mr->flush_coalesced_mmio = false;
  1942. while (!QTAILQ_EMPTY(&mr->coalesced)) {
  1943. cmr = QTAILQ_FIRST(&mr->coalesced);
  1944. QTAILQ_REMOVE(&mr->coalesced, cmr, link);
  1945. g_free(cmr);
  1946. updated = true;
  1947. }
  1948. if (updated) {
  1949. memory_region_update_coalesced_range(mr);
  1950. }
  1951. }
  1952. void memory_region_set_flush_coalesced(MemoryRegion *mr)
  1953. {
  1954. mr->flush_coalesced_mmio = true;
  1955. }
  1956. void memory_region_clear_flush_coalesced(MemoryRegion *mr)
  1957. {
  1958. qemu_flush_coalesced_mmio_buffer();
  1959. if (QTAILQ_EMPTY(&mr->coalesced)) {
  1960. mr->flush_coalesced_mmio = false;
  1961. }
  1962. }
  1963. void memory_region_clear_global_locking(MemoryRegion *mr)
  1964. {
  1965. mr->global_locking = false;
  1966. }
  1967. static bool userspace_eventfd_warning;
  1968. void memory_region_add_eventfd(MemoryRegion *mr,
  1969. hwaddr addr,
  1970. unsigned size,
  1971. bool match_data,
  1972. uint64_t data,
  1973. EventNotifier *e)
  1974. {
  1975. MemoryRegionIoeventfd mrfd = {
  1976. .addr.start = int128_make64(addr),
  1977. .addr.size = int128_make64(size),
  1978. .match_data = match_data,
  1979. .data = data,
  1980. .e = e,
  1981. };
  1982. unsigned i;
  1983. if (kvm_enabled() && (!(kvm_eventfds_enabled() ||
  1984. userspace_eventfd_warning))) {
  1985. userspace_eventfd_warning = true;
  1986. error_report("Using eventfd without MMIO binding in KVM. "
  1987. "Suboptimal performance expected");
  1988. }
  1989. if (size) {
  1990. adjust_endianness(mr, &mrfd.data, size);
  1991. }
  1992. memory_region_transaction_begin();
  1993. for (i = 0; i < mr->ioeventfd_nb; ++i) {
  1994. if (memory_region_ioeventfd_before(&mrfd, &mr->ioeventfds[i])) {
  1995. break;
  1996. }
  1997. }
  1998. ++mr->ioeventfd_nb;
  1999. mr->ioeventfds = g_realloc(mr->ioeventfds,
  2000. sizeof(*mr->ioeventfds) * mr->ioeventfd_nb);
  2001. memmove(&mr->ioeventfds[i+1], &mr->ioeventfds[i],
  2002. sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb-1 - i));
  2003. mr->ioeventfds[i] = mrfd;
  2004. ioeventfd_update_pending |= mr->enabled;
  2005. memory_region_transaction_commit();
  2006. }
  2007. void memory_region_del_eventfd(MemoryRegion *mr,
  2008. hwaddr addr,
  2009. unsigned size,
  2010. bool match_data,
  2011. uint64_t data,
  2012. EventNotifier *e)
  2013. {
  2014. MemoryRegionIoeventfd mrfd = {
  2015. .addr.start = int128_make64(addr),
  2016. .addr.size = int128_make64(size),
  2017. .match_data = match_data,
  2018. .data = data,
  2019. .e = e,
  2020. };
  2021. unsigned i;
  2022. if (size) {
  2023. adjust_endianness(mr, &mrfd.data, size);
  2024. }
  2025. memory_region_transaction_begin();
  2026. for (i = 0; i < mr->ioeventfd_nb; ++i) {
  2027. if (memory_region_ioeventfd_equal(&mrfd, &mr->ioeventfds[i])) {
  2028. break;
  2029. }
  2030. }
  2031. assert(i != mr->ioeventfd_nb);
  2032. memmove(&mr->ioeventfds[i], &mr->ioeventfds[i+1],
  2033. sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb - (i+1)));
  2034. --mr->ioeventfd_nb;
  2035. mr->ioeventfds = g_realloc(mr->ioeventfds,
  2036. sizeof(*mr->ioeventfds)*mr->ioeventfd_nb + 1);
  2037. ioeventfd_update_pending |= mr->enabled;
  2038. memory_region_transaction_commit();
  2039. }
  2040. static void memory_region_update_container_subregions(MemoryRegion *subregion)
  2041. {
  2042. MemoryRegion *mr = subregion->container;
  2043. MemoryRegion *other;
  2044. memory_region_transaction_begin();
  2045. memory_region_ref(subregion);
  2046. QTAILQ_FOREACH(other, &mr->subregions, subregions_link) {
  2047. if (subregion->priority >= other->priority) {
  2048. QTAILQ_INSERT_BEFORE(other, subregion, subregions_link);
  2049. goto done;
  2050. }
  2051. }
  2052. QTAILQ_INSERT_TAIL(&mr->subregions, subregion, subregions_link);
  2053. done:
  2054. memory_region_update_pending |= mr->enabled && subregion->enabled;
  2055. memory_region_transaction_commit();
  2056. }
  2057. static void memory_region_add_subregion_common(MemoryRegion *mr,
  2058. hwaddr offset,
  2059. MemoryRegion *subregion)
  2060. {
  2061. assert(!subregion->container);
  2062. subregion->container = mr;
  2063. subregion->addr = offset;
  2064. memory_region_update_container_subregions(subregion);
  2065. }
  2066. void memory_region_add_subregion(MemoryRegion *mr,
  2067. hwaddr offset,
  2068. MemoryRegion *subregion)
  2069. {
  2070. subregion->priority = 0;
  2071. memory_region_add_subregion_common(mr, offset, subregion);
  2072. }
  2073. void memory_region_add_subregion_overlap(MemoryRegion *mr,
  2074. hwaddr offset,
  2075. MemoryRegion *subregion,
  2076. int priority)
  2077. {
  2078. subregion->priority = priority;
  2079. memory_region_add_subregion_common(mr, offset, subregion);
  2080. }
  2081. void memory_region_del_subregion(MemoryRegion *mr,
  2082. MemoryRegion *subregion)
  2083. {
  2084. memory_region_transaction_begin();
  2085. assert(subregion->container == mr);
  2086. subregion->container = NULL;
  2087. QTAILQ_REMOVE(&mr->subregions, subregion, subregions_link);
  2088. memory_region_unref(subregion);
  2089. memory_region_update_pending |= mr->enabled && subregion->enabled;
  2090. memory_region_transaction_commit();
  2091. }
  2092. void memory_region_set_enabled(MemoryRegion *mr, bool enabled)
  2093. {
  2094. if (enabled == mr->enabled) {
  2095. return;
  2096. }
  2097. memory_region_transaction_begin();
  2098. mr->enabled = enabled;
  2099. memory_region_update_pending = true;
  2100. memory_region_transaction_commit();
  2101. }
  2102. void memory_region_set_size(MemoryRegion *mr, uint64_t size)
  2103. {
  2104. Int128 s = int128_make64(size);
  2105. if (size == UINT64_MAX) {
  2106. s = int128_2_64();
  2107. }
  2108. if (int128_eq(s, mr->size)) {
  2109. return;
  2110. }
  2111. memory_region_transaction_begin();
  2112. mr->size = s;
  2113. memory_region_update_pending = true;
  2114. memory_region_transaction_commit();
  2115. }
  2116. static void memory_region_readd_subregion(MemoryRegion *mr)
  2117. {
  2118. MemoryRegion *container = mr->container;
  2119. if (container) {
  2120. memory_region_transaction_begin();
  2121. memory_region_ref(mr);
  2122. memory_region_del_subregion(container, mr);
  2123. mr->container = container;
  2124. memory_region_update_container_subregions(mr);
  2125. memory_region_unref(mr);
  2126. memory_region_transaction_commit();
  2127. }
  2128. }
  2129. void memory_region_set_address(MemoryRegion *mr, hwaddr addr)
  2130. {
  2131. if (addr != mr->addr) {
  2132. mr->addr = addr;
  2133. memory_region_readd_subregion(mr);
  2134. }
  2135. }
  2136. void memory_region_set_alias_offset(MemoryRegion *mr, hwaddr offset)
  2137. {
  2138. assert(mr->alias);
  2139. if (offset == mr->alias_offset) {
  2140. return;
  2141. }
  2142. memory_region_transaction_begin();
  2143. mr->alias_offset = offset;
  2144. memory_region_update_pending |= mr->enabled;
  2145. memory_region_transaction_commit();
  2146. }
  2147. uint64_t memory_region_get_alignment(const MemoryRegion *mr)
  2148. {
  2149. return mr->align;
  2150. }
  2151. static int cmp_flatrange_addr(const void *addr_, const void *fr_)
  2152. {
  2153. const AddrRange *addr = addr_;
  2154. const FlatRange *fr = fr_;
  2155. if (int128_le(addrrange_end(*addr), fr->addr.start)) {
  2156. return -1;
  2157. } else if (int128_ge(addr->start, addrrange_end(fr->addr))) {
  2158. return 1;
  2159. }
  2160. return 0;
  2161. }
  2162. static FlatRange *flatview_lookup(FlatView *view, AddrRange addr)
  2163. {
  2164. return bsearch(&addr, view->ranges, view->nr,
  2165. sizeof(FlatRange), cmp_flatrange_addr);
  2166. }
  2167. bool memory_region_is_mapped(MemoryRegion *mr)
  2168. {
  2169. return mr->container ? true : false;
  2170. }
  2171. /* Same as memory_region_find, but it does not add a reference to the
  2172. * returned region. It must be called from an RCU critical section.
  2173. */
  2174. static MemoryRegionSection memory_region_find_rcu(MemoryRegion *mr,
  2175. hwaddr addr, uint64_t size)
  2176. {
  2177. MemoryRegionSection ret = { .mr = NULL };
  2178. MemoryRegion *root;
  2179. AddressSpace *as;
  2180. AddrRange range;
  2181. FlatView *view;
  2182. FlatRange *fr;
  2183. addr += mr->addr;
  2184. for (root = mr; root->container; ) {
  2185. root = root->container;
  2186. addr += root->addr;
  2187. }
  2188. as = memory_region_to_address_space(root);
  2189. if (!as) {
  2190. return ret;
  2191. }
  2192. range = addrrange_make(int128_make64(addr), int128_make64(size));
  2193. view = address_space_to_flatview(as);
  2194. fr = flatview_lookup(view, range);
  2195. if (!fr) {
  2196. return ret;
  2197. }
  2198. while (fr > view->ranges && addrrange_intersects(fr[-1].addr, range)) {
  2199. --fr;
  2200. }
  2201. ret.mr = fr->mr;
  2202. ret.fv = view;
  2203. range = addrrange_intersection(range, fr->addr);
  2204. ret.offset_within_region = fr->offset_in_region;
  2205. ret.offset_within_region += int128_get64(int128_sub(range.start,
  2206. fr->addr.start));
  2207. ret.size = range.size;
  2208. ret.offset_within_address_space = int128_get64(range.start);
  2209. ret.readonly = fr->readonly;
  2210. ret.nonvolatile = fr->nonvolatile;
  2211. return ret;
  2212. }
  2213. MemoryRegionSection memory_region_find(MemoryRegion *mr,
  2214. hwaddr addr, uint64_t size)
  2215. {
  2216. MemoryRegionSection ret;
  2217. rcu_read_lock();
  2218. ret = memory_region_find_rcu(mr, addr, size);
  2219. if (ret.mr) {
  2220. memory_region_ref(ret.mr);
  2221. }
  2222. rcu_read_unlock();
  2223. return ret;
  2224. }
  2225. bool memory_region_present(MemoryRegion *container, hwaddr addr)
  2226. {
  2227. MemoryRegion *mr;
  2228. rcu_read_lock();
  2229. mr = memory_region_find_rcu(container, addr, 1).mr;
  2230. rcu_read_unlock();
  2231. return mr && mr != container;
  2232. }
  2233. void memory_global_dirty_log_sync(void)
  2234. {
  2235. memory_region_sync_dirty_bitmap(NULL);
  2236. }
  2237. static VMChangeStateEntry *vmstate_change;
  2238. void memory_global_dirty_log_start(void)
  2239. {
  2240. if (vmstate_change) {
  2241. qemu_del_vm_change_state_handler(vmstate_change);
  2242. vmstate_change = NULL;
  2243. }
  2244. global_dirty_log = true;
  2245. MEMORY_LISTENER_CALL_GLOBAL(log_global_start, Forward);
  2246. /* Refresh DIRTY_MEMORY_MIGRATION bit. */
  2247. memory_region_transaction_begin();
  2248. memory_region_update_pending = true;
  2249. memory_region_transaction_commit();
  2250. }
  2251. static void memory_global_dirty_log_do_stop(void)
  2252. {
  2253. global_dirty_log = false;
  2254. /* Refresh DIRTY_MEMORY_MIGRATION bit. */
  2255. memory_region_transaction_begin();
  2256. memory_region_update_pending = true;
  2257. memory_region_transaction_commit();
  2258. MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse);
  2259. }
  2260. static void memory_vm_change_state_handler(void *opaque, int running,
  2261. RunState state)
  2262. {
  2263. if (running) {
  2264. memory_global_dirty_log_do_stop();
  2265. if (vmstate_change) {
  2266. qemu_del_vm_change_state_handler(vmstate_change);
  2267. vmstate_change = NULL;
  2268. }
  2269. }
  2270. }
  2271. void memory_global_dirty_log_stop(void)
  2272. {
  2273. if (!runstate_is_running()) {
  2274. if (vmstate_change) {
  2275. return;
  2276. }
  2277. vmstate_change = qemu_add_vm_change_state_handler(
  2278. memory_vm_change_state_handler, NULL);
  2279. return;
  2280. }
  2281. memory_global_dirty_log_do_stop();
  2282. }
  2283. static void listener_add_address_space(MemoryListener *listener,
  2284. AddressSpace *as)
  2285. {
  2286. FlatView *view;
  2287. FlatRange *fr;
  2288. if (listener->begin) {
  2289. listener->begin(listener);
  2290. }
  2291. if (global_dirty_log) {
  2292. if (listener->log_global_start) {
  2293. listener->log_global_start(listener);
  2294. }
  2295. }
  2296. view = address_space_get_flatview(as);
  2297. FOR_EACH_FLAT_RANGE(fr, view) {
  2298. MemoryRegionSection section = section_from_flat_range(fr, view);
  2299. if (listener->region_add) {
  2300. listener->region_add(listener, &section);
  2301. }
  2302. if (fr->dirty_log_mask && listener->log_start) {
  2303. listener->log_start(listener, &section, 0, fr->dirty_log_mask);
  2304. }
  2305. }
  2306. if (listener->commit) {
  2307. listener->commit(listener);
  2308. }
  2309. flatview_unref(view);
  2310. }
  2311. static void listener_del_address_space(MemoryListener *listener,
  2312. AddressSpace *as)
  2313. {
  2314. FlatView *view;
  2315. FlatRange *fr;
  2316. if (listener->begin) {
  2317. listener->begin(listener);
  2318. }
  2319. view = address_space_get_flatview(as);
  2320. FOR_EACH_FLAT_RANGE(fr, view) {
  2321. MemoryRegionSection section = section_from_flat_range(fr, view);
  2322. if (fr->dirty_log_mask && listener->log_stop) {
  2323. listener->log_stop(listener, &section, fr->dirty_log_mask, 0);
  2324. }
  2325. if (listener->region_del) {
  2326. listener->region_del(listener, &section);
  2327. }
  2328. }
  2329. if (listener->commit) {
  2330. listener->commit(listener);
  2331. }
  2332. flatview_unref(view);
  2333. }
  2334. void memory_listener_register(MemoryListener *listener, AddressSpace *as)
  2335. {
  2336. MemoryListener *other = NULL;
  2337. listener->address_space = as;
  2338. if (QTAILQ_EMPTY(&memory_listeners)
  2339. || listener->priority >= QTAILQ_LAST(&memory_listeners)->priority) {
  2340. QTAILQ_INSERT_TAIL(&memory_listeners, listener, link);
  2341. } else {
  2342. QTAILQ_FOREACH(other, &memory_listeners, link) {
  2343. if (listener->priority < other->priority) {
  2344. break;
  2345. }
  2346. }
  2347. QTAILQ_INSERT_BEFORE(other, listener, link);
  2348. }
  2349. if (QTAILQ_EMPTY(&as->listeners)
  2350. || listener->priority >= QTAILQ_LAST(&as->listeners)->priority) {
  2351. QTAILQ_INSERT_TAIL(&as->listeners, listener, link_as);
  2352. } else {
  2353. QTAILQ_FOREACH(other, &as->listeners, link_as) {
  2354. if (listener->priority < other->priority) {
  2355. break;
  2356. }
  2357. }
  2358. QTAILQ_INSERT_BEFORE(other, listener, link_as);
  2359. }
  2360. listener_add_address_space(listener, as);
  2361. }
  2362. void memory_listener_unregister(MemoryListener *listener)
  2363. {
  2364. if (!listener->address_space) {
  2365. return;
  2366. }
  2367. listener_del_address_space(listener, listener->address_space);
  2368. QTAILQ_REMOVE(&memory_listeners, listener, link);
  2369. QTAILQ_REMOVE(&listener->address_space->listeners, listener, link_as);
  2370. listener->address_space = NULL;
  2371. }
  2372. void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name)
  2373. {
  2374. memory_region_ref(root);
  2375. as->root = root;
  2376. as->current_map = NULL;
  2377. as->ioeventfd_nb = 0;
  2378. as->ioeventfds = NULL;
  2379. QTAILQ_INIT(&as->listeners);
  2380. QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link);
  2381. as->name = g_strdup(name ? name : "anonymous");
  2382. address_space_update_topology(as);
  2383. address_space_update_ioeventfds(as);
  2384. }
  2385. static void do_address_space_destroy(AddressSpace *as)
  2386. {
  2387. assert(QTAILQ_EMPTY(&as->listeners));
  2388. flatview_unref(as->current_map);
  2389. g_free(as->name);
  2390. g_free(as->ioeventfds);
  2391. memory_region_unref(as->root);
  2392. }
  2393. void address_space_destroy(AddressSpace *as)
  2394. {
  2395. MemoryRegion *root = as->root;
  2396. /* Flush out anything from MemoryListeners listening in on this */
  2397. memory_region_transaction_begin();
  2398. as->root = NULL;
  2399. memory_region_transaction_commit();
  2400. QTAILQ_REMOVE(&address_spaces, as, address_spaces_link);
  2401. /* At this point, as->dispatch and as->current_map are dummy
  2402. * entries that the guest should never use. Wait for the old
  2403. * values to expire before freeing the data.
  2404. */
  2405. as->root = root;
  2406. call_rcu(as, do_address_space_destroy, rcu);
  2407. }
  2408. static const char *memory_region_type(MemoryRegion *mr)
  2409. {
  2410. if (memory_region_is_ram_device(mr)) {
  2411. return "ramd";
  2412. } else if (memory_region_is_romd(mr)) {
  2413. return "romd";
  2414. } else if (memory_region_is_rom(mr)) {
  2415. return "rom";
  2416. } else if (memory_region_is_ram(mr)) {
  2417. return "ram";
  2418. } else {
  2419. return "i/o";
  2420. }
  2421. }
  2422. typedef struct MemoryRegionList MemoryRegionList;
  2423. struct MemoryRegionList {
  2424. const MemoryRegion *mr;
  2425. QTAILQ_ENTRY(MemoryRegionList) mrqueue;
  2426. };
  2427. typedef QTAILQ_HEAD(, MemoryRegionList) MemoryRegionListHead;
  2428. #define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \
  2429. int128_sub((size), int128_one())) : 0)
  2430. #define MTREE_INDENT " "
  2431. static void mtree_expand_owner(const char *label, Object *obj)
  2432. {
  2433. DeviceState *dev = (DeviceState *) object_dynamic_cast(obj, TYPE_DEVICE);
  2434. qemu_printf(" %s:{%s", label, dev ? "dev" : "obj");
  2435. if (dev && dev->id) {
  2436. qemu_printf(" id=%s", dev->id);
  2437. } else {
  2438. gchar *canonical_path = object_get_canonical_path(obj);
  2439. if (canonical_path) {
  2440. qemu_printf(" path=%s", canonical_path);
  2441. g_free(canonical_path);
  2442. } else {
  2443. qemu_printf(" type=%s", object_get_typename(obj));
  2444. }
  2445. }
  2446. qemu_printf("}");
  2447. }
  2448. static void mtree_print_mr_owner(const MemoryRegion *mr)
  2449. {
  2450. Object *owner = mr->owner;
  2451. Object *parent = memory_region_owner((MemoryRegion *)mr);
  2452. if (!owner && !parent) {
  2453. qemu_printf(" orphan");
  2454. return;
  2455. }
  2456. if (owner) {
  2457. mtree_expand_owner("owner", owner);
  2458. }
  2459. if (parent && parent != owner) {
  2460. mtree_expand_owner("parent", parent);
  2461. }
  2462. }
  2463. static void mtree_print_mr(const MemoryRegion *mr, unsigned int level,
  2464. hwaddr base,
  2465. MemoryRegionListHead *alias_print_queue,
  2466. bool owner)
  2467. {
  2468. MemoryRegionList *new_ml, *ml, *next_ml;
  2469. MemoryRegionListHead submr_print_queue;
  2470. const MemoryRegion *submr;
  2471. unsigned int i;
  2472. hwaddr cur_start, cur_end;
  2473. if (!mr) {
  2474. return;
  2475. }
  2476. for (i = 0; i < level; i++) {
  2477. qemu_printf(MTREE_INDENT);
  2478. }
  2479. cur_start = base + mr->addr;
  2480. cur_end = cur_start + MR_SIZE(mr->size);
  2481. /*
  2482. * Try to detect overflow of memory region. This should never
  2483. * happen normally. When it happens, we dump something to warn the
  2484. * user who is observing this.
  2485. */
  2486. if (cur_start < base || cur_end < cur_start) {
  2487. qemu_printf("[DETECTED OVERFLOW!] ");
  2488. }
  2489. if (mr->alias) {
  2490. MemoryRegionList *ml;
  2491. bool found = false;
  2492. /* check if the alias is already in the queue */
  2493. QTAILQ_FOREACH(ml, alias_print_queue, mrqueue) {
  2494. if (ml->mr == mr->alias) {
  2495. found = true;
  2496. }
  2497. }
  2498. if (!found) {
  2499. ml = g_new(MemoryRegionList, 1);
  2500. ml->mr = mr->alias;
  2501. QTAILQ_INSERT_TAIL(alias_print_queue, ml, mrqueue);
  2502. }
  2503. qemu_printf(TARGET_FMT_plx "-" TARGET_FMT_plx
  2504. " (prio %d, %s%s): alias %s @%s " TARGET_FMT_plx
  2505. "-" TARGET_FMT_plx "%s",
  2506. cur_start, cur_end,
  2507. mr->priority,
  2508. mr->nonvolatile ? "nv-" : "",
  2509. memory_region_type((MemoryRegion *)mr),
  2510. memory_region_name(mr),
  2511. memory_region_name(mr->alias),
  2512. mr->alias_offset,
  2513. mr->alias_offset + MR_SIZE(mr->size),
  2514. mr->enabled ? "" : " [disabled]");
  2515. if (owner) {
  2516. mtree_print_mr_owner(mr);
  2517. }
  2518. } else {
  2519. qemu_printf(TARGET_FMT_plx "-" TARGET_FMT_plx
  2520. " (prio %d, %s%s): %s%s",
  2521. cur_start, cur_end,
  2522. mr->priority,
  2523. mr->nonvolatile ? "nv-" : "",
  2524. memory_region_type((MemoryRegion *)mr),
  2525. memory_region_name(mr),
  2526. mr->enabled ? "" : " [disabled]");
  2527. if (owner) {
  2528. mtree_print_mr_owner(mr);
  2529. }
  2530. }
  2531. qemu_printf("\n");
  2532. QTAILQ_INIT(&submr_print_queue);
  2533. QTAILQ_FOREACH(submr, &mr->subregions, subregions_link) {
  2534. new_ml = g_new(MemoryRegionList, 1);
  2535. new_ml->mr = submr;
  2536. QTAILQ_FOREACH(ml, &submr_print_queue, mrqueue) {
  2537. if (new_ml->mr->addr < ml->mr->addr ||
  2538. (new_ml->mr->addr == ml->mr->addr &&
  2539. new_ml->mr->priority > ml->mr->priority)) {
  2540. QTAILQ_INSERT_BEFORE(ml, new_ml, mrqueue);
  2541. new_ml = NULL;
  2542. break;
  2543. }
  2544. }
  2545. if (new_ml) {
  2546. QTAILQ_INSERT_TAIL(&submr_print_queue, new_ml, mrqueue);
  2547. }
  2548. }
  2549. QTAILQ_FOREACH(ml, &submr_print_queue, mrqueue) {
  2550. mtree_print_mr(ml->mr, level + 1, cur_start,
  2551. alias_print_queue, owner);
  2552. }
  2553. QTAILQ_FOREACH_SAFE(ml, &submr_print_queue, mrqueue, next_ml) {
  2554. g_free(ml);
  2555. }
  2556. }
  2557. struct FlatViewInfo {
  2558. int counter;
  2559. bool dispatch_tree;
  2560. bool owner;
  2561. };
  2562. static void mtree_print_flatview(gpointer key, gpointer value,
  2563. gpointer user_data)
  2564. {
  2565. FlatView *view = key;
  2566. GArray *fv_address_spaces = value;
  2567. struct FlatViewInfo *fvi = user_data;
  2568. FlatRange *range = &view->ranges[0];
  2569. MemoryRegion *mr;
  2570. int n = view->nr;
  2571. int i;
  2572. AddressSpace *as;
  2573. qemu_printf("FlatView #%d\n", fvi->counter);
  2574. ++fvi->counter;
  2575. for (i = 0; i < fv_address_spaces->len; ++i) {
  2576. as = g_array_index(fv_address_spaces, AddressSpace*, i);
  2577. qemu_printf(" AS \"%s\", root: %s",
  2578. as->name, memory_region_name(as->root));
  2579. if (as->root->alias) {
  2580. qemu_printf(", alias %s", memory_region_name(as->root->alias));
  2581. }
  2582. qemu_printf("\n");
  2583. }
  2584. qemu_printf(" Root memory region: %s\n",
  2585. view->root ? memory_region_name(view->root) : "(none)");
  2586. if (n <= 0) {
  2587. qemu_printf(MTREE_INDENT "No rendered FlatView\n\n");
  2588. return;
  2589. }
  2590. while (n--) {
  2591. mr = range->mr;
  2592. if (range->offset_in_region) {
  2593. qemu_printf(MTREE_INDENT TARGET_FMT_plx "-" TARGET_FMT_plx
  2594. " (prio %d, %s%s): %s @" TARGET_FMT_plx,
  2595. int128_get64(range->addr.start),
  2596. int128_get64(range->addr.start)
  2597. + MR_SIZE(range->addr.size),
  2598. mr->priority,
  2599. range->nonvolatile ? "nv-" : "",
  2600. range->readonly ? "rom" : memory_region_type(mr),
  2601. memory_region_name(mr),
  2602. range->offset_in_region);
  2603. } else {
  2604. qemu_printf(MTREE_INDENT TARGET_FMT_plx "-" TARGET_FMT_plx
  2605. " (prio %d, %s%s): %s",
  2606. int128_get64(range->addr.start),
  2607. int128_get64(range->addr.start)
  2608. + MR_SIZE(range->addr.size),
  2609. mr->priority,
  2610. range->nonvolatile ? "nv-" : "",
  2611. range->readonly ? "rom" : memory_region_type(mr),
  2612. memory_region_name(mr));
  2613. }
  2614. if (fvi->owner) {
  2615. mtree_print_mr_owner(mr);
  2616. }
  2617. qemu_printf("\n");
  2618. range++;
  2619. }
  2620. #if !defined(CONFIG_USER_ONLY)
  2621. if (fvi->dispatch_tree && view->root) {
  2622. mtree_print_dispatch(view->dispatch, view->root);
  2623. }
  2624. #endif
  2625. qemu_printf("\n");
  2626. }
  2627. static gboolean mtree_info_flatview_free(gpointer key, gpointer value,
  2628. gpointer user_data)
  2629. {
  2630. FlatView *view = key;
  2631. GArray *fv_address_spaces = value;
  2632. g_array_unref(fv_address_spaces);
  2633. flatview_unref(view);
  2634. return true;
  2635. }
  2636. void mtree_info(bool flatview, bool dispatch_tree, bool owner)
  2637. {
  2638. MemoryRegionListHead ml_head;
  2639. MemoryRegionList *ml, *ml2;
  2640. AddressSpace *as;
  2641. if (flatview) {
  2642. FlatView *view;
  2643. struct FlatViewInfo fvi = {
  2644. .counter = 0,
  2645. .dispatch_tree = dispatch_tree,
  2646. .owner = owner,
  2647. };
  2648. GArray *fv_address_spaces;
  2649. GHashTable *views = g_hash_table_new(g_direct_hash, g_direct_equal);
  2650. /* Gather all FVs in one table */
  2651. QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
  2652. view = address_space_get_flatview(as);
  2653. fv_address_spaces = g_hash_table_lookup(views, view);
  2654. if (!fv_address_spaces) {
  2655. fv_address_spaces = g_array_new(false, false, sizeof(as));
  2656. g_hash_table_insert(views, view, fv_address_spaces);
  2657. }
  2658. g_array_append_val(fv_address_spaces, as);
  2659. }
  2660. /* Print */
  2661. g_hash_table_foreach(views, mtree_print_flatview, &fvi);
  2662. /* Free */
  2663. g_hash_table_foreach_remove(views, mtree_info_flatview_free, 0);
  2664. g_hash_table_unref(views);
  2665. return;
  2666. }
  2667. QTAILQ_INIT(&ml_head);
  2668. QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
  2669. qemu_printf("address-space: %s\n", as->name);
  2670. mtree_print_mr(as->root, 1, 0, &ml_head, owner);
  2671. qemu_printf("\n");
  2672. }
  2673. /* print aliased regions */
  2674. QTAILQ_FOREACH(ml, &ml_head, mrqueue) {
  2675. qemu_printf("memory-region: %s\n", memory_region_name(ml->mr));
  2676. mtree_print_mr(ml->mr, 1, 0, &ml_head, owner);
  2677. qemu_printf("\n");
  2678. }
  2679. QTAILQ_FOREACH_SAFE(ml, &ml_head, mrqueue, ml2) {
  2680. g_free(ml);
  2681. }
  2682. }
  2683. void memory_region_init_ram(MemoryRegion *mr,
  2684. struct Object *owner,
  2685. const char *name,
  2686. uint64_t size,
  2687. Error **errp)
  2688. {
  2689. DeviceState *owner_dev;
  2690. Error *err = NULL;
  2691. memory_region_init_ram_nomigrate(mr, owner, name, size, &err);
  2692. if (err) {
  2693. error_propagate(errp, err);
  2694. return;
  2695. }
  2696. /* This will assert if owner is neither NULL nor a DeviceState.
  2697. * We only want the owner here for the purposes of defining a
  2698. * unique name for migration. TODO: Ideally we should implement
  2699. * a naming scheme for Objects which are not DeviceStates, in
  2700. * which case we can relax this restriction.
  2701. */
  2702. owner_dev = DEVICE(owner);
  2703. vmstate_register_ram(mr, owner_dev);
  2704. }
  2705. void memory_region_init_rom(MemoryRegion *mr,
  2706. struct Object *owner,
  2707. const char *name,
  2708. uint64_t size,
  2709. Error **errp)
  2710. {
  2711. DeviceState *owner_dev;
  2712. Error *err = NULL;
  2713. memory_region_init_rom_nomigrate(mr, owner, name, size, &err);
  2714. if (err) {
  2715. error_propagate(errp, err);
  2716. return;
  2717. }
  2718. /* This will assert if owner is neither NULL nor a DeviceState.
  2719. * We only want the owner here for the purposes of defining a
  2720. * unique name for migration. TODO: Ideally we should implement
  2721. * a naming scheme for Objects which are not DeviceStates, in
  2722. * which case we can relax this restriction.
  2723. */
  2724. owner_dev = DEVICE(owner);
  2725. vmstate_register_ram(mr, owner_dev);
  2726. }
  2727. void memory_region_init_rom_device(MemoryRegion *mr,
  2728. struct Object *owner,
  2729. const MemoryRegionOps *ops,
  2730. void *opaque,
  2731. const char *name,
  2732. uint64_t size,
  2733. Error **errp)
  2734. {
  2735. DeviceState *owner_dev;
  2736. Error *err = NULL;
  2737. memory_region_init_rom_device_nomigrate(mr, owner, ops, opaque,
  2738. name, size, &err);
  2739. if (err) {
  2740. error_propagate(errp, err);
  2741. return;
  2742. }
  2743. /* This will assert if owner is neither NULL nor a DeviceState.
  2744. * We only want the owner here for the purposes of defining a
  2745. * unique name for migration. TODO: Ideally we should implement
  2746. * a naming scheme for Objects which are not DeviceStates, in
  2747. * which case we can relax this restriction.
  2748. */
  2749. owner_dev = DEVICE(owner);
  2750. vmstate_register_ram(mr, owner_dev);
  2751. }
  2752. static const TypeInfo memory_region_info = {
  2753. .parent = TYPE_OBJECT,
  2754. .name = TYPE_MEMORY_REGION,
  2755. .instance_size = sizeof(MemoryRegion),
  2756. .instance_init = memory_region_initfn,
  2757. .instance_finalize = memory_region_finalize,
  2758. };
  2759. static const TypeInfo iommu_memory_region_info = {
  2760. .parent = TYPE_MEMORY_REGION,
  2761. .name = TYPE_IOMMU_MEMORY_REGION,
  2762. .class_size = sizeof(IOMMUMemoryRegionClass),
  2763. .instance_size = sizeof(IOMMUMemoryRegion),
  2764. .instance_init = iommu_memory_region_initfn,
  2765. .abstract = true,
  2766. };
  2767. static void memory_register_types(void)
  2768. {
  2769. type_register_static(&memory_region_info);
  2770. type_register_static(&iommu_memory_region_info);
  2771. }
  2772. type_init(memory_register_types)