You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

exec.c 124KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256
  1. /*
  2. * Virtual page mapping
  3. *
  4. * Copyright (c) 2003 Fabrice Bellard
  5. *
  6. * This library is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU Lesser General Public
  8. * License as published by the Free Software Foundation; either
  9. * version 2 of the License, or (at your option) any later version.
  10. *
  11. * This library is distributed in the hope that it will be useful,
  12. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. * Lesser General Public License for more details.
  15. *
  16. * You should have received a copy of the GNU Lesser General Public
  17. * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18. */
  19. #include "qemu/osdep.h"
  20. #include "qemu-common.h"
  21. #include "qapi/error.h"
  22. #include "qemu/cutils.h"
  23. #include "cpu.h"
  24. #include "exec/exec-all.h"
  25. #include "exec/target_page.h"
  26. #include "tcg.h"
  27. #include "hw/qdev-core.h"
  28. #include "hw/qdev-properties.h"
  29. #if !defined(CONFIG_USER_ONLY)
  30. #include "hw/boards.h"
  31. #include "hw/xen/xen.h"
  32. #endif
  33. #include "sysemu/kvm.h"
  34. #include "sysemu/sysemu.h"
  35. #include "sysemu/tcg.h"
  36. #include "qemu/timer.h"
  37. #include "qemu/config-file.h"
  38. #include "qemu/error-report.h"
  39. #include "qemu/qemu-print.h"
  40. #if defined(CONFIG_USER_ONLY)
  41. #include "qemu.h"
  42. #else /* !CONFIG_USER_ONLY */
  43. #include "exec/memory.h"
  44. #include "exec/ioport.h"
  45. #include "sysemu/dma.h"
  46. #include "sysemu/hostmem.h"
  47. #include "sysemu/hw_accel.h"
  48. #include "exec/address-spaces.h"
  49. #include "sysemu/xen-mapcache.h"
  50. #include "trace-root.h"
  51. #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
  52. #include <linux/falloc.h>
  53. #endif
  54. #endif
  55. #include "qemu/rcu_queue.h"
  56. #include "qemu/main-loop.h"
  57. #include "translate-all.h"
  58. #include "sysemu/replay.h"
  59. #include "exec/memory-internal.h"
  60. #include "exec/ram_addr.h"
  61. #include "exec/log.h"
  62. #include "migration/vmstate.h"
  63. #include "qemu/range.h"
  64. #ifndef _WIN32
  65. #include "qemu/mmap-alloc.h"
  66. #endif
  67. #include "monitor/monitor.h"
  68. //#define DEBUG_SUBPAGE
  69. #if !defined(CONFIG_USER_ONLY)
  70. /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
  71. * are protected by the ramlist lock.
  72. */
  73. RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
  74. static MemoryRegion *system_memory;
  75. static MemoryRegion *system_io;
  76. AddressSpace address_space_io;
  77. AddressSpace address_space_memory;
  78. MemoryRegion io_mem_rom, io_mem_notdirty;
  79. static MemoryRegion io_mem_unassigned;
  80. #endif
  81. #ifdef TARGET_PAGE_BITS_VARY
  82. int target_page_bits;
  83. bool target_page_bits_decided;
  84. #endif
  85. CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
  86. /* current CPU in the current thread. It is only valid inside
  87. cpu_exec() */
  88. __thread CPUState *current_cpu;
  89. /* 0 = Do not count executed instructions.
  90. 1 = Precise instruction counting.
  91. 2 = Adaptive rate instruction counting. */
  92. int use_icount;
  93. uintptr_t qemu_host_page_size;
  94. intptr_t qemu_host_page_mask;
  95. bool set_preferred_target_page_bits(int bits)
  96. {
  97. /* The target page size is the lowest common denominator for all
  98. * the CPUs in the system, so we can only make it smaller, never
  99. * larger. And we can't make it smaller once we've committed to
  100. * a particular size.
  101. */
  102. #ifdef TARGET_PAGE_BITS_VARY
  103. assert(bits >= TARGET_PAGE_BITS_MIN);
  104. if (target_page_bits == 0 || target_page_bits > bits) {
  105. if (target_page_bits_decided) {
  106. return false;
  107. }
  108. target_page_bits = bits;
  109. }
  110. #endif
  111. return true;
  112. }
  113. #if !defined(CONFIG_USER_ONLY)
  114. static void finalize_target_page_bits(void)
  115. {
  116. #ifdef TARGET_PAGE_BITS_VARY
  117. if (target_page_bits == 0) {
  118. target_page_bits = TARGET_PAGE_BITS_MIN;
  119. }
  120. target_page_bits_decided = true;
  121. #endif
  122. }
  123. typedef struct PhysPageEntry PhysPageEntry;
  124. struct PhysPageEntry {
  125. /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
  126. uint32_t skip : 6;
  127. /* index into phys_sections (!skip) or phys_map_nodes (skip) */
  128. uint32_t ptr : 26;
  129. };
  130. #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
  131. /* Size of the L2 (and L3, etc) page tables. */
  132. #define ADDR_SPACE_BITS 64
  133. #define P_L2_BITS 9
  134. #define P_L2_SIZE (1 << P_L2_BITS)
  135. #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
  136. typedef PhysPageEntry Node[P_L2_SIZE];
  137. typedef struct PhysPageMap {
  138. struct rcu_head rcu;
  139. unsigned sections_nb;
  140. unsigned sections_nb_alloc;
  141. unsigned nodes_nb;
  142. unsigned nodes_nb_alloc;
  143. Node *nodes;
  144. MemoryRegionSection *sections;
  145. } PhysPageMap;
  146. struct AddressSpaceDispatch {
  147. MemoryRegionSection *mru_section;
  148. /* This is a multi-level map on the physical address space.
  149. * The bottom level has pointers to MemoryRegionSections.
  150. */
  151. PhysPageEntry phys_map;
  152. PhysPageMap map;
  153. };
  154. #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
  155. typedef struct subpage_t {
  156. MemoryRegion iomem;
  157. FlatView *fv;
  158. hwaddr base;
  159. uint16_t sub_section[];
  160. } subpage_t;
  161. #define PHYS_SECTION_UNASSIGNED 0
  162. #define PHYS_SECTION_NOTDIRTY 1
  163. #define PHYS_SECTION_ROM 2
  164. #define PHYS_SECTION_WATCH 3
  165. static void io_mem_init(void);
  166. static void memory_map_init(void);
  167. static void tcg_commit(MemoryListener *listener);
  168. static MemoryRegion io_mem_watch;
  169. /**
  170. * CPUAddressSpace: all the information a CPU needs about an AddressSpace
  171. * @cpu: the CPU whose AddressSpace this is
  172. * @as: the AddressSpace itself
  173. * @memory_dispatch: its dispatch pointer (cached, RCU protected)
  174. * @tcg_as_listener: listener for tracking changes to the AddressSpace
  175. */
  176. struct CPUAddressSpace {
  177. CPUState *cpu;
  178. AddressSpace *as;
  179. struct AddressSpaceDispatch *memory_dispatch;
  180. MemoryListener tcg_as_listener;
  181. };
  182. struct DirtyBitmapSnapshot {
  183. ram_addr_t start;
  184. ram_addr_t end;
  185. unsigned long dirty[];
  186. };
  187. #endif
  188. #if !defined(CONFIG_USER_ONLY)
  189. static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
  190. {
  191. static unsigned alloc_hint = 16;
  192. if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
  193. map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, alloc_hint);
  194. map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
  195. map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
  196. alloc_hint = map->nodes_nb_alloc;
  197. }
  198. }
  199. static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
  200. {
  201. unsigned i;
  202. uint32_t ret;
  203. PhysPageEntry e;
  204. PhysPageEntry *p;
  205. ret = map->nodes_nb++;
  206. p = map->nodes[ret];
  207. assert(ret != PHYS_MAP_NODE_NIL);
  208. assert(ret != map->nodes_nb_alloc);
  209. e.skip = leaf ? 0 : 1;
  210. e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
  211. for (i = 0; i < P_L2_SIZE; ++i) {
  212. memcpy(&p[i], &e, sizeof(e));
  213. }
  214. return ret;
  215. }
  216. static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
  217. hwaddr *index, hwaddr *nb, uint16_t leaf,
  218. int level)
  219. {
  220. PhysPageEntry *p;
  221. hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
  222. if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
  223. lp->ptr = phys_map_node_alloc(map, level == 0);
  224. }
  225. p = map->nodes[lp->ptr];
  226. lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
  227. while (*nb && lp < &p[P_L2_SIZE]) {
  228. if ((*index & (step - 1)) == 0 && *nb >= step) {
  229. lp->skip = 0;
  230. lp->ptr = leaf;
  231. *index += step;
  232. *nb -= step;
  233. } else {
  234. phys_page_set_level(map, lp, index, nb, leaf, level - 1);
  235. }
  236. ++lp;
  237. }
  238. }
  239. static void phys_page_set(AddressSpaceDispatch *d,
  240. hwaddr index, hwaddr nb,
  241. uint16_t leaf)
  242. {
  243. /* Wildly overreserve - it doesn't matter much. */
  244. phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
  245. phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
  246. }
  247. /* Compact a non leaf page entry. Simply detect that the entry has a single child,
  248. * and update our entry so we can skip it and go directly to the destination.
  249. */
  250. static void phys_page_compact(PhysPageEntry *lp, Node *nodes)
  251. {
  252. unsigned valid_ptr = P_L2_SIZE;
  253. int valid = 0;
  254. PhysPageEntry *p;
  255. int i;
  256. if (lp->ptr == PHYS_MAP_NODE_NIL) {
  257. return;
  258. }
  259. p = nodes[lp->ptr];
  260. for (i = 0; i < P_L2_SIZE; i++) {
  261. if (p[i].ptr == PHYS_MAP_NODE_NIL) {
  262. continue;
  263. }
  264. valid_ptr = i;
  265. valid++;
  266. if (p[i].skip) {
  267. phys_page_compact(&p[i], nodes);
  268. }
  269. }
  270. /* We can only compress if there's only one child. */
  271. if (valid != 1) {
  272. return;
  273. }
  274. assert(valid_ptr < P_L2_SIZE);
  275. /* Don't compress if it won't fit in the # of bits we have. */
  276. if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
  277. return;
  278. }
  279. lp->ptr = p[valid_ptr].ptr;
  280. if (!p[valid_ptr].skip) {
  281. /* If our only child is a leaf, make this a leaf. */
  282. /* By design, we should have made this node a leaf to begin with so we
  283. * should never reach here.
  284. * But since it's so simple to handle this, let's do it just in case we
  285. * change this rule.
  286. */
  287. lp->skip = 0;
  288. } else {
  289. lp->skip += p[valid_ptr].skip;
  290. }
  291. }
  292. void address_space_dispatch_compact(AddressSpaceDispatch *d)
  293. {
  294. if (d->phys_map.skip) {
  295. phys_page_compact(&d->phys_map, d->map.nodes);
  296. }
  297. }
  298. static inline bool section_covers_addr(const MemoryRegionSection *section,
  299. hwaddr addr)
  300. {
  301. /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
  302. * the section must cover the entire address space.
  303. */
  304. return int128_gethi(section->size) ||
  305. range_covers_byte(section->offset_within_address_space,
  306. int128_getlo(section->size), addr);
  307. }
  308. static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr addr)
  309. {
  310. PhysPageEntry lp = d->phys_map, *p;
  311. Node *nodes = d->map.nodes;
  312. MemoryRegionSection *sections = d->map.sections;
  313. hwaddr index = addr >> TARGET_PAGE_BITS;
  314. int i;
  315. for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
  316. if (lp.ptr == PHYS_MAP_NODE_NIL) {
  317. return &sections[PHYS_SECTION_UNASSIGNED];
  318. }
  319. p = nodes[lp.ptr];
  320. lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
  321. }
  322. if (section_covers_addr(&sections[lp.ptr], addr)) {
  323. return &sections[lp.ptr];
  324. } else {
  325. return &sections[PHYS_SECTION_UNASSIGNED];
  326. }
  327. }
  328. /* Called from RCU critical section */
  329. static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
  330. hwaddr addr,
  331. bool resolve_subpage)
  332. {
  333. MemoryRegionSection *section = atomic_read(&d->mru_section);
  334. subpage_t *subpage;
  335. if (!section || section == &d->map.sections[PHYS_SECTION_UNASSIGNED] ||
  336. !section_covers_addr(section, addr)) {
  337. section = phys_page_find(d, addr);
  338. atomic_set(&d->mru_section, section);
  339. }
  340. if (resolve_subpage && section->mr->subpage) {
  341. subpage = container_of(section->mr, subpage_t, iomem);
  342. section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
  343. }
  344. return section;
  345. }
  346. /* Called from RCU critical section */
  347. static MemoryRegionSection *
  348. address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
  349. hwaddr *plen, bool resolve_subpage)
  350. {
  351. MemoryRegionSection *section;
  352. MemoryRegion *mr;
  353. Int128 diff;
  354. section = address_space_lookup_region(d, addr, resolve_subpage);
  355. /* Compute offset within MemoryRegionSection */
  356. addr -= section->offset_within_address_space;
  357. /* Compute offset within MemoryRegion */
  358. *xlat = addr + section->offset_within_region;
  359. mr = section->mr;
  360. /* MMIO registers can be expected to perform full-width accesses based only
  361. * on their address, without considering adjacent registers that could
  362. * decode to completely different MemoryRegions. When such registers
  363. * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
  364. * regions overlap wildly. For this reason we cannot clamp the accesses
  365. * here.
  366. *
  367. * If the length is small (as is the case for address_space_ldl/stl),
  368. * everything works fine. If the incoming length is large, however,
  369. * the caller really has to do the clamping through memory_access_size.
  370. */
  371. if (memory_region_is_ram(mr)) {
  372. diff = int128_sub(section->size, int128_make64(addr));
  373. *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
  374. }
  375. return section;
  376. }
  377. /**
  378. * address_space_translate_iommu - translate an address through an IOMMU
  379. * memory region and then through the target address space.
  380. *
  381. * @iommu_mr: the IOMMU memory region that we start the translation from
  382. * @addr: the address to be translated through the MMU
  383. * @xlat: the translated address offset within the destination memory region.
  384. * It cannot be %NULL.
  385. * @plen_out: valid read/write length of the translated address. It
  386. * cannot be %NULL.
  387. * @page_mask_out: page mask for the translated address. This
  388. * should only be meaningful for IOMMU translated
  389. * addresses, since there may be huge pages that this bit
  390. * would tell. It can be %NULL if we don't care about it.
  391. * @is_write: whether the translation operation is for write
  392. * @is_mmio: whether this can be MMIO, set true if it can
  393. * @target_as: the address space targeted by the IOMMU
  394. * @attrs: transaction attributes
  395. *
  396. * This function is called from RCU critical section. It is the common
  397. * part of flatview_do_translate and address_space_translate_cached.
  398. */
  399. static MemoryRegionSection address_space_translate_iommu(IOMMUMemoryRegion *iommu_mr,
  400. hwaddr *xlat,
  401. hwaddr *plen_out,
  402. hwaddr *page_mask_out,
  403. bool is_write,
  404. bool is_mmio,
  405. AddressSpace **target_as,
  406. MemTxAttrs attrs)
  407. {
  408. MemoryRegionSection *section;
  409. hwaddr page_mask = (hwaddr)-1;
  410. do {
  411. hwaddr addr = *xlat;
  412. IOMMUMemoryRegionClass *imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
  413. int iommu_idx = 0;
  414. IOMMUTLBEntry iotlb;
  415. if (imrc->attrs_to_index) {
  416. iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
  417. }
  418. iotlb = imrc->translate(iommu_mr, addr, is_write ?
  419. IOMMU_WO : IOMMU_RO, iommu_idx);
  420. if (!(iotlb.perm & (1 << is_write))) {
  421. goto unassigned;
  422. }
  423. addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
  424. | (addr & iotlb.addr_mask));
  425. page_mask &= iotlb.addr_mask;
  426. *plen_out = MIN(*plen_out, (addr | iotlb.addr_mask) - addr + 1);
  427. *target_as = iotlb.target_as;
  428. section = address_space_translate_internal(
  429. address_space_to_dispatch(iotlb.target_as), addr, xlat,
  430. plen_out, is_mmio);
  431. iommu_mr = memory_region_get_iommu(section->mr);
  432. } while (unlikely(iommu_mr));
  433. if (page_mask_out) {
  434. *page_mask_out = page_mask;
  435. }
  436. return *section;
  437. unassigned:
  438. return (MemoryRegionSection) { .mr = &io_mem_unassigned };
  439. }
  440. /**
  441. * flatview_do_translate - translate an address in FlatView
  442. *
  443. * @fv: the flat view that we want to translate on
  444. * @addr: the address to be translated in above address space
  445. * @xlat: the translated address offset within memory region. It
  446. * cannot be @NULL.
  447. * @plen_out: valid read/write length of the translated address. It
  448. * can be @NULL when we don't care about it.
  449. * @page_mask_out: page mask for the translated address. This
  450. * should only be meaningful for IOMMU translated
  451. * addresses, since there may be huge pages that this bit
  452. * would tell. It can be @NULL if we don't care about it.
  453. * @is_write: whether the translation operation is for write
  454. * @is_mmio: whether this can be MMIO, set true if it can
  455. * @target_as: the address space targeted by the IOMMU
  456. * @attrs: memory transaction attributes
  457. *
  458. * This function is called from RCU critical section
  459. */
  460. static MemoryRegionSection flatview_do_translate(FlatView *fv,
  461. hwaddr addr,
  462. hwaddr *xlat,
  463. hwaddr *plen_out,
  464. hwaddr *page_mask_out,
  465. bool is_write,
  466. bool is_mmio,
  467. AddressSpace **target_as,
  468. MemTxAttrs attrs)
  469. {
  470. MemoryRegionSection *section;
  471. IOMMUMemoryRegion *iommu_mr;
  472. hwaddr plen = (hwaddr)(-1);
  473. if (!plen_out) {
  474. plen_out = &plen;
  475. }
  476. section = address_space_translate_internal(
  477. flatview_to_dispatch(fv), addr, xlat,
  478. plen_out, is_mmio);
  479. iommu_mr = memory_region_get_iommu(section->mr);
  480. if (unlikely(iommu_mr)) {
  481. return address_space_translate_iommu(iommu_mr, xlat,
  482. plen_out, page_mask_out,
  483. is_write, is_mmio,
  484. target_as, attrs);
  485. }
  486. if (page_mask_out) {
  487. /* Not behind an IOMMU, use default page size. */
  488. *page_mask_out = ~TARGET_PAGE_MASK;
  489. }
  490. return *section;
  491. }
  492. /* Called from RCU critical section */
  493. IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr,
  494. bool is_write, MemTxAttrs attrs)
  495. {
  496. MemoryRegionSection section;
  497. hwaddr xlat, page_mask;
  498. /*
  499. * This can never be MMIO, and we don't really care about plen,
  500. * but page mask.
  501. */
  502. section = flatview_do_translate(address_space_to_flatview(as), addr, &xlat,
  503. NULL, &page_mask, is_write, false, &as,
  504. attrs);
  505. /* Illegal translation */
  506. if (section.mr == &io_mem_unassigned) {
  507. goto iotlb_fail;
  508. }
  509. /* Convert memory region offset into address space offset */
  510. xlat += section.offset_within_address_space -
  511. section.offset_within_region;
  512. return (IOMMUTLBEntry) {
  513. .target_as = as,
  514. .iova = addr & ~page_mask,
  515. .translated_addr = xlat & ~page_mask,
  516. .addr_mask = page_mask,
  517. /* IOTLBs are for DMAs, and DMA only allows on RAMs. */
  518. .perm = IOMMU_RW,
  519. };
  520. iotlb_fail:
  521. return (IOMMUTLBEntry) {0};
  522. }
  523. /* Called from RCU critical section */
  524. MemoryRegion *flatview_translate(FlatView *fv, hwaddr addr, hwaddr *xlat,
  525. hwaddr *plen, bool is_write,
  526. MemTxAttrs attrs)
  527. {
  528. MemoryRegion *mr;
  529. MemoryRegionSection section;
  530. AddressSpace *as = NULL;
  531. /* This can be MMIO, so setup MMIO bit. */
  532. section = flatview_do_translate(fv, addr, xlat, plen, NULL,
  533. is_write, true, &as, attrs);
  534. mr = section.mr;
  535. if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
  536. hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
  537. *plen = MIN(page, *plen);
  538. }
  539. return mr;
  540. }
  541. typedef struct TCGIOMMUNotifier {
  542. IOMMUNotifier n;
  543. MemoryRegion *mr;
  544. CPUState *cpu;
  545. int iommu_idx;
  546. bool active;
  547. } TCGIOMMUNotifier;
  548. static void tcg_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
  549. {
  550. TCGIOMMUNotifier *notifier = container_of(n, TCGIOMMUNotifier, n);
  551. if (!notifier->active) {
  552. return;
  553. }
  554. tlb_flush(notifier->cpu);
  555. notifier->active = false;
  556. /* We leave the notifier struct on the list to avoid reallocating it later.
  557. * Generally the number of IOMMUs a CPU deals with will be small.
  558. * In any case we can't unregister the iommu notifier from a notify
  559. * callback.
  560. */
  561. }
  562. static void tcg_register_iommu_notifier(CPUState *cpu,
  563. IOMMUMemoryRegion *iommu_mr,
  564. int iommu_idx)
  565. {
  566. /* Make sure this CPU has an IOMMU notifier registered for this
  567. * IOMMU/IOMMU index combination, so that we can flush its TLB
  568. * when the IOMMU tells us the mappings we've cached have changed.
  569. */
  570. MemoryRegion *mr = MEMORY_REGION(iommu_mr);
  571. TCGIOMMUNotifier *notifier;
  572. int i;
  573. for (i = 0; i < cpu->iommu_notifiers->len; i++) {
  574. notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
  575. if (notifier->mr == mr && notifier->iommu_idx == iommu_idx) {
  576. break;
  577. }
  578. }
  579. if (i == cpu->iommu_notifiers->len) {
  580. /* Not found, add a new entry at the end of the array */
  581. cpu->iommu_notifiers = g_array_set_size(cpu->iommu_notifiers, i + 1);
  582. notifier = g_new0(TCGIOMMUNotifier, 1);
  583. g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i) = notifier;
  584. notifier->mr = mr;
  585. notifier->iommu_idx = iommu_idx;
  586. notifier->cpu = cpu;
  587. /* Rather than trying to register interest in the specific part
  588. * of the iommu's address space that we've accessed and then
  589. * expand it later as subsequent accesses touch more of it, we
  590. * just register interest in the whole thing, on the assumption
  591. * that iommu reconfiguration will be rare.
  592. */
  593. iommu_notifier_init(&notifier->n,
  594. tcg_iommu_unmap_notify,
  595. IOMMU_NOTIFIER_UNMAP,
  596. 0,
  597. HWADDR_MAX,
  598. iommu_idx);
  599. memory_region_register_iommu_notifier(notifier->mr, &notifier->n);
  600. }
  601. if (!notifier->active) {
  602. notifier->active = true;
  603. }
  604. }
  605. static void tcg_iommu_free_notifier_list(CPUState *cpu)
  606. {
  607. /* Destroy the CPU's notifier list */
  608. int i;
  609. TCGIOMMUNotifier *notifier;
  610. for (i = 0; i < cpu->iommu_notifiers->len; i++) {
  611. notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
  612. memory_region_unregister_iommu_notifier(notifier->mr, &notifier->n);
  613. g_free(notifier);
  614. }
  615. g_array_free(cpu->iommu_notifiers, true);
  616. }
  617. /* Called from RCU critical section */
  618. MemoryRegionSection *
  619. address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
  620. hwaddr *xlat, hwaddr *plen,
  621. MemTxAttrs attrs, int *prot)
  622. {
  623. MemoryRegionSection *section;
  624. IOMMUMemoryRegion *iommu_mr;
  625. IOMMUMemoryRegionClass *imrc;
  626. IOMMUTLBEntry iotlb;
  627. int iommu_idx;
  628. AddressSpaceDispatch *d = atomic_rcu_read(&cpu->cpu_ases[asidx].memory_dispatch);
  629. for (;;) {
  630. section = address_space_translate_internal(d, addr, &addr, plen, false);
  631. iommu_mr = memory_region_get_iommu(section->mr);
  632. if (!iommu_mr) {
  633. break;
  634. }
  635. imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
  636. iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
  637. tcg_register_iommu_notifier(cpu, iommu_mr, iommu_idx);
  638. /* We need all the permissions, so pass IOMMU_NONE so the IOMMU
  639. * doesn't short-cut its translation table walk.
  640. */
  641. iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, iommu_idx);
  642. addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
  643. | (addr & iotlb.addr_mask));
  644. /* Update the caller's prot bits to remove permissions the IOMMU
  645. * is giving us a failure response for. If we get down to no
  646. * permissions left at all we can give up now.
  647. */
  648. if (!(iotlb.perm & IOMMU_RO)) {
  649. *prot &= ~(PAGE_READ | PAGE_EXEC);
  650. }
  651. if (!(iotlb.perm & IOMMU_WO)) {
  652. *prot &= ~PAGE_WRITE;
  653. }
  654. if (!*prot) {
  655. goto translate_fail;
  656. }
  657. d = flatview_to_dispatch(address_space_to_flatview(iotlb.target_as));
  658. }
  659. assert(!memory_region_is_iommu(section->mr));
  660. *xlat = addr;
  661. return section;
  662. translate_fail:
  663. return &d->map.sections[PHYS_SECTION_UNASSIGNED];
  664. }
  665. #endif
  666. #if !defined(CONFIG_USER_ONLY)
  667. static int cpu_common_post_load(void *opaque, int version_id)
  668. {
  669. CPUState *cpu = opaque;
  670. /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
  671. version_id is increased. */
  672. cpu->interrupt_request &= ~0x01;
  673. tlb_flush(cpu);
  674. /* loadvm has just updated the content of RAM, bypassing the
  675. * usual mechanisms that ensure we flush TBs for writes to
  676. * memory we've translated code from. So we must flush all TBs,
  677. * which will now be stale.
  678. */
  679. tb_flush(cpu);
  680. return 0;
  681. }
  682. static int cpu_common_pre_load(void *opaque)
  683. {
  684. CPUState *cpu = opaque;
  685. cpu->exception_index = -1;
  686. return 0;
  687. }
  688. static bool cpu_common_exception_index_needed(void *opaque)
  689. {
  690. CPUState *cpu = opaque;
  691. return tcg_enabled() && cpu->exception_index != -1;
  692. }
  693. static const VMStateDescription vmstate_cpu_common_exception_index = {
  694. .name = "cpu_common/exception_index",
  695. .version_id = 1,
  696. .minimum_version_id = 1,
  697. .needed = cpu_common_exception_index_needed,
  698. .fields = (VMStateField[]) {
  699. VMSTATE_INT32(exception_index, CPUState),
  700. VMSTATE_END_OF_LIST()
  701. }
  702. };
  703. static bool cpu_common_crash_occurred_needed(void *opaque)
  704. {
  705. CPUState *cpu = opaque;
  706. return cpu->crash_occurred;
  707. }
  708. static const VMStateDescription vmstate_cpu_common_crash_occurred = {
  709. .name = "cpu_common/crash_occurred",
  710. .version_id = 1,
  711. .minimum_version_id = 1,
  712. .needed = cpu_common_crash_occurred_needed,
  713. .fields = (VMStateField[]) {
  714. VMSTATE_BOOL(crash_occurred, CPUState),
  715. VMSTATE_END_OF_LIST()
  716. }
  717. };
  718. const VMStateDescription vmstate_cpu_common = {
  719. .name = "cpu_common",
  720. .version_id = 1,
  721. .minimum_version_id = 1,
  722. .pre_load = cpu_common_pre_load,
  723. .post_load = cpu_common_post_load,
  724. .fields = (VMStateField[]) {
  725. VMSTATE_UINT32(halted, CPUState),
  726. VMSTATE_UINT32(interrupt_request, CPUState),
  727. VMSTATE_END_OF_LIST()
  728. },
  729. .subsections = (const VMStateDescription*[]) {
  730. &vmstate_cpu_common_exception_index,
  731. &vmstate_cpu_common_crash_occurred,
  732. NULL
  733. }
  734. };
  735. #endif
  736. CPUState *qemu_get_cpu(int index)
  737. {
  738. CPUState *cpu;
  739. CPU_FOREACH(cpu) {
  740. if (cpu->cpu_index == index) {
  741. return cpu;
  742. }
  743. }
  744. return NULL;
  745. }
  746. #if !defined(CONFIG_USER_ONLY)
  747. void cpu_address_space_init(CPUState *cpu, int asidx,
  748. const char *prefix, MemoryRegion *mr)
  749. {
  750. CPUAddressSpace *newas;
  751. AddressSpace *as = g_new0(AddressSpace, 1);
  752. char *as_name;
  753. assert(mr);
  754. as_name = g_strdup_printf("%s-%d", prefix, cpu->cpu_index);
  755. address_space_init(as, mr, as_name);
  756. g_free(as_name);
  757. /* Target code should have set num_ases before calling us */
  758. assert(asidx < cpu->num_ases);
  759. if (asidx == 0) {
  760. /* address space 0 gets the convenience alias */
  761. cpu->as = as;
  762. }
  763. /* KVM cannot currently support multiple address spaces. */
  764. assert(asidx == 0 || !kvm_enabled());
  765. if (!cpu->cpu_ases) {
  766. cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
  767. }
  768. newas = &cpu->cpu_ases[asidx];
  769. newas->cpu = cpu;
  770. newas->as = as;
  771. if (tcg_enabled()) {
  772. newas->tcg_as_listener.commit = tcg_commit;
  773. memory_listener_register(&newas->tcg_as_listener, as);
  774. }
  775. }
  776. AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
  777. {
  778. /* Return the AddressSpace corresponding to the specified index */
  779. return cpu->cpu_ases[asidx].as;
  780. }
  781. #endif
  782. void cpu_exec_unrealizefn(CPUState *cpu)
  783. {
  784. CPUClass *cc = CPU_GET_CLASS(cpu);
  785. cpu_list_remove(cpu);
  786. if (cc->vmsd != NULL) {
  787. vmstate_unregister(NULL, cc->vmsd, cpu);
  788. }
  789. if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
  790. vmstate_unregister(NULL, &vmstate_cpu_common, cpu);
  791. }
  792. #ifndef CONFIG_USER_ONLY
  793. tcg_iommu_free_notifier_list(cpu);
  794. #endif
  795. }
  796. Property cpu_common_props[] = {
  797. #ifndef CONFIG_USER_ONLY
  798. /* Create a memory property for softmmu CPU object,
  799. * so users can wire up its memory. (This can't go in qom/cpu.c
  800. * because that file is compiled only once for both user-mode
  801. * and system builds.) The default if no link is set up is to use
  802. * the system address space.
  803. */
  804. DEFINE_PROP_LINK("memory", CPUState, memory, TYPE_MEMORY_REGION,
  805. MemoryRegion *),
  806. #endif
  807. DEFINE_PROP_END_OF_LIST(),
  808. };
  809. void cpu_exec_initfn(CPUState *cpu)
  810. {
  811. cpu->as = NULL;
  812. cpu->num_ases = 0;
  813. #ifndef CONFIG_USER_ONLY
  814. cpu->thread_id = qemu_get_thread_id();
  815. cpu->memory = system_memory;
  816. object_ref(OBJECT(cpu->memory));
  817. #endif
  818. }
  819. void cpu_exec_realizefn(CPUState *cpu, Error **errp)
  820. {
  821. CPUClass *cc = CPU_GET_CLASS(cpu);
  822. static bool tcg_target_initialized;
  823. cpu_list_add(cpu);
  824. if (tcg_enabled() && !tcg_target_initialized) {
  825. tcg_target_initialized = true;
  826. cc->tcg_initialize();
  827. }
  828. tlb_init(cpu);
  829. #ifndef CONFIG_USER_ONLY
  830. if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
  831. vmstate_register(NULL, cpu->cpu_index, &vmstate_cpu_common, cpu);
  832. }
  833. if (cc->vmsd != NULL) {
  834. vmstate_register(NULL, cpu->cpu_index, cc->vmsd, cpu);
  835. }
  836. cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier *));
  837. #endif
  838. }
  839. const char *parse_cpu_option(const char *cpu_option)
  840. {
  841. ObjectClass *oc;
  842. CPUClass *cc;
  843. gchar **model_pieces;
  844. const char *cpu_type;
  845. model_pieces = g_strsplit(cpu_option, ",", 2);
  846. if (!model_pieces[0]) {
  847. error_report("-cpu option cannot be empty");
  848. exit(1);
  849. }
  850. oc = cpu_class_by_name(CPU_RESOLVING_TYPE, model_pieces[0]);
  851. if (oc == NULL) {
  852. error_report("unable to find CPU model '%s'", model_pieces[0]);
  853. g_strfreev(model_pieces);
  854. exit(EXIT_FAILURE);
  855. }
  856. cpu_type = object_class_get_name(oc);
  857. cc = CPU_CLASS(oc);
  858. cc->parse_features(cpu_type, model_pieces[1], &error_fatal);
  859. g_strfreev(model_pieces);
  860. return cpu_type;
  861. }
  862. #if defined(CONFIG_USER_ONLY)
  863. void tb_invalidate_phys_addr(target_ulong addr)
  864. {
  865. mmap_lock();
  866. tb_invalidate_phys_page_range(addr, addr + 1, 0);
  867. mmap_unlock();
  868. }
  869. static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
  870. {
  871. tb_invalidate_phys_addr(pc);
  872. }
  873. #else
  874. void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs)
  875. {
  876. ram_addr_t ram_addr;
  877. MemoryRegion *mr;
  878. hwaddr l = 1;
  879. if (!tcg_enabled()) {
  880. return;
  881. }
  882. rcu_read_lock();
  883. mr = address_space_translate(as, addr, &addr, &l, false, attrs);
  884. if (!(memory_region_is_ram(mr)
  885. || memory_region_is_romd(mr))) {
  886. rcu_read_unlock();
  887. return;
  888. }
  889. ram_addr = memory_region_get_ram_addr(mr) + addr;
  890. tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
  891. rcu_read_unlock();
  892. }
  893. static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
  894. {
  895. MemTxAttrs attrs;
  896. hwaddr phys = cpu_get_phys_page_attrs_debug(cpu, pc, &attrs);
  897. int asidx = cpu_asidx_from_attrs(cpu, attrs);
  898. if (phys != -1) {
  899. /* Locks grabbed by tb_invalidate_phys_addr */
  900. tb_invalidate_phys_addr(cpu->cpu_ases[asidx].as,
  901. phys | (pc & ~TARGET_PAGE_MASK), attrs);
  902. }
  903. }
  904. #endif
  905. #if defined(CONFIG_USER_ONLY)
  906. void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
  907. {
  908. }
  909. int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
  910. int flags)
  911. {
  912. return -ENOSYS;
  913. }
  914. void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
  915. {
  916. }
  917. int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
  918. int flags, CPUWatchpoint **watchpoint)
  919. {
  920. return -ENOSYS;
  921. }
  922. #else
  923. /* Add a watchpoint. */
  924. int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
  925. int flags, CPUWatchpoint **watchpoint)
  926. {
  927. CPUWatchpoint *wp;
  928. /* forbid ranges which are empty or run off the end of the address space */
  929. if (len == 0 || (addr + len - 1) < addr) {
  930. error_report("tried to set invalid watchpoint at %"
  931. VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
  932. return -EINVAL;
  933. }
  934. wp = g_malloc(sizeof(*wp));
  935. wp->vaddr = addr;
  936. wp->len = len;
  937. wp->flags = flags;
  938. /* keep all GDB-injected watchpoints in front */
  939. if (flags & BP_GDB) {
  940. QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
  941. } else {
  942. QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
  943. }
  944. tlb_flush_page(cpu, addr);
  945. if (watchpoint)
  946. *watchpoint = wp;
  947. return 0;
  948. }
  949. /* Remove a specific watchpoint. */
  950. int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
  951. int flags)
  952. {
  953. CPUWatchpoint *wp;
  954. QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
  955. if (addr == wp->vaddr && len == wp->len
  956. && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
  957. cpu_watchpoint_remove_by_ref(cpu, wp);
  958. return 0;
  959. }
  960. }
  961. return -ENOENT;
  962. }
  963. /* Remove a specific watchpoint by reference. */
  964. void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
  965. {
  966. QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
  967. tlb_flush_page(cpu, watchpoint->vaddr);
  968. g_free(watchpoint);
  969. }
  970. /* Remove all matching watchpoints. */
  971. void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
  972. {
  973. CPUWatchpoint *wp, *next;
  974. QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
  975. if (wp->flags & mask) {
  976. cpu_watchpoint_remove_by_ref(cpu, wp);
  977. }
  978. }
  979. }
  980. /* Return true if this watchpoint address matches the specified
  981. * access (ie the address range covered by the watchpoint overlaps
  982. * partially or completely with the address range covered by the
  983. * access).
  984. */
  985. static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
  986. vaddr addr,
  987. vaddr len)
  988. {
  989. /* We know the lengths are non-zero, but a little caution is
  990. * required to avoid errors in the case where the range ends
  991. * exactly at the top of the address space and so addr + len
  992. * wraps round to zero.
  993. */
  994. vaddr wpend = wp->vaddr + wp->len - 1;
  995. vaddr addrend = addr + len - 1;
  996. return !(addr > wpend || wp->vaddr > addrend);
  997. }
  998. #endif
  999. /* Add a breakpoint. */
  1000. int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
  1001. CPUBreakpoint **breakpoint)
  1002. {
  1003. CPUBreakpoint *bp;
  1004. bp = g_malloc(sizeof(*bp));
  1005. bp->pc = pc;
  1006. bp->flags = flags;
  1007. /* keep all GDB-injected breakpoints in front */
  1008. if (flags & BP_GDB) {
  1009. QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
  1010. } else {
  1011. QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
  1012. }
  1013. breakpoint_invalidate(cpu, pc);
  1014. if (breakpoint) {
  1015. *breakpoint = bp;
  1016. }
  1017. return 0;
  1018. }
  1019. /* Remove a specific breakpoint. */
  1020. int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
  1021. {
  1022. CPUBreakpoint *bp;
  1023. QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
  1024. if (bp->pc == pc && bp->flags == flags) {
  1025. cpu_breakpoint_remove_by_ref(cpu, bp);
  1026. return 0;
  1027. }
  1028. }
  1029. return -ENOENT;
  1030. }
  1031. /* Remove a specific breakpoint by reference. */
  1032. void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
  1033. {
  1034. QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
  1035. breakpoint_invalidate(cpu, breakpoint->pc);
  1036. g_free(breakpoint);
  1037. }
  1038. /* Remove all matching breakpoints. */
  1039. void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
  1040. {
  1041. CPUBreakpoint *bp, *next;
  1042. QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
  1043. if (bp->flags & mask) {
  1044. cpu_breakpoint_remove_by_ref(cpu, bp);
  1045. }
  1046. }
  1047. }
  1048. /* enable or disable single step mode. EXCP_DEBUG is returned by the
  1049. CPU loop after each instruction */
  1050. void cpu_single_step(CPUState *cpu, int enabled)
  1051. {
  1052. if (cpu->singlestep_enabled != enabled) {
  1053. cpu->singlestep_enabled = enabled;
  1054. if (kvm_enabled()) {
  1055. kvm_update_guest_debug(cpu, 0);
  1056. } else {
  1057. /* must flush all the translated code to avoid inconsistencies */
  1058. /* XXX: only flush what is necessary */
  1059. tb_flush(cpu);
  1060. }
  1061. }
  1062. }
  1063. void cpu_abort(CPUState *cpu, const char *fmt, ...)
  1064. {
  1065. va_list ap;
  1066. va_list ap2;
  1067. va_start(ap, fmt);
  1068. va_copy(ap2, ap);
  1069. fprintf(stderr, "qemu: fatal: ");
  1070. vfprintf(stderr, fmt, ap);
  1071. fprintf(stderr, "\n");
  1072. cpu_dump_state(cpu, stderr, CPU_DUMP_FPU | CPU_DUMP_CCOP);
  1073. if (qemu_log_separate()) {
  1074. qemu_log_lock();
  1075. qemu_log("qemu: fatal: ");
  1076. qemu_log_vprintf(fmt, ap2);
  1077. qemu_log("\n");
  1078. log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
  1079. qemu_log_flush();
  1080. qemu_log_unlock();
  1081. qemu_log_close();
  1082. }
  1083. va_end(ap2);
  1084. va_end(ap);
  1085. replay_finish();
  1086. #if defined(CONFIG_USER_ONLY)
  1087. {
  1088. struct sigaction act;
  1089. sigfillset(&act.sa_mask);
  1090. act.sa_handler = SIG_DFL;
  1091. act.sa_flags = 0;
  1092. sigaction(SIGABRT, &act, NULL);
  1093. }
  1094. #endif
  1095. abort();
  1096. }
  1097. #if !defined(CONFIG_USER_ONLY)
  1098. /* Called from RCU critical section */
  1099. static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
  1100. {
  1101. RAMBlock *block;
  1102. block = atomic_rcu_read(&ram_list.mru_block);
  1103. if (block && addr - block->offset < block->max_length) {
  1104. return block;
  1105. }
  1106. RAMBLOCK_FOREACH(block) {
  1107. if (addr - block->offset < block->max_length) {
  1108. goto found;
  1109. }
  1110. }
  1111. fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
  1112. abort();
  1113. found:
  1114. /* It is safe to write mru_block outside the iothread lock. This
  1115. * is what happens:
  1116. *
  1117. * mru_block = xxx
  1118. * rcu_read_unlock()
  1119. * xxx removed from list
  1120. * rcu_read_lock()
  1121. * read mru_block
  1122. * mru_block = NULL;
  1123. * call_rcu(reclaim_ramblock, xxx);
  1124. * rcu_read_unlock()
  1125. *
  1126. * atomic_rcu_set is not needed here. The block was already published
  1127. * when it was placed into the list. Here we're just making an extra
  1128. * copy of the pointer.
  1129. */
  1130. ram_list.mru_block = block;
  1131. return block;
  1132. }
  1133. static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
  1134. {
  1135. CPUState *cpu;
  1136. ram_addr_t start1;
  1137. RAMBlock *block;
  1138. ram_addr_t end;
  1139. assert(tcg_enabled());
  1140. end = TARGET_PAGE_ALIGN(start + length);
  1141. start &= TARGET_PAGE_MASK;
  1142. rcu_read_lock();
  1143. block = qemu_get_ram_block(start);
  1144. assert(block == qemu_get_ram_block(end - 1));
  1145. start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
  1146. CPU_FOREACH(cpu) {
  1147. tlb_reset_dirty(cpu, start1, length);
  1148. }
  1149. rcu_read_unlock();
  1150. }
  1151. /* Note: start and end must be within the same ram block. */
  1152. bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
  1153. ram_addr_t length,
  1154. unsigned client)
  1155. {
  1156. DirtyMemoryBlocks *blocks;
  1157. unsigned long end, page;
  1158. bool dirty = false;
  1159. RAMBlock *ramblock;
  1160. uint64_t mr_offset, mr_size;
  1161. if (length == 0) {
  1162. return false;
  1163. }
  1164. end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
  1165. page = start >> TARGET_PAGE_BITS;
  1166. rcu_read_lock();
  1167. blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
  1168. ramblock = qemu_get_ram_block(start);
  1169. /* Range sanity check on the ramblock */
  1170. assert(start >= ramblock->offset &&
  1171. start + length <= ramblock->offset + ramblock->used_length);
  1172. while (page < end) {
  1173. unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
  1174. unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
  1175. unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
  1176. dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
  1177. offset, num);
  1178. page += num;
  1179. }
  1180. mr_offset = (ram_addr_t)(page << TARGET_PAGE_BITS) - ramblock->offset;
  1181. mr_size = (end - page) << TARGET_PAGE_BITS;
  1182. memory_region_clear_dirty_bitmap(ramblock->mr, mr_offset, mr_size);
  1183. rcu_read_unlock();
  1184. if (dirty && tcg_enabled()) {
  1185. tlb_reset_dirty_range_all(start, length);
  1186. }
  1187. return dirty;
  1188. }
  1189. DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
  1190. (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client)
  1191. {
  1192. DirtyMemoryBlocks *blocks;
  1193. ram_addr_t start = memory_region_get_ram_addr(mr) + offset;
  1194. unsigned long align = 1UL << (TARGET_PAGE_BITS + BITS_PER_LEVEL);
  1195. ram_addr_t first = QEMU_ALIGN_DOWN(start, align);
  1196. ram_addr_t last = QEMU_ALIGN_UP(start + length, align);
  1197. DirtyBitmapSnapshot *snap;
  1198. unsigned long page, end, dest;
  1199. snap = g_malloc0(sizeof(*snap) +
  1200. ((last - first) >> (TARGET_PAGE_BITS + 3)));
  1201. snap->start = first;
  1202. snap->end = last;
  1203. page = first >> TARGET_PAGE_BITS;
  1204. end = last >> TARGET_PAGE_BITS;
  1205. dest = 0;
  1206. rcu_read_lock();
  1207. blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
  1208. while (page < end) {
  1209. unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
  1210. unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
  1211. unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
  1212. assert(QEMU_IS_ALIGNED(offset, (1 << BITS_PER_LEVEL)));
  1213. assert(QEMU_IS_ALIGNED(num, (1 << BITS_PER_LEVEL)));
  1214. offset >>= BITS_PER_LEVEL;
  1215. bitmap_copy_and_clear_atomic(snap->dirty + dest,
  1216. blocks->blocks[idx] + offset,
  1217. num);
  1218. page += num;
  1219. dest += num >> BITS_PER_LEVEL;
  1220. }
  1221. rcu_read_unlock();
  1222. if (tcg_enabled()) {
  1223. tlb_reset_dirty_range_all(start, length);
  1224. }
  1225. memory_region_clear_dirty_bitmap(mr, offset, length);
  1226. return snap;
  1227. }
  1228. bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
  1229. ram_addr_t start,
  1230. ram_addr_t length)
  1231. {
  1232. unsigned long page, end;
  1233. assert(start >= snap->start);
  1234. assert(start + length <= snap->end);
  1235. end = TARGET_PAGE_ALIGN(start + length - snap->start) >> TARGET_PAGE_BITS;
  1236. page = (start - snap->start) >> TARGET_PAGE_BITS;
  1237. while (page < end) {
  1238. if (test_bit(page, snap->dirty)) {
  1239. return true;
  1240. }
  1241. page++;
  1242. }
  1243. return false;
  1244. }
  1245. /* Called from RCU critical section */
  1246. hwaddr memory_region_section_get_iotlb(CPUState *cpu,
  1247. MemoryRegionSection *section,
  1248. target_ulong vaddr,
  1249. hwaddr paddr, hwaddr xlat,
  1250. int prot,
  1251. target_ulong *address)
  1252. {
  1253. hwaddr iotlb;
  1254. CPUWatchpoint *wp;
  1255. if (memory_region_is_ram(section->mr)) {
  1256. /* Normal RAM. */
  1257. iotlb = memory_region_get_ram_addr(section->mr) + xlat;
  1258. if (!section->readonly) {
  1259. iotlb |= PHYS_SECTION_NOTDIRTY;
  1260. } else {
  1261. iotlb |= PHYS_SECTION_ROM;
  1262. }
  1263. } else {
  1264. AddressSpaceDispatch *d;
  1265. d = flatview_to_dispatch(section->fv);
  1266. iotlb = section - d->map.sections;
  1267. iotlb += xlat;
  1268. }
  1269. /* Make accesses to pages with watchpoints go via the
  1270. watchpoint trap routines. */
  1271. QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
  1272. if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
  1273. /* Avoid trapping reads of pages with a write breakpoint. */
  1274. if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
  1275. iotlb = PHYS_SECTION_WATCH + paddr;
  1276. *address |= TLB_MMIO;
  1277. break;
  1278. }
  1279. }
  1280. }
  1281. return iotlb;
  1282. }
  1283. #endif /* defined(CONFIG_USER_ONLY) */
  1284. #if !defined(CONFIG_USER_ONLY)
  1285. static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
  1286. uint16_t section);
  1287. static subpage_t *subpage_init(FlatView *fv, hwaddr base);
  1288. static void *(*phys_mem_alloc)(size_t size, uint64_t *align, bool shared) =
  1289. qemu_anon_ram_alloc;
  1290. /*
  1291. * Set a custom physical guest memory alloator.
  1292. * Accelerators with unusual needs may need this. Hopefully, we can
  1293. * get rid of it eventually.
  1294. */
  1295. void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align, bool shared))
  1296. {
  1297. phys_mem_alloc = alloc;
  1298. }
  1299. static uint16_t phys_section_add(PhysPageMap *map,
  1300. MemoryRegionSection *section)
  1301. {
  1302. /* The physical section number is ORed with a page-aligned
  1303. * pointer to produce the iotlb entries. Thus it should
  1304. * never overflow into the page-aligned value.
  1305. */
  1306. assert(map->sections_nb < TARGET_PAGE_SIZE);
  1307. if (map->sections_nb == map->sections_nb_alloc) {
  1308. map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
  1309. map->sections = g_renew(MemoryRegionSection, map->sections,
  1310. map->sections_nb_alloc);
  1311. }
  1312. map->sections[map->sections_nb] = *section;
  1313. memory_region_ref(section->mr);
  1314. return map->sections_nb++;
  1315. }
  1316. static void phys_section_destroy(MemoryRegion *mr)
  1317. {
  1318. bool have_sub_page = mr->subpage;
  1319. memory_region_unref(mr);
  1320. if (have_sub_page) {
  1321. subpage_t *subpage = container_of(mr, subpage_t, iomem);
  1322. object_unref(OBJECT(&subpage->iomem));
  1323. g_free(subpage);
  1324. }
  1325. }
  1326. static void phys_sections_free(PhysPageMap *map)
  1327. {
  1328. while (map->sections_nb > 0) {
  1329. MemoryRegionSection *section = &map->sections[--map->sections_nb];
  1330. phys_section_destroy(section->mr);
  1331. }
  1332. g_free(map->sections);
  1333. g_free(map->nodes);
  1334. }
  1335. static void register_subpage(FlatView *fv, MemoryRegionSection *section)
  1336. {
  1337. AddressSpaceDispatch *d = flatview_to_dispatch(fv);
  1338. subpage_t *subpage;
  1339. hwaddr base = section->offset_within_address_space
  1340. & TARGET_PAGE_MASK;
  1341. MemoryRegionSection *existing = phys_page_find(d, base);
  1342. MemoryRegionSection subsection = {
  1343. .offset_within_address_space = base,
  1344. .size = int128_make64(TARGET_PAGE_SIZE),
  1345. };
  1346. hwaddr start, end;
  1347. assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
  1348. if (!(existing->mr->subpage)) {
  1349. subpage = subpage_init(fv, base);
  1350. subsection.fv = fv;
  1351. subsection.mr = &subpage->iomem;
  1352. phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
  1353. phys_section_add(&d->map, &subsection));
  1354. } else {
  1355. subpage = container_of(existing->mr, subpage_t, iomem);
  1356. }
  1357. start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
  1358. end = start + int128_get64(section->size) - 1;
  1359. subpage_register(subpage, start, end,
  1360. phys_section_add(&d->map, section));
  1361. }
  1362. static void register_multipage(FlatView *fv,
  1363. MemoryRegionSection *section)
  1364. {
  1365. AddressSpaceDispatch *d = flatview_to_dispatch(fv);
  1366. hwaddr start_addr = section->offset_within_address_space;
  1367. uint16_t section_index = phys_section_add(&d->map, section);
  1368. uint64_t num_pages = int128_get64(int128_rshift(section->size,
  1369. TARGET_PAGE_BITS));
  1370. assert(num_pages);
  1371. phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
  1372. }
  1373. /*
  1374. * The range in *section* may look like this:
  1375. *
  1376. * |s|PPPPPPP|s|
  1377. *
  1378. * where s stands for subpage and P for page.
  1379. */
  1380. void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section)
  1381. {
  1382. MemoryRegionSection remain = *section;
  1383. Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
  1384. /* register first subpage */
  1385. if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
  1386. uint64_t left = TARGET_PAGE_ALIGN(remain.offset_within_address_space)
  1387. - remain.offset_within_address_space;
  1388. MemoryRegionSection now = remain;
  1389. now.size = int128_min(int128_make64(left), now.size);
  1390. register_subpage(fv, &now);
  1391. if (int128_eq(remain.size, now.size)) {
  1392. return;
  1393. }
  1394. remain.size = int128_sub(remain.size, now.size);
  1395. remain.offset_within_address_space += int128_get64(now.size);
  1396. remain.offset_within_region += int128_get64(now.size);
  1397. }
  1398. /* register whole pages */
  1399. if (int128_ge(remain.size, page_size)) {
  1400. MemoryRegionSection now = remain;
  1401. now.size = int128_and(now.size, int128_neg(page_size));
  1402. register_multipage(fv, &now);
  1403. if (int128_eq(remain.size, now.size)) {
  1404. return;
  1405. }
  1406. remain.size = int128_sub(remain.size, now.size);
  1407. remain.offset_within_address_space += int128_get64(now.size);
  1408. remain.offset_within_region += int128_get64(now.size);
  1409. }
  1410. /* register last subpage */
  1411. register_subpage(fv, &remain);
  1412. }
  1413. void qemu_flush_coalesced_mmio_buffer(void)
  1414. {
  1415. if (kvm_enabled())
  1416. kvm_flush_coalesced_mmio_buffer();
  1417. }
  1418. void qemu_mutex_lock_ramlist(void)
  1419. {
  1420. qemu_mutex_lock(&ram_list.mutex);
  1421. }
  1422. void qemu_mutex_unlock_ramlist(void)
  1423. {
  1424. qemu_mutex_unlock(&ram_list.mutex);
  1425. }
  1426. void ram_block_dump(Monitor *mon)
  1427. {
  1428. RAMBlock *block;
  1429. char *psize;
  1430. rcu_read_lock();
  1431. monitor_printf(mon, "%24s %8s %18s %18s %18s\n",
  1432. "Block Name", "PSize", "Offset", "Used", "Total");
  1433. RAMBLOCK_FOREACH(block) {
  1434. psize = size_to_str(block->page_size);
  1435. monitor_printf(mon, "%24s %8s 0x%016" PRIx64 " 0x%016" PRIx64
  1436. " 0x%016" PRIx64 "\n", block->idstr, psize,
  1437. (uint64_t)block->offset,
  1438. (uint64_t)block->used_length,
  1439. (uint64_t)block->max_length);
  1440. g_free(psize);
  1441. }
  1442. rcu_read_unlock();
  1443. }
  1444. #ifdef __linux__
  1445. /*
  1446. * FIXME TOCTTOU: this iterates over memory backends' mem-path, which
  1447. * may or may not name the same files / on the same filesystem now as
  1448. * when we actually open and map them. Iterate over the file
  1449. * descriptors instead, and use qemu_fd_getpagesize().
  1450. */
  1451. static int find_min_backend_pagesize(Object *obj, void *opaque)
  1452. {
  1453. long *hpsize_min = opaque;
  1454. if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
  1455. HostMemoryBackend *backend = MEMORY_BACKEND(obj);
  1456. long hpsize = host_memory_backend_pagesize(backend);
  1457. if (host_memory_backend_is_mapped(backend) && (hpsize < *hpsize_min)) {
  1458. *hpsize_min = hpsize;
  1459. }
  1460. }
  1461. return 0;
  1462. }
  1463. static int find_max_backend_pagesize(Object *obj, void *opaque)
  1464. {
  1465. long *hpsize_max = opaque;
  1466. if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
  1467. HostMemoryBackend *backend = MEMORY_BACKEND(obj);
  1468. long hpsize = host_memory_backend_pagesize(backend);
  1469. if (host_memory_backend_is_mapped(backend) && (hpsize > *hpsize_max)) {
  1470. *hpsize_max = hpsize;
  1471. }
  1472. }
  1473. return 0;
  1474. }
  1475. /*
  1476. * TODO: We assume right now that all mapped host memory backends are
  1477. * used as RAM, however some might be used for different purposes.
  1478. */
  1479. long qemu_minrampagesize(void)
  1480. {
  1481. long hpsize = LONG_MAX;
  1482. long mainrampagesize;
  1483. Object *memdev_root;
  1484. mainrampagesize = qemu_mempath_getpagesize(mem_path);
  1485. /* it's possible we have memory-backend objects with
  1486. * hugepage-backed RAM. these may get mapped into system
  1487. * address space via -numa parameters or memory hotplug
  1488. * hooks. we want to take these into account, but we
  1489. * also want to make sure these supported hugepage
  1490. * sizes are applicable across the entire range of memory
  1491. * we may boot from, so we take the min across all
  1492. * backends, and assume normal pages in cases where a
  1493. * backend isn't backed by hugepages.
  1494. */
  1495. memdev_root = object_resolve_path("/objects", NULL);
  1496. if (memdev_root) {
  1497. object_child_foreach(memdev_root, find_min_backend_pagesize, &hpsize);
  1498. }
  1499. if (hpsize == LONG_MAX) {
  1500. /* No additional memory regions found ==> Report main RAM page size */
  1501. return mainrampagesize;
  1502. }
  1503. /* If NUMA is disabled or the NUMA nodes are not backed with a
  1504. * memory-backend, then there is at least one node using "normal" RAM,
  1505. * so if its page size is smaller we have got to report that size instead.
  1506. */
  1507. if (hpsize > mainrampagesize &&
  1508. (nb_numa_nodes == 0 || numa_info[0].node_memdev == NULL)) {
  1509. static bool warned;
  1510. if (!warned) {
  1511. error_report("Huge page support disabled (n/a for main memory).");
  1512. warned = true;
  1513. }
  1514. return mainrampagesize;
  1515. }
  1516. return hpsize;
  1517. }
  1518. long qemu_maxrampagesize(void)
  1519. {
  1520. long pagesize = qemu_mempath_getpagesize(mem_path);
  1521. Object *memdev_root = object_resolve_path("/objects", NULL);
  1522. if (memdev_root) {
  1523. object_child_foreach(memdev_root, find_max_backend_pagesize,
  1524. &pagesize);
  1525. }
  1526. return pagesize;
  1527. }
  1528. #else
  1529. long qemu_minrampagesize(void)
  1530. {
  1531. return getpagesize();
  1532. }
  1533. long qemu_maxrampagesize(void)
  1534. {
  1535. return getpagesize();
  1536. }
  1537. #endif
  1538. #ifdef CONFIG_POSIX
  1539. static int64_t get_file_size(int fd)
  1540. {
  1541. int64_t size = lseek(fd, 0, SEEK_END);
  1542. if (size < 0) {
  1543. return -errno;
  1544. }
  1545. return size;
  1546. }
  1547. static int file_ram_open(const char *path,
  1548. const char *region_name,
  1549. bool *created,
  1550. Error **errp)
  1551. {
  1552. char *filename;
  1553. char *sanitized_name;
  1554. char *c;
  1555. int fd = -1;
  1556. *created = false;
  1557. for (;;) {
  1558. fd = open(path, O_RDWR);
  1559. if (fd >= 0) {
  1560. /* @path names an existing file, use it */
  1561. break;
  1562. }
  1563. if (errno == ENOENT) {
  1564. /* @path names a file that doesn't exist, create it */
  1565. fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
  1566. if (fd >= 0) {
  1567. *created = true;
  1568. break;
  1569. }
  1570. } else if (errno == EISDIR) {
  1571. /* @path names a directory, create a file there */
  1572. /* Make name safe to use with mkstemp by replacing '/' with '_'. */
  1573. sanitized_name = g_strdup(region_name);
  1574. for (c = sanitized_name; *c != '\0'; c++) {
  1575. if (*c == '/') {
  1576. *c = '_';
  1577. }
  1578. }
  1579. filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
  1580. sanitized_name);
  1581. g_free(sanitized_name);
  1582. fd = mkstemp(filename);
  1583. if (fd >= 0) {
  1584. unlink(filename);
  1585. g_free(filename);
  1586. break;
  1587. }
  1588. g_free(filename);
  1589. }
  1590. if (errno != EEXIST && errno != EINTR) {
  1591. error_setg_errno(errp, errno,
  1592. "can't open backing store %s for guest RAM",
  1593. path);
  1594. return -1;
  1595. }
  1596. /*
  1597. * Try again on EINTR and EEXIST. The latter happens when
  1598. * something else creates the file between our two open().
  1599. */
  1600. }
  1601. return fd;
  1602. }
  1603. static void *file_ram_alloc(RAMBlock *block,
  1604. ram_addr_t memory,
  1605. int fd,
  1606. bool truncate,
  1607. Error **errp)
  1608. {
  1609. MachineState *ms = MACHINE(qdev_get_machine());
  1610. void *area;
  1611. block->page_size = qemu_fd_getpagesize(fd);
  1612. if (block->mr->align % block->page_size) {
  1613. error_setg(errp, "alignment 0x%" PRIx64
  1614. " must be multiples of page size 0x%zx",
  1615. block->mr->align, block->page_size);
  1616. return NULL;
  1617. } else if (block->mr->align && !is_power_of_2(block->mr->align)) {
  1618. error_setg(errp, "alignment 0x%" PRIx64
  1619. " must be a power of two", block->mr->align);
  1620. return NULL;
  1621. }
  1622. block->mr->align = MAX(block->page_size, block->mr->align);
  1623. #if defined(__s390x__)
  1624. if (kvm_enabled()) {
  1625. block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN);
  1626. }
  1627. #endif
  1628. if (memory < block->page_size) {
  1629. error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
  1630. "or larger than page size 0x%zx",
  1631. memory, block->page_size);
  1632. return NULL;
  1633. }
  1634. memory = ROUND_UP(memory, block->page_size);
  1635. /*
  1636. * ftruncate is not supported by hugetlbfs in older
  1637. * hosts, so don't bother bailing out on errors.
  1638. * If anything goes wrong with it under other filesystems,
  1639. * mmap will fail.
  1640. *
  1641. * Do not truncate the non-empty backend file to avoid corrupting
  1642. * the existing data in the file. Disabling shrinking is not
  1643. * enough. For example, the current vNVDIMM implementation stores
  1644. * the guest NVDIMM labels at the end of the backend file. If the
  1645. * backend file is later extended, QEMU will not be able to find
  1646. * those labels. Therefore, extending the non-empty backend file
  1647. * is disabled as well.
  1648. */
  1649. if (truncate && ftruncate(fd, memory)) {
  1650. perror("ftruncate");
  1651. }
  1652. area = qemu_ram_mmap(fd, memory, block->mr->align,
  1653. block->flags & RAM_SHARED, block->flags & RAM_PMEM);
  1654. if (area == MAP_FAILED) {
  1655. error_setg_errno(errp, errno,
  1656. "unable to map backing store for guest RAM");
  1657. return NULL;
  1658. }
  1659. if (mem_prealloc) {
  1660. os_mem_prealloc(fd, area, memory, ms->smp.cpus, errp);
  1661. if (errp && *errp) {
  1662. qemu_ram_munmap(fd, area, memory);
  1663. return NULL;
  1664. }
  1665. }
  1666. block->fd = fd;
  1667. return area;
  1668. }
  1669. #endif
  1670. /* Allocate space within the ram_addr_t space that governs the
  1671. * dirty bitmaps.
  1672. * Called with the ramlist lock held.
  1673. */
  1674. static ram_addr_t find_ram_offset(ram_addr_t size)
  1675. {
  1676. RAMBlock *block, *next_block;
  1677. ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
  1678. assert(size != 0); /* it would hand out same offset multiple times */
  1679. if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
  1680. return 0;
  1681. }
  1682. RAMBLOCK_FOREACH(block) {
  1683. ram_addr_t candidate, next = RAM_ADDR_MAX;
  1684. /* Align blocks to start on a 'long' in the bitmap
  1685. * which makes the bitmap sync'ing take the fast path.
  1686. */
  1687. candidate = block->offset + block->max_length;
  1688. candidate = ROUND_UP(candidate, BITS_PER_LONG << TARGET_PAGE_BITS);
  1689. /* Search for the closest following block
  1690. * and find the gap.
  1691. */
  1692. RAMBLOCK_FOREACH(next_block) {
  1693. if (next_block->offset >= candidate) {
  1694. next = MIN(next, next_block->offset);
  1695. }
  1696. }
  1697. /* If it fits remember our place and remember the size
  1698. * of gap, but keep going so that we might find a smaller
  1699. * gap to fill so avoiding fragmentation.
  1700. */
  1701. if (next - candidate >= size && next - candidate < mingap) {
  1702. offset = candidate;
  1703. mingap = next - candidate;
  1704. }
  1705. trace_find_ram_offset_loop(size, candidate, offset, next, mingap);
  1706. }
  1707. if (offset == RAM_ADDR_MAX) {
  1708. fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
  1709. (uint64_t)size);
  1710. abort();
  1711. }
  1712. trace_find_ram_offset(size, offset);
  1713. return offset;
  1714. }
  1715. static unsigned long last_ram_page(void)
  1716. {
  1717. RAMBlock *block;
  1718. ram_addr_t last = 0;
  1719. rcu_read_lock();
  1720. RAMBLOCK_FOREACH(block) {
  1721. last = MAX(last, block->offset + block->max_length);
  1722. }
  1723. rcu_read_unlock();
  1724. return last >> TARGET_PAGE_BITS;
  1725. }
  1726. static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
  1727. {
  1728. int ret;
  1729. /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
  1730. if (!machine_dump_guest_core(current_machine)) {
  1731. ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
  1732. if (ret) {
  1733. perror("qemu_madvise");
  1734. fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
  1735. "but dump_guest_core=off specified\n");
  1736. }
  1737. }
  1738. }
  1739. const char *qemu_ram_get_idstr(RAMBlock *rb)
  1740. {
  1741. return rb->idstr;
  1742. }
  1743. void *qemu_ram_get_host_addr(RAMBlock *rb)
  1744. {
  1745. return rb->host;
  1746. }
  1747. ram_addr_t qemu_ram_get_offset(RAMBlock *rb)
  1748. {
  1749. return rb->offset;
  1750. }
  1751. ram_addr_t qemu_ram_get_used_length(RAMBlock *rb)
  1752. {
  1753. return rb->used_length;
  1754. }
  1755. bool qemu_ram_is_shared(RAMBlock *rb)
  1756. {
  1757. return rb->flags & RAM_SHARED;
  1758. }
  1759. /* Note: Only set at the start of postcopy */
  1760. bool qemu_ram_is_uf_zeroable(RAMBlock *rb)
  1761. {
  1762. return rb->flags & RAM_UF_ZEROPAGE;
  1763. }
  1764. void qemu_ram_set_uf_zeroable(RAMBlock *rb)
  1765. {
  1766. rb->flags |= RAM_UF_ZEROPAGE;
  1767. }
  1768. bool qemu_ram_is_migratable(RAMBlock *rb)
  1769. {
  1770. return rb->flags & RAM_MIGRATABLE;
  1771. }
  1772. void qemu_ram_set_migratable(RAMBlock *rb)
  1773. {
  1774. rb->flags |= RAM_MIGRATABLE;
  1775. }
  1776. void qemu_ram_unset_migratable(RAMBlock *rb)
  1777. {
  1778. rb->flags &= ~RAM_MIGRATABLE;
  1779. }
  1780. /* Called with iothread lock held. */
  1781. void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
  1782. {
  1783. RAMBlock *block;
  1784. assert(new_block);
  1785. assert(!new_block->idstr[0]);
  1786. if (dev) {
  1787. char *id = qdev_get_dev_path(dev);
  1788. if (id) {
  1789. snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
  1790. g_free(id);
  1791. }
  1792. }
  1793. pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
  1794. rcu_read_lock();
  1795. RAMBLOCK_FOREACH(block) {
  1796. if (block != new_block &&
  1797. !strcmp(block->idstr, new_block->idstr)) {
  1798. fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
  1799. new_block->idstr);
  1800. abort();
  1801. }
  1802. }
  1803. rcu_read_unlock();
  1804. }
  1805. /* Called with iothread lock held. */
  1806. void qemu_ram_unset_idstr(RAMBlock *block)
  1807. {
  1808. /* FIXME: arch_init.c assumes that this is not called throughout
  1809. * migration. Ignore the problem since hot-unplug during migration
  1810. * does not work anyway.
  1811. */
  1812. if (block) {
  1813. memset(block->idstr, 0, sizeof(block->idstr));
  1814. }
  1815. }
  1816. size_t qemu_ram_pagesize(RAMBlock *rb)
  1817. {
  1818. return rb->page_size;
  1819. }
  1820. /* Returns the largest size of page in use */
  1821. size_t qemu_ram_pagesize_largest(void)
  1822. {
  1823. RAMBlock *block;
  1824. size_t largest = 0;
  1825. RAMBLOCK_FOREACH(block) {
  1826. largest = MAX(largest, qemu_ram_pagesize(block));
  1827. }
  1828. return largest;
  1829. }
  1830. static int memory_try_enable_merging(void *addr, size_t len)
  1831. {
  1832. if (!machine_mem_merge(current_machine)) {
  1833. /* disabled by the user */
  1834. return 0;
  1835. }
  1836. return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
  1837. }
  1838. /* Only legal before guest might have detected the memory size: e.g. on
  1839. * incoming migration, or right after reset.
  1840. *
  1841. * As memory core doesn't know how is memory accessed, it is up to
  1842. * resize callback to update device state and/or add assertions to detect
  1843. * misuse, if necessary.
  1844. */
  1845. int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
  1846. {
  1847. assert(block);
  1848. newsize = HOST_PAGE_ALIGN(newsize);
  1849. if (block->used_length == newsize) {
  1850. return 0;
  1851. }
  1852. if (!(block->flags & RAM_RESIZEABLE)) {
  1853. error_setg_errno(errp, EINVAL,
  1854. "Length mismatch: %s: 0x" RAM_ADDR_FMT
  1855. " in != 0x" RAM_ADDR_FMT, block->idstr,
  1856. newsize, block->used_length);
  1857. return -EINVAL;
  1858. }
  1859. if (block->max_length < newsize) {
  1860. error_setg_errno(errp, EINVAL,
  1861. "Length too large: %s: 0x" RAM_ADDR_FMT
  1862. " > 0x" RAM_ADDR_FMT, block->idstr,
  1863. newsize, block->max_length);
  1864. return -EINVAL;
  1865. }
  1866. cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
  1867. block->used_length = newsize;
  1868. cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
  1869. DIRTY_CLIENTS_ALL);
  1870. memory_region_set_size(block->mr, newsize);
  1871. if (block->resized) {
  1872. block->resized(block->idstr, newsize, block->host);
  1873. }
  1874. return 0;
  1875. }
  1876. /* Called with ram_list.mutex held */
  1877. static void dirty_memory_extend(ram_addr_t old_ram_size,
  1878. ram_addr_t new_ram_size)
  1879. {
  1880. ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size,
  1881. DIRTY_MEMORY_BLOCK_SIZE);
  1882. ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size,
  1883. DIRTY_MEMORY_BLOCK_SIZE);
  1884. int i;
  1885. /* Only need to extend if block count increased */
  1886. if (new_num_blocks <= old_num_blocks) {
  1887. return;
  1888. }
  1889. for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
  1890. DirtyMemoryBlocks *old_blocks;
  1891. DirtyMemoryBlocks *new_blocks;
  1892. int j;
  1893. old_blocks = atomic_rcu_read(&ram_list.dirty_memory[i]);
  1894. new_blocks = g_malloc(sizeof(*new_blocks) +
  1895. sizeof(new_blocks->blocks[0]) * new_num_blocks);
  1896. if (old_num_blocks) {
  1897. memcpy(new_blocks->blocks, old_blocks->blocks,
  1898. old_num_blocks * sizeof(old_blocks->blocks[0]));
  1899. }
  1900. for (j = old_num_blocks; j < new_num_blocks; j++) {
  1901. new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
  1902. }
  1903. atomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
  1904. if (old_blocks) {
  1905. g_free_rcu(old_blocks, rcu);
  1906. }
  1907. }
  1908. }
  1909. static void ram_block_add(RAMBlock *new_block, Error **errp, bool shared)
  1910. {
  1911. RAMBlock *block;
  1912. RAMBlock *last_block = NULL;
  1913. ram_addr_t old_ram_size, new_ram_size;
  1914. Error *err = NULL;
  1915. old_ram_size = last_ram_page();
  1916. qemu_mutex_lock_ramlist();
  1917. new_block->offset = find_ram_offset(new_block->max_length);
  1918. if (!new_block->host) {
  1919. if (xen_enabled()) {
  1920. xen_ram_alloc(new_block->offset, new_block->max_length,
  1921. new_block->mr, &err);
  1922. if (err) {
  1923. error_propagate(errp, err);
  1924. qemu_mutex_unlock_ramlist();
  1925. return;
  1926. }
  1927. } else {
  1928. new_block->host = phys_mem_alloc(new_block->max_length,
  1929. &new_block->mr->align, shared);
  1930. if (!new_block->host) {
  1931. error_setg_errno(errp, errno,
  1932. "cannot set up guest memory '%s'",
  1933. memory_region_name(new_block->mr));
  1934. qemu_mutex_unlock_ramlist();
  1935. return;
  1936. }
  1937. memory_try_enable_merging(new_block->host, new_block->max_length);
  1938. }
  1939. }
  1940. new_ram_size = MAX(old_ram_size,
  1941. (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
  1942. if (new_ram_size > old_ram_size) {
  1943. dirty_memory_extend(old_ram_size, new_ram_size);
  1944. }
  1945. /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
  1946. * QLIST (which has an RCU-friendly variant) does not have insertion at
  1947. * tail, so save the last element in last_block.
  1948. */
  1949. RAMBLOCK_FOREACH(block) {
  1950. last_block = block;
  1951. if (block->max_length < new_block->max_length) {
  1952. break;
  1953. }
  1954. }
  1955. if (block) {
  1956. QLIST_INSERT_BEFORE_RCU(block, new_block, next);
  1957. } else if (last_block) {
  1958. QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
  1959. } else { /* list is empty */
  1960. QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
  1961. }
  1962. ram_list.mru_block = NULL;
  1963. /* Write list before version */
  1964. smp_wmb();
  1965. ram_list.version++;
  1966. qemu_mutex_unlock_ramlist();
  1967. cpu_physical_memory_set_dirty_range(new_block->offset,
  1968. new_block->used_length,
  1969. DIRTY_CLIENTS_ALL);
  1970. if (new_block->host) {
  1971. qemu_ram_setup_dump(new_block->host, new_block->max_length);
  1972. qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
  1973. /* MADV_DONTFORK is also needed by KVM in absence of synchronous MMU */
  1974. qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
  1975. ram_block_notify_add(new_block->host, new_block->max_length);
  1976. }
  1977. }
  1978. #ifdef CONFIG_POSIX
  1979. RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
  1980. uint32_t ram_flags, int fd,
  1981. Error **errp)
  1982. {
  1983. RAMBlock *new_block;
  1984. Error *local_err = NULL;
  1985. int64_t file_size;
  1986. /* Just support these ram flags by now. */
  1987. assert((ram_flags & ~(RAM_SHARED | RAM_PMEM)) == 0);
  1988. if (xen_enabled()) {
  1989. error_setg(errp, "-mem-path not supported with Xen");
  1990. return NULL;
  1991. }
  1992. if (kvm_enabled() && !kvm_has_sync_mmu()) {
  1993. error_setg(errp,
  1994. "host lacks kvm mmu notifiers, -mem-path unsupported");
  1995. return NULL;
  1996. }
  1997. if (phys_mem_alloc != qemu_anon_ram_alloc) {
  1998. /*
  1999. * file_ram_alloc() needs to allocate just like
  2000. * phys_mem_alloc, but we haven't bothered to provide
  2001. * a hook there.
  2002. */
  2003. error_setg(errp,
  2004. "-mem-path not supported with this accelerator");
  2005. return NULL;
  2006. }
  2007. size = HOST_PAGE_ALIGN(size);
  2008. file_size = get_file_size(fd);
  2009. if (file_size > 0 && file_size < size) {
  2010. error_setg(errp, "backing store %s size 0x%" PRIx64
  2011. " does not match 'size' option 0x" RAM_ADDR_FMT,
  2012. mem_path, file_size, size);
  2013. return NULL;
  2014. }
  2015. new_block = g_malloc0(sizeof(*new_block));
  2016. new_block->mr = mr;
  2017. new_block->used_length = size;
  2018. new_block->max_length = size;
  2019. new_block->flags = ram_flags;
  2020. new_block->host = file_ram_alloc(new_block, size, fd, !file_size, errp);
  2021. if (!new_block->host) {
  2022. g_free(new_block);
  2023. return NULL;
  2024. }
  2025. ram_block_add(new_block, &local_err, ram_flags & RAM_SHARED);
  2026. if (local_err) {
  2027. g_free(new_block);
  2028. error_propagate(errp, local_err);
  2029. return NULL;
  2030. }
  2031. return new_block;
  2032. }
  2033. RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
  2034. uint32_t ram_flags, const char *mem_path,
  2035. Error **errp)
  2036. {
  2037. int fd;
  2038. bool created;
  2039. RAMBlock *block;
  2040. fd = file_ram_open(mem_path, memory_region_name(mr), &created, errp);
  2041. if (fd < 0) {
  2042. return NULL;
  2043. }
  2044. block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, errp);
  2045. if (!block) {
  2046. if (created) {
  2047. unlink(mem_path);
  2048. }
  2049. close(fd);
  2050. return NULL;
  2051. }
  2052. return block;
  2053. }
  2054. #endif
  2055. static
  2056. RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
  2057. void (*resized)(const char*,
  2058. uint64_t length,
  2059. void *host),
  2060. void *host, bool resizeable, bool share,
  2061. MemoryRegion *mr, Error **errp)
  2062. {
  2063. RAMBlock *new_block;
  2064. Error *local_err = NULL;
  2065. size = HOST_PAGE_ALIGN(size);
  2066. max_size = HOST_PAGE_ALIGN(max_size);
  2067. new_block = g_malloc0(sizeof(*new_block));
  2068. new_block->mr = mr;
  2069. new_block->resized = resized;
  2070. new_block->used_length = size;
  2071. new_block->max_length = max_size;
  2072. assert(max_size >= size);
  2073. new_block->fd = -1;
  2074. new_block->page_size = getpagesize();
  2075. new_block->host = host;
  2076. if (host) {
  2077. new_block->flags |= RAM_PREALLOC;
  2078. }
  2079. if (resizeable) {
  2080. new_block->flags |= RAM_RESIZEABLE;
  2081. }
  2082. ram_block_add(new_block, &local_err, share);
  2083. if (local_err) {
  2084. g_free(new_block);
  2085. error_propagate(errp, local_err);
  2086. return NULL;
  2087. }
  2088. return new_block;
  2089. }
  2090. RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
  2091. MemoryRegion *mr, Error **errp)
  2092. {
  2093. return qemu_ram_alloc_internal(size, size, NULL, host, false,
  2094. false, mr, errp);
  2095. }
  2096. RAMBlock *qemu_ram_alloc(ram_addr_t size, bool share,
  2097. MemoryRegion *mr, Error **errp)
  2098. {
  2099. return qemu_ram_alloc_internal(size, size, NULL, NULL, false,
  2100. share, mr, errp);
  2101. }
  2102. RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
  2103. void (*resized)(const char*,
  2104. uint64_t length,
  2105. void *host),
  2106. MemoryRegion *mr, Error **errp)
  2107. {
  2108. return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true,
  2109. false, mr, errp);
  2110. }
  2111. static void reclaim_ramblock(RAMBlock *block)
  2112. {
  2113. if (block->flags & RAM_PREALLOC) {
  2114. ;
  2115. } else if (xen_enabled()) {
  2116. xen_invalidate_map_cache_entry(block->host);
  2117. #ifndef _WIN32
  2118. } else if (block->fd >= 0) {
  2119. qemu_ram_munmap(block->fd, block->host, block->max_length);
  2120. close(block->fd);
  2121. #endif
  2122. } else {
  2123. qemu_anon_ram_free(block->host, block->max_length);
  2124. }
  2125. g_free(block);
  2126. }
  2127. void qemu_ram_free(RAMBlock *block)
  2128. {
  2129. if (!block) {
  2130. return;
  2131. }
  2132. if (block->host) {
  2133. ram_block_notify_remove(block->host, block->max_length);
  2134. }
  2135. qemu_mutex_lock_ramlist();
  2136. QLIST_REMOVE_RCU(block, next);
  2137. ram_list.mru_block = NULL;
  2138. /* Write list before version */
  2139. smp_wmb();
  2140. ram_list.version++;
  2141. call_rcu(block, reclaim_ramblock, rcu);
  2142. qemu_mutex_unlock_ramlist();
  2143. }
  2144. #ifndef _WIN32
  2145. void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
  2146. {
  2147. RAMBlock *block;
  2148. ram_addr_t offset;
  2149. int flags;
  2150. void *area, *vaddr;
  2151. RAMBLOCK_FOREACH(block) {
  2152. offset = addr - block->offset;
  2153. if (offset < block->max_length) {
  2154. vaddr = ramblock_ptr(block, offset);
  2155. if (block->flags & RAM_PREALLOC) {
  2156. ;
  2157. } else if (xen_enabled()) {
  2158. abort();
  2159. } else {
  2160. flags = MAP_FIXED;
  2161. if (block->fd >= 0) {
  2162. flags |= (block->flags & RAM_SHARED ?
  2163. MAP_SHARED : MAP_PRIVATE);
  2164. area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
  2165. flags, block->fd, offset);
  2166. } else {
  2167. /*
  2168. * Remap needs to match alloc. Accelerators that
  2169. * set phys_mem_alloc never remap. If they did,
  2170. * we'd need a remap hook here.
  2171. */
  2172. assert(phys_mem_alloc == qemu_anon_ram_alloc);
  2173. flags |= MAP_PRIVATE | MAP_ANONYMOUS;
  2174. area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
  2175. flags, -1, 0);
  2176. }
  2177. if (area != vaddr) {
  2178. error_report("Could not remap addr: "
  2179. RAM_ADDR_FMT "@" RAM_ADDR_FMT "",
  2180. length, addr);
  2181. exit(1);
  2182. }
  2183. memory_try_enable_merging(vaddr, length);
  2184. qemu_ram_setup_dump(vaddr, length);
  2185. }
  2186. }
  2187. }
  2188. }
  2189. #endif /* !_WIN32 */
  2190. /* Return a host pointer to ram allocated with qemu_ram_alloc.
  2191. * This should not be used for general purpose DMA. Use address_space_map
  2192. * or address_space_rw instead. For local memory (e.g. video ram) that the
  2193. * device owns, use memory_region_get_ram_ptr.
  2194. *
  2195. * Called within RCU critical section.
  2196. */
  2197. void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
  2198. {
  2199. RAMBlock *block = ram_block;
  2200. if (block == NULL) {
  2201. block = qemu_get_ram_block(addr);
  2202. addr -= block->offset;
  2203. }
  2204. if (xen_enabled() && block->host == NULL) {
  2205. /* We need to check if the requested address is in the RAM
  2206. * because we don't want to map the entire memory in QEMU.
  2207. * In that case just map until the end of the page.
  2208. */
  2209. if (block->offset == 0) {
  2210. return xen_map_cache(addr, 0, 0, false);
  2211. }
  2212. block->host = xen_map_cache(block->offset, block->max_length, 1, false);
  2213. }
  2214. return ramblock_ptr(block, addr);
  2215. }
  2216. /* Return a host pointer to guest's ram. Similar to qemu_map_ram_ptr
  2217. * but takes a size argument.
  2218. *
  2219. * Called within RCU critical section.
  2220. */
  2221. static void *qemu_ram_ptr_length(RAMBlock *ram_block, ram_addr_t addr,
  2222. hwaddr *size, bool lock)
  2223. {
  2224. RAMBlock *block = ram_block;
  2225. if (*size == 0) {
  2226. return NULL;
  2227. }
  2228. if (block == NULL) {
  2229. block = qemu_get_ram_block(addr);
  2230. addr -= block->offset;
  2231. }
  2232. *size = MIN(*size, block->max_length - addr);
  2233. if (xen_enabled() && block->host == NULL) {
  2234. /* We need to check if the requested address is in the RAM
  2235. * because we don't want to map the entire memory in QEMU.
  2236. * In that case just map the requested area.
  2237. */
  2238. if (block->offset == 0) {
  2239. return xen_map_cache(addr, *size, lock, lock);
  2240. }
  2241. block->host = xen_map_cache(block->offset, block->max_length, 1, lock);
  2242. }
  2243. return ramblock_ptr(block, addr);
  2244. }
  2245. /* Return the offset of a hostpointer within a ramblock */
  2246. ram_addr_t qemu_ram_block_host_offset(RAMBlock *rb, void *host)
  2247. {
  2248. ram_addr_t res = (uint8_t *)host - (uint8_t *)rb->host;
  2249. assert((uintptr_t)host >= (uintptr_t)rb->host);
  2250. assert(res < rb->max_length);
  2251. return res;
  2252. }
  2253. /*
  2254. * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
  2255. * in that RAMBlock.
  2256. *
  2257. * ptr: Host pointer to look up
  2258. * round_offset: If true round the result offset down to a page boundary
  2259. * *ram_addr: set to result ram_addr
  2260. * *offset: set to result offset within the RAMBlock
  2261. *
  2262. * Returns: RAMBlock (or NULL if not found)
  2263. *
  2264. * By the time this function returns, the returned pointer is not protected
  2265. * by RCU anymore. If the caller is not within an RCU critical section and
  2266. * does not hold the iothread lock, it must have other means of protecting the
  2267. * pointer, such as a reference to the region that includes the incoming
  2268. * ram_addr_t.
  2269. */
  2270. RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
  2271. ram_addr_t *offset)
  2272. {
  2273. RAMBlock *block;
  2274. uint8_t *host = ptr;
  2275. if (xen_enabled()) {
  2276. ram_addr_t ram_addr;
  2277. rcu_read_lock();
  2278. ram_addr = xen_ram_addr_from_mapcache(ptr);
  2279. block = qemu_get_ram_block(ram_addr);
  2280. if (block) {
  2281. *offset = ram_addr - block->offset;
  2282. }
  2283. rcu_read_unlock();
  2284. return block;
  2285. }
  2286. rcu_read_lock();
  2287. block = atomic_rcu_read(&ram_list.mru_block);
  2288. if (block && block->host && host - block->host < block->max_length) {
  2289. goto found;
  2290. }
  2291. RAMBLOCK_FOREACH(block) {
  2292. /* This case append when the block is not mapped. */
  2293. if (block->host == NULL) {
  2294. continue;
  2295. }
  2296. if (host - block->host < block->max_length) {
  2297. goto found;
  2298. }
  2299. }
  2300. rcu_read_unlock();
  2301. return NULL;
  2302. found:
  2303. *offset = (host - block->host);
  2304. if (round_offset) {
  2305. *offset &= TARGET_PAGE_MASK;
  2306. }
  2307. rcu_read_unlock();
  2308. return block;
  2309. }
  2310. /*
  2311. * Finds the named RAMBlock
  2312. *
  2313. * name: The name of RAMBlock to find
  2314. *
  2315. * Returns: RAMBlock (or NULL if not found)
  2316. */
  2317. RAMBlock *qemu_ram_block_by_name(const char *name)
  2318. {
  2319. RAMBlock *block;
  2320. RAMBLOCK_FOREACH(block) {
  2321. if (!strcmp(name, block->idstr)) {
  2322. return block;
  2323. }
  2324. }
  2325. return NULL;
  2326. }
  2327. /* Some of the softmmu routines need to translate from a host pointer
  2328. (typically a TLB entry) back to a ram offset. */
  2329. ram_addr_t qemu_ram_addr_from_host(void *ptr)
  2330. {
  2331. RAMBlock *block;
  2332. ram_addr_t offset;
  2333. block = qemu_ram_block_from_host(ptr, false, &offset);
  2334. if (!block) {
  2335. return RAM_ADDR_INVALID;
  2336. }
  2337. return block->offset + offset;
  2338. }
  2339. /* Called within RCU critical section. */
  2340. void memory_notdirty_write_prepare(NotDirtyInfo *ndi,
  2341. CPUState *cpu,
  2342. vaddr mem_vaddr,
  2343. ram_addr_t ram_addr,
  2344. unsigned size)
  2345. {
  2346. ndi->cpu = cpu;
  2347. ndi->ram_addr = ram_addr;
  2348. ndi->mem_vaddr = mem_vaddr;
  2349. ndi->size = size;
  2350. ndi->pages = NULL;
  2351. assert(tcg_enabled());
  2352. if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
  2353. ndi->pages = page_collection_lock(ram_addr, ram_addr + size);
  2354. tb_invalidate_phys_page_fast(ndi->pages, ram_addr, size);
  2355. }
  2356. }
  2357. /* Called within RCU critical section. */
  2358. void memory_notdirty_write_complete(NotDirtyInfo *ndi)
  2359. {
  2360. if (ndi->pages) {
  2361. assert(tcg_enabled());
  2362. page_collection_unlock(ndi->pages);
  2363. ndi->pages = NULL;
  2364. }
  2365. /* Set both VGA and migration bits for simplicity and to remove
  2366. * the notdirty callback faster.
  2367. */
  2368. cpu_physical_memory_set_dirty_range(ndi->ram_addr, ndi->size,
  2369. DIRTY_CLIENTS_NOCODE);
  2370. /* we remove the notdirty callback only if the code has been
  2371. flushed */
  2372. if (!cpu_physical_memory_is_clean(ndi->ram_addr)) {
  2373. tlb_set_dirty(ndi->cpu, ndi->mem_vaddr);
  2374. }
  2375. }
  2376. /* Called within RCU critical section. */
  2377. static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
  2378. uint64_t val, unsigned size)
  2379. {
  2380. NotDirtyInfo ndi;
  2381. memory_notdirty_write_prepare(&ndi, current_cpu, current_cpu->mem_io_vaddr,
  2382. ram_addr, size);
  2383. stn_p(qemu_map_ram_ptr(NULL, ram_addr), size, val);
  2384. memory_notdirty_write_complete(&ndi);
  2385. }
  2386. static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
  2387. unsigned size, bool is_write,
  2388. MemTxAttrs attrs)
  2389. {
  2390. return is_write;
  2391. }
  2392. static const MemoryRegionOps notdirty_mem_ops = {
  2393. .write = notdirty_mem_write,
  2394. .valid.accepts = notdirty_mem_accepts,
  2395. .endianness = DEVICE_NATIVE_ENDIAN,
  2396. .valid = {
  2397. .min_access_size = 1,
  2398. .max_access_size = 8,
  2399. .unaligned = false,
  2400. },
  2401. .impl = {
  2402. .min_access_size = 1,
  2403. .max_access_size = 8,
  2404. .unaligned = false,
  2405. },
  2406. };
  2407. /* Generate a debug exception if a watchpoint has been hit. */
  2408. static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
  2409. {
  2410. CPUState *cpu = current_cpu;
  2411. CPUClass *cc = CPU_GET_CLASS(cpu);
  2412. target_ulong vaddr;
  2413. CPUWatchpoint *wp;
  2414. assert(tcg_enabled());
  2415. if (cpu->watchpoint_hit) {
  2416. /* We re-entered the check after replacing the TB. Now raise
  2417. * the debug interrupt so that is will trigger after the
  2418. * current instruction. */
  2419. cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
  2420. return;
  2421. }
  2422. vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
  2423. vaddr = cc->adjust_watchpoint_address(cpu, vaddr, len);
  2424. QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
  2425. if (cpu_watchpoint_address_matches(wp, vaddr, len)
  2426. && (wp->flags & flags)) {
  2427. if (flags == BP_MEM_READ) {
  2428. wp->flags |= BP_WATCHPOINT_HIT_READ;
  2429. } else {
  2430. wp->flags |= BP_WATCHPOINT_HIT_WRITE;
  2431. }
  2432. wp->hitaddr = vaddr;
  2433. wp->hitattrs = attrs;
  2434. if (!cpu->watchpoint_hit) {
  2435. if (wp->flags & BP_CPU &&
  2436. !cc->debug_check_watchpoint(cpu, wp)) {
  2437. wp->flags &= ~BP_WATCHPOINT_HIT;
  2438. continue;
  2439. }
  2440. cpu->watchpoint_hit = wp;
  2441. mmap_lock();
  2442. tb_check_watchpoint(cpu);
  2443. if (wp->flags & BP_STOP_BEFORE_ACCESS) {
  2444. cpu->exception_index = EXCP_DEBUG;
  2445. mmap_unlock();
  2446. cpu_loop_exit(cpu);
  2447. } else {
  2448. /* Force execution of one insn next time. */
  2449. cpu->cflags_next_tb = 1 | curr_cflags();
  2450. mmap_unlock();
  2451. cpu_loop_exit_noexc(cpu);
  2452. }
  2453. }
  2454. } else {
  2455. wp->flags &= ~BP_WATCHPOINT_HIT;
  2456. }
  2457. }
  2458. }
  2459. /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
  2460. so these check for a hit then pass through to the normal out-of-line
  2461. phys routines. */
  2462. static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
  2463. unsigned size, MemTxAttrs attrs)
  2464. {
  2465. MemTxResult res;
  2466. uint64_t data;
  2467. int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
  2468. AddressSpace *as = current_cpu->cpu_ases[asidx].as;
  2469. check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
  2470. switch (size) {
  2471. case 1:
  2472. data = address_space_ldub(as, addr, attrs, &res);
  2473. break;
  2474. case 2:
  2475. data = address_space_lduw(as, addr, attrs, &res);
  2476. break;
  2477. case 4:
  2478. data = address_space_ldl(as, addr, attrs, &res);
  2479. break;
  2480. case 8:
  2481. data = address_space_ldq(as, addr, attrs, &res);
  2482. break;
  2483. default: abort();
  2484. }
  2485. *pdata = data;
  2486. return res;
  2487. }
  2488. static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
  2489. uint64_t val, unsigned size,
  2490. MemTxAttrs attrs)
  2491. {
  2492. MemTxResult res;
  2493. int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
  2494. AddressSpace *as = current_cpu->cpu_ases[asidx].as;
  2495. check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
  2496. switch (size) {
  2497. case 1:
  2498. address_space_stb(as, addr, val, attrs, &res);
  2499. break;
  2500. case 2:
  2501. address_space_stw(as, addr, val, attrs, &res);
  2502. break;
  2503. case 4:
  2504. address_space_stl(as, addr, val, attrs, &res);
  2505. break;
  2506. case 8:
  2507. address_space_stq(as, addr, val, attrs, &res);
  2508. break;
  2509. default: abort();
  2510. }
  2511. return res;
  2512. }
  2513. static const MemoryRegionOps watch_mem_ops = {
  2514. .read_with_attrs = watch_mem_read,
  2515. .write_with_attrs = watch_mem_write,
  2516. .endianness = DEVICE_NATIVE_ENDIAN,
  2517. .valid = {
  2518. .min_access_size = 1,
  2519. .max_access_size = 8,
  2520. .unaligned = false,
  2521. },
  2522. .impl = {
  2523. .min_access_size = 1,
  2524. .max_access_size = 8,
  2525. .unaligned = false,
  2526. },
  2527. };
  2528. static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
  2529. MemTxAttrs attrs, uint8_t *buf, hwaddr len);
  2530. static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
  2531. const uint8_t *buf, hwaddr len);
  2532. static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
  2533. bool is_write, MemTxAttrs attrs);
  2534. static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
  2535. unsigned len, MemTxAttrs attrs)
  2536. {
  2537. subpage_t *subpage = opaque;
  2538. uint8_t buf[8];
  2539. MemTxResult res;
  2540. #if defined(DEBUG_SUBPAGE)
  2541. printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
  2542. subpage, len, addr);
  2543. #endif
  2544. res = flatview_read(subpage->fv, addr + subpage->base, attrs, buf, len);
  2545. if (res) {
  2546. return res;
  2547. }
  2548. *data = ldn_p(buf, len);
  2549. return MEMTX_OK;
  2550. }
  2551. static MemTxResult subpage_write(void *opaque, hwaddr addr,
  2552. uint64_t value, unsigned len, MemTxAttrs attrs)
  2553. {
  2554. subpage_t *subpage = opaque;
  2555. uint8_t buf[8];
  2556. #if defined(DEBUG_SUBPAGE)
  2557. printf("%s: subpage %p len %u addr " TARGET_FMT_plx
  2558. " value %"PRIx64"\n",
  2559. __func__, subpage, len, addr, value);
  2560. #endif
  2561. stn_p(buf, len, value);
  2562. return flatview_write(subpage->fv, addr + subpage->base, attrs, buf, len);
  2563. }
  2564. static bool subpage_accepts(void *opaque, hwaddr addr,
  2565. unsigned len, bool is_write,
  2566. MemTxAttrs attrs)
  2567. {
  2568. subpage_t *subpage = opaque;
  2569. #if defined(DEBUG_SUBPAGE)
  2570. printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
  2571. __func__, subpage, is_write ? 'w' : 'r', len, addr);
  2572. #endif
  2573. return flatview_access_valid(subpage->fv, addr + subpage->base,
  2574. len, is_write, attrs);
  2575. }
  2576. static const MemoryRegionOps subpage_ops = {
  2577. .read_with_attrs = subpage_read,
  2578. .write_with_attrs = subpage_write,
  2579. .impl.min_access_size = 1,
  2580. .impl.max_access_size = 8,
  2581. .valid.min_access_size = 1,
  2582. .valid.max_access_size = 8,
  2583. .valid.accepts = subpage_accepts,
  2584. .endianness = DEVICE_NATIVE_ENDIAN,
  2585. };
  2586. static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
  2587. uint16_t section)
  2588. {
  2589. int idx, eidx;
  2590. if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
  2591. return -1;
  2592. idx = SUBPAGE_IDX(start);
  2593. eidx = SUBPAGE_IDX(end);
  2594. #if defined(DEBUG_SUBPAGE)
  2595. printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
  2596. __func__, mmio, start, end, idx, eidx, section);
  2597. #endif
  2598. for (; idx <= eidx; idx++) {
  2599. mmio->sub_section[idx] = section;
  2600. }
  2601. return 0;
  2602. }
  2603. static subpage_t *subpage_init(FlatView *fv, hwaddr base)
  2604. {
  2605. subpage_t *mmio;
  2606. mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t));
  2607. mmio->fv = fv;
  2608. mmio->base = base;
  2609. memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
  2610. NULL, TARGET_PAGE_SIZE);
  2611. mmio->iomem.subpage = true;
  2612. #if defined(DEBUG_SUBPAGE)
  2613. printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
  2614. mmio, base, TARGET_PAGE_SIZE);
  2615. #endif
  2616. subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
  2617. return mmio;
  2618. }
  2619. static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr)
  2620. {
  2621. assert(fv);
  2622. MemoryRegionSection section = {
  2623. .fv = fv,
  2624. .mr = mr,
  2625. .offset_within_address_space = 0,
  2626. .offset_within_region = 0,
  2627. .size = int128_2_64(),
  2628. };
  2629. return phys_section_add(map, &section);
  2630. }
  2631. static void readonly_mem_write(void *opaque, hwaddr addr,
  2632. uint64_t val, unsigned size)
  2633. {
  2634. /* Ignore any write to ROM. */
  2635. }
  2636. static bool readonly_mem_accepts(void *opaque, hwaddr addr,
  2637. unsigned size, bool is_write,
  2638. MemTxAttrs attrs)
  2639. {
  2640. return is_write;
  2641. }
  2642. /* This will only be used for writes, because reads are special cased
  2643. * to directly access the underlying host ram.
  2644. */
  2645. static const MemoryRegionOps readonly_mem_ops = {
  2646. .write = readonly_mem_write,
  2647. .valid.accepts = readonly_mem_accepts,
  2648. .endianness = DEVICE_NATIVE_ENDIAN,
  2649. .valid = {
  2650. .min_access_size = 1,
  2651. .max_access_size = 8,
  2652. .unaligned = false,
  2653. },
  2654. .impl = {
  2655. .min_access_size = 1,
  2656. .max_access_size = 8,
  2657. .unaligned = false,
  2658. },
  2659. };
  2660. MemoryRegionSection *iotlb_to_section(CPUState *cpu,
  2661. hwaddr index, MemTxAttrs attrs)
  2662. {
  2663. int asidx = cpu_asidx_from_attrs(cpu, attrs);
  2664. CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
  2665. AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
  2666. MemoryRegionSection *sections = d->map.sections;
  2667. return &sections[index & ~TARGET_PAGE_MASK];
  2668. }
  2669. static void io_mem_init(void)
  2670. {
  2671. memory_region_init_io(&io_mem_rom, NULL, &readonly_mem_ops,
  2672. NULL, NULL, UINT64_MAX);
  2673. memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
  2674. NULL, UINT64_MAX);
  2675. /* io_mem_notdirty calls tb_invalidate_phys_page_fast,
  2676. * which can be called without the iothread mutex.
  2677. */
  2678. memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
  2679. NULL, UINT64_MAX);
  2680. memory_region_clear_global_locking(&io_mem_notdirty);
  2681. memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
  2682. NULL, UINT64_MAX);
  2683. }
  2684. AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv)
  2685. {
  2686. AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
  2687. uint16_t n;
  2688. n = dummy_section(&d->map, fv, &io_mem_unassigned);
  2689. assert(n == PHYS_SECTION_UNASSIGNED);
  2690. n = dummy_section(&d->map, fv, &io_mem_notdirty);
  2691. assert(n == PHYS_SECTION_NOTDIRTY);
  2692. n = dummy_section(&d->map, fv, &io_mem_rom);
  2693. assert(n == PHYS_SECTION_ROM);
  2694. n = dummy_section(&d->map, fv, &io_mem_watch);
  2695. assert(n == PHYS_SECTION_WATCH);
  2696. d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
  2697. return d;
  2698. }
  2699. void address_space_dispatch_free(AddressSpaceDispatch *d)
  2700. {
  2701. phys_sections_free(&d->map);
  2702. g_free(d);
  2703. }
  2704. static void tcg_commit(MemoryListener *listener)
  2705. {
  2706. CPUAddressSpace *cpuas;
  2707. AddressSpaceDispatch *d;
  2708. assert(tcg_enabled());
  2709. /* since each CPU stores ram addresses in its TLB cache, we must
  2710. reset the modified entries */
  2711. cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
  2712. cpu_reloading_memory_map();
  2713. /* The CPU and TLB are protected by the iothread lock.
  2714. * We reload the dispatch pointer now because cpu_reloading_memory_map()
  2715. * may have split the RCU critical section.
  2716. */
  2717. d = address_space_to_dispatch(cpuas->as);
  2718. atomic_rcu_set(&cpuas->memory_dispatch, d);
  2719. tlb_flush(cpuas->cpu);
  2720. }
  2721. static void memory_map_init(void)
  2722. {
  2723. system_memory = g_malloc(sizeof(*system_memory));
  2724. memory_region_init(system_memory, NULL, "system", UINT64_MAX);
  2725. address_space_init(&address_space_memory, system_memory, "memory");
  2726. system_io = g_malloc(sizeof(*system_io));
  2727. memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
  2728. 65536);
  2729. address_space_init(&address_space_io, system_io, "I/O");
  2730. }
  2731. MemoryRegion *get_system_memory(void)
  2732. {
  2733. return system_memory;
  2734. }
  2735. MemoryRegion *get_system_io(void)
  2736. {
  2737. return system_io;
  2738. }
  2739. #endif /* !defined(CONFIG_USER_ONLY) */
  2740. /* physical memory access (slow version, mainly for debug) */
  2741. #if defined(CONFIG_USER_ONLY)
  2742. int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
  2743. uint8_t *buf, target_ulong len, int is_write)
  2744. {
  2745. int flags;
  2746. target_ulong l, page;
  2747. void * p;
  2748. while (len > 0) {
  2749. page = addr & TARGET_PAGE_MASK;
  2750. l = (page + TARGET_PAGE_SIZE) - addr;
  2751. if (l > len)
  2752. l = len;
  2753. flags = page_get_flags(page);
  2754. if (!(flags & PAGE_VALID))
  2755. return -1;
  2756. if (is_write) {
  2757. if (!(flags & PAGE_WRITE))
  2758. return -1;
  2759. /* XXX: this code should not depend on lock_user */
  2760. if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
  2761. return -1;
  2762. memcpy(p, buf, l);
  2763. unlock_user(p, addr, l);
  2764. } else {
  2765. if (!(flags & PAGE_READ))
  2766. return -1;
  2767. /* XXX: this code should not depend on lock_user */
  2768. if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
  2769. return -1;
  2770. memcpy(buf, p, l);
  2771. unlock_user(p, addr, 0);
  2772. }
  2773. len -= l;
  2774. buf += l;
  2775. addr += l;
  2776. }
  2777. return 0;
  2778. }
  2779. #else
  2780. static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
  2781. hwaddr length)
  2782. {
  2783. uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
  2784. addr += memory_region_get_ram_addr(mr);
  2785. /* No early return if dirty_log_mask is or becomes 0, because
  2786. * cpu_physical_memory_set_dirty_range will still call
  2787. * xen_modified_memory.
  2788. */
  2789. if (dirty_log_mask) {
  2790. dirty_log_mask =
  2791. cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
  2792. }
  2793. if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
  2794. assert(tcg_enabled());
  2795. tb_invalidate_phys_range(addr, addr + length);
  2796. dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
  2797. }
  2798. cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
  2799. }
  2800. void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size)
  2801. {
  2802. /*
  2803. * In principle this function would work on other memory region types too,
  2804. * but the ROM device use case is the only one where this operation is
  2805. * necessary. Other memory regions should use the
  2806. * address_space_read/write() APIs.
  2807. */
  2808. assert(memory_region_is_romd(mr));
  2809. invalidate_and_set_dirty(mr, addr, size);
  2810. }
  2811. static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
  2812. {
  2813. unsigned access_size_max = mr->ops->valid.max_access_size;
  2814. /* Regions are assumed to support 1-4 byte accesses unless
  2815. otherwise specified. */
  2816. if (access_size_max == 0) {
  2817. access_size_max = 4;
  2818. }
  2819. /* Bound the maximum access by the alignment of the address. */
  2820. if (!mr->ops->impl.unaligned) {
  2821. unsigned align_size_max = addr & -addr;
  2822. if (align_size_max != 0 && align_size_max < access_size_max) {
  2823. access_size_max = align_size_max;
  2824. }
  2825. }
  2826. /* Don't attempt accesses larger than the maximum. */
  2827. if (l > access_size_max) {
  2828. l = access_size_max;
  2829. }
  2830. l = pow2floor(l);
  2831. return l;
  2832. }
  2833. static bool prepare_mmio_access(MemoryRegion *mr)
  2834. {
  2835. bool unlocked = !qemu_mutex_iothread_locked();
  2836. bool release_lock = false;
  2837. if (unlocked && mr->global_locking) {
  2838. qemu_mutex_lock_iothread();
  2839. unlocked = false;
  2840. release_lock = true;
  2841. }
  2842. if (mr->flush_coalesced_mmio) {
  2843. if (unlocked) {
  2844. qemu_mutex_lock_iothread();
  2845. }
  2846. qemu_flush_coalesced_mmio_buffer();
  2847. if (unlocked) {
  2848. qemu_mutex_unlock_iothread();
  2849. }
  2850. }
  2851. return release_lock;
  2852. }
  2853. /* Called within RCU critical section. */
  2854. static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr,
  2855. MemTxAttrs attrs,
  2856. const uint8_t *buf,
  2857. hwaddr len, hwaddr addr1,
  2858. hwaddr l, MemoryRegion *mr)
  2859. {
  2860. uint8_t *ptr;
  2861. uint64_t val;
  2862. MemTxResult result = MEMTX_OK;
  2863. bool release_lock = false;
  2864. for (;;) {
  2865. if (!memory_access_is_direct(mr, true)) {
  2866. release_lock |= prepare_mmio_access(mr);
  2867. l = memory_access_size(mr, l, addr1);
  2868. /* XXX: could force current_cpu to NULL to avoid
  2869. potential bugs */
  2870. val = ldn_p(buf, l);
  2871. result |= memory_region_dispatch_write(mr, addr1, val, l, attrs);
  2872. } else {
  2873. /* RAM case */
  2874. ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false);
  2875. memcpy(ptr, buf, l);
  2876. invalidate_and_set_dirty(mr, addr1, l);
  2877. }
  2878. if (release_lock) {
  2879. qemu_mutex_unlock_iothread();
  2880. release_lock = false;
  2881. }
  2882. len -= l;
  2883. buf += l;
  2884. addr += l;
  2885. if (!len) {
  2886. break;
  2887. }
  2888. l = len;
  2889. mr = flatview_translate(fv, addr, &addr1, &l, true, attrs);
  2890. }
  2891. return result;
  2892. }
  2893. /* Called from RCU critical section. */
  2894. static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
  2895. const uint8_t *buf, hwaddr len)
  2896. {
  2897. hwaddr l;
  2898. hwaddr addr1;
  2899. MemoryRegion *mr;
  2900. MemTxResult result = MEMTX_OK;
  2901. l = len;
  2902. mr = flatview_translate(fv, addr, &addr1, &l, true, attrs);
  2903. result = flatview_write_continue(fv, addr, attrs, buf, len,
  2904. addr1, l, mr);
  2905. return result;
  2906. }
  2907. /* Called within RCU critical section. */
  2908. MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr,
  2909. MemTxAttrs attrs, uint8_t *buf,
  2910. hwaddr len, hwaddr addr1, hwaddr l,
  2911. MemoryRegion *mr)
  2912. {
  2913. uint8_t *ptr;
  2914. uint64_t val;
  2915. MemTxResult result = MEMTX_OK;
  2916. bool release_lock = false;
  2917. for (;;) {
  2918. if (!memory_access_is_direct(mr, false)) {
  2919. /* I/O case */
  2920. release_lock |= prepare_mmio_access(mr);
  2921. l = memory_access_size(mr, l, addr1);
  2922. result |= memory_region_dispatch_read(mr, addr1, &val, l, attrs);
  2923. stn_p(buf, l, val);
  2924. } else {
  2925. /* RAM case */
  2926. ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false);
  2927. memcpy(buf, ptr, l);
  2928. }
  2929. if (release_lock) {
  2930. qemu_mutex_unlock_iothread();
  2931. release_lock = false;
  2932. }
  2933. len -= l;
  2934. buf += l;
  2935. addr += l;
  2936. if (!len) {
  2937. break;
  2938. }
  2939. l = len;
  2940. mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
  2941. }
  2942. return result;
  2943. }
  2944. /* Called from RCU critical section. */
  2945. static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
  2946. MemTxAttrs attrs, uint8_t *buf, hwaddr len)
  2947. {
  2948. hwaddr l;
  2949. hwaddr addr1;
  2950. MemoryRegion *mr;
  2951. l = len;
  2952. mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
  2953. return flatview_read_continue(fv, addr, attrs, buf, len,
  2954. addr1, l, mr);
  2955. }
  2956. MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
  2957. MemTxAttrs attrs, uint8_t *buf, hwaddr len)
  2958. {
  2959. MemTxResult result = MEMTX_OK;
  2960. FlatView *fv;
  2961. if (len > 0) {
  2962. rcu_read_lock();
  2963. fv = address_space_to_flatview(as);
  2964. result = flatview_read(fv, addr, attrs, buf, len);
  2965. rcu_read_unlock();
  2966. }
  2967. return result;
  2968. }
  2969. MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
  2970. MemTxAttrs attrs,
  2971. const uint8_t *buf, hwaddr len)
  2972. {
  2973. MemTxResult result = MEMTX_OK;
  2974. FlatView *fv;
  2975. if (len > 0) {
  2976. rcu_read_lock();
  2977. fv = address_space_to_flatview(as);
  2978. result = flatview_write(fv, addr, attrs, buf, len);
  2979. rcu_read_unlock();
  2980. }
  2981. return result;
  2982. }
  2983. MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
  2984. uint8_t *buf, hwaddr len, bool is_write)
  2985. {
  2986. if (is_write) {
  2987. return address_space_write(as, addr, attrs, buf, len);
  2988. } else {
  2989. return address_space_read_full(as, addr, attrs, buf, len);
  2990. }
  2991. }
  2992. void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
  2993. hwaddr len, int is_write)
  2994. {
  2995. address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
  2996. buf, len, is_write);
  2997. }
  2998. enum write_rom_type {
  2999. WRITE_DATA,
  3000. FLUSH_CACHE,
  3001. };
  3002. static inline MemTxResult address_space_write_rom_internal(AddressSpace *as,
  3003. hwaddr addr,
  3004. MemTxAttrs attrs,
  3005. const uint8_t *buf,
  3006. hwaddr len,
  3007. enum write_rom_type type)
  3008. {
  3009. hwaddr l;
  3010. uint8_t *ptr;
  3011. hwaddr addr1;
  3012. MemoryRegion *mr;
  3013. rcu_read_lock();
  3014. while (len > 0) {
  3015. l = len;
  3016. mr = address_space_translate(as, addr, &addr1, &l, true, attrs);
  3017. if (!(memory_region_is_ram(mr) ||
  3018. memory_region_is_romd(mr))) {
  3019. l = memory_access_size(mr, l, addr1);
  3020. } else {
  3021. /* ROM/RAM case */
  3022. ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
  3023. switch (type) {
  3024. case WRITE_DATA:
  3025. memcpy(ptr, buf, l);
  3026. invalidate_and_set_dirty(mr, addr1, l);
  3027. break;
  3028. case FLUSH_CACHE:
  3029. flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
  3030. break;
  3031. }
  3032. }
  3033. len -= l;
  3034. buf += l;
  3035. addr += l;
  3036. }
  3037. rcu_read_unlock();
  3038. return MEMTX_OK;
  3039. }
  3040. /* used for ROM loading : can write in RAM and ROM */
  3041. MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr,
  3042. MemTxAttrs attrs,
  3043. const uint8_t *buf, hwaddr len)
  3044. {
  3045. return address_space_write_rom_internal(as, addr, attrs,
  3046. buf, len, WRITE_DATA);
  3047. }
  3048. void cpu_flush_icache_range(hwaddr start, hwaddr len)
  3049. {
  3050. /*
  3051. * This function should do the same thing as an icache flush that was
  3052. * triggered from within the guest. For TCG we are always cache coherent,
  3053. * so there is no need to flush anything. For KVM / Xen we need to flush
  3054. * the host's instruction cache at least.
  3055. */
  3056. if (tcg_enabled()) {
  3057. return;
  3058. }
  3059. address_space_write_rom_internal(&address_space_memory,
  3060. start, MEMTXATTRS_UNSPECIFIED,
  3061. NULL, len, FLUSH_CACHE);
  3062. }
  3063. typedef struct {
  3064. MemoryRegion *mr;
  3065. void *buffer;
  3066. hwaddr addr;
  3067. hwaddr len;
  3068. bool in_use;
  3069. } BounceBuffer;
  3070. static BounceBuffer bounce;
  3071. typedef struct MapClient {
  3072. QEMUBH *bh;
  3073. QLIST_ENTRY(MapClient) link;
  3074. } MapClient;
  3075. QemuMutex map_client_list_lock;
  3076. static QLIST_HEAD(, MapClient) map_client_list
  3077. = QLIST_HEAD_INITIALIZER(map_client_list);
  3078. static void cpu_unregister_map_client_do(MapClient *client)
  3079. {
  3080. QLIST_REMOVE(client, link);
  3081. g_free(client);
  3082. }
  3083. static void cpu_notify_map_clients_locked(void)
  3084. {
  3085. MapClient *client;
  3086. while (!QLIST_EMPTY(&map_client_list)) {
  3087. client = QLIST_FIRST(&map_client_list);
  3088. qemu_bh_schedule(client->bh);
  3089. cpu_unregister_map_client_do(client);
  3090. }
  3091. }
  3092. void cpu_register_map_client(QEMUBH *bh)
  3093. {
  3094. MapClient *client = g_malloc(sizeof(*client));
  3095. qemu_mutex_lock(&map_client_list_lock);
  3096. client->bh = bh;
  3097. QLIST_INSERT_HEAD(&map_client_list, client, link);
  3098. if (!atomic_read(&bounce.in_use)) {
  3099. cpu_notify_map_clients_locked();
  3100. }
  3101. qemu_mutex_unlock(&map_client_list_lock);
  3102. }
  3103. void cpu_exec_init_all(void)
  3104. {
  3105. qemu_mutex_init(&ram_list.mutex);
  3106. /* The data structures we set up here depend on knowing the page size,
  3107. * so no more changes can be made after this point.
  3108. * In an ideal world, nothing we did before we had finished the
  3109. * machine setup would care about the target page size, and we could
  3110. * do this much later, rather than requiring board models to state
  3111. * up front what their requirements are.
  3112. */
  3113. finalize_target_page_bits();
  3114. io_mem_init();
  3115. memory_map_init();
  3116. qemu_mutex_init(&map_client_list_lock);
  3117. }
  3118. void cpu_unregister_map_client(QEMUBH *bh)
  3119. {
  3120. MapClient *client;
  3121. qemu_mutex_lock(&map_client_list_lock);
  3122. QLIST_FOREACH(client, &map_client_list, link) {
  3123. if (client->bh == bh) {
  3124. cpu_unregister_map_client_do(client);
  3125. break;
  3126. }
  3127. }
  3128. qemu_mutex_unlock(&map_client_list_lock);
  3129. }
  3130. static void cpu_notify_map_clients(void)
  3131. {
  3132. qemu_mutex_lock(&map_client_list_lock);
  3133. cpu_notify_map_clients_locked();
  3134. qemu_mutex_unlock(&map_client_list_lock);
  3135. }
  3136. static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
  3137. bool is_write, MemTxAttrs attrs)
  3138. {
  3139. MemoryRegion *mr;
  3140. hwaddr l, xlat;
  3141. while (len > 0) {
  3142. l = len;
  3143. mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
  3144. if (!memory_access_is_direct(mr, is_write)) {
  3145. l = memory_access_size(mr, l, addr);
  3146. if (!memory_region_access_valid(mr, xlat, l, is_write, attrs)) {
  3147. return false;
  3148. }
  3149. }
  3150. len -= l;
  3151. addr += l;
  3152. }
  3153. return true;
  3154. }
  3155. bool address_space_access_valid(AddressSpace *as, hwaddr addr,
  3156. hwaddr len, bool is_write,
  3157. MemTxAttrs attrs)
  3158. {
  3159. FlatView *fv;
  3160. bool result;
  3161. rcu_read_lock();
  3162. fv = address_space_to_flatview(as);
  3163. result = flatview_access_valid(fv, addr, len, is_write, attrs);
  3164. rcu_read_unlock();
  3165. return result;
  3166. }
  3167. static hwaddr
  3168. flatview_extend_translation(FlatView *fv, hwaddr addr,
  3169. hwaddr target_len,
  3170. MemoryRegion *mr, hwaddr base, hwaddr len,
  3171. bool is_write, MemTxAttrs attrs)
  3172. {
  3173. hwaddr done = 0;
  3174. hwaddr xlat;
  3175. MemoryRegion *this_mr;
  3176. for (;;) {
  3177. target_len -= len;
  3178. addr += len;
  3179. done += len;
  3180. if (target_len == 0) {
  3181. return done;
  3182. }
  3183. len = target_len;
  3184. this_mr = flatview_translate(fv, addr, &xlat,
  3185. &len, is_write, attrs);
  3186. if (this_mr != mr || xlat != base + done) {
  3187. return done;
  3188. }
  3189. }
  3190. }
  3191. /* Map a physical memory region into a host virtual address.
  3192. * May map a subset of the requested range, given by and returned in *plen.
  3193. * May return NULL if resources needed to perform the mapping are exhausted.
  3194. * Use only for reads OR writes - not for read-modify-write operations.
  3195. * Use cpu_register_map_client() to know when retrying the map operation is
  3196. * likely to succeed.
  3197. */
  3198. void *address_space_map(AddressSpace *as,
  3199. hwaddr addr,
  3200. hwaddr *plen,
  3201. bool is_write,
  3202. MemTxAttrs attrs)
  3203. {
  3204. hwaddr len = *plen;
  3205. hwaddr l, xlat;
  3206. MemoryRegion *mr;
  3207. void *ptr;
  3208. FlatView *fv;
  3209. if (len == 0) {
  3210. return NULL;
  3211. }
  3212. l = len;
  3213. rcu_read_lock();
  3214. fv = address_space_to_flatview(as);
  3215. mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
  3216. if (!memory_access_is_direct(mr, is_write)) {
  3217. if (atomic_xchg(&bounce.in_use, true)) {
  3218. rcu_read_unlock();
  3219. return NULL;
  3220. }
  3221. /* Avoid unbounded allocations */
  3222. l = MIN(l, TARGET_PAGE_SIZE);
  3223. bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
  3224. bounce.addr = addr;
  3225. bounce.len = l;
  3226. memory_region_ref(mr);
  3227. bounce.mr = mr;
  3228. if (!is_write) {
  3229. flatview_read(fv, addr, MEMTXATTRS_UNSPECIFIED,
  3230. bounce.buffer, l);
  3231. }
  3232. rcu_read_unlock();
  3233. *plen = l;
  3234. return bounce.buffer;
  3235. }
  3236. memory_region_ref(mr);
  3237. *plen = flatview_extend_translation(fv, addr, len, mr, xlat,
  3238. l, is_write, attrs);
  3239. ptr = qemu_ram_ptr_length(mr->ram_block, xlat, plen, true);
  3240. rcu_read_unlock();
  3241. return ptr;
  3242. }
  3243. /* Unmaps a memory region previously mapped by address_space_map().
  3244. * Will also mark the memory as dirty if is_write == 1. access_len gives
  3245. * the amount of memory that was actually read or written by the caller.
  3246. */
  3247. void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
  3248. int is_write, hwaddr access_len)
  3249. {
  3250. if (buffer != bounce.buffer) {
  3251. MemoryRegion *mr;
  3252. ram_addr_t addr1;
  3253. mr = memory_region_from_host(buffer, &addr1);
  3254. assert(mr != NULL);
  3255. if (is_write) {
  3256. invalidate_and_set_dirty(mr, addr1, access_len);
  3257. }
  3258. if (xen_enabled()) {
  3259. xen_invalidate_map_cache_entry(buffer);
  3260. }
  3261. memory_region_unref(mr);
  3262. return;
  3263. }
  3264. if (is_write) {
  3265. address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
  3266. bounce.buffer, access_len);
  3267. }
  3268. qemu_vfree(bounce.buffer);
  3269. bounce.buffer = NULL;
  3270. memory_region_unref(bounce.mr);
  3271. atomic_mb_set(&bounce.in_use, false);
  3272. cpu_notify_map_clients();
  3273. }
  3274. void *cpu_physical_memory_map(hwaddr addr,
  3275. hwaddr *plen,
  3276. int is_write)
  3277. {
  3278. return address_space_map(&address_space_memory, addr, plen, is_write,
  3279. MEMTXATTRS_UNSPECIFIED);
  3280. }
  3281. void cpu_physical_memory_unmap(void *buffer, hwaddr len,
  3282. int is_write, hwaddr access_len)
  3283. {
  3284. return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
  3285. }
  3286. #define ARG1_DECL AddressSpace *as
  3287. #define ARG1 as
  3288. #define SUFFIX
  3289. #define TRANSLATE(...) address_space_translate(as, __VA_ARGS__)
  3290. #define RCU_READ_LOCK(...) rcu_read_lock()
  3291. #define RCU_READ_UNLOCK(...) rcu_read_unlock()
  3292. #include "memory_ldst.inc.c"
  3293. int64_t address_space_cache_init(MemoryRegionCache *cache,
  3294. AddressSpace *as,
  3295. hwaddr addr,
  3296. hwaddr len,
  3297. bool is_write)
  3298. {
  3299. AddressSpaceDispatch *d;
  3300. hwaddr l;
  3301. MemoryRegion *mr;
  3302. assert(len > 0);
  3303. l = len;
  3304. cache->fv = address_space_get_flatview(as);
  3305. d = flatview_to_dispatch(cache->fv);
  3306. cache->mrs = *address_space_translate_internal(d, addr, &cache->xlat, &l, true);
  3307. mr = cache->mrs.mr;
  3308. memory_region_ref(mr);
  3309. if (memory_access_is_direct(mr, is_write)) {
  3310. /* We don't care about the memory attributes here as we're only
  3311. * doing this if we found actual RAM, which behaves the same
  3312. * regardless of attributes; so UNSPECIFIED is fine.
  3313. */
  3314. l = flatview_extend_translation(cache->fv, addr, len, mr,
  3315. cache->xlat, l, is_write,
  3316. MEMTXATTRS_UNSPECIFIED);
  3317. cache->ptr = qemu_ram_ptr_length(mr->ram_block, cache->xlat, &l, true);
  3318. } else {
  3319. cache->ptr = NULL;
  3320. }
  3321. cache->len = l;
  3322. cache->is_write = is_write;
  3323. return l;
  3324. }
  3325. void address_space_cache_invalidate(MemoryRegionCache *cache,
  3326. hwaddr addr,
  3327. hwaddr access_len)
  3328. {
  3329. assert(cache->is_write);
  3330. if (likely(cache->ptr)) {
  3331. invalidate_and_set_dirty(cache->mrs.mr, addr + cache->xlat, access_len);
  3332. }
  3333. }
  3334. void address_space_cache_destroy(MemoryRegionCache *cache)
  3335. {
  3336. if (!cache->mrs.mr) {
  3337. return;
  3338. }
  3339. if (xen_enabled()) {
  3340. xen_invalidate_map_cache_entry(cache->ptr);
  3341. }
  3342. memory_region_unref(cache->mrs.mr);
  3343. flatview_unref(cache->fv);
  3344. cache->mrs.mr = NULL;
  3345. cache->fv = NULL;
  3346. }
  3347. /* Called from RCU critical section. This function has the same
  3348. * semantics as address_space_translate, but it only works on a
  3349. * predefined range of a MemoryRegion that was mapped with
  3350. * address_space_cache_init.
  3351. */
  3352. static inline MemoryRegion *address_space_translate_cached(
  3353. MemoryRegionCache *cache, hwaddr addr, hwaddr *xlat,
  3354. hwaddr *plen, bool is_write, MemTxAttrs attrs)
  3355. {
  3356. MemoryRegionSection section;
  3357. MemoryRegion *mr;
  3358. IOMMUMemoryRegion *iommu_mr;
  3359. AddressSpace *target_as;
  3360. assert(!cache->ptr);
  3361. *xlat = addr + cache->xlat;
  3362. mr = cache->mrs.mr;
  3363. iommu_mr = memory_region_get_iommu(mr);
  3364. if (!iommu_mr) {
  3365. /* MMIO region. */
  3366. return mr;
  3367. }
  3368. section = address_space_translate_iommu(iommu_mr, xlat, plen,
  3369. NULL, is_write, true,
  3370. &target_as, attrs);
  3371. return section.mr;
  3372. }
  3373. /* Called from RCU critical section. address_space_read_cached uses this
  3374. * out of line function when the target is an MMIO or IOMMU region.
  3375. */
  3376. void
  3377. address_space_read_cached_slow(MemoryRegionCache *cache, hwaddr addr,
  3378. void *buf, hwaddr len)
  3379. {
  3380. hwaddr addr1, l;
  3381. MemoryRegion *mr;
  3382. l = len;
  3383. mr = address_space_translate_cached(cache, addr, &addr1, &l, false,
  3384. MEMTXATTRS_UNSPECIFIED);
  3385. flatview_read_continue(cache->fv,
  3386. addr, MEMTXATTRS_UNSPECIFIED, buf, len,
  3387. addr1, l, mr);
  3388. }
  3389. /* Called from RCU critical section. address_space_write_cached uses this
  3390. * out of line function when the target is an MMIO or IOMMU region.
  3391. */
  3392. void
  3393. address_space_write_cached_slow(MemoryRegionCache *cache, hwaddr addr,
  3394. const void *buf, hwaddr len)
  3395. {
  3396. hwaddr addr1, l;
  3397. MemoryRegion *mr;
  3398. l = len;
  3399. mr = address_space_translate_cached(cache, addr, &addr1, &l, true,
  3400. MEMTXATTRS_UNSPECIFIED);
  3401. flatview_write_continue(cache->fv,
  3402. addr, MEMTXATTRS_UNSPECIFIED, buf, len,
  3403. addr1, l, mr);
  3404. }
  3405. #define ARG1_DECL MemoryRegionCache *cache
  3406. #define ARG1 cache
  3407. #define SUFFIX _cached_slow
  3408. #define TRANSLATE(...) address_space_translate_cached(cache, __VA_ARGS__)
  3409. #define RCU_READ_LOCK() ((void)0)
  3410. #define RCU_READ_UNLOCK() ((void)0)
  3411. #include "memory_ldst.inc.c"
  3412. /* virtual memory access for debug (includes writing to ROM) */
  3413. int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
  3414. uint8_t *buf, target_ulong len, int is_write)
  3415. {
  3416. hwaddr phys_addr;
  3417. target_ulong l, page;
  3418. cpu_synchronize_state(cpu);
  3419. while (len > 0) {
  3420. int asidx;
  3421. MemTxAttrs attrs;
  3422. page = addr & TARGET_PAGE_MASK;
  3423. phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
  3424. asidx = cpu_asidx_from_attrs(cpu, attrs);
  3425. /* if no physical page mapped, return an error */
  3426. if (phys_addr == -1)
  3427. return -1;
  3428. l = (page + TARGET_PAGE_SIZE) - addr;
  3429. if (l > len)
  3430. l = len;
  3431. phys_addr += (addr & ~TARGET_PAGE_MASK);
  3432. if (is_write) {
  3433. address_space_write_rom(cpu->cpu_ases[asidx].as, phys_addr,
  3434. attrs, buf, l);
  3435. } else {
  3436. address_space_rw(cpu->cpu_ases[asidx].as, phys_addr,
  3437. attrs, buf, l, 0);
  3438. }
  3439. len -= l;
  3440. buf += l;
  3441. addr += l;
  3442. }
  3443. return 0;
  3444. }
  3445. /*
  3446. * Allows code that needs to deal with migration bitmaps etc to still be built
  3447. * target independent.
  3448. */
  3449. size_t qemu_target_page_size(void)
  3450. {
  3451. return TARGET_PAGE_SIZE;
  3452. }
  3453. int qemu_target_page_bits(void)
  3454. {
  3455. return TARGET_PAGE_BITS;
  3456. }
  3457. int qemu_target_page_bits_min(void)
  3458. {
  3459. return TARGET_PAGE_BITS_MIN;
  3460. }
  3461. #endif
  3462. bool target_words_bigendian(void)
  3463. {
  3464. #if defined(TARGET_WORDS_BIGENDIAN)
  3465. return true;
  3466. #else
  3467. return false;
  3468. #endif
  3469. }
  3470. #ifndef CONFIG_USER_ONLY
  3471. bool cpu_physical_memory_is_io(hwaddr phys_addr)
  3472. {
  3473. MemoryRegion*mr;
  3474. hwaddr l = 1;
  3475. bool res;
  3476. rcu_read_lock();
  3477. mr = address_space_translate(&address_space_memory,
  3478. phys_addr, &phys_addr, &l, false,
  3479. MEMTXATTRS_UNSPECIFIED);
  3480. res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
  3481. rcu_read_unlock();
  3482. return res;
  3483. }
  3484. int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
  3485. {
  3486. RAMBlock *block;
  3487. int ret = 0;
  3488. rcu_read_lock();
  3489. RAMBLOCK_FOREACH(block) {
  3490. ret = func(block, opaque);
  3491. if (ret) {
  3492. break;
  3493. }
  3494. }
  3495. rcu_read_unlock();
  3496. return ret;
  3497. }
  3498. /*
  3499. * Unmap pages of memory from start to start+length such that
  3500. * they a) read as 0, b) Trigger whatever fault mechanism
  3501. * the OS provides for postcopy.
  3502. * The pages must be unmapped by the end of the function.
  3503. * Returns: 0 on success, none-0 on failure
  3504. *
  3505. */
  3506. int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length)
  3507. {
  3508. int ret = -1;
  3509. uint8_t *host_startaddr = rb->host + start;
  3510. if ((uintptr_t)host_startaddr & (rb->page_size - 1)) {
  3511. error_report("ram_block_discard_range: Unaligned start address: %p",
  3512. host_startaddr);
  3513. goto err;
  3514. }
  3515. if ((start + length) <= rb->used_length) {
  3516. bool need_madvise, need_fallocate;
  3517. uint8_t *host_endaddr = host_startaddr + length;
  3518. if ((uintptr_t)host_endaddr & (rb->page_size - 1)) {
  3519. error_report("ram_block_discard_range: Unaligned end address: %p",
  3520. host_endaddr);
  3521. goto err;
  3522. }
  3523. errno = ENOTSUP; /* If we are missing MADVISE etc */
  3524. /* The logic here is messy;
  3525. * madvise DONTNEED fails for hugepages
  3526. * fallocate works on hugepages and shmem
  3527. */
  3528. need_madvise = (rb->page_size == qemu_host_page_size);
  3529. need_fallocate = rb->fd != -1;
  3530. if (need_fallocate) {
  3531. /* For a file, this causes the area of the file to be zero'd
  3532. * if read, and for hugetlbfs also causes it to be unmapped
  3533. * so a userfault will trigger.
  3534. */
  3535. #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
  3536. ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
  3537. start, length);
  3538. if (ret) {
  3539. ret = -errno;
  3540. error_report("ram_block_discard_range: Failed to fallocate "
  3541. "%s:%" PRIx64 " +%zx (%d)",
  3542. rb->idstr, start, length, ret);
  3543. goto err;
  3544. }
  3545. #else
  3546. ret = -ENOSYS;
  3547. error_report("ram_block_discard_range: fallocate not available/file"
  3548. "%s:%" PRIx64 " +%zx (%d)",
  3549. rb->idstr, start, length, ret);
  3550. goto err;
  3551. #endif
  3552. }
  3553. if (need_madvise) {
  3554. /* For normal RAM this causes it to be unmapped,
  3555. * for shared memory it causes the local mapping to disappear
  3556. * and to fall back on the file contents (which we just
  3557. * fallocate'd away).
  3558. */
  3559. #if defined(CONFIG_MADVISE)
  3560. ret = madvise(host_startaddr, length, MADV_DONTNEED);
  3561. if (ret) {
  3562. ret = -errno;
  3563. error_report("ram_block_discard_range: Failed to discard range "
  3564. "%s:%" PRIx64 " +%zx (%d)",
  3565. rb->idstr, start, length, ret);
  3566. goto err;
  3567. }
  3568. #else
  3569. ret = -ENOSYS;
  3570. error_report("ram_block_discard_range: MADVISE not available"
  3571. "%s:%" PRIx64 " +%zx (%d)",
  3572. rb->idstr, start, length, ret);
  3573. goto err;
  3574. #endif
  3575. }
  3576. trace_ram_block_discard_range(rb->idstr, host_startaddr, length,
  3577. need_madvise, need_fallocate, ret);
  3578. } else {
  3579. error_report("ram_block_discard_range: Overrun block '%s' (%" PRIu64
  3580. "/%zx/" RAM_ADDR_FMT")",
  3581. rb->idstr, start, length, rb->used_length);
  3582. }
  3583. err:
  3584. return ret;
  3585. }
  3586. bool ramblock_is_pmem(RAMBlock *rb)
  3587. {
  3588. return rb->flags & RAM_PMEM;
  3589. }
  3590. #endif
  3591. void page_size_init(void)
  3592. {
  3593. /* NOTE: we can always suppose that qemu_host_page_size >=
  3594. TARGET_PAGE_SIZE */
  3595. if (qemu_host_page_size == 0) {
  3596. qemu_host_page_size = qemu_real_host_page_size;
  3597. }
  3598. if (qemu_host_page_size < TARGET_PAGE_SIZE) {
  3599. qemu_host_page_size = TARGET_PAGE_SIZE;
  3600. }
  3601. qemu_host_page_mask = -(intptr_t)qemu_host_page_size;
  3602. }
  3603. #if !defined(CONFIG_USER_ONLY)
  3604. static void mtree_print_phys_entries(int start, int end, int skip, int ptr)
  3605. {
  3606. if (start == end - 1) {
  3607. qemu_printf("\t%3d ", start);
  3608. } else {
  3609. qemu_printf("\t%3d..%-3d ", start, end - 1);
  3610. }
  3611. qemu_printf(" skip=%d ", skip);
  3612. if (ptr == PHYS_MAP_NODE_NIL) {
  3613. qemu_printf(" ptr=NIL");
  3614. } else if (!skip) {
  3615. qemu_printf(" ptr=#%d", ptr);
  3616. } else {
  3617. qemu_printf(" ptr=[%d]", ptr);
  3618. }
  3619. qemu_printf("\n");
  3620. }
  3621. #define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \
  3622. int128_sub((size), int128_one())) : 0)
  3623. void mtree_print_dispatch(AddressSpaceDispatch *d, MemoryRegion *root)
  3624. {
  3625. int i;
  3626. qemu_printf(" Dispatch\n");
  3627. qemu_printf(" Physical sections\n");
  3628. for (i = 0; i < d->map.sections_nb; ++i) {
  3629. MemoryRegionSection *s = d->map.sections + i;
  3630. const char *names[] = { " [unassigned]", " [not dirty]",
  3631. " [ROM]", " [watch]" };
  3632. qemu_printf(" #%d @" TARGET_FMT_plx ".." TARGET_FMT_plx
  3633. " %s%s%s%s%s",
  3634. i,
  3635. s->offset_within_address_space,
  3636. s->offset_within_address_space + MR_SIZE(s->mr->size),
  3637. s->mr->name ? s->mr->name : "(noname)",
  3638. i < ARRAY_SIZE(names) ? names[i] : "",
  3639. s->mr == root ? " [ROOT]" : "",
  3640. s == d->mru_section ? " [MRU]" : "",
  3641. s->mr->is_iommu ? " [iommu]" : "");
  3642. if (s->mr->alias) {
  3643. qemu_printf(" alias=%s", s->mr->alias->name ?
  3644. s->mr->alias->name : "noname");
  3645. }
  3646. qemu_printf("\n");
  3647. }
  3648. qemu_printf(" Nodes (%d bits per level, %d levels) ptr=[%d] skip=%d\n",
  3649. P_L2_BITS, P_L2_LEVELS, d->phys_map.ptr, d->phys_map.skip);
  3650. for (i = 0; i < d->map.nodes_nb; ++i) {
  3651. int j, jprev;
  3652. PhysPageEntry prev;
  3653. Node *n = d->map.nodes + i;
  3654. qemu_printf(" [%d]\n", i);
  3655. for (j = 0, jprev = 0, prev = *n[0]; j < ARRAY_SIZE(*n); ++j) {
  3656. PhysPageEntry *pe = *n + j;
  3657. if (pe->ptr == prev.ptr && pe->skip == prev.skip) {
  3658. continue;
  3659. }
  3660. mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
  3661. jprev = j;
  3662. prev = *pe;
  3663. }
  3664. if (jprev != ARRAY_SIZE(*n)) {
  3665. mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
  3666. }
  3667. }
  3668. }
  3669. #endif