aboutsummaryrefslogtreecommitdiff
path: root/opcodes/ChangeLog-2008
blob: bb9316e73a5687df37a779c97b235cb97857c538 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
2008-12-30  Martin Schwidefsky  <schwidefskyy@de.ibm.com>

	* s390-opc.txt: Add ptff instruction.

2008-12-24  Jan Kratochvil  <jan.kratochvil@redhat.com>

	* Makefile.am (CFILES, ALL_MACHINES): Add LM32 source and object files.
	* Makefile.in: Regenerate.

2008-12-23  Jon Beniston <jon@beniston.com>

	* Makefile.am: Add LM32 object files and dependencies.
	* Makefile.in: Regenerate.
	* configure.in: Add LM32 target.
	* configure: Regenerate.
	* disassemble.c: Add LM32 disassembler.
	* cgen-asm.in: Update copyright year.
	* cgen-dis.in: Update copyright year.
	* cgen-ibld.in: Update copyright year.
	* lm32-asm.c: New file.
	* lm32-desc.c: New file.
	* lm32-desc.h: New file.
	* lm32-dis.c: New file.
	* lm32-ibld.c: New file.
	* lm32-opc.c: New file.
	* lm32-opc.h: New file.
	* lm32-opinst.c: New file.

2008-12-23  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-dis.c (EXdS): New.
	(EXdVexS): Likewise.
	(EXqVexS): Likewise.
	(d_swap_mode): Likewise.
	(q_mode): Updated.
	(prefix_table): Use EXdS on movss and EXqS on movsd.
	(vex_len_table): Use EXdVexS on vmovss and EXqVexS on vmovsd.
	(intel_operand_size): Handle d_swap_mode.
	(OP_EX): Likewise.

	* i386-opc.h (S): Update comments.

	* i386-opc.tbl: Add S to movss, movsd, vmovss and vmovsd.
	* i386-tbl.h: Regenerated.

2008-12-23  Nick Clifton  <nickc@redhat.com>

	* po/ga.po: Updated Irish translation.

2008-12-20  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-dis.c (EbS): New.
	(EvS): Likewise.
	(EMS): Likewise.
	(EXqS): Likewise.
	(EXxS): Likewise.
	(b_swap_mode): Likewise.
	(v_swap_mode): Likewise.
	(q_swap_mode): Likewise.
	(x_swap_mode): Likewise.
	(v_mode): Updated.
	(w_mode): Likewise.
	(t_mode): Likewise.
	(xmm_mode): Likewise.
	(swap_operand): Likewise.
	(dis386): Use EbS on movB.  Use EvS on moveS.
	(dis386_twobyte): Use EXxS on movapX.
	(prefix_table): Use EXxS on movups, movupd, movdqu, movdqa,
	vmovups, vmovdqu, vmovdqa. Use EMS and EXqS on movq.
	(vex_table): Use EXxS on vmovapX.
	(vex_len_table): Use EXqS on vmovq.
	(intel_operand_size): Handle b_swap_mode, v_swap_mode,
	q_swap_mode and x_swap_mode.
	(OP_E_register): Handle b_swap_mode and v_swap_mode.
	(OP_EM): Handle v_swap_mode.
	(OP_EX): x_swap_mode and q_swap_mode.

	* i386-gen.c (opcode_modifiers): Add S.

	* i386-opc.h (S): New.
	(Modrm): Updated.
	(i386_opcode_modifier): Add s.

	* i386-opc.tbl: Add S to movapd, movaps, movdqa, movdqu, movq,
	movupd, movups, vmovapd, vmovaps, vmovdqa, vmovdqu and vmovq.
	* i386-tbl.h: Regenerated.

2008-12-18  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-dis.c (mnemonicendp): New.
	(op): Likewise.
	(print_insn): Use mnemonicendp.
	(OP_3DNowSuffix): Likewise.
	(CMP_Fixup): Likewise.
	(CMPXCHG8B_Fixup): Likewise.
	(CRC32_Fixup): Likewise.
	(OP_DREX_FCMP): Likewise.
	(OP_DREX_ICMP): Likewise.
	(VZERO_Fixup): Likewise.
	(VCMP_Fixup): Likewise.
	(PCLMUL_Fixup): Likewise.
	(VPERMIL2_Fixup): Likewise.
	(MOVBE_Fixup): Likewise.
	(putop): Update mnemonicendp.
	(oappend): Use stpcpy.
	(simd_cmp_op): Changed to struct op.
	(vex_cmp_op): Likewise.
	(pclmul_op): Likewise.
	(vpermil2_op): Likewise.

2008-12-18  Ralf Wildenhues  <Ralf.Wildenhues@gmx.de>

	* configure: Regenerate.

2008-12-15  Richard Earnshaw  <rearnsha@arm.com>

	* arm-dis.c (coprocessor_opcodes): Disassemble VFP instructions using
	unified syntax.

2008-12-08  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c (opcode_modifiers): Move VexNDS before VexNDD.

2008-12-08  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-dis.c (putop): Remove strayed comments.

2008-12-04  Ben Elliston  <bje@au.ibm.com>

	* ppc-dis.c (powerpc_init_dialect): Do not set PPC_OPCODE_BOOKE
	for -Mbooke.
	(print_ppc_disassembler_options): Update usage.
	* ppc-opc.c (DE, DES, DEO, DE_MASK): Remove.
	(BOOKE64): Remove.
	(PPCCHLK64): Likewise.
	(powerpc_opcodes): Remove all BOOKE64 instructions.

2008-11-28  Joshua Kinard  <kumba@gentoo.org>

	* mips-dis.c (mips_arch_choices): Add r14000, r16000.

2008-11-27  M R Swami Reddy <MR.Swami.Reddy@nsc.com>

	* cr16-dis.c (match_opcode): Truncate mcode to 32 bit and
	adjusted the mask for 32-bit branch instruction.

2008-11-27  Alan Modra  <amodra@bigpond.net.au>

	* ppc-opc.c (extract_sprg): Correct operand range check.

2008-11-26  Andreas Schwab  <schwab@suse.de>

	* m68k-dis.c (NEXTBYTE, NEXTWORD, NEXTLONG, NEXTULONG, NEXTSINGLE)
	(NEXTDOUBLE, NEXTEXTEND, NEXTPACKED): Fix error handling.
	(save_printer, save_print_address): Remove.
	(fetch_data): Don't use them.
	(match_insn_m68k): Always restore printing functions.
	(print_insn_m68k): Don't save/restore printing functions.

2008-11-25  Nick Clifton  <nickc@redhat.com>

	* m68k-dis.c: Rewrite to remove use of setjmp/longjmp.

2008-11-18  Catherine Moore  <clm@codesourcery.com>

	* arm-dis.c (coprocessor_opcodes): Add half-precision vcvt
	instructions.
	(neon_opcodes): Likewise.
	(print_insn_coprocessor): Print 't' or 'b' for vcvt
	instructions.

2008-11-14  Tristan Gingold  <gingold@adacore.com>

	* makefile.vms (OBJS): Update list of objects.
	(DEFS): Update
	(CFLAGS): Update.

2008-11-06  Chao-ying Fu  <fu@mips.com>

	* mips-opc.c (synciobdma, syncs, syncw, syncws): Move these
	before sync.
	(sync): New instruction with 5-bit sync type.
	* mips-dis.c (print_insn_args): Add case '1' to print 5-bit values.

2008-11-06  Nick Clifton  <nickc@redhat.com>

	* avr-dis.c: Replace uses of sprintf without a format string with
	calls to strcpy.

2008-11-03  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-opc.tbl: Add cmovpe and cmovpo.
	* i386-tbl.h: Regenerated.

2008-10-22  Nick Clifton  <nickc@redhat.com>

	PR 6937
	* configure.in (SHARED_LIBADD): Revert previous change.
	Add a comment explaining why.
	(SHARED_DEPENDENCIES): Revert previous change.
	* configure: Regenerate.

2008-10-10  Nick Clifton  <nickc@redhat.com>

	PR 6937
	* configure.in (SHARED_LIBADD): Add libiberty.a.
	(SHARED_DEPENDENCIES): Add libiberty.a.

2008-09-30  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c: Include "hashtab.h".
	(next_field): Take a new argument, last.  Check last.
	(process_i386_cpu_flag): Updated.
	(process_i386_opcode_modifier): Likewise.
	(process_i386_operand_type): Likewise.
	(process_i386_registers): Likewise.
	(output_i386_opcode): New.
	(opcode_hash_entry): Likewise.
	(opcode_hash_table): Likewise.
	(opcode_hash_hash): Likewise.
	(opcode_hash_eq): Likewise.
	(process_i386_opcodes): Use opcode hash table and opcode array.

2008-09-30  Andreas Krebbel  <Andreas.Krebbel@de.ibm.com>

	* s390-opc.txt (stdy, stey): Fix description

2008-09-30  Alan Modra  <amodra@bigpond.net.au>

	* Makefile.am: Run "make dep-am".
	* Makefile.in: Regenerate.

2008-09-29  H.J. Lu  <hongjiu.lu@intel.com>

	* aclocal.m4: Regenerated.
	* configure: Likewise.
	* Makefile.in: Likewise.

2008-09-29  Nick Clifton  <nickc@redhat.com>

	* po/vi.po: Updated Vietnamese translation.
	* po/fr.po: Updated French translation.

2008-09-26  Florian Krohm  <fkrohm@us.ibm.com>

	* s390-opc.txt (thder, thdr): Change RRE_RR to RRE_FF.
	(cfxr, cfdr, cfer, clclu): Add esa flag.
	(sqd): Instruction added.
	(qadtr, qaxtr): Change RRF_FFFU to RRF_FUFF.
	* s390-opc.c: (INSTR_RRF_FFFU, MASK_RRF_FFFU): Removed.

2008-09-14  Arnold Metselaar  <arnold.metselaar@planet.nl>

	* z80-dis.c (prt_rr_nn): Fix register pair for two byte opcodes.
	(tab_elt opc_ed): Add "ld r,a" and "ld r,a" instructions.

2008-09-11  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-opc.tbl: Fix memory operand size for cmpXXXs[sd].
	* i386-tbl.h: Regenerated.

2008-08-28  Jan Beulich  <jbeulich@novell.com>

	* i386-dis.c (dis386): Adjust far return mnemonics.
	* i386-opc.tbl: Add retf.
	* i386-tbl.h: Re-generate.

2008-08-28  Jan Beulich  <jbeulich@novell.com>

	* i386-dis.c (dis386_twobyte): Adjust cmovXX mnemonics.

2008-08-28  H.J. Lu  <hongjiu.lu@intel.com>

	* ia64-dis.c (print_insn_ia64): Handle cr.iib0 and cr.iib1.
	* ia64-gen.c (lookup_specifier): Likewise.

	* ia64-ic.tbl: Add support for cr.iib0 and cr.iib1.
	* ia64-raw.tbl: Likewise.
	* ia64-waw.tbl: Likewise.
	* ia64-asmtab.c: Regenerated.

2008-08-27  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-opc.tbl: Correct fidivr operand size.

	* i386-tbl.h: Regenerated.

2008-08-24  Alan Modra  <amodra@bigpond.net.au>

	* configure.in: Update a number of obsolete autoconf macros.
	* aclocal.m4: Regenerate.

2008-08-20  H.J. Lu  <hongjiu.lu@intel.com>

	AVX Programming Reference (August, 2008)
	* i386-dis.c (PREFIX_VEX_38DB): New.
	(PREFIX_VEX_38DC): Likewise.
	(PREFIX_VEX_38DD): Likewise.
	(PREFIX_VEX_38DE): Likewise.
	(PREFIX_VEX_38DF): Likewise.
	(PREFIX_VEX_3ADF): Likewise.
	(VEX_LEN_38DB_P_2): Likewise.
	(VEX_LEN_38DC_P_2): Likewise.
	(VEX_LEN_38DD_P_2): Likewise.
	(VEX_LEN_38DE_P_2): Likewise.
	(VEX_LEN_38DF_P_2): Likewise.
	(VEX_LEN_3ADF_P_2): Likewise.
	(PREFIX_VEX_3A04): Updated.
	(VEX_LEN_3A06_P_2): Likewise.
	(prefix_table): Add PREFIX_VEX_38DB, PREFIX_VEX_38DC,
	PREFIX_VEX_38DD, PREFIX_VEX_38DE and PREFIX_VEX_3ADF.
	(x86_64_table): Likewise.
	(vex_len_table): Add VEX_LEN_38DB_P_2, VEX_LEN_38DC_P_2,
	VEX_LEN_38DD_P_2, VEX_LEN_38DE_P_2, VEX_LEN_38DF_P_2 and
	VEX_LEN_3ADF_P_2.

	* i386-opc.tbl: Add AES + AVX instructions.
	* i386-init.h: Regenerated.
	* i386-tbl.h: Likewise.

2008-08-15  Andreas Krebbel  <Andreas.Krebbel@de.ibm.com>

	* s390-opc.c (INSTR_RRF_FFRU, MASK_RRF_FFRU): New instruction format.
	* s390-opc.txt (lxr, rrdtr, rrxtr): Fix instruction format.

2008-08-15  Alan Modra  <amodra@bigpond.net.au>

	PR 6526
	* configure.in: Invoke AC_USE_SYSTEM_EXTENSIONS.
	* Makefile.in: Regenerate.
	* aclocal.m4: Regenerate.
	* config.in: Regenerate.
	* configure: Regenerate.

2008-08-14  Sebastian Huber  <sebastian.huber@embedded-brains.de>

	PR 6825
	* ppc-opc.c (powerpc_opcodes): Enable rfci, mfpmr, mtpmr for e300.

2008-08-12  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-opc.tbl: Add syscall and sysret for Cpu64.

	* i386-tbl.h: Regenerated.

2008-08-04  Alan Modra  <amodra@bigpond.net.au>

	* Makefile.am (POTFILES.in): Set LC_ALL=C.
	* Makefile.in: Regenerate.
	* po/POTFILES.in: Regenerate.

2008-08-01  Peter Bergner  <bergner@vnet.ibm.com>

	* ppc-dis.c (powerpc_init_dialect): Handle power7 and vsx options.
	(print_insn_powerpc): Prepend 'vs' when printing VSX registers.
	(print_ppc_disassembler_options): Document -Mpower7 and -Mvsx.
	* ppc-opc.c (insert_xt6): New static function.
	(extract_xt6): Likewise.
	(insert_xa6): Likewise.
	(extract_xa6: Likewise.
	(insert_xb6): Likewise.
	(extract_xb6): Likewise.
	(insert_xb6s): Likewise.
	(extract_xb6s): Likewise.
	(XS6, XT6, XA6, XB6, XB6S, DM, XX3, XX3DM, XX1_MASK, XX3_MASK,
	XX3DM_MASK, PPCVSX): New.
	(powerpc_opcodes): Add opcodes "lxvd2x", "lxvd2ux", "stxvd2x",
	"stxvd2ux", "xxmrghd", "xxmrgld", "xxpermdi", "xvmovdp", "xvcpsgndp".

2008-08-01  Pedro Alves  <pedro@codesourcery.com>

	* Makefile.am ($(srcdir)/ia64-asmtab.c): Remove line continuation.
	* Makefile.in: Regenerate.

2008-08-01  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-reg.tbl: Use Dw2Inval on AVX registers.
	* i386-tbl.h: Regenerated.

2008-07-30  Michael J. Eager  <eager@eagercon.com>

	* ppc-dis.c (print_insn_powerpc): Disassemble FSL/FCR/UDI fields.
	* ppc-opc.c (powerpc_operands): Add Xilinx APU related operands.
	(insert_sprg, PPC405): Use PPC_OPCODE_405.
	(powerpc_opcodes): Add Xilinx APU related opcodes.

2008-07-30  Alan Modra  <amodra@bigpond.net.au>

	* bfin-dis.c, cris-dis.c, i386-dis.c, or32-opc.c: Silence gcc warnings.

2008-07-10  Richard Sandiford  <rdsandiford@googlemail.com>

	* mips-dis.c (_print_insn_mips): Use ELF_ST_IS_MIPS16.

2008-07-07  Adam Nemet  <anemet@caviumnetworks.com>

	* mips-opc.c (CP): New macro.
	(mips_builtin_opcodes): Mark c0, c2 and c3 as CP.  Add Octeon to the
	membership of di, dmfc0, dmtc0, ei, mfc0 and mtc0.  Add dmfc2 and
	dmtc2 Octeon instructions.

2008-07-07  Stan Shebs  <stan@codesourcery.com>

	* dis-init.c (init_disassemble_info): Init endian_code field.
	* arm-dis.c (print_insn): Disassemble code according to
	setting of endian_code.
	(print_insn_big_arm): Detect when BE8 extension flag has been set.

2008-06-30  Richard Sandiford  <rdsandiford@googlemail.com>

	* mips-dis.c (_print_insn_mips): Use bfd_asymbol_flavour to check
	for ELF symbols.

2008-06-25  Peter Bergner  <bergner@vnet.ibm.com>

	* ppc-dis.c (powerpc_init_dialect): Handle -M464.
	(print_ppc_disassembler_options): Likewise.
	* ppc-opc.c (PPC464): Define.
	(powerpc_opcodes): Add mfdcrux and mtdcrux.

2008-06-17  Ralf Wildenhues  <Ralf.Wildenhues@gmx.de>

	* configure: Regenerate.

2008-06-13  Peter Bergner  <bergner@vnet.ibm.com>

	* ppc-dis.c (print_insn_powerpc): Update prototye to use new
	ppc_cpu_t typedef.
	(struct dis_private): New.
	(POWERPC_DIALECT): New define.
	(powerpc_dialect): Renamed to...
	(powerpc_init_dialect): This.  Update to use ppc_cpu_t and
	struct dis_private.
	(print_insn_big_powerpc): Update for using structure in
	info->private_data.
	(print_insn_little_powerpc): Likewise.
	(operand_value_powerpc): Change type of dialect param to ppc_cpu_t.
	(skip_optional_operands): Likewise.
	(print_insn_powerpc): Likewise.  Remove initialization of dialect.
	* ppc-opc.c (extract_bat, extract_bba, extract_bdm, extract_bdp,
	extract_bo, extract_boe, extract_fxm, extract_mb6, extract_mbe,
	extract_nb, extract_nsi, extract_rbs, extract_sh6, extract_spr,
	extract_sprg, extract_tbr insert_bat, insert_bba, insert_bdm,
	insert_bdp, insert_bo, insert_boe, insert_fxm, insert_mb6, insert_mbe,
	insert_nsi, insert_ral, insert_ram, insert_raq, insert_ras, insert_rbs,
	insert_sh6, insert_spr, insert_sprg, insert_tbr): Change the dialect
	param to be of type ppc_cpu_t.  Update prototype.

2008-06-12  Adam Nemet  <anemet@caviumnetworks.com>

	* mips-dis.c (print_insn_args): Handle field descriptors +x, +p,
	+s, +S.
	* mips-opc.c (mips_builtin_opcodes): Add Octeon instructions
	baddu, bbit*, cins*, dmul, pop, dpop, exts*, mtm*, mtp*, syncs,
	syncw, syncws, vm3mulu, vm0 and vmulu.

	* mips-dis.c (print_insn_args): Handle field descriptor +Q.
	* mips-opc.c (mips_builtin_opcodes): Add Octeon instructions seq,
	seqi, sne and snei.

2008-05-30  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-opc.tbl: Add vmovd with 64bit operand.
	* i386-tbl.h: Regenerated.

2008-05-27  Martin Schwidefsky  <schwidefsky@de.ibm.com>

	* s390-opc.c (INSTR_RRF_R0RR): Fix RRF_R0RR operand format.

2008-05-22  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-opc.tbl: Add NoAVX to cvtpd2pi, cvtpi2pd and cvttpd2pi.
	* i386-tbl.h: Regenerated.

2008-05-22  H.J. Lu  <hongjiu.lu@intel.com>

	PR gas/6517
	* i386-opc.tbl: Break cvtsi2ss/cvtsi2sd/vcvtsi2sd/vcvtsi2ss
	into 32bit and 64bit.  Remove Reg64|Qword and add
	IgnoreSize|No_qSuf on 32bit version.
	* i386-tbl.h: Regenerated.

2008-05-21  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-opc.tbl: Add NoAVX to movdq2q and movq2dq.
	* i386-tbl.h: Regenerated.

2008-05-21  M R Swami Reddy <MR.Swami.Reddy@nsc.com>

	* cr16-dis.c (build_mask): Adjust the mask for 32-bit bcond.

2008-05-14  Alan Modra  <amodra@bigpond.net.au>

	* Makefile.am: Run "make dep-am".
	* Makefile.in: Regenerate.

2008-05-02  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-dis.c (MOVBE_Fixup): New.
	(Mo): Likewise.
	(PREFIX_0F3880): Likewise.
	(PREFIX_0F3881): Likewise.
	(PREFIX_0F38F0): Updated.
	(prefix_table): Add PREFIX_0F3880 and PREFIX_0F3881.  Update
	PREFIX_0F38F0 and PREFIX_0F38F1 for movbe.
	(three_byte_table): Use PREFIX_0F3880 and PREFIX_0F3881.

	* i386-gen.c (cpu_flag_init): Add CPU_MOVBE_FLAGS and
	CPU_EPT_FLAGS.
	(cpu_flags): Add CpuMovbe and CpuEPT.

	* i386-opc.h (CpuMovbe): New.
	(CpuEPT): Likewise.
	(CpuLM): Updated.
	(i386_cpu_flags): Add cpumovbe and cpuept.

	* i386-opc.tbl: Add entries for movbe and EPT instructions.
	* i386-init.h: Regenerated.
	* i386-tbl.h: Likewise.

2008-04-29  Adam Nemet  <anemet@caviumnetworks.com>

	* mips-opc.c (mips_builtin_opcodes): Set field `match' to 0 for
	the two drem and the two dremu macros.

2008-04-28  Adam Nemet  <anemet@caviumnetworks.com>

	* mips-opc.c (mips_builtin_opcodes): Mark prefx and c1
	instructions FP_S.  Mark l.s, li.s, lwc1, swc1, s.s, trunc.w.s and
	cop1 macros INSN2_M_FP_S.  Mark l.d, li.d, ldc1 and sdc1 macros
	INSN2_M_FP_D.  Mark trunc.w.d macro INSN2_M_FP_S and INSN2_M_FP_D.

2008-04-25  David S. Miller  <davem@davemloft.net>

	* sparc-dis.c: Emit %stick instead of %sys_tick, and %stick_cmpr
	instead of %sys_tick_cmpr, as suggested in architecture manuals.

2008-04-23  Paolo Bonzini  <bonzini@gnu.org>

	* aclocal.m4: Regenerate.
	* configure: Regenerate.

2008-04-23  David S. Miller  <davem@davemloft.net>

	* sparc-opc.c (asi_table): Add UltraSPARC and Niagara
	extended values.
	(prefetch_table): Add missing values.

2008-04-22  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c (opcode_modifiers): Add NoAVX.

	* i386-opc.h (NoAVX): New.
	(OldGcc): Updated.
	(i386_opcode_modifier): Add noavx.

	* i386-opc.tbl: Add NoAVX to SSE, SSE2, SSE3 and SSSE3
	instructions which don't have AVX equivalent.
	* i386-tbl.h: Regenerated.

2008-04-18  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-dis.c (OP_VEX_FMA): New.
	(OP_EX_VexImmW): Likewise.
	(VexFMA): Likewise.
	(Vex128FMA): Likewise.
	(EXVexImmW): Likewise.
	(get_vex_imm8): Likewise.
	(OP_EX_VexReg): Likewise.
	(vex_i4_done): Renamed to ...
	(vex_w_done): This.
	(prefix_table): Replace EXVexW with EXVexImmW on vpermil2ps
	and vpermil2pd.  Replace Vex/Vex128 with VexFMA/Vex128FMA on
	FMA instructions.
	(print_insn): Updated.
	(OP_EX_VexW): Rewrite to swap register in VEX with EX.
	(OP_REG_VexI4): Check invalid high registers.

2008-04-16  Dwarakanath Rajagopal  <dwarak.rajagopal@amd.com>
	    Michael Meissner  <michael.meissner@amd.com>

	* i386-opc.tbl: Fix protX to allow memory in the middle operand.
	* i386-tbl.h: Regenerate from i386-opc.tbl.

2008-04-14  Edmar Wienskoski  <edmar@freescale.com>

	* ppc-dis.c (powerpc_dialect): Handle "e500mc".  Extend "e500" to
	accept Power E500MC instructions.
	(print_ppc_disassembler_options): Document -Me500mc.
	* ppc-opc.c (DUIS, DUI, T): New.
	(XRT, XRTRA): Likewise.
	(E500MC): Likewise.
	(powerpc_opcodes): Add new Power E500MC instructions.

2008-04-10  Andreas Krebbel  <krebbel1@de.ibm.com>

	* s390-dis.c (init_disasm): Evaluate disassembler_options.
	(print_s390_disassembler_options): New function.
	* disassemble.c (disassembler_usage): Invoke
	print_s390_disassembler_options.

2008-04-10  Andreas Krebbel  <krebbel1@de.ibm.com>

	* s390-mkopc.c (insertExpandedMnemonic): Expand string sizes
	of local variables used for mnemonic parsing: prefix, suffix and
	number.

2008-04-10  Andreas Krebbel  <krebbel1@de.ibm.com>

	* s390-mkopc.c (s390_cond_ext_format): Add back the mnemonic
	extensions for conditional jumps (o, p, m, nz, z, nm, np, no).
	(s390_crb_extensions): New extensions table.
	(insertExpandedMnemonic): Handle '$' tag.
	* s390-opc.txt: Remove conditional jump variants which can now
	be expanded automatically.
	Replace '*' tag with '$' in the compare and branch instructions.

2008-04-07  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-dis.c (PREFIX_VEX_38XX): Add a tab.
	(PREFIX_VEX_3AXX): Likewis.

2008-04-07  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-opc.tbl: Remove 4 extra blank lines.

2008-04-04  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c (cpu_flag_init): Replace CPU_CLMUL_FLAGS/CpuCLMUL
	with CPU_PCLMUL_FLAGS/CpuPCLMUL.
	(cpu_flags): Replace CpuCLMUL with CpuPCLMUL.
	* i386-opc.tbl: Likewise.

	* i386-opc.h (CpuCLMUL): Renamed to ...
	(CpuPCLMUL): This.
	(CpuFMA): Updated.
	(i386_cpu_flags): Replace cpuclmul with cpupclmul.

	* i386-init.h: Regenerated.

2008-04-03  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-dis.c (OP_E_register): New.
	(OP_E_memory): Likewise.
	(OP_VEX): Likewise.
	(OP_EX_Vex): Likewise.
	(OP_EX_VexW): Likewise.
	(OP_XMM_Vex): Likewise.
	(OP_XMM_VexW): Likewise.
	(OP_REG_VexI4): Likewise.
	(PCLMUL_Fixup): Likewise.
	(VEXI4_Fixup): Likewise.
	(VZERO_Fixup): Likewise.
	(VCMP_Fixup): Likewise.
	(VPERMIL2_Fixup): Likewise.
	(rex_original): Likewise.
	(rex_ignored): Likewise.
	(Mxmm): Likewise.
	(XMM): Likewise.
	(EXxmm): Likewise.
	(EXxmmq): Likewise.
	(EXymmq): Likewise.
	(Vex): Likewise.
	(Vex128): Likewise.
	(Vex256): Likewise.
	(VexI4): Likewise.
	(EXdVex): Likewise.
	(EXqVex): Likewise.
	(EXVexW): Likewise.
	(EXdVexW): Likewise.
	(EXqVexW): Likewise.
	(XMVex): Likewise.
	(XMVexW): Likewise.
	(XMVexI4): Likewise.
	(PCLMUL): Likewise.
	(VZERO): Likewise.
	(VCMP): Likewise.
	(VPERMIL2): Likewise.
	(xmm_mode): Likewise.
	(xmmq_mode): Likewise.
	(ymmq_mode): Likewise.
	(vex_mode): Likewise.
	(vex128_mode): Likewise.
	(vex256_mode): Likewise.
	(USE_VEX_C4_TABLE): Likewise.
	(USE_VEX_C5_TABLE): Likewise.
	(USE_VEX_LEN_TABLE): Likewise.
	(VEX_C4_TABLE): Likewise.
	(VEX_C5_TABLE): Likewise.
	(VEX_LEN_TABLE): Likewise.
	(REG_VEX_XX): Likewise.
	(MOD_VEX_XXX): Likewise.
	(PREFIX_0F38DB..PREFIX_0F38DF): Likewise.
	(PREFIX_0F3A44): Likewise.
	(PREFIX_0F3ADF): Likewise.
	(PREFIX_VEX_XXX): Likewise.
	(VEX_OF): Likewise.
	(VEX_OF38): Likewise.
	(VEX_OF3A): Likewise.
	(VEX_LEN_XXX): Likewise.
	(vex): Likewise.
	(need_vex): Likewise.
	(need_vex_reg): Likewise.
	(vex_i4_done): Likewise.
	(vex_table): Likewise.
	(vex_len_table): Likewise.
	(OP_REG_VexI4): Likewise.
	(vex_cmp_op): Likewise.
	(pclmul_op): Likewise.
	(vpermil2_op): Likewise.
	(m_mode): Updated.
	(es_reg): Likewise.
	(PREFIX_0F38F0): Likewise.
	(PREFIX_0F3A60): Likewise.
	(reg_table): Add REG_VEX_71...REG_VEX_73 and REG_VEX_AE.
	(prefix_table): Add PREFIX_0F38DB..PREFIX_0F38DF, PREFIX_0F3ADF
	and PREFIX_VEX_XXX entries.
	(x86_64_table): Use VEX_C4_TABLE and VEX_C5_TABLE.
	(three_byte_table): Use PREFIX_0F38DB..PREFIX_0F38DF and
	PREFIX_0F3ADF.
	(mod_table): Use VEX_C4_TABLE, VEX_C5_TABLE and VEX_LEN_TABLE.
	Add MOD_VEX_XXX entries.
	(ckprefix): Initialize rex_original and rex_ignored.  Store the
	REX byte in rex_original.
	(get_valid_dis386): Handle the implicit prefix in VEX prefix
	bytes and USE_VEX_LEN_TABLE/USE_VEX_C4_TABLE/USE_VEX_C5_TABLE.
	(print_insn): Set need_vex/need_vex_reg/vex_i4_done to 0 before
	calling get_valid_dis386.  Use rex_original and rex_ignored when
	printing out REX.
	(putop): Handle "XY".
	(intel_operand_size): Handle VEX, xmm_mode, xmmq_mode and
	ymmq_mode.
	(OP_E_extended): Updated to use OP_E_register and
	OP_E_memory.
	(OP_XMM): Handle VEX.
	(OP_EX): Likewise.
	(XMM_Fixup): Likewise.
	(CMP_Fixup): Use ARRAY_SIZE.

	* i386-gen.c (cpu_flag_init): Add CpuAES, CPU_CLMUL_FLAGS,
	CPU_FMA_FLAGS and CPU_AVX_FLAGS.
	(operand_type_init): Add OPERAND_TYPE_REGYMM and
	OPERAND_TYPE_VEX_IMM4.
	(cpu_flags): Add CpuAVX, CpuAES, CpuCLMUL and CpuFMA.
	(opcode_modifiers): Add Implicit1stXmm0, Vex, Vex256, VexNDD,
	VexNDS, VexW0, VexW1, Vex0F, Vex0F38, Vex0F3A, Vex3Sources,
	VexImmExt and SSE2AVX.
	(operand_types): Add RegYMM, Ymmword and Vex_Imm4.

	* i386-opc.h (CpuAVX): New.
	(CpuAES): Likewise.
	(CpuCLMUL): Likewise.
	(CpuFMA): Likewise.
	(Vex): Likewise.
	(Vex256): Likewise.
	(VexNDS): Likewise.
	(VexNDD): Likewise.
	(VexW0): Likewise.
	(VexW1): Likewise.
	(Vex0F): Likewise.
	(Vex0F38): Likewise.
	(Vex0F3A): Likewise.
	(Vex3Sources): Likewise.
	(VexImmExt): Likewise.
	(SSE2AVX): Likewise.
	(RegYMM): Likewise.
	(Ymmword): Likewise.
	(Vex_Imm4): Likewise.
	(Implicit1stXmm0): Likewise.
	(CpuXsave): Updated.
	(CpuLM): Likewise.
	(ByteOkIntel): Likewise.
	(OldGcc): Likewise.
	(Control): Likewise.
	(Unspecified): Likewise.
	(OTMax): Likewise.
	(i386_cpu_flags): Add cpuavx, cpuaes, cpuclmul and cpufma.
	(i386_opcode_modifier): Add implicit1stxmm0, vex, vex256,
	vexnds, vexndd, vexw0, vexw1, vex0f, vex0f38, vex0f3a,
	vex3sources, veximmext and sse2avx.
	(i386_operand_type): Add regymm, ymmword and vex_imm4.

	* i386-opc.tbl: Add AES, CLMUL, AVX and FMA new instructions.

	* i386-reg.tbl: Add AVX registers, ymm0..ymm15.

	* i386-init.h: Regenerated.
	* i386-tbl.h: Likewise.

2008-03-26  Bernd Schmidt  <bernd.schmidt@analog.com>

	From  Robin Getz  <robin.getz@analog.com>
	* bfin-dis.c (bu32): Typedef.
	(enum const_forms_t): Add c_uimm32 and c_huimm32.
	(constant_formats[]): Add uimm32 and huimm16.
	(fmtconst_val): New.
	(uimm32): Define.
	(huimm32): Define.
	(imm16_val): Define.
	(luimm16_val): Define.
	(struct saved_state): Define.
	(GREG, DPREG, DREG, PREG, SPREG, FPREG, IREG, MREG, BREG, LREG,
	A0XREG, A0WREG, A1XREG, A1WREG,CCREG, LC0REG, LT0REG, LB0REG,
	LC1REG, LT1REG, LB1REG, RETSREG, PCREG): Define.
	(get_allreg): New.
	(decode_LDIMMhalf_0): Print out the whole register value.

	From Jie Zhang  <jie.zhang@analog.com>
	* bfin-dis.c (decode_dsp32mac_0): Decode (IU) option for
	multiply and multiply-accumulate to data register instruction.

	* bfin-dis.c: (c_uimm4s4d, c_imm5d, c_imm7d, c_imm16d, c_uimm16s4d,
	c_imm32, c_huimm32e): Define.
	(constant_formats): Add flags for printing decimal, leading spaces, and
	exact symbols.
	(comment, parallel): Add global flags in all disassembly.
	(fmtconst): Take advantage of new flags, and print default in hex.
	(fmtconst_val): Likewise.
	(decode_macfunc): Be consistant with spaces, tabs, comments,
	capitalization in disassembly, fix minor coding style issues.
	(reg_names, amod0, amod1, amod0amod2, aligndir, get_allreg): Likewise.
	(decode_ProgCtrl_0, decode_PushPopMultiple_0, decode_CCflag_0,
	decode_CC2dreg_0, decode_CC2stat_0, decode_BRCC_0, decode_UJUMP_0,
	decode_REGMV_0, decode_ALU2op_0, decode_PTR2op_0, decode_LOGI2op_0,
	decode_COMP3op_0, decode_COMPI2opD_0, decode_COMPI2opP_0,
	decode_LDSTpmod_0, decode_dagMODim_0, decode_dagMODik_0,
	decode_dspLDST_0, decode_LDST_0, decode_LDSTiiFP_0, decode_LDSTii_0,
	decode_LoopSetup_0, decode_LDIMMhalf_0, decode_CALLa_0,
	decode_LDSTidxI_0, decode_linkage_0, decode_dsp32alu_0,
	decode_dsp32shift_0, decode_dsp32shiftimm_0, decode_pseudodbg_assert_0,
	_print_insn_bfin, print_insn_bfin): Likewise.

2008-03-17  Ralf Wildenhues  <Ralf.Wildenhues@gmx.de>

	* aclocal.m4: Regenerate.
	* configure: Likewise.
	* Makefile.in: Likewise.

2008-03-13  Alan Modra  <amodra@bigpond.net.au>

	* Makefile.am: Run "make dep-am".
	* Makefile.in: Regenerate.
	* configure: Regenerate.

2008-03-07  Alan Modra  <amodra@bigpond.net.au>

	* ppc-opc.c (powerpc_opcodes): Order and format.

2008-03-01  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-opc.tbl: Allow 16-bit near indirect branches for x86-64.
	* i386-tbl.h: Regenerated.

2008-02-23  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-opc.tbl: Disallow 16-bit near indirect branches for
	x86-64.
	* i386-tbl.h: Regenerated.

2008-02-21  Jan Beulich  <jbeulich@novell.com>

	* i386-opc.tbl: Allow Dword for far indirect call. Allow Dword
	and Fword for far indirect jmp. Allow Reg16 and Word for near
	indirect jmp on x86-64. Disallow Fword for lcall.
	* i386-tbl.h: Re-generate.

2008-02-18  M R Swami Reddy <MR.Swami.Reddy@nsc.com>

	* cr16-opc.c  (cr16_num_optab): Defined

2008-02-16  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c  (operand_type_init): Add OPERAND_TYPE_INOUTPORTREG.
	* i386-init.h: Regenerated.

2008-02-14  Nick Clifton  <nickc@redhat.com>

	PR binutils/5524
	* configure.in (SHARED_LIBADD): Select the correct host specific
	file extension for shared libraries.
	* configure: Regenerate.

2008-02-13  Jan Beulich  <jbeulich@novell.com>

	* i386-opc.h (RegFlat): New.
	* i386-reg.tbl (flat): Add.
	* i386-tbl.h: Re-generate.

2008-02-13  Jan Beulich  <jbeulich@novell.com>

	* i386-dis.c (a_mode): New.
	(cond_jump_mode): Adjust.
	(Ma): Change to a_mode.
	(intel_operand_size): Handle a_mode.
	* i386-opc.tbl: Allow Dword and Qword for bound.
	* i386-tbl.h: Re-generate.

2008-02-13  Jan Beulich  <jbeulich@novell.com>

	* i386-gen.c (process_i386_registers): Process new fields.
	* i386-opc.h (reg_entry): Shrink reg_flags and reg_num to
	unsigned char. Add dw2_regnum and Dw2Inval.
	* i386-reg.tbl: Provide initializers for dw2_regnum. Add pseudo
	register names.
	* i386-tbl.h: Re-generate.

2008-02-11  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c (cpu_flag_init): Add CPU_XSAVE_FLAGS.
	* i386-init.h: Updated.

2008-02-11  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c (cpu_flags): Add CpuXsave.

	* i386-opc.h (CpuXsave): New.
	(CpuLM): Updated.
	(i386_cpu_flags): Add cpuxsave.

	* i386-dis.c (MOD_0FAE_REG_4): New.
	(RM_0F01_REG_2): Likewise.
	(MOD_0FAE_REG_5): Updated.
	(RM_0F01_REG_3): Likewise.
	(reg_table): Use MOD_0FAE_REG_4.
	(mod_table): Use RM_0F01_REG_2.  Add MOD_0FAE_REG_4.  Updated
	for xrstor.
	(rm_table): Add RM_0F01_REG_2.

	* i386-opc.tbl: Add xsave, xrstor, xgetbv and xsetbv.
	* i386-init.h: Regenerated.
	* i386-tbl.h: Likewise.

2008-02-11  Jan Beulich  <jbeulich@novell.com>

	* i386-opc.tbl: Remove Disp32S from CpuNo64 opcodes. Remove
	Disp16 from Cpu64 non-jump opcodes (including loop and j?cxz).
	* i386-tbl.h: Re-generate.

2008-02-04  H.J. Lu  <hongjiu.lu@intel.com>

	PR 5715
	* configure: Regenerated.

2008-02-04  Adam Nemet  <anemet@caviumnetworks.com>

	* mips-dis.c: Update copyright.
	(mips_arch_choices): Add Octeon.
	* mips-opc.c: Update copyright.
	(IOCT): New macro.
	(mips_builtin_opcodes): Add Octeon instruction synciobdma.

2008-01-29  Alan Modra  <amodra@bigpond.net.au>

	* ppc-opc.c: Support optional L form mtmsr.

2008-01-24  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-dis.c (OP_E_extended): Handle r12 like rsp.

2008-01-23  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c (cpu_flag_init): Add CpuLM to CPU_GENERIC64_FLAGS.
	* i386-init.h: Regenerated.

2008-01-23  Tristan Gingold  <gingold@adacore.com>

	* ia64-dis.c (print_insn_ia64): Display symbolic name of ar.fcr,
	ar.eflag, ar.csd, ar.ssd, ar.cflg, ar.fsr, ar.fir and ar.fdr.

2008-01-22  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c (cpu_flag_init): Remove CpuMMX2.
	(cpu_flags): Likewise.

	* i386-opc.h (CpuMMX2): Removed.
	(CpuSSE): Updated.

	* i386-opc.tbl: Replace CpuMMX2 with CpuSSE|Cpu3dnowA.
	* i386-init.h: Regenerated.
	* i386-tbl.h: Likewise.

2008-01-22  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c (cpu_flag_init): Add CPU_VMX_FLAGS and
	CPU_SMX_FLAGS.
	* i386-init.h: Regenerated.

2008-01-15  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-opc.tbl: Use Qword on movddup.
	* i386-tbl.h: Regenerated.

2008-01-15  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-opc.tbl: Put back 16bit movsx/movzx for AT&T syntax.
	* i386-tbl.h: Regenerated.

2008-01-15  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-dis.c (Mx): New.
	(PREFIX_0FC3): Likewise.
	(PREFIX_0FC7_REG_6): Updated.
	(dis386_twobyte): Use PREFIX_0FC3.
	(prefix_table): Add PREFIX_0FC3.  Use Mq on movntq and movntsd.
	Use Mx on movntps, movntpd, movntdq and movntdqa.  Use Md on
	movntss.

2008-01-14  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c (opcode_modifiers): Add IntelSyntax.
	(operand_types): Add Mem.

	* i386-opc.h (IntelSyntax): New.
	* i386-opc.h (Mem): New.
	(Byte): Updated.
	(Opcode_Modifier_Max): Updated.
	(i386_opcode_modifier): Add intelsyntax.
	(i386_operand_type): Add mem.

	* i386-opc.tbl: Remove Reg16 from movnti.  Add sizes to more
	instructions.

	* i386-reg.tbl: Add size for accumulator.

	* i386-init.h: Regenerated.
	* i386-tbl.h: Likewise.

2008-01-13  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-opc.h (Byte): Fix a typo.

2008-01-12  H.J. Lu  <hongjiu.lu@intel.com>

	PR gas/5534
	* i386-gen.c (operand_type_init): Add Dword to
	OPERAND_TYPE_ACC32.  Add Qword to OPERAND_TYPE_ACC64.
	(opcode_modifiers): Remove CheckSize, Byte, Word, Dword,
	Qword and Xmmword.
	(operand_types): Add Byte, Word, Dword, Fword, Qword, Tbyte,
	Xmmword, Unspecified and Anysize.
	(set_bitfield): Make Mmword an alias of Qword.  Make Oword
	an alias of Xmmword.

	* i386-opc.h (CheckSize): Removed.
	(Byte): Updated.
	(Word): Likewise.
	(Dword): Likewise.
	(Qword): Likewise.
	(Xmmword): Likewise.
	(FWait): Updated.
	(OTMax): Likewise.
	(i386_opcode_modifier): Remove checksize, byte, word, dword,
	qword and xmmword.
	(Fword): New.
	(TBYTE): Likewise.
	(Unspecified): Likewise.
	(Anysize): Likewise.
	(i386_operand_type): Add byte, word, dword, fword, qword,
	tbyte xmmword, unspecified and anysize.

	* i386-opc.tbl: Updated to use Byte, Word, Dword, Fword, Qword,
	Tbyte, Xmmword, Unspecified and Anysize.

	* i386-reg.tbl: Add size for accumulator.

	* i386-init.h: Regenerated.
	* i386-tbl.h: Likewise.

2008-01-10  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-dis.c (REG_0F0E): Renamed to REG_0F0D.
	(REG_0F18): Updated.
	(reg_table): Updated.
	(dis386_twobyte): Updated.  Use "nopQ" on 0x19 to 0x1e.
	(twobyte_has_modrm): Set 1 for 0x19 to 0x1e.

2008-01-08  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c (set_bitfield): Use fail () on error.

2008-01-08  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c (lineno): New.
	(filename): Likewise.
	(set_bitfield): Report filename and line numer on error.
	(process_i386_opcodes): Set filename and update lineno.
	(process_i386_registers): Likewise.

2008-01-05  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c (opcode_modifiers): Rename IntelMnemonic to
	ATTSyntax.

	* i386-opc.h (IntelMnemonic): Renamed to ..
	(ATTSyntax): This
	(Opcode_Modifier_Max): Updated.
	(i386_opcode_modifier): Remove intelmnemonic. Add attsyntax
	and intelsyntax.

	* i386-opc.tbl: Remove IntelMnemonic and update with ATTSyntax
	on fsub, fubp, fsubr, fsubrp, div, fdivp, fdivr and fdivrp.
	* i386-tbl.h: Regenerated.

2008-01-04  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c: Update copyright to 2008.
	* i386-opc.h: Likewise.
	* i386-opc.tbl: Likewise.

	* i386-init.h: Regenerated.
	* i386-tbl.h: Likewise.

2008-01-04  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-opc.tbl: Add NoRex64 to extractps, movmskpd, movmskps,
	pextrb, pextrw, pinsrb, pinsrw and pmovmskb.
	* i386-tbl.h: Regenerated.

2008-01-03  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c (cpu_flag_init): Remove CpuSSE4_1_Or_5 and
	CpuSSE4_2_Or_ABM.
	(cpu_flags): Likewise.

	* i386-opc.h (CpuSSE4_1_Or_5): Removed.
	(CpuSSE4_2_Or_ABM): Likewise.
	(CpuLM): Updated.
	(i386_cpu_flags): Remove cpusse4_1_or_5 and cpusse4_2_or_abm.

	* i386-opc.tbl: Replace CpuSSE4_1_Or_5, CpuSSE4_2_Or_ABM and
	Cpu686|CpuPadLock with CpuSSE4_1|CpuSSE5, CpuABM|CpuSSE4_2
	and CpuPadLock, respectively.
	* i386-init.h: Regenerated.
	* i386-tbl.h: Likewise.

2008-01-03  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c (opcode_modifiers): Remove No_xSuf.

	* i386-opc.h (No_xSuf): Removed.
	(CheckSize): Updated.

	* i386-tbl.h: Regenerated.

2008-01-02  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c (cpu_flag_init): Add CpuSSE4_2_Or_ABM to
	CPU_AMDFAM10_FLAGS, CPU_SSE4_2_FLAGS, CpuABM and
	CPU_SSE5_FLAGS.
	(cpu_flags): Add CpuSSE4_2_Or_ABM.

	* i386-opc.h (CpuSSE4_2_Or_ABM): New.
	(CpuLM): Updated.
	(i386_cpu_flags): Add cpusse4_2_or_abm.

	* i386-opc.tbl: Use CpuSSE4_2_Or_ABM instead of
	CpuABM|CpuSSE4_2 on popcnt.
	* i386-init.h: Regenerated.
	* i386-tbl.h: Likewise.

2008-01-02  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-opc.h: Update comments.

2008-01-02  H.J. Lu  <hongjiu.lu@intel.com>

	* i386-gen.c (opcode_modifiers): Use Qword instead of QWord.
	* i386-opc.h: Likewise.
	* i386-opc.tbl: Likewise.

2008-01-02  H.J. Lu  <hongjiu.lu@intel.com>

	PR gas/5534
	* i386-gen.c (opcode_modifiers): Add No_xSuf, CheckSize,
	Byte, Word, Dword, QWord and Xmmword.

	* i386-opc.h (No_xSuf): New.
	(CheckSize): Likewise.
	(Byte): Likewise.
	(Word): Likewise.
	(Dword): Likewise.
	(QWord): Likewise.
	(Xmmword): Likewise.
	(FWait): Updated.
	(i386_opcode_modifier): Add No_xSuf, CheckSize, Byte, Word,
	Dword, QWord and Xmmword.

	* i386-opc.tbl: Add CheckSize|QWord to movq if IgnoreSize is
	used.
	* i386-tbl.h: Regenerated.

2008-01-02  Mark Kettenis  <kettenis@gnu.org>

	* m88k-dis.c (instructions): Fix fcvt.* instructions.
	From Miod Vallat.

For older changes see ChangeLog-2007

Copyright (C) 2008 Free Software Foundation, Inc.

Copying and distribution of this file, with or without modification,
are permitted in any medium without royalty provided the copyright
notice and this notice are preserved.

Local Variables:
mode: change-log
left-margin: 8
fill-column: 74
version-control: never
End:
f='#n3004'>3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618
/*
 *  Virtual page mapping
 *
 *  Copyright (c) 2003 Fabrice Bellard
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 */
#include "config.h"
#ifndef _WIN32
#include <sys/types.h>
#include <sys/mman.h>
#endif

#include "qemu-common.h"
#include "cpu.h"
#include "tcg.h"
#include "hw/hw.h"
#if !defined(CONFIG_USER_ONLY)
#include "hw/boards.h"
#endif
#include "hw/qdev.h"
#include "qemu/osdep.h"
#include "sysemu/kvm.h"
#include "sysemu/sysemu.h"
#include "hw/xen/xen.h"
#include "qemu/timer.h"
#include "qemu/config-file.h"
#include "qemu/error-report.h"
#include "exec/memory.h"
#include "sysemu/dma.h"
#include "exec/address-spaces.h"
#if defined(CONFIG_USER_ONLY)
#include <qemu.h>
#else /* !CONFIG_USER_ONLY */
#include "sysemu/xen-mapcache.h"
#include "trace.h"
#endif
#include "exec/cpu-all.h"
#include "qemu/rcu_queue.h"
#include "qemu/main-loop.h"
#include "translate-all.h"
#include "sysemu/replay.h"

#include "exec/memory-internal.h"
#include "exec/ram_addr.h"

#include "qemu/range.h"
#ifndef _WIN32
#include "qemu/mmap-alloc.h"
#endif

//#define DEBUG_SUBPAGE

#if !defined(CONFIG_USER_ONLY)
/* ram_list is read under rcu_read_lock()/rcu_read_unlock().  Writes
 * are protected by the ramlist lock.
 */
RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };

static MemoryRegion *system_memory;
static MemoryRegion *system_io;

AddressSpace address_space_io;
AddressSpace address_space_memory;

MemoryRegion io_mem_rom, io_mem_notdirty;
static MemoryRegion io_mem_unassigned;

/* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
#define RAM_PREALLOC   (1 << 0)

/* RAM is mmap-ed with MAP_SHARED */
#define RAM_SHARED     (1 << 1)

/* Only a portion of RAM (used_length) is actually used, and migrated.
 * This used_length size can change across reboots.
 */
#define RAM_RESIZEABLE (1 << 2)

/* RAM is backed by an mmapped file.
 */
#define RAM_FILE (1 << 3)
#endif

struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
/* current CPU in the current thread. It is only valid inside
   cpu_exec() */
__thread CPUState *current_cpu;
/* 0 = Do not count executed instructions.
   1 = Precise instruction counting.
   2 = Adaptive rate instruction counting.  */
int use_icount;

#if !defined(CONFIG_USER_ONLY)

typedef struct PhysPageEntry PhysPageEntry;

struct PhysPageEntry {
    /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
    uint32_t skip : 6;
     /* index into phys_sections (!skip) or phys_map_nodes (skip) */
    uint32_t ptr : 26;
};

#define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)

/* Size of the L2 (and L3, etc) page tables.  */
#define ADDR_SPACE_BITS 64

#define P_L2_BITS 9
#define P_L2_SIZE (1 << P_L2_BITS)

#define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)

typedef PhysPageEntry Node[P_L2_SIZE];

typedef struct PhysPageMap {
    struct rcu_head rcu;

    unsigned sections_nb;
    unsigned sections_nb_alloc;
    unsigned nodes_nb;
    unsigned nodes_nb_alloc;
    Node *nodes;
    MemoryRegionSection *sections;
} PhysPageMap;

struct AddressSpaceDispatch {
    struct rcu_head rcu;

    /* This is a multi-level map on the physical address space.
     * The bottom level has pointers to MemoryRegionSections.
     */
    PhysPageEntry phys_map;
    PhysPageMap map;
    AddressSpace *as;
};

#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
typedef struct subpage_t {
    MemoryRegion iomem;
    AddressSpace *as;
    hwaddr base;
    uint16_t sub_section[TARGET_PAGE_SIZE];
} subpage_t;

#define PHYS_SECTION_UNASSIGNED 0
#define PHYS_SECTION_NOTDIRTY 1
#define PHYS_SECTION_ROM 2
#define PHYS_SECTION_WATCH 3

static void io_mem_init(void);
static void memory_map_init(void);
static void tcg_commit(MemoryListener *listener);

static MemoryRegion io_mem_watch;

/**
 * CPUAddressSpace: all the information a CPU needs about an AddressSpace
 * @cpu: the CPU whose AddressSpace this is
 * @as: the AddressSpace itself
 * @memory_dispatch: its dispatch pointer (cached, RCU protected)
 * @tcg_as_listener: listener for tracking changes to the AddressSpace
 */
struct CPUAddressSpace {
    CPUState *cpu;
    AddressSpace *as;
    struct AddressSpaceDispatch *memory_dispatch;
    MemoryListener tcg_as_listener;
};

#endif

#if !defined(CONFIG_USER_ONLY)

static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
{
    if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
        map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
        map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
        map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
    }
}

static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
{
    unsigned i;
    uint32_t ret;
    PhysPageEntry e;
    PhysPageEntry *p;

    ret = map->nodes_nb++;
    p = map->nodes[ret];
    assert(ret != PHYS_MAP_NODE_NIL);
    assert(ret != map->nodes_nb_alloc);

    e.skip = leaf ? 0 : 1;
    e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
    for (i = 0; i < P_L2_SIZE; ++i) {
        memcpy(&p[i], &e, sizeof(e));
    }
    return ret;
}

static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
                                hwaddr *index, hwaddr *nb, uint16_t leaf,
                                int level)
{
    PhysPageEntry *p;
    hwaddr step = (hwaddr)1 << (level * P_L2_BITS);

    if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
        lp->ptr = phys_map_node_alloc(map, level == 0);
    }
    p = map->nodes[lp->ptr];
    lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];

    while (*nb && lp < &p[P_L2_SIZE]) {
        if ((*index & (step - 1)) == 0 && *nb >= step) {
            lp->skip = 0;
            lp->ptr = leaf;
            *index += step;
            *nb -= step;
        } else {
            phys_page_set_level(map, lp, index, nb, leaf, level - 1);
        }
        ++lp;
    }
}

static void phys_page_set(AddressSpaceDispatch *d,
                          hwaddr index, hwaddr nb,
                          uint16_t leaf)
{
    /* Wildly overreserve - it doesn't matter much. */
    phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);

    phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
}

/* Compact a non leaf page entry. Simply detect that the entry has a single child,
 * and update our entry so we can skip it and go directly to the destination.
 */
static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
{
    unsigned valid_ptr = P_L2_SIZE;
    int valid = 0;
    PhysPageEntry *p;
    int i;

    if (lp->ptr == PHYS_MAP_NODE_NIL) {
        return;
    }

    p = nodes[lp->ptr];
    for (i = 0; i < P_L2_SIZE; i++) {
        if (p[i].ptr == PHYS_MAP_NODE_NIL) {
            continue;
        }

        valid_ptr = i;
        valid++;
        if (p[i].skip) {
            phys_page_compact(&p[i], nodes, compacted);
        }
    }

    /* We can only compress if there's only one child. */
    if (valid != 1) {
        return;
    }

    assert(valid_ptr < P_L2_SIZE);

    /* Don't compress if it won't fit in the # of bits we have. */
    if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
        return;
    }

    lp->ptr = p[valid_ptr].ptr;
    if (!p[valid_ptr].skip) {
        /* If our only child is a leaf, make this a leaf. */
        /* By design, we should have made this node a leaf to begin with so we
         * should never reach here.
         * But since it's so simple to handle this, let's do it just in case we
         * change this rule.
         */
        lp->skip = 0;
    } else {
        lp->skip += p[valid_ptr].skip;
    }
}

static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
{
    DECLARE_BITMAP(compacted, nodes_nb);

    if (d->phys_map.skip) {
        phys_page_compact(&d->phys_map, d->map.nodes, compacted);
    }
}

static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
                                           Node *nodes, MemoryRegionSection *sections)
{
    PhysPageEntry *p;
    hwaddr index = addr >> TARGET_PAGE_BITS;
    int i;

    for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
        if (lp.ptr == PHYS_MAP_NODE_NIL) {
            return &sections[PHYS_SECTION_UNASSIGNED];
        }
        p = nodes[lp.ptr];
        lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
    }

    if (sections[lp.ptr].size.hi ||
        range_covers_byte(sections[lp.ptr].offset_within_address_space,
                          sections[lp.ptr].size.lo, addr)) {
        return &sections[lp.ptr];
    } else {
        return &sections[PHYS_SECTION_UNASSIGNED];
    }
}

bool memory_region_is_unassigned(MemoryRegion *mr)
{
    return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
        && mr != &io_mem_watch;
}

/* Called from RCU critical section */
static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
                                                        hwaddr addr,
                                                        bool resolve_subpage)
{
    MemoryRegionSection *section;
    subpage_t *subpage;

    section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
    if (resolve_subpage && section->mr->subpage) {
        subpage = container_of(section->mr, subpage_t, iomem);
        section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
    }
    return section;
}

/* Called from RCU critical section */
static MemoryRegionSection *
address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
                                 hwaddr *plen, bool resolve_subpage)
{
    MemoryRegionSection *section;
    MemoryRegion *mr;
    Int128 diff;

    section = address_space_lookup_region(d, addr, resolve_subpage);
    /* Compute offset within MemoryRegionSection */
    addr -= section->offset_within_address_space;

    /* Compute offset within MemoryRegion */
    *xlat = addr + section->offset_within_region;

    mr = section->mr;

    /* MMIO registers can be expected to perform full-width accesses based only
     * on their address, without considering adjacent registers that could
     * decode to completely different MemoryRegions.  When such registers
     * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
     * regions overlap wildly.  For this reason we cannot clamp the accesses
     * here.
     *
     * If the length is small (as is the case for address_space_ldl/stl),
     * everything works fine.  If the incoming length is large, however,
     * the caller really has to do the clamping through memory_access_size.
     */
    if (memory_region_is_ram(mr)) {
        diff = int128_sub(section->size, int128_make64(addr));
        *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
    }
    return section;
}

static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
{
    if (memory_region_is_ram(mr)) {
        return !(is_write && mr->readonly);
    }
    if (memory_region_is_romd(mr)) {
        return !is_write;
    }

    return false;
}

/* Called from RCU critical section */
MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
                                      hwaddr *xlat, hwaddr *plen,
                                      bool is_write)
{
    IOMMUTLBEntry iotlb;
    MemoryRegionSection *section;
    MemoryRegion *mr;

    for (;;) {
        AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch);
        section = address_space_translate_internal(d, addr, &addr, plen, true);
        mr = section->mr;

        if (!mr->iommu_ops) {
            break;
        }

        iotlb = mr->iommu_ops->translate(mr, addr, is_write);
        addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
                | (addr & iotlb.addr_mask));
        *plen = MIN(*plen, (addr | iotlb.addr_mask) - addr + 1);
        if (!(iotlb.perm & (1 << is_write))) {
            mr = &io_mem_unassigned;
            break;
        }

        as = iotlb.target_as;
    }

    if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
        hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
        *plen = MIN(page, *plen);
    }

    *xlat = addr;
    return mr;
}

/* Called from RCU critical section */
MemoryRegionSection *
address_space_translate_for_iotlb(CPUState *cpu, hwaddr addr,
                                  hwaddr *xlat, hwaddr *plen)
{
    MemoryRegionSection *section;
    section = address_space_translate_internal(cpu->cpu_ases[0].memory_dispatch,
                                               addr, xlat, plen, false);

    assert(!section->mr->iommu_ops);
    return section;
}
#endif

#if !defined(CONFIG_USER_ONLY)

static int cpu_common_post_load(void *opaque, int version_id)
{
    CPUState *cpu = opaque;

    /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
       version_id is increased. */
    cpu->interrupt_request &= ~0x01;
    tlb_flush(cpu, 1);

    return 0;
}

static int cpu_common_pre_load(void *opaque)
{
    CPUState *cpu = opaque;

    cpu->exception_index = -1;

    return 0;
}

static bool cpu_common_exception_index_needed(void *opaque)
{
    CPUState *cpu = opaque;

    return tcg_enabled() && cpu->exception_index != -1;
}

static const VMStateDescription vmstate_cpu_common_exception_index = {
    .name = "cpu_common/exception_index",
    .version_id = 1,
    .minimum_version_id = 1,
    .needed = cpu_common_exception_index_needed,
    .fields = (VMStateField[]) {
        VMSTATE_INT32(exception_index, CPUState),
        VMSTATE_END_OF_LIST()
    }
};

static bool cpu_common_crash_occurred_needed(void *opaque)
{
    CPUState *cpu = opaque;

    return cpu->crash_occurred;
}

static const VMStateDescription vmstate_cpu_common_crash_occurred = {
    .name = "cpu_common/crash_occurred",
    .version_id = 1,
    .minimum_version_id = 1,
    .needed = cpu_common_crash_occurred_needed,
    .fields = (VMStateField[]) {
        VMSTATE_BOOL(crash_occurred, CPUState),
        VMSTATE_END_OF_LIST()
    }
};

const VMStateDescription vmstate_cpu_common = {
    .name = "cpu_common",
    .version_id = 1,
    .minimum_version_id = 1,
    .pre_load = cpu_common_pre_load,
    .post_load = cpu_common_post_load,
    .fields = (VMStateField[]) {
        VMSTATE_UINT32(halted, CPUState),
        VMSTATE_UINT32(interrupt_request, CPUState),
        VMSTATE_END_OF_LIST()
    },
    .subsections = (const VMStateDescription*[]) {
        &vmstate_cpu_common_exception_index,
        &vmstate_cpu_common_crash_occurred,
        NULL
    }
};

#endif

CPUState *qemu_get_cpu(int index)
{
    CPUState *cpu;

    CPU_FOREACH(cpu) {
        if (cpu->cpu_index == index) {
            return cpu;
        }
    }

    return NULL;
}

#if !defined(CONFIG_USER_ONLY)
void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as)
{
    /* We only support one address space per cpu at the moment.  */
    assert(cpu->as == as);

    if (cpu->cpu_ases) {
        /* We've already registered the listener for our only AS */
        return;
    }

    cpu->cpu_ases = g_new0(CPUAddressSpace, 1);
    cpu->cpu_ases[0].cpu = cpu;
    cpu->cpu_ases[0].as = as;
    cpu->cpu_ases[0].tcg_as_listener.commit = tcg_commit;
    memory_listener_register(&cpu->cpu_ases[0].tcg_as_listener, as);
}
#endif

#ifndef CONFIG_USER_ONLY
static DECLARE_BITMAP(cpu_index_map, MAX_CPUMASK_BITS);

static int cpu_get_free_index(Error **errp)
{
    int cpu = find_first_zero_bit(cpu_index_map, MAX_CPUMASK_BITS);

    if (cpu >= MAX_CPUMASK_BITS) {
        error_setg(errp, "Trying to use more CPUs than max of %d",
                   MAX_CPUMASK_BITS);
        return -1;
    }

    bitmap_set(cpu_index_map, cpu, 1);
    return cpu;
}

void cpu_exec_exit(CPUState *cpu)
{
    if (cpu->cpu_index == -1) {
        /* cpu_index was never allocated by this @cpu or was already freed. */
        return;
    }

    bitmap_clear(cpu_index_map, cpu->cpu_index, 1);
    cpu->cpu_index = -1;
}
#else

static int cpu_get_free_index(Error **errp)
{
    CPUState *some_cpu;
    int cpu_index = 0;

    CPU_FOREACH(some_cpu) {
        cpu_index++;
    }
    return cpu_index;
}

void cpu_exec_exit(CPUState *cpu)
{
}
#endif

void cpu_exec_init(CPUState *cpu, Error **errp)
{
    CPUClass *cc = CPU_GET_CLASS(cpu);
    int cpu_index;
    Error *local_err = NULL;

#ifndef CONFIG_USER_ONLY
    cpu->as = &address_space_memory;
    cpu->thread_id = qemu_get_thread_id();
#endif

#if defined(CONFIG_USER_ONLY)
    cpu_list_lock();
#endif
    cpu_index = cpu->cpu_index = cpu_get_free_index(&local_err);
    if (local_err) {
        error_propagate(errp, local_err);
#if defined(CONFIG_USER_ONLY)
        cpu_list_unlock();
#endif
        return;
    }
    QTAILQ_INSERT_TAIL(&cpus, cpu, node);
#if defined(CONFIG_USER_ONLY)
    cpu_list_unlock();
#endif
    if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
        vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
    }
#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
    register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
                    cpu_save, cpu_load, cpu->env_ptr);
    assert(cc->vmsd == NULL);
    assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
#endif
    if (cc->vmsd != NULL) {
        vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
    }
}

#if defined(CONFIG_USER_ONLY)
static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
{
    tb_invalidate_phys_page_range(pc, pc + 1, 0);
}
#else
static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
{
    hwaddr phys = cpu_get_phys_page_debug(cpu, pc);
    if (phys != -1) {
        tb_invalidate_phys_addr(cpu->as,
                                phys | (pc & ~TARGET_PAGE_MASK));
    }
}
#endif

#if defined(CONFIG_USER_ONLY)
void cpu_watchpoint_remove_all(CPUState *cpu, int mask)

{
}

int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
                          int flags)
{
    return -ENOSYS;
}

void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
{
}

int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
                          int flags, CPUWatchpoint **watchpoint)
{
    return -ENOSYS;
}
#else
/* Add a watchpoint.  */
int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
                          int flags, CPUWatchpoint **watchpoint)
{
    CPUWatchpoint *wp;

    /* forbid ranges which are empty or run off the end of the address space */
    if (len == 0 || (addr + len - 1) < addr) {
        error_report("tried to set invalid watchpoint at %"
                     VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
        return -EINVAL;
    }
    wp = g_malloc(sizeof(*wp));

    wp->vaddr = addr;
    wp->len = len;
    wp->flags = flags;

    /* keep all GDB-injected watchpoints in front */
    if (flags & BP_GDB) {
        QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
    } else {
        QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
    }

    tlb_flush_page(cpu, addr);

    if (watchpoint)
        *watchpoint = wp;
    return 0;
}

/* Remove a specific watchpoint.  */
int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
                          int flags)
{
    CPUWatchpoint *wp;

    QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
        if (addr == wp->vaddr && len == wp->len
                && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
            cpu_watchpoint_remove_by_ref(cpu, wp);
            return 0;
        }
    }
    return -ENOENT;
}

/* Remove a specific watchpoint by reference.  */
void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
{
    QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);

    tlb_flush_page(cpu, watchpoint->vaddr);

    g_free(watchpoint);
}

/* Remove all matching watchpoints.  */
void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
{
    CPUWatchpoint *wp, *next;

    QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
        if (wp->flags & mask) {
            cpu_watchpoint_remove_by_ref(cpu, wp);
        }
    }
}

/* Return true if this watchpoint address matches the specified
 * access (ie the address range covered by the watchpoint overlaps
 * partially or completely with the address range covered by the
 * access).
 */
static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
                                                  vaddr addr,
                                                  vaddr len)
{
    /* We know the lengths are non-zero, but a little caution is
     * required to avoid errors in the case where the range ends
     * exactly at the top of the address space and so addr + len
     * wraps round to zero.
     */
    vaddr wpend = wp->vaddr + wp->len - 1;
    vaddr addrend = addr + len - 1;

    return !(addr > wpend || wp->vaddr > addrend);
}

#endif

/* Add a breakpoint.  */
int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
                          CPUBreakpoint **breakpoint)
{
    CPUBreakpoint *bp;

    bp = g_malloc(sizeof(*bp));

    bp->pc = pc;
    bp->flags = flags;

    /* keep all GDB-injected breakpoints in front */
    if (flags & BP_GDB) {
        QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
    } else {
        QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
    }

    breakpoint_invalidate(cpu, pc);

    if (breakpoint) {
        *breakpoint = bp;
    }
    return 0;
}

/* Remove a specific breakpoint.  */
int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
{
    CPUBreakpoint *bp;

    QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
        if (bp->pc == pc && bp->flags == flags) {
            cpu_breakpoint_remove_by_ref(cpu, bp);
            return 0;
        }
    }
    return -ENOENT;
}

/* Remove a specific breakpoint by reference.  */
void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
{
    QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);

    breakpoint_invalidate(cpu, breakpoint->pc);

    g_free(breakpoint);
}

/* Remove all matching breakpoints. */
void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
{
    CPUBreakpoint *bp, *next;

    QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
        if (bp->flags & mask) {
            cpu_breakpoint_remove_by_ref(cpu, bp);
        }
    }
}

/* enable or disable single step mode. EXCP_DEBUG is returned by the
   CPU loop after each instruction */
void cpu_single_step(CPUState *cpu, int enabled)
{
    if (cpu->singlestep_enabled != enabled) {
        cpu->singlestep_enabled = enabled;
        if (kvm_enabled()) {
            kvm_update_guest_debug(cpu, 0);
        } else {
            /* must flush all the translated code to avoid inconsistencies */
            /* XXX: only flush what is necessary */
            tb_flush(cpu);
        }
    }
}

void cpu_abort(CPUState *cpu, const char *fmt, ...)
{
    va_list ap;
    va_list ap2;

    va_start(ap, fmt);
    va_copy(ap2, ap);
    fprintf(stderr, "qemu: fatal: ");
    vfprintf(stderr, fmt, ap);
    fprintf(stderr, "\n");
    cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
    if (qemu_log_enabled()) {
        qemu_log("qemu: fatal: ");
        qemu_log_vprintf(fmt, ap2);
        qemu_log("\n");
        log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
        qemu_log_flush();
        qemu_log_close();
    }
    va_end(ap2);
    va_end(ap);
    replay_finish();
#if defined(CONFIG_USER_ONLY)
    {
        struct sigaction act;
        sigfillset(&act.sa_mask);
        act.sa_handler = SIG_DFL;
        sigaction(SIGABRT, &act, NULL);
    }
#endif
    abort();
}

#if !defined(CONFIG_USER_ONLY)
/* Called from RCU critical section */
static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
{
    RAMBlock *block;

    block = atomic_rcu_read(&ram_list.mru_block);
    if (block && addr - block->offset < block->max_length) {
        return block;
    }
    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
        if (addr - block->offset < block->max_length) {
            goto found;
        }
    }

    fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
    abort();

found:
    /* It is safe to write mru_block outside the iothread lock.  This
     * is what happens:
     *
     *     mru_block = xxx
     *     rcu_read_unlock()
     *                                        xxx removed from list
     *                  rcu_read_lock()
     *                  read mru_block
     *                                        mru_block = NULL;
     *                                        call_rcu(reclaim_ramblock, xxx);
     *                  rcu_read_unlock()
     *
     * atomic_rcu_set is not needed here.  The block was already published
     * when it was placed into the list.  Here we're just making an extra
     * copy of the pointer.
     */
    ram_list.mru_block = block;
    return block;
}

static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
{
    CPUState *cpu;
    ram_addr_t start1;
    RAMBlock *block;
    ram_addr_t end;

    end = TARGET_PAGE_ALIGN(start + length);
    start &= TARGET_PAGE_MASK;

    rcu_read_lock();
    block = qemu_get_ram_block(start);
    assert(block == qemu_get_ram_block(end - 1));
    start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
    CPU_FOREACH(cpu) {
        tlb_reset_dirty(cpu, start1, length);
    }
    rcu_read_unlock();
}

/* Note: start and end must be within the same ram block.  */
bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
                                              ram_addr_t length,
                                              unsigned client)
{
    unsigned long end, page;
    bool dirty;

    if (length == 0) {
        return false;
    }

    end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
    page = start >> TARGET_PAGE_BITS;
    dirty = bitmap_test_and_clear_atomic(ram_list.dirty_memory[client],
                                         page, end - page);

    if (dirty && tcg_enabled()) {
        tlb_reset_dirty_range_all(start, length);
    }

    return dirty;
}

/* Called from RCU critical section */
hwaddr memory_region_section_get_iotlb(CPUState *cpu,
                                       MemoryRegionSection *section,
                                       target_ulong vaddr,
                                       hwaddr paddr, hwaddr xlat,
                                       int prot,
                                       target_ulong *address)
{
    hwaddr iotlb;
    CPUWatchpoint *wp;

    if (memory_region_is_ram(section->mr)) {
        /* Normal RAM.  */
        iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
            + xlat;
        if (!section->readonly) {
            iotlb |= PHYS_SECTION_NOTDIRTY;
        } else {
            iotlb |= PHYS_SECTION_ROM;
        }
    } else {
        AddressSpaceDispatch *d;

        d = atomic_rcu_read(&section->address_space->dispatch);
        iotlb = section - d->map.sections;
        iotlb += xlat;
    }

    /* Make accesses to pages with watchpoints go via the
       watchpoint trap routines.  */
    QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
        if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
            /* Avoid trapping reads of pages with a write breakpoint. */
            if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
                iotlb = PHYS_SECTION_WATCH + paddr;
                *address |= TLB_MMIO;
                break;
            }
        }
    }

    return iotlb;
}
#endif /* defined(CONFIG_USER_ONLY) */

#if !defined(CONFIG_USER_ONLY)

static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
                             uint16_t section);
static subpage_t *subpage_init(AddressSpace *as, hwaddr base);

static void *(*phys_mem_alloc)(size_t size, uint64_t *align) =
                               qemu_anon_ram_alloc;

/*
 * Set a custom physical guest memory alloator.
 * Accelerators with unusual needs may need this.  Hopefully, we can
 * get rid of it eventually.
 */
void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align))
{
    phys_mem_alloc = alloc;
}

static uint16_t phys_section_add(PhysPageMap *map,
                                 MemoryRegionSection *section)
{
    /* The physical section number is ORed with a page-aligned
     * pointer to produce the iotlb entries.  Thus it should
     * never overflow into the page-aligned value.
     */
    assert(map->sections_nb < TARGET_PAGE_SIZE);

    if (map->sections_nb == map->sections_nb_alloc) {
        map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
        map->sections = g_renew(MemoryRegionSection, map->sections,
                                map->sections_nb_alloc);
    }
    map->sections[map->sections_nb] = *section;
    memory_region_ref(section->mr);
    return map->sections_nb++;
}

static void phys_section_destroy(MemoryRegion *mr)
{
    bool have_sub_page = mr->subpage;

    memory_region_unref(mr);

    if (have_sub_page) {
        subpage_t *subpage = container_of(mr, subpage_t, iomem);
        object_unref(OBJECT(&subpage->iomem));
        g_free(subpage);
    }
}

static void phys_sections_free(PhysPageMap *map)
{
    while (map->sections_nb > 0) {
        MemoryRegionSection *section = &map->sections[--map->sections_nb];
        phys_section_destroy(section->mr);
    }
    g_free(map->sections);
    g_free(map->nodes);
}

static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
{
    subpage_t *subpage;
    hwaddr base = section->offset_within_address_space
        & TARGET_PAGE_MASK;
    MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
                                                   d->map.nodes, d->map.sections);
    MemoryRegionSection subsection = {
        .offset_within_address_space = base,
        .size = int128_make64(TARGET_PAGE_SIZE),
    };
    hwaddr start, end;

    assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);

    if (!(existing->mr->subpage)) {
        subpage = subpage_init(d->as, base);
        subsection.address_space = d->as;
        subsection.mr = &subpage->iomem;
        phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
                      phys_section_add(&d->map, &subsection));
    } else {
        subpage = container_of(existing->mr, subpage_t, iomem);
    }
    start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
    end = start + int128_get64(section->size) - 1;
    subpage_register(subpage, start, end,
                     phys_section_add(&d->map, section));
}


static void register_multipage(AddressSpaceDispatch *d,
                               MemoryRegionSection *section)
{
    hwaddr start_addr = section->offset_within_address_space;
    uint16_t section_index = phys_section_add(&d->map, section);
    uint64_t num_pages = int128_get64(int128_rshift(section->size,
                                                    TARGET_PAGE_BITS));

    assert(num_pages);
    phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
}

static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
{
    AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
    AddressSpaceDispatch *d = as->next_dispatch;
    MemoryRegionSection now = *section, remain = *section;
    Int128 page_size = int128_make64(TARGET_PAGE_SIZE);

    if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
        uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
                       - now.offset_within_address_space;

        now.size = int128_min(int128_make64(left), now.size);
        register_subpage(d, &now);
    } else {
        now.size = int128_zero();
    }
    while (int128_ne(remain.size, now.size)) {
        remain.size = int128_sub(remain.size, now.size);
        remain.offset_within_address_space += int128_get64(now.size);
        remain.offset_within_region += int128_get64(now.size);
        now = remain;
        if (int128_lt(remain.size, page_size)) {
            register_subpage(d, &now);
        } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
            now.size = page_size;
            register_subpage(d, &now);
        } else {
            now.size = int128_and(now.size, int128_neg(page_size));
            register_multipage(d, &now);
        }
    }
}

void qemu_flush_coalesced_mmio_buffer(void)
{
    if (kvm_enabled())
        kvm_flush_coalesced_mmio_buffer();
}

void qemu_mutex_lock_ramlist(void)
{
    qemu_mutex_lock(&ram_list.mutex);
}

void qemu_mutex_unlock_ramlist(void)
{
    qemu_mutex_unlock(&ram_list.mutex);
}

#ifdef __linux__

#include <sys/vfs.h>

#define HUGETLBFS_MAGIC       0x958458f6

static long gethugepagesize(const char *path, Error **errp)
{
    struct statfs fs;
    int ret;

    do {
        ret = statfs(path, &fs);
    } while (ret != 0 && errno == EINTR);

    if (ret != 0) {
        error_setg_errno(errp, errno, "failed to get page size of file %s",
                         path);
        return 0;
    }

    return fs.f_bsize;
}

static void *file_ram_alloc(RAMBlock *block,
                            ram_addr_t memory,
                            const char *path,
                            Error **errp)
{
    struct stat st;
    char *filename;
    char *sanitized_name;
    char *c;
    void *area;
    int fd;
    uint64_t hpagesize;
    Error *local_err = NULL;

    hpagesize = gethugepagesize(path, &local_err);
    if (local_err) {
        error_propagate(errp, local_err);
        goto error;
    }
    block->mr->align = hpagesize;

    if (memory < hpagesize) {
        error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
                   "or larger than huge page size 0x%" PRIx64,
                   memory, hpagesize);
        goto error;
    }

    if (kvm_enabled() && !kvm_has_sync_mmu()) {
        error_setg(errp,
                   "host lacks kvm mmu notifiers, -mem-path unsupported");
        goto error;
    }

    if (!stat(path, &st) && S_ISDIR(st.st_mode)) {
        /* Make name safe to use with mkstemp by replacing '/' with '_'. */
        sanitized_name = g_strdup(memory_region_name(block->mr));
        for (c = sanitized_name; *c != '\0'; c++) {
            if (*c == '/') {
                *c = '_';
            }
        }

        filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
                                   sanitized_name);
        g_free(sanitized_name);

        fd = mkstemp(filename);
        if (fd >= 0) {
            unlink(filename);
        }
        g_free(filename);
    } else {
        fd = open(path, O_RDWR | O_CREAT, 0644);
    }

    if (fd < 0) {
        error_setg_errno(errp, errno,
                         "unable to create backing store for hugepages");
        goto error;
    }

    memory = ROUND_UP(memory, hpagesize);

    /*
     * ftruncate is not supported by hugetlbfs in older
     * hosts, so don't bother bailing out on errors.
     * If anything goes wrong with it under other filesystems,
     * mmap will fail.
     */
    if (ftruncate(fd, memory)) {
        perror("ftruncate");
    }

    area = qemu_ram_mmap(fd, memory, hpagesize, block->flags & RAM_SHARED);
    if (area == MAP_FAILED) {
        error_setg_errno(errp, errno,
                         "unable to map backing store for hugepages");
        close(fd);
        goto error;
    }

    if (mem_prealloc) {
        os_mem_prealloc(fd, area, memory);
    }

    block->fd = fd;
    return area;

error:
    return NULL;
}
#endif

/* Called with the ramlist lock held.  */
static ram_addr_t find_ram_offset(ram_addr_t size)
{
    RAMBlock *block, *next_block;
    ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;

    assert(size != 0); /* it would hand out same offset multiple times */

    if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
        return 0;
    }

    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
        ram_addr_t end, next = RAM_ADDR_MAX;

        end = block->offset + block->max_length;

        QLIST_FOREACH_RCU(next_block, &ram_list.blocks, next) {
            if (next_block->offset >= end) {
                next = MIN(next, next_block->offset);
            }
        }
        if (next - end >= size && next - end < mingap) {
            offset = end;
            mingap = next - end;
        }
    }

    if (offset == RAM_ADDR_MAX) {
        fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
                (uint64_t)size);
        abort();
    }

    return offset;
}

ram_addr_t last_ram_offset(void)
{
    RAMBlock *block;
    ram_addr_t last = 0;

    rcu_read_lock();
    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
        last = MAX(last, block->offset + block->max_length);
    }
    rcu_read_unlock();
    return last;
}

static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
{
    int ret;

    /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
    if (!machine_dump_guest_core(current_machine)) {
        ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
        if (ret) {
            perror("qemu_madvise");
            fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
                            "but dump_guest_core=off specified\n");
        }
    }
}

/* Called within an RCU critical section, or while the ramlist lock
 * is held.
 */
static RAMBlock *find_ram_block(ram_addr_t addr)
{
    RAMBlock *block;

    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
        if (block->offset == addr) {
            return block;
        }
    }

    return NULL;
}

const char *qemu_ram_get_idstr(RAMBlock *rb)
{
    return rb->idstr;
}

/* Called with iothread lock held.  */
void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
{
    RAMBlock *new_block, *block;

    rcu_read_lock();
    new_block = find_ram_block(addr);
    assert(new_block);
    assert(!new_block->idstr[0]);

    if (dev) {
        char *id = qdev_get_dev_path(dev);
        if (id) {
            snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
            g_free(id);
        }
    }
    pstrcat(new_block->idstr, sizeof(new_block->idstr), name);

    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
        if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
            fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
                    new_block->idstr);
            abort();
        }
    }
    rcu_read_unlock();
}

/* Called with iothread lock held.  */
void qemu_ram_unset_idstr(ram_addr_t addr)
{
    RAMBlock *block;

    /* FIXME: arch_init.c assumes that this is not called throughout
     * migration.  Ignore the problem since hot-unplug during migration
     * does not work anyway.
     */

    rcu_read_lock();
    block = find_ram_block(addr);
    if (block) {
        memset(block->idstr, 0, sizeof(block->idstr));
    }
    rcu_read_unlock();
}

static int memory_try_enable_merging(void *addr, size_t len)
{
    if (!machine_mem_merge(current_machine)) {
        /* disabled by the user */
        return 0;
    }

    return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
}

/* Only legal before guest might have detected the memory size: e.g. on
 * incoming migration, or right after reset.
 *
 * As memory core doesn't know how is memory accessed, it is up to
 * resize callback to update device state and/or add assertions to detect
 * misuse, if necessary.
 */
int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp)
{
    RAMBlock *block = find_ram_block(base);

    assert(block);

    newsize = HOST_PAGE_ALIGN(newsize);

    if (block->used_length == newsize) {
        return 0;
    }

    if (!(block->flags & RAM_RESIZEABLE)) {
        error_setg_errno(errp, EINVAL,
                         "Length mismatch: %s: 0x" RAM_ADDR_FMT
                         " in != 0x" RAM_ADDR_FMT, block->idstr,
                         newsize, block->used_length);
        return -EINVAL;
    }

    if (block->max_length < newsize) {
        error_setg_errno(errp, EINVAL,
                         "Length too large: %s: 0x" RAM_ADDR_FMT
                         " > 0x" RAM_ADDR_FMT, block->idstr,
                         newsize, block->max_length);
        return -EINVAL;
    }

    cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
    block->used_length = newsize;
    cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
                                        DIRTY_CLIENTS_ALL);
    memory_region_set_size(block->mr, newsize);
    if (block->resized) {
        block->resized(block->idstr, newsize, block->host);
    }
    return 0;
}

static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
{
    RAMBlock *block;
    RAMBlock *last_block = NULL;
    ram_addr_t old_ram_size, new_ram_size;

    old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;

    qemu_mutex_lock_ramlist();
    new_block->offset = find_ram_offset(new_block->max_length);

    if (!new_block->host) {
        if (xen_enabled()) {
            xen_ram_alloc(new_block->offset, new_block->max_length,
                          new_block->mr);
        } else {
            new_block->host = phys_mem_alloc(new_block->max_length,
                                             &new_block->mr->align);
            if (!new_block->host) {
                error_setg_errno(errp, errno,
                                 "cannot set up guest memory '%s'",
                                 memory_region_name(new_block->mr));
                qemu_mutex_unlock_ramlist();
                return -1;
            }
            memory_try_enable_merging(new_block->host, new_block->max_length);
        }
    }

    new_ram_size = MAX(old_ram_size,
              (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
    if (new_ram_size > old_ram_size) {
        migration_bitmap_extend(old_ram_size, new_ram_size);
    }
    /* Keep the list sorted from biggest to smallest block.  Unlike QTAILQ,
     * QLIST (which has an RCU-friendly variant) does not have insertion at
     * tail, so save the last element in last_block.
     */
    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
        last_block = block;
        if (block->max_length < new_block->max_length) {
            break;
        }
    }
    if (block) {
        QLIST_INSERT_BEFORE_RCU(block, new_block, next);
    } else if (last_block) {
        QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
    } else { /* list is empty */
        QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
    }
    ram_list.mru_block = NULL;

    /* Write list before version */
    smp_wmb();
    ram_list.version++;
    qemu_mutex_unlock_ramlist();

    new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;

    if (new_ram_size > old_ram_size) {
        int i;

        /* ram_list.dirty_memory[] is protected by the iothread lock.  */
        for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
            ram_list.dirty_memory[i] =
                bitmap_zero_extend(ram_list.dirty_memory[i],
                                   old_ram_size, new_ram_size);
       }
    }
    cpu_physical_memory_set_dirty_range(new_block->offset,
                                        new_block->used_length,
                                        DIRTY_CLIENTS_ALL);

    if (new_block->host) {
        qemu_ram_setup_dump(new_block->host, new_block->max_length);
        qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
        qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
        if (kvm_enabled()) {
            kvm_setup_guest_memory(new_block->host, new_block->max_length);
        }
    }

    return new_block->offset;
}

#ifdef __linux__
ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
                                    bool share, const char *mem_path,
                                    Error **errp)
{
    RAMBlock *new_block;
    ram_addr_t addr;
    Error *local_err = NULL;

    if (xen_enabled()) {
        error_setg(errp, "-mem-path not supported with Xen");
        return -1;
    }

    if (phys_mem_alloc != qemu_anon_ram_alloc) {
        /*
         * file_ram_alloc() needs to allocate just like
         * phys_mem_alloc, but we haven't bothered to provide
         * a hook there.
         */
        error_setg(errp,
                   "-mem-path not supported with this accelerator");
        return -1;
    }

    size = HOST_PAGE_ALIGN(size);
    new_block = g_malloc0(sizeof(*new_block));
    new_block->mr = mr;
    new_block->used_length = size;
    new_block->max_length = size;
    new_block->flags = share ? RAM_SHARED : 0;
    new_block->flags |= RAM_FILE;
    new_block->host = file_ram_alloc(new_block, size,
                                     mem_path, errp);
    if (!new_block->host) {
        g_free(new_block);
        return -1;
    }

    addr = ram_block_add(new_block, &local_err);
    if (local_err) {
        g_free(new_block);
        error_propagate(errp, local_err);
        return -1;
    }
    return addr;
}
#endif

static
ram_addr_t qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
                                   void (*resized)(const char*,
                                                   uint64_t length,
                                                   void *host),
                                   void *host, bool resizeable,
                                   MemoryRegion *mr, Error **errp)
{
    RAMBlock *new_block;
    ram_addr_t addr;
    Error *local_err = NULL;

    size = HOST_PAGE_ALIGN(size);
    max_size = HOST_PAGE_ALIGN(max_size);
    new_block = g_malloc0(sizeof(*new_block));
    new_block->mr = mr;
    new_block->resized = resized;
    new_block->used_length = size;
    new_block->max_length = max_size;
    assert(max_size >= size);
    new_block->fd = -1;
    new_block->host = host;
    if (host) {
        new_block->flags |= RAM_PREALLOC;
    }
    if (resizeable) {
        new_block->flags |= RAM_RESIZEABLE;
    }
    addr = ram_block_add(new_block, &local_err);
    if (local_err) {
        g_free(new_block);
        error_propagate(errp, local_err);
        return -1;
    }
    return addr;
}

ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
                                   MemoryRegion *mr, Error **errp)
{
    return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
}

ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
{
    return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
}

ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
                                     void (*resized)(const char*,
                                                     uint64_t length,
                                                     void *host),
                                     MemoryRegion *mr, Error **errp)
{
    return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);
}

void qemu_ram_free_from_ptr(ram_addr_t addr)
{
    RAMBlock *block;

    qemu_mutex_lock_ramlist();
    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
        if (addr == block->offset) {
            QLIST_REMOVE_RCU(block, next);
            ram_list.mru_block = NULL;
            /* Write list before version */
            smp_wmb();
            ram_list.version++;
            g_free_rcu(block, rcu);
            break;
        }
    }
    qemu_mutex_unlock_ramlist();
}

static void reclaim_ramblock(RAMBlock *block)
{
    if (block->flags & RAM_PREALLOC) {
        ;
    } else if (xen_enabled()) {
        xen_invalidate_map_cache_entry(block->host);
#ifndef _WIN32
    } else if (block->fd >= 0) {
        if (block->flags & RAM_FILE) {
            qemu_ram_munmap(block->host, block->max_length);
        } else {
            munmap(block->host, block->max_length);
        }
        close(block->fd);
#endif
    } else {
        qemu_anon_ram_free(block->host, block->max_length);
    }
    g_free(block);
}

void qemu_ram_free(ram_addr_t addr)
{
    RAMBlock *block;

    qemu_mutex_lock_ramlist();
    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
        if (addr == block->offset) {
            QLIST_REMOVE_RCU(block, next);
            ram_list.mru_block = NULL;
            /* Write list before version */
            smp_wmb();
            ram_list.version++;
            call_rcu(block, reclaim_ramblock, rcu);
            break;
        }
    }
    qemu_mutex_unlock_ramlist();
}

#ifndef _WIN32
void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
{
    RAMBlock *block;
    ram_addr_t offset;
    int flags;
    void *area, *vaddr;

    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
        offset = addr - block->offset;
        if (offset < block->max_length) {
            vaddr = ramblock_ptr(block, offset);
            if (block->flags & RAM_PREALLOC) {
                ;
            } else if (xen_enabled()) {
                abort();
            } else {
                flags = MAP_FIXED;
                if (block->fd >= 0) {
                    flags |= (block->flags & RAM_SHARED ?
                              MAP_SHARED : MAP_PRIVATE);
                    area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
                                flags, block->fd, offset);
                } else {
                    /*
                     * Remap needs to match alloc.  Accelerators that
                     * set phys_mem_alloc never remap.  If they did,
                     * we'd need a remap hook here.
                     */
                    assert(phys_mem_alloc == qemu_anon_ram_alloc);

                    flags |= MAP_PRIVATE | MAP_ANONYMOUS;
                    area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
                                flags, -1, 0);
                }
                if (area != vaddr) {
                    fprintf(stderr, "Could not remap addr: "
                            RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
                            length, addr);
                    exit(1);
                }
                memory_try_enable_merging(vaddr, length);
                qemu_ram_setup_dump(vaddr, length);
            }
        }
    }
}
#endif /* !_WIN32 */

int qemu_get_ram_fd(ram_addr_t addr)
{
    RAMBlock *block;
    int fd;

    rcu_read_lock();
    block = qemu_get_ram_block(addr);
    fd = block->fd;
    rcu_read_unlock();
    return fd;
}

void *qemu_get_ram_block_host_ptr(ram_addr_t addr)
{
    RAMBlock *block;
    void *ptr;

    rcu_read_lock();
    block = qemu_get_ram_block(addr);
    ptr = ramblock_ptr(block, 0);
    rcu_read_unlock();
    return ptr;
}

/* Return a host pointer to ram allocated with qemu_ram_alloc.
 * This should not be used for general purpose DMA.  Use address_space_map
 * or address_space_rw instead. For local memory (e.g. video ram) that the
 * device owns, use memory_region_get_ram_ptr.
 *
 * By the time this function returns, the returned pointer is not protected
 * by RCU anymore.  If the caller is not within an RCU critical section and
 * does not hold the iothread lock, it must have other means of protecting the
 * pointer, such as a reference to the region that includes the incoming
 * ram_addr_t.
 */
void *qemu_get_ram_ptr(ram_addr_t addr)
{
    RAMBlock *block;
    void *ptr;

    rcu_read_lock();
    block = qemu_get_ram_block(addr);

    if (xen_enabled() && block->host == NULL) {
        /* We need to check if the requested address is in the RAM
         * because we don't want to map the entire memory in QEMU.
         * In that case just map until the end of the page.
         */
        if (block->offset == 0) {
            ptr = xen_map_cache(addr, 0, 0);
            goto unlock;
        }

        block->host = xen_map_cache(block->offset, block->max_length, 1);
    }
    ptr = ramblock_ptr(block, addr - block->offset);

unlock:
    rcu_read_unlock();
    return ptr;
}

/* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
 * but takes a size argument.
 *
 * By the time this function returns, the returned pointer is not protected
 * by RCU anymore.  If the caller is not within an RCU critical section and
 * does not hold the iothread lock, it must have other means of protecting the
 * pointer, such as a reference to the region that includes the incoming
 * ram_addr_t.
 */
static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
{
    void *ptr;
    if (*size == 0) {
        return NULL;
    }
    if (xen_enabled()) {
        return xen_map_cache(addr, *size, 1);
    } else {
        RAMBlock *block;
        rcu_read_lock();
        QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
            if (addr - block->offset < block->max_length) {
                if (addr - block->offset + *size > block->max_length)
                    *size = block->max_length - addr + block->offset;
                ptr = ramblock_ptr(block, addr - block->offset);
                rcu_read_unlock();
                return ptr;
            }
        }

        fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
        abort();
    }
}

/*
 * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
 * in that RAMBlock.
 *
 * ptr: Host pointer to look up
 * round_offset: If true round the result offset down to a page boundary
 * *ram_addr: set to result ram_addr
 * *offset: set to result offset within the RAMBlock
 *
 * Returns: RAMBlock (or NULL if not found)
 *
 * By the time this function returns, the returned pointer is not protected
 * by RCU anymore.  If the caller is not within an RCU critical section and
 * does not hold the iothread lock, it must have other means of protecting the
 * pointer, such as a reference to the region that includes the incoming
 * ram_addr_t.
 */
RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
                                   ram_addr_t *ram_addr,
                                   ram_addr_t *offset)
{
    RAMBlock *block;
    uint8_t *host = ptr;

    if (xen_enabled()) {
        rcu_read_lock();
        *ram_addr = xen_ram_addr_from_mapcache(ptr);
        block = qemu_get_ram_block(*ram_addr);
        if (block) {
            *offset = (host - block->host);
        }
        rcu_read_unlock();
        return block;
    }

    rcu_read_lock();
    block = atomic_rcu_read(&ram_list.mru_block);
    if (block && block->host && host - block->host < block->max_length) {
        goto found;
    }

    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
        /* This case append when the block is not mapped. */
        if (block->host == NULL) {
            continue;
        }
        if (host - block->host < block->max_length) {
            goto found;
        }
    }

    rcu_read_unlock();
    return NULL;

found:
    *offset = (host - block->host);
    if (round_offset) {
        *offset &= TARGET_PAGE_MASK;
    }
    *ram_addr = block->offset + *offset;
    rcu_read_unlock();
    return block;
}

/*
 * Finds the named RAMBlock
 *
 * name: The name of RAMBlock to find
 *
 * Returns: RAMBlock (or NULL if not found)
 */
RAMBlock *qemu_ram_block_by_name(const char *name)
{
    RAMBlock *block;

    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
        if (!strcmp(name, block->idstr)) {
            return block;
        }
    }

    return NULL;
}

/* Some of the softmmu routines need to translate from a host pointer
   (typically a TLB entry) back to a ram offset.  */
MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
{
    RAMBlock *block;
    ram_addr_t offset; /* Not used */

    block = qemu_ram_block_from_host(ptr, false, ram_addr, &offset);

    if (!block) {
        return NULL;
    }

    return block->mr;
}

static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
                               uint64_t val, unsigned size)
{
    if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
        tb_invalidate_phys_page_fast(ram_addr, size);
    }
    switch (size) {
    case 1:
        stb_p(qemu_get_ram_ptr(ram_addr), val);
        break;
    case 2:
        stw_p(qemu_get_ram_ptr(ram_addr), val);
        break;
    case 4:
        stl_p(qemu_get_ram_ptr(ram_addr), val);
        break;
    default:
        abort();
    }
    /* Set both VGA and migration bits for simplicity and to remove
     * the notdirty callback faster.
     */
    cpu_physical_memory_set_dirty_range(ram_addr, size,
                                        DIRTY_CLIENTS_NOCODE);
    /* we remove the notdirty callback only if the code has been
       flushed */
    if (!cpu_physical_memory_is_clean(ram_addr)) {
        tlb_set_dirty(current_cpu, current_cpu->mem_io_vaddr);
    }
}

static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
                                 unsigned size, bool is_write)
{
    return is_write;
}

static const MemoryRegionOps notdirty_mem_ops = {
    .write = notdirty_mem_write,
    .valid.accepts = notdirty_mem_accepts,
    .endianness = DEVICE_NATIVE_ENDIAN,
};

/* Generate a debug exception if a watchpoint has been hit.  */
static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
{
    CPUState *cpu = current_cpu;
    CPUArchState *env = cpu->env_ptr;
    target_ulong pc, cs_base;
    target_ulong vaddr;
    CPUWatchpoint *wp;
    int cpu_flags;

    if (cpu->watchpoint_hit) {
        /* We re-entered the check after replacing the TB. Now raise
         * the debug interrupt so that is will trigger after the
         * current instruction. */
        cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
        return;
    }
    vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
    QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
        if (cpu_watchpoint_address_matches(wp, vaddr, len)
            && (wp->flags & flags)) {
            if (flags == BP_MEM_READ) {
                wp->flags |= BP_WATCHPOINT_HIT_READ;
            } else {
                wp->flags |= BP_WATCHPOINT_HIT_WRITE;
            }
            wp->hitaddr = vaddr;
            wp->hitattrs = attrs;
            if (!cpu->watchpoint_hit) {
                cpu->watchpoint_hit = wp;
                tb_check_watchpoint(cpu);
                if (wp->flags & BP_STOP_BEFORE_ACCESS) {
                    cpu->exception_index = EXCP_DEBUG;
                    cpu_loop_exit(cpu);
                } else {
                    cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
                    tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
                    cpu_resume_from_signal(cpu, NULL);
                }
            }
        } else {
            wp->flags &= ~BP_WATCHPOINT_HIT;
        }
    }
}

/* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
   so these check for a hit then pass through to the normal out-of-line
   phys routines.  */
static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
                                  unsigned size, MemTxAttrs attrs)
{
    MemTxResult res;
    uint64_t data;

    check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
    switch (size) {
    case 1:
        data = address_space_ldub(&address_space_memory, addr, attrs, &res);
        break;
    case 2:
        data = address_space_lduw(&address_space_memory, addr, attrs, &res);
        break;
    case 4:
        data = address_space_ldl(&address_space_memory, addr, attrs, &res);
        break;
    default: abort();
    }
    *pdata = data;
    return res;
}

static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
                                   uint64_t val, unsigned size,
                                   MemTxAttrs attrs)
{
    MemTxResult res;

    check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
    switch (size) {
    case 1:
        address_space_stb(&address_space_memory, addr, val, attrs, &res);
        break;
    case 2:
        address_space_stw(&address_space_memory, addr, val, attrs, &res);
        break;
    case 4:
        address_space_stl(&address_space_memory, addr, val, attrs, &res);
        break;
    default: abort();
    }
    return res;
}

static const MemoryRegionOps watch_mem_ops = {
    .read_with_attrs = watch_mem_read,
    .write_with_attrs = watch_mem_write,
    .endianness = DEVICE_NATIVE_ENDIAN,
};

static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
                                unsigned len, MemTxAttrs attrs)
{
    subpage_t *subpage = opaque;
    uint8_t buf[8];
    MemTxResult res;

#if defined(DEBUG_SUBPAGE)
    printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
           subpage, len, addr);
#endif
    res = address_space_read(subpage->as, addr + subpage->base,
                             attrs, buf, len);
    if (res) {
        return res;
    }
    switch (len) {
    case 1:
        *data = ldub_p(buf);
        return MEMTX_OK;
    case 2:
        *data = lduw_p(buf);
        return MEMTX_OK;
    case 4:
        *data = ldl_p(buf);
        return MEMTX_OK;
    case 8:
        *data = ldq_p(buf);
        return MEMTX_OK;
    default:
        abort();
    }
}

static MemTxResult subpage_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned len, MemTxAttrs attrs)
{
    subpage_t *subpage = opaque;
    uint8_t buf[8];

#if defined(DEBUG_SUBPAGE)
    printf("%s: subpage %p len %u addr " TARGET_FMT_plx
           " value %"PRIx64"\n",
           __func__, subpage, len, addr, value);
#endif
    switch (len) {
    case 1:
        stb_p(buf, value);
        break;
    case 2:
        stw_p(buf, value);
        break;
    case 4:
        stl_p(buf, value);
        break;
    case 8:
        stq_p(buf, value);
        break;
    default:
        abort();
    }
    return address_space_write(subpage->as, addr + subpage->base,
                               attrs, buf, len);
}

static bool subpage_accepts(void *opaque, hwaddr addr,
                            unsigned len, bool is_write)
{
    subpage_t *subpage = opaque;
#if defined(DEBUG_SUBPAGE)
    printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
           __func__, subpage, is_write ? 'w' : 'r', len, addr);
#endif

    return address_space_access_valid(subpage->as, addr + subpage->base,
                                      len, is_write);
}

static const MemoryRegionOps subpage_ops = {
    .read_with_attrs = subpage_read,
    .write_with_attrs = subpage_write,
    .impl.min_access_size = 1,
    .impl.max_access_size = 8,
    .valid.min_access_size = 1,
    .valid.max_access_size = 8,
    .valid.accepts = subpage_accepts,
    .endianness = DEVICE_NATIVE_ENDIAN,
};

static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
                             uint16_t section)
{
    int idx, eidx;

    if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
        return -1;
    idx = SUBPAGE_IDX(start);
    eidx = SUBPAGE_IDX(end);
#if defined(DEBUG_SUBPAGE)
    printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
           __func__, mmio, start, end, idx, eidx, section);
#endif
    for (; idx <= eidx; idx++) {
        mmio->sub_section[idx] = section;
    }

    return 0;
}

static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
{
    subpage_t *mmio;

    mmio = g_malloc0(sizeof(subpage_t));

    mmio->as = as;
    mmio->base = base;
    memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
                          NULL, TARGET_PAGE_SIZE);
    mmio->iomem.subpage = true;
#if defined(DEBUG_SUBPAGE)
    printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
           mmio, base, TARGET_PAGE_SIZE);
#endif
    subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);

    return mmio;
}

static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
                              MemoryRegion *mr)
{
    assert(as);
    MemoryRegionSection section = {
        .address_space = as,
        .mr = mr,
        .offset_within_address_space = 0,
        .offset_within_region = 0,
        .size = int128_2_64(),
    };

    return phys_section_add(map, &section);
}

MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index)
{
    CPUAddressSpace *cpuas = &cpu->cpu_ases[0];
    AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
    MemoryRegionSection *sections = d->map.sections;

    return sections[index & ~TARGET_PAGE_MASK].mr;
}

static void io_mem_init(void)
{
    memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, NULL, UINT64_MAX);
    memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
                          NULL, UINT64_MAX);
    memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
                          NULL, UINT64_MAX);
    memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
                          NULL, UINT64_MAX);
}

static void mem_begin(MemoryListener *listener)
{
    AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
    AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
    uint16_t n;

    n = dummy_section(&d->map, as, &io_mem_unassigned);
    assert(n == PHYS_SECTION_UNASSIGNED);
    n = dummy_section(&d->map, as, &io_mem_notdirty);
    assert(n == PHYS_SECTION_NOTDIRTY);
    n = dummy_section(&d->map, as, &io_mem_rom);
    assert(n == PHYS_SECTION_ROM);
    n = dummy_section(&d->map, as, &io_mem_watch);
    assert(n == PHYS_SECTION_WATCH);

    d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
    d->as = as;
    as->next_dispatch = d;
}

static void address_space_dispatch_free(AddressSpaceDispatch *d)
{
    phys_sections_free(&d->map);
    g_free(d);
}

static void mem_commit(MemoryListener *listener)
{
    AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
    AddressSpaceDispatch *cur = as->dispatch;
    AddressSpaceDispatch *next = as->next_dispatch;

    phys_page_compact_all(next, next->map.nodes_nb);

    atomic_rcu_set(&as->dispatch, next);
    if (cur) {
        call_rcu(cur, address_space_dispatch_free, rcu);
    }
}

static void tcg_commit(MemoryListener *listener)
{
    CPUAddressSpace *cpuas;
    AddressSpaceDispatch *d;

    /* since each CPU stores ram addresses in its TLB cache, we must
       reset the modified entries */
    cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
    cpu_reloading_memory_map();
    /* The CPU and TLB are protected by the iothread lock.
     * We reload the dispatch pointer now because cpu_reloading_memory_map()
     * may have split the RCU critical section.
     */
    d = atomic_rcu_read(&cpuas->as->dispatch);
    cpuas->memory_dispatch = d;
    tlb_flush(cpuas->cpu, 1);
}

void address_space_init_dispatch(AddressSpace *as)
{
    as->dispatch = NULL;
    as->dispatch_listener = (MemoryListener) {
        .begin = mem_begin,
        .commit = mem_commit,
        .region_add = mem_add,
        .region_nop = mem_add,
        .priority = 0,
    };
    memory_listener_register(&as->dispatch_listener, as);
}

void address_space_unregister(AddressSpace *as)
{
    memory_listener_unregister(&as->dispatch_listener);
}

void address_space_destroy_dispatch(AddressSpace *as)
{
    AddressSpaceDispatch *d = as->dispatch;

    atomic_rcu_set(&as->dispatch, NULL);
    if (d) {
        call_rcu(d, address_space_dispatch_free, rcu);
    }
}

static void memory_map_init(void)
{
    system_memory = g_malloc(sizeof(*system_memory));

    memory_region_init(system_memory, NULL, "system", UINT64_MAX);
    address_space_init(&address_space_memory, system_memory, "memory");

    system_io = g_malloc(sizeof(*system_io));
    memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
                          65536);
    address_space_init(&address_space_io, system_io, "I/O");
}

MemoryRegion *get_system_memory(void)
{
    return system_memory;
}

MemoryRegion *get_system_io(void)
{
    return system_io;
}

#endif /* !defined(CONFIG_USER_ONLY) */

/* physical memory access (slow version, mainly for debug) */
#if defined(CONFIG_USER_ONLY)
int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
                        uint8_t *buf, int len, int is_write)
{
    int l, flags;
    target_ulong page;
    void * p;

    while (len > 0) {
        page = addr & TARGET_PAGE_MASK;
        l = (page + TARGET_PAGE_SIZE) - addr;
        if (l > len)
            l = len;
        flags = page_get_flags(page);
        if (!(flags & PAGE_VALID))
            return -1;
        if (is_write) {
            if (!(flags & PAGE_WRITE))
                return -1;
            /* XXX: this code should not depend on lock_user */
            if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
                return -1;
            memcpy(p, buf, l);
            unlock_user(p, addr, l);
        } else {
            if (!(flags & PAGE_READ))
                return -1;
            /* XXX: this code should not depend on lock_user */
            if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
                return -1;
            memcpy(buf, p, l);
            unlock_user(p, addr, 0);
        }
        len -= l;
        buf += l;
        addr += l;
    }
    return 0;
}

#else

static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
                                     hwaddr length)
{
    uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
    /* No early return if dirty_log_mask is or becomes 0, because
     * cpu_physical_memory_set_dirty_range will still call
     * xen_modified_memory.
     */
    if (dirty_log_mask) {
        dirty_log_mask =
            cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
    }
    if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
        tb_invalidate_phys_range(addr, addr + length);
        dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
    }
    cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
}

static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
{
    unsigned access_size_max = mr->ops->valid.max_access_size;

    /* Regions are assumed to support 1-4 byte accesses unless
       otherwise specified.  */
    if (access_size_max == 0) {
        access_size_max = 4;
    }

    /* Bound the maximum access by the alignment of the address.  */
    if (!mr->ops->impl.unaligned) {
        unsigned align_size_max = addr & -addr;
        if (align_size_max != 0 && align_size_max < access_size_max) {
            access_size_max = align_size_max;
        }
    }

    /* Don't attempt accesses larger than the maximum.  */
    if (l > access_size_max) {
        l = access_size_max;
    }
    l = pow2floor(l);

    return l;
}

static bool prepare_mmio_access(MemoryRegion *mr)
{
    bool unlocked = !qemu_mutex_iothread_locked();
    bool release_lock = false;

    if (unlocked && mr->global_locking) {
        qemu_mutex_lock_iothread();
        unlocked = false;
        release_lock = true;
    }
    if (mr->flush_coalesced_mmio) {
        if (unlocked) {
            qemu_mutex_lock_iothread();
        }
        qemu_flush_coalesced_mmio_buffer();
        if (unlocked) {
            qemu_mutex_unlock_iothread();
        }
    }

    return release_lock;
}

MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
                             uint8_t *buf, int len, bool is_write)
{
    hwaddr l;
    uint8_t *ptr;
    uint64_t val;
    hwaddr addr1;
    MemoryRegion *mr;
    MemTxResult result = MEMTX_OK;
    bool release_lock = false;

    rcu_read_lock();
    while (len > 0) {
        l = len;
        mr = address_space_translate(as, addr, &addr1, &l, is_write);

        if (is_write) {
            if (!memory_access_is_direct(mr, is_write)) {
                release_lock |= prepare_mmio_access(mr);
                l = memory_access_size(mr, l, addr1);
                /* XXX: could force current_cpu to NULL to avoid
                   potential bugs */
                switch (l) {
                case 8:
                    /* 64 bit write access */
                    val = ldq_p(buf);
                    result |= memory_region_dispatch_write(mr, addr1, val, 8,
                                                           attrs);
                    break;
                case 4:
                    /* 32 bit write access */
                    val = ldl_p(buf);
                    result |= memory_region_dispatch_write(mr, addr1, val, 4,
                                                           attrs);
                    break;
                case 2:
                    /* 16 bit write access */
                    val = lduw_p(buf);
                    result |= memory_region_dispatch_write(mr, addr1, val, 2,
                                                           attrs);
                    break;
                case 1:
                    /* 8 bit write access */
                    val = ldub_p(buf);
                    result |= memory_region_dispatch_write(mr, addr1, val, 1,
                                                           attrs);
                    break;
                default:
                    abort();
                }
            } else {
                addr1 += memory_region_get_ram_addr(mr);
                /* RAM case */
                ptr = qemu_get_ram_ptr(addr1);
                memcpy(ptr, buf, l);
                invalidate_and_set_dirty(mr, addr1, l);
            }
        } else {
            if (!memory_access_is_direct(mr, is_write)) {
                /* I/O case */
                release_lock |= prepare_mmio_access(mr);
                l = memory_access_size(mr, l, addr1);
                switch (l) {
                case 8:
                    /* 64 bit read access */
                    result |= memory_region_dispatch_read(mr, addr1, &val, 8,
                                                          attrs);
                    stq_p(buf, val);
                    break;
                case 4:
                    /* 32 bit read access */
                    result |= memory_region_dispatch_read(mr, addr1, &val, 4,
                                                          attrs);
                    stl_p(buf, val);
                    break;
                case 2:
                    /* 16 bit read access */
                    result |= memory_region_dispatch_read(mr, addr1, &val, 2,
                                                          attrs);
                    stw_p(buf, val);
                    break;
                case 1:
                    /* 8 bit read access */
                    result |= memory_region_dispatch_read(mr, addr1, &val, 1,
                                                          attrs);
                    stb_p(buf, val);
                    break;
                default:
                    abort();
                }
            } else {
                /* RAM case */
                ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
                memcpy(buf, ptr, l);
            }
        }

        if (release_lock) {
            qemu_mutex_unlock_iothread();
            release_lock = false;
        }

        len -= l;
        buf += l;
        addr += l;
    }
    rcu_read_unlock();

    return result;
}

MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
                                const uint8_t *buf, int len)
{
    return address_space_rw(as, addr, attrs, (uint8_t *)buf, len, true);
}

MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
                               uint8_t *buf, int len)
{
    return address_space_rw(as, addr, attrs, buf, len, false);
}


void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
                            int len, int is_write)
{
    address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
                     buf, len, is_write);
}

enum write_rom_type {
    WRITE_DATA,
    FLUSH_CACHE,
};

static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
    hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
{
    hwaddr l;
    uint8_t *ptr;
    hwaddr addr1;
    MemoryRegion *mr;

    rcu_read_lock();
    while (len > 0) {
        l = len;
        mr = address_space_translate(as, addr, &addr1, &l, true);

        if (!(memory_region_is_ram(mr) ||
              memory_region_is_romd(mr))) {
            l = memory_access_size(mr, l, addr1);
        } else {
            addr1 += memory_region_get_ram_addr(mr);
            /* ROM/RAM case */
            ptr = qemu_get_ram_ptr(addr1);
            switch (type) {
            case WRITE_DATA:
                memcpy(ptr, buf, l);
                invalidate_and_set_dirty(mr, addr1, l);
                break;
            case FLUSH_CACHE:
                flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
                break;
            }
        }
        len -= l;
        buf += l;
        addr += l;
    }
    rcu_read_unlock();
}

/* used for ROM loading : can write in RAM and ROM */
void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
                                   const uint8_t *buf, int len)
{
    cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
}

void cpu_flush_icache_range(hwaddr start, int len)
{
    /*
     * This function should do the same thing as an icache flush that was
     * triggered from within the guest. For TCG we are always cache coherent,
     * so there is no need to flush anything. For KVM / Xen we need to flush
     * the host's instruction cache at least.
     */
    if (tcg_enabled()) {
        return;
    }

    cpu_physical_memory_write_rom_internal(&address_space_memory,
                                           start, NULL, len, FLUSH_CACHE);
}

typedef struct {
    MemoryRegion *mr;
    void *buffer;
    hwaddr addr;
    hwaddr len;
    bool in_use;
} BounceBuffer;

static BounceBuffer bounce;

typedef struct MapClient {
    QEMUBH *bh;
    QLIST_ENTRY(MapClient) link;
} MapClient;

QemuMutex map_client_list_lock;
static QLIST_HEAD(map_client_list, MapClient) map_client_list
    = QLIST_HEAD_INITIALIZER(map_client_list);

static void cpu_unregister_map_client_do(MapClient *client)
{
    QLIST_REMOVE(client, link);
    g_free(client);
}

static void cpu_notify_map_clients_locked(void)
{
    MapClient *client;

    while (!QLIST_EMPTY(&map_client_list)) {
        client = QLIST_FIRST(&map_client_list);
        qemu_bh_schedule(client->bh);
        cpu_unregister_map_client_do(client);
    }
}

void cpu_register_map_client(QEMUBH *bh)
{
    MapClient *client = g_malloc(sizeof(*client));

    qemu_mutex_lock(&map_client_list_lock);
    client->bh = bh;
    QLIST_INSERT_HEAD(&map_client_list, client, link);
    if (!atomic_read(&bounce.in_use)) {
        cpu_notify_map_clients_locked();
    }
    qemu_mutex_unlock(&map_client_list_lock);
}

void cpu_exec_init_all(void)
{
    qemu_mutex_init(&ram_list.mutex);
    io_mem_init();
    memory_map_init();
    qemu_mutex_init(&map_client_list_lock);
}

void cpu_unregister_map_client(QEMUBH *bh)
{
    MapClient *client;

    qemu_mutex_lock(&map_client_list_lock);
    QLIST_FOREACH(client, &map_client_list, link) {
        if (client->bh == bh) {
            cpu_unregister_map_client_do(client);
            break;
        }
    }
    qemu_mutex_unlock(&map_client_list_lock);
}

static void cpu_notify_map_clients(void)
{
    qemu_mutex_lock(&map_client_list_lock);
    cpu_notify_map_clients_locked();
    qemu_mutex_unlock(&map_client_list_lock);
}

bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
{
    MemoryRegion *mr;
    hwaddr l, xlat;

    rcu_read_lock();
    while (len > 0) {
        l = len;
        mr = address_space_translate(as, addr, &xlat, &l, is_write);
        if (!memory_access_is_direct(mr, is_write)) {
            l = memory_access_size(mr, l, addr);
            if (!memory_region_access_valid(mr, xlat, l, is_write)) {
                return false;
            }
        }

        len -= l;
        addr += l;
    }
    rcu_read_unlock();
    return true;
}

/* Map a physical memory region into a host virtual address.
 * May map a subset of the requested range, given by and returned in *plen.
 * May return NULL if resources needed to perform the mapping are exhausted.
 * Use only for reads OR writes - not for read-modify-write operations.
 * Use cpu_register_map_client() to know when retrying the map operation is
 * likely to succeed.
 */
void *address_space_map(AddressSpace *as,
                        hwaddr addr,
                        hwaddr *plen,
                        bool is_write)
{
    hwaddr len = *plen;
    hwaddr done = 0;
    hwaddr l, xlat, base;
    MemoryRegion *mr, *this_mr;
    ram_addr_t raddr;

    if (len == 0) {
        return NULL;
    }

    l = len;
    rcu_read_lock();
    mr = address_space_translate(as, addr, &xlat, &l, is_write);

    if (!memory_access_is_direct(mr, is_write)) {
        if (atomic_xchg(&bounce.in_use, true)) {
            rcu_read_unlock();
            return NULL;
        }
        /* Avoid unbounded allocations */
        l = MIN(l, TARGET_PAGE_SIZE);
        bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
        bounce.addr = addr;
        bounce.len = l;

        memory_region_ref(mr);
        bounce.mr = mr;
        if (!is_write) {
            address_space_read(as, addr, MEMTXATTRS_UNSPECIFIED,
                               bounce.buffer, l);
        }

        rcu_read_unlock();
        *plen = l;
        return bounce.buffer;
    }

    base = xlat;
    raddr = memory_region_get_ram_addr(mr);

    for (;;) {
        len -= l;
        addr += l;
        done += l;
        if (len == 0) {
            break;
        }

        l = len;
        this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
        if (this_mr != mr || xlat != base + done) {
            break;
        }
    }

    memory_region_ref(mr);
    rcu_read_unlock();
    *plen = done;
    return qemu_ram_ptr_length(raddr + base, plen);
}

/* Unmaps a memory region previously mapped by address_space_map().
 * Will also mark the memory as dirty if is_write == 1.  access_len gives
 * the amount of memory that was actually read or written by the caller.
 */
void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
                         int is_write, hwaddr access_len)
{
    if (buffer != bounce.buffer) {
        MemoryRegion *mr;
        ram_addr_t addr1;

        mr = qemu_ram_addr_from_host(buffer, &addr1);
        assert(mr != NULL);
        if (is_write) {
            invalidate_and_set_dirty(mr, addr1, access_len);
        }
        if (xen_enabled()) {
            xen_invalidate_map_cache_entry(buffer);
        }
        memory_region_unref(mr);
        return;
    }
    if (is_write) {
        address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
                            bounce.buffer, access_len);
    }
    qemu_vfree(bounce.buffer);
    bounce.buffer = NULL;
    memory_region_unref(bounce.mr);
    atomic_mb_set(&bounce.in_use, false);
    cpu_notify_map_clients();
}

void *cpu_physical_memory_map(hwaddr addr,
                              hwaddr *plen,
                              int is_write)
{
    return address_space_map(&address_space_memory, addr, plen, is_write);
}

void cpu_physical_memory_unmap(void *buffer, hwaddr len,
                               int is_write, hwaddr access_len)
{
    return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
}

/* warning: addr must be aligned */
static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
                                                  MemTxAttrs attrs,
                                                  MemTxResult *result,
                                                  enum device_endian endian)
{
    uint8_t *ptr;
    uint64_t val;
    MemoryRegion *mr;
    hwaddr l = 4;
    hwaddr addr1;
    MemTxResult r;
    bool release_lock = false;

    rcu_read_lock();
    mr = address_space_translate(as, addr, &addr1, &l, false);
    if (l < 4 || !memory_access_is_direct(mr, false)) {
        release_lock |= prepare_mmio_access(mr);

        /* I/O case */
        r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
#if defined(TARGET_WORDS_BIGENDIAN)
        if (endian == DEVICE_LITTLE_ENDIAN) {
            val = bswap32(val);
        }
#else
        if (endian == DEVICE_BIG_ENDIAN) {
            val = bswap32(val);
        }
#endif
    } else {
        /* RAM case */
        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
                                & TARGET_PAGE_MASK)
                               + addr1);
        switch (endian) {
        case DEVICE_LITTLE_ENDIAN:
            val = ldl_le_p(ptr);
            break;
        case DEVICE_BIG_ENDIAN:
            val = ldl_be_p(ptr);
            break;
        default:
            val = ldl_p(ptr);
            break;
        }
        r = MEMTX_OK;
    }
    if (result) {
        *result = r;
    }
    if (release_lock) {
        qemu_mutex_unlock_iothread();
    }
    rcu_read_unlock();
    return val;
}

uint32_t address_space_ldl(AddressSpace *as, hwaddr addr,
                           MemTxAttrs attrs, MemTxResult *result)
{
    return address_space_ldl_internal(as, addr, attrs, result,
                                      DEVICE_NATIVE_ENDIAN);
}

uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
                              MemTxAttrs attrs, MemTxResult *result)
{
    return address_space_ldl_internal(as, addr, attrs, result,
                                      DEVICE_LITTLE_ENDIAN);
}

uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
                              MemTxAttrs attrs, MemTxResult *result)
{
    return address_space_ldl_internal(as, addr, attrs, result,
                                      DEVICE_BIG_ENDIAN);
}

uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
{
    return address_space_ldl(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
}

uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
{
    return address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
}

uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
{
    return address_space_ldl_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
}

/* warning: addr must be aligned */
static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
                                                  MemTxAttrs attrs,
                                                  MemTxResult *result,
                                                  enum device_endian endian)
{
    uint8_t *ptr;
    uint64_t val;
    MemoryRegion *mr;
    hwaddr l = 8;
    hwaddr addr1;
    MemTxResult r;
    bool release_lock = false;

    rcu_read_lock();
    mr = address_space_translate(as, addr, &addr1, &l,
                                 false);
    if (l < 8 || !memory_access_is_direct(mr, false)) {
        release_lock |= prepare_mmio_access(mr);

        /* I/O case */
        r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
#if defined(TARGET_WORDS_BIGENDIAN)
        if (endian == DEVICE_LITTLE_ENDIAN) {
            val = bswap64(val);
        }
#else
        if (endian == DEVICE_BIG_ENDIAN) {
            val = bswap64(val);
        }
#endif
    } else {
        /* RAM case */
        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
                                & TARGET_PAGE_MASK)
                               + addr1);
        switch (endian) {
        case DEVICE_LITTLE_ENDIAN:
            val = ldq_le_p(ptr);
            break;
        case DEVICE_BIG_ENDIAN:
            val = ldq_be_p(ptr);
            break;
        default:
            val = ldq_p(ptr);
            break;
        }
        r = MEMTX_OK;
    }
    if (result) {
        *result = r;
    }
    if (release_lock) {
        qemu_mutex_unlock_iothread();
    }
    rcu_read_unlock();
    return val;
}

uint64_t address_space_ldq(AddressSpace *as, hwaddr addr,
                           MemTxAttrs attrs, MemTxResult *result)
{
    return address_space_ldq_internal(as, addr, attrs, result,
                                      DEVICE_NATIVE_ENDIAN);
}

uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
                           MemTxAttrs attrs, MemTxResult *result)
{
    return address_space_ldq_internal(as, addr, attrs, result,
                                      DEVICE_LITTLE_ENDIAN);
}

uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
                           MemTxAttrs attrs, MemTxResult *result)
{
    return address_space_ldq_internal(as, addr, attrs, result,
                                      DEVICE_BIG_ENDIAN);
}

uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
{
    return address_space_ldq(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
}

uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
{
    return address_space_ldq_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
}

uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
{
    return address_space_ldq_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
}

/* XXX: optimize */
uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
                            MemTxAttrs attrs, MemTxResult *result)
{
    uint8_t val;
    MemTxResult r;

    r = address_space_rw(as, addr, attrs, &val, 1, 0);
    if (result) {
        *result = r;
    }
    return val;
}

uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
{
    return address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
}

/* warning: addr must be aligned */
static inline uint32_t address_space_lduw_internal(AddressSpace *as,
                                                   hwaddr addr,
                                                   MemTxAttrs attrs,
                                                   MemTxResult *result,
                                                   enum device_endian endian)
{
    uint8_t *ptr;
    uint64_t val;
    MemoryRegion *mr;
    hwaddr l = 2;
    hwaddr addr1;
    MemTxResult r;
    bool release_lock = false;

    rcu_read_lock();
    mr = address_space_translate(as, addr, &addr1, &l,
                                 false);
    if (l < 2 || !memory_access_is_direct(mr, false)) {
        release_lock |= prepare_mmio_access(mr);

        /* I/O case */
        r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
#if defined(TARGET_WORDS_BIGENDIAN)
        if (endian == DEVICE_LITTLE_ENDIAN) {
            val = bswap16(val);
        }
#else
        if (endian == DEVICE_BIG_ENDIAN) {
            val = bswap16(val);
        }
#endif
    } else {
        /* RAM case */
        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
                                & TARGET_PAGE_MASK)
                               + addr1);
        switch (endian) {
        case DEVICE_LITTLE_ENDIAN:
            val = lduw_le_p(ptr);
            break;
        case DEVICE_BIG_ENDIAN:
            val = lduw_be_p(ptr);
            break;
        default:
            val = lduw_p(ptr);
            break;
        }
        r = MEMTX_OK;
    }
    if (result) {
        *result = r;
    }
    if (release_lock) {
        qemu_mutex_unlock_iothread();
    }
    rcu_read_unlock();
    return val;
}

uint32_t address_space_lduw(AddressSpace *as, hwaddr addr,
                           MemTxAttrs attrs, MemTxResult *result)
{
    return address_space_lduw_internal(as, addr, attrs, result,
                                       DEVICE_NATIVE_ENDIAN);
}

uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr,
                           MemTxAttrs attrs, MemTxResult *result)
{
    return address_space_lduw_internal(as, addr, attrs, result,
                                       DEVICE_LITTLE_ENDIAN);
}

uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr,
                           MemTxAttrs attrs, MemTxResult *result)
{
    return address_space_lduw_internal(as, addr, attrs, result,
                                       DEVICE_BIG_ENDIAN);
}

uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
{
    return address_space_lduw(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
}

uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
{
    return address_space_lduw_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
}

uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
{
    return address_space_lduw_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
}

/* warning: addr must be aligned. The ram page is not masked as dirty
   and the code inside is not invalidated. It is useful if the dirty
   bits are used to track modified PTEs */
void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
                                MemTxAttrs attrs, MemTxResult *result)
{
    uint8_t *ptr;
    MemoryRegion *mr;
    hwaddr l = 4;
    hwaddr addr1;
    MemTxResult r;
    uint8_t dirty_log_mask;
    bool release_lock = false;

    rcu_read_lock();
    mr = address_space_translate(as, addr, &addr1, &l,
                                 true);
    if (l < 4 || !memory_access_is_direct(mr, true)) {
        release_lock |= prepare_mmio_access(mr);

        r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
    } else {
        addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
        ptr = qemu_get_ram_ptr(addr1);
        stl_p(ptr, val);

        dirty_log_mask = memory_region_get_dirty_log_mask(mr);
        dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
        cpu_physical_memory_set_dirty_range(addr1, 4, dirty_log_mask);
        r = MEMTX_OK;
    }
    if (result) {
        *result = r;
    }
    if (release_lock) {
        qemu_mutex_unlock_iothread();
    }
    rcu_read_unlock();
}

void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
{
    address_space_stl_notdirty(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
}

/* warning: addr must be aligned */
static inline void address_space_stl_internal(AddressSpace *as,
                                              hwaddr addr, uint32_t val,
                                              MemTxAttrs attrs,
                                              MemTxResult *result,
                                              enum device_endian endian)
{
    uint8_t *ptr;
    MemoryRegion *mr;
    hwaddr l = 4;
    hwaddr addr1;
    MemTxResult r;
    bool release_lock = false;

    rcu_read_lock();
    mr = address_space_translate(as, addr, &addr1, &l,
                                 true);
    if (l < 4 || !memory_access_is_direct(mr, true)) {
        release_lock |= prepare_mmio_access(mr);

#if defined(TARGET_WORDS_BIGENDIAN)
        if (endian == DEVICE_LITTLE_ENDIAN) {
            val = bswap32(val);
        }
#else
        if (endian == DEVICE_BIG_ENDIAN) {
            val = bswap32(val);
        }
#endif
        r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
    } else {
        /* RAM case */
        addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
        ptr = qemu_get_ram_ptr(addr1);
        switch (endian) {
        case DEVICE_LITTLE_ENDIAN:
            stl_le_p(ptr, val);
            break;
        case DEVICE_BIG_ENDIAN:
            stl_be_p(ptr, val);
            break;
        default:
            stl_p(ptr, val);
            break;
        }
        invalidate_and_set_dirty(mr, addr1, 4);
        r = MEMTX_OK;
    }
    if (result) {
        *result = r;
    }
    if (release_lock) {
        qemu_mutex_unlock_iothread();
    }
    rcu_read_unlock();
}

void address_space_stl(AddressSpace *as, hwaddr addr, uint32_t val,
                       MemTxAttrs attrs, MemTxResult *result)
{
    address_space_stl_internal(as, addr, val, attrs, result,
                               DEVICE_NATIVE_ENDIAN);
}

void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val,
                       MemTxAttrs attrs, MemTxResult *result)
{
    address_space_stl_internal(as, addr, val, attrs, result,
                               DEVICE_LITTLE_ENDIAN);
}

void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val,
                       MemTxAttrs attrs, MemTxResult *result)
{
    address_space_stl_internal(as, addr, val, attrs, result,
                               DEVICE_BIG_ENDIAN);
}

void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
{
    address_space_stl(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
}

void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
{
    address_space_stl_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
}

void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
{
    address_space_stl_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
}

/* XXX: optimize */
void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val,
                       MemTxAttrs attrs, MemTxResult *result)
{
    uint8_t v = val;
    MemTxResult r;

    r = address_space_rw(as, addr, attrs, &v, 1, 1);
    if (result) {
        *result = r;
    }
}

void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
{
    address_space_stb(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
}

/* warning: addr must be aligned */
static inline void address_space_stw_internal(AddressSpace *as,
                                              hwaddr addr, uint32_t val,
                                              MemTxAttrs attrs,
                                              MemTxResult *result,
                                              enum device_endian endian)
{
    uint8_t *ptr;
    MemoryRegion *mr;
    hwaddr l = 2;
    hwaddr addr1;
    MemTxResult r;
    bool release_lock = false;

    rcu_read_lock();
    mr = address_space_translate(as, addr, &addr1, &l, true);
    if (l < 2 || !memory_access_is_direct(mr, true)) {
        release_lock |= prepare_mmio_access(mr);

#if defined(TARGET_WORDS_BIGENDIAN)
        if (endian == DEVICE_LITTLE_ENDIAN) {
            val = bswap16(val);
        }
#else
        if (endian == DEVICE_BIG_ENDIAN) {
            val = bswap16(val);
        }
#endif
        r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
    } else {
        /* RAM case */
        addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
        ptr = qemu_get_ram_ptr(addr1);
        switch (endian) {
        case DEVICE_LITTLE_ENDIAN:
            stw_le_p(ptr, val);
            break;
        case DEVICE_BIG_ENDIAN:
            stw_be_p(ptr, val);
            break;
        default:
            stw_p(ptr, val);
            break;
        }
        invalidate_and_set_dirty(mr, addr1, 2);
        r = MEMTX_OK;
    }
    if (result) {
        *result = r;
    }
    if (release_lock) {
        qemu_mutex_unlock_iothread();
    }
    rcu_read_unlock();
}

void address_space_stw(AddressSpace *as, hwaddr addr, uint32_t val,
                       MemTxAttrs attrs, MemTxResult *result)
{
    address_space_stw_internal(as, addr, val, attrs, result,
                               DEVICE_NATIVE_ENDIAN);
}

void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val,
                       MemTxAttrs attrs, MemTxResult *result)
{
    address_space_stw_internal(as, addr, val, attrs, result,
                               DEVICE_LITTLE_ENDIAN);
}

void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val,
                       MemTxAttrs attrs, MemTxResult *result)
{
    address_space_stw_internal(as, addr, val, attrs, result,
                               DEVICE_BIG_ENDIAN);
}

void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
{
    address_space_stw(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
}

void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
{
    address_space_stw_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
}

void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
{
    address_space_stw_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
}

/* XXX: optimize */
void address_space_stq(AddressSpace *as, hwaddr addr, uint64_t val,
                       MemTxAttrs attrs, MemTxResult *result)
{
    MemTxResult r;
    val = tswap64(val);
    r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
    if (result) {
        *result = r;
    }
}

void address_space_stq_le(AddressSpace *as, hwaddr addr, uint64_t val,
                       MemTxAttrs attrs, MemTxResult *result)
{
    MemTxResult r;
    val = cpu_to_le64(val);
    r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
    if (result) {
        *result = r;
    }
}
void address_space_stq_be(AddressSpace *as, hwaddr addr, uint64_t val,
                       MemTxAttrs attrs, MemTxResult *result)
{
    MemTxResult r;
    val = cpu_to_be64(val);
    r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
    if (result) {
        *result = r;
    }
}