2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301 USA
24 #include <sys/types.h>
37 #include "qemu-common.h"
42 #if defined(CONFIG_USER_ONLY)
46 //#define DEBUG_TB_INVALIDATE
49 //#define DEBUG_UNASSIGNED
51 /* make various TB consistency checks */
52 //#define DEBUG_TB_CHECK
53 //#define DEBUG_TLB_CHECK
55 //#define DEBUG_IOPORT
56 //#define DEBUG_SUBPAGE
58 #if !defined(CONFIG_USER_ONLY)
59 /* TB consistency checks only implemented for usermode emulation. */
63 #define SMC_BITMAP_USE_THRESHOLD 10
65 #if defined(TARGET_SPARC64)
66 #define TARGET_PHYS_ADDR_SPACE_BITS 41
67 #elif defined(TARGET_SPARC)
68 #define TARGET_PHYS_ADDR_SPACE_BITS 36
69 #elif defined(TARGET_ALPHA)
70 #define TARGET_PHYS_ADDR_SPACE_BITS 42
71 #define TARGET_VIRT_ADDR_SPACE_BITS 42
72 #elif defined(TARGET_PPC64)
73 #define TARGET_PHYS_ADDR_SPACE_BITS 42
74 #elif defined(TARGET_X86_64) && !defined(USE_KQEMU)
75 #define TARGET_PHYS_ADDR_SPACE_BITS 42
76 #elif defined(TARGET_I386) && !defined(USE_KQEMU)
77 #define TARGET_PHYS_ADDR_SPACE_BITS 36
79 /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
80 #define TARGET_PHYS_ADDR_SPACE_BITS 32
83 static TranslationBlock *tbs;
84 int code_gen_max_blocks;
85 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
87 /* any access to the tbs or the page table must use this lock */
88 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
90 #if defined(__arm__) || defined(__sparc_v9__)
91 /* The prologue must be reachable with a direct jump. ARM and Sparc64
92 have limited branch ranges (possibly also PPC) so place it in a
93 section close to code segment. */
94 #define code_gen_section \
95 __attribute__((__section__(".gen_code"))) \
96 __attribute__((aligned (32)))
98 #define code_gen_section \
99 __attribute__((aligned (32)))
102 uint8_t code_gen_prologue[1024] code_gen_section;
103 static uint8_t *code_gen_buffer;
104 static unsigned long code_gen_buffer_size;
105 /* threshold to flush the translated code buffer */
106 static unsigned long code_gen_buffer_max_size;
107 uint8_t *code_gen_ptr;
109 #if !defined(CONFIG_USER_ONLY)
110 ram_addr_t phys_ram_size;
112 uint8_t *phys_ram_base;
113 uint8_t *phys_ram_dirty;
114 static int in_migration;
115 static ram_addr_t phys_ram_alloc_offset = 0;
119 /* current CPU in the current thread. It is only valid inside
121 CPUState *cpu_single_env;
122 /* 0 = Do not count executed instructions.
123 1 = Precise instruction counting.
124 2 = Adaptive rate instruction counting. */
126 /* Current instruction counter. While executing translated code this may
127 include some instructions that have not yet been executed. */
130 typedef struct PageDesc {
131 /* list of TBs intersecting this ram page */
132 TranslationBlock *first_tb;
133 /* in order to optimize self modifying code, we count the number
134 of lookups we do to a given page to use a bitmap */
135 unsigned int code_write_count;
136 uint8_t *code_bitmap;
137 #if defined(CONFIG_USER_ONLY)
142 typedef struct PhysPageDesc {
143 /* offset in host memory of the page + io_index in the low bits */
144 ram_addr_t phys_offset;
145 ram_addr_t region_offset;
149 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
150 /* XXX: this is a temporary hack for alpha target.
151 * In the future, this is to be replaced by a multi-level table
152 * to actually be able to handle the complete 64 bits address space.
154 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
156 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
159 #define L1_SIZE (1 << L1_BITS)
160 #define L2_SIZE (1 << L2_BITS)
162 unsigned long qemu_real_host_page_size;
163 unsigned long qemu_host_page_bits;
164 unsigned long qemu_host_page_size;
165 unsigned long qemu_host_page_mask;
167 /* XXX: for system emulation, it could just be an array */
168 static PageDesc *l1_map[L1_SIZE];
169 static PhysPageDesc **l1_phys_map;
171 #if !defined(CONFIG_USER_ONLY)
172 static void io_mem_init(void);
174 /* io memory support */
175 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
176 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
177 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
178 static char io_mem_used[IO_MEM_NB_ENTRIES];
179 static int io_mem_watch;
183 static const char *logfilename = "/tmp/qemu.log";
186 static int log_append = 0;
189 static int tlb_flush_count;
190 static int tb_flush_count;
191 static int tb_phys_invalidate_count;
193 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
194 typedef struct subpage_t {
195 target_phys_addr_t base;
196 CPUReadMemoryFunc **mem_read[TARGET_PAGE_SIZE][4];
197 CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
198 void *opaque[TARGET_PAGE_SIZE][2][4];
199 ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
203 static void map_exec(void *addr, long size)
206 VirtualProtect(addr, size,
207 PAGE_EXECUTE_READWRITE, &old_protect);
211 static void map_exec(void *addr, long size)
213 unsigned long start, end, page_size;
215 page_size = getpagesize();
216 start = (unsigned long)addr;
217 start &= ~(page_size - 1);
219 end = (unsigned long)addr + size;
220 end += page_size - 1;
221 end &= ~(page_size - 1);
223 mprotect((void *)start, end - start,
224 PROT_READ | PROT_WRITE | PROT_EXEC);
228 static void page_init(void)
230 /* NOTE: we can always suppose that qemu_host_page_size >=
234 SYSTEM_INFO system_info;
236 GetSystemInfo(&system_info);
237 qemu_real_host_page_size = system_info.dwPageSize;
240 qemu_real_host_page_size = getpagesize();
242 if (qemu_host_page_size == 0)
243 qemu_host_page_size = qemu_real_host_page_size;
244 if (qemu_host_page_size < TARGET_PAGE_SIZE)
245 qemu_host_page_size = TARGET_PAGE_SIZE;
246 qemu_host_page_bits = 0;
247 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
248 qemu_host_page_bits++;
249 qemu_host_page_mask = ~(qemu_host_page_size - 1);
250 l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
251 memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
253 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
255 long long startaddr, endaddr;
260 last_brk = (unsigned long)sbrk(0);
261 f = fopen("/proc/self/maps", "r");
264 n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
266 startaddr = MIN(startaddr,
267 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
268 endaddr = MIN(endaddr,
269 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
270 page_set_flags(startaddr & TARGET_PAGE_MASK,
271 TARGET_PAGE_ALIGN(endaddr),
282 static inline PageDesc **page_l1_map(target_ulong index)
284 #if TARGET_LONG_BITS > 32
285 /* Host memory outside guest VM. For 32-bit targets we have already
286 excluded high addresses. */
287 if (index > ((target_ulong)L2_SIZE * L1_SIZE))
290 return &l1_map[index >> L2_BITS];
293 static inline PageDesc *page_find_alloc(target_ulong index)
296 lp = page_l1_map(index);
302 /* allocate if not found */
303 #if defined(CONFIG_USER_ONLY)
304 size_t len = sizeof(PageDesc) * L2_SIZE;
305 /* Don't use qemu_malloc because it may recurse. */
306 p = mmap(0, len, PROT_READ | PROT_WRITE,
307 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
310 unsigned long addr = h2g(p);
311 page_set_flags(addr & TARGET_PAGE_MASK,
312 TARGET_PAGE_ALIGN(addr + len),
316 p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
320 return p + (index & (L2_SIZE - 1));
323 static inline PageDesc *page_find(target_ulong index)
326 lp = page_l1_map(index);
333 return p + (index & (L2_SIZE - 1));
336 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
341 p = (void **)l1_phys_map;
342 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
344 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
345 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
347 lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
350 /* allocate if not found */
353 p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
354 memset(p, 0, sizeof(void *) * L1_SIZE);
358 lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
362 /* allocate if not found */
365 pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
367 for (i = 0; i < L2_SIZE; i++) {
368 pd[i].phys_offset = IO_MEM_UNASSIGNED;
369 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
372 return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
375 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
377 return phys_page_find_alloc(index, 0);
380 #if !defined(CONFIG_USER_ONLY)
381 static void tlb_protect_code(ram_addr_t ram_addr);
382 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
384 #define mmap_lock() do { } while(0)
385 #define mmap_unlock() do { } while(0)
388 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
390 #if defined(CONFIG_USER_ONLY)
391 /* Currently it is not recommanded to allocate big chunks of data in
392 user mode. It will change when a dedicated libc will be used */
393 #define USE_STATIC_CODE_GEN_BUFFER
396 #ifdef USE_STATIC_CODE_GEN_BUFFER
397 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
400 static void code_gen_alloc(unsigned long tb_size)
402 #ifdef USE_STATIC_CODE_GEN_BUFFER
403 code_gen_buffer = static_code_gen_buffer;
404 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
405 map_exec(code_gen_buffer, code_gen_buffer_size);
407 code_gen_buffer_size = tb_size;
408 if (code_gen_buffer_size == 0) {
409 #if defined(CONFIG_USER_ONLY)
410 /* in user mode, phys_ram_size is not meaningful */
411 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
413 /* XXX: needs ajustments */
414 code_gen_buffer_size = (unsigned long)(phys_ram_size / 4);
417 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
418 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
419 /* The code gen buffer location may have constraints depending on
420 the host cpu and OS */
421 #if defined(__linux__)
426 flags = MAP_PRIVATE | MAP_ANONYMOUS;
427 #if defined(__x86_64__)
429 /* Cannot map more than that */
430 if (code_gen_buffer_size > (800 * 1024 * 1024))
431 code_gen_buffer_size = (800 * 1024 * 1024);
432 #elif defined(__sparc_v9__)
433 // Map the buffer below 2G, so we can use direct calls and branches
435 start = (void *) 0x60000000UL;
436 if (code_gen_buffer_size > (512 * 1024 * 1024))
437 code_gen_buffer_size = (512 * 1024 * 1024);
438 #elif defined(__arm__)
439 /* Map the buffer below 32M, so we can use direct calls and branches */
441 start = (void *) 0x01000000UL;
442 if (code_gen_buffer_size > 16 * 1024 * 1024)
443 code_gen_buffer_size = 16 * 1024 * 1024;
445 code_gen_buffer = mmap(start, code_gen_buffer_size,
446 PROT_WRITE | PROT_READ | PROT_EXEC,
448 if (code_gen_buffer == MAP_FAILED) {
449 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
453 #elif defined(__FreeBSD__) || defined(__DragonFly__)
457 flags = MAP_PRIVATE | MAP_ANONYMOUS;
458 #if defined(__x86_64__)
459 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
460 * 0x40000000 is free */
462 addr = (void *)0x40000000;
463 /* Cannot map more than that */
464 if (code_gen_buffer_size > (800 * 1024 * 1024))
465 code_gen_buffer_size = (800 * 1024 * 1024);
467 code_gen_buffer = mmap(addr, code_gen_buffer_size,
468 PROT_WRITE | PROT_READ | PROT_EXEC,
470 if (code_gen_buffer == MAP_FAILED) {
471 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
476 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
477 map_exec(code_gen_buffer, code_gen_buffer_size);
479 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
480 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
481 code_gen_buffer_max_size = code_gen_buffer_size -
482 code_gen_max_block_size();
483 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
484 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
487 /* Must be called before using the QEMU cpus. 'tb_size' is the size
488 (in bytes) allocated to the translation buffer. Zero means default
490 void cpu_exec_init_all(unsigned long tb_size)
493 code_gen_alloc(tb_size);
494 code_gen_ptr = code_gen_buffer;
496 #if !defined(CONFIG_USER_ONLY)
501 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
503 #define CPU_COMMON_SAVE_VERSION 1
505 static void cpu_common_save(QEMUFile *f, void *opaque)
507 CPUState *env = opaque;
509 qemu_put_be32s(f, &env->halted);
510 qemu_put_be32s(f, &env->interrupt_request);
513 static int cpu_common_load(QEMUFile *f, void *opaque, int version_id)
515 CPUState *env = opaque;
517 if (version_id != CPU_COMMON_SAVE_VERSION)
520 qemu_get_be32s(f, &env->halted);
521 qemu_get_be32s(f, &env->interrupt_request);
522 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
523 version_id is increased. */
524 env->interrupt_request &= ~0x01;
531 void cpu_exec_init(CPUState *env)
536 #if defined(CONFIG_USER_ONLY)
539 env->next_cpu = NULL;
542 while (*penv != NULL) {
543 penv = (CPUState **)&(*penv)->next_cpu;
546 env->cpu_index = cpu_index;
547 TAILQ_INIT(&env->breakpoints);
548 TAILQ_INIT(&env->watchpoints);
550 #if defined(CONFIG_USER_ONLY)
553 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
554 register_savevm("cpu_common", cpu_index, CPU_COMMON_SAVE_VERSION,
555 cpu_common_save, cpu_common_load, env);
556 register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
557 cpu_save, cpu_load, env);
561 static inline void invalidate_page_bitmap(PageDesc *p)
563 if (p->code_bitmap) {
564 qemu_free(p->code_bitmap);
565 p->code_bitmap = NULL;
567 p->code_write_count = 0;
570 /* set to NULL all the 'first_tb' fields in all PageDescs */
571 static void page_flush_tb(void)
576 for(i = 0; i < L1_SIZE; i++) {
579 for(j = 0; j < L2_SIZE; j++) {
581 invalidate_page_bitmap(p);
588 /* flush all the translation blocks */
589 /* XXX: tb_flush is currently not thread safe */
590 void tb_flush(CPUState *env1)
593 #if defined(DEBUG_FLUSH)
594 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
595 (unsigned long)(code_gen_ptr - code_gen_buffer),
597 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
599 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
600 cpu_abort(env1, "Internal error: code buffer overflow\n");
604 for(env = first_cpu; env != NULL; env = env->next_cpu) {
605 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
608 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
611 code_gen_ptr = code_gen_buffer;
612 /* XXX: flush processor icache at this point if cache flush is
617 #ifdef DEBUG_TB_CHECK
619 static void tb_invalidate_check(target_ulong address)
621 TranslationBlock *tb;
623 address &= TARGET_PAGE_MASK;
624 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
625 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
626 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
627 address >= tb->pc + tb->size)) {
628 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
629 address, (long)tb->pc, tb->size);
635 /* verify that all the pages have correct rights for code */
636 static void tb_page_check(void)
638 TranslationBlock *tb;
639 int i, flags1, flags2;
641 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
642 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
643 flags1 = page_get_flags(tb->pc);
644 flags2 = page_get_flags(tb->pc + tb->size - 1);
645 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
646 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
647 (long)tb->pc, tb->size, flags1, flags2);
653 static void tb_jmp_check(TranslationBlock *tb)
655 TranslationBlock *tb1;
658 /* suppress any remaining jumps to this TB */
662 tb1 = (TranslationBlock *)((long)tb1 & ~3);
665 tb1 = tb1->jmp_next[n1];
667 /* check end of list */
669 printf("ERROR: jmp_list from 0x%08lx\n", (long)tb);
675 /* invalidate one TB */
676 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
679 TranslationBlock *tb1;
683 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
686 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
690 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
692 TranslationBlock *tb1;
698 tb1 = (TranslationBlock *)((long)tb1 & ~3);
700 *ptb = tb1->page_next[n1];
703 ptb = &tb1->page_next[n1];
707 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
709 TranslationBlock *tb1, **ptb;
712 ptb = &tb->jmp_next[n];
715 /* find tb(n) in circular list */
719 tb1 = (TranslationBlock *)((long)tb1 & ~3);
720 if (n1 == n && tb1 == tb)
723 ptb = &tb1->jmp_first;
725 ptb = &tb1->jmp_next[n1];
728 /* now we can suppress tb(n) from the list */
729 *ptb = tb->jmp_next[n];
731 tb->jmp_next[n] = NULL;
735 /* reset the jump entry 'n' of a TB so that it is not chained to
737 static inline void tb_reset_jump(TranslationBlock *tb, int n)
739 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
742 void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
747 target_phys_addr_t phys_pc;
748 TranslationBlock *tb1, *tb2;
750 /* remove the TB from the hash list */
751 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
752 h = tb_phys_hash_func(phys_pc);
753 tb_remove(&tb_phys_hash[h], tb,
754 offsetof(TranslationBlock, phys_hash_next));
756 /* remove the TB from the page list */
757 if (tb->page_addr[0] != page_addr) {
758 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
759 tb_page_remove(&p->first_tb, tb);
760 invalidate_page_bitmap(p);
762 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
763 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
764 tb_page_remove(&p->first_tb, tb);
765 invalidate_page_bitmap(p);
768 tb_invalidated_flag = 1;
770 /* remove the TB from the hash list */
771 h = tb_jmp_cache_hash_func(tb->pc);
772 for(env = first_cpu; env != NULL; env = env->next_cpu) {
773 if (env->tb_jmp_cache[h] == tb)
774 env->tb_jmp_cache[h] = NULL;
777 /* suppress this TB from the two jump lists */
778 tb_jmp_remove(tb, 0);
779 tb_jmp_remove(tb, 1);
781 /* suppress any remaining jumps to this TB */
787 tb1 = (TranslationBlock *)((long)tb1 & ~3);
788 tb2 = tb1->jmp_next[n1];
789 tb_reset_jump(tb1, n1);
790 tb1->jmp_next[n1] = NULL;
793 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
795 tb_phys_invalidate_count++;
798 static inline void set_bits(uint8_t *tab, int start, int len)
804 mask = 0xff << (start & 7);
805 if ((start & ~7) == (end & ~7)) {
807 mask &= ~(0xff << (end & 7));
812 start = (start + 8) & ~7;
814 while (start < end1) {
819 mask = ~(0xff << (end & 7));
825 static void build_page_bitmap(PageDesc *p)
827 int n, tb_start, tb_end;
828 TranslationBlock *tb;
830 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
835 tb = (TranslationBlock *)((long)tb & ~3);
836 /* NOTE: this is subtle as a TB may span two physical pages */
838 /* NOTE: tb_end may be after the end of the page, but
839 it is not a problem */
840 tb_start = tb->pc & ~TARGET_PAGE_MASK;
841 tb_end = tb_start + tb->size;
842 if (tb_end > TARGET_PAGE_SIZE)
843 tb_end = TARGET_PAGE_SIZE;
846 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
848 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
849 tb = tb->page_next[n];
853 TranslationBlock *tb_gen_code(CPUState *env,
854 target_ulong pc, target_ulong cs_base,
855 int flags, int cflags)
857 TranslationBlock *tb;
859 target_ulong phys_pc, phys_page2, virt_page2;
862 phys_pc = get_phys_addr_code(env, pc);
865 /* flush must be done */
867 /* cannot fail at this point */
869 /* Don't forget to invalidate previous TB info. */
870 tb_invalidated_flag = 1;
872 tc_ptr = code_gen_ptr;
874 tb->cs_base = cs_base;
877 cpu_gen_code(env, tb, &code_gen_size);
878 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
880 /* check next page if needed */
881 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
883 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
884 phys_page2 = get_phys_addr_code(env, virt_page2);
886 tb_link_phys(tb, phys_pc, phys_page2);
890 /* invalidate all TBs which intersect with the target physical page
891 starting in range [start;end[. NOTE: start and end must refer to
892 the same physical page. 'is_cpu_write_access' should be true if called
893 from a real cpu write access: the virtual CPU will exit the current
894 TB if code is modified inside this TB. */
895 void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
896 int is_cpu_write_access)
898 TranslationBlock *tb, *tb_next, *saved_tb;
899 CPUState *env = cpu_single_env;
900 target_ulong tb_start, tb_end;
903 #ifdef TARGET_HAS_PRECISE_SMC
904 int current_tb_not_found = is_cpu_write_access;
905 TranslationBlock *current_tb = NULL;
906 int current_tb_modified = 0;
907 target_ulong current_pc = 0;
908 target_ulong current_cs_base = 0;
909 int current_flags = 0;
910 #endif /* TARGET_HAS_PRECISE_SMC */
912 p = page_find(start >> TARGET_PAGE_BITS);
915 if (!p->code_bitmap &&
916 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
917 is_cpu_write_access) {
918 /* build code bitmap */
919 build_page_bitmap(p);
922 /* we remove all the TBs in the range [start, end[ */
923 /* XXX: see if in some cases it could be faster to invalidate all the code */
927 tb = (TranslationBlock *)((long)tb & ~3);
928 tb_next = tb->page_next[n];
929 /* NOTE: this is subtle as a TB may span two physical pages */
931 /* NOTE: tb_end may be after the end of the page, but
932 it is not a problem */
933 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
934 tb_end = tb_start + tb->size;
936 tb_start = tb->page_addr[1];
937 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
939 if (!(tb_end <= start || tb_start >= end)) {
940 #ifdef TARGET_HAS_PRECISE_SMC
941 if (current_tb_not_found) {
942 current_tb_not_found = 0;
944 if (env->mem_io_pc) {
945 /* now we have a real cpu fault */
946 current_tb = tb_find_pc(env->mem_io_pc);
949 if (current_tb == tb &&
950 (current_tb->cflags & CF_COUNT_MASK) != 1) {
951 /* If we are modifying the current TB, we must stop
952 its execution. We could be more precise by checking
953 that the modification is after the current PC, but it
954 would require a specialized function to partially
955 restore the CPU state */
957 current_tb_modified = 1;
958 cpu_restore_state(current_tb, env,
959 env->mem_io_pc, NULL);
960 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base,
963 #endif /* TARGET_HAS_PRECISE_SMC */
964 /* we need to do that to handle the case where a signal
965 occurs while doing tb_phys_invalidate() */
968 saved_tb = env->current_tb;
969 env->current_tb = NULL;
971 tb_phys_invalidate(tb, -1);
973 env->current_tb = saved_tb;
974 if (env->interrupt_request && env->current_tb)
975 cpu_interrupt(env, env->interrupt_request);
980 #if !defined(CONFIG_USER_ONLY)
981 /* if no code remaining, no need to continue to use slow writes */
983 invalidate_page_bitmap(p);
984 if (is_cpu_write_access) {
985 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
989 #ifdef TARGET_HAS_PRECISE_SMC
990 if (current_tb_modified) {
991 /* we generate a block containing just the instruction
992 modifying the memory. It will ensure that it cannot modify
994 env->current_tb = NULL;
995 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
996 cpu_resume_from_signal(env, NULL);
1001 /* len must be <= 8 and start must be a multiple of len */
1002 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1008 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1009 cpu_single_env->mem_io_vaddr, len,
1010 cpu_single_env->eip,
1011 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1014 p = page_find(start >> TARGET_PAGE_BITS);
1017 if (p->code_bitmap) {
1018 offset = start & ~TARGET_PAGE_MASK;
1019 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1020 if (b & ((1 << len) - 1))
1024 tb_invalidate_phys_page_range(start, start + len, 1);
1028 #if !defined(CONFIG_SOFTMMU)
1029 static void tb_invalidate_phys_page(target_phys_addr_t addr,
1030 unsigned long pc, void *puc)
1032 TranslationBlock *tb;
1035 #ifdef TARGET_HAS_PRECISE_SMC
1036 TranslationBlock *current_tb = NULL;
1037 CPUState *env = cpu_single_env;
1038 int current_tb_modified = 0;
1039 target_ulong current_pc = 0;
1040 target_ulong current_cs_base = 0;
1041 int current_flags = 0;
1044 addr &= TARGET_PAGE_MASK;
1045 p = page_find(addr >> TARGET_PAGE_BITS);
1049 #ifdef TARGET_HAS_PRECISE_SMC
1050 if (tb && pc != 0) {
1051 current_tb = tb_find_pc(pc);
1054 while (tb != NULL) {
1056 tb = (TranslationBlock *)((long)tb & ~3);
1057 #ifdef TARGET_HAS_PRECISE_SMC
1058 if (current_tb == tb &&
1059 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1060 /* If we are modifying the current TB, we must stop
1061 its execution. We could be more precise by checking
1062 that the modification is after the current PC, but it
1063 would require a specialized function to partially
1064 restore the CPU state */
1066 current_tb_modified = 1;
1067 cpu_restore_state(current_tb, env, pc, puc);
1068 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base,
1071 #endif /* TARGET_HAS_PRECISE_SMC */
1072 tb_phys_invalidate(tb, addr);
1073 tb = tb->page_next[n];
1076 #ifdef TARGET_HAS_PRECISE_SMC
1077 if (current_tb_modified) {
1078 /* we generate a block containing just the instruction
1079 modifying the memory. It will ensure that it cannot modify
1081 env->current_tb = NULL;
1082 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1083 cpu_resume_from_signal(env, puc);
1089 /* add the tb in the target page and protect it if necessary */
1090 static inline void tb_alloc_page(TranslationBlock *tb,
1091 unsigned int n, target_ulong page_addr)
1094 TranslationBlock *last_first_tb;
1096 tb->page_addr[n] = page_addr;
1097 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1098 tb->page_next[n] = p->first_tb;
1099 last_first_tb = p->first_tb;
1100 p->first_tb = (TranslationBlock *)((long)tb | n);
1101 invalidate_page_bitmap(p);
1103 #if defined(TARGET_HAS_SMC) || 1
1105 #if defined(CONFIG_USER_ONLY)
1106 if (p->flags & PAGE_WRITE) {
1111 /* force the host page as non writable (writes will have a
1112 page fault + mprotect overhead) */
1113 page_addr &= qemu_host_page_mask;
1115 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1116 addr += TARGET_PAGE_SIZE) {
1118 p2 = page_find (addr >> TARGET_PAGE_BITS);
1122 p2->flags &= ~PAGE_WRITE;
1123 page_get_flags(addr);
1125 mprotect(g2h(page_addr), qemu_host_page_size,
1126 (prot & PAGE_BITS) & ~PAGE_WRITE);
1127 #ifdef DEBUG_TB_INVALIDATE
1128 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1133 /* if some code is already present, then the pages are already
1134 protected. So we handle the case where only the first TB is
1135 allocated in a physical page */
1136 if (!last_first_tb) {
1137 tlb_protect_code(page_addr);
1141 #endif /* TARGET_HAS_SMC */
1144 /* Allocate a new translation block. Flush the translation buffer if
1145 too many translation blocks or too much generated code. */
1146 TranslationBlock *tb_alloc(target_ulong pc)
1148 TranslationBlock *tb;
1150 if (nb_tbs >= code_gen_max_blocks ||
1151 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1153 tb = &tbs[nb_tbs++];
1159 void tb_free(TranslationBlock *tb)
1161 /* In practice this is mostly used for single use temporary TB
1162 Ignore the hard cases and just back up if this TB happens to
1163 be the last one generated. */
1164 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1165 code_gen_ptr = tb->tc_ptr;
1170 /* add a new TB and link it to the physical page tables. phys_page2 is
1171 (-1) to indicate that only one page contains the TB. */
1172 void tb_link_phys(TranslationBlock *tb,
1173 target_ulong phys_pc, target_ulong phys_page2)
1176 TranslationBlock **ptb;
1178 /* Grab the mmap lock to stop another thread invalidating this TB
1179 before we are done. */
1181 /* add in the physical hash table */
1182 h = tb_phys_hash_func(phys_pc);
1183 ptb = &tb_phys_hash[h];
1184 tb->phys_hash_next = *ptb;
1187 /* add in the page list */
1188 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1189 if (phys_page2 != -1)
1190 tb_alloc_page(tb, 1, phys_page2);
1192 tb->page_addr[1] = -1;
1194 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1195 tb->jmp_next[0] = NULL;
1196 tb->jmp_next[1] = NULL;
1198 /* init original jump addresses */
1199 if (tb->tb_next_offset[0] != 0xffff)
1200 tb_reset_jump(tb, 0);
1201 if (tb->tb_next_offset[1] != 0xffff)
1202 tb_reset_jump(tb, 1);
1204 #ifdef DEBUG_TB_CHECK
1210 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1211 tb[1].tc_ptr. Return NULL if not found */
1212 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1214 int m_min, m_max, m;
1216 TranslationBlock *tb;
1220 if (tc_ptr < (unsigned long)code_gen_buffer ||
1221 tc_ptr >= (unsigned long)code_gen_ptr)
1223 /* binary search (cf Knuth) */
1226 while (m_min <= m_max) {
1227 m = (m_min + m_max) >> 1;
1229 v = (unsigned long)tb->tc_ptr;
1232 else if (tc_ptr < v) {
1241 static void tb_reset_jump_recursive(TranslationBlock *tb);
1243 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1245 TranslationBlock *tb1, *tb_next, **ptb;
1248 tb1 = tb->jmp_next[n];
1250 /* find head of list */
1253 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1256 tb1 = tb1->jmp_next[n1];
1258 /* we are now sure now that tb jumps to tb1 */
1261 /* remove tb from the jmp_first list */
1262 ptb = &tb_next->jmp_first;
1266 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1267 if (n1 == n && tb1 == tb)
1269 ptb = &tb1->jmp_next[n1];
1271 *ptb = tb->jmp_next[n];
1272 tb->jmp_next[n] = NULL;
1274 /* suppress the jump to next tb in generated code */
1275 tb_reset_jump(tb, n);
1277 /* suppress jumps in the tb on which we could have jumped */
1278 tb_reset_jump_recursive(tb_next);
1282 static void tb_reset_jump_recursive(TranslationBlock *tb)
1284 tb_reset_jump_recursive2(tb, 0);
1285 tb_reset_jump_recursive2(tb, 1);
1288 #if defined(TARGET_HAS_ICE)
1289 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1291 target_phys_addr_t addr;
1293 ram_addr_t ram_addr;
1296 addr = cpu_get_phys_page_debug(env, pc);
1297 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1299 pd = IO_MEM_UNASSIGNED;
1301 pd = p->phys_offset;
1303 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1304 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1308 /* Add a watchpoint. */
1309 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1310 int flags, CPUWatchpoint **watchpoint)
1312 target_ulong len_mask = ~(len - 1);
1315 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1316 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1317 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1318 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1321 wp = qemu_malloc(sizeof(*wp));
1324 wp->len_mask = len_mask;
1327 /* keep all GDB-injected watchpoints in front */
1329 TAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1331 TAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1333 tlb_flush_page(env, addr);
1340 /* Remove a specific watchpoint. */
1341 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1344 target_ulong len_mask = ~(len - 1);
1347 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1348 if (addr == wp->vaddr && len_mask == wp->len_mask
1349 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1350 cpu_watchpoint_remove_by_ref(env, wp);
1357 /* Remove a specific watchpoint by reference. */
1358 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1360 TAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1362 tlb_flush_page(env, watchpoint->vaddr);
1364 qemu_free(watchpoint);
1367 /* Remove all matching watchpoints. */
1368 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1370 CPUWatchpoint *wp, *next;
1372 TAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1373 if (wp->flags & mask)
1374 cpu_watchpoint_remove_by_ref(env, wp);
1378 /* Add a breakpoint. */
1379 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1380 CPUBreakpoint **breakpoint)
1382 #if defined(TARGET_HAS_ICE)
1385 bp = qemu_malloc(sizeof(*bp));
1390 /* keep all GDB-injected breakpoints in front */
1392 TAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1394 TAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1396 breakpoint_invalidate(env, pc);
1406 /* Remove a specific breakpoint. */
1407 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1409 #if defined(TARGET_HAS_ICE)
1412 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1413 if (bp->pc == pc && bp->flags == flags) {
1414 cpu_breakpoint_remove_by_ref(env, bp);
1424 /* Remove a specific breakpoint by reference. */
1425 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1427 #if defined(TARGET_HAS_ICE)
1428 TAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1430 breakpoint_invalidate(env, breakpoint->pc);
1432 qemu_free(breakpoint);
1436 /* Remove all matching breakpoints. */
1437 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1439 #if defined(TARGET_HAS_ICE)
1440 CPUBreakpoint *bp, *next;
1442 TAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1443 if (bp->flags & mask)
1444 cpu_breakpoint_remove_by_ref(env, bp);
1449 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1450 CPU loop after each instruction */
1451 void cpu_single_step(CPUState *env, int enabled)
1453 #if defined(TARGET_HAS_ICE)
1454 if (env->singlestep_enabled != enabled) {
1455 env->singlestep_enabled = enabled;
1457 kvm_update_guest_debug(env, 0);
1459 /* must flush all the translated code to avoid inconsistancies */
1460 /* XXX: only flush what is necessary */
1467 /* enable or disable low levels log */
1468 void cpu_set_log(int log_flags)
1470 loglevel = log_flags;
1471 if (loglevel && !logfile) {
1472 logfile = fopen(logfilename, log_append ? "a" : "w");
1474 perror(logfilename);
1477 #if !defined(CONFIG_SOFTMMU)
1478 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1480 static char logfile_buf[4096];
1481 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1484 setvbuf(logfile, NULL, _IOLBF, 0);
1488 if (!loglevel && logfile) {
1494 void cpu_set_log_filename(const char *filename)
1496 logfilename = strdup(filename);
1501 cpu_set_log(loglevel);
1504 static void cpu_unlink_tb(CPUState *env)
1506 #if defined(USE_NPTL)
1507 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1508 problem and hope the cpu will stop of its own accord. For userspace
1509 emulation this often isn't actually as bad as it sounds. Often
1510 signals are used primarily to interrupt blocking syscalls. */
1512 TranslationBlock *tb;
1513 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1515 tb = env->current_tb;
1516 /* if the cpu is currently executing code, we must unlink it and
1517 all the potentially executing TB */
1518 if (tb && !testandset(&interrupt_lock)) {
1519 env->current_tb = NULL;
1520 tb_reset_jump_recursive(tb);
1521 resetlock(&interrupt_lock);
1526 /* mask must never be zero, except for A20 change call */
1527 void cpu_interrupt(CPUState *env, int mask)
1531 old_mask = env->interrupt_request;
1532 env->interrupt_request |= mask;
1535 env->icount_decr.u16.high = 0xffff;
1536 #ifndef CONFIG_USER_ONLY
1538 && (mask & ~old_mask) != 0) {
1539 cpu_abort(env, "Raised interrupt while not in I/O function");
1547 void cpu_reset_interrupt(CPUState *env, int mask)
1549 env->interrupt_request &= ~mask;
1552 void cpu_exit(CPUState *env)
1554 env->exit_request = 1;
1558 const CPULogItem cpu_log_items[] = {
1559 { CPU_LOG_TB_OUT_ASM, "out_asm",
1560 "show generated host assembly code for each compiled TB" },
1561 { CPU_LOG_TB_IN_ASM, "in_asm",
1562 "show target assembly code for each compiled TB" },
1563 { CPU_LOG_TB_OP, "op",
1564 "show micro ops for each compiled TB" },
1565 { CPU_LOG_TB_OP_OPT, "op_opt",
1568 "before eflags optimization and "
1570 "after liveness analysis" },
1571 { CPU_LOG_INT, "int",
1572 "show interrupts/exceptions in short format" },
1573 { CPU_LOG_EXEC, "exec",
1574 "show trace before each executed TB (lots of logs)" },
1575 { CPU_LOG_TB_CPU, "cpu",
1576 "show CPU state before block translation" },
1578 { CPU_LOG_PCALL, "pcall",
1579 "show protected mode far calls/returns/exceptions" },
1580 { CPU_LOG_RESET, "cpu_reset",
1581 "show CPU state before CPU resets" },
1584 { CPU_LOG_IOPORT, "ioport",
1585 "show all i/o ports accesses" },
1590 static int cmp1(const char *s1, int n, const char *s2)
1592 if (strlen(s2) != n)
1594 return memcmp(s1, s2, n) == 0;
1597 /* takes a comma separated list of log masks. Return 0 if error. */
1598 int cpu_str_to_log_mask(const char *str)
1600 const CPULogItem *item;
1607 p1 = strchr(p, ',');
1610 if(cmp1(p,p1-p,"all")) {
1611 for(item = cpu_log_items; item->mask != 0; item++) {
1615 for(item = cpu_log_items; item->mask != 0; item++) {
1616 if (cmp1(p, p1 - p, item->name))
1630 void cpu_abort(CPUState *env, const char *fmt, ...)
1637 fprintf(stderr, "qemu: fatal: ");
1638 vfprintf(stderr, fmt, ap);
1639 fprintf(stderr, "\n");
1641 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1643 cpu_dump_state(env, stderr, fprintf, 0);
1645 if (qemu_log_enabled()) {
1646 qemu_log("qemu: fatal: ");
1647 qemu_log_vprintf(fmt, ap2);
1650 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1652 log_cpu_state(env, 0);
1662 CPUState *cpu_copy(CPUState *env)
1664 CPUState *new_env = cpu_init(env->cpu_model_str);
1665 CPUState *next_cpu = new_env->next_cpu;
1666 int cpu_index = new_env->cpu_index;
1667 #if defined(TARGET_HAS_ICE)
1672 memcpy(new_env, env, sizeof(CPUState));
1674 /* Preserve chaining and index. */
1675 new_env->next_cpu = next_cpu;
1676 new_env->cpu_index = cpu_index;
1678 /* Clone all break/watchpoints.
1679 Note: Once we support ptrace with hw-debug register access, make sure
1680 BP_CPU break/watchpoints are handled correctly on clone. */
1681 TAILQ_INIT(&env->breakpoints);
1682 TAILQ_INIT(&env->watchpoints);
1683 #if defined(TARGET_HAS_ICE)
1684 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1685 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1687 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1688 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1696 #if !defined(CONFIG_USER_ONLY)
1698 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1702 /* Discard jump cache entries for any tb which might potentially
1703 overlap the flushed page. */
1704 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1705 memset (&env->tb_jmp_cache[i], 0,
1706 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1708 i = tb_jmp_cache_hash_page(addr);
1709 memset (&env->tb_jmp_cache[i], 0,
1710 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1713 /* NOTE: if flush_global is true, also flush global entries (not
1715 void tlb_flush(CPUState *env, int flush_global)
1719 #if defined(DEBUG_TLB)
1720 printf("tlb_flush:\n");
1722 /* must reset current TB so that interrupts cannot modify the
1723 links while we are modifying them */
1724 env->current_tb = NULL;
1726 for(i = 0; i < CPU_TLB_SIZE; i++) {
1727 env->tlb_table[0][i].addr_read = -1;
1728 env->tlb_table[0][i].addr_write = -1;
1729 env->tlb_table[0][i].addr_code = -1;
1730 env->tlb_table[1][i].addr_read = -1;
1731 env->tlb_table[1][i].addr_write = -1;
1732 env->tlb_table[1][i].addr_code = -1;
1733 #if (NB_MMU_MODES >= 3)
1734 env->tlb_table[2][i].addr_read = -1;
1735 env->tlb_table[2][i].addr_write = -1;
1736 env->tlb_table[2][i].addr_code = -1;
1737 #if (NB_MMU_MODES == 4)
1738 env->tlb_table[3][i].addr_read = -1;
1739 env->tlb_table[3][i].addr_write = -1;
1740 env->tlb_table[3][i].addr_code = -1;
1745 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1748 if (env->kqemu_enabled) {
1749 kqemu_flush(env, flush_global);
1755 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1757 if (addr == (tlb_entry->addr_read &
1758 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1759 addr == (tlb_entry->addr_write &
1760 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1761 addr == (tlb_entry->addr_code &
1762 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1763 tlb_entry->addr_read = -1;
1764 tlb_entry->addr_write = -1;
1765 tlb_entry->addr_code = -1;
1769 void tlb_flush_page(CPUState *env, target_ulong addr)
1773 #if defined(DEBUG_TLB)
1774 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1776 /* must reset current TB so that interrupts cannot modify the
1777 links while we are modifying them */
1778 env->current_tb = NULL;
1780 addr &= TARGET_PAGE_MASK;
1781 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1782 tlb_flush_entry(&env->tlb_table[0][i], addr);
1783 tlb_flush_entry(&env->tlb_table[1][i], addr);
1784 #if (NB_MMU_MODES >= 3)
1785 tlb_flush_entry(&env->tlb_table[2][i], addr);
1786 #if (NB_MMU_MODES == 4)
1787 tlb_flush_entry(&env->tlb_table[3][i], addr);
1791 tlb_flush_jmp_cache(env, addr);
1794 if (env->kqemu_enabled) {
1795 kqemu_flush_page(env, addr);
1800 /* update the TLBs so that writes to code in the virtual page 'addr'
1802 static void tlb_protect_code(ram_addr_t ram_addr)
1804 cpu_physical_memory_reset_dirty(ram_addr,
1805 ram_addr + TARGET_PAGE_SIZE,
1809 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1810 tested for self modifying code */
1811 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1814 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1817 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1818 unsigned long start, unsigned long length)
1821 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1822 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1823 if ((addr - start) < length) {
1824 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1829 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
1833 unsigned long length, start1;
1837 start &= TARGET_PAGE_MASK;
1838 end = TARGET_PAGE_ALIGN(end);
1840 length = end - start;
1843 len = length >> TARGET_PAGE_BITS;
1845 /* XXX: should not depend on cpu context */
1847 if (env->kqemu_enabled) {
1850 for(i = 0; i < len; i++) {
1851 kqemu_set_notdirty(env, addr);
1852 addr += TARGET_PAGE_SIZE;
1856 mask = ~dirty_flags;
1857 p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
1858 for(i = 0; i < len; i++)
1861 /* we modify the TLB cache so that the dirty bit will be set again
1862 when accessing the range */
1863 start1 = start + (unsigned long)phys_ram_base;
1864 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1865 for(i = 0; i < CPU_TLB_SIZE; i++)
1866 tlb_reset_dirty_range(&env->tlb_table[0][i], start1, length);
1867 for(i = 0; i < CPU_TLB_SIZE; i++)
1868 tlb_reset_dirty_range(&env->tlb_table[1][i], start1, length);
1869 #if (NB_MMU_MODES >= 3)
1870 for(i = 0; i < CPU_TLB_SIZE; i++)
1871 tlb_reset_dirty_range(&env->tlb_table[2][i], start1, length);
1872 #if (NB_MMU_MODES == 4)
1873 for(i = 0; i < CPU_TLB_SIZE; i++)
1874 tlb_reset_dirty_range(&env->tlb_table[3][i], start1, length);
1880 int cpu_physical_memory_set_dirty_tracking(int enable)
1882 in_migration = enable;
1886 int cpu_physical_memory_get_dirty_tracking(void)
1888 return in_migration;
1891 void cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr, target_phys_addr_t end_addr)
1894 kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
1897 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
1899 ram_addr_t ram_addr;
1901 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1902 ram_addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) +
1903 tlb_entry->addend - (unsigned long)phys_ram_base;
1904 if (!cpu_physical_memory_is_dirty(ram_addr)) {
1905 tlb_entry->addr_write |= TLB_NOTDIRTY;
1910 /* update the TLB according to the current state of the dirty bits */
1911 void cpu_tlb_update_dirty(CPUState *env)
1914 for(i = 0; i < CPU_TLB_SIZE; i++)
1915 tlb_update_dirty(&env->tlb_table[0][i]);
1916 for(i = 0; i < CPU_TLB_SIZE; i++)
1917 tlb_update_dirty(&env->tlb_table[1][i]);
1918 #if (NB_MMU_MODES >= 3)
1919 for(i = 0; i < CPU_TLB_SIZE; i++)
1920 tlb_update_dirty(&env->tlb_table[2][i]);
1921 #if (NB_MMU_MODES == 4)
1922 for(i = 0; i < CPU_TLB_SIZE; i++)
1923 tlb_update_dirty(&env->tlb_table[3][i]);
1928 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1930 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
1931 tlb_entry->addr_write = vaddr;
1934 /* update the TLB corresponding to virtual page vaddr
1935 so that it is no longer dirty */
1936 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
1940 vaddr &= TARGET_PAGE_MASK;
1941 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1942 tlb_set_dirty1(&env->tlb_table[0][i], vaddr);
1943 tlb_set_dirty1(&env->tlb_table[1][i], vaddr);
1944 #if (NB_MMU_MODES >= 3)
1945 tlb_set_dirty1(&env->tlb_table[2][i], vaddr);
1946 #if (NB_MMU_MODES == 4)
1947 tlb_set_dirty1(&env->tlb_table[3][i], vaddr);
1952 /* add a new TLB entry. At most one entry for a given virtual address
1953 is permitted. Return 0 if OK or 2 if the page could not be mapped
1954 (can only happen in non SOFTMMU mode for I/O pages or pages
1955 conflicting with the host address space). */
1956 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
1957 target_phys_addr_t paddr, int prot,
1958 int mmu_idx, int is_softmmu)
1963 target_ulong address;
1964 target_ulong code_address;
1965 target_phys_addr_t addend;
1969 target_phys_addr_t iotlb;
1971 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
1973 pd = IO_MEM_UNASSIGNED;
1975 pd = p->phys_offset;
1977 #if defined(DEBUG_TLB)
1978 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1979 vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
1984 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
1985 /* IO memory case (romd handled later) */
1986 address |= TLB_MMIO;
1988 addend = (unsigned long)phys_ram_base + (pd & TARGET_PAGE_MASK);
1989 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
1991 iotlb = pd & TARGET_PAGE_MASK;
1992 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
1993 iotlb |= IO_MEM_NOTDIRTY;
1995 iotlb |= IO_MEM_ROM;
1997 /* IO handlers are currently passed a phsical address.
1998 It would be nice to pass an offset from the base address
1999 of that region. This would avoid having to special case RAM,
2000 and avoid full address decoding in every device.
2001 We can't use the high bits of pd for this because
2002 IO_MEM_ROMD uses these as a ram address. */
2003 iotlb = (pd & ~TARGET_PAGE_MASK);
2005 iotlb += p->region_offset;
2011 code_address = address;
2012 /* Make accesses to pages with watchpoints go via the
2013 watchpoint trap routines. */
2014 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2015 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2016 iotlb = io_mem_watch + paddr;
2017 /* TODO: The memory case can be optimized by not trapping
2018 reads of pages with a write breakpoint. */
2019 address |= TLB_MMIO;
2023 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2024 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2025 te = &env->tlb_table[mmu_idx][index];
2026 te->addend = addend - vaddr;
2027 if (prot & PAGE_READ) {
2028 te->addr_read = address;
2033 if (prot & PAGE_EXEC) {
2034 te->addr_code = code_address;
2038 if (prot & PAGE_WRITE) {
2039 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2040 (pd & IO_MEM_ROMD)) {
2041 /* Write access calls the I/O callback. */
2042 te->addr_write = address | TLB_MMIO;
2043 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2044 !cpu_physical_memory_is_dirty(pd)) {
2045 te->addr_write = address | TLB_NOTDIRTY;
2047 te->addr_write = address;
2050 te->addr_write = -1;
2057 void tlb_flush(CPUState *env, int flush_global)
2061 void tlb_flush_page(CPUState *env, target_ulong addr)
2065 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2066 target_phys_addr_t paddr, int prot,
2067 int mmu_idx, int is_softmmu)
2072 void walk_memory_regions(void *priv,
2073 int (*fn)(void *, unsigned long, unsigned long, unsigned long))
2075 unsigned long start, end;
2077 int i, j, prot, prot1;
2082 for (i = 0; i <= L1_SIZE; i++) {
2083 p = (i < L1_SIZE) ? l1_map[i] : NULL;
2084 for (j = 0; j < L2_SIZE; j++) {
2085 prot1 = (p == NULL) ? 0 : p[j].flags;
2087 * "region" is one continuous chunk of memory
2088 * that has same protection flags set.
2090 if (prot1 != prot) {
2091 end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
2093 if ((*fn)(priv, start, end, prot) != 0)
2107 ; /* null statement to make compiler happy */
2110 static int dump_region(void *priv, unsigned long start,
2111 unsigned long end, unsigned long prot)
2113 FILE *f = (FILE *)priv;
2115 (void) fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2116 start, end, end - start,
2117 ((prot & PAGE_READ) ? 'r' : '-'),
2118 ((prot & PAGE_WRITE) ? 'w' : '-'),
2119 ((prot & PAGE_EXEC) ? 'x' : '-'));
2124 /* dump memory mappings */
2125 void page_dump(FILE *f)
2127 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2128 "start", "end", "size", "prot");
2129 walk_memory_regions(f, dump_region);
2132 int page_get_flags(target_ulong address)
2136 p = page_find(address >> TARGET_PAGE_BITS);
2142 /* modify the flags of a page and invalidate the code if
2143 necessary. The flag PAGE_WRITE_ORG is positionned automatically
2144 depending on PAGE_WRITE */
2145 void page_set_flags(target_ulong start, target_ulong end, int flags)
2150 /* mmap_lock should already be held. */
2151 start = start & TARGET_PAGE_MASK;
2152 end = TARGET_PAGE_ALIGN(end);
2153 if (flags & PAGE_WRITE)
2154 flags |= PAGE_WRITE_ORG;
2155 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2156 p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2157 /* We may be called for host regions that are outside guest
2161 /* if the write protection is set, then we invalidate the code
2163 if (!(p->flags & PAGE_WRITE) &&
2164 (flags & PAGE_WRITE) &&
2166 tb_invalidate_phys_page(addr, 0, NULL);
2172 int page_check_range(target_ulong start, target_ulong len, int flags)
2178 if (start + len < start)
2179 /* we've wrapped around */
2182 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2183 start = start & TARGET_PAGE_MASK;
2185 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2186 p = page_find(addr >> TARGET_PAGE_BITS);
2189 if( !(p->flags & PAGE_VALID) )
2192 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2194 if (flags & PAGE_WRITE) {
2195 if (!(p->flags & PAGE_WRITE_ORG))
2197 /* unprotect the page if it was put read-only because it
2198 contains translated code */
2199 if (!(p->flags & PAGE_WRITE)) {
2200 if (!page_unprotect(addr, 0, NULL))
2209 /* called from signal handler: invalidate the code and unprotect the
2210 page. Return TRUE if the fault was succesfully handled. */
2211 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2213 unsigned int page_index, prot, pindex;
2215 target_ulong host_start, host_end, addr;
2217 /* Technically this isn't safe inside a signal handler. However we
2218 know this only ever happens in a synchronous SEGV handler, so in
2219 practice it seems to be ok. */
2222 host_start = address & qemu_host_page_mask;
2223 page_index = host_start >> TARGET_PAGE_BITS;
2224 p1 = page_find(page_index);
2229 host_end = host_start + qemu_host_page_size;
2232 for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2236 /* if the page was really writable, then we change its
2237 protection back to writable */
2238 if (prot & PAGE_WRITE_ORG) {
2239 pindex = (address - host_start) >> TARGET_PAGE_BITS;
2240 if (!(p1[pindex].flags & PAGE_WRITE)) {
2241 mprotect((void *)g2h(host_start), qemu_host_page_size,
2242 (prot & PAGE_BITS) | PAGE_WRITE);
2243 p1[pindex].flags |= PAGE_WRITE;
2244 /* and since the content will be modified, we must invalidate
2245 the corresponding translated code. */
2246 tb_invalidate_phys_page(address, pc, puc);
2247 #ifdef DEBUG_TB_CHECK
2248 tb_invalidate_check(address);
2258 static inline void tlb_set_dirty(CPUState *env,
2259 unsigned long addr, target_ulong vaddr)
2262 #endif /* defined(CONFIG_USER_ONLY) */
2264 #if !defined(CONFIG_USER_ONLY)
2266 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2267 ram_addr_t memory, ram_addr_t region_offset);
2268 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2269 ram_addr_t orig_memory, ram_addr_t region_offset);
2270 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2273 if (addr > start_addr) \
2276 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2277 if (start_addr2 > 0) \
2281 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2282 end_addr2 = TARGET_PAGE_SIZE - 1; \
2284 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2285 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2290 /* register physical memory. 'size' must be a multiple of the target
2291 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2292 io memory page. The address used when calling the IO function is
2293 the offset from the start of the region, plus region_offset. Both
2294 start_region and regon_offset are rounded down to a page boundary
2295 before calculating this offset. This should not be a problem unless
2296 the low bits of start_addr and region_offset differ. */
2297 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2299 ram_addr_t phys_offset,
2300 ram_addr_t region_offset)
2302 target_phys_addr_t addr, end_addr;
2305 ram_addr_t orig_size = size;
2309 /* XXX: should not depend on cpu context */
2311 if (env->kqemu_enabled) {
2312 kqemu_set_phys_mem(start_addr, size, phys_offset);
2316 kvm_set_phys_mem(start_addr, size, phys_offset);
2318 if (phys_offset == IO_MEM_UNASSIGNED) {
2319 region_offset = start_addr;
2321 region_offset &= TARGET_PAGE_MASK;
2322 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2323 end_addr = start_addr + (target_phys_addr_t)size;
2324 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2325 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2326 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2327 ram_addr_t orig_memory = p->phys_offset;
2328 target_phys_addr_t start_addr2, end_addr2;
2329 int need_subpage = 0;
2331 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2333 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2334 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2335 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2336 &p->phys_offset, orig_memory,
2339 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2342 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2344 p->region_offset = 0;
2346 p->phys_offset = phys_offset;
2347 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2348 (phys_offset & IO_MEM_ROMD))
2349 phys_offset += TARGET_PAGE_SIZE;
2352 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2353 p->phys_offset = phys_offset;
2354 p->region_offset = region_offset;
2355 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2356 (phys_offset & IO_MEM_ROMD)) {
2357 phys_offset += TARGET_PAGE_SIZE;
2359 target_phys_addr_t start_addr2, end_addr2;
2360 int need_subpage = 0;
2362 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2363 end_addr2, need_subpage);
2365 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2366 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2367 &p->phys_offset, IO_MEM_UNASSIGNED,
2368 addr & TARGET_PAGE_MASK);
2369 subpage_register(subpage, start_addr2, end_addr2,
2370 phys_offset, region_offset);
2371 p->region_offset = 0;
2375 region_offset += TARGET_PAGE_SIZE;
2378 /* since each CPU stores ram addresses in its TLB cache, we must
2379 reset the modified entries */
2381 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2386 /* XXX: temporary until new memory mapping API */
2387 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2391 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2393 return IO_MEM_UNASSIGNED;
2394 return p->phys_offset;
2397 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2400 kvm_coalesce_mmio_region(addr, size);
2403 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2406 kvm_uncoalesce_mmio_region(addr, size);
2409 /* XXX: better than nothing */
2410 ram_addr_t qemu_ram_alloc(ram_addr_t size)
2413 if ((phys_ram_alloc_offset + size) > phys_ram_size) {
2414 fprintf(stderr, "Not enough memory (requested_size = %" PRIu64 ", max memory = %" PRIu64 ")\n",
2415 (uint64_t)size, (uint64_t)phys_ram_size);
2418 addr = phys_ram_alloc_offset;
2419 phys_ram_alloc_offset = TARGET_PAGE_ALIGN(phys_ram_alloc_offset + size);
2423 void qemu_ram_free(ram_addr_t addr)
2427 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2429 #ifdef DEBUG_UNASSIGNED
2430 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2432 #if defined(TARGET_SPARC)
2433 do_unassigned_access(addr, 0, 0, 0, 1);
2438 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2440 #ifdef DEBUG_UNASSIGNED
2441 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2443 #if defined(TARGET_SPARC)
2444 do_unassigned_access(addr, 0, 0, 0, 2);
2449 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2451 #ifdef DEBUG_UNASSIGNED
2452 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2454 #if defined(TARGET_SPARC)
2455 do_unassigned_access(addr, 0, 0, 0, 4);
2460 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2462 #ifdef DEBUG_UNASSIGNED
2463 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2465 #if defined(TARGET_SPARC)
2466 do_unassigned_access(addr, 1, 0, 0, 1);
2470 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2472 #ifdef DEBUG_UNASSIGNED
2473 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2475 #if defined(TARGET_SPARC)
2476 do_unassigned_access(addr, 1, 0, 0, 2);
2480 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2482 #ifdef DEBUG_UNASSIGNED
2483 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2485 #if defined(TARGET_SPARC)
2486 do_unassigned_access(addr, 1, 0, 0, 4);
2490 static CPUReadMemoryFunc *unassigned_mem_read[3] = {
2491 unassigned_mem_readb,
2492 unassigned_mem_readw,
2493 unassigned_mem_readl,
2496 static CPUWriteMemoryFunc *unassigned_mem_write[3] = {
2497 unassigned_mem_writeb,
2498 unassigned_mem_writew,
2499 unassigned_mem_writel,
2502 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2506 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2507 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2508 #if !defined(CONFIG_USER_ONLY)
2509 tb_invalidate_phys_page_fast(ram_addr, 1);
2510 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2513 stb_p(phys_ram_base + ram_addr, val);
2515 if (cpu_single_env->kqemu_enabled &&
2516 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2517 kqemu_modify_page(cpu_single_env, ram_addr);
2519 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2520 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2521 /* we remove the notdirty callback only if the code has been
2523 if (dirty_flags == 0xff)
2524 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2527 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2531 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2532 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2533 #if !defined(CONFIG_USER_ONLY)
2534 tb_invalidate_phys_page_fast(ram_addr, 2);
2535 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2538 stw_p(phys_ram_base + ram_addr, val);
2540 if (cpu_single_env->kqemu_enabled &&
2541 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2542 kqemu_modify_page(cpu_single_env, ram_addr);
2544 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2545 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2546 /* we remove the notdirty callback only if the code has been
2548 if (dirty_flags == 0xff)
2549 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2552 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
2556 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2557 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2558 #if !defined(CONFIG_USER_ONLY)
2559 tb_invalidate_phys_page_fast(ram_addr, 4);
2560 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2563 stl_p(phys_ram_base + ram_addr, val);
2565 if (cpu_single_env->kqemu_enabled &&
2566 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2567 kqemu_modify_page(cpu_single_env, ram_addr);
2569 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2570 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2571 /* we remove the notdirty callback only if the code has been
2573 if (dirty_flags == 0xff)
2574 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2577 static CPUReadMemoryFunc *error_mem_read[3] = {
2578 NULL, /* never used */
2579 NULL, /* never used */
2580 NULL, /* never used */
2583 static CPUWriteMemoryFunc *notdirty_mem_write[3] = {
2584 notdirty_mem_writeb,
2585 notdirty_mem_writew,
2586 notdirty_mem_writel,
2589 /* Generate a debug exception if a watchpoint has been hit. */
2590 static void check_watchpoint(int offset, int len_mask, int flags)
2592 CPUState *env = cpu_single_env;
2593 target_ulong pc, cs_base;
2594 TranslationBlock *tb;
2599 if (env->watchpoint_hit) {
2600 /* We re-entered the check after replacing the TB. Now raise
2601 * the debug interrupt so that is will trigger after the
2602 * current instruction. */
2603 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
2606 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2607 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2608 if ((vaddr == (wp->vaddr & len_mask) ||
2609 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
2610 wp->flags |= BP_WATCHPOINT_HIT;
2611 if (!env->watchpoint_hit) {
2612 env->watchpoint_hit = wp;
2613 tb = tb_find_pc(env->mem_io_pc);
2615 cpu_abort(env, "check_watchpoint: could not find TB for "
2616 "pc=%p", (void *)env->mem_io_pc);
2618 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
2619 tb_phys_invalidate(tb, -1);
2620 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2621 env->exception_index = EXCP_DEBUG;
2623 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2624 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
2626 cpu_resume_from_signal(env, NULL);
2629 wp->flags &= ~BP_WATCHPOINT_HIT;
2634 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2635 so these check for a hit then pass through to the normal out-of-line
2637 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
2639 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
2640 return ldub_phys(addr);
2643 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
2645 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
2646 return lduw_phys(addr);
2649 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
2651 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
2652 return ldl_phys(addr);
2655 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
2658 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
2659 stb_phys(addr, val);
2662 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
2665 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
2666 stw_phys(addr, val);
2669 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
2672 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
2673 stl_phys(addr, val);
2676 static CPUReadMemoryFunc *watch_mem_read[3] = {
2682 static CPUWriteMemoryFunc *watch_mem_write[3] = {
2688 static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
2694 idx = SUBPAGE_IDX(addr);
2695 #if defined(DEBUG_SUBPAGE)
2696 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
2697 mmio, len, addr, idx);
2699 ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
2700 addr + mmio->region_offset[idx][0][len]);
2705 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
2706 uint32_t value, unsigned int len)
2710 idx = SUBPAGE_IDX(addr);
2711 #if defined(DEBUG_SUBPAGE)
2712 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
2713 mmio, len, addr, idx, value);
2715 (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
2716 addr + mmio->region_offset[idx][1][len],
2720 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
2722 #if defined(DEBUG_SUBPAGE)
2723 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2726 return subpage_readlen(opaque, addr, 0);
2729 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
2732 #if defined(DEBUG_SUBPAGE)
2733 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2735 subpage_writelen(opaque, addr, value, 0);
2738 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
2740 #if defined(DEBUG_SUBPAGE)
2741 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2744 return subpage_readlen(opaque, addr, 1);
2747 static void subpage_writew (void *opaque, target_phys_addr_t addr,
2750 #if defined(DEBUG_SUBPAGE)
2751 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2753 subpage_writelen(opaque, addr, value, 1);
2756 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
2758 #if defined(DEBUG_SUBPAGE)
2759 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2762 return subpage_readlen(opaque, addr, 2);
2765 static void subpage_writel (void *opaque,
2766 target_phys_addr_t addr, uint32_t value)
2768 #if defined(DEBUG_SUBPAGE)
2769 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2771 subpage_writelen(opaque, addr, value, 2);
2774 static CPUReadMemoryFunc *subpage_read[] = {
2780 static CPUWriteMemoryFunc *subpage_write[] = {
2786 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2787 ram_addr_t memory, ram_addr_t region_offset)
2792 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2794 idx = SUBPAGE_IDX(start);
2795 eidx = SUBPAGE_IDX(end);
2796 #if defined(DEBUG_SUBPAGE)
2797 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__,
2798 mmio, start, end, idx, eidx, memory);
2800 memory >>= IO_MEM_SHIFT;
2801 for (; idx <= eidx; idx++) {
2802 for (i = 0; i < 4; i++) {
2803 if (io_mem_read[memory][i]) {
2804 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
2805 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
2806 mmio->region_offset[idx][0][i] = region_offset;
2808 if (io_mem_write[memory][i]) {
2809 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
2810 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
2811 mmio->region_offset[idx][1][i] = region_offset;
2819 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2820 ram_addr_t orig_memory, ram_addr_t region_offset)
2825 mmio = qemu_mallocz(sizeof(subpage_t));
2828 subpage_memory = cpu_register_io_memory(0, subpage_read, subpage_write, mmio);
2829 #if defined(DEBUG_SUBPAGE)
2830 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
2831 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
2833 *phys = subpage_memory | IO_MEM_SUBPAGE;
2834 subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
2840 static int get_free_io_mem_idx(void)
2844 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
2845 if (!io_mem_used[i]) {
2849 fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
2853 static void io_mem_init(void)
2857 cpu_register_io_memory(IO_MEM_ROM >> IO_MEM_SHIFT, error_mem_read, unassigned_mem_write, NULL);
2858 cpu_register_io_memory(IO_MEM_UNASSIGNED >> IO_MEM_SHIFT, unassigned_mem_read, unassigned_mem_write, NULL);
2859 cpu_register_io_memory(IO_MEM_NOTDIRTY >> IO_MEM_SHIFT, error_mem_read, notdirty_mem_write, NULL);
2863 io_mem_watch = cpu_register_io_memory(0, watch_mem_read,
2864 watch_mem_write, NULL);
2865 /* alloc dirty bits array */
2866 phys_ram_dirty = qemu_vmalloc(phys_ram_size >> TARGET_PAGE_BITS);
2867 memset(phys_ram_dirty, 0xff, phys_ram_size >> TARGET_PAGE_BITS);
2870 /* mem_read and mem_write are arrays of functions containing the
2871 function to access byte (index 0), word (index 1) and dword (index
2872 2). Functions can be omitted with a NULL function pointer. The
2873 registered functions may be modified dynamically later.
2874 If io_index is non zero, the corresponding io zone is
2875 modified. If it is zero, a new io zone is allocated. The return
2876 value can be used with cpu_register_physical_memory(). (-1) is
2877 returned if error. */
2878 int cpu_register_io_memory(int io_index,
2879 CPUReadMemoryFunc **mem_read,
2880 CPUWriteMemoryFunc **mem_write,
2883 int i, subwidth = 0;
2885 if (io_index <= 0) {
2886 io_index = get_free_io_mem_idx();
2890 if (io_index >= IO_MEM_NB_ENTRIES)
2894 for(i = 0;i < 3; i++) {
2895 if (!mem_read[i] || !mem_write[i])
2896 subwidth = IO_MEM_SUBWIDTH;
2897 io_mem_read[io_index][i] = mem_read[i];
2898 io_mem_write[io_index][i] = mem_write[i];
2900 io_mem_opaque[io_index] = opaque;
2901 return (io_index << IO_MEM_SHIFT) | subwidth;
2904 void cpu_unregister_io_memory(int io_table_address)
2907 int io_index = io_table_address >> IO_MEM_SHIFT;
2909 for (i=0;i < 3; i++) {
2910 io_mem_read[io_index][i] = unassigned_mem_read[i];
2911 io_mem_write[io_index][i] = unassigned_mem_write[i];
2913 io_mem_opaque[io_index] = NULL;
2914 io_mem_used[io_index] = 0;
2917 CPUWriteMemoryFunc **cpu_get_io_memory_write(int io_index)
2919 return io_mem_write[io_index >> IO_MEM_SHIFT];
2922 CPUReadMemoryFunc **cpu_get_io_memory_read(int io_index)
2924 return io_mem_read[io_index >> IO_MEM_SHIFT];
2927 #endif /* !defined(CONFIG_USER_ONLY) */
2929 /* physical memory access (slow version, mainly for debug) */
2930 #if defined(CONFIG_USER_ONLY)
2931 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2932 int len, int is_write)
2939 page = addr & TARGET_PAGE_MASK;
2940 l = (page + TARGET_PAGE_SIZE) - addr;
2943 flags = page_get_flags(page);
2944 if (!(flags & PAGE_VALID))
2947 if (!(flags & PAGE_WRITE))
2949 /* XXX: this code should not depend on lock_user */
2950 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2951 /* FIXME - should this return an error rather than just fail? */
2954 unlock_user(p, addr, l);
2956 if (!(flags & PAGE_READ))
2958 /* XXX: this code should not depend on lock_user */
2959 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2960 /* FIXME - should this return an error rather than just fail? */
2963 unlock_user(p, addr, 0);
2972 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2973 int len, int is_write)
2978 target_phys_addr_t page;
2983 page = addr & TARGET_PAGE_MASK;
2984 l = (page + TARGET_PAGE_SIZE) - addr;
2987 p = phys_page_find(page >> TARGET_PAGE_BITS);
2989 pd = IO_MEM_UNASSIGNED;
2991 pd = p->phys_offset;
2995 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
2996 target_phys_addr_t addr1 = addr;
2997 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
2999 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3000 /* XXX: could force cpu_single_env to NULL to avoid
3002 if (l >= 4 && ((addr1 & 3) == 0)) {
3003 /* 32 bit write access */
3005 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3007 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3008 /* 16 bit write access */
3010 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3013 /* 8 bit write access */
3015 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3019 unsigned long addr1;
3020 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3022 ptr = phys_ram_base + addr1;
3023 memcpy(ptr, buf, l);
3024 if (!cpu_physical_memory_is_dirty(addr1)) {
3025 /* invalidate code */
3026 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3028 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3029 (0xff & ~CODE_DIRTY_FLAG);
3033 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3034 !(pd & IO_MEM_ROMD)) {
3035 target_phys_addr_t addr1 = addr;
3037 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3039 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3040 if (l >= 4 && ((addr1 & 3) == 0)) {
3041 /* 32 bit read access */
3042 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3045 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3046 /* 16 bit read access */
3047 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3051 /* 8 bit read access */
3052 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3058 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3059 (addr & ~TARGET_PAGE_MASK);
3060 memcpy(buf, ptr, l);
3069 /* used for ROM loading : can write in RAM and ROM */
3070 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3071 const uint8_t *buf, int len)
3075 target_phys_addr_t page;
3080 page = addr & TARGET_PAGE_MASK;
3081 l = (page + TARGET_PAGE_SIZE) - addr;
3084 p = phys_page_find(page >> TARGET_PAGE_BITS);
3086 pd = IO_MEM_UNASSIGNED;
3088 pd = p->phys_offset;
3091 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3092 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3093 !(pd & IO_MEM_ROMD)) {
3096 unsigned long addr1;
3097 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3099 ptr = phys_ram_base + addr1;
3100 memcpy(ptr, buf, l);
3110 target_phys_addr_t addr;
3111 target_phys_addr_t len;
3114 static BounceBuffer bounce;
3116 typedef struct MapClient {
3118 void (*callback)(void *opaque);
3119 LIST_ENTRY(MapClient) link;
3122 static LIST_HEAD(map_client_list, MapClient) map_client_list
3123 = LIST_HEAD_INITIALIZER(map_client_list);
3125 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3127 MapClient *client = qemu_malloc(sizeof(*client));
3129 client->opaque = opaque;
3130 client->callback = callback;
3131 LIST_INSERT_HEAD(&map_client_list, client, link);
3135 void cpu_unregister_map_client(void *_client)
3137 MapClient *client = (MapClient *)_client;
3139 LIST_REMOVE(client, link);
3142 static void cpu_notify_map_clients(void)
3146 while (!LIST_EMPTY(&map_client_list)) {
3147 client = LIST_FIRST(&map_client_list);
3148 client->callback(client->opaque);
3149 LIST_REMOVE(client, link);
3153 /* Map a physical memory region into a host virtual address.
3154 * May map a subset of the requested range, given by and returned in *plen.
3155 * May return NULL if resources needed to perform the mapping are exhausted.
3156 * Use only for reads OR writes - not for read-modify-write operations.
3157 * Use cpu_register_map_client() to know when retrying the map operation is
3158 * likely to succeed.
3160 void *cpu_physical_memory_map(target_phys_addr_t addr,
3161 target_phys_addr_t *plen,
3164 target_phys_addr_t len = *plen;
3165 target_phys_addr_t done = 0;
3167 uint8_t *ret = NULL;
3169 target_phys_addr_t page;
3172 unsigned long addr1;
3175 page = addr & TARGET_PAGE_MASK;
3176 l = (page + TARGET_PAGE_SIZE) - addr;
3179 p = phys_page_find(page >> TARGET_PAGE_BITS);
3181 pd = IO_MEM_UNASSIGNED;
3183 pd = p->phys_offset;
3186 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3187 if (done || bounce.buffer) {
3190 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3194 cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3196 ptr = bounce.buffer;
3198 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3199 ptr = phys_ram_base + addr1;
3203 } else if (ret + done != ptr) {
3215 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3216 * Will also mark the memory as dirty if is_write == 1. access_len gives
3217 * the amount of memory that was actually read or written by the caller.
3219 void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3220 int is_write, target_phys_addr_t access_len)
3222 if (buffer != bounce.buffer) {
3224 unsigned long addr1 = (uint8_t *)buffer - phys_ram_base;
3225 while (access_len) {
3227 l = TARGET_PAGE_SIZE;
3230 if (!cpu_physical_memory_is_dirty(addr1)) {
3231 /* invalidate code */
3232 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3234 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3235 (0xff & ~CODE_DIRTY_FLAG);
3244 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3246 qemu_free(bounce.buffer);
3247 bounce.buffer = NULL;
3248 cpu_notify_map_clients();
3251 /* warning: addr must be aligned */
3252 uint32_t ldl_phys(target_phys_addr_t addr)
3260 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3262 pd = IO_MEM_UNASSIGNED;
3264 pd = p->phys_offset;
3267 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3268 !(pd & IO_MEM_ROMD)) {
3270 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3272 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3273 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3276 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3277 (addr & ~TARGET_PAGE_MASK);
3283 /* warning: addr must be aligned */
3284 uint64_t ldq_phys(target_phys_addr_t addr)
3292 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3294 pd = IO_MEM_UNASSIGNED;
3296 pd = p->phys_offset;
3299 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3300 !(pd & IO_MEM_ROMD)) {
3302 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3304 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3305 #ifdef TARGET_WORDS_BIGENDIAN
3306 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3307 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3309 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3310 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3314 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3315 (addr & ~TARGET_PAGE_MASK);
3322 uint32_t ldub_phys(target_phys_addr_t addr)
3325 cpu_physical_memory_read(addr, &val, 1);
3330 uint32_t lduw_phys(target_phys_addr_t addr)
3333 cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3334 return tswap16(val);
3337 /* warning: addr must be aligned. The ram page is not masked as dirty
3338 and the code inside is not invalidated. It is useful if the dirty
3339 bits are used to track modified PTEs */
3340 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3347 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3349 pd = IO_MEM_UNASSIGNED;
3351 pd = p->phys_offset;
3354 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3355 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3357 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3358 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3360 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3361 ptr = phys_ram_base + addr1;
3364 if (unlikely(in_migration)) {
3365 if (!cpu_physical_memory_is_dirty(addr1)) {
3366 /* invalidate code */
3367 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3369 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3370 (0xff & ~CODE_DIRTY_FLAG);
3376 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3383 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3385 pd = IO_MEM_UNASSIGNED;
3387 pd = p->phys_offset;
3390 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3391 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3393 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3394 #ifdef TARGET_WORDS_BIGENDIAN
3395 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3396 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3398 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3399 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3402 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3403 (addr & ~TARGET_PAGE_MASK);
3408 /* warning: addr must be aligned */
3409 void stl_phys(target_phys_addr_t addr, uint32_t val)
3416 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3418 pd = IO_MEM_UNASSIGNED;
3420 pd = p->phys_offset;
3423 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3424 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3426 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3427 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3429 unsigned long addr1;
3430 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3432 ptr = phys_ram_base + addr1;
3434 if (!cpu_physical_memory_is_dirty(addr1)) {
3435 /* invalidate code */
3436 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3438 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3439 (0xff & ~CODE_DIRTY_FLAG);
3445 void stb_phys(target_phys_addr_t addr, uint32_t val)
3448 cpu_physical_memory_write(addr, &v, 1);
3452 void stw_phys(target_phys_addr_t addr, uint32_t val)
3454 uint16_t v = tswap16(val);
3455 cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3459 void stq_phys(target_phys_addr_t addr, uint64_t val)
3462 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3467 /* virtual memory access for debug (includes writing to ROM) */
3468 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3469 uint8_t *buf, int len, int is_write)
3472 target_phys_addr_t phys_addr;
3476 page = addr & TARGET_PAGE_MASK;
3477 phys_addr = cpu_get_phys_page_debug(env, page);
3478 /* if no physical page mapped, return an error */
3479 if (phys_addr == -1)
3481 l = (page + TARGET_PAGE_SIZE) - addr;
3484 phys_addr += (addr & ~TARGET_PAGE_MASK);
3485 #if !defined(CONFIG_USER_ONLY)
3487 cpu_physical_memory_write_rom(phys_addr, buf, l);
3490 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
3498 /* in deterministic execution mode, instructions doing device I/Os
3499 must be at the end of the TB */
3500 void cpu_io_recompile(CPUState *env, void *retaddr)
3502 TranslationBlock *tb;
3504 target_ulong pc, cs_base;
3507 tb = tb_find_pc((unsigned long)retaddr);
3509 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3512 n = env->icount_decr.u16.low + tb->icount;
3513 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3514 /* Calculate how many instructions had been executed before the fault
3516 n = n - env->icount_decr.u16.low;
3517 /* Generate a new TB ending on the I/O insn. */
3519 /* On MIPS and SH, delay slot instructions can only be restarted if
3520 they were already the first instruction in the TB. If this is not
3521 the first instruction in a TB then re-execute the preceding
3523 #if defined(TARGET_MIPS)
3524 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3525 env->active_tc.PC -= 4;
3526 env->icount_decr.u16.low++;
3527 env->hflags &= ~MIPS_HFLAG_BMASK;
3529 #elif defined(TARGET_SH4)
3530 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3533 env->icount_decr.u16.low++;
3534 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3537 /* This should never happen. */
3538 if (n > CF_COUNT_MASK)
3539 cpu_abort(env, "TB too big during recompile");
3541 cflags = n | CF_LAST_IO;
3543 cs_base = tb->cs_base;
3545 tb_phys_invalidate(tb, -1);
3546 /* FIXME: In theory this could raise an exception. In practice
3547 we have already translated the block once so it's probably ok. */
3548 tb_gen_code(env, pc, cs_base, flags, cflags);
3549 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3550 the first in the TB) then we end up generating a whole new TB and
3551 repeating the fault, which is horribly inefficient.
3552 Better would be to execute just this insn uncached, or generate a
3554 cpu_resume_from_signal(env, NULL);
3557 void dump_exec_info(FILE *f,
3558 int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
3560 int i, target_code_size, max_target_code_size;
3561 int direct_jmp_count, direct_jmp2_count, cross_page;
3562 TranslationBlock *tb;
3564 target_code_size = 0;
3565 max_target_code_size = 0;
3567 direct_jmp_count = 0;
3568 direct_jmp2_count = 0;
3569 for(i = 0; i < nb_tbs; i++) {
3571 target_code_size += tb->size;
3572 if (tb->size > max_target_code_size)
3573 max_target_code_size = tb->size;
3574 if (tb->page_addr[1] != -1)
3576 if (tb->tb_next_offset[0] != 0xffff) {
3578 if (tb->tb_next_offset[1] != 0xffff) {
3579 direct_jmp2_count++;
3583 /* XXX: avoid using doubles ? */
3584 cpu_fprintf(f, "Translation buffer state:\n");
3585 cpu_fprintf(f, "gen code size %ld/%ld\n",
3586 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
3587 cpu_fprintf(f, "TB count %d/%d\n",
3588 nb_tbs, code_gen_max_blocks);
3589 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
3590 nb_tbs ? target_code_size / nb_tbs : 0,
3591 max_target_code_size);
3592 cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3593 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
3594 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
3595 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
3597 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
3598 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3600 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
3602 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
3603 cpu_fprintf(f, "\nStatistics:\n");
3604 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
3605 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
3606 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
3607 tcg_dump_info(f, cpu_fprintf);
3610 #if !defined(CONFIG_USER_ONLY)
3612 #define MMUSUFFIX _cmmu
3613 #define GETPC() NULL
3614 #define env cpu_single_env
3615 #define SOFTMMU_CODE_ACCESS
3618 #include "softmmu_template.h"
3621 #include "softmmu_template.h"
3624 #include "softmmu_template.h"
3627 #include "softmmu_template.h"