Annotation of qemu/exec.c, revision 1.1.1.13

1.1       root        1: /*
                      2:  *  virtual page mapping and translated block handling
1.1.1.6   root        3:  *
1.1       root        4:  *  Copyright (c) 2003 Fabrice Bellard
                      5:  *
                      6:  * This library is free software; you can redistribute it and/or
                      7:  * modify it under the terms of the GNU Lesser General Public
                      8:  * License as published by the Free Software Foundation; either
                      9:  * version 2 of the License, or (at your option) any later version.
                     10:  *
                     11:  * This library is distributed in the hope that it will be useful,
                     12:  * but WITHOUT ANY WARRANTY; without even the implied warranty of
                     13:  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
                     14:  * Lesser General Public License for more details.
                     15:  *
                     16:  * You should have received a copy of the GNU Lesser General Public
1.1.1.10  root       17:  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
1.1       root       18:  */
                     19: #include "config.h"
                     20: #ifdef _WIN32
                     21: #include <windows.h>
                     22: #else
                     23: #include <sys/types.h>
                     24: #include <sys/mman.h>
                     25: #endif
                     26: #include <stdlib.h>
                     27: #include <stdio.h>
                     28: #include <stdarg.h>
                     29: #include <string.h>
                     30: #include <errno.h>
                     31: #include <unistd.h>
                     32: #include <inttypes.h>
                     33: 
                     34: #include "cpu.h"
                     35: #include "exec-all.h"
1.1.1.7   root       36: #include "qemu-common.h"
                     37: #include "tcg.h"
                     38: #include "hw/hw.h"
1.1.1.13! root       39: #include "hw/qdev.h"
1.1.1.7   root       40: #include "osdep.h"
                     41: #include "kvm.h"
1.1.1.13! root       42: #include "qemu-timer.h"
1.1.1.3   root       43: #if defined(CONFIG_USER_ONLY)
                     44: #include <qemu.h>
1.1.1.13! root       45: #include <signal.h>
        !            46: #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
        !            47: #include <sys/param.h>
        !            48: #if __FreeBSD_version >= 700104
        !            49: #define HAVE_KINFO_GETVMMAP
        !            50: #define sigqueue sigqueue_freebsd  /* avoid redefinition */
        !            51: #include <sys/time.h>
        !            52: #include <sys/proc.h>
        !            53: #include <machine/profile.h>
        !            54: #define _KERNEL
        !            55: #include <sys/user.h>
        !            56: #undef _KERNEL
        !            57: #undef sigqueue
        !            58: #include <libutil.h>
        !            59: #endif
        !            60: #endif
1.1.1.3   root       61: #endif
1.1       root       62: 
                     63: //#define DEBUG_TB_INVALIDATE
                     64: //#define DEBUG_FLUSH
                     65: //#define DEBUG_TLB
1.1.1.5   root       66: //#define DEBUG_UNASSIGNED
1.1       root       67: 
                     68: /* make various TB consistency checks */
1.1.1.6   root       69: //#define DEBUG_TB_CHECK
                     70: //#define DEBUG_TLB_CHECK
                     71: 
                     72: //#define DEBUG_IOPORT
                     73: //#define DEBUG_SUBPAGE
1.1       root       74: 
1.1.1.3   root       75: #if !defined(CONFIG_USER_ONLY)
                     76: /* TB consistency checks only implemented for usermode emulation.  */
                     77: #undef DEBUG_TB_CHECK
                     78: #endif
                     79: 
1.1       root       80: #define SMC_BITMAP_USE_THRESHOLD 10
                     81: 
1.1.1.7   root       82: static TranslationBlock *tbs;
1.1.1.13! root       83: static int code_gen_max_blocks;
1.1       root       84: TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
1.1.1.7   root       85: static int nb_tbs;
1.1       root       86: /* any access to the tbs or the page table must use this lock */
                     87: spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
                     88: 
1.1.1.7   root       89: #if defined(__arm__) || defined(__sparc_v9__)
                     90: /* The prologue must be reachable with a direct jump. ARM and Sparc64
                     91:  have limited branch ranges (possibly also PPC) so place it in a
                     92:  section close to code segment. */
                     93: #define code_gen_section                                \
                     94:     __attribute__((__section__(".gen_code")))           \
                     95:     __attribute__((aligned (32)))
1.1.1.10  root       96: #elif defined(_WIN32)
                     97: /* Maximum alignment for Win32 is 16. */
                     98: #define code_gen_section                                \
                     99:     __attribute__((aligned (16)))
1.1.1.7   root      100: #else
                    101: #define code_gen_section                                \
                    102:     __attribute__((aligned (32)))
                    103: #endif
                    104: 
                    105: uint8_t code_gen_prologue[1024] code_gen_section;
                    106: static uint8_t *code_gen_buffer;
                    107: static unsigned long code_gen_buffer_size;
                    108: /* threshold to flush the translated code buffer */
                    109: static unsigned long code_gen_buffer_max_size;
1.1.1.13! root      110: static uint8_t *code_gen_ptr;
1.1       root      111: 
1.1.1.7   root      112: #if !defined(CONFIG_USER_ONLY)
1.1       root      113: int phys_ram_fd;
1.1.1.7   root      114: static int in_migration;
1.1.1.10  root      115: 
1.1.1.13! root      116: RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list) };
1.1.1.7   root      117: #endif
1.1       root      118: 
1.1.1.2   root      119: CPUState *first_cpu;
                    120: /* current CPU in the current thread. It is only valid inside
                    121:    cpu_exec() */
1.1.1.6   root      122: CPUState *cpu_single_env;
1.1.1.7   root      123: /* 0 = Do not count executed instructions.
                    124:    1 = Precise instruction counting.
                    125:    2 = Adaptive rate instruction counting.  */
                    126: int use_icount = 0;
                    127: /* Current instruction counter.  While executing translated code this may
                    128:    include some instructions that have not yet been executed.  */
                    129: int64_t qemu_icount;
1.1.1.2   root      130: 
1.1       root      131: typedef struct PageDesc {
                    132:     /* list of TBs intersecting this ram page */
                    133:     TranslationBlock *first_tb;
                    134:     /* in order to optimize self modifying code, we count the number
                    135:        of lookups we do to a given page to use a bitmap */
                    136:     unsigned int code_write_count;
                    137:     uint8_t *code_bitmap;
                    138: #if defined(CONFIG_USER_ONLY)
                    139:     unsigned long flags;
                    140: #endif
                    141: } PageDesc;
                    142: 
1.1.1.13! root      143: /* In system mode we want L1_MAP to be based on ram offsets,
        !           144:    while in user mode we want it to be based on virtual addresses.  */
        !           145: #if !defined(CONFIG_USER_ONLY)
        !           146: #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
        !           147: # define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS
        !           148: #else
        !           149: # define L1_MAP_ADDR_SPACE_BITS  TARGET_PHYS_ADDR_SPACE_BITS
        !           150: #endif
        !           151: #else
        !           152: # define L1_MAP_ADDR_SPACE_BITS  TARGET_VIRT_ADDR_SPACE_BITS
        !           153: #endif
1.1       root      154: 
1.1.1.13! root      155: /* Size of the L2 (and L3, etc) page tables.  */
1.1       root      156: #define L2_BITS 10
1.1.1.13! root      157: #define L2_SIZE (1 << L2_BITS)
        !           158: 
        !           159: /* The bits remaining after N lower levels of page tables.  */
        !           160: #define P_L1_BITS_REM \
        !           161:     ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
        !           162: #define V_L1_BITS_REM \
        !           163:     ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
        !           164: 
        !           165: /* Size of the L1 page table.  Avoid silly small sizes.  */
        !           166: #if P_L1_BITS_REM < 4
        !           167: #define P_L1_BITS  (P_L1_BITS_REM + L2_BITS)
1.1.1.6   root      168: #else
1.1.1.13! root      169: #define P_L1_BITS  P_L1_BITS_REM
1.1.1.6   root      170: #endif
1.1       root      171: 
1.1.1.13! root      172: #if V_L1_BITS_REM < 4
        !           173: #define V_L1_BITS  (V_L1_BITS_REM + L2_BITS)
        !           174: #else
        !           175: #define V_L1_BITS  V_L1_BITS_REM
        !           176: #endif
        !           177: 
        !           178: #define P_L1_SIZE  ((target_phys_addr_t)1 << P_L1_BITS)
        !           179: #define V_L1_SIZE  ((target_ulong)1 << V_L1_BITS)
        !           180: 
        !           181: #define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
        !           182: #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
1.1       root      183: 
                    184: unsigned long qemu_real_host_page_size;
                    185: unsigned long qemu_host_page_bits;
                    186: unsigned long qemu_host_page_size;
                    187: unsigned long qemu_host_page_mask;
                    188: 
1.1.1.13! root      189: /* This is a multi-level map on the virtual address space.
        !           190:    The bottom level has pointers to PageDesc.  */
        !           191: static void *l1_map[V_L1_SIZE];
1.1.1.7   root      192: 
                    193: #if !defined(CONFIG_USER_ONLY)
1.1.1.13! root      194: typedef struct PhysPageDesc {
        !           195:     /* offset in host memory of the page + io_index in the low bits */
        !           196:     ram_addr_t phys_offset;
        !           197:     ram_addr_t region_offset;
        !           198: } PhysPageDesc;
        !           199: 
        !           200: /* This is a multi-level map on the physical address space.
        !           201:    The bottom level has pointers to PhysPageDesc.  */
        !           202: static void *l1_phys_map[P_L1_SIZE];
        !           203: 
1.1.1.7   root      204: static void io_mem_init(void);
1.1       root      205: 
                    206: /* io memory support */
                    207: CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
                    208: CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
                    209: void *io_mem_opaque[IO_MEM_NB_ENTRIES];
1.1.1.10  root      210: static char io_mem_used[IO_MEM_NB_ENTRIES];
1.1.1.6   root      211: static int io_mem_watch;
                    212: #endif
1.1       root      213: 
                    214: /* log support */
1.1.1.13! root      215: #ifdef WIN32
        !           216: static const char *logfilename = "qemu.log";
        !           217: #else
1.1.1.7   root      218: static const char *logfilename = "/tmp/qemu.log";
1.1.1.13! root      219: #endif
1.1       root      220: FILE *logfile;
                    221: int loglevel;
1.1.1.6   root      222: static int log_append = 0;
1.1       root      223: 
                    224: /* statistics */
1.1.1.13! root      225: #if !defined(CONFIG_USER_ONLY)
1.1       root      226: static int tlb_flush_count;
1.1.1.13! root      227: #endif
1.1       root      228: static int tb_flush_count;
                    229: static int tb_phys_invalidate_count;
                    230: 
1.1.1.7   root      231: #ifdef _WIN32
                    232: static void map_exec(void *addr, long size)
                    233: {
                    234:     DWORD old_protect;
                    235:     VirtualProtect(addr, size,
                    236:                    PAGE_EXECUTE_READWRITE, &old_protect);
                    237:     
                    238: }
                    239: #else
                    240: static void map_exec(void *addr, long size)
                    241: {
                    242:     unsigned long start, end, page_size;
                    243:     
                    244:     page_size = getpagesize();
                    245:     start = (unsigned long)addr;
                    246:     start &= ~(page_size - 1);
                    247:     
                    248:     end = (unsigned long)addr + size;
                    249:     end += page_size - 1;
                    250:     end &= ~(page_size - 1);
                    251:     
                    252:     mprotect((void *)start, end - start,
                    253:              PROT_READ | PROT_WRITE | PROT_EXEC);
                    254: }
                    255: #endif
                    256: 
1.1       root      257: static void page_init(void)
                    258: {
                    259:     /* NOTE: we can always suppose that qemu_host_page_size >=
                    260:        TARGET_PAGE_SIZE */
                    261: #ifdef _WIN32
                    262:     {
                    263:         SYSTEM_INFO system_info;
1.1.1.6   root      264: 
1.1       root      265:         GetSystemInfo(&system_info);
                    266:         qemu_real_host_page_size = system_info.dwPageSize;
                    267:     }
                    268: #else
                    269:     qemu_real_host_page_size = getpagesize();
                    270: #endif
                    271:     if (qemu_host_page_size == 0)
                    272:         qemu_host_page_size = qemu_real_host_page_size;
                    273:     if (qemu_host_page_size < TARGET_PAGE_SIZE)
                    274:         qemu_host_page_size = TARGET_PAGE_SIZE;
                    275:     qemu_host_page_bits = 0;
                    276:     while ((1 << qemu_host_page_bits) < qemu_host_page_size)
                    277:         qemu_host_page_bits++;
                    278:     qemu_host_page_mask = ~(qemu_host_page_size - 1);
1.1.1.6   root      279: 
1.1.1.13! root      280: #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
1.1.1.6   root      281:     {
1.1.1.13! root      282: #ifdef HAVE_KINFO_GETVMMAP
        !           283:         struct kinfo_vmentry *freep;
        !           284:         int i, cnt;
        !           285: 
        !           286:         freep = kinfo_getvmmap(getpid(), &cnt);
        !           287:         if (freep) {
        !           288:             mmap_lock();
        !           289:             for (i = 0; i < cnt; i++) {
        !           290:                 unsigned long startaddr, endaddr;
        !           291: 
        !           292:                 startaddr = freep[i].kve_start;
        !           293:                 endaddr = freep[i].kve_end;
        !           294:                 if (h2g_valid(startaddr)) {
        !           295:                     startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
        !           296: 
        !           297:                     if (h2g_valid(endaddr)) {
        !           298:                         endaddr = h2g(endaddr);
        !           299:                         page_set_flags(startaddr, endaddr, PAGE_RESERVED);
        !           300:                     } else {
        !           301: #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
        !           302:                         endaddr = ~0ul;
        !           303:                         page_set_flags(startaddr, endaddr, PAGE_RESERVED);
        !           304: #endif
        !           305:                     }
        !           306:                 }
        !           307:             }
        !           308:             free(freep);
        !           309:             mmap_unlock();
        !           310:         }
        !           311: #else
1.1.1.6   root      312:         FILE *f;
                    313: 
1.1.1.7   root      314:         last_brk = (unsigned long)sbrk(0);
1.1.1.13! root      315: 
        !           316:         f = fopen("/compat/linux/proc/self/maps", "r");
1.1.1.6   root      317:         if (f) {
1.1.1.13! root      318:             mmap_lock();
        !           319: 
1.1.1.6   root      320:             do {
1.1.1.13! root      321:                 unsigned long startaddr, endaddr;
        !           322:                 int n;
        !           323: 
        !           324:                 n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
        !           325: 
        !           326:                 if (n == 2 && h2g_valid(startaddr)) {
        !           327:                     startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
        !           328: 
        !           329:                     if (h2g_valid(endaddr)) {
        !           330:                         endaddr = h2g(endaddr);
        !           331:                     } else {
        !           332:                         endaddr = ~0ul;
        !           333:                     }
        !           334:                     page_set_flags(startaddr, endaddr, PAGE_RESERVED);
1.1.1.6   root      335:                 }
                    336:             } while (!feof(f));
1.1.1.13! root      337: 
1.1.1.6   root      338:             fclose(f);
1.1.1.13! root      339:             mmap_unlock();
1.1.1.6   root      340:         }
                    341: #endif
1.1.1.13! root      342:     }
1.1.1.7   root      343: #endif
                    344: }
                    345: 
1.1.1.13! root      346: static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
1.1       root      347: {
1.1.1.13! root      348:     PageDesc *pd;
        !           349:     void **lp;
        !           350:     int i;
1.1       root      351: 
1.1.1.7   root      352: #if defined(CONFIG_USER_ONLY)
1.1.1.13! root      353:     /* We can't use qemu_malloc because it may recurse into a locked mutex. */
        !           354: # define ALLOC(P, SIZE)                                 \
        !           355:     do {                                                \
        !           356:         P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,    \
        !           357:                  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);   \
        !           358:     } while (0)
1.1.1.7   root      359: #else
1.1.1.13! root      360: # define ALLOC(P, SIZE) \
        !           361:     do { P = qemu_mallocz(SIZE); } while (0)
1.1.1.7   root      362: #endif
1.1.1.13! root      363: 
        !           364:     /* Level 1.  Always allocated.  */
        !           365:     lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
        !           366: 
        !           367:     /* Level 2..N-1.  */
        !           368:     for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
        !           369:         void **p = *lp;
        !           370: 
        !           371:         if (p == NULL) {
        !           372:             if (!alloc) {
        !           373:                 return NULL;
        !           374:             }
        !           375:             ALLOC(p, sizeof(void *) * L2_SIZE);
        !           376:             *lp = p;
        !           377:         }
        !           378: 
        !           379:         lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
        !           380:     }
        !           381: 
        !           382:     pd = *lp;
        !           383:     if (pd == NULL) {
        !           384:         if (!alloc) {
        !           385:             return NULL;
        !           386:         }
        !           387:         ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
        !           388:         *lp = pd;
1.1       root      389:     }
1.1.1.13! root      390: 
        !           391: #undef ALLOC
        !           392: 
        !           393:     return pd + (index & (L2_SIZE - 1));
1.1       root      394: }
                    395: 
1.1.1.13! root      396: static inline PageDesc *page_find(tb_page_addr_t index)
1.1       root      397: {
1.1.1.13! root      398:     return page_find_alloc(index, 0);
1.1       root      399: }
                    400: 
1.1.1.13! root      401: #if !defined(CONFIG_USER_ONLY)
1.1       root      402: static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
                    403: {
1.1.1.3   root      404:     PhysPageDesc *pd;
1.1.1.13! root      405:     void **lp;
        !           406:     int i;
1.1       root      407: 
1.1.1.13! root      408:     /* Level 1.  Always allocated.  */
        !           409:     lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
1.1       root      410: 
1.1.1.13! root      411:     /* Level 2..N-1.  */
        !           412:     for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
        !           413:         void **p = *lp;
        !           414:         if (p == NULL) {
        !           415:             if (!alloc) {
        !           416:                 return NULL;
        !           417:             }
        !           418:             *lp = p = qemu_mallocz(sizeof(void *) * L2_SIZE);
        !           419:         }
        !           420:         lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
1.1       root      421:     }
1.1.1.13! root      422: 
1.1.1.3   root      423:     pd = *lp;
1.1.1.13! root      424:     if (pd == NULL) {
1.1.1.3   root      425:         int i;
1.1.1.13! root      426: 
        !           427:         if (!alloc) {
1.1       root      428:             return NULL;
1.1.1.13! root      429:         }
        !           430: 
        !           431:         *lp = pd = qemu_malloc(sizeof(PhysPageDesc) * L2_SIZE);
        !           432: 
1.1.1.7   root      433:         for (i = 0; i < L2_SIZE; i++) {
1.1.1.13! root      434:             pd[i].phys_offset = IO_MEM_UNASSIGNED;
        !           435:             pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
1.1.1.7   root      436:         }
1.1       root      437:     }
1.1.1.13! root      438: 
        !           439:     return pd + (index & (L2_SIZE - 1));
1.1       root      440: }
                    441: 
                    442: static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
                    443: {
                    444:     return phys_page_find_alloc(index, 0);
                    445: }
                    446: 
1.1.1.2   root      447: static void tlb_protect_code(ram_addr_t ram_addr);
1.1.1.6   root      448: static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1.1       root      449:                                     target_ulong vaddr);
1.1.1.7   root      450: #define mmap_lock() do { } while(0)
                    451: #define mmap_unlock() do { } while(0)
                    452: #endif
                    453: 
                    454: #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
                    455: 
                    456: #if defined(CONFIG_USER_ONLY)
1.1.1.10  root      457: /* Currently it is not recommended to allocate big chunks of data in
1.1.1.7   root      458:    user mode. It will change when a dedicated libc will be used */
                    459: #define USE_STATIC_CODE_GEN_BUFFER
                    460: #endif
                    461: 
                    462: #ifdef USE_STATIC_CODE_GEN_BUFFER
1.1.1.13! root      463: static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
        !           464:                __attribute__((aligned (CODE_GEN_ALIGN)));
1.1.1.7   root      465: #endif
                    466: 
                    467: static void code_gen_alloc(unsigned long tb_size)
                    468: {
                    469: #ifdef USE_STATIC_CODE_GEN_BUFFER
                    470:     code_gen_buffer = static_code_gen_buffer;
                    471:     code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
                    472:     map_exec(code_gen_buffer, code_gen_buffer_size);
                    473: #else
                    474:     code_gen_buffer_size = tb_size;
                    475:     if (code_gen_buffer_size == 0) {
                    476: #if defined(CONFIG_USER_ONLY)
                    477:         /* in user mode, phys_ram_size is not meaningful */
                    478:         code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
                    479: #else
1.1.1.10  root      480:         /* XXX: needs adjustments */
                    481:         code_gen_buffer_size = (unsigned long)(ram_size / 4);
1.1.1.7   root      482: #endif
                    483:     }
                    484:     if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
                    485:         code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
                    486:     /* The code gen buffer location may have constraints depending on
                    487:        the host cpu and OS */
                    488: #if defined(__linux__) 
                    489:     {
                    490:         int flags;
                    491:         void *start = NULL;
                    492: 
                    493:         flags = MAP_PRIVATE | MAP_ANONYMOUS;
                    494: #if defined(__x86_64__)
                    495:         flags |= MAP_32BIT;
                    496:         /* Cannot map more than that */
                    497:         if (code_gen_buffer_size > (800 * 1024 * 1024))
                    498:             code_gen_buffer_size = (800 * 1024 * 1024);
                    499: #elif defined(__sparc_v9__)
                    500:         // Map the buffer below 2G, so we can use direct calls and branches
                    501:         flags |= MAP_FIXED;
                    502:         start = (void *) 0x60000000UL;
                    503:         if (code_gen_buffer_size > (512 * 1024 * 1024))
                    504:             code_gen_buffer_size = (512 * 1024 * 1024);
                    505: #elif defined(__arm__)
                    506:         /* Map the buffer below 32M, so we can use direct calls and branches */
                    507:         flags |= MAP_FIXED;
                    508:         start = (void *) 0x01000000UL;
                    509:         if (code_gen_buffer_size > 16 * 1024 * 1024)
                    510:             code_gen_buffer_size = 16 * 1024 * 1024;
1.1.1.13! root      511: #elif defined(__s390x__)
        !           512:         /* Map the buffer so that we can use direct calls and branches.  */
        !           513:         /* We have a +- 4GB range on the branches; leave some slop.  */
        !           514:         if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {
        !           515:             code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;
        !           516:         }
        !           517:         start = (void *)0x90000000UL;
1.1.1.7   root      518: #endif
                    519:         code_gen_buffer = mmap(start, code_gen_buffer_size,
                    520:                                PROT_WRITE | PROT_READ | PROT_EXEC,
                    521:                                flags, -1, 0);
                    522:         if (code_gen_buffer == MAP_FAILED) {
                    523:             fprintf(stderr, "Could not allocate dynamic translator buffer\n");
                    524:             exit(1);
                    525:         }
                    526:     }
1.1.1.11  root      527: #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
1.1.1.7   root      528:     {
                    529:         int flags;
                    530:         void *addr = NULL;
                    531:         flags = MAP_PRIVATE | MAP_ANONYMOUS;
                    532: #if defined(__x86_64__)
                    533:         /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
                    534:          * 0x40000000 is free */
                    535:         flags |= MAP_FIXED;
                    536:         addr = (void *)0x40000000;
                    537:         /* Cannot map more than that */
                    538:         if (code_gen_buffer_size > (800 * 1024 * 1024))
                    539:             code_gen_buffer_size = (800 * 1024 * 1024);
                    540: #endif
                    541:         code_gen_buffer = mmap(addr, code_gen_buffer_size,
                    542:                                PROT_WRITE | PROT_READ | PROT_EXEC, 
                    543:                                flags, -1, 0);
                    544:         if (code_gen_buffer == MAP_FAILED) {
                    545:             fprintf(stderr, "Could not allocate dynamic translator buffer\n");
                    546:             exit(1);
                    547:         }
                    548:     }
                    549: #else
                    550:     code_gen_buffer = qemu_malloc(code_gen_buffer_size);
                    551:     map_exec(code_gen_buffer, code_gen_buffer_size);
                    552: #endif
                    553: #endif /* !USE_STATIC_CODE_GEN_BUFFER */
                    554:     map_exec(code_gen_prologue, sizeof(code_gen_prologue));
                    555:     code_gen_buffer_max_size = code_gen_buffer_size - 
1.1.1.13! root      556:         (TCG_MAX_OP_SIZE * OPC_MAX_SIZE);
1.1.1.7   root      557:     code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
                    558:     tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
                    559: }
                    560: 
                    561: /* Must be called before using the QEMU cpus. 'tb_size' is the size
                    562:    (in bytes) allocated to the translation buffer. Zero means default
                    563:    size. */
                    564: void cpu_exec_init_all(unsigned long tb_size)
                    565: {
                    566:     cpu_gen_init();
                    567:     code_gen_alloc(tb_size);
                    568:     code_gen_ptr = code_gen_buffer;
                    569:     page_init();
                    570: #if !defined(CONFIG_USER_ONLY)
                    571:     io_mem_init();
                    572: #endif
1.1.1.13! root      573: #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
        !           574:     /* There's no guest base to take into account, so go ahead and
        !           575:        initialize the prologue now.  */
        !           576:     tcg_prologue_init(&tcg_ctx);
        !           577: #endif
1.1.1.7   root      578: }
                    579: 
                    580: #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
                    581: 
1.1.1.11  root      582: static int cpu_common_post_load(void *opaque, int version_id)
                    583: {
                    584:     CPUState *env = opaque;
1.1.1.7   root      585: 
1.1.1.10  root      586:     /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
                    587:        version_id is increased. */
                    588:     env->interrupt_request &= ~0x01;
1.1.1.7   root      589:     tlb_flush(env, 1);
                    590: 
                    591:     return 0;
                    592: }
1.1.1.11  root      593: 
                    594: static const VMStateDescription vmstate_cpu_common = {
                    595:     .name = "cpu_common",
                    596:     .version_id = 1,
                    597:     .minimum_version_id = 1,
                    598:     .minimum_version_id_old = 1,
                    599:     .post_load = cpu_common_post_load,
                    600:     .fields      = (VMStateField []) {
                    601:         VMSTATE_UINT32(halted, CPUState),
                    602:         VMSTATE_UINT32(interrupt_request, CPUState),
                    603:         VMSTATE_END_OF_LIST()
                    604:     }
                    605: };
1.1       root      606: #endif
                    607: 
1.1.1.10  root      608: CPUState *qemu_get_cpu(int cpu)
                    609: {
                    610:     CPUState *env = first_cpu;
                    611: 
                    612:     while (env) {
                    613:         if (env->cpu_index == cpu)
                    614:             break;
                    615:         env = env->next_cpu;
                    616:     }
                    617: 
                    618:     return env;
                    619: }
                    620: 
1.1.1.2   root      621: void cpu_exec_init(CPUState *env)
1.1       root      622: {
1.1.1.2   root      623:     CPUState **penv;
                    624:     int cpu_index;
1.1       root      625: 
1.1.1.10  root      626: #if defined(CONFIG_USER_ONLY)
                    627:     cpu_list_lock();
                    628: #endif
1.1.1.2   root      629:     env->next_cpu = NULL;
                    630:     penv = &first_cpu;
                    631:     cpu_index = 0;
                    632:     while (*penv != NULL) {
1.1.1.10  root      633:         penv = &(*penv)->next_cpu;
1.1.1.2   root      634:         cpu_index++;
                    635:     }
                    636:     env->cpu_index = cpu_index;
1.1.1.10  root      637:     env->numa_node = 0;
1.1.1.11  root      638:     QTAILQ_INIT(&env->breakpoints);
                    639:     QTAILQ_INIT(&env->watchpoints);
1.1.1.2   root      640:     *penv = env;
1.1.1.10  root      641: #if defined(CONFIG_USER_ONLY)
                    642:     cpu_list_unlock();
                    643: #endif
1.1.1.7   root      644: #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
1.1.1.13! root      645:     vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
        !           646:     register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
1.1.1.7   root      647:                     cpu_save, cpu_load, env);
                    648: #endif
1.1       root      649: }
                    650: 
                    651: static inline void invalidate_page_bitmap(PageDesc *p)
                    652: {
                    653:     if (p->code_bitmap) {
                    654:         qemu_free(p->code_bitmap);
                    655:         p->code_bitmap = NULL;
                    656:     }
                    657:     p->code_write_count = 0;
                    658: }
                    659: 
1.1.1.13! root      660: /* Set to NULL all the 'first_tb' fields in all PageDescs. */
        !           661: 
        !           662: static void page_flush_tb_1 (int level, void **lp)
1.1       root      663: {
1.1.1.13! root      664:     int i;
1.1       root      665: 
1.1.1.13! root      666:     if (*lp == NULL) {
        !           667:         return;
        !           668:     }
        !           669:     if (level == 0) {
        !           670:         PageDesc *pd = *lp;
        !           671:         for (i = 0; i < L2_SIZE; ++i) {
        !           672:             pd[i].first_tb = NULL;
        !           673:             invalidate_page_bitmap(pd + i);
        !           674:         }
        !           675:     } else {
        !           676:         void **pp = *lp;
        !           677:         for (i = 0; i < L2_SIZE; ++i) {
        !           678:             page_flush_tb_1 (level - 1, pp + i);
1.1       root      679:         }
                    680:     }
                    681: }
                    682: 
1.1.1.13! root      683: static void page_flush_tb(void)
        !           684: {
        !           685:     int i;
        !           686:     for (i = 0; i < V_L1_SIZE; i++) {
        !           687:         page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
        !           688:     }
        !           689: }
        !           690: 
1.1       root      691: /* flush all the translation blocks */
                    692: /* XXX: tb_flush is currently not thread safe */
1.1.1.2   root      693: void tb_flush(CPUState *env1)
1.1       root      694: {
1.1.1.2   root      695:     CPUState *env;
1.1       root      696: #if defined(DEBUG_FLUSH)
1.1.1.6   root      697:     printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
                    698:            (unsigned long)(code_gen_ptr - code_gen_buffer),
                    699:            nb_tbs, nb_tbs > 0 ?
                    700:            ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
1.1       root      701: #endif
1.1.1.7   root      702:     if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
                    703:         cpu_abort(env1, "Internal error: code buffer overflow\n");
                    704: 
1.1       root      705:     nb_tbs = 0;
1.1.1.6   root      706: 
1.1.1.2   root      707:     for(env = first_cpu; env != NULL; env = env->next_cpu) {
                    708:         memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
                    709:     }
1.1       root      710: 
                    711:     memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
                    712:     page_flush_tb();
                    713: 
                    714:     code_gen_ptr = code_gen_buffer;
                    715:     /* XXX: flush processor icache at this point if cache flush is
                    716:        expensive */
                    717:     tb_flush_count++;
                    718: }
                    719: 
                    720: #ifdef DEBUG_TB_CHECK
                    721: 
1.1.1.6   root      722: static void tb_invalidate_check(target_ulong address)
1.1       root      723: {
                    724:     TranslationBlock *tb;
                    725:     int i;
                    726:     address &= TARGET_PAGE_MASK;
1.1.1.3   root      727:     for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
                    728:         for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
1.1       root      729:             if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
                    730:                   address >= tb->pc + tb->size)) {
1.1.1.10  root      731:                 printf("ERROR invalidate: address=" TARGET_FMT_lx
                    732:                        " PC=%08lx size=%04x\n",
1.1.1.3   root      733:                        address, (long)tb->pc, tb->size);
1.1       root      734:             }
                    735:         }
                    736:     }
                    737: }
                    738: 
                    739: /* verify that all the pages have correct rights for code */
                    740: static void tb_page_check(void)
                    741: {
                    742:     TranslationBlock *tb;
                    743:     int i, flags1, flags2;
1.1.1.6   root      744: 
1.1.1.3   root      745:     for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
                    746:         for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
1.1       root      747:             flags1 = page_get_flags(tb->pc);
                    748:             flags2 = page_get_flags(tb->pc + tb->size - 1);
                    749:             if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
                    750:                 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
1.1.1.3   root      751:                        (long)tb->pc, tb->size, flags1, flags2);
1.1       root      752:             }
                    753:         }
                    754:     }
                    755: }
                    756: 
                    757: #endif
                    758: 
                    759: /* invalidate one TB */
                    760: static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
                    761:                              int next_offset)
                    762: {
                    763:     TranslationBlock *tb1;
                    764:     for(;;) {
                    765:         tb1 = *ptb;
                    766:         if (tb1 == tb) {
                    767:             *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
                    768:             break;
                    769:         }
                    770:         ptb = (TranslationBlock **)((char *)tb1 + next_offset);
                    771:     }
                    772: }
                    773: 
                    774: static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
                    775: {
                    776:     TranslationBlock *tb1;
                    777:     unsigned int n1;
                    778: 
                    779:     for(;;) {
                    780:         tb1 = *ptb;
                    781:         n1 = (long)tb1 & 3;
                    782:         tb1 = (TranslationBlock *)((long)tb1 & ~3);
                    783:         if (tb1 == tb) {
                    784:             *ptb = tb1->page_next[n1];
                    785:             break;
                    786:         }
                    787:         ptb = &tb1->page_next[n1];
                    788:     }
                    789: }
                    790: 
                    791: static inline void tb_jmp_remove(TranslationBlock *tb, int n)
                    792: {
                    793:     TranslationBlock *tb1, **ptb;
                    794:     unsigned int n1;
                    795: 
                    796:     ptb = &tb->jmp_next[n];
                    797:     tb1 = *ptb;
                    798:     if (tb1) {
                    799:         /* find tb(n) in circular list */
                    800:         for(;;) {
                    801:             tb1 = *ptb;
                    802:             n1 = (long)tb1 & 3;
                    803:             tb1 = (TranslationBlock *)((long)tb1 & ~3);
                    804:             if (n1 == n && tb1 == tb)
                    805:                 break;
                    806:             if (n1 == 2) {
                    807:                 ptb = &tb1->jmp_first;
                    808:             } else {
                    809:                 ptb = &tb1->jmp_next[n1];
                    810:             }
                    811:         }
                    812:         /* now we can suppress tb(n) from the list */
                    813:         *ptb = tb->jmp_next[n];
                    814: 
                    815:         tb->jmp_next[n] = NULL;
                    816:     }
                    817: }
                    818: 
                    819: /* reset the jump entry 'n' of a TB so that it is not chained to
                    820:    another TB */
                    821: static inline void tb_reset_jump(TranslationBlock *tb, int n)
                    822: {
                    823:     tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
                    824: }
                    825: 
1.1.1.13! root      826: void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
1.1       root      827: {
1.1.1.2   root      828:     CPUState *env;
                    829:     PageDesc *p;
1.1       root      830:     unsigned int h, n1;
1.1.1.13! root      831:     tb_page_addr_t phys_pc;
1.1.1.2   root      832:     TranslationBlock *tb1, *tb2;
1.1.1.6   root      833: 
1.1.1.2   root      834:     /* remove the TB from the hash list */
                    835:     phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
                    836:     h = tb_phys_hash_func(phys_pc);
1.1.1.6   root      837:     tb_remove(&tb_phys_hash[h], tb,
1.1.1.2   root      838:               offsetof(TranslationBlock, phys_hash_next));
                    839: 
                    840:     /* remove the TB from the page list */
                    841:     if (tb->page_addr[0] != page_addr) {
                    842:         p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
                    843:         tb_page_remove(&p->first_tb, tb);
                    844:         invalidate_page_bitmap(p);
                    845:     }
                    846:     if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
                    847:         p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
                    848:         tb_page_remove(&p->first_tb, tb);
                    849:         invalidate_page_bitmap(p);
                    850:     }
                    851: 
1.1       root      852:     tb_invalidated_flag = 1;
                    853: 
                    854:     /* remove the TB from the hash list */
1.1.1.2   root      855:     h = tb_jmp_cache_hash_func(tb->pc);
                    856:     for(env = first_cpu; env != NULL; env = env->next_cpu) {
                    857:         if (env->tb_jmp_cache[h] == tb)
                    858:             env->tb_jmp_cache[h] = NULL;
1.1       root      859:     }
                    860: 
                    861:     /* suppress this TB from the two jump lists */
                    862:     tb_jmp_remove(tb, 0);
                    863:     tb_jmp_remove(tb, 1);
                    864: 
                    865:     /* suppress any remaining jumps to this TB */
                    866:     tb1 = tb->jmp_first;
                    867:     for(;;) {
                    868:         n1 = (long)tb1 & 3;
                    869:         if (n1 == 2)
                    870:             break;
                    871:         tb1 = (TranslationBlock *)((long)tb1 & ~3);
                    872:         tb2 = tb1->jmp_next[n1];
                    873:         tb_reset_jump(tb1, n1);
                    874:         tb1->jmp_next[n1] = NULL;
                    875:         tb1 = tb2;
                    876:     }
                    877:     tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
                    878: 
                    879:     tb_phys_invalidate_count++;
                    880: }
                    881: 
                    882: static inline void set_bits(uint8_t *tab, int start, int len)
                    883: {
                    884:     int end, mask, end1;
                    885: 
                    886:     end = start + len;
                    887:     tab += start >> 3;
                    888:     mask = 0xff << (start & 7);
                    889:     if ((start & ~7) == (end & ~7)) {
                    890:         if (start < end) {
                    891:             mask &= ~(0xff << (end & 7));
                    892:             *tab |= mask;
                    893:         }
                    894:     } else {
                    895:         *tab++ |= mask;
                    896:         start = (start + 8) & ~7;
                    897:         end1 = end & ~7;
                    898:         while (start < end1) {
                    899:             *tab++ = 0xff;
                    900:             start += 8;
                    901:         }
                    902:         if (start < end) {
                    903:             mask = ~(0xff << (end & 7));
                    904:             *tab |= mask;
                    905:         }
                    906:     }
                    907: }
                    908: 
                    909: static void build_page_bitmap(PageDesc *p)
                    910: {
                    911:     int n, tb_start, tb_end;
                    912:     TranslationBlock *tb;
1.1.1.6   root      913: 
1.1.1.7   root      914:     p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
1.1       root      915: 
                    916:     tb = p->first_tb;
                    917:     while (tb != NULL) {
                    918:         n = (long)tb & 3;
                    919:         tb = (TranslationBlock *)((long)tb & ~3);
                    920:         /* NOTE: this is subtle as a TB may span two physical pages */
                    921:         if (n == 0) {
                    922:             /* NOTE: tb_end may be after the end of the page, but
                    923:                it is not a problem */
                    924:             tb_start = tb->pc & ~TARGET_PAGE_MASK;
                    925:             tb_end = tb_start + tb->size;
                    926:             if (tb_end > TARGET_PAGE_SIZE)
                    927:                 tb_end = TARGET_PAGE_SIZE;
                    928:         } else {
                    929:             tb_start = 0;
                    930:             tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
                    931:         }
                    932:         set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
                    933:         tb = tb->page_next[n];
                    934:     }
                    935: }
                    936: 
1.1.1.7   root      937: TranslationBlock *tb_gen_code(CPUState *env,
                    938:                               target_ulong pc, target_ulong cs_base,
                    939:                               int flags, int cflags)
1.1       root      940: {
                    941:     TranslationBlock *tb;
                    942:     uint8_t *tc_ptr;
1.1.1.13! root      943:     tb_page_addr_t phys_pc, phys_page2;
        !           944:     target_ulong virt_page2;
1.1       root      945:     int code_gen_size;
                    946: 
1.1.1.13! root      947:     phys_pc = get_page_addr_code(env, pc);
1.1       root      948:     tb = tb_alloc(pc);
                    949:     if (!tb) {
                    950:         /* flush must be done */
                    951:         tb_flush(env);
                    952:         /* cannot fail at this point */
                    953:         tb = tb_alloc(pc);
1.1.1.7   root      954:         /* Don't forget to invalidate previous TB info.  */
                    955:         tb_invalidated_flag = 1;
1.1       root      956:     }
                    957:     tc_ptr = code_gen_ptr;
                    958:     tb->tc_ptr = tc_ptr;
                    959:     tb->cs_base = cs_base;
                    960:     tb->flags = flags;
                    961:     tb->cflags = cflags;
1.1.1.6   root      962:     cpu_gen_code(env, tb, &code_gen_size);
1.1       root      963:     code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
1.1.1.6   root      964: 
1.1       root      965:     /* check next page if needed */
                    966:     virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
                    967:     phys_page2 = -1;
                    968:     if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1.1.1.13! root      969:         phys_page2 = get_page_addr_code(env, virt_page2);
1.1       root      970:     }
1.1.1.13! root      971:     tb_link_page(tb, phys_pc, phys_page2);
1.1.1.7   root      972:     return tb;
1.1       root      973: }
1.1.1.6   root      974: 
1.1       root      975: /* invalidate all TBs which intersect with the target physical page
                    976:    starting in range [start;end[. NOTE: start and end must refer to
                    977:    the same physical page. 'is_cpu_write_access' should be true if called
                    978:    from a real cpu write access: the virtual CPU will exit the current
                    979:    TB if code is modified inside this TB. */
1.1.1.13! root      980: void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1.1       root      981:                                    int is_cpu_write_access)
                    982: {
1.1.1.7   root      983:     TranslationBlock *tb, *tb_next, *saved_tb;
1.1       root      984:     CPUState *env = cpu_single_env;
1.1.1.13! root      985:     tb_page_addr_t tb_start, tb_end;
1.1.1.7   root      986:     PageDesc *p;
                    987:     int n;
                    988: #ifdef TARGET_HAS_PRECISE_SMC
                    989:     int current_tb_not_found = is_cpu_write_access;
                    990:     TranslationBlock *current_tb = NULL;
                    991:     int current_tb_modified = 0;
                    992:     target_ulong current_pc = 0;
                    993:     target_ulong current_cs_base = 0;
                    994:     int current_flags = 0;
                    995: #endif /* TARGET_HAS_PRECISE_SMC */
1.1       root      996: 
                    997:     p = page_find(start >> TARGET_PAGE_BITS);
1.1.1.6   root      998:     if (!p)
1.1       root      999:         return;
1.1.1.6   root     1000:     if (!p->code_bitmap &&
1.1       root     1001:         ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
                   1002:         is_cpu_write_access) {
                   1003:         /* build code bitmap */
                   1004:         build_page_bitmap(p);
                   1005:     }
                   1006: 
                   1007:     /* we remove all the TBs in the range [start, end[ */
                   1008:     /* XXX: see if in some cases it could be faster to invalidate all the code */
                   1009:     tb = p->first_tb;
                   1010:     while (tb != NULL) {
                   1011:         n = (long)tb & 3;
                   1012:         tb = (TranslationBlock *)((long)tb & ~3);
                   1013:         tb_next = tb->page_next[n];
                   1014:         /* NOTE: this is subtle as a TB may span two physical pages */
                   1015:         if (n == 0) {
                   1016:             /* NOTE: tb_end may be after the end of the page, but
                   1017:                it is not a problem */
                   1018:             tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
                   1019:             tb_end = tb_start + tb->size;
                   1020:         } else {
                   1021:             tb_start = tb->page_addr[1];
                   1022:             tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
                   1023:         }
                   1024:         if (!(tb_end <= start || tb_start >= end)) {
                   1025: #ifdef TARGET_HAS_PRECISE_SMC
                   1026:             if (current_tb_not_found) {
                   1027:                 current_tb_not_found = 0;
                   1028:                 current_tb = NULL;
1.1.1.7   root     1029:                 if (env->mem_io_pc) {
1.1       root     1030:                     /* now we have a real cpu fault */
1.1.1.7   root     1031:                     current_tb = tb_find_pc(env->mem_io_pc);
1.1       root     1032:                 }
                   1033:             }
                   1034:             if (current_tb == tb &&
1.1.1.7   root     1035:                 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1.1       root     1036:                 /* If we are modifying the current TB, we must stop
                   1037:                 its execution. We could be more precise by checking
                   1038:                 that the modification is after the current PC, but it
                   1039:                 would require a specialized function to partially
                   1040:                 restore the CPU state */
1.1.1.6   root     1041: 
1.1       root     1042:                 current_tb_modified = 1;
1.1.1.6   root     1043:                 cpu_restore_state(current_tb, env,
1.1.1.7   root     1044:                                   env->mem_io_pc, NULL);
                   1045:                 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
                   1046:                                      &current_flags);
1.1       root     1047:             }
                   1048: #endif /* TARGET_HAS_PRECISE_SMC */
1.1.1.2   root     1049:             /* we need to do that to handle the case where a signal
                   1050:                occurs while doing tb_phys_invalidate() */
                   1051:             saved_tb = NULL;
                   1052:             if (env) {
                   1053:                 saved_tb = env->current_tb;
                   1054:                 env->current_tb = NULL;
                   1055:             }
1.1       root     1056:             tb_phys_invalidate(tb, -1);
1.1.1.2   root     1057:             if (env) {
                   1058:                 env->current_tb = saved_tb;
                   1059:                 if (env->interrupt_request && env->current_tb)
                   1060:                     cpu_interrupt(env, env->interrupt_request);
                   1061:             }
1.1       root     1062:         }
                   1063:         tb = tb_next;
                   1064:     }
                   1065: #if !defined(CONFIG_USER_ONLY)
                   1066:     /* if no code remaining, no need to continue to use slow writes */
                   1067:     if (!p->first_tb) {
                   1068:         invalidate_page_bitmap(p);
                   1069:         if (is_cpu_write_access) {
1.1.1.7   root     1070:             tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1.1       root     1071:         }
                   1072:     }
                   1073: #endif
                   1074: #ifdef TARGET_HAS_PRECISE_SMC
                   1075:     if (current_tb_modified) {
                   1076:         /* we generate a block containing just the instruction
                   1077:            modifying the memory. It will ensure that it cannot modify
                   1078:            itself */
                   1079:         env->current_tb = NULL;
1.1.1.7   root     1080:         tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1.1       root     1081:         cpu_resume_from_signal(env, NULL);
                   1082:     }
                   1083: #endif
                   1084: }
                   1085: 
                   1086: /* len must be <= 8 and start must be a multiple of len */
1.1.1.13! root     1087: static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1.1       root     1088: {
                   1089:     PageDesc *p;
                   1090:     int offset, b;
                   1091: #if 0
                   1092:     if (1) {
1.1.1.7   root     1093:         qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
                   1094:                   cpu_single_env->mem_io_vaddr, len,
                   1095:                   cpu_single_env->eip,
                   1096:                   cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1.1       root     1097:     }
                   1098: #endif
                   1099:     p = page_find(start >> TARGET_PAGE_BITS);
1.1.1.6   root     1100:     if (!p)
1.1       root     1101:         return;
                   1102:     if (p->code_bitmap) {
                   1103:         offset = start & ~TARGET_PAGE_MASK;
                   1104:         b = p->code_bitmap[offset >> 3] >> (offset & 7);
                   1105:         if (b & ((1 << len) - 1))
                   1106:             goto do_invalidate;
                   1107:     } else {
                   1108:     do_invalidate:
                   1109:         tb_invalidate_phys_page_range(start, start + len, 1);
                   1110:     }
                   1111: }
                   1112: 
                   1113: #if !defined(CONFIG_SOFTMMU)
1.1.1.13! root     1114: static void tb_invalidate_phys_page(tb_page_addr_t addr,
1.1       root     1115:                                     unsigned long pc, void *puc)
                   1116: {
1.1.1.7   root     1117:     TranslationBlock *tb;
1.1       root     1118:     PageDesc *p;
1.1.1.7   root     1119:     int n;
1.1       root     1120: #ifdef TARGET_HAS_PRECISE_SMC
1.1.1.7   root     1121:     TranslationBlock *current_tb = NULL;
1.1       root     1122:     CPUState *env = cpu_single_env;
1.1.1.7   root     1123:     int current_tb_modified = 0;
                   1124:     target_ulong current_pc = 0;
                   1125:     target_ulong current_cs_base = 0;
                   1126:     int current_flags = 0;
1.1       root     1127: #endif
                   1128: 
                   1129:     addr &= TARGET_PAGE_MASK;
                   1130:     p = page_find(addr >> TARGET_PAGE_BITS);
1.1.1.6   root     1131:     if (!p)
1.1       root     1132:         return;
                   1133:     tb = p->first_tb;
                   1134: #ifdef TARGET_HAS_PRECISE_SMC
                   1135:     if (tb && pc != 0) {
                   1136:         current_tb = tb_find_pc(pc);
                   1137:     }
                   1138: #endif
                   1139:     while (tb != NULL) {
                   1140:         n = (long)tb & 3;
                   1141:         tb = (TranslationBlock *)((long)tb & ~3);
                   1142: #ifdef TARGET_HAS_PRECISE_SMC
                   1143:         if (current_tb == tb &&
1.1.1.7   root     1144:             (current_tb->cflags & CF_COUNT_MASK) != 1) {
1.1       root     1145:                 /* If we are modifying the current TB, we must stop
                   1146:                    its execution. We could be more precise by checking
                   1147:                    that the modification is after the current PC, but it
                   1148:                    would require a specialized function to partially
                   1149:                    restore the CPU state */
1.1.1.6   root     1150: 
1.1       root     1151:             current_tb_modified = 1;
                   1152:             cpu_restore_state(current_tb, env, pc, puc);
1.1.1.7   root     1153:             cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
                   1154:                                  &current_flags);
1.1       root     1155:         }
                   1156: #endif /* TARGET_HAS_PRECISE_SMC */
                   1157:         tb_phys_invalidate(tb, addr);
                   1158:         tb = tb->page_next[n];
                   1159:     }
                   1160:     p->first_tb = NULL;
                   1161: #ifdef TARGET_HAS_PRECISE_SMC
                   1162:     if (current_tb_modified) {
                   1163:         /* we generate a block containing just the instruction
                   1164:            modifying the memory. It will ensure that it cannot modify
                   1165:            itself */
                   1166:         env->current_tb = NULL;
1.1.1.7   root     1167:         tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1.1       root     1168:         cpu_resume_from_signal(env, puc);
                   1169:     }
                   1170: #endif
                   1171: }
                   1172: #endif
                   1173: 
                   1174: /* add the tb in the target page and protect it if necessary */
1.1.1.6   root     1175: static inline void tb_alloc_page(TranslationBlock *tb,
1.1.1.13! root     1176:                                  unsigned int n, tb_page_addr_t page_addr)
1.1       root     1177: {
                   1178:     PageDesc *p;
                   1179:     TranslationBlock *last_first_tb;
                   1180: 
                   1181:     tb->page_addr[n] = page_addr;
1.1.1.13! root     1182:     p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1.1       root     1183:     tb->page_next[n] = p->first_tb;
                   1184:     last_first_tb = p->first_tb;
                   1185:     p->first_tb = (TranslationBlock *)((long)tb | n);
                   1186:     invalidate_page_bitmap(p);
                   1187: 
                   1188: #if defined(TARGET_HAS_SMC) || 1
                   1189: 
                   1190: #if defined(CONFIG_USER_ONLY)
                   1191:     if (p->flags & PAGE_WRITE) {
1.1.1.3   root     1192:         target_ulong addr;
                   1193:         PageDesc *p2;
1.1       root     1194:         int prot;
                   1195: 
                   1196:         /* force the host page as non writable (writes will have a
                   1197:            page fault + mprotect overhead) */
1.1.1.3   root     1198:         page_addr &= qemu_host_page_mask;
1.1       root     1199:         prot = 0;
1.1.1.3   root     1200:         for(addr = page_addr; addr < page_addr + qemu_host_page_size;
                   1201:             addr += TARGET_PAGE_SIZE) {
                   1202: 
                   1203:             p2 = page_find (addr >> TARGET_PAGE_BITS);
                   1204:             if (!p2)
                   1205:                 continue;
                   1206:             prot |= p2->flags;
                   1207:             p2->flags &= ~PAGE_WRITE;
                   1208:           }
1.1.1.6   root     1209:         mprotect(g2h(page_addr), qemu_host_page_size,
1.1       root     1210:                  (prot & PAGE_BITS) & ~PAGE_WRITE);
                   1211: #ifdef DEBUG_TB_INVALIDATE
1.1.1.6   root     1212:         printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1.1.1.3   root     1213:                page_addr);
1.1       root     1214: #endif
                   1215:     }
                   1216: #else
                   1217:     /* if some code is already present, then the pages are already
                   1218:        protected. So we handle the case where only the first TB is
                   1219:        allocated in a physical page */
                   1220:     if (!last_first_tb) {
1.1.1.2   root     1221:         tlb_protect_code(page_addr);
1.1       root     1222:     }
                   1223: #endif
                   1224: 
                   1225: #endif /* TARGET_HAS_SMC */
                   1226: }
                   1227: 
                   1228: /* Allocate a new translation block. Flush the translation buffer if
                   1229:    too many translation blocks or too much generated code. */
                   1230: TranslationBlock *tb_alloc(target_ulong pc)
                   1231: {
                   1232:     TranslationBlock *tb;
                   1233: 
1.1.1.7   root     1234:     if (nb_tbs >= code_gen_max_blocks ||
                   1235:         (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1.1       root     1236:         return NULL;
                   1237:     tb = &tbs[nb_tbs++];
                   1238:     tb->pc = pc;
                   1239:     tb->cflags = 0;
                   1240:     return tb;
                   1241: }
                   1242: 
1.1.1.7   root     1243: void tb_free(TranslationBlock *tb)
                   1244: {
                   1245:     /* In practice this is mostly used for single use temporary TB
                   1246:        Ignore the hard cases and just back up if this TB happens to
                   1247:        be the last one generated.  */
                   1248:     if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
                   1249:         code_gen_ptr = tb->tc_ptr;
                   1250:         nb_tbs--;
                   1251:     }
                   1252: }
                   1253: 
1.1       root     1254: /* add a new TB and link it to the physical page tables. phys_page2 is
                   1255:    (-1) to indicate that only one page contains the TB. */
1.1.1.13! root     1256: void tb_link_page(TranslationBlock *tb,
        !          1257:                   tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
1.1       root     1258: {
                   1259:     unsigned int h;
                   1260:     TranslationBlock **ptb;
                   1261: 
1.1.1.7   root     1262:     /* Grab the mmap lock to stop another thread invalidating this TB
                   1263:        before we are done.  */
                   1264:     mmap_lock();
1.1       root     1265:     /* add in the physical hash table */
                   1266:     h = tb_phys_hash_func(phys_pc);
                   1267:     ptb = &tb_phys_hash[h];
                   1268:     tb->phys_hash_next = *ptb;
                   1269:     *ptb = tb;
                   1270: 
                   1271:     /* add in the page list */
                   1272:     tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
                   1273:     if (phys_page2 != -1)
                   1274:         tb_alloc_page(tb, 1, phys_page2);
                   1275:     else
                   1276:         tb->page_addr[1] = -1;
                   1277: 
                   1278:     tb->jmp_first = (TranslationBlock *)((long)tb | 2);
                   1279:     tb->jmp_next[0] = NULL;
                   1280:     tb->jmp_next[1] = NULL;
                   1281: 
                   1282:     /* init original jump addresses */
                   1283:     if (tb->tb_next_offset[0] != 0xffff)
                   1284:         tb_reset_jump(tb, 0);
                   1285:     if (tb->tb_next_offset[1] != 0xffff)
                   1286:         tb_reset_jump(tb, 1);
1.1.1.2   root     1287: 
                   1288: #ifdef DEBUG_TB_CHECK
                   1289:     tb_page_check();
                   1290: #endif
1.1.1.7   root     1291:     mmap_unlock();
1.1       root     1292: }
                   1293: 
                   1294: /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
                   1295:    tb[1].tc_ptr. Return NULL if not found */
                   1296: TranslationBlock *tb_find_pc(unsigned long tc_ptr)
                   1297: {
                   1298:     int m_min, m_max, m;
                   1299:     unsigned long v;
                   1300:     TranslationBlock *tb;
                   1301: 
                   1302:     if (nb_tbs <= 0)
                   1303:         return NULL;
                   1304:     if (tc_ptr < (unsigned long)code_gen_buffer ||
                   1305:         tc_ptr >= (unsigned long)code_gen_ptr)
                   1306:         return NULL;
                   1307:     /* binary search (cf Knuth) */
                   1308:     m_min = 0;
                   1309:     m_max = nb_tbs - 1;
                   1310:     while (m_min <= m_max) {
                   1311:         m = (m_min + m_max) >> 1;
                   1312:         tb = &tbs[m];
                   1313:         v = (unsigned long)tb->tc_ptr;
                   1314:         if (v == tc_ptr)
                   1315:             return tb;
                   1316:         else if (tc_ptr < v) {
                   1317:             m_max = m - 1;
                   1318:         } else {
                   1319:             m_min = m + 1;
                   1320:         }
1.1.1.6   root     1321:     }
1.1       root     1322:     return &tbs[m_max];
                   1323: }
                   1324: 
                   1325: static void tb_reset_jump_recursive(TranslationBlock *tb);
                   1326: 
                   1327: static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
                   1328: {
                   1329:     TranslationBlock *tb1, *tb_next, **ptb;
                   1330:     unsigned int n1;
                   1331: 
                   1332:     tb1 = tb->jmp_next[n];
                   1333:     if (tb1 != NULL) {
                   1334:         /* find head of list */
                   1335:         for(;;) {
                   1336:             n1 = (long)tb1 & 3;
                   1337:             tb1 = (TranslationBlock *)((long)tb1 & ~3);
                   1338:             if (n1 == 2)
                   1339:                 break;
                   1340:             tb1 = tb1->jmp_next[n1];
                   1341:         }
                   1342:         /* we are now sure now that tb jumps to tb1 */
                   1343:         tb_next = tb1;
                   1344: 
                   1345:         /* remove tb from the jmp_first list */
                   1346:         ptb = &tb_next->jmp_first;
                   1347:         for(;;) {
                   1348:             tb1 = *ptb;
                   1349:             n1 = (long)tb1 & 3;
                   1350:             tb1 = (TranslationBlock *)((long)tb1 & ~3);
                   1351:             if (n1 == n && tb1 == tb)
                   1352:                 break;
                   1353:             ptb = &tb1->jmp_next[n1];
                   1354:         }
                   1355:         *ptb = tb->jmp_next[n];
                   1356:         tb->jmp_next[n] = NULL;
1.1.1.6   root     1357: 
1.1       root     1358:         /* suppress the jump to next tb in generated code */
                   1359:         tb_reset_jump(tb, n);
                   1360: 
                   1361:         /* suppress jumps in the tb on which we could have jumped */
                   1362:         tb_reset_jump_recursive(tb_next);
                   1363:     }
                   1364: }
                   1365: 
                   1366: static void tb_reset_jump_recursive(TranslationBlock *tb)
                   1367: {
                   1368:     tb_reset_jump_recursive2(tb, 0);
                   1369:     tb_reset_jump_recursive2(tb, 1);
                   1370: }
                   1371: 
                   1372: #if defined(TARGET_HAS_ICE)
1.1.1.13! root     1373: #if defined(CONFIG_USER_ONLY)
        !          1374: static void breakpoint_invalidate(CPUState *env, target_ulong pc)
        !          1375: {
        !          1376:     tb_invalidate_phys_page_range(pc, pc + 1, 0);
        !          1377: }
        !          1378: #else
1.1       root     1379: static void breakpoint_invalidate(CPUState *env, target_ulong pc)
                   1380: {
1.1.1.6   root     1381:     target_phys_addr_t addr;
                   1382:     target_ulong pd;
1.1.1.3   root     1383:     ram_addr_t ram_addr;
                   1384:     PhysPageDesc *p;
1.1       root     1385: 
1.1.1.3   root     1386:     addr = cpu_get_phys_page_debug(env, pc);
                   1387:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   1388:     if (!p) {
                   1389:         pd = IO_MEM_UNASSIGNED;
                   1390:     } else {
                   1391:         pd = p->phys_offset;
                   1392:     }
                   1393:     ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
                   1394:     tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1.1       root     1395: }
                   1396: #endif
1.1.1.13! root     1397: #endif /* TARGET_HAS_ICE */
        !          1398: 
        !          1399: #if defined(CONFIG_USER_ONLY)
        !          1400: void cpu_watchpoint_remove_all(CPUState *env, int mask)
        !          1401: 
        !          1402: {
        !          1403: }
1.1       root     1404: 
1.1.1.13! root     1405: int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
        !          1406:                           int flags, CPUWatchpoint **watchpoint)
        !          1407: {
        !          1408:     return -ENOSYS;
        !          1409: }
        !          1410: #else
1.1.1.6   root     1411: /* Add a watchpoint.  */
1.1.1.7   root     1412: int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
                   1413:                           int flags, CPUWatchpoint **watchpoint)
1.1.1.6   root     1414: {
1.1.1.7   root     1415:     target_ulong len_mask = ~(len - 1);
                   1416:     CPUWatchpoint *wp;
                   1417: 
                   1418:     /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
                   1419:     if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
                   1420:         fprintf(stderr, "qemu: tried to set invalid watchpoint at "
                   1421:                 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
                   1422:         return -EINVAL;
                   1423:     }
                   1424:     wp = qemu_malloc(sizeof(*wp));
                   1425: 
                   1426:     wp->vaddr = addr;
                   1427:     wp->len_mask = len_mask;
                   1428:     wp->flags = flags;
                   1429: 
                   1430:     /* keep all GDB-injected watchpoints in front */
                   1431:     if (flags & BP_GDB)
1.1.1.11  root     1432:         QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1.1.1.7   root     1433:     else
1.1.1.11  root     1434:         QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1.1.1.7   root     1435: 
                   1436:     tlb_flush_page(env, addr);
                   1437: 
                   1438:     if (watchpoint)
                   1439:         *watchpoint = wp;
                   1440:     return 0;
                   1441: }
1.1.1.6   root     1442: 
1.1.1.7   root     1443: /* Remove a specific watchpoint.  */
                   1444: int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
                   1445:                           int flags)
                   1446: {
                   1447:     target_ulong len_mask = ~(len - 1);
                   1448:     CPUWatchpoint *wp;
                   1449: 
1.1.1.11  root     1450:     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1.1.1.7   root     1451:         if (addr == wp->vaddr && len_mask == wp->len_mask
                   1452:                 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
                   1453:             cpu_watchpoint_remove_by_ref(env, wp);
1.1.1.6   root     1454:             return 0;
1.1.1.7   root     1455:         }
1.1.1.6   root     1456:     }
1.1.1.7   root     1457:     return -ENOENT;
                   1458: }
1.1.1.6   root     1459: 
1.1.1.7   root     1460: /* Remove a specific watchpoint by reference.  */
                   1461: void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
                   1462: {
1.1.1.11  root     1463:     QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1.1.1.7   root     1464: 
                   1465:     tlb_flush_page(env, watchpoint->vaddr);
                   1466: 
                   1467:     qemu_free(watchpoint);
1.1.1.6   root     1468: }
                   1469: 
1.1.1.7   root     1470: /* Remove all matching watchpoints.  */
                   1471: void cpu_watchpoint_remove_all(CPUState *env, int mask)
1.1.1.6   root     1472: {
1.1.1.7   root     1473:     CPUWatchpoint *wp, *next;
1.1.1.6   root     1474: 
1.1.1.11  root     1475:     QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1.1.1.7   root     1476:         if (wp->flags & mask)
                   1477:             cpu_watchpoint_remove_by_ref(env, wp);
1.1.1.6   root     1478:     }
                   1479: }
1.1.1.13! root     1480: #endif
1.1.1.6   root     1481: 
1.1.1.7   root     1482: /* Add a breakpoint.  */
                   1483: int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
                   1484:                           CPUBreakpoint **breakpoint)
1.1       root     1485: {
                   1486: #if defined(TARGET_HAS_ICE)
1.1.1.7   root     1487:     CPUBreakpoint *bp;
1.1.1.6   root     1488: 
1.1.1.7   root     1489:     bp = qemu_malloc(sizeof(*bp));
1.1       root     1490: 
1.1.1.7   root     1491:     bp->pc = pc;
                   1492:     bp->flags = flags;
                   1493: 
                   1494:     /* keep all GDB-injected breakpoints in front */
                   1495:     if (flags & BP_GDB)
1.1.1.11  root     1496:         QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1.1.1.7   root     1497:     else
1.1.1.11  root     1498:         QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1.1.1.6   root     1499: 
1.1       root     1500:     breakpoint_invalidate(env, pc);
1.1.1.7   root     1501: 
                   1502:     if (breakpoint)
                   1503:         *breakpoint = bp;
1.1       root     1504:     return 0;
                   1505: #else
1.1.1.7   root     1506:     return -ENOSYS;
1.1       root     1507: #endif
                   1508: }
                   1509: 
1.1.1.7   root     1510: /* Remove a specific breakpoint.  */
                   1511: int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1.1       root     1512: {
                   1513: #if defined(TARGET_HAS_ICE)
1.1.1.7   root     1514:     CPUBreakpoint *bp;
1.1       root     1515: 
1.1.1.11  root     1516:     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1.1.1.7   root     1517:         if (bp->pc == pc && bp->flags == flags) {
                   1518:             cpu_breakpoint_remove_by_ref(env, bp);
                   1519:             return 0;
                   1520:         }
                   1521:     }
                   1522:     return -ENOENT;
1.1       root     1523: #else
1.1.1.7   root     1524:     return -ENOSYS;
                   1525: #endif
                   1526: }
                   1527: 
                   1528: /* Remove a specific breakpoint by reference.  */
                   1529: void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
                   1530: {
                   1531: #if defined(TARGET_HAS_ICE)
1.1.1.11  root     1532:     QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1.1.1.7   root     1533: 
                   1534:     breakpoint_invalidate(env, breakpoint->pc);
                   1535: 
                   1536:     qemu_free(breakpoint);
                   1537: #endif
                   1538: }
                   1539: 
                   1540: /* Remove all matching breakpoints. */
                   1541: void cpu_breakpoint_remove_all(CPUState *env, int mask)
                   1542: {
                   1543: #if defined(TARGET_HAS_ICE)
                   1544:     CPUBreakpoint *bp, *next;
                   1545: 
1.1.1.11  root     1546:     QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1.1.1.7   root     1547:         if (bp->flags & mask)
                   1548:             cpu_breakpoint_remove_by_ref(env, bp);
                   1549:     }
1.1       root     1550: #endif
                   1551: }
                   1552: 
                   1553: /* enable or disable single step mode. EXCP_DEBUG is returned by the
                   1554:    CPU loop after each instruction */
                   1555: void cpu_single_step(CPUState *env, int enabled)
                   1556: {
                   1557: #if defined(TARGET_HAS_ICE)
                   1558:     if (env->singlestep_enabled != enabled) {
                   1559:         env->singlestep_enabled = enabled;
1.1.1.10  root     1560:         if (kvm_enabled())
                   1561:             kvm_update_guest_debug(env, 0);
                   1562:         else {
                   1563:             /* must flush all the translated code to avoid inconsistencies */
                   1564:             /* XXX: only flush what is necessary */
                   1565:             tb_flush(env);
                   1566:         }
1.1       root     1567:     }
                   1568: #endif
                   1569: }
                   1570: 
                   1571: /* enable or disable low levels log */
                   1572: void cpu_set_log(int log_flags)
                   1573: {
                   1574:     loglevel = log_flags;
                   1575:     if (loglevel && !logfile) {
1.1.1.6   root     1576:         logfile = fopen(logfilename, log_append ? "a" : "w");
1.1       root     1577:         if (!logfile) {
                   1578:             perror(logfilename);
                   1579:             _exit(1);
                   1580:         }
                   1581: #if !defined(CONFIG_SOFTMMU)
                   1582:         /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
                   1583:         {
1.1.1.7   root     1584:             static char logfile_buf[4096];
1.1       root     1585:             setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
                   1586:         }
1.1.1.11  root     1587: #elif !defined(_WIN32)
                   1588:         /* Win32 doesn't support line-buffering and requires size >= 2 */
1.1       root     1589:         setvbuf(logfile, NULL, _IOLBF, 0);
                   1590: #endif
1.1.1.6   root     1591:         log_append = 1;
                   1592:     }
                   1593:     if (!loglevel && logfile) {
                   1594:         fclose(logfile);
                   1595:         logfile = NULL;
1.1       root     1596:     }
                   1597: }
                   1598: 
                   1599: void cpu_set_log_filename(const char *filename)
                   1600: {
                   1601:     logfilename = strdup(filename);
1.1.1.6   root     1602:     if (logfile) {
                   1603:         fclose(logfile);
                   1604:         logfile = NULL;
                   1605:     }
                   1606:     cpu_set_log(loglevel);
1.1       root     1607: }
                   1608: 
1.1.1.10  root     1609: static void cpu_unlink_tb(CPUState *env)
1.1       root     1610: {
1.1.1.10  root     1611:     /* FIXME: TB unchaining isn't SMP safe.  For now just ignore the
                   1612:        problem and hope the cpu will stop of its own accord.  For userspace
                   1613:        emulation this often isn't actually as bad as it sounds.  Often
                   1614:        signals are used primarily to interrupt blocking syscalls.  */
1.1       root     1615:     TranslationBlock *tb;
1.1.1.7   root     1616:     static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1.1       root     1617: 
1.1.1.13! root     1618:     spin_lock(&interrupt_lock);
1.1.1.10  root     1619:     tb = env->current_tb;
                   1620:     /* if the cpu is currently executing code, we must unlink it and
                   1621:        all the potentially executing TB */
1.1.1.13! root     1622:     if (tb) {
1.1.1.10  root     1623:         env->current_tb = NULL;
                   1624:         tb_reset_jump_recursive(tb);
1.1.1.8   root     1625:     }
1.1.1.13! root     1626:     spin_unlock(&interrupt_lock);
1.1.1.10  root     1627: }
                   1628: 
                   1629: /* mask must never be zero, except for A20 change call */
                   1630: void cpu_interrupt(CPUState *env, int mask)
                   1631: {
                   1632:     int old_mask;
1.1.1.8   root     1633: 
1.1.1.7   root     1634:     old_mask = env->interrupt_request;
1.1       root     1635:     env->interrupt_request |= mask;
1.1.1.10  root     1636: 
                   1637: #ifndef CONFIG_USER_ONLY
                   1638:     /*
                   1639:      * If called from iothread context, wake the target cpu in
                   1640:      * case its halted.
                   1641:      */
                   1642:     if (!qemu_cpu_self(env)) {
                   1643:         qemu_cpu_kick(env);
                   1644:         return;
                   1645:     }
                   1646: #endif
                   1647: 
1.1.1.7   root     1648:     if (use_icount) {
                   1649:         env->icount_decr.u16.high = 0xffff;
                   1650: #ifndef CONFIG_USER_ONLY
                   1651:         if (!can_do_io(env)
1.1.1.8   root     1652:             && (mask & ~old_mask) != 0) {
1.1.1.7   root     1653:             cpu_abort(env, "Raised interrupt while not in I/O function");
                   1654:         }
                   1655: #endif
                   1656:     } else {
1.1.1.10  root     1657:         cpu_unlink_tb(env);
1.1       root     1658:     }
                   1659: }
                   1660: 
                   1661: void cpu_reset_interrupt(CPUState *env, int mask)
                   1662: {
                   1663:     env->interrupt_request &= ~mask;
                   1664: }
                   1665: 
1.1.1.10  root     1666: void cpu_exit(CPUState *env)
                   1667: {
                   1668:     env->exit_request = 1;
                   1669:     cpu_unlink_tb(env);
                   1670: }
                   1671: 
1.1.1.7   root     1672: const CPULogItem cpu_log_items[] = {
1.1.1.6   root     1673:     { CPU_LOG_TB_OUT_ASM, "out_asm",
1.1       root     1674:       "show generated host assembly code for each compiled TB" },
                   1675:     { CPU_LOG_TB_IN_ASM, "in_asm",
                   1676:       "show target assembly code for each compiled TB" },
1.1.1.6   root     1677:     { CPU_LOG_TB_OP, "op",
1.1.1.7   root     1678:       "show micro ops for each compiled TB" },
1.1       root     1679:     { CPU_LOG_TB_OP_OPT, "op_opt",
1.1.1.7   root     1680:       "show micro ops "
                   1681: #ifdef TARGET_I386
                   1682:       "before eflags optimization and "
1.1       root     1683: #endif
1.1.1.7   root     1684:       "after liveness analysis" },
1.1       root     1685:     { CPU_LOG_INT, "int",
                   1686:       "show interrupts/exceptions in short format" },
                   1687:     { CPU_LOG_EXEC, "exec",
                   1688:       "show trace before each executed TB (lots of logs)" },
                   1689:     { CPU_LOG_TB_CPU, "cpu",
1.1.1.6   root     1690:       "show CPU state before block translation" },
1.1       root     1691: #ifdef TARGET_I386
                   1692:     { CPU_LOG_PCALL, "pcall",
                   1693:       "show protected mode far calls/returns/exceptions" },
1.1.1.7   root     1694:     { CPU_LOG_RESET, "cpu_reset",
                   1695:       "show CPU state before CPU resets" },
1.1       root     1696: #endif
                   1697: #ifdef DEBUG_IOPORT
                   1698:     { CPU_LOG_IOPORT, "ioport",
                   1699:       "show all i/o ports accesses" },
                   1700: #endif
                   1701:     { 0, NULL, NULL },
                   1702: };
                   1703: 
1.1.1.13! root     1704: #ifndef CONFIG_USER_ONLY
        !          1705: static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
        !          1706:     = QLIST_HEAD_INITIALIZER(memory_client_list);
        !          1707: 
        !          1708: static void cpu_notify_set_memory(target_phys_addr_t start_addr,
        !          1709:                                  ram_addr_t size,
        !          1710:                                  ram_addr_t phys_offset)
        !          1711: {
        !          1712:     CPUPhysMemoryClient *client;
        !          1713:     QLIST_FOREACH(client, &memory_client_list, list) {
        !          1714:         client->set_memory(client, start_addr, size, phys_offset);
        !          1715:     }
        !          1716: }
        !          1717: 
        !          1718: static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
        !          1719:                                        target_phys_addr_t end)
        !          1720: {
        !          1721:     CPUPhysMemoryClient *client;
        !          1722:     QLIST_FOREACH(client, &memory_client_list, list) {
        !          1723:         int r = client->sync_dirty_bitmap(client, start, end);
        !          1724:         if (r < 0)
        !          1725:             return r;
        !          1726:     }
        !          1727:     return 0;
        !          1728: }
        !          1729: 
        !          1730: static int cpu_notify_migration_log(int enable)
        !          1731: {
        !          1732:     CPUPhysMemoryClient *client;
        !          1733:     QLIST_FOREACH(client, &memory_client_list, list) {
        !          1734:         int r = client->migration_log(client, enable);
        !          1735:         if (r < 0)
        !          1736:             return r;
        !          1737:     }
        !          1738:     return 0;
        !          1739: }
        !          1740: 
        !          1741: static void phys_page_for_each_1(CPUPhysMemoryClient *client,
        !          1742:                                  int level, void **lp)
        !          1743: {
        !          1744:     int i;
        !          1745: 
        !          1746:     if (*lp == NULL) {
        !          1747:         return;
        !          1748:     }
        !          1749:     if (level == 0) {
        !          1750:         PhysPageDesc *pd = *lp;
        !          1751:         for (i = 0; i < L2_SIZE; ++i) {
        !          1752:             if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
        !          1753:                 client->set_memory(client, pd[i].region_offset,
        !          1754:                                    TARGET_PAGE_SIZE, pd[i].phys_offset);
        !          1755:             }
        !          1756:         }
        !          1757:     } else {
        !          1758:         void **pp = *lp;
        !          1759:         for (i = 0; i < L2_SIZE; ++i) {
        !          1760:             phys_page_for_each_1(client, level - 1, pp + i);
        !          1761:         }
        !          1762:     }
        !          1763: }
        !          1764: 
        !          1765: static void phys_page_for_each(CPUPhysMemoryClient *client)
        !          1766: {
        !          1767:     int i;
        !          1768:     for (i = 0; i < P_L1_SIZE; ++i) {
        !          1769:         phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
        !          1770:                              l1_phys_map + 1);
        !          1771:     }
        !          1772: }
        !          1773: 
        !          1774: void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
        !          1775: {
        !          1776:     QLIST_INSERT_HEAD(&memory_client_list, client, list);
        !          1777:     phys_page_for_each(client);
        !          1778: }
        !          1779: 
        !          1780: void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
        !          1781: {
        !          1782:     QLIST_REMOVE(client, list);
        !          1783: }
        !          1784: #endif
        !          1785: 
1.1       root     1786: static int cmp1(const char *s1, int n, const char *s2)
                   1787: {
                   1788:     if (strlen(s2) != n)
                   1789:         return 0;
                   1790:     return memcmp(s1, s2, n) == 0;
                   1791: }
1.1.1.6   root     1792: 
1.1       root     1793: /* takes a comma separated list of log masks. Return 0 if error. */
                   1794: int cpu_str_to_log_mask(const char *str)
                   1795: {
1.1.1.7   root     1796:     const CPULogItem *item;
1.1       root     1797:     int mask;
                   1798:     const char *p, *p1;
                   1799: 
                   1800:     p = str;
                   1801:     mask = 0;
                   1802:     for(;;) {
                   1803:         p1 = strchr(p, ',');
                   1804:         if (!p1)
                   1805:             p1 = p + strlen(p);
                   1806:        if(cmp1(p,p1-p,"all")) {
                   1807:                for(item = cpu_log_items; item->mask != 0; item++) {
                   1808:                        mask |= item->mask;
                   1809:                }
                   1810:        } else {
                   1811:         for(item = cpu_log_items; item->mask != 0; item++) {
                   1812:             if (cmp1(p, p1 - p, item->name))
                   1813:                 goto found;
                   1814:         }
                   1815:         return 0;
                   1816:        }
                   1817:     found:
                   1818:         mask |= item->mask;
                   1819:         if (*p1 != ',')
                   1820:             break;
                   1821:         p = p1 + 1;
                   1822:     }
                   1823:     return mask;
                   1824: }
                   1825: 
                   1826: void cpu_abort(CPUState *env, const char *fmt, ...)
                   1827: {
                   1828:     va_list ap;
1.1.1.6   root     1829:     va_list ap2;
1.1       root     1830: 
                   1831:     va_start(ap, fmt);
1.1.1.6   root     1832:     va_copy(ap2, ap);
1.1       root     1833:     fprintf(stderr, "qemu: fatal: ");
                   1834:     vfprintf(stderr, fmt, ap);
                   1835:     fprintf(stderr, "\n");
                   1836: #ifdef TARGET_I386
                   1837:     cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
                   1838: #else
                   1839:     cpu_dump_state(env, stderr, fprintf, 0);
                   1840: #endif
1.1.1.7   root     1841:     if (qemu_log_enabled()) {
                   1842:         qemu_log("qemu: fatal: ");
                   1843:         qemu_log_vprintf(fmt, ap2);
                   1844:         qemu_log("\n");
1.1.1.6   root     1845: #ifdef TARGET_I386
1.1.1.7   root     1846:         log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1.1.1.6   root     1847: #else
1.1.1.7   root     1848:         log_cpu_state(env, 0);
1.1.1.6   root     1849: #endif
1.1.1.7   root     1850:         qemu_log_flush();
                   1851:         qemu_log_close();
1.1.1.6   root     1852:     }
                   1853:     va_end(ap2);
1.1       root     1854:     va_end(ap);
1.1.1.13! root     1855: #if defined(CONFIG_USER_ONLY)
        !          1856:     {
        !          1857:         struct sigaction act;
        !          1858:         sigfillset(&act.sa_mask);
        !          1859:         act.sa_handler = SIG_DFL;
        !          1860:         sigaction(SIGABRT, &act, NULL);
        !          1861:     }
        !          1862: #endif
1.1       root     1863:     abort();
                   1864: }
                   1865: 
1.1.1.6   root     1866: CPUState *cpu_copy(CPUState *env)
                   1867: {
                   1868:     CPUState *new_env = cpu_init(env->cpu_model_str);
                   1869:     CPUState *next_cpu = new_env->next_cpu;
                   1870:     int cpu_index = new_env->cpu_index;
1.1.1.7   root     1871: #if defined(TARGET_HAS_ICE)
                   1872:     CPUBreakpoint *bp;
                   1873:     CPUWatchpoint *wp;
                   1874: #endif
                   1875: 
1.1.1.6   root     1876:     memcpy(new_env, env, sizeof(CPUState));
1.1.1.7   root     1877: 
                   1878:     /* Preserve chaining and index. */
1.1.1.6   root     1879:     new_env->next_cpu = next_cpu;
                   1880:     new_env->cpu_index = cpu_index;
1.1.1.7   root     1881: 
                   1882:     /* Clone all break/watchpoints.
                   1883:        Note: Once we support ptrace with hw-debug register access, make sure
                   1884:        BP_CPU break/watchpoints are handled correctly on clone. */
1.1.1.11  root     1885:     QTAILQ_INIT(&env->breakpoints);
                   1886:     QTAILQ_INIT(&env->watchpoints);
1.1.1.7   root     1887: #if defined(TARGET_HAS_ICE)
1.1.1.11  root     1888:     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1.1.1.7   root     1889:         cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
                   1890:     }
1.1.1.11  root     1891:     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1.1.1.7   root     1892:         cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
                   1893:                               wp->flags, NULL);
                   1894:     }
                   1895: #endif
                   1896: 
1.1.1.6   root     1897:     return new_env;
                   1898: }
                   1899: 
1.1       root     1900: #if !defined(CONFIG_USER_ONLY)
                   1901: 
1.1.1.7   root     1902: static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
                   1903: {
                   1904:     unsigned int i;
                   1905: 
                   1906:     /* Discard jump cache entries for any tb which might potentially
                   1907:        overlap the flushed page.  */
                   1908:     i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
                   1909:     memset (&env->tb_jmp_cache[i], 0, 
                   1910:            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
                   1911: 
                   1912:     i = tb_jmp_cache_hash_page(addr);
                   1913:     memset (&env->tb_jmp_cache[i], 0, 
                   1914:            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
                   1915: }
                   1916: 
1.1.1.10  root     1917: static CPUTLBEntry s_cputlb_empty_entry = {
                   1918:     .addr_read  = -1,
                   1919:     .addr_write = -1,
                   1920:     .addr_code  = -1,
                   1921:     .addend     = -1,
                   1922: };
                   1923: 
1.1       root     1924: /* NOTE: if flush_global is true, also flush global entries (not
                   1925:    implemented yet) */
                   1926: void tlb_flush(CPUState *env, int flush_global)
                   1927: {
                   1928:     int i;
                   1929: 
                   1930: #if defined(DEBUG_TLB)
                   1931:     printf("tlb_flush:\n");
                   1932: #endif
                   1933:     /* must reset current TB so that interrupts cannot modify the
                   1934:        links while we are modifying them */
                   1935:     env->current_tb = NULL;
                   1936: 
                   1937:     for(i = 0; i < CPU_TLB_SIZE; i++) {
1.1.1.10  root     1938:         int mmu_idx;
                   1939:         for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
                   1940:             env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
                   1941:         }
1.1       root     1942:     }
                   1943: 
1.1.1.2   root     1944:     memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1.1       root     1945: 
1.1.1.13! root     1946:     env->tlb_flush_addr = -1;
        !          1947:     env->tlb_flush_mask = 0;
1.1       root     1948:     tlb_flush_count++;
                   1949: }
                   1950: 
                   1951: static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
                   1952: {
1.1.1.6   root     1953:     if (addr == (tlb_entry->addr_read &
1.1.1.2   root     1954:                  (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1.1.1.6   root     1955:         addr == (tlb_entry->addr_write &
1.1.1.2   root     1956:                  (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1.1.1.6   root     1957:         addr == (tlb_entry->addr_code &
1.1.1.2   root     1958:                  (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1.1.1.10  root     1959:         *tlb_entry = s_cputlb_empty_entry;
1.1.1.2   root     1960:     }
1.1       root     1961: }
                   1962: 
                   1963: void tlb_flush_page(CPUState *env, target_ulong addr)
                   1964: {
1.1.1.2   root     1965:     int i;
1.1.1.10  root     1966:     int mmu_idx;
1.1       root     1967: 
                   1968: #if defined(DEBUG_TLB)
                   1969:     printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
                   1970: #endif
1.1.1.13! root     1971:     /* Check if we need to flush due to large pages.  */
        !          1972:     if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
        !          1973: #if defined(DEBUG_TLB)
        !          1974:         printf("tlb_flush_page: forced full flush ("
        !          1975:                TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
        !          1976:                env->tlb_flush_addr, env->tlb_flush_mask);
        !          1977: #endif
        !          1978:         tlb_flush(env, 1);
        !          1979:         return;
        !          1980:     }
1.1       root     1981:     /* must reset current TB so that interrupts cannot modify the
                   1982:        links while we are modifying them */
                   1983:     env->current_tb = NULL;
                   1984: 
                   1985:     addr &= TARGET_PAGE_MASK;
                   1986:     i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1.1.1.10  root     1987:     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
                   1988:         tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
1.1       root     1989: 
1.1.1.7   root     1990:     tlb_flush_jmp_cache(env, addr);
1.1       root     1991: }
                   1992: 
                   1993: /* update the TLBs so that writes to code in the virtual page 'addr'
                   1994:    can be detected */
1.1.1.2   root     1995: static void tlb_protect_code(ram_addr_t ram_addr)
1.1       root     1996: {
1.1.1.6   root     1997:     cpu_physical_memory_reset_dirty(ram_addr,
1.1.1.2   root     1998:                                     ram_addr + TARGET_PAGE_SIZE,
                   1999:                                     CODE_DIRTY_FLAG);
1.1       root     2000: }
                   2001: 
                   2002: /* update the TLB so that writes in physical page 'phys_addr' are no longer
                   2003:    tested for self modifying code */
1.1.1.6   root     2004: static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1.1       root     2005:                                     target_ulong vaddr)
                   2006: {
1.1.1.13! root     2007:     cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
1.1       root     2008: }
                   2009: 
1.1.1.6   root     2010: static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1.1       root     2011:                                          unsigned long start, unsigned long length)
                   2012: {
                   2013:     unsigned long addr;
1.1.1.2   root     2014:     if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
                   2015:         addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1.1       root     2016:         if ((addr - start) < length) {
1.1.1.7   root     2017:             tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1.1       root     2018:         }
                   2019:     }
                   2020: }
                   2021: 
1.1.1.10  root     2022: /* Note: start and end must be within the same ram block.  */
1.1       root     2023: void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
                   2024:                                      int dirty_flags)
                   2025: {
                   2026:     CPUState *env;
                   2027:     unsigned long length, start1;
1.1.1.13! root     2028:     int i;
1.1       root     2029: 
                   2030:     start &= TARGET_PAGE_MASK;
                   2031:     end = TARGET_PAGE_ALIGN(end);
                   2032: 
                   2033:     length = end - start;
                   2034:     if (length == 0)
                   2035:         return;
1.1.1.13! root     2036:     cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
1.1       root     2037: 
                   2038:     /* we modify the TLB cache so that the dirty bit will be set again
                   2039:        when accessing the range */
1.1.1.10  root     2040:     start1 = (unsigned long)qemu_get_ram_ptr(start);
                   2041:     /* Chek that we don't span multiple blocks - this breaks the
                   2042:        address comparisons below.  */
                   2043:     if ((unsigned long)qemu_get_ram_ptr(end - 1) - start1
                   2044:             != (end - 1) - start) {
                   2045:         abort();
                   2046:     }
                   2047: 
1.1.1.2   root     2048:     for(env = first_cpu; env != NULL; env = env->next_cpu) {
1.1.1.10  root     2049:         int mmu_idx;
                   2050:         for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
                   2051:             for(i = 0; i < CPU_TLB_SIZE; i++)
                   2052:                 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
                   2053:                                       start1, length);
                   2054:         }
1.1.1.2   root     2055:     }
1.1.1.7   root     2056: }
1.1       root     2057: 
1.1.1.7   root     2058: int cpu_physical_memory_set_dirty_tracking(int enable)
                   2059: {
1.1.1.13! root     2060:     int ret = 0;
1.1.1.7   root     2061:     in_migration = enable;
1.1.1.13! root     2062:     ret = cpu_notify_migration_log(!!enable);
        !          2063:     return ret;
1.1.1.7   root     2064: }
1.1       root     2065: 
1.1.1.7   root     2066: int cpu_physical_memory_get_dirty_tracking(void)
                   2067: {
                   2068:     return in_migration;
                   2069: }
                   2070: 
1.1.1.10  root     2071: int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
                   2072:                                    target_phys_addr_t end_addr)
1.1.1.7   root     2073: {
1.1.1.13! root     2074:     int ret;
1.1.1.10  root     2075: 
1.1.1.13! root     2076:     ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
1.1.1.10  root     2077:     return ret;
1.1       root     2078: }
                   2079: 
                   2080: static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
                   2081: {
                   2082:     ram_addr_t ram_addr;
1.1.1.10  root     2083:     void *p;
1.1       root     2084: 
1.1.1.2   root     2085:     if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1.1.1.10  root     2086:         p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
                   2087:             + tlb_entry->addend);
                   2088:         ram_addr = qemu_ram_addr_from_host(p);
1.1       root     2089:         if (!cpu_physical_memory_is_dirty(ram_addr)) {
1.1.1.7   root     2090:             tlb_entry->addr_write |= TLB_NOTDIRTY;
1.1       root     2091:         }
                   2092:     }
                   2093: }
                   2094: 
                   2095: /* update the TLB according to the current state of the dirty bits */
                   2096: void cpu_tlb_update_dirty(CPUState *env)
                   2097: {
                   2098:     int i;
1.1.1.10  root     2099:     int mmu_idx;
                   2100:     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
                   2101:         for(i = 0; i < CPU_TLB_SIZE; i++)
                   2102:             tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
                   2103:     }
1.1       root     2104: }
                   2105: 
1.1.1.7   root     2106: static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1.1       root     2107: {
1.1.1.7   root     2108:     if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
                   2109:         tlb_entry->addr_write = vaddr;
1.1       root     2110: }
                   2111: 
1.1.1.7   root     2112: /* update the TLB corresponding to virtual page vaddr
                   2113:    so that it is no longer dirty */
                   2114: static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
1.1       root     2115: {
                   2116:     int i;
1.1.1.10  root     2117:     int mmu_idx;
1.1       root     2118: 
1.1.1.7   root     2119:     vaddr &= TARGET_PAGE_MASK;
1.1       root     2120:     i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1.1.1.10  root     2121:     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
                   2122:         tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
1.1       root     2123: }
                   2124: 
1.1.1.13! root     2125: /* Our TLB does not support large pages, so remember the area covered by
        !          2126:    large pages and trigger a full TLB flush if these are invalidated.  */
        !          2127: static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
        !          2128:                                target_ulong size)
        !          2129: {
        !          2130:     target_ulong mask = ~(size - 1);
        !          2131: 
        !          2132:     if (env->tlb_flush_addr == (target_ulong)-1) {
        !          2133:         env->tlb_flush_addr = vaddr & mask;
        !          2134:         env->tlb_flush_mask = mask;
        !          2135:         return;
        !          2136:     }
        !          2137:     /* Extend the existing region to include the new page.
        !          2138:        This is a compromise between unnecessary flushes and the cost
        !          2139:        of maintaining a full variable size TLB.  */
        !          2140:     mask &= env->tlb_flush_mask;
        !          2141:     while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
        !          2142:         mask <<= 1;
        !          2143:     }
        !          2144:     env->tlb_flush_addr &= mask;
        !          2145:     env->tlb_flush_mask = mask;
        !          2146: }
        !          2147: 
        !          2148: /* Add a new TLB entry. At most one entry for a given virtual address
        !          2149:    is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
        !          2150:    supplied size is only used by tlb_flush_page.  */
        !          2151: void tlb_set_page(CPUState *env, target_ulong vaddr,
        !          2152:                   target_phys_addr_t paddr, int prot,
        !          2153:                   int mmu_idx, target_ulong size)
1.1       root     2154: {
                   2155:     PhysPageDesc *p;
                   2156:     unsigned long pd;
                   2157:     unsigned int index;
                   2158:     target_ulong address;
1.1.1.7   root     2159:     target_ulong code_address;
1.1.1.13! root     2160:     unsigned long addend;
1.1.1.2   root     2161:     CPUTLBEntry *te;
1.1.1.7   root     2162:     CPUWatchpoint *wp;
                   2163:     target_phys_addr_t iotlb;
1.1       root     2164: 
1.1.1.13! root     2165:     assert(size >= TARGET_PAGE_SIZE);
        !          2166:     if (size != TARGET_PAGE_SIZE) {
        !          2167:         tlb_add_large_page(env, vaddr, size);
        !          2168:     }
1.1       root     2169:     p = phys_page_find(paddr >> TARGET_PAGE_BITS);
                   2170:     if (!p) {
                   2171:         pd = IO_MEM_UNASSIGNED;
                   2172:     } else {
                   2173:         pd = p->phys_offset;
                   2174:     }
                   2175: #if defined(DEBUG_TLB)
1.1.1.6   root     2176:     printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
                   2177:            vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
1.1       root     2178: #endif
                   2179: 
1.1.1.7   root     2180:     address = vaddr;
                   2181:     if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
                   2182:         /* IO memory case (romd handled later) */
                   2183:         address |= TLB_MMIO;
                   2184:     }
1.1.1.10  root     2185:     addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
1.1.1.7   root     2186:     if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
                   2187:         /* Normal RAM.  */
                   2188:         iotlb = pd & TARGET_PAGE_MASK;
                   2189:         if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
                   2190:             iotlb |= IO_MEM_NOTDIRTY;
                   2191:         else
                   2192:             iotlb |= IO_MEM_ROM;
                   2193:     } else {
1.1.1.10  root     2194:         /* IO handlers are currently passed a physical address.
1.1.1.7   root     2195:            It would be nice to pass an offset from the base address
                   2196:            of that region.  This would avoid having to special case RAM,
                   2197:            and avoid full address decoding in every device.
                   2198:            We can't use the high bits of pd for this because
                   2199:            IO_MEM_ROMD uses these as a ram address.  */
                   2200:         iotlb = (pd & ~TARGET_PAGE_MASK);
                   2201:         if (p) {
                   2202:             iotlb += p->region_offset;
1.1       root     2203:         } else {
1.1.1.7   root     2204:             iotlb += paddr;
1.1       root     2205:         }
1.1.1.7   root     2206:     }
1.1.1.6   root     2207: 
1.1.1.7   root     2208:     code_address = address;
                   2209:     /* Make accesses to pages with watchpoints go via the
                   2210:        watchpoint trap routines.  */
1.1.1.11  root     2211:     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1.1.1.7   root     2212:         if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
1.1.1.13! root     2213:             /* Avoid trapping reads of pages with a write breakpoint. */
        !          2214:             if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
        !          2215:                 iotlb = io_mem_watch + paddr;
        !          2216:                 address |= TLB_MMIO;
        !          2217:                 break;
        !          2218:             }
1.1.1.6   root     2219:         }
1.1.1.7   root     2220:     }
1.1.1.6   root     2221: 
1.1.1.7   root     2222:     index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
                   2223:     env->iotlb[mmu_idx][index] = iotlb - vaddr;
                   2224:     te = &env->tlb_table[mmu_idx][index];
                   2225:     te->addend = addend - vaddr;
                   2226:     if (prot & PAGE_READ) {
                   2227:         te->addr_read = address;
                   2228:     } else {
                   2229:         te->addr_read = -1;
1.1       root     2230:     }
                   2231: 
1.1.1.7   root     2232:     if (prot & PAGE_EXEC) {
                   2233:         te->addr_code = code_address;
                   2234:     } else {
                   2235:         te->addr_code = -1;
                   2236:     }
                   2237:     if (prot & PAGE_WRITE) {
                   2238:         if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
                   2239:             (pd & IO_MEM_ROMD)) {
                   2240:             /* Write access calls the I/O callback.  */
                   2241:             te->addr_write = address | TLB_MMIO;
                   2242:         } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
                   2243:                    !cpu_physical_memory_is_dirty(pd)) {
                   2244:             te->addr_write = address | TLB_NOTDIRTY;
                   2245:         } else {
                   2246:             te->addr_write = address;
1.1       root     2247:         }
1.1.1.7   root     2248:     } else {
                   2249:         te->addr_write = -1;
1.1       root     2250:     }
                   2251: }
                   2252: 
                   2253: #else
                   2254: 
                   2255: void tlb_flush(CPUState *env, int flush_global)
                   2256: {
                   2257: }
                   2258: 
                   2259: void tlb_flush_page(CPUState *env, target_ulong addr)
                   2260: {
                   2261: }
                   2262: 
1.1.1.10  root     2263: /*
                   2264:  * Walks guest process memory "regions" one by one
                   2265:  * and calls callback function 'fn' for each region.
                   2266:  */
1.1.1.13! root     2267: 
        !          2268: struct walk_memory_regions_data
        !          2269: {
        !          2270:     walk_memory_regions_fn fn;
        !          2271:     void *priv;
        !          2272:     unsigned long start;
        !          2273:     int prot;
        !          2274: };
        !          2275: 
        !          2276: static int walk_memory_regions_end(struct walk_memory_regions_data *data,
        !          2277:                                    abi_ulong end, int new_prot)
1.1       root     2278: {
1.1.1.13! root     2279:     if (data->start != -1ul) {
        !          2280:         int rc = data->fn(data->priv, data->start, end, data->prot);
        !          2281:         if (rc != 0) {
        !          2282:             return rc;
        !          2283:         }
        !          2284:     }
        !          2285: 
        !          2286:     data->start = (new_prot ? end : -1ul);
        !          2287:     data->prot = new_prot;
        !          2288: 
        !          2289:     return 0;
        !          2290: }
        !          2291: 
        !          2292: static int walk_memory_regions_1(struct walk_memory_regions_data *data,
        !          2293:                                  abi_ulong base, int level, void **lp)
        !          2294: {
        !          2295:     abi_ulong pa;
        !          2296:     int i, rc;
        !          2297: 
        !          2298:     if (*lp == NULL) {
        !          2299:         return walk_memory_regions_end(data, base, 0);
        !          2300:     }
        !          2301: 
        !          2302:     if (level == 0) {
        !          2303:         PageDesc *pd = *lp;
        !          2304:         for (i = 0; i < L2_SIZE; ++i) {
        !          2305:             int prot = pd[i].flags;
        !          2306: 
        !          2307:             pa = base | (i << TARGET_PAGE_BITS);
        !          2308:             if (prot != data->prot) {
        !          2309:                 rc = walk_memory_regions_end(data, pa, prot);
        !          2310:                 if (rc != 0) {
        !          2311:                     return rc;
1.1       root     2312:                 }
                   2313:             }
1.1.1.13! root     2314:         }
        !          2315:     } else {
        !          2316:         void **pp = *lp;
        !          2317:         for (i = 0; i < L2_SIZE; ++i) {
        !          2318:             pa = base | ((abi_ulong)i <<
        !          2319:                 (TARGET_PAGE_BITS + L2_BITS * level));
        !          2320:             rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
        !          2321:             if (rc != 0) {
        !          2322:                 return rc;
        !          2323:             }
1.1       root     2324:         }
                   2325:     }
1.1.1.13! root     2326: 
        !          2327:     return 0;
1.1.1.10  root     2328: }
                   2329: 
1.1.1.13! root     2330: int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
        !          2331: {
        !          2332:     struct walk_memory_regions_data data;
        !          2333:     unsigned long i;
        !          2334: 
        !          2335:     data.fn = fn;
        !          2336:     data.priv = priv;
        !          2337:     data.start = -1ul;
        !          2338:     data.prot = 0;
        !          2339: 
        !          2340:     for (i = 0; i < V_L1_SIZE; i++) {
        !          2341:         int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
        !          2342:                                        V_L1_SHIFT / L2_BITS - 1, l1_map + i);
        !          2343:         if (rc != 0) {
        !          2344:             return rc;
        !          2345:         }
        !          2346:     }
        !          2347: 
        !          2348:     return walk_memory_regions_end(&data, 0, 0);
        !          2349: }
        !          2350: 
        !          2351: static int dump_region(void *priv, abi_ulong start,
        !          2352:     abi_ulong end, unsigned long prot)
1.1.1.10  root     2353: {
                   2354:     FILE *f = (FILE *)priv;
                   2355: 
1.1.1.13! root     2356:     (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
        !          2357:         " "TARGET_ABI_FMT_lx" %c%c%c\n",
1.1.1.10  root     2358:         start, end, end - start,
                   2359:         ((prot & PAGE_READ) ? 'r' : '-'),
                   2360:         ((prot & PAGE_WRITE) ? 'w' : '-'),
                   2361:         ((prot & PAGE_EXEC) ? 'x' : '-'));
                   2362: 
                   2363:     return (0);
                   2364: }
                   2365: 
                   2366: /* dump memory mappings */
                   2367: void page_dump(FILE *f)
                   2368: {
                   2369:     (void) fprintf(f, "%-8s %-8s %-8s %s\n",
                   2370:             "start", "end", "size", "prot");
                   2371:     walk_memory_regions(f, dump_region);
1.1       root     2372: }
                   2373: 
1.1.1.3   root     2374: int page_get_flags(target_ulong address)
1.1       root     2375: {
                   2376:     PageDesc *p;
                   2377: 
                   2378:     p = page_find(address >> TARGET_PAGE_BITS);
                   2379:     if (!p)
                   2380:         return 0;
                   2381:     return p->flags;
                   2382: }
                   2383: 
1.1.1.13! root     2384: /* Modify the flags of a page and invalidate the code if necessary.
        !          2385:    The flag PAGE_WRITE_ORG is positioned automatically depending
        !          2386:    on PAGE_WRITE.  The mmap_lock should already be held.  */
1.1.1.3   root     2387: void page_set_flags(target_ulong start, target_ulong end, int flags)
1.1       root     2388: {
1.1.1.13! root     2389:     target_ulong addr, len;
        !          2390: 
        !          2391:     /* This function should never be called with addresses outside the
        !          2392:        guest address space.  If this assert fires, it probably indicates
        !          2393:        a missing call to h2g_valid.  */
        !          2394: #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
        !          2395:     assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
        !          2396: #endif
        !          2397:     assert(start < end);
1.1       root     2398: 
                   2399:     start = start & TARGET_PAGE_MASK;
                   2400:     end = TARGET_PAGE_ALIGN(end);
1.1.1.13! root     2401: 
        !          2402:     if (flags & PAGE_WRITE) {
1.1       root     2403:         flags |= PAGE_WRITE_ORG;
1.1.1.13! root     2404:     }
        !          2405: 
        !          2406:     for (addr = start, len = end - start;
        !          2407:          len != 0;
        !          2408:          len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
        !          2409:         PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
        !          2410: 
        !          2411:         /* If the write protection bit is set, then we invalidate
        !          2412:            the code inside.  */
1.1.1.6   root     2413:         if (!(p->flags & PAGE_WRITE) &&
1.1       root     2414:             (flags & PAGE_WRITE) &&
                   2415:             p->first_tb) {
                   2416:             tb_invalidate_phys_page(addr, 0, NULL);
                   2417:         }
                   2418:         p->flags = flags;
                   2419:     }
                   2420: }
                   2421: 
1.1.1.6   root     2422: int page_check_range(target_ulong start, target_ulong len, int flags)
                   2423: {
                   2424:     PageDesc *p;
                   2425:     target_ulong end;
                   2426:     target_ulong addr;
                   2427: 
1.1.1.13! root     2428:     /* This function should never be called with addresses outside the
        !          2429:        guest address space.  If this assert fires, it probably indicates
        !          2430:        a missing call to h2g_valid.  */
        !          2431: #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
        !          2432:     assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
        !          2433: #endif
        !          2434: 
        !          2435:     if (len == 0) {
        !          2436:         return 0;
        !          2437:     }
        !          2438:     if (start + len - 1 < start) {
        !          2439:         /* We've wrapped around.  */
1.1.1.7   root     2440:         return -1;
1.1.1.13! root     2441:     }
1.1.1.7   root     2442: 
1.1.1.6   root     2443:     end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
                   2444:     start = start & TARGET_PAGE_MASK;
                   2445: 
1.1.1.13! root     2446:     for (addr = start, len = end - start;
        !          2447:          len != 0;
        !          2448:          len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1.1.1.6   root     2449:         p = page_find(addr >> TARGET_PAGE_BITS);
                   2450:         if( !p )
                   2451:             return -1;
                   2452:         if( !(p->flags & PAGE_VALID) )
                   2453:             return -1;
                   2454: 
                   2455:         if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
                   2456:             return -1;
                   2457:         if (flags & PAGE_WRITE) {
                   2458:             if (!(p->flags & PAGE_WRITE_ORG))
                   2459:                 return -1;
                   2460:             /* unprotect the page if it was put read-only because it
                   2461:                contains translated code */
                   2462:             if (!(p->flags & PAGE_WRITE)) {
                   2463:                 if (!page_unprotect(addr, 0, NULL))
                   2464:                     return -1;
                   2465:             }
                   2466:             return 0;
                   2467:         }
                   2468:     }
                   2469:     return 0;
                   2470: }
                   2471: 
1.1       root     2472: /* called from signal handler: invalidate the code and unprotect the
1.1.1.10  root     2473:    page. Return TRUE if the fault was successfully handled. */
1.1.1.3   root     2474: int page_unprotect(target_ulong address, unsigned long pc, void *puc)
1.1       root     2475: {
1.1.1.13! root     2476:     unsigned int prot;
        !          2477:     PageDesc *p;
1.1.1.3   root     2478:     target_ulong host_start, host_end, addr;
1.1       root     2479: 
1.1.1.7   root     2480:     /* Technically this isn't safe inside a signal handler.  However we
                   2481:        know this only ever happens in a synchronous SEGV handler, so in
                   2482:        practice it seems to be ok.  */
                   2483:     mmap_lock();
                   2484: 
1.1.1.13! root     2485:     p = page_find(address >> TARGET_PAGE_BITS);
        !          2486:     if (!p) {
1.1.1.7   root     2487:         mmap_unlock();
1.1       root     2488:         return 0;
1.1.1.7   root     2489:     }
1.1.1.13! root     2490: 
1.1       root     2491:     /* if the page was really writable, then we change its
                   2492:        protection back to writable */
1.1.1.13! root     2493:     if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
        !          2494:         host_start = address & qemu_host_page_mask;
        !          2495:         host_end = host_start + qemu_host_page_size;
        !          2496: 
        !          2497:         prot = 0;
        !          2498:         for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
        !          2499:             p = page_find(addr >> TARGET_PAGE_BITS);
        !          2500:             p->flags |= PAGE_WRITE;
        !          2501:             prot |= p->flags;
        !          2502: 
1.1       root     2503:             /* and since the content will be modified, we must invalidate
                   2504:                the corresponding translated code. */
1.1.1.13! root     2505:             tb_invalidate_phys_page(addr, pc, puc);
1.1       root     2506: #ifdef DEBUG_TB_CHECK
1.1.1.13! root     2507:             tb_invalidate_check(addr);
1.1       root     2508: #endif
                   2509:         }
1.1.1.13! root     2510:         mprotect((void *)g2h(host_start), qemu_host_page_size,
        !          2511:                  prot & PAGE_BITS);
        !          2512: 
        !          2513:         mmap_unlock();
        !          2514:         return 1;
1.1       root     2515:     }
1.1.1.7   root     2516:     mmap_unlock();
1.1       root     2517:     return 0;
                   2518: }
                   2519: 
1.1.1.2   root     2520: static inline void tlb_set_dirty(CPUState *env,
                   2521:                                  unsigned long addr, target_ulong vaddr)
1.1       root     2522: {
                   2523: }
                   2524: #endif /* defined(CONFIG_USER_ONLY) */
                   2525: 
1.1.1.7   root     2526: #if !defined(CONFIG_USER_ONLY)
                   2527: 
1.1.1.13! root     2528: #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
        !          2529: typedef struct subpage_t {
        !          2530:     target_phys_addr_t base;
        !          2531:     ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
        !          2532:     ram_addr_t region_offset[TARGET_PAGE_SIZE];
        !          2533: } subpage_t;
        !          2534: 
1.1.1.6   root     2535: static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1.1.1.7   root     2536:                              ram_addr_t memory, ram_addr_t region_offset);
1.1.1.13! root     2537: static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
        !          2538:                                 ram_addr_t orig_memory,
        !          2539:                                 ram_addr_t region_offset);
1.1.1.6   root     2540: #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
                   2541:                       need_subpage)                                     \
                   2542:     do {                                                                \
                   2543:         if (addr > start_addr)                                          \
                   2544:             start_addr2 = 0;                                            \
                   2545:         else {                                                          \
                   2546:             start_addr2 = start_addr & ~TARGET_PAGE_MASK;               \
                   2547:             if (start_addr2 > 0)                                        \
                   2548:                 need_subpage = 1;                                       \
                   2549:         }                                                               \
                   2550:                                                                         \
                   2551:         if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE)        \
                   2552:             end_addr2 = TARGET_PAGE_SIZE - 1;                           \
                   2553:         else {                                                          \
                   2554:             end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
                   2555:             if (end_addr2 < TARGET_PAGE_SIZE - 1)                       \
                   2556:                 need_subpage = 1;                                       \
                   2557:         }                                                               \
                   2558:     } while (0)
                   2559: 
1.1.1.11  root     2560: /* register physical memory.
                   2561:    For RAM, 'size' must be a multiple of the target page size.
                   2562:    If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
1.1.1.7   root     2563:    io memory page.  The address used when calling the IO function is
                   2564:    the offset from the start of the region, plus region_offset.  Both
1.1.1.10  root     2565:    start_addr and region_offset are rounded down to a page boundary
1.1.1.7   root     2566:    before calculating this offset.  This should not be a problem unless
                   2567:    the low bits of start_addr and region_offset differ.  */
                   2568: void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
                   2569:                                          ram_addr_t size,
                   2570:                                          ram_addr_t phys_offset,
                   2571:                                          ram_addr_t region_offset)
1.1       root     2572: {
                   2573:     target_phys_addr_t addr, end_addr;
                   2574:     PhysPageDesc *p;
1.1.1.4   root     2575:     CPUState *env;
1.1.1.7   root     2576:     ram_addr_t orig_size = size;
1.1.1.13! root     2577:     subpage_t *subpage;
1.1       root     2578: 
1.1.1.13! root     2579:     cpu_notify_set_memory(start_addr, size, phys_offset);
1.1.1.7   root     2580: 
                   2581:     if (phys_offset == IO_MEM_UNASSIGNED) {
                   2582:         region_offset = start_addr;
                   2583:     }
                   2584:     region_offset &= TARGET_PAGE_MASK;
1.1       root     2585:     size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
1.1.1.6   root     2586:     end_addr = start_addr + (target_phys_addr_t)size;
1.1       root     2587:     for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
1.1.1.6   root     2588:         p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   2589:         if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
1.1.1.7   root     2590:             ram_addr_t orig_memory = p->phys_offset;
1.1.1.6   root     2591:             target_phys_addr_t start_addr2, end_addr2;
                   2592:             int need_subpage = 0;
                   2593: 
                   2594:             CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
                   2595:                           need_subpage);
1.1.1.13! root     2596:             if (need_subpage) {
1.1.1.6   root     2597:                 if (!(orig_memory & IO_MEM_SUBPAGE)) {
                   2598:                     subpage = subpage_init((addr & TARGET_PAGE_MASK),
1.1.1.7   root     2599:                                            &p->phys_offset, orig_memory,
                   2600:                                            p->region_offset);
1.1.1.6   root     2601:                 } else {
                   2602:                     subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
                   2603:                                             >> IO_MEM_SHIFT];
                   2604:                 }
1.1.1.7   root     2605:                 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
                   2606:                                  region_offset);
                   2607:                 p->region_offset = 0;
1.1.1.6   root     2608:             } else {
                   2609:                 p->phys_offset = phys_offset;
                   2610:                 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
                   2611:                     (phys_offset & IO_MEM_ROMD))
                   2612:                     phys_offset += TARGET_PAGE_SIZE;
                   2613:             }
                   2614:         } else {
                   2615:             p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
                   2616:             p->phys_offset = phys_offset;
1.1.1.7   root     2617:             p->region_offset = region_offset;
1.1.1.6   root     2618:             if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
1.1.1.7   root     2619:                 (phys_offset & IO_MEM_ROMD)) {
1.1.1.6   root     2620:                 phys_offset += TARGET_PAGE_SIZE;
1.1.1.7   root     2621:             } else {
1.1.1.6   root     2622:                 target_phys_addr_t start_addr2, end_addr2;
                   2623:                 int need_subpage = 0;
                   2624: 
                   2625:                 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
                   2626:                               end_addr2, need_subpage);
                   2627: 
1.1.1.13! root     2628:                 if (need_subpage) {
1.1.1.6   root     2629:                     subpage = subpage_init((addr & TARGET_PAGE_MASK),
1.1.1.7   root     2630:                                            &p->phys_offset, IO_MEM_UNASSIGNED,
                   2631:                                            addr & TARGET_PAGE_MASK);
1.1.1.6   root     2632:                     subpage_register(subpage, start_addr2, end_addr2,
1.1.1.7   root     2633:                                      phys_offset, region_offset);
                   2634:                     p->region_offset = 0;
1.1.1.6   root     2635:                 }
                   2636:             }
                   2637:         }
1.1.1.7   root     2638:         region_offset += TARGET_PAGE_SIZE;
1.1       root     2639:     }
1.1.1.6   root     2640: 
1.1.1.4   root     2641:     /* since each CPU stores ram addresses in its TLB cache, we must
                   2642:        reset the modified entries */
                   2643:     /* XXX: slow ! */
                   2644:     for(env = first_cpu; env != NULL; env = env->next_cpu) {
                   2645:         tlb_flush(env, 1);
                   2646:     }
1.1       root     2647: }
                   2648: 
1.1.1.5   root     2649: /* XXX: temporary until new memory mapping API */
1.1.1.7   root     2650: ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
1.1.1.5   root     2651: {
                   2652:     PhysPageDesc *p;
                   2653: 
                   2654:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   2655:     if (!p)
                   2656:         return IO_MEM_UNASSIGNED;
                   2657:     return p->phys_offset;
                   2658: }
                   2659: 
1.1.1.7   root     2660: void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
                   2661: {
                   2662:     if (kvm_enabled())
                   2663:         kvm_coalesce_mmio_region(addr, size);
                   2664: }
                   2665: 
                   2666: void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
                   2667: {
                   2668:     if (kvm_enabled())
                   2669:         kvm_uncoalesce_mmio_region(addr, size);
                   2670: }
                   2671: 
1.1.1.13! root     2672: void qemu_flush_coalesced_mmio_buffer(void)
1.1.1.10  root     2673: {
1.1.1.13! root     2674:     if (kvm_enabled())
        !          2675:         kvm_flush_coalesced_mmio_buffer();
        !          2676: }
        !          2677: 
        !          2678: #if defined(__linux__) && !defined(TARGET_S390X)
        !          2679: 
        !          2680: #include <sys/vfs.h>
        !          2681: 
        !          2682: #define HUGETLBFS_MAGIC       0x958458f6
        !          2683: 
        !          2684: static long gethugepagesize(const char *path)
        !          2685: {
        !          2686:     struct statfs fs;
        !          2687:     int ret;
        !          2688: 
        !          2689:     do {
        !          2690:            ret = statfs(path, &fs);
        !          2691:     } while (ret != 0 && errno == EINTR);
        !          2692: 
        !          2693:     if (ret != 0) {
        !          2694:            perror(path);
        !          2695:            return 0;
        !          2696:     }
        !          2697: 
        !          2698:     if (fs.f_type != HUGETLBFS_MAGIC)
        !          2699:            fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
        !          2700: 
        !          2701:     return fs.f_bsize;
        !          2702: }
        !          2703: 
        !          2704: static void *file_ram_alloc(RAMBlock *block,
        !          2705:                             ram_addr_t memory,
        !          2706:                             const char *path)
        !          2707: {
        !          2708:     char *filename;
        !          2709:     void *area;
        !          2710:     int fd;
        !          2711: #ifdef MAP_POPULATE
        !          2712:     int flags;
        !          2713: #endif
        !          2714:     unsigned long hpagesize;
        !          2715: 
        !          2716:     hpagesize = gethugepagesize(path);
        !          2717:     if (!hpagesize) {
        !          2718:        return NULL;
        !          2719:     }
        !          2720: 
        !          2721:     if (memory < hpagesize) {
        !          2722:         return NULL;
        !          2723:     }
        !          2724: 
        !          2725:     if (kvm_enabled() && !kvm_has_sync_mmu()) {
        !          2726:         fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
        !          2727:         return NULL;
        !          2728:     }
        !          2729: 
        !          2730:     if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
        !          2731:        return NULL;
        !          2732:     }
        !          2733: 
        !          2734:     fd = mkstemp(filename);
        !          2735:     if (fd < 0) {
        !          2736:        perror("unable to create backing store for hugepages");
        !          2737:        free(filename);
        !          2738:        return NULL;
        !          2739:     }
        !          2740:     unlink(filename);
        !          2741:     free(filename);
        !          2742: 
        !          2743:     memory = (memory+hpagesize-1) & ~(hpagesize-1);
        !          2744: 
        !          2745:     /*
        !          2746:      * ftruncate is not supported by hugetlbfs in older
        !          2747:      * hosts, so don't bother bailing out on errors.
        !          2748:      * If anything goes wrong with it under other filesystems,
        !          2749:      * mmap will fail.
        !          2750:      */
        !          2751:     if (ftruncate(fd, memory))
        !          2752:        perror("ftruncate");
        !          2753: 
        !          2754: #ifdef MAP_POPULATE
        !          2755:     /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
        !          2756:      * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
        !          2757:      * to sidestep this quirk.
        !          2758:      */
        !          2759:     flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
        !          2760:     area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
        !          2761: #else
        !          2762:     area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
        !          2763: #endif
        !          2764:     if (area == MAP_FAILED) {
        !          2765:        perror("file_ram_alloc: can't mmap RAM pages");
        !          2766:        close(fd);
        !          2767:        return (NULL);
        !          2768:     }
        !          2769:     block->fd = fd;
        !          2770:     return area;
        !          2771: }
        !          2772: #endif
        !          2773: 
        !          2774: static ram_addr_t find_ram_offset(ram_addr_t size)
        !          2775: {
        !          2776:     RAMBlock *block, *next_block;
        !          2777:     ram_addr_t offset = 0, mingap = ULONG_MAX;
        !          2778: 
        !          2779:     if (QLIST_EMPTY(&ram_list.blocks))
        !          2780:         return 0;
        !          2781: 
        !          2782:     QLIST_FOREACH(block, &ram_list.blocks, next) {
        !          2783:         ram_addr_t end, next = ULONG_MAX;
        !          2784: 
        !          2785:         end = block->offset + block->length;
        !          2786: 
        !          2787:         QLIST_FOREACH(next_block, &ram_list.blocks, next) {
        !          2788:             if (next_block->offset >= end) {
        !          2789:                 next = MIN(next, next_block->offset);
        !          2790:             }
        !          2791:         }
        !          2792:         if (next - end >= size && next - end < mingap) {
        !          2793:             offset =  end;
        !          2794:             mingap = next - end;
        !          2795:         }
        !          2796:     }
        !          2797:     return offset;
        !          2798: }
        !          2799: 
        !          2800: static ram_addr_t last_ram_offset(void)
        !          2801: {
        !          2802:     RAMBlock *block;
        !          2803:     ram_addr_t last = 0;
        !          2804: 
        !          2805:     QLIST_FOREACH(block, &ram_list.blocks, next)
        !          2806:         last = MAX(last, block->offset + block->length);
        !          2807: 
        !          2808:     return last;
        !          2809: }
        !          2810: 
        !          2811: ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
        !          2812:                         ram_addr_t size, void *host)
        !          2813: {
        !          2814:     RAMBlock *new_block, *block;
        !          2815: 
        !          2816:     size = TARGET_PAGE_ALIGN(size);
        !          2817:     new_block = qemu_mallocz(sizeof(*new_block));
        !          2818: 
        !          2819:     if (dev && dev->parent_bus && dev->parent_bus->info->get_dev_path) {
        !          2820:         char *id = dev->parent_bus->info->get_dev_path(dev);
        !          2821:         if (id) {
        !          2822:             snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
        !          2823:             qemu_free(id);
        !          2824:         }
        !          2825:     }
        !          2826:     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
        !          2827: 
        !          2828:     QLIST_FOREACH(block, &ram_list.blocks, next) {
        !          2829:         if (!strcmp(block->idstr, new_block->idstr)) {
        !          2830:             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
        !          2831:                     new_block->idstr);
        !          2832:             abort();
        !          2833:         }
        !          2834:     }
        !          2835: 
        !          2836:     new_block->host = host;
        !          2837: 
        !          2838:     new_block->offset = find_ram_offset(size);
        !          2839:     new_block->length = size;
        !          2840: 
        !          2841:     QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
        !          2842: 
        !          2843:     ram_list.phys_dirty = qemu_realloc(ram_list.phys_dirty,
        !          2844:                                        last_ram_offset() >> TARGET_PAGE_BITS);
        !          2845:     memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
        !          2846:            0xff, size >> TARGET_PAGE_BITS);
        !          2847: 
        !          2848:     if (kvm_enabled())
        !          2849:         kvm_setup_guest_memory(new_block->host, size);
        !          2850: 
        !          2851:     return new_block->offset;
        !          2852: }
        !          2853: 
        !          2854: ram_addr_t qemu_ram_alloc(DeviceState *dev, const char *name, ram_addr_t size)
        !          2855: {
        !          2856:     RAMBlock *new_block, *block;
1.1.1.10  root     2857: 
                   2858:     size = TARGET_PAGE_ALIGN(size);
1.1.1.13! root     2859:     new_block = qemu_mallocz(sizeof(*new_block));
        !          2860: 
        !          2861:     if (dev && dev->parent_bus && dev->parent_bus->info->get_dev_path) {
        !          2862:         char *id = dev->parent_bus->info->get_dev_path(dev);
        !          2863:         if (id) {
        !          2864:             snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
        !          2865:             qemu_free(id);
        !          2866:         }
        !          2867:     }
        !          2868:     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1.1.1.10  root     2869: 
1.1.1.13! root     2870:     QLIST_FOREACH(block, &ram_list.blocks, next) {
        !          2871:         if (!strcmp(block->idstr, new_block->idstr)) {
        !          2872:             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
        !          2873:                     new_block->idstr);
        !          2874:             abort();
        !          2875:         }
        !          2876:     }
        !          2877: 
        !          2878:     if (mem_path) {
        !          2879: #if defined (__linux__) && !defined(TARGET_S390X)
        !          2880:         new_block->host = file_ram_alloc(new_block, size, mem_path);
        !          2881:         if (!new_block->host) {
        !          2882:             new_block->host = qemu_vmalloc(size);
        !          2883: #ifdef MADV_MERGEABLE
        !          2884:             madvise(new_block->host, size, MADV_MERGEABLE);
        !          2885: #endif
        !          2886:         }
        !          2887: #else
        !          2888:         fprintf(stderr, "-mem-path option unsupported\n");
        !          2889:         exit(1);
        !          2890: #endif
        !          2891:     } else {
1.1.1.11  root     2892: #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1.1.1.13! root     2893:         /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
        !          2894:         new_block->host = mmap((void*)0x1000000, size,
        !          2895:                                 PROT_EXEC|PROT_READ|PROT_WRITE,
        !          2896:                                 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1.1.1.11  root     2897: #else
1.1.1.13! root     2898:         new_block->host = qemu_vmalloc(size);
1.1.1.11  root     2899: #endif
                   2900: #ifdef MADV_MERGEABLE
1.1.1.13! root     2901:         madvise(new_block->host, size, MADV_MERGEABLE);
1.1.1.11  root     2902: #endif
1.1.1.13! root     2903:     }
        !          2904:     new_block->offset = find_ram_offset(size);
1.1.1.10  root     2905:     new_block->length = size;
                   2906: 
1.1.1.13! root     2907:     QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
1.1.1.10  root     2908: 
1.1.1.13! root     2909:     ram_list.phys_dirty = qemu_realloc(ram_list.phys_dirty,
        !          2910:                                        last_ram_offset() >> TARGET_PAGE_BITS);
        !          2911:     memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1.1.1.10  root     2912:            0xff, size >> TARGET_PAGE_BITS);
                   2913: 
1.1.1.9   root     2914:     if (kvm_enabled())
1.1.1.10  root     2915:         kvm_setup_guest_memory(new_block->host, size);
1.1.1.9   root     2916: 
1.1.1.10  root     2917:     return new_block->offset;
1.1.1.6   root     2918: }
                   2919: 
                   2920: void qemu_ram_free(ram_addr_t addr)
                   2921: {
1.1.1.13! root     2922:     RAMBlock *block;
        !          2923: 
        !          2924:     QLIST_FOREACH(block, &ram_list.blocks, next) {
        !          2925:         if (addr == block->offset) {
        !          2926:             QLIST_REMOVE(block, next);
        !          2927:             if (mem_path) {
        !          2928: #if defined (__linux__) && !defined(TARGET_S390X)
        !          2929:                 if (block->fd) {
        !          2930:                     munmap(block->host, block->length);
        !          2931:                     close(block->fd);
        !          2932:                 } else {
        !          2933:                     qemu_vfree(block->host);
        !          2934:                 }
        !          2935: #endif
        !          2936:             } else {
        !          2937: #if defined(TARGET_S390X) && defined(CONFIG_KVM)
        !          2938:                 munmap(block->host, block->length);
        !          2939: #else
        !          2940:                 qemu_vfree(block->host);
        !          2941: #endif
        !          2942:             }
        !          2943:             qemu_free(block);
        !          2944:             return;
        !          2945:         }
        !          2946:     }
        !          2947: 
1.1.1.10  root     2948: }
                   2949: 
                   2950: /* Return a host pointer to ram allocated with qemu_ram_alloc.
                   2951:    With the exception of the softmmu code in this file, this should
                   2952:    only be used for local memory (e.g. video ram) that the device owns,
                   2953:    and knows it isn't going to access beyond the end of the block.
                   2954: 
                   2955:    It should not be used for general purpose DMA.
                   2956:    Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
                   2957:  */
                   2958: void *qemu_get_ram_ptr(ram_addr_t addr)
                   2959: {
                   2960:     RAMBlock *block;
                   2961: 
1.1.1.13! root     2962:     QLIST_FOREACH(block, &ram_list.blocks, next) {
        !          2963:         if (addr - block->offset < block->length) {
        !          2964:             QLIST_REMOVE(block, next);
        !          2965:             QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
        !          2966:             return block->host + (addr - block->offset);
        !          2967:         }
1.1.1.10  root     2968:     }
1.1.1.13! root     2969: 
        !          2970:     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
        !          2971:     abort();
        !          2972: 
        !          2973:     return NULL;
1.1.1.10  root     2974: }
                   2975: 
                   2976: /* Some of the softmmu routines need to translate from a host pointer
                   2977:    (typically a TLB entry) back to a ram offset.  */
                   2978: ram_addr_t qemu_ram_addr_from_host(void *ptr)
                   2979: {
                   2980:     RAMBlock *block;
                   2981:     uint8_t *host = ptr;
                   2982: 
1.1.1.13! root     2983:     QLIST_FOREACH(block, &ram_list.blocks, next) {
        !          2984:         if (host - block->host < block->length) {
        !          2985:             return block->offset + (host - block->host);
        !          2986:         }
1.1.1.10  root     2987:     }
1.1.1.13! root     2988: 
        !          2989:     fprintf(stderr, "Bad ram pointer %p\n", ptr);
        !          2990:     abort();
        !          2991: 
        !          2992:     return 0;
1.1.1.6   root     2993: }
                   2994: 
1.1       root     2995: static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
                   2996: {
1.1.1.5   root     2997: #ifdef DEBUG_UNASSIGNED
1.1.1.6   root     2998:     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
                   2999: #endif
1.1.1.11  root     3000: #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1.1.1.7   root     3001:     do_unassigned_access(addr, 0, 0, 0, 1);
                   3002: #endif
                   3003:     return 0;
                   3004: }
                   3005: 
                   3006: static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
                   3007: {
                   3008: #ifdef DEBUG_UNASSIGNED
                   3009:     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
                   3010: #endif
1.1.1.11  root     3011: #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1.1.1.7   root     3012:     do_unassigned_access(addr, 0, 0, 0, 2);
                   3013: #endif
                   3014:     return 0;
                   3015: }
                   3016: 
                   3017: static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
                   3018: {
                   3019: #ifdef DEBUG_UNASSIGNED
                   3020:     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
                   3021: #endif
1.1.1.11  root     3022: #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1.1.1.7   root     3023:     do_unassigned_access(addr, 0, 0, 0, 4);
1.1.1.5   root     3024: #endif
1.1       root     3025:     return 0;
                   3026: }
                   3027: 
                   3028: static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
                   3029: {
1.1.1.5   root     3030: #ifdef DEBUG_UNASSIGNED
1.1.1.6   root     3031:     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
                   3032: #endif
1.1.1.11  root     3033: #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1.1.1.7   root     3034:     do_unassigned_access(addr, 1, 0, 0, 1);
                   3035: #endif
                   3036: }
                   3037: 
                   3038: static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
                   3039: {
                   3040: #ifdef DEBUG_UNASSIGNED
                   3041:     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
                   3042: #endif
1.1.1.11  root     3043: #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1.1.1.7   root     3044:     do_unassigned_access(addr, 1, 0, 0, 2);
                   3045: #endif
                   3046: }
                   3047: 
                   3048: static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
                   3049: {
                   3050: #ifdef DEBUG_UNASSIGNED
                   3051:     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
                   3052: #endif
1.1.1.11  root     3053: #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1.1.1.7   root     3054:     do_unassigned_access(addr, 1, 0, 0, 4);
1.1.1.5   root     3055: #endif
1.1       root     3056: }
                   3057: 
1.1.1.11  root     3058: static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
1.1       root     3059:     unassigned_mem_readb,
1.1.1.7   root     3060:     unassigned_mem_readw,
                   3061:     unassigned_mem_readl,
1.1       root     3062: };
                   3063: 
1.1.1.11  root     3064: static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
1.1       root     3065:     unassigned_mem_writeb,
1.1.1.7   root     3066:     unassigned_mem_writew,
                   3067:     unassigned_mem_writel,
1.1       root     3068: };
                   3069: 
1.1.1.7   root     3070: static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
                   3071:                                 uint32_t val)
1.1       root     3072: {
                   3073:     int dirty_flags;
1.1.1.13! root     3074:     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1.1       root     3075:     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
                   3076: #if !defined(CONFIG_USER_ONLY)
                   3077:         tb_invalidate_phys_page_fast(ram_addr, 1);
1.1.1.13! root     3078:         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1.1       root     3079: #endif
                   3080:     }
1.1.1.10  root     3081:     stb_p(qemu_get_ram_ptr(ram_addr), val);
1.1       root     3082:     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1.1.1.13! root     3083:     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1.1       root     3084:     /* we remove the notdirty callback only if the code has been
                   3085:        flushed */
                   3086:     if (dirty_flags == 0xff)
1.1.1.7   root     3087:         tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1.1       root     3088: }
                   3089: 
1.1.1.7   root     3090: static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
                   3091:                                 uint32_t val)
1.1       root     3092: {
                   3093:     int dirty_flags;
1.1.1.13! root     3094:     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1.1       root     3095:     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
                   3096: #if !defined(CONFIG_USER_ONLY)
                   3097:         tb_invalidate_phys_page_fast(ram_addr, 2);
1.1.1.13! root     3098:         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1.1       root     3099: #endif
                   3100:     }
1.1.1.10  root     3101:     stw_p(qemu_get_ram_ptr(ram_addr), val);
1.1       root     3102:     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1.1.1.13! root     3103:     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1.1       root     3104:     /* we remove the notdirty callback only if the code has been
                   3105:        flushed */
                   3106:     if (dirty_flags == 0xff)
1.1.1.7   root     3107:         tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1.1       root     3108: }
                   3109: 
1.1.1.7   root     3110: static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
                   3111:                                 uint32_t val)
1.1       root     3112: {
                   3113:     int dirty_flags;
1.1.1.13! root     3114:     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1.1       root     3115:     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
                   3116: #if !defined(CONFIG_USER_ONLY)
                   3117:         tb_invalidate_phys_page_fast(ram_addr, 4);
1.1.1.13! root     3118:         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1.1       root     3119: #endif
                   3120:     }
1.1.1.10  root     3121:     stl_p(qemu_get_ram_ptr(ram_addr), val);
1.1       root     3122:     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1.1.1.13! root     3123:     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1.1       root     3124:     /* we remove the notdirty callback only if the code has been
                   3125:        flushed */
                   3126:     if (dirty_flags == 0xff)
1.1.1.7   root     3127:         tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1.1       root     3128: }
                   3129: 
1.1.1.11  root     3130: static CPUReadMemoryFunc * const error_mem_read[3] = {
1.1       root     3131:     NULL, /* never used */
                   3132:     NULL, /* never used */
                   3133:     NULL, /* never used */
                   3134: };
                   3135: 
1.1.1.11  root     3136: static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
1.1       root     3137:     notdirty_mem_writeb,
                   3138:     notdirty_mem_writew,
                   3139:     notdirty_mem_writel,
                   3140: };
                   3141: 
1.1.1.7   root     3142: /* Generate a debug exception if a watchpoint has been hit.  */
                   3143: static void check_watchpoint(int offset, int len_mask, int flags)
                   3144: {
                   3145:     CPUState *env = cpu_single_env;
                   3146:     target_ulong pc, cs_base;
                   3147:     TranslationBlock *tb;
                   3148:     target_ulong vaddr;
                   3149:     CPUWatchpoint *wp;
                   3150:     int cpu_flags;
                   3151: 
                   3152:     if (env->watchpoint_hit) {
                   3153:         /* We re-entered the check after replacing the TB. Now raise
                   3154:          * the debug interrupt so that is will trigger after the
                   3155:          * current instruction. */
                   3156:         cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
                   3157:         return;
                   3158:     }
                   3159:     vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1.1.1.11  root     3160:     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1.1.1.7   root     3161:         if ((vaddr == (wp->vaddr & len_mask) ||
                   3162:              (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
                   3163:             wp->flags |= BP_WATCHPOINT_HIT;
                   3164:             if (!env->watchpoint_hit) {
                   3165:                 env->watchpoint_hit = wp;
                   3166:                 tb = tb_find_pc(env->mem_io_pc);
                   3167:                 if (!tb) {
                   3168:                     cpu_abort(env, "check_watchpoint: could not find TB for "
                   3169:                               "pc=%p", (void *)env->mem_io_pc);
                   3170:                 }
                   3171:                 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
                   3172:                 tb_phys_invalidate(tb, -1);
                   3173:                 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
                   3174:                     env->exception_index = EXCP_DEBUG;
                   3175:                 } else {
                   3176:                     cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
                   3177:                     tb_gen_code(env, pc, cs_base, cpu_flags, 1);
                   3178:                 }
                   3179:                 cpu_resume_from_signal(env, NULL);
                   3180:             }
                   3181:         } else {
                   3182:             wp->flags &= ~BP_WATCHPOINT_HIT;
                   3183:         }
                   3184:     }
                   3185: }
                   3186: 
1.1.1.6   root     3187: /* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
                   3188:    so these check for a hit then pass through to the normal out-of-line
                   3189:    phys routines.  */
                   3190: static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
                   3191: {
1.1.1.7   root     3192:     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
1.1.1.6   root     3193:     return ldub_phys(addr);
                   3194: }
                   3195: 
                   3196: static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
                   3197: {
1.1.1.7   root     3198:     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
1.1.1.6   root     3199:     return lduw_phys(addr);
                   3200: }
                   3201: 
                   3202: static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
                   3203: {
1.1.1.7   root     3204:     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
1.1.1.6   root     3205:     return ldl_phys(addr);
                   3206: }
                   3207: 
                   3208: static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
                   3209:                              uint32_t val)
                   3210: {
1.1.1.7   root     3211:     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
1.1.1.6   root     3212:     stb_phys(addr, val);
                   3213: }
                   3214: 
                   3215: static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
                   3216:                              uint32_t val)
                   3217: {
1.1.1.7   root     3218:     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
1.1.1.6   root     3219:     stw_phys(addr, val);
                   3220: }
                   3221: 
                   3222: static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
                   3223:                              uint32_t val)
                   3224: {
1.1.1.7   root     3225:     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
1.1.1.6   root     3226:     stl_phys(addr, val);
                   3227: }
                   3228: 
1.1.1.11  root     3229: static CPUReadMemoryFunc * const watch_mem_read[3] = {
1.1.1.6   root     3230:     watch_mem_readb,
                   3231:     watch_mem_readw,
                   3232:     watch_mem_readl,
                   3233: };
                   3234: 
1.1.1.11  root     3235: static CPUWriteMemoryFunc * const watch_mem_write[3] = {
1.1.1.6   root     3236:     watch_mem_writeb,
                   3237:     watch_mem_writew,
                   3238:     watch_mem_writel,
                   3239: };
                   3240: 
1.1.1.13! root     3241: static inline uint32_t subpage_readlen (subpage_t *mmio,
        !          3242:                                         target_phys_addr_t addr,
        !          3243:                                         unsigned int len)
1.1.1.6   root     3244: {
1.1.1.13! root     3245:     unsigned int idx = SUBPAGE_IDX(addr);
1.1.1.6   root     3246: #if defined(DEBUG_SUBPAGE)
                   3247:     printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
                   3248:            mmio, len, addr, idx);
                   3249: #endif
                   3250: 
1.1.1.13! root     3251:     addr += mmio->region_offset[idx];
        !          3252:     idx = mmio->sub_io_index[idx];
        !          3253:     return io_mem_read[idx][len](io_mem_opaque[idx], addr);
1.1.1.6   root     3254: }
                   3255: 
                   3256: static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
1.1.1.13! root     3257:                                      uint32_t value, unsigned int len)
1.1.1.6   root     3258: {
1.1.1.13! root     3259:     unsigned int idx = SUBPAGE_IDX(addr);
1.1.1.6   root     3260: #if defined(DEBUG_SUBPAGE)
1.1.1.13! root     3261:     printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n",
        !          3262:            __func__, mmio, len, addr, idx, value);
1.1.1.6   root     3263: #endif
1.1.1.13! root     3264: 
        !          3265:     addr += mmio->region_offset[idx];
        !          3266:     idx = mmio->sub_io_index[idx];
        !          3267:     io_mem_write[idx][len](io_mem_opaque[idx], addr, value);
1.1.1.6   root     3268: }
                   3269: 
                   3270: static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
                   3271: {
                   3272:     return subpage_readlen(opaque, addr, 0);
                   3273: }
                   3274: 
                   3275: static void subpage_writeb (void *opaque, target_phys_addr_t addr,
                   3276:                             uint32_t value)
                   3277: {
                   3278:     subpage_writelen(opaque, addr, value, 0);
                   3279: }
                   3280: 
                   3281: static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
                   3282: {
                   3283:     return subpage_readlen(opaque, addr, 1);
                   3284: }
                   3285: 
                   3286: static void subpage_writew (void *opaque, target_phys_addr_t addr,
                   3287:                             uint32_t value)
                   3288: {
                   3289:     subpage_writelen(opaque, addr, value, 1);
                   3290: }
                   3291: 
                   3292: static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
                   3293: {
                   3294:     return subpage_readlen(opaque, addr, 2);
                   3295: }
                   3296: 
1.1.1.13! root     3297: static void subpage_writel (void *opaque, target_phys_addr_t addr,
        !          3298:                             uint32_t value)
1.1.1.6   root     3299: {
                   3300:     subpage_writelen(opaque, addr, value, 2);
                   3301: }
                   3302: 
1.1.1.11  root     3303: static CPUReadMemoryFunc * const subpage_read[] = {
1.1.1.6   root     3304:     &subpage_readb,
                   3305:     &subpage_readw,
                   3306:     &subpage_readl,
                   3307: };
                   3308: 
1.1.1.11  root     3309: static CPUWriteMemoryFunc * const subpage_write[] = {
1.1.1.6   root     3310:     &subpage_writeb,
                   3311:     &subpage_writew,
                   3312:     &subpage_writel,
                   3313: };
                   3314: 
                   3315: static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1.1.1.7   root     3316:                              ram_addr_t memory, ram_addr_t region_offset)
1.1.1.6   root     3317: {
                   3318:     int idx, eidx;
                   3319: 
                   3320:     if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
                   3321:         return -1;
                   3322:     idx = SUBPAGE_IDX(start);
                   3323:     eidx = SUBPAGE_IDX(end);
                   3324: #if defined(DEBUG_SUBPAGE)
1.1.1.10  root     3325:     printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1.1.1.6   root     3326:            mmio, start, end, idx, eidx, memory);
                   3327: #endif
1.1.1.13! root     3328:     memory = (memory >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
1.1.1.6   root     3329:     for (; idx <= eidx; idx++) {
1.1.1.13! root     3330:         mmio->sub_io_index[idx] = memory;
        !          3331:         mmio->region_offset[idx] = region_offset;
1.1.1.6   root     3332:     }
                   3333: 
                   3334:     return 0;
                   3335: }
                   3336: 
1.1.1.13! root     3337: static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
        !          3338:                                 ram_addr_t orig_memory,
        !          3339:                                 ram_addr_t region_offset)
1.1.1.6   root     3340: {
                   3341:     subpage_t *mmio;
                   3342:     int subpage_memory;
                   3343: 
                   3344:     mmio = qemu_mallocz(sizeof(subpage_t));
1.1.1.7   root     3345: 
                   3346:     mmio->base = base;
1.1.1.10  root     3347:     subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio);
1.1.1.6   root     3348: #if defined(DEBUG_SUBPAGE)
1.1.1.7   root     3349:     printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
                   3350:            mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1.1.1.6   root     3351: #endif
1.1.1.7   root     3352:     *phys = subpage_memory | IO_MEM_SUBPAGE;
1.1.1.13! root     3353:     subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset);
1.1.1.6   root     3354: 
                   3355:     return mmio;
                   3356: }
                   3357: 
1.1.1.7   root     3358: static int get_free_io_mem_idx(void)
                   3359: {
                   3360:     int i;
                   3361: 
                   3362:     for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
                   3363:         if (!io_mem_used[i]) {
                   3364:             io_mem_used[i] = 1;
                   3365:             return i;
                   3366:         }
1.1.1.13! root     3367:     fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
1.1.1.7   root     3368:     return -1;
                   3369: }
                   3370: 
1.1       root     3371: /* mem_read and mem_write are arrays of functions containing the
                   3372:    function to access byte (index 0), word (index 1) and dword (index
1.1.1.10  root     3373:    2). Functions can be omitted with a NULL function pointer.
1.1.1.6   root     3374:    If io_index is non zero, the corresponding io zone is
                   3375:    modified. If it is zero, a new io zone is allocated. The return
                   3376:    value can be used with cpu_register_physical_memory(). (-1) is
                   3377:    returned if error. */
1.1.1.10  root     3378: static int cpu_register_io_memory_fixed(int io_index,
1.1.1.11  root     3379:                                         CPUReadMemoryFunc * const *mem_read,
                   3380:                                         CPUWriteMemoryFunc * const *mem_write,
1.1.1.10  root     3381:                                         void *opaque)
1.1       root     3382: {
1.1.1.13! root     3383:     int i;
1.1       root     3384: 
                   3385:     if (io_index <= 0) {
1.1.1.7   root     3386:         io_index = get_free_io_mem_idx();
                   3387:         if (io_index == -1)
                   3388:             return io_index;
1.1       root     3389:     } else {
1.1.1.10  root     3390:         io_index >>= IO_MEM_SHIFT;
1.1       root     3391:         if (io_index >= IO_MEM_NB_ENTRIES)
                   3392:             return -1;
                   3393:     }
1.1.1.2   root     3394: 
1.1.1.13! root     3395:     for (i = 0; i < 3; ++i) {
        !          3396:         io_mem_read[io_index][i]
        !          3397:             = (mem_read[i] ? mem_read[i] : unassigned_mem_read[i]);
        !          3398:     }
        !          3399:     for (i = 0; i < 3; ++i) {
        !          3400:         io_mem_write[io_index][i]
        !          3401:             = (mem_write[i] ? mem_write[i] : unassigned_mem_write[i]);
1.1       root     3402:     }
                   3403:     io_mem_opaque[io_index] = opaque;
1.1.1.13! root     3404: 
        !          3405:     return (io_index << IO_MEM_SHIFT);
1.1       root     3406: }
                   3407: 
1.1.1.11  root     3408: int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
                   3409:                            CPUWriteMemoryFunc * const *mem_write,
1.1.1.10  root     3410:                            void *opaque)
                   3411: {
                   3412:     return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque);
                   3413: }
                   3414: 
1.1.1.7   root     3415: void cpu_unregister_io_memory(int io_table_address)
                   3416: {
                   3417:     int i;
                   3418:     int io_index = io_table_address >> IO_MEM_SHIFT;
                   3419: 
                   3420:     for (i=0;i < 3; i++) {
                   3421:         io_mem_read[io_index][i] = unassigned_mem_read[i];
                   3422:         io_mem_write[io_index][i] = unassigned_mem_write[i];
                   3423:     }
                   3424:     io_mem_opaque[io_index] = NULL;
                   3425:     io_mem_used[io_index] = 0;
                   3426: }
                   3427: 
1.1.1.10  root     3428: static void io_mem_init(void)
1.1       root     3429: {
1.1.1.10  root     3430:     int i;
1.1       root     3431: 
1.1.1.10  root     3432:     cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read, unassigned_mem_write, NULL);
                   3433:     cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read, unassigned_mem_write, NULL);
                   3434:     cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read, notdirty_mem_write, NULL);
                   3435:     for (i=0; i<5; i++)
                   3436:         io_mem_used[i] = 1;
                   3437: 
                   3438:     io_mem_watch = cpu_register_io_memory(watch_mem_read,
                   3439:                                           watch_mem_write, NULL);
1.1       root     3440: }
                   3441: 
1.1.1.7   root     3442: #endif /* !defined(CONFIG_USER_ONLY) */
                   3443: 
1.1       root     3444: /* physical memory access (slow version, mainly for debug) */
                   3445: #if defined(CONFIG_USER_ONLY)
1.1.1.13! root     3446: int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
        !          3447:                         uint8_t *buf, int len, int is_write)
1.1       root     3448: {
                   3449:     int l, flags;
                   3450:     target_ulong page;
1.1.1.3   root     3451:     void * p;
1.1       root     3452: 
                   3453:     while (len > 0) {
                   3454:         page = addr & TARGET_PAGE_MASK;
                   3455:         l = (page + TARGET_PAGE_SIZE) - addr;
                   3456:         if (l > len)
                   3457:             l = len;
                   3458:         flags = page_get_flags(page);
                   3459:         if (!(flags & PAGE_VALID))
1.1.1.13! root     3460:             return -1;
1.1       root     3461:         if (is_write) {
                   3462:             if (!(flags & PAGE_WRITE))
1.1.1.13! root     3463:                 return -1;
1.1.1.6   root     3464:             /* XXX: this code should not depend on lock_user */
1.1.1.7   root     3465:             if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1.1.1.13! root     3466:                 return -1;
1.1.1.7   root     3467:             memcpy(p, buf, l);
                   3468:             unlock_user(p, addr, l);
1.1       root     3469:         } else {
                   3470:             if (!(flags & PAGE_READ))
1.1.1.13! root     3471:                 return -1;
1.1.1.6   root     3472:             /* XXX: this code should not depend on lock_user */
1.1.1.7   root     3473:             if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1.1.1.13! root     3474:                 return -1;
1.1.1.7   root     3475:             memcpy(buf, p, l);
1.1.1.3   root     3476:             unlock_user(p, addr, 0);
1.1       root     3477:         }
                   3478:         len -= l;
                   3479:         buf += l;
                   3480:         addr += l;
                   3481:     }
1.1.1.13! root     3482:     return 0;
1.1       root     3483: }
                   3484: 
                   3485: #else
1.1.1.6   root     3486: void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
1.1       root     3487:                             int len, int is_write)
                   3488: {
                   3489:     int l, io_index;
                   3490:     uint8_t *ptr;
                   3491:     uint32_t val;
                   3492:     target_phys_addr_t page;
                   3493:     unsigned long pd;
                   3494:     PhysPageDesc *p;
1.1.1.6   root     3495: 
1.1       root     3496:     while (len > 0) {
                   3497:         page = addr & TARGET_PAGE_MASK;
                   3498:         l = (page + TARGET_PAGE_SIZE) - addr;
                   3499:         if (l > len)
                   3500:             l = len;
                   3501:         p = phys_page_find(page >> TARGET_PAGE_BITS);
                   3502:         if (!p) {
                   3503:             pd = IO_MEM_UNASSIGNED;
                   3504:         } else {
                   3505:             pd = p->phys_offset;
                   3506:         }
1.1.1.6   root     3507: 
1.1       root     3508:         if (is_write) {
                   3509:             if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
1.1.1.7   root     3510:                 target_phys_addr_t addr1 = addr;
1.1       root     3511:                 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
1.1.1.7   root     3512:                 if (p)
                   3513:                     addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
1.1.1.2   root     3514:                 /* XXX: could force cpu_single_env to NULL to avoid
                   3515:                    potential bugs */
1.1.1.7   root     3516:                 if (l >= 4 && ((addr1 & 3) == 0)) {
1.1       root     3517:                     /* 32 bit write access */
                   3518:                     val = ldl_p(buf);
1.1.1.7   root     3519:                     io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
1.1       root     3520:                     l = 4;
1.1.1.7   root     3521:                 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1.1       root     3522:                     /* 16 bit write access */
                   3523:                     val = lduw_p(buf);
1.1.1.7   root     3524:                     io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
1.1       root     3525:                     l = 2;
                   3526:                 } else {
                   3527:                     /* 8 bit write access */
                   3528:                     val = ldub_p(buf);
1.1.1.7   root     3529:                     io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
1.1       root     3530:                     l = 1;
                   3531:                 }
                   3532:             } else {
                   3533:                 unsigned long addr1;
                   3534:                 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
                   3535:                 /* RAM case */
1.1.1.10  root     3536:                 ptr = qemu_get_ram_ptr(addr1);
1.1       root     3537:                 memcpy(ptr, buf, l);
                   3538:                 if (!cpu_physical_memory_is_dirty(addr1)) {
                   3539:                     /* invalidate code */
                   3540:                     tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
                   3541:                     /* set dirty bit */
1.1.1.13! root     3542:                     cpu_physical_memory_set_dirty_flags(
        !          3543:                         addr1, (0xff & ~CODE_DIRTY_FLAG));
1.1       root     3544:                 }
                   3545:             }
                   3546:         } else {
1.1.1.6   root     3547:             if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
1.1.1.4   root     3548:                 !(pd & IO_MEM_ROMD)) {
1.1.1.7   root     3549:                 target_phys_addr_t addr1 = addr;
1.1       root     3550:                 /* I/O case */
                   3551:                 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
1.1.1.7   root     3552:                 if (p)
                   3553:                     addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
                   3554:                 if (l >= 4 && ((addr1 & 3) == 0)) {
1.1       root     3555:                     /* 32 bit read access */
1.1.1.7   root     3556:                     val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
1.1       root     3557:                     stl_p(buf, val);
                   3558:                     l = 4;
1.1.1.7   root     3559:                 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1.1       root     3560:                     /* 16 bit read access */
1.1.1.7   root     3561:                     val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
1.1       root     3562:                     stw_p(buf, val);
                   3563:                     l = 2;
                   3564:                 } else {
                   3565:                     /* 8 bit read access */
1.1.1.7   root     3566:                     val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
1.1       root     3567:                     stb_p(buf, val);
                   3568:                     l = 1;
                   3569:                 }
                   3570:             } else {
                   3571:                 /* RAM case */
1.1.1.10  root     3572:                 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
1.1       root     3573:                     (addr & ~TARGET_PAGE_MASK);
                   3574:                 memcpy(buf, ptr, l);
                   3575:             }
                   3576:         }
                   3577:         len -= l;
                   3578:         buf += l;
                   3579:         addr += l;
                   3580:     }
                   3581: }
                   3582: 
1.1.1.3   root     3583: /* used for ROM loading : can write in RAM and ROM */
1.1.1.6   root     3584: void cpu_physical_memory_write_rom(target_phys_addr_t addr,
1.1.1.3   root     3585:                                    const uint8_t *buf, int len)
                   3586: {
                   3587:     int l;
                   3588:     uint8_t *ptr;
                   3589:     target_phys_addr_t page;
                   3590:     unsigned long pd;
                   3591:     PhysPageDesc *p;
1.1.1.6   root     3592: 
1.1.1.3   root     3593:     while (len > 0) {
                   3594:         page = addr & TARGET_PAGE_MASK;
                   3595:         l = (page + TARGET_PAGE_SIZE) - addr;
                   3596:         if (l > len)
                   3597:             l = len;
                   3598:         p = phys_page_find(page >> TARGET_PAGE_BITS);
                   3599:         if (!p) {
                   3600:             pd = IO_MEM_UNASSIGNED;
                   3601:         } else {
                   3602:             pd = p->phys_offset;
                   3603:         }
1.1.1.6   root     3604: 
1.1.1.3   root     3605:         if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
1.1.1.4   root     3606:             (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
                   3607:             !(pd & IO_MEM_ROMD)) {
1.1.1.3   root     3608:             /* do nothing */
                   3609:         } else {
                   3610:             unsigned long addr1;
                   3611:             addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
                   3612:             /* ROM/RAM case */
1.1.1.10  root     3613:             ptr = qemu_get_ram_ptr(addr1);
1.1.1.3   root     3614:             memcpy(ptr, buf, l);
                   3615:         }
                   3616:         len -= l;
                   3617:         buf += l;
                   3618:         addr += l;
                   3619:     }
                   3620: }
                   3621: 
1.1.1.7   root     3622: typedef struct {
                   3623:     void *buffer;
                   3624:     target_phys_addr_t addr;
                   3625:     target_phys_addr_t len;
                   3626: } BounceBuffer;
                   3627: 
                   3628: static BounceBuffer bounce;
                   3629: 
                   3630: typedef struct MapClient {
                   3631:     void *opaque;
                   3632:     void (*callback)(void *opaque);
1.1.1.11  root     3633:     QLIST_ENTRY(MapClient) link;
1.1.1.7   root     3634: } MapClient;
                   3635: 
1.1.1.11  root     3636: static QLIST_HEAD(map_client_list, MapClient) map_client_list
                   3637:     = QLIST_HEAD_INITIALIZER(map_client_list);
1.1.1.7   root     3638: 
                   3639: void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
                   3640: {
                   3641:     MapClient *client = qemu_malloc(sizeof(*client));
                   3642: 
                   3643:     client->opaque = opaque;
                   3644:     client->callback = callback;
1.1.1.11  root     3645:     QLIST_INSERT_HEAD(&map_client_list, client, link);
1.1.1.7   root     3646:     return client;
                   3647: }
                   3648: 
                   3649: void cpu_unregister_map_client(void *_client)
                   3650: {
                   3651:     MapClient *client = (MapClient *)_client;
                   3652: 
1.1.1.11  root     3653:     QLIST_REMOVE(client, link);
1.1.1.10  root     3654:     qemu_free(client);
1.1.1.7   root     3655: }
                   3656: 
                   3657: static void cpu_notify_map_clients(void)
                   3658: {
                   3659:     MapClient *client;
                   3660: 
1.1.1.11  root     3661:     while (!QLIST_EMPTY(&map_client_list)) {
                   3662:         client = QLIST_FIRST(&map_client_list);
1.1.1.7   root     3663:         client->callback(client->opaque);
1.1.1.10  root     3664:         cpu_unregister_map_client(client);
1.1.1.7   root     3665:     }
                   3666: }
                   3667: 
                   3668: /* Map a physical memory region into a host virtual address.
                   3669:  * May map a subset of the requested range, given by and returned in *plen.
                   3670:  * May return NULL if resources needed to perform the mapping are exhausted.
                   3671:  * Use only for reads OR writes - not for read-modify-write operations.
                   3672:  * Use cpu_register_map_client() to know when retrying the map operation is
                   3673:  * likely to succeed.
                   3674:  */
                   3675: void *cpu_physical_memory_map(target_phys_addr_t addr,
                   3676:                               target_phys_addr_t *plen,
                   3677:                               int is_write)
                   3678: {
                   3679:     target_phys_addr_t len = *plen;
                   3680:     target_phys_addr_t done = 0;
                   3681:     int l;
                   3682:     uint8_t *ret = NULL;
                   3683:     uint8_t *ptr;
                   3684:     target_phys_addr_t page;
                   3685:     unsigned long pd;
                   3686:     PhysPageDesc *p;
                   3687:     unsigned long addr1;
                   3688: 
                   3689:     while (len > 0) {
                   3690:         page = addr & TARGET_PAGE_MASK;
                   3691:         l = (page + TARGET_PAGE_SIZE) - addr;
                   3692:         if (l > len)
                   3693:             l = len;
                   3694:         p = phys_page_find(page >> TARGET_PAGE_BITS);
                   3695:         if (!p) {
                   3696:             pd = IO_MEM_UNASSIGNED;
                   3697:         } else {
                   3698:             pd = p->phys_offset;
                   3699:         }
                   3700: 
                   3701:         if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
                   3702:             if (done || bounce.buffer) {
                   3703:                 break;
                   3704:             }
                   3705:             bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
                   3706:             bounce.addr = addr;
                   3707:             bounce.len = l;
                   3708:             if (!is_write) {
                   3709:                 cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
                   3710:             }
                   3711:             ptr = bounce.buffer;
                   3712:         } else {
                   3713:             addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
1.1.1.10  root     3714:             ptr = qemu_get_ram_ptr(addr1);
1.1.1.7   root     3715:         }
                   3716:         if (!done) {
                   3717:             ret = ptr;
                   3718:         } else if (ret + done != ptr) {
                   3719:             break;
                   3720:         }
                   3721: 
                   3722:         len -= l;
                   3723:         addr += l;
                   3724:         done += l;
                   3725:     }
                   3726:     *plen = done;
                   3727:     return ret;
                   3728: }
                   3729: 
                   3730: /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
                   3731:  * Will also mark the memory as dirty if is_write == 1.  access_len gives
                   3732:  * the amount of memory that was actually read or written by the caller.
                   3733:  */
                   3734: void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
                   3735:                                int is_write, target_phys_addr_t access_len)
                   3736: {
                   3737:     if (buffer != bounce.buffer) {
                   3738:         if (is_write) {
1.1.1.10  root     3739:             ram_addr_t addr1 = qemu_ram_addr_from_host(buffer);
1.1.1.7   root     3740:             while (access_len) {
                   3741:                 unsigned l;
                   3742:                 l = TARGET_PAGE_SIZE;
                   3743:                 if (l > access_len)
                   3744:                     l = access_len;
                   3745:                 if (!cpu_physical_memory_is_dirty(addr1)) {
                   3746:                     /* invalidate code */
                   3747:                     tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
                   3748:                     /* set dirty bit */
1.1.1.13! root     3749:                     cpu_physical_memory_set_dirty_flags(
        !          3750:                         addr1, (0xff & ~CODE_DIRTY_FLAG));
1.1.1.7   root     3751:                 }
                   3752:                 addr1 += l;
                   3753:                 access_len -= l;
                   3754:             }
                   3755:         }
                   3756:         return;
                   3757:     }
                   3758:     if (is_write) {
                   3759:         cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
                   3760:     }
1.1.1.12  root     3761:     qemu_vfree(bounce.buffer);
1.1.1.7   root     3762:     bounce.buffer = NULL;
                   3763:     cpu_notify_map_clients();
                   3764: }
1.1.1.3   root     3765: 
1.1       root     3766: /* warning: addr must be aligned */
                   3767: uint32_t ldl_phys(target_phys_addr_t addr)
                   3768: {
                   3769:     int io_index;
                   3770:     uint8_t *ptr;
                   3771:     uint32_t val;
                   3772:     unsigned long pd;
                   3773:     PhysPageDesc *p;
                   3774: 
                   3775:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   3776:     if (!p) {
                   3777:         pd = IO_MEM_UNASSIGNED;
                   3778:     } else {
                   3779:         pd = p->phys_offset;
                   3780:     }
1.1.1.6   root     3781: 
                   3782:     if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
1.1.1.4   root     3783:         !(pd & IO_MEM_ROMD)) {
1.1       root     3784:         /* I/O case */
                   3785:         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
1.1.1.7   root     3786:         if (p)
                   3787:             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
1.1       root     3788:         val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
                   3789:     } else {
                   3790:         /* RAM case */
1.1.1.10  root     3791:         ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
1.1       root     3792:             (addr & ~TARGET_PAGE_MASK);
                   3793:         val = ldl_p(ptr);
                   3794:     }
                   3795:     return val;
                   3796: }
                   3797: 
1.1.1.2   root     3798: /* warning: addr must be aligned */
                   3799: uint64_t ldq_phys(target_phys_addr_t addr)
                   3800: {
                   3801:     int io_index;
                   3802:     uint8_t *ptr;
                   3803:     uint64_t val;
                   3804:     unsigned long pd;
                   3805:     PhysPageDesc *p;
                   3806: 
                   3807:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   3808:     if (!p) {
                   3809:         pd = IO_MEM_UNASSIGNED;
                   3810:     } else {
                   3811:         pd = p->phys_offset;
                   3812:     }
1.1.1.6   root     3813: 
1.1.1.4   root     3814:     if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
                   3815:         !(pd & IO_MEM_ROMD)) {
1.1.1.2   root     3816:         /* I/O case */
                   3817:         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
1.1.1.7   root     3818:         if (p)
                   3819:             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
1.1.1.2   root     3820: #ifdef TARGET_WORDS_BIGENDIAN
                   3821:         val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
                   3822:         val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
                   3823: #else
                   3824:         val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
                   3825:         val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
                   3826: #endif
                   3827:     } else {
                   3828:         /* RAM case */
1.1.1.10  root     3829:         ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
1.1.1.2   root     3830:             (addr & ~TARGET_PAGE_MASK);
                   3831:         val = ldq_p(ptr);
                   3832:     }
                   3833:     return val;
                   3834: }
                   3835: 
                   3836: /* XXX: optimize */
                   3837: uint32_t ldub_phys(target_phys_addr_t addr)
                   3838: {
                   3839:     uint8_t val;
                   3840:     cpu_physical_memory_read(addr, &val, 1);
                   3841:     return val;
                   3842: }
                   3843: 
1.1.1.13! root     3844: /* warning: addr must be aligned */
1.1.1.2   root     3845: uint32_t lduw_phys(target_phys_addr_t addr)
                   3846: {
1.1.1.13! root     3847:     int io_index;
        !          3848:     uint8_t *ptr;
        !          3849:     uint64_t val;
        !          3850:     unsigned long pd;
        !          3851:     PhysPageDesc *p;
        !          3852: 
        !          3853:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
        !          3854:     if (!p) {
        !          3855:         pd = IO_MEM_UNASSIGNED;
        !          3856:     } else {
        !          3857:         pd = p->phys_offset;
        !          3858:     }
        !          3859: 
        !          3860:     if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
        !          3861:         !(pd & IO_MEM_ROMD)) {
        !          3862:         /* I/O case */
        !          3863:         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
        !          3864:         if (p)
        !          3865:             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
        !          3866:         val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
        !          3867:     } else {
        !          3868:         /* RAM case */
        !          3869:         ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
        !          3870:             (addr & ~TARGET_PAGE_MASK);
        !          3871:         val = lduw_p(ptr);
        !          3872:     }
        !          3873:     return val;
1.1.1.2   root     3874: }
                   3875: 
1.1       root     3876: /* warning: addr must be aligned. The ram page is not masked as dirty
                   3877:    and the code inside is not invalidated. It is useful if the dirty
                   3878:    bits are used to track modified PTEs */
                   3879: void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
                   3880: {
                   3881:     int io_index;
                   3882:     uint8_t *ptr;
                   3883:     unsigned long pd;
                   3884:     PhysPageDesc *p;
                   3885: 
                   3886:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   3887:     if (!p) {
                   3888:         pd = IO_MEM_UNASSIGNED;
                   3889:     } else {
                   3890:         pd = p->phys_offset;
                   3891:     }
1.1.1.6   root     3892: 
1.1       root     3893:     if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
                   3894:         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
1.1.1.7   root     3895:         if (p)
                   3896:             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
1.1       root     3897:         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
                   3898:     } else {
1.1.1.7   root     3899:         unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
1.1.1.10  root     3900:         ptr = qemu_get_ram_ptr(addr1);
1.1       root     3901:         stl_p(ptr, val);
1.1.1.7   root     3902: 
                   3903:         if (unlikely(in_migration)) {
                   3904:             if (!cpu_physical_memory_is_dirty(addr1)) {
                   3905:                 /* invalidate code */
                   3906:                 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
                   3907:                 /* set dirty bit */
1.1.1.13! root     3908:                 cpu_physical_memory_set_dirty_flags(
        !          3909:                     addr1, (0xff & ~CODE_DIRTY_FLAG));
1.1.1.7   root     3910:             }
                   3911:         }
1.1       root     3912:     }
                   3913: }
                   3914: 
1.1.1.6   root     3915: void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
                   3916: {
                   3917:     int io_index;
                   3918:     uint8_t *ptr;
                   3919:     unsigned long pd;
                   3920:     PhysPageDesc *p;
                   3921: 
                   3922:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   3923:     if (!p) {
                   3924:         pd = IO_MEM_UNASSIGNED;
                   3925:     } else {
                   3926:         pd = p->phys_offset;
                   3927:     }
                   3928: 
                   3929:     if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
                   3930:         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
1.1.1.7   root     3931:         if (p)
                   3932:             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
1.1.1.6   root     3933: #ifdef TARGET_WORDS_BIGENDIAN
                   3934:         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
                   3935:         io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
                   3936: #else
                   3937:         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
                   3938:         io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
                   3939: #endif
                   3940:     } else {
1.1.1.10  root     3941:         ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
1.1.1.6   root     3942:             (addr & ~TARGET_PAGE_MASK);
                   3943:         stq_p(ptr, val);
                   3944:     }
                   3945: }
                   3946: 
1.1       root     3947: /* warning: addr must be aligned */
                   3948: void stl_phys(target_phys_addr_t addr, uint32_t val)
                   3949: {
                   3950:     int io_index;
                   3951:     uint8_t *ptr;
                   3952:     unsigned long pd;
                   3953:     PhysPageDesc *p;
                   3954: 
                   3955:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   3956:     if (!p) {
                   3957:         pd = IO_MEM_UNASSIGNED;
                   3958:     } else {
                   3959:         pd = p->phys_offset;
                   3960:     }
1.1.1.6   root     3961: 
1.1       root     3962:     if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
                   3963:         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
1.1.1.7   root     3964:         if (p)
                   3965:             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
1.1       root     3966:         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
                   3967:     } else {
                   3968:         unsigned long addr1;
                   3969:         addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
                   3970:         /* RAM case */
1.1.1.10  root     3971:         ptr = qemu_get_ram_ptr(addr1);
1.1       root     3972:         stl_p(ptr, val);
                   3973:         if (!cpu_physical_memory_is_dirty(addr1)) {
                   3974:             /* invalidate code */
                   3975:             tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
                   3976:             /* set dirty bit */
1.1.1.13! root     3977:             cpu_physical_memory_set_dirty_flags(addr1,
        !          3978:                 (0xff & ~CODE_DIRTY_FLAG));
1.1       root     3979:         }
                   3980:     }
                   3981: }
                   3982: 
1.1.1.2   root     3983: /* XXX: optimize */
                   3984: void stb_phys(target_phys_addr_t addr, uint32_t val)
                   3985: {
                   3986:     uint8_t v = val;
                   3987:     cpu_physical_memory_write(addr, &v, 1);
                   3988: }
                   3989: 
1.1.1.13! root     3990: /* warning: addr must be aligned */
1.1.1.2   root     3991: void stw_phys(target_phys_addr_t addr, uint32_t val)
                   3992: {
1.1.1.13! root     3993:     int io_index;
        !          3994:     uint8_t *ptr;
        !          3995:     unsigned long pd;
        !          3996:     PhysPageDesc *p;
        !          3997: 
        !          3998:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
        !          3999:     if (!p) {
        !          4000:         pd = IO_MEM_UNASSIGNED;
        !          4001:     } else {
        !          4002:         pd = p->phys_offset;
        !          4003:     }
        !          4004: 
        !          4005:     if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
        !          4006:         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
        !          4007:         if (p)
        !          4008:             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
        !          4009:         io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
        !          4010:     } else {
        !          4011:         unsigned long addr1;
        !          4012:         addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
        !          4013:         /* RAM case */
        !          4014:         ptr = qemu_get_ram_ptr(addr1);
        !          4015:         stw_p(ptr, val);
        !          4016:         if (!cpu_physical_memory_is_dirty(addr1)) {
        !          4017:             /* invalidate code */
        !          4018:             tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
        !          4019:             /* set dirty bit */
        !          4020:             cpu_physical_memory_set_dirty_flags(addr1,
        !          4021:                 (0xff & ~CODE_DIRTY_FLAG));
        !          4022:         }
        !          4023:     }
1.1.1.2   root     4024: }
                   4025: 
                   4026: /* XXX: optimize */
                   4027: void stq_phys(target_phys_addr_t addr, uint64_t val)
                   4028: {
                   4029:     val = tswap64(val);
                   4030:     cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
                   4031: }
                   4032: 
1.1.1.10  root     4033: /* virtual memory access for debug (includes writing to ROM) */
1.1.1.6   root     4034: int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
1.1       root     4035:                         uint8_t *buf, int len, int is_write)
                   4036: {
                   4037:     int l;
1.1.1.6   root     4038:     target_phys_addr_t phys_addr;
                   4039:     target_ulong page;
1.1       root     4040: 
                   4041:     while (len > 0) {
                   4042:         page = addr & TARGET_PAGE_MASK;
                   4043:         phys_addr = cpu_get_phys_page_debug(env, page);
                   4044:         /* if no physical page mapped, return an error */
                   4045:         if (phys_addr == -1)
                   4046:             return -1;
                   4047:         l = (page + TARGET_PAGE_SIZE) - addr;
                   4048:         if (l > len)
                   4049:             l = len;
1.1.1.10  root     4050:         phys_addr += (addr & ~TARGET_PAGE_MASK);
                   4051:         if (is_write)
                   4052:             cpu_physical_memory_write_rom(phys_addr, buf, l);
                   4053:         else
                   4054:             cpu_physical_memory_rw(phys_addr, buf, l, is_write);
1.1       root     4055:         len -= l;
                   4056:         buf += l;
                   4057:         addr += l;
                   4058:     }
                   4059:     return 0;
                   4060: }
1.1.1.13! root     4061: #endif
1.1       root     4062: 
1.1.1.7   root     4063: /* in deterministic execution mode, instructions doing device I/Os
                   4064:    must be at the end of the TB */
                   4065: void cpu_io_recompile(CPUState *env, void *retaddr)
                   4066: {
                   4067:     TranslationBlock *tb;
                   4068:     uint32_t n, cflags;
                   4069:     target_ulong pc, cs_base;
                   4070:     uint64_t flags;
                   4071: 
                   4072:     tb = tb_find_pc((unsigned long)retaddr);
                   4073:     if (!tb) {
                   4074:         cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p", 
                   4075:                   retaddr);
                   4076:     }
                   4077:     n = env->icount_decr.u16.low + tb->icount;
                   4078:     cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
                   4079:     /* Calculate how many instructions had been executed before the fault
                   4080:        occurred.  */
                   4081:     n = n - env->icount_decr.u16.low;
                   4082:     /* Generate a new TB ending on the I/O insn.  */
                   4083:     n++;
                   4084:     /* On MIPS and SH, delay slot instructions can only be restarted if
                   4085:        they were already the first instruction in the TB.  If this is not
                   4086:        the first instruction in a TB then re-execute the preceding
                   4087:        branch.  */
                   4088: #if defined(TARGET_MIPS)
                   4089:     if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
                   4090:         env->active_tc.PC -= 4;
                   4091:         env->icount_decr.u16.low++;
                   4092:         env->hflags &= ~MIPS_HFLAG_BMASK;
                   4093:     }
                   4094: #elif defined(TARGET_SH4)
                   4095:     if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
                   4096:             && n > 1) {
                   4097:         env->pc -= 2;
                   4098:         env->icount_decr.u16.low++;
                   4099:         env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
                   4100:     }
                   4101: #endif
                   4102:     /* This should never happen.  */
                   4103:     if (n > CF_COUNT_MASK)
                   4104:         cpu_abort(env, "TB too big during recompile");
                   4105: 
                   4106:     cflags = n | CF_LAST_IO;
                   4107:     pc = tb->pc;
                   4108:     cs_base = tb->cs_base;
                   4109:     flags = tb->flags;
                   4110:     tb_phys_invalidate(tb, -1);
                   4111:     /* FIXME: In theory this could raise an exception.  In practice
                   4112:        we have already translated the block once so it's probably ok.  */
                   4113:     tb_gen_code(env, pc, cs_base, flags, cflags);
                   4114:     /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
                   4115:        the first in the TB) then we end up generating a whole new TB and
                   4116:        repeating the fault, which is horribly inefficient.
                   4117:        Better would be to execute just this insn uncached, or generate a
                   4118:        second new TB.  */
                   4119:     cpu_resume_from_signal(env, NULL);
                   4120: }
                   4121: 
1.1.1.13! root     4122: #if !defined(CONFIG_USER_ONLY)
        !          4123: 
1.1       root     4124: void dump_exec_info(FILE *f,
                   4125:                     int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
                   4126: {
                   4127:     int i, target_code_size, max_target_code_size;
                   4128:     int direct_jmp_count, direct_jmp2_count, cross_page;
                   4129:     TranslationBlock *tb;
1.1.1.6   root     4130: 
1.1       root     4131:     target_code_size = 0;
                   4132:     max_target_code_size = 0;
                   4133:     cross_page = 0;
                   4134:     direct_jmp_count = 0;
                   4135:     direct_jmp2_count = 0;
                   4136:     for(i = 0; i < nb_tbs; i++) {
                   4137:         tb = &tbs[i];
                   4138:         target_code_size += tb->size;
                   4139:         if (tb->size > max_target_code_size)
                   4140:             max_target_code_size = tb->size;
                   4141:         if (tb->page_addr[1] != -1)
                   4142:             cross_page++;
                   4143:         if (tb->tb_next_offset[0] != 0xffff) {
                   4144:             direct_jmp_count++;
                   4145:             if (tb->tb_next_offset[1] != 0xffff) {
                   4146:                 direct_jmp2_count++;
                   4147:             }
                   4148:         }
                   4149:     }
                   4150:     /* XXX: avoid using doubles ? */
1.1.1.7   root     4151:     cpu_fprintf(f, "Translation buffer state:\n");
                   4152:     cpu_fprintf(f, "gen code size       %ld/%ld\n",
                   4153:                 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
                   4154:     cpu_fprintf(f, "TB count            %d/%d\n", 
                   4155:                 nb_tbs, code_gen_max_blocks);
1.1.1.6   root     4156:     cpu_fprintf(f, "TB avg target size  %d max=%d bytes\n",
1.1       root     4157:                 nb_tbs ? target_code_size / nb_tbs : 0,
                   4158:                 max_target_code_size);
1.1.1.6   root     4159:     cpu_fprintf(f, "TB avg host size    %d bytes (expansion ratio: %0.1f)\n",
1.1       root     4160:                 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
                   4161:                 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
1.1.1.6   root     4162:     cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
                   4163:             cross_page,
1.1       root     4164:             nb_tbs ? (cross_page * 100) / nb_tbs : 0);
                   4165:     cpu_fprintf(f, "direct jump count   %d (%d%%) (2 jumps=%d %d%%)\n",
1.1.1.6   root     4166:                 direct_jmp_count,
1.1       root     4167:                 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
                   4168:                 direct_jmp2_count,
                   4169:                 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
1.1.1.7   root     4170:     cpu_fprintf(f, "\nStatistics:\n");
1.1       root     4171:     cpu_fprintf(f, "TB flush count      %d\n", tb_flush_count);
                   4172:     cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
                   4173:     cpu_fprintf(f, "TLB flush count     %d\n", tlb_flush_count);
1.1.1.7   root     4174:     tcg_dump_info(f, cpu_fprintf);
1.1       root     4175: }
                   4176: 
                   4177: #define MMUSUFFIX _cmmu
                   4178: #define GETPC() NULL
                   4179: #define env cpu_single_env
                   4180: #define SOFTMMU_CODE_ACCESS
                   4181: 
                   4182: #define SHIFT 0
                   4183: #include "softmmu_template.h"
                   4184: 
                   4185: #define SHIFT 1
                   4186: #include "softmmu_template.h"
                   4187: 
                   4188: #define SHIFT 2
                   4189: #include "softmmu_template.h"
                   4190: 
                   4191: #define SHIFT 3
                   4192: #include "softmmu_template.h"
                   4193: 
                   4194: #undef env
                   4195: 
                   4196: #endif

unix.superglobalmegacorp.com