Annotation of qemu/exec.c, revision 1.1.1.16

1.1       root        1: /*
                      2:  *  virtual page mapping and translated block handling
1.1.1.6   root        3:  *
1.1       root        4:  *  Copyright (c) 2003 Fabrice Bellard
                      5:  *
                      6:  * This library is free software; you can redistribute it and/or
                      7:  * modify it under the terms of the GNU Lesser General Public
                      8:  * License as published by the Free Software Foundation; either
                      9:  * version 2 of the License, or (at your option) any later version.
                     10:  *
                     11:  * This library is distributed in the hope that it will be useful,
                     12:  * but WITHOUT ANY WARRANTY; without even the implied warranty of
                     13:  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
                     14:  * Lesser General Public License for more details.
                     15:  *
                     16:  * You should have received a copy of the GNU Lesser General Public
1.1.1.10  root       17:  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
1.1       root       18:  */
                     19: #include "config.h"
                     20: #ifdef _WIN32
                     21: #include <windows.h>
                     22: #else
                     23: #include <sys/types.h>
                     24: #include <sys/mman.h>
                     25: #endif
                     26: 
1.1.1.14  root       27: #include "qemu-common.h"
1.1       root       28: #include "cpu.h"
1.1.1.7   root       29: #include "tcg.h"
                     30: #include "hw/hw.h"
1.1.1.13  root       31: #include "hw/qdev.h"
1.1.1.7   root       32: #include "osdep.h"
                     33: #include "kvm.h"
1.1.1.15  root       34: #include "hw/xen.h"
1.1.1.13  root       35: #include "qemu-timer.h"
1.1.1.16! root       36: #include "memory.h"
        !            37: #include "exec-memory.h"
1.1.1.3   root       38: #if defined(CONFIG_USER_ONLY)
                     39: #include <qemu.h>
1.1.1.13  root       40: #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
                     41: #include <sys/param.h>
                     42: #if __FreeBSD_version >= 700104
                     43: #define HAVE_KINFO_GETVMMAP
                     44: #define sigqueue sigqueue_freebsd  /* avoid redefinition */
                     45: #include <sys/time.h>
                     46: #include <sys/proc.h>
                     47: #include <machine/profile.h>
                     48: #define _KERNEL
                     49: #include <sys/user.h>
                     50: #undef _KERNEL
                     51: #undef sigqueue
                     52: #include <libutil.h>
                     53: #endif
                     54: #endif
1.1.1.15  root       55: #else /* !CONFIG_USER_ONLY */
                     56: #include "xen-mapcache.h"
                     57: #include "trace.h"
1.1.1.3   root       58: #endif
1.1       root       59: 
                     60: //#define DEBUG_TB_INVALIDATE
                     61: //#define DEBUG_FLUSH
                     62: //#define DEBUG_TLB
1.1.1.5   root       63: //#define DEBUG_UNASSIGNED
1.1       root       64: 
                     65: /* make various TB consistency checks */
1.1.1.6   root       66: //#define DEBUG_TB_CHECK
                     67: //#define DEBUG_TLB_CHECK
                     68: 
                     69: //#define DEBUG_IOPORT
                     70: //#define DEBUG_SUBPAGE
1.1       root       71: 
1.1.1.3   root       72: #if !defined(CONFIG_USER_ONLY)
                     73: /* TB consistency checks only implemented for usermode emulation.  */
                     74: #undef DEBUG_TB_CHECK
                     75: #endif
                     76: 
1.1       root       77: #define SMC_BITMAP_USE_THRESHOLD 10
                     78: 
1.1.1.7   root       79: static TranslationBlock *tbs;
1.1.1.13  root       80: static int code_gen_max_blocks;
1.1       root       81: TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
1.1.1.7   root       82: static int nb_tbs;
1.1       root       83: /* any access to the tbs or the page table must use this lock */
                     84: spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
                     85: 
1.1.1.7   root       86: #if defined(__arm__) || defined(__sparc_v9__)
                     87: /* The prologue must be reachable with a direct jump. ARM and Sparc64
                     88:  have limited branch ranges (possibly also PPC) so place it in a
                     89:  section close to code segment. */
                     90: #define code_gen_section                                \
                     91:     __attribute__((__section__(".gen_code")))           \
                     92:     __attribute__((aligned (32)))
1.1.1.10  root       93: #elif defined(_WIN32)
                     94: /* Maximum alignment for Win32 is 16. */
                     95: #define code_gen_section                                \
                     96:     __attribute__((aligned (16)))
1.1.1.7   root       97: #else
                     98: #define code_gen_section                                \
                     99:     __attribute__((aligned (32)))
                    100: #endif
                    101: 
                    102: uint8_t code_gen_prologue[1024] code_gen_section;
                    103: static uint8_t *code_gen_buffer;
                    104: static unsigned long code_gen_buffer_size;
                    105: /* threshold to flush the translated code buffer */
                    106: static unsigned long code_gen_buffer_max_size;
1.1.1.13  root      107: static uint8_t *code_gen_ptr;
1.1       root      108: 
1.1.1.7   root      109: #if !defined(CONFIG_USER_ONLY)
1.1       root      110: int phys_ram_fd;
1.1.1.7   root      111: static int in_migration;
1.1.1.10  root      112: 
1.1.1.16! root      113: RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
        !           114: 
        !           115: static MemoryRegion *system_memory;
        !           116: static MemoryRegion *system_io;
        !           117: 
1.1.1.7   root      118: #endif
1.1       root      119: 
1.1.1.2   root      120: CPUState *first_cpu;
                    121: /* current CPU in the current thread. It is only valid inside
                    122:    cpu_exec() */
1.1.1.16! root      123: DEFINE_TLS(CPUState *,cpu_single_env);
1.1.1.7   root      124: /* 0 = Do not count executed instructions.
                    125:    1 = Precise instruction counting.
                    126:    2 = Adaptive rate instruction counting.  */
                    127: int use_icount = 0;
1.1.1.2   root      128: 
1.1       root      129: typedef struct PageDesc {
                    130:     /* list of TBs intersecting this ram page */
                    131:     TranslationBlock *first_tb;
                    132:     /* in order to optimize self modifying code, we count the number
                    133:        of lookups we do to a given page to use a bitmap */
                    134:     unsigned int code_write_count;
                    135:     uint8_t *code_bitmap;
                    136: #if defined(CONFIG_USER_ONLY)
                    137:     unsigned long flags;
                    138: #endif
                    139: } PageDesc;
                    140: 
1.1.1.13  root      141: /* In system mode we want L1_MAP to be based on ram offsets,
                    142:    while in user mode we want it to be based on virtual addresses.  */
                    143: #if !defined(CONFIG_USER_ONLY)
                    144: #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
                    145: # define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS
                    146: #else
                    147: # define L1_MAP_ADDR_SPACE_BITS  TARGET_PHYS_ADDR_SPACE_BITS
                    148: #endif
                    149: #else
                    150: # define L1_MAP_ADDR_SPACE_BITS  TARGET_VIRT_ADDR_SPACE_BITS
                    151: #endif
1.1       root      152: 
1.1.1.13  root      153: /* Size of the L2 (and L3, etc) page tables.  */
1.1       root      154: #define L2_BITS 10
1.1.1.13  root      155: #define L2_SIZE (1 << L2_BITS)
                    156: 
                    157: /* The bits remaining after N lower levels of page tables.  */
                    158: #define P_L1_BITS_REM \
                    159:     ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
                    160: #define V_L1_BITS_REM \
                    161:     ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
                    162: 
                    163: /* Size of the L1 page table.  Avoid silly small sizes.  */
                    164: #if P_L1_BITS_REM < 4
                    165: #define P_L1_BITS  (P_L1_BITS_REM + L2_BITS)
1.1.1.6   root      166: #else
1.1.1.13  root      167: #define P_L1_BITS  P_L1_BITS_REM
1.1.1.6   root      168: #endif
1.1       root      169: 
1.1.1.13  root      170: #if V_L1_BITS_REM < 4
                    171: #define V_L1_BITS  (V_L1_BITS_REM + L2_BITS)
                    172: #else
                    173: #define V_L1_BITS  V_L1_BITS_REM
                    174: #endif
                    175: 
                    176: #define P_L1_SIZE  ((target_phys_addr_t)1 << P_L1_BITS)
                    177: #define V_L1_SIZE  ((target_ulong)1 << V_L1_BITS)
                    178: 
                    179: #define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
                    180: #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
1.1       root      181: 
                    182: unsigned long qemu_real_host_page_size;
                    183: unsigned long qemu_host_page_size;
                    184: unsigned long qemu_host_page_mask;
                    185: 
1.1.1.13  root      186: /* This is a multi-level map on the virtual address space.
                    187:    The bottom level has pointers to PageDesc.  */
                    188: static void *l1_map[V_L1_SIZE];
1.1.1.7   root      189: 
                    190: #if !defined(CONFIG_USER_ONLY)
1.1.1.13  root      191: typedef struct PhysPageDesc {
                    192:     /* offset in host memory of the page + io_index in the low bits */
                    193:     ram_addr_t phys_offset;
                    194:     ram_addr_t region_offset;
                    195: } PhysPageDesc;
                    196: 
                    197: /* This is a multi-level map on the physical address space.
                    198:    The bottom level has pointers to PhysPageDesc.  */
                    199: static void *l1_phys_map[P_L1_SIZE];
                    200: 
1.1.1.7   root      201: static void io_mem_init(void);
1.1.1.16! root      202: static void memory_map_init(void);
1.1       root      203: 
                    204: /* io memory support */
                    205: CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
                    206: CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
                    207: void *io_mem_opaque[IO_MEM_NB_ENTRIES];
1.1.1.10  root      208: static char io_mem_used[IO_MEM_NB_ENTRIES];
1.1.1.6   root      209: static int io_mem_watch;
                    210: #endif
1.1       root      211: 
                    212: /* log support */
1.1.1.13  root      213: #ifdef WIN32
                    214: static const char *logfilename = "qemu.log";
                    215: #else
1.1.1.7   root      216: static const char *logfilename = "/tmp/qemu.log";
1.1.1.13  root      217: #endif
1.1       root      218: FILE *logfile;
                    219: int loglevel;
1.1.1.6   root      220: static int log_append = 0;
1.1       root      221: 
                    222: /* statistics */
1.1.1.13  root      223: #if !defined(CONFIG_USER_ONLY)
1.1       root      224: static int tlb_flush_count;
1.1.1.13  root      225: #endif
1.1       root      226: static int tb_flush_count;
                    227: static int tb_phys_invalidate_count;
                    228: 
1.1.1.7   root      229: #ifdef _WIN32
                    230: static void map_exec(void *addr, long size)
                    231: {
                    232:     DWORD old_protect;
                    233:     VirtualProtect(addr, size,
                    234:                    PAGE_EXECUTE_READWRITE, &old_protect);
                    235:     
                    236: }
                    237: #else
                    238: static void map_exec(void *addr, long size)
                    239: {
                    240:     unsigned long start, end, page_size;
                    241:     
                    242:     page_size = getpagesize();
                    243:     start = (unsigned long)addr;
                    244:     start &= ~(page_size - 1);
                    245:     
                    246:     end = (unsigned long)addr + size;
                    247:     end += page_size - 1;
                    248:     end &= ~(page_size - 1);
                    249:     
                    250:     mprotect((void *)start, end - start,
                    251:              PROT_READ | PROT_WRITE | PROT_EXEC);
                    252: }
                    253: #endif
                    254: 
1.1       root      255: static void page_init(void)
                    256: {
                    257:     /* NOTE: we can always suppose that qemu_host_page_size >=
                    258:        TARGET_PAGE_SIZE */
                    259: #ifdef _WIN32
                    260:     {
                    261:         SYSTEM_INFO system_info;
1.1.1.6   root      262: 
1.1       root      263:         GetSystemInfo(&system_info);
                    264:         qemu_real_host_page_size = system_info.dwPageSize;
                    265:     }
                    266: #else
                    267:     qemu_real_host_page_size = getpagesize();
                    268: #endif
                    269:     if (qemu_host_page_size == 0)
                    270:         qemu_host_page_size = qemu_real_host_page_size;
                    271:     if (qemu_host_page_size < TARGET_PAGE_SIZE)
                    272:         qemu_host_page_size = TARGET_PAGE_SIZE;
                    273:     qemu_host_page_mask = ~(qemu_host_page_size - 1);
1.1.1.6   root      274: 
1.1.1.13  root      275: #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
1.1.1.6   root      276:     {
1.1.1.13  root      277: #ifdef HAVE_KINFO_GETVMMAP
                    278:         struct kinfo_vmentry *freep;
                    279:         int i, cnt;
                    280: 
                    281:         freep = kinfo_getvmmap(getpid(), &cnt);
                    282:         if (freep) {
                    283:             mmap_lock();
                    284:             for (i = 0; i < cnt; i++) {
                    285:                 unsigned long startaddr, endaddr;
                    286: 
                    287:                 startaddr = freep[i].kve_start;
                    288:                 endaddr = freep[i].kve_end;
                    289:                 if (h2g_valid(startaddr)) {
                    290:                     startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
                    291: 
                    292:                     if (h2g_valid(endaddr)) {
                    293:                         endaddr = h2g(endaddr);
                    294:                         page_set_flags(startaddr, endaddr, PAGE_RESERVED);
                    295:                     } else {
                    296: #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
                    297:                         endaddr = ~0ul;
                    298:                         page_set_flags(startaddr, endaddr, PAGE_RESERVED);
                    299: #endif
                    300:                     }
                    301:                 }
                    302:             }
                    303:             free(freep);
                    304:             mmap_unlock();
                    305:         }
                    306: #else
1.1.1.6   root      307:         FILE *f;
                    308: 
1.1.1.7   root      309:         last_brk = (unsigned long)sbrk(0);
1.1.1.13  root      310: 
                    311:         f = fopen("/compat/linux/proc/self/maps", "r");
1.1.1.6   root      312:         if (f) {
1.1.1.13  root      313:             mmap_lock();
                    314: 
1.1.1.6   root      315:             do {
1.1.1.13  root      316:                 unsigned long startaddr, endaddr;
                    317:                 int n;
                    318: 
                    319:                 n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
                    320: 
                    321:                 if (n == 2 && h2g_valid(startaddr)) {
                    322:                     startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
                    323: 
                    324:                     if (h2g_valid(endaddr)) {
                    325:                         endaddr = h2g(endaddr);
                    326:                     } else {
                    327:                         endaddr = ~0ul;
                    328:                     }
                    329:                     page_set_flags(startaddr, endaddr, PAGE_RESERVED);
1.1.1.6   root      330:                 }
                    331:             } while (!feof(f));
1.1.1.13  root      332: 
1.1.1.6   root      333:             fclose(f);
1.1.1.13  root      334:             mmap_unlock();
1.1.1.6   root      335:         }
                    336: #endif
1.1.1.13  root      337:     }
1.1.1.7   root      338: #endif
                    339: }
                    340: 
1.1.1.13  root      341: static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
1.1       root      342: {
1.1.1.13  root      343:     PageDesc *pd;
                    344:     void **lp;
                    345:     int i;
1.1       root      346: 
1.1.1.7   root      347: #if defined(CONFIG_USER_ONLY)
1.1.1.16! root      348:     /* We can't use g_malloc because it may recurse into a locked mutex. */
1.1.1.13  root      349: # define ALLOC(P, SIZE)                                 \
                    350:     do {                                                \
                    351:         P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,    \
                    352:                  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);   \
                    353:     } while (0)
1.1.1.7   root      354: #else
1.1.1.13  root      355: # define ALLOC(P, SIZE) \
1.1.1.16! root      356:     do { P = g_malloc0(SIZE); } while (0)
1.1.1.7   root      357: #endif
1.1.1.13  root      358: 
                    359:     /* Level 1.  Always allocated.  */
                    360:     lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
                    361: 
                    362:     /* Level 2..N-1.  */
                    363:     for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
                    364:         void **p = *lp;
                    365: 
                    366:         if (p == NULL) {
                    367:             if (!alloc) {
                    368:                 return NULL;
                    369:             }
                    370:             ALLOC(p, sizeof(void *) * L2_SIZE);
                    371:             *lp = p;
                    372:         }
                    373: 
                    374:         lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
                    375:     }
                    376: 
                    377:     pd = *lp;
                    378:     if (pd == NULL) {
                    379:         if (!alloc) {
                    380:             return NULL;
                    381:         }
                    382:         ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
                    383:         *lp = pd;
1.1       root      384:     }
1.1.1.13  root      385: 
                    386: #undef ALLOC
                    387: 
                    388:     return pd + (index & (L2_SIZE - 1));
1.1       root      389: }
                    390: 
1.1.1.13  root      391: static inline PageDesc *page_find(tb_page_addr_t index)
1.1       root      392: {
1.1.1.13  root      393:     return page_find_alloc(index, 0);
1.1       root      394: }
                    395: 
1.1.1.13  root      396: #if !defined(CONFIG_USER_ONLY)
1.1       root      397: static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
                    398: {
1.1.1.3   root      399:     PhysPageDesc *pd;
1.1.1.13  root      400:     void **lp;
                    401:     int i;
1.1       root      402: 
1.1.1.13  root      403:     /* Level 1.  Always allocated.  */
                    404:     lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
1.1       root      405: 
1.1.1.13  root      406:     /* Level 2..N-1.  */
                    407:     for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
                    408:         void **p = *lp;
                    409:         if (p == NULL) {
                    410:             if (!alloc) {
                    411:                 return NULL;
                    412:             }
1.1.1.16! root      413:             *lp = p = g_malloc0(sizeof(void *) * L2_SIZE);
1.1.1.13  root      414:         }
                    415:         lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
1.1       root      416:     }
1.1.1.13  root      417: 
1.1.1.3   root      418:     pd = *lp;
1.1.1.13  root      419:     if (pd == NULL) {
1.1.1.3   root      420:         int i;
1.1.1.13  root      421: 
                    422:         if (!alloc) {
1.1       root      423:             return NULL;
1.1.1.13  root      424:         }
                    425: 
1.1.1.16! root      426:         *lp = pd = g_malloc(sizeof(PhysPageDesc) * L2_SIZE);
1.1.1.13  root      427: 
1.1.1.7   root      428:         for (i = 0; i < L2_SIZE; i++) {
1.1.1.13  root      429:             pd[i].phys_offset = IO_MEM_UNASSIGNED;
                    430:             pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
1.1.1.7   root      431:         }
1.1       root      432:     }
1.1.1.13  root      433: 
                    434:     return pd + (index & (L2_SIZE - 1));
1.1       root      435: }
                    436: 
                    437: static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
                    438: {
                    439:     return phys_page_find_alloc(index, 0);
                    440: }
                    441: 
1.1.1.2   root      442: static void tlb_protect_code(ram_addr_t ram_addr);
1.1.1.6   root      443: static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1.1       root      444:                                     target_ulong vaddr);
1.1.1.7   root      445: #define mmap_lock() do { } while(0)
                    446: #define mmap_unlock() do { } while(0)
                    447: #endif
                    448: 
                    449: #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
                    450: 
                    451: #if defined(CONFIG_USER_ONLY)
1.1.1.10  root      452: /* Currently it is not recommended to allocate big chunks of data in
1.1.1.7   root      453:    user mode. It will change when a dedicated libc will be used */
                    454: #define USE_STATIC_CODE_GEN_BUFFER
                    455: #endif
                    456: 
                    457: #ifdef USE_STATIC_CODE_GEN_BUFFER
1.1.1.13  root      458: static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
                    459:                __attribute__((aligned (CODE_GEN_ALIGN)));
1.1.1.7   root      460: #endif
                    461: 
                    462: static void code_gen_alloc(unsigned long tb_size)
                    463: {
                    464: #ifdef USE_STATIC_CODE_GEN_BUFFER
                    465:     code_gen_buffer = static_code_gen_buffer;
                    466:     code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
                    467:     map_exec(code_gen_buffer, code_gen_buffer_size);
                    468: #else
                    469:     code_gen_buffer_size = tb_size;
                    470:     if (code_gen_buffer_size == 0) {
                    471: #if defined(CONFIG_USER_ONLY)
                    472:         code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
                    473: #else
1.1.1.10  root      474:         /* XXX: needs adjustments */
                    475:         code_gen_buffer_size = (unsigned long)(ram_size / 4);
1.1.1.7   root      476: #endif
                    477:     }
                    478:     if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
                    479:         code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
                    480:     /* The code gen buffer location may have constraints depending on
                    481:        the host cpu and OS */
                    482: #if defined(__linux__) 
                    483:     {
                    484:         int flags;
                    485:         void *start = NULL;
                    486: 
                    487:         flags = MAP_PRIVATE | MAP_ANONYMOUS;
                    488: #if defined(__x86_64__)
                    489:         flags |= MAP_32BIT;
                    490:         /* Cannot map more than that */
                    491:         if (code_gen_buffer_size > (800 * 1024 * 1024))
                    492:             code_gen_buffer_size = (800 * 1024 * 1024);
                    493: #elif defined(__sparc_v9__)
                    494:         // Map the buffer below 2G, so we can use direct calls and branches
                    495:         flags |= MAP_FIXED;
                    496:         start = (void *) 0x60000000UL;
                    497:         if (code_gen_buffer_size > (512 * 1024 * 1024))
                    498:             code_gen_buffer_size = (512 * 1024 * 1024);
                    499: #elif defined(__arm__)
                    500:         /* Map the buffer below 32M, so we can use direct calls and branches */
                    501:         flags |= MAP_FIXED;
                    502:         start = (void *) 0x01000000UL;
                    503:         if (code_gen_buffer_size > 16 * 1024 * 1024)
                    504:             code_gen_buffer_size = 16 * 1024 * 1024;
1.1.1.13  root      505: #elif defined(__s390x__)
                    506:         /* Map the buffer so that we can use direct calls and branches.  */
                    507:         /* We have a +- 4GB range on the branches; leave some slop.  */
                    508:         if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {
                    509:             code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;
                    510:         }
                    511:         start = (void *)0x90000000UL;
1.1.1.7   root      512: #endif
                    513:         code_gen_buffer = mmap(start, code_gen_buffer_size,
                    514:                                PROT_WRITE | PROT_READ | PROT_EXEC,
                    515:                                flags, -1, 0);
                    516:         if (code_gen_buffer == MAP_FAILED) {
                    517:             fprintf(stderr, "Could not allocate dynamic translator buffer\n");
                    518:             exit(1);
                    519:         }
                    520:     }
1.1.1.14  root      521: #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
1.1.1.16! root      522:     || defined(__DragonFly__) || defined(__OpenBSD__) \
        !           523:     || defined(__NetBSD__)
1.1.1.7   root      524:     {
                    525:         int flags;
                    526:         void *addr = NULL;
                    527:         flags = MAP_PRIVATE | MAP_ANONYMOUS;
                    528: #if defined(__x86_64__)
                    529:         /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
                    530:          * 0x40000000 is free */
                    531:         flags |= MAP_FIXED;
                    532:         addr = (void *)0x40000000;
                    533:         /* Cannot map more than that */
                    534:         if (code_gen_buffer_size > (800 * 1024 * 1024))
                    535:             code_gen_buffer_size = (800 * 1024 * 1024);
1.1.1.14  root      536: #elif defined(__sparc_v9__)
                    537:         // Map the buffer below 2G, so we can use direct calls and branches
                    538:         flags |= MAP_FIXED;
                    539:         addr = (void *) 0x60000000UL;
                    540:         if (code_gen_buffer_size > (512 * 1024 * 1024)) {
                    541:             code_gen_buffer_size = (512 * 1024 * 1024);
                    542:         }
1.1.1.7   root      543: #endif
                    544:         code_gen_buffer = mmap(addr, code_gen_buffer_size,
                    545:                                PROT_WRITE | PROT_READ | PROT_EXEC, 
                    546:                                flags, -1, 0);
                    547:         if (code_gen_buffer == MAP_FAILED) {
                    548:             fprintf(stderr, "Could not allocate dynamic translator buffer\n");
                    549:             exit(1);
                    550:         }
                    551:     }
                    552: #else
1.1.1.16! root      553:     code_gen_buffer = g_malloc(code_gen_buffer_size);
1.1.1.7   root      554:     map_exec(code_gen_buffer, code_gen_buffer_size);
                    555: #endif
                    556: #endif /* !USE_STATIC_CODE_GEN_BUFFER */
                    557:     map_exec(code_gen_prologue, sizeof(code_gen_prologue));
1.1.1.15  root      558:     code_gen_buffer_max_size = code_gen_buffer_size -
                    559:         (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
1.1.1.7   root      560:     code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
1.1.1.16! root      561:     tbs = g_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
1.1.1.7   root      562: }
                    563: 
                    564: /* Must be called before using the QEMU cpus. 'tb_size' is the size
                    565:    (in bytes) allocated to the translation buffer. Zero means default
                    566:    size. */
1.1.1.16! root      567: void tcg_exec_init(unsigned long tb_size)
1.1.1.7   root      568: {
                    569:     cpu_gen_init();
                    570:     code_gen_alloc(tb_size);
                    571:     code_gen_ptr = code_gen_buffer;
                    572:     page_init();
1.1.1.13  root      573: #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
                    574:     /* There's no guest base to take into account, so go ahead and
                    575:        initialize the prologue now.  */
                    576:     tcg_prologue_init(&tcg_ctx);
                    577: #endif
1.1.1.7   root      578: }
                    579: 
1.1.1.16! root      580: bool tcg_enabled(void)
        !           581: {
        !           582:     return code_gen_buffer != NULL;
        !           583: }
        !           584: 
        !           585: void cpu_exec_init_all(void)
        !           586: {
        !           587: #if !defined(CONFIG_USER_ONLY)
        !           588:     memory_map_init();
        !           589:     io_mem_init();
        !           590: #endif
        !           591: }
        !           592: 
1.1.1.7   root      593: #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
                    594: 
1.1.1.11  root      595: static int cpu_common_post_load(void *opaque, int version_id)
                    596: {
                    597:     CPUState *env = opaque;
1.1.1.7   root      598: 
1.1.1.10  root      599:     /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
                    600:        version_id is increased. */
                    601:     env->interrupt_request &= ~0x01;
1.1.1.7   root      602:     tlb_flush(env, 1);
                    603: 
                    604:     return 0;
                    605: }
1.1.1.11  root      606: 
                    607: static const VMStateDescription vmstate_cpu_common = {
                    608:     .name = "cpu_common",
                    609:     .version_id = 1,
                    610:     .minimum_version_id = 1,
                    611:     .minimum_version_id_old = 1,
                    612:     .post_load = cpu_common_post_load,
                    613:     .fields      = (VMStateField []) {
                    614:         VMSTATE_UINT32(halted, CPUState),
                    615:         VMSTATE_UINT32(interrupt_request, CPUState),
                    616:         VMSTATE_END_OF_LIST()
                    617:     }
                    618: };
1.1       root      619: #endif
                    620: 
1.1.1.10  root      621: CPUState *qemu_get_cpu(int cpu)
                    622: {
                    623:     CPUState *env = first_cpu;
                    624: 
                    625:     while (env) {
                    626:         if (env->cpu_index == cpu)
                    627:             break;
                    628:         env = env->next_cpu;
                    629:     }
                    630: 
                    631:     return env;
                    632: }
                    633: 
1.1.1.2   root      634: void cpu_exec_init(CPUState *env)
1.1       root      635: {
1.1.1.2   root      636:     CPUState **penv;
                    637:     int cpu_index;
1.1       root      638: 
1.1.1.10  root      639: #if defined(CONFIG_USER_ONLY)
                    640:     cpu_list_lock();
                    641: #endif
1.1.1.2   root      642:     env->next_cpu = NULL;
                    643:     penv = &first_cpu;
                    644:     cpu_index = 0;
                    645:     while (*penv != NULL) {
1.1.1.10  root      646:         penv = &(*penv)->next_cpu;
1.1.1.2   root      647:         cpu_index++;
                    648:     }
                    649:     env->cpu_index = cpu_index;
1.1.1.10  root      650:     env->numa_node = 0;
1.1.1.11  root      651:     QTAILQ_INIT(&env->breakpoints);
                    652:     QTAILQ_INIT(&env->watchpoints);
1.1.1.15  root      653: #ifndef CONFIG_USER_ONLY
                    654:     env->thread_id = qemu_get_thread_id();
                    655: #endif
1.1.1.2   root      656:     *penv = env;
1.1.1.10  root      657: #if defined(CONFIG_USER_ONLY)
                    658:     cpu_list_unlock();
                    659: #endif
1.1.1.7   root      660: #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
1.1.1.13  root      661:     vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
                    662:     register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
1.1.1.7   root      663:                     cpu_save, cpu_load, env);
                    664: #endif
1.1       root      665: }
                    666: 
1.1.1.15  root      667: /* Allocate a new translation block. Flush the translation buffer if
                    668:    too many translation blocks or too much generated code. */
                    669: static TranslationBlock *tb_alloc(target_ulong pc)
                    670: {
                    671:     TranslationBlock *tb;
                    672: 
                    673:     if (nb_tbs >= code_gen_max_blocks ||
                    674:         (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
                    675:         return NULL;
                    676:     tb = &tbs[nb_tbs++];
                    677:     tb->pc = pc;
                    678:     tb->cflags = 0;
                    679:     return tb;
                    680: }
                    681: 
                    682: void tb_free(TranslationBlock *tb)
                    683: {
                    684:     /* In practice this is mostly used for single use temporary TB
                    685:        Ignore the hard cases and just back up if this TB happens to
                    686:        be the last one generated.  */
                    687:     if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
                    688:         code_gen_ptr = tb->tc_ptr;
                    689:         nb_tbs--;
                    690:     }
                    691: }
                    692: 
1.1       root      693: static inline void invalidate_page_bitmap(PageDesc *p)
                    694: {
                    695:     if (p->code_bitmap) {
1.1.1.16! root      696:         g_free(p->code_bitmap);
1.1       root      697:         p->code_bitmap = NULL;
                    698:     }
                    699:     p->code_write_count = 0;
                    700: }
                    701: 
1.1.1.13  root      702: /* Set to NULL all the 'first_tb' fields in all PageDescs. */
                    703: 
                    704: static void page_flush_tb_1 (int level, void **lp)
1.1       root      705: {
1.1.1.13  root      706:     int i;
1.1       root      707: 
1.1.1.13  root      708:     if (*lp == NULL) {
                    709:         return;
                    710:     }
                    711:     if (level == 0) {
                    712:         PageDesc *pd = *lp;
                    713:         for (i = 0; i < L2_SIZE; ++i) {
                    714:             pd[i].first_tb = NULL;
                    715:             invalidate_page_bitmap(pd + i);
                    716:         }
                    717:     } else {
                    718:         void **pp = *lp;
                    719:         for (i = 0; i < L2_SIZE; ++i) {
                    720:             page_flush_tb_1 (level - 1, pp + i);
1.1       root      721:         }
                    722:     }
                    723: }
                    724: 
1.1.1.13  root      725: static void page_flush_tb(void)
                    726: {
                    727:     int i;
                    728:     for (i = 0; i < V_L1_SIZE; i++) {
                    729:         page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
                    730:     }
                    731: }
                    732: 
1.1       root      733: /* flush all the translation blocks */
                    734: /* XXX: tb_flush is currently not thread safe */
1.1.1.2   root      735: void tb_flush(CPUState *env1)
1.1       root      736: {
1.1.1.2   root      737:     CPUState *env;
1.1       root      738: #if defined(DEBUG_FLUSH)
1.1.1.6   root      739:     printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
                    740:            (unsigned long)(code_gen_ptr - code_gen_buffer),
                    741:            nb_tbs, nb_tbs > 0 ?
                    742:            ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
1.1       root      743: #endif
1.1.1.7   root      744:     if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
                    745:         cpu_abort(env1, "Internal error: code buffer overflow\n");
                    746: 
1.1       root      747:     nb_tbs = 0;
1.1.1.6   root      748: 
1.1.1.2   root      749:     for(env = first_cpu; env != NULL; env = env->next_cpu) {
                    750:         memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
                    751:     }
1.1       root      752: 
                    753:     memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
                    754:     page_flush_tb();
                    755: 
                    756:     code_gen_ptr = code_gen_buffer;
                    757:     /* XXX: flush processor icache at this point if cache flush is
                    758:        expensive */
                    759:     tb_flush_count++;
                    760: }
                    761: 
                    762: #ifdef DEBUG_TB_CHECK
                    763: 
1.1.1.6   root      764: static void tb_invalidate_check(target_ulong address)
1.1       root      765: {
                    766:     TranslationBlock *tb;
                    767:     int i;
                    768:     address &= TARGET_PAGE_MASK;
1.1.1.3   root      769:     for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
                    770:         for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
1.1       root      771:             if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
                    772:                   address >= tb->pc + tb->size)) {
1.1.1.10  root      773:                 printf("ERROR invalidate: address=" TARGET_FMT_lx
                    774:                        " PC=%08lx size=%04x\n",
1.1.1.3   root      775:                        address, (long)tb->pc, tb->size);
1.1       root      776:             }
                    777:         }
                    778:     }
                    779: }
                    780: 
                    781: /* verify that all the pages have correct rights for code */
                    782: static void tb_page_check(void)
                    783: {
                    784:     TranslationBlock *tb;
                    785:     int i, flags1, flags2;
1.1.1.6   root      786: 
1.1.1.3   root      787:     for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
                    788:         for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
1.1       root      789:             flags1 = page_get_flags(tb->pc);
                    790:             flags2 = page_get_flags(tb->pc + tb->size - 1);
                    791:             if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
                    792:                 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
1.1.1.3   root      793:                        (long)tb->pc, tb->size, flags1, flags2);
1.1       root      794:             }
                    795:         }
                    796:     }
                    797: }
                    798: 
                    799: #endif
                    800: 
                    801: /* invalidate one TB */
                    802: static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
                    803:                              int next_offset)
                    804: {
                    805:     TranslationBlock *tb1;
                    806:     for(;;) {
                    807:         tb1 = *ptb;
                    808:         if (tb1 == tb) {
                    809:             *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
                    810:             break;
                    811:         }
                    812:         ptb = (TranslationBlock **)((char *)tb1 + next_offset);
                    813:     }
                    814: }
                    815: 
                    816: static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
                    817: {
                    818:     TranslationBlock *tb1;
                    819:     unsigned int n1;
                    820: 
                    821:     for(;;) {
                    822:         tb1 = *ptb;
                    823:         n1 = (long)tb1 & 3;
                    824:         tb1 = (TranslationBlock *)((long)tb1 & ~3);
                    825:         if (tb1 == tb) {
                    826:             *ptb = tb1->page_next[n1];
                    827:             break;
                    828:         }
                    829:         ptb = &tb1->page_next[n1];
                    830:     }
                    831: }
                    832: 
                    833: static inline void tb_jmp_remove(TranslationBlock *tb, int n)
                    834: {
                    835:     TranslationBlock *tb1, **ptb;
                    836:     unsigned int n1;
                    837: 
                    838:     ptb = &tb->jmp_next[n];
                    839:     tb1 = *ptb;
                    840:     if (tb1) {
                    841:         /* find tb(n) in circular list */
                    842:         for(;;) {
                    843:             tb1 = *ptb;
                    844:             n1 = (long)tb1 & 3;
                    845:             tb1 = (TranslationBlock *)((long)tb1 & ~3);
                    846:             if (n1 == n && tb1 == tb)
                    847:                 break;
                    848:             if (n1 == 2) {
                    849:                 ptb = &tb1->jmp_first;
                    850:             } else {
                    851:                 ptb = &tb1->jmp_next[n1];
                    852:             }
                    853:         }
                    854:         /* now we can suppress tb(n) from the list */
                    855:         *ptb = tb->jmp_next[n];
                    856: 
                    857:         tb->jmp_next[n] = NULL;
                    858:     }
                    859: }
                    860: 
                    861: /* reset the jump entry 'n' of a TB so that it is not chained to
                    862:    another TB */
                    863: static inline void tb_reset_jump(TranslationBlock *tb, int n)
                    864: {
                    865:     tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
                    866: }
                    867: 
1.1.1.13  root      868: void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
1.1       root      869: {
1.1.1.2   root      870:     CPUState *env;
                    871:     PageDesc *p;
1.1       root      872:     unsigned int h, n1;
1.1.1.13  root      873:     tb_page_addr_t phys_pc;
1.1.1.2   root      874:     TranslationBlock *tb1, *tb2;
1.1.1.6   root      875: 
1.1.1.2   root      876:     /* remove the TB from the hash list */
                    877:     phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
                    878:     h = tb_phys_hash_func(phys_pc);
1.1.1.6   root      879:     tb_remove(&tb_phys_hash[h], tb,
1.1.1.2   root      880:               offsetof(TranslationBlock, phys_hash_next));
                    881: 
                    882:     /* remove the TB from the page list */
                    883:     if (tb->page_addr[0] != page_addr) {
                    884:         p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
                    885:         tb_page_remove(&p->first_tb, tb);
                    886:         invalidate_page_bitmap(p);
                    887:     }
                    888:     if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
                    889:         p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
                    890:         tb_page_remove(&p->first_tb, tb);
                    891:         invalidate_page_bitmap(p);
                    892:     }
                    893: 
1.1       root      894:     tb_invalidated_flag = 1;
                    895: 
                    896:     /* remove the TB from the hash list */
1.1.1.2   root      897:     h = tb_jmp_cache_hash_func(tb->pc);
                    898:     for(env = first_cpu; env != NULL; env = env->next_cpu) {
                    899:         if (env->tb_jmp_cache[h] == tb)
                    900:             env->tb_jmp_cache[h] = NULL;
1.1       root      901:     }
                    902: 
                    903:     /* suppress this TB from the two jump lists */
                    904:     tb_jmp_remove(tb, 0);
                    905:     tb_jmp_remove(tb, 1);
                    906: 
                    907:     /* suppress any remaining jumps to this TB */
                    908:     tb1 = tb->jmp_first;
                    909:     for(;;) {
                    910:         n1 = (long)tb1 & 3;
                    911:         if (n1 == 2)
                    912:             break;
                    913:         tb1 = (TranslationBlock *)((long)tb1 & ~3);
                    914:         tb2 = tb1->jmp_next[n1];
                    915:         tb_reset_jump(tb1, n1);
                    916:         tb1->jmp_next[n1] = NULL;
                    917:         tb1 = tb2;
                    918:     }
                    919:     tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
                    920: 
                    921:     tb_phys_invalidate_count++;
                    922: }
                    923: 
                    924: static inline void set_bits(uint8_t *tab, int start, int len)
                    925: {
                    926:     int end, mask, end1;
                    927: 
                    928:     end = start + len;
                    929:     tab += start >> 3;
                    930:     mask = 0xff << (start & 7);
                    931:     if ((start & ~7) == (end & ~7)) {
                    932:         if (start < end) {
                    933:             mask &= ~(0xff << (end & 7));
                    934:             *tab |= mask;
                    935:         }
                    936:     } else {
                    937:         *tab++ |= mask;
                    938:         start = (start + 8) & ~7;
                    939:         end1 = end & ~7;
                    940:         while (start < end1) {
                    941:             *tab++ = 0xff;
                    942:             start += 8;
                    943:         }
                    944:         if (start < end) {
                    945:             mask = ~(0xff << (end & 7));
                    946:             *tab |= mask;
                    947:         }
                    948:     }
                    949: }
                    950: 
                    951: static void build_page_bitmap(PageDesc *p)
                    952: {
                    953:     int n, tb_start, tb_end;
                    954:     TranslationBlock *tb;
1.1.1.6   root      955: 
1.1.1.16! root      956:     p->code_bitmap = g_malloc0(TARGET_PAGE_SIZE / 8);
1.1       root      957: 
                    958:     tb = p->first_tb;
                    959:     while (tb != NULL) {
                    960:         n = (long)tb & 3;
                    961:         tb = (TranslationBlock *)((long)tb & ~3);
                    962:         /* NOTE: this is subtle as a TB may span two physical pages */
                    963:         if (n == 0) {
                    964:             /* NOTE: tb_end may be after the end of the page, but
                    965:                it is not a problem */
                    966:             tb_start = tb->pc & ~TARGET_PAGE_MASK;
                    967:             tb_end = tb_start + tb->size;
                    968:             if (tb_end > TARGET_PAGE_SIZE)
                    969:                 tb_end = TARGET_PAGE_SIZE;
                    970:         } else {
                    971:             tb_start = 0;
                    972:             tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
                    973:         }
                    974:         set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
                    975:         tb = tb->page_next[n];
                    976:     }
                    977: }
                    978: 
1.1.1.7   root      979: TranslationBlock *tb_gen_code(CPUState *env,
                    980:                               target_ulong pc, target_ulong cs_base,
                    981:                               int flags, int cflags)
1.1       root      982: {
                    983:     TranslationBlock *tb;
                    984:     uint8_t *tc_ptr;
1.1.1.13  root      985:     tb_page_addr_t phys_pc, phys_page2;
                    986:     target_ulong virt_page2;
1.1       root      987:     int code_gen_size;
                    988: 
1.1.1.13  root      989:     phys_pc = get_page_addr_code(env, pc);
1.1       root      990:     tb = tb_alloc(pc);
                    991:     if (!tb) {
                    992:         /* flush must be done */
                    993:         tb_flush(env);
                    994:         /* cannot fail at this point */
                    995:         tb = tb_alloc(pc);
1.1.1.7   root      996:         /* Don't forget to invalidate previous TB info.  */
                    997:         tb_invalidated_flag = 1;
1.1       root      998:     }
                    999:     tc_ptr = code_gen_ptr;
                   1000:     tb->tc_ptr = tc_ptr;
                   1001:     tb->cs_base = cs_base;
                   1002:     tb->flags = flags;
                   1003:     tb->cflags = cflags;
1.1.1.6   root     1004:     cpu_gen_code(env, tb, &code_gen_size);
1.1       root     1005:     code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
1.1.1.6   root     1006: 
1.1       root     1007:     /* check next page if needed */
                   1008:     virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
                   1009:     phys_page2 = -1;
                   1010:     if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1.1.1.13  root     1011:         phys_page2 = get_page_addr_code(env, virt_page2);
1.1       root     1012:     }
1.1.1.13  root     1013:     tb_link_page(tb, phys_pc, phys_page2);
1.1.1.7   root     1014:     return tb;
1.1       root     1015: }
1.1.1.6   root     1016: 
1.1       root     1017: /* invalidate all TBs which intersect with the target physical page
                   1018:    starting in range [start;end[. NOTE: start and end must refer to
                   1019:    the same physical page. 'is_cpu_write_access' should be true if called
                   1020:    from a real cpu write access: the virtual CPU will exit the current
                   1021:    TB if code is modified inside this TB. */
1.1.1.13  root     1022: void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1.1       root     1023:                                    int is_cpu_write_access)
                   1024: {
1.1.1.7   root     1025:     TranslationBlock *tb, *tb_next, *saved_tb;
1.1       root     1026:     CPUState *env = cpu_single_env;
1.1.1.13  root     1027:     tb_page_addr_t tb_start, tb_end;
1.1.1.7   root     1028:     PageDesc *p;
                   1029:     int n;
                   1030: #ifdef TARGET_HAS_PRECISE_SMC
                   1031:     int current_tb_not_found = is_cpu_write_access;
                   1032:     TranslationBlock *current_tb = NULL;
                   1033:     int current_tb_modified = 0;
                   1034:     target_ulong current_pc = 0;
                   1035:     target_ulong current_cs_base = 0;
                   1036:     int current_flags = 0;
                   1037: #endif /* TARGET_HAS_PRECISE_SMC */
1.1       root     1038: 
                   1039:     p = page_find(start >> TARGET_PAGE_BITS);
1.1.1.6   root     1040:     if (!p)
1.1       root     1041:         return;
1.1.1.6   root     1042:     if (!p->code_bitmap &&
1.1       root     1043:         ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
                   1044:         is_cpu_write_access) {
                   1045:         /* build code bitmap */
                   1046:         build_page_bitmap(p);
                   1047:     }
                   1048: 
                   1049:     /* we remove all the TBs in the range [start, end[ */
                   1050:     /* XXX: see if in some cases it could be faster to invalidate all the code */
                   1051:     tb = p->first_tb;
                   1052:     while (tb != NULL) {
                   1053:         n = (long)tb & 3;
                   1054:         tb = (TranslationBlock *)((long)tb & ~3);
                   1055:         tb_next = tb->page_next[n];
                   1056:         /* NOTE: this is subtle as a TB may span two physical pages */
                   1057:         if (n == 0) {
                   1058:             /* NOTE: tb_end may be after the end of the page, but
                   1059:                it is not a problem */
                   1060:             tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
                   1061:             tb_end = tb_start + tb->size;
                   1062:         } else {
                   1063:             tb_start = tb->page_addr[1];
                   1064:             tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
                   1065:         }
                   1066:         if (!(tb_end <= start || tb_start >= end)) {
                   1067: #ifdef TARGET_HAS_PRECISE_SMC
                   1068:             if (current_tb_not_found) {
                   1069:                 current_tb_not_found = 0;
                   1070:                 current_tb = NULL;
1.1.1.7   root     1071:                 if (env->mem_io_pc) {
1.1       root     1072:                     /* now we have a real cpu fault */
1.1.1.7   root     1073:                     current_tb = tb_find_pc(env->mem_io_pc);
1.1       root     1074:                 }
                   1075:             }
                   1076:             if (current_tb == tb &&
1.1.1.7   root     1077:                 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1.1       root     1078:                 /* If we are modifying the current TB, we must stop
                   1079:                 its execution. We could be more precise by checking
                   1080:                 that the modification is after the current PC, but it
                   1081:                 would require a specialized function to partially
                   1082:                 restore the CPU state */
1.1.1.6   root     1083: 
1.1       root     1084:                 current_tb_modified = 1;
1.1.1.15  root     1085:                 cpu_restore_state(current_tb, env, env->mem_io_pc);
1.1.1.7   root     1086:                 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
                   1087:                                      &current_flags);
1.1       root     1088:             }
                   1089: #endif /* TARGET_HAS_PRECISE_SMC */
1.1.1.2   root     1090:             /* we need to do that to handle the case where a signal
                   1091:                occurs while doing tb_phys_invalidate() */
                   1092:             saved_tb = NULL;
                   1093:             if (env) {
                   1094:                 saved_tb = env->current_tb;
                   1095:                 env->current_tb = NULL;
                   1096:             }
1.1       root     1097:             tb_phys_invalidate(tb, -1);
1.1.1.2   root     1098:             if (env) {
                   1099:                 env->current_tb = saved_tb;
                   1100:                 if (env->interrupt_request && env->current_tb)
                   1101:                     cpu_interrupt(env, env->interrupt_request);
                   1102:             }
1.1       root     1103:         }
                   1104:         tb = tb_next;
                   1105:     }
                   1106: #if !defined(CONFIG_USER_ONLY)
                   1107:     /* if no code remaining, no need to continue to use slow writes */
                   1108:     if (!p->first_tb) {
                   1109:         invalidate_page_bitmap(p);
                   1110:         if (is_cpu_write_access) {
1.1.1.7   root     1111:             tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1.1       root     1112:         }
                   1113:     }
                   1114: #endif
                   1115: #ifdef TARGET_HAS_PRECISE_SMC
                   1116:     if (current_tb_modified) {
                   1117:         /* we generate a block containing just the instruction
                   1118:            modifying the memory. It will ensure that it cannot modify
                   1119:            itself */
                   1120:         env->current_tb = NULL;
1.1.1.7   root     1121:         tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1.1       root     1122:         cpu_resume_from_signal(env, NULL);
                   1123:     }
                   1124: #endif
                   1125: }
                   1126: 
                   1127: /* len must be <= 8 and start must be a multiple of len */
1.1.1.13  root     1128: static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1.1       root     1129: {
                   1130:     PageDesc *p;
                   1131:     int offset, b;
                   1132: #if 0
                   1133:     if (1) {
1.1.1.7   root     1134:         qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
                   1135:                   cpu_single_env->mem_io_vaddr, len,
                   1136:                   cpu_single_env->eip,
                   1137:                   cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1.1       root     1138:     }
                   1139: #endif
                   1140:     p = page_find(start >> TARGET_PAGE_BITS);
1.1.1.6   root     1141:     if (!p)
1.1       root     1142:         return;
                   1143:     if (p->code_bitmap) {
                   1144:         offset = start & ~TARGET_PAGE_MASK;
                   1145:         b = p->code_bitmap[offset >> 3] >> (offset & 7);
                   1146:         if (b & ((1 << len) - 1))
                   1147:             goto do_invalidate;
                   1148:     } else {
                   1149:     do_invalidate:
                   1150:         tb_invalidate_phys_page_range(start, start + len, 1);
                   1151:     }
                   1152: }
                   1153: 
                   1154: #if !defined(CONFIG_SOFTMMU)
1.1.1.13  root     1155: static void tb_invalidate_phys_page(tb_page_addr_t addr,
1.1       root     1156:                                     unsigned long pc, void *puc)
                   1157: {
1.1.1.7   root     1158:     TranslationBlock *tb;
1.1       root     1159:     PageDesc *p;
1.1.1.7   root     1160:     int n;
1.1       root     1161: #ifdef TARGET_HAS_PRECISE_SMC
1.1.1.7   root     1162:     TranslationBlock *current_tb = NULL;
1.1       root     1163:     CPUState *env = cpu_single_env;
1.1.1.7   root     1164:     int current_tb_modified = 0;
                   1165:     target_ulong current_pc = 0;
                   1166:     target_ulong current_cs_base = 0;
                   1167:     int current_flags = 0;
1.1       root     1168: #endif
                   1169: 
                   1170:     addr &= TARGET_PAGE_MASK;
                   1171:     p = page_find(addr >> TARGET_PAGE_BITS);
1.1.1.6   root     1172:     if (!p)
1.1       root     1173:         return;
                   1174:     tb = p->first_tb;
                   1175: #ifdef TARGET_HAS_PRECISE_SMC
                   1176:     if (tb && pc != 0) {
                   1177:         current_tb = tb_find_pc(pc);
                   1178:     }
                   1179: #endif
                   1180:     while (tb != NULL) {
                   1181:         n = (long)tb & 3;
                   1182:         tb = (TranslationBlock *)((long)tb & ~3);
                   1183: #ifdef TARGET_HAS_PRECISE_SMC
                   1184:         if (current_tb == tb &&
1.1.1.7   root     1185:             (current_tb->cflags & CF_COUNT_MASK) != 1) {
1.1       root     1186:                 /* If we are modifying the current TB, we must stop
                   1187:                    its execution. We could be more precise by checking
                   1188:                    that the modification is after the current PC, but it
                   1189:                    would require a specialized function to partially
                   1190:                    restore the CPU state */
1.1.1.6   root     1191: 
1.1       root     1192:             current_tb_modified = 1;
1.1.1.15  root     1193:             cpu_restore_state(current_tb, env, pc);
1.1.1.7   root     1194:             cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
                   1195:                                  &current_flags);
1.1       root     1196:         }
                   1197: #endif /* TARGET_HAS_PRECISE_SMC */
                   1198:         tb_phys_invalidate(tb, addr);
                   1199:         tb = tb->page_next[n];
                   1200:     }
                   1201:     p->first_tb = NULL;
                   1202: #ifdef TARGET_HAS_PRECISE_SMC
                   1203:     if (current_tb_modified) {
                   1204:         /* we generate a block containing just the instruction
                   1205:            modifying the memory. It will ensure that it cannot modify
                   1206:            itself */
                   1207:         env->current_tb = NULL;
1.1.1.7   root     1208:         tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1.1       root     1209:         cpu_resume_from_signal(env, puc);
                   1210:     }
                   1211: #endif
                   1212: }
                   1213: #endif
                   1214: 
                   1215: /* add the tb in the target page and protect it if necessary */
1.1.1.6   root     1216: static inline void tb_alloc_page(TranslationBlock *tb,
1.1.1.13  root     1217:                                  unsigned int n, tb_page_addr_t page_addr)
1.1       root     1218: {
                   1219:     PageDesc *p;
1.1.1.15  root     1220: #ifndef CONFIG_USER_ONLY
                   1221:     bool page_already_protected;
                   1222: #endif
1.1       root     1223: 
                   1224:     tb->page_addr[n] = page_addr;
1.1.1.13  root     1225:     p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1.1       root     1226:     tb->page_next[n] = p->first_tb;
1.1.1.15  root     1227: #ifndef CONFIG_USER_ONLY
                   1228:     page_already_protected = p->first_tb != NULL;
                   1229: #endif
1.1       root     1230:     p->first_tb = (TranslationBlock *)((long)tb | n);
                   1231:     invalidate_page_bitmap(p);
                   1232: 
                   1233: #if defined(TARGET_HAS_SMC) || 1
                   1234: 
                   1235: #if defined(CONFIG_USER_ONLY)
                   1236:     if (p->flags & PAGE_WRITE) {
1.1.1.3   root     1237:         target_ulong addr;
                   1238:         PageDesc *p2;
1.1       root     1239:         int prot;
                   1240: 
                   1241:         /* force the host page as non writable (writes will have a
                   1242:            page fault + mprotect overhead) */
1.1.1.3   root     1243:         page_addr &= qemu_host_page_mask;
1.1       root     1244:         prot = 0;
1.1.1.3   root     1245:         for(addr = page_addr; addr < page_addr + qemu_host_page_size;
                   1246:             addr += TARGET_PAGE_SIZE) {
                   1247: 
                   1248:             p2 = page_find (addr >> TARGET_PAGE_BITS);
                   1249:             if (!p2)
                   1250:                 continue;
                   1251:             prot |= p2->flags;
                   1252:             p2->flags &= ~PAGE_WRITE;
                   1253:           }
1.1.1.6   root     1254:         mprotect(g2h(page_addr), qemu_host_page_size,
1.1       root     1255:                  (prot & PAGE_BITS) & ~PAGE_WRITE);
                   1256: #ifdef DEBUG_TB_INVALIDATE
1.1.1.6   root     1257:         printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1.1.1.3   root     1258:                page_addr);
1.1       root     1259: #endif
                   1260:     }
                   1261: #else
                   1262:     /* if some code is already present, then the pages are already
                   1263:        protected. So we handle the case where only the first TB is
                   1264:        allocated in a physical page */
1.1.1.15  root     1265:     if (!page_already_protected) {
1.1.1.2   root     1266:         tlb_protect_code(page_addr);
1.1       root     1267:     }
                   1268: #endif
                   1269: 
                   1270: #endif /* TARGET_HAS_SMC */
                   1271: }
                   1272: 
                   1273: /* add a new TB and link it to the physical page tables. phys_page2 is
                   1274:    (-1) to indicate that only one page contains the TB. */
1.1.1.13  root     1275: void tb_link_page(TranslationBlock *tb,
                   1276:                   tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
1.1       root     1277: {
                   1278:     unsigned int h;
                   1279:     TranslationBlock **ptb;
                   1280: 
1.1.1.7   root     1281:     /* Grab the mmap lock to stop another thread invalidating this TB
                   1282:        before we are done.  */
                   1283:     mmap_lock();
1.1       root     1284:     /* add in the physical hash table */
                   1285:     h = tb_phys_hash_func(phys_pc);
                   1286:     ptb = &tb_phys_hash[h];
                   1287:     tb->phys_hash_next = *ptb;
                   1288:     *ptb = tb;
                   1289: 
                   1290:     /* add in the page list */
                   1291:     tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
                   1292:     if (phys_page2 != -1)
                   1293:         tb_alloc_page(tb, 1, phys_page2);
                   1294:     else
                   1295:         tb->page_addr[1] = -1;
                   1296: 
                   1297:     tb->jmp_first = (TranslationBlock *)((long)tb | 2);
                   1298:     tb->jmp_next[0] = NULL;
                   1299:     tb->jmp_next[1] = NULL;
                   1300: 
                   1301:     /* init original jump addresses */
                   1302:     if (tb->tb_next_offset[0] != 0xffff)
                   1303:         tb_reset_jump(tb, 0);
                   1304:     if (tb->tb_next_offset[1] != 0xffff)
                   1305:         tb_reset_jump(tb, 1);
1.1.1.2   root     1306: 
                   1307: #ifdef DEBUG_TB_CHECK
                   1308:     tb_page_check();
                   1309: #endif
1.1.1.7   root     1310:     mmap_unlock();
1.1       root     1311: }
                   1312: 
                   1313: /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
                   1314:    tb[1].tc_ptr. Return NULL if not found */
                   1315: TranslationBlock *tb_find_pc(unsigned long tc_ptr)
                   1316: {
                   1317:     int m_min, m_max, m;
                   1318:     unsigned long v;
                   1319:     TranslationBlock *tb;
                   1320: 
                   1321:     if (nb_tbs <= 0)
                   1322:         return NULL;
                   1323:     if (tc_ptr < (unsigned long)code_gen_buffer ||
                   1324:         tc_ptr >= (unsigned long)code_gen_ptr)
                   1325:         return NULL;
                   1326:     /* binary search (cf Knuth) */
                   1327:     m_min = 0;
                   1328:     m_max = nb_tbs - 1;
                   1329:     while (m_min <= m_max) {
                   1330:         m = (m_min + m_max) >> 1;
                   1331:         tb = &tbs[m];
                   1332:         v = (unsigned long)tb->tc_ptr;
                   1333:         if (v == tc_ptr)
                   1334:             return tb;
                   1335:         else if (tc_ptr < v) {
                   1336:             m_max = m - 1;
                   1337:         } else {
                   1338:             m_min = m + 1;
                   1339:         }
1.1.1.6   root     1340:     }
1.1       root     1341:     return &tbs[m_max];
                   1342: }
                   1343: 
                   1344: static void tb_reset_jump_recursive(TranslationBlock *tb);
                   1345: 
                   1346: static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
                   1347: {
                   1348:     TranslationBlock *tb1, *tb_next, **ptb;
                   1349:     unsigned int n1;
                   1350: 
                   1351:     tb1 = tb->jmp_next[n];
                   1352:     if (tb1 != NULL) {
                   1353:         /* find head of list */
                   1354:         for(;;) {
                   1355:             n1 = (long)tb1 & 3;
                   1356:             tb1 = (TranslationBlock *)((long)tb1 & ~3);
                   1357:             if (n1 == 2)
                   1358:                 break;
                   1359:             tb1 = tb1->jmp_next[n1];
                   1360:         }
                   1361:         /* we are now sure now that tb jumps to tb1 */
                   1362:         tb_next = tb1;
                   1363: 
                   1364:         /* remove tb from the jmp_first list */
                   1365:         ptb = &tb_next->jmp_first;
                   1366:         for(;;) {
                   1367:             tb1 = *ptb;
                   1368:             n1 = (long)tb1 & 3;
                   1369:             tb1 = (TranslationBlock *)((long)tb1 & ~3);
                   1370:             if (n1 == n && tb1 == tb)
                   1371:                 break;
                   1372:             ptb = &tb1->jmp_next[n1];
                   1373:         }
                   1374:         *ptb = tb->jmp_next[n];
                   1375:         tb->jmp_next[n] = NULL;
1.1.1.6   root     1376: 
1.1       root     1377:         /* suppress the jump to next tb in generated code */
                   1378:         tb_reset_jump(tb, n);
                   1379: 
                   1380:         /* suppress jumps in the tb on which we could have jumped */
                   1381:         tb_reset_jump_recursive(tb_next);
                   1382:     }
                   1383: }
                   1384: 
                   1385: static void tb_reset_jump_recursive(TranslationBlock *tb)
                   1386: {
                   1387:     tb_reset_jump_recursive2(tb, 0);
                   1388:     tb_reset_jump_recursive2(tb, 1);
                   1389: }
                   1390: 
                   1391: #if defined(TARGET_HAS_ICE)
1.1.1.13  root     1392: #if defined(CONFIG_USER_ONLY)
                   1393: static void breakpoint_invalidate(CPUState *env, target_ulong pc)
                   1394: {
                   1395:     tb_invalidate_phys_page_range(pc, pc + 1, 0);
                   1396: }
                   1397: #else
1.1       root     1398: static void breakpoint_invalidate(CPUState *env, target_ulong pc)
                   1399: {
1.1.1.6   root     1400:     target_phys_addr_t addr;
                   1401:     target_ulong pd;
1.1.1.3   root     1402:     ram_addr_t ram_addr;
                   1403:     PhysPageDesc *p;
1.1       root     1404: 
1.1.1.3   root     1405:     addr = cpu_get_phys_page_debug(env, pc);
                   1406:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   1407:     if (!p) {
                   1408:         pd = IO_MEM_UNASSIGNED;
                   1409:     } else {
                   1410:         pd = p->phys_offset;
                   1411:     }
                   1412:     ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
                   1413:     tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1.1       root     1414: }
                   1415: #endif
1.1.1.13  root     1416: #endif /* TARGET_HAS_ICE */
                   1417: 
                   1418: #if defined(CONFIG_USER_ONLY)
                   1419: void cpu_watchpoint_remove_all(CPUState *env, int mask)
                   1420: 
                   1421: {
                   1422: }
1.1       root     1423: 
1.1.1.13  root     1424: int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
                   1425:                           int flags, CPUWatchpoint **watchpoint)
                   1426: {
                   1427:     return -ENOSYS;
                   1428: }
                   1429: #else
1.1.1.6   root     1430: /* Add a watchpoint.  */
1.1.1.7   root     1431: int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
                   1432:                           int flags, CPUWatchpoint **watchpoint)
1.1.1.6   root     1433: {
1.1.1.7   root     1434:     target_ulong len_mask = ~(len - 1);
                   1435:     CPUWatchpoint *wp;
                   1436: 
                   1437:     /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
                   1438:     if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
                   1439:         fprintf(stderr, "qemu: tried to set invalid watchpoint at "
                   1440:                 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
                   1441:         return -EINVAL;
                   1442:     }
1.1.1.16! root     1443:     wp = g_malloc(sizeof(*wp));
1.1.1.7   root     1444: 
                   1445:     wp->vaddr = addr;
                   1446:     wp->len_mask = len_mask;
                   1447:     wp->flags = flags;
                   1448: 
                   1449:     /* keep all GDB-injected watchpoints in front */
                   1450:     if (flags & BP_GDB)
1.1.1.11  root     1451:         QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1.1.1.7   root     1452:     else
1.1.1.11  root     1453:         QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1.1.1.7   root     1454: 
                   1455:     tlb_flush_page(env, addr);
                   1456: 
                   1457:     if (watchpoint)
                   1458:         *watchpoint = wp;
                   1459:     return 0;
                   1460: }
1.1.1.6   root     1461: 
1.1.1.7   root     1462: /* Remove a specific watchpoint.  */
                   1463: int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
                   1464:                           int flags)
                   1465: {
                   1466:     target_ulong len_mask = ~(len - 1);
                   1467:     CPUWatchpoint *wp;
                   1468: 
1.1.1.11  root     1469:     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1.1.1.7   root     1470:         if (addr == wp->vaddr && len_mask == wp->len_mask
                   1471:                 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
                   1472:             cpu_watchpoint_remove_by_ref(env, wp);
1.1.1.6   root     1473:             return 0;
1.1.1.7   root     1474:         }
1.1.1.6   root     1475:     }
1.1.1.7   root     1476:     return -ENOENT;
                   1477: }
1.1.1.6   root     1478: 
1.1.1.7   root     1479: /* Remove a specific watchpoint by reference.  */
                   1480: void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
                   1481: {
1.1.1.11  root     1482:     QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1.1.1.7   root     1483: 
                   1484:     tlb_flush_page(env, watchpoint->vaddr);
                   1485: 
1.1.1.16! root     1486:     g_free(watchpoint);
1.1.1.6   root     1487: }
                   1488: 
1.1.1.7   root     1489: /* Remove all matching watchpoints.  */
                   1490: void cpu_watchpoint_remove_all(CPUState *env, int mask)
1.1.1.6   root     1491: {
1.1.1.7   root     1492:     CPUWatchpoint *wp, *next;
1.1.1.6   root     1493: 
1.1.1.11  root     1494:     QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1.1.1.7   root     1495:         if (wp->flags & mask)
                   1496:             cpu_watchpoint_remove_by_ref(env, wp);
1.1.1.6   root     1497:     }
                   1498: }
1.1.1.13  root     1499: #endif
1.1.1.6   root     1500: 
1.1.1.7   root     1501: /* Add a breakpoint.  */
                   1502: int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
                   1503:                           CPUBreakpoint **breakpoint)
1.1       root     1504: {
                   1505: #if defined(TARGET_HAS_ICE)
1.1.1.7   root     1506:     CPUBreakpoint *bp;
1.1.1.6   root     1507: 
1.1.1.16! root     1508:     bp = g_malloc(sizeof(*bp));
1.1       root     1509: 
1.1.1.7   root     1510:     bp->pc = pc;
                   1511:     bp->flags = flags;
                   1512: 
                   1513:     /* keep all GDB-injected breakpoints in front */
                   1514:     if (flags & BP_GDB)
1.1.1.11  root     1515:         QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1.1.1.7   root     1516:     else
1.1.1.11  root     1517:         QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1.1.1.6   root     1518: 
1.1       root     1519:     breakpoint_invalidate(env, pc);
1.1.1.7   root     1520: 
                   1521:     if (breakpoint)
                   1522:         *breakpoint = bp;
1.1       root     1523:     return 0;
                   1524: #else
1.1.1.7   root     1525:     return -ENOSYS;
1.1       root     1526: #endif
                   1527: }
                   1528: 
1.1.1.7   root     1529: /* Remove a specific breakpoint.  */
                   1530: int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1.1       root     1531: {
                   1532: #if defined(TARGET_HAS_ICE)
1.1.1.7   root     1533:     CPUBreakpoint *bp;
1.1       root     1534: 
1.1.1.11  root     1535:     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1.1.1.7   root     1536:         if (bp->pc == pc && bp->flags == flags) {
                   1537:             cpu_breakpoint_remove_by_ref(env, bp);
                   1538:             return 0;
                   1539:         }
                   1540:     }
                   1541:     return -ENOENT;
1.1       root     1542: #else
1.1.1.7   root     1543:     return -ENOSYS;
                   1544: #endif
                   1545: }
                   1546: 
                   1547: /* Remove a specific breakpoint by reference.  */
                   1548: void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
                   1549: {
                   1550: #if defined(TARGET_HAS_ICE)
1.1.1.11  root     1551:     QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1.1.1.7   root     1552: 
                   1553:     breakpoint_invalidate(env, breakpoint->pc);
                   1554: 
1.1.1.16! root     1555:     g_free(breakpoint);
1.1.1.7   root     1556: #endif
                   1557: }
                   1558: 
                   1559: /* Remove all matching breakpoints. */
                   1560: void cpu_breakpoint_remove_all(CPUState *env, int mask)
                   1561: {
                   1562: #if defined(TARGET_HAS_ICE)
                   1563:     CPUBreakpoint *bp, *next;
                   1564: 
1.1.1.11  root     1565:     QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1.1.1.7   root     1566:         if (bp->flags & mask)
                   1567:             cpu_breakpoint_remove_by_ref(env, bp);
                   1568:     }
1.1       root     1569: #endif
                   1570: }
                   1571: 
                   1572: /* enable or disable single step mode. EXCP_DEBUG is returned by the
                   1573:    CPU loop after each instruction */
                   1574: void cpu_single_step(CPUState *env, int enabled)
                   1575: {
                   1576: #if defined(TARGET_HAS_ICE)
                   1577:     if (env->singlestep_enabled != enabled) {
                   1578:         env->singlestep_enabled = enabled;
1.1.1.10  root     1579:         if (kvm_enabled())
                   1580:             kvm_update_guest_debug(env, 0);
                   1581:         else {
                   1582:             /* must flush all the translated code to avoid inconsistencies */
                   1583:             /* XXX: only flush what is necessary */
                   1584:             tb_flush(env);
                   1585:         }
1.1       root     1586:     }
                   1587: #endif
                   1588: }
                   1589: 
                   1590: /* enable or disable low levels log */
                   1591: void cpu_set_log(int log_flags)
                   1592: {
                   1593:     loglevel = log_flags;
                   1594:     if (loglevel && !logfile) {
1.1.1.6   root     1595:         logfile = fopen(logfilename, log_append ? "a" : "w");
1.1       root     1596:         if (!logfile) {
                   1597:             perror(logfilename);
                   1598:             _exit(1);
                   1599:         }
                   1600: #if !defined(CONFIG_SOFTMMU)
                   1601:         /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
                   1602:         {
1.1.1.7   root     1603:             static char logfile_buf[4096];
1.1       root     1604:             setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
                   1605:         }
1.1.1.11  root     1606: #elif !defined(_WIN32)
                   1607:         /* Win32 doesn't support line-buffering and requires size >= 2 */
1.1       root     1608:         setvbuf(logfile, NULL, _IOLBF, 0);
                   1609: #endif
1.1.1.6   root     1610:         log_append = 1;
                   1611:     }
                   1612:     if (!loglevel && logfile) {
                   1613:         fclose(logfile);
                   1614:         logfile = NULL;
1.1       root     1615:     }
                   1616: }
                   1617: 
                   1618: void cpu_set_log_filename(const char *filename)
                   1619: {
                   1620:     logfilename = strdup(filename);
1.1.1.6   root     1621:     if (logfile) {
                   1622:         fclose(logfile);
                   1623:         logfile = NULL;
                   1624:     }
                   1625:     cpu_set_log(loglevel);
1.1       root     1626: }
                   1627: 
1.1.1.10  root     1628: static void cpu_unlink_tb(CPUState *env)
1.1       root     1629: {
1.1.1.10  root     1630:     /* FIXME: TB unchaining isn't SMP safe.  For now just ignore the
                   1631:        problem and hope the cpu will stop of its own accord.  For userspace
                   1632:        emulation this often isn't actually as bad as it sounds.  Often
                   1633:        signals are used primarily to interrupt blocking syscalls.  */
1.1       root     1634:     TranslationBlock *tb;
1.1.1.7   root     1635:     static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1.1       root     1636: 
1.1.1.13  root     1637:     spin_lock(&interrupt_lock);
1.1.1.10  root     1638:     tb = env->current_tb;
                   1639:     /* if the cpu is currently executing code, we must unlink it and
                   1640:        all the potentially executing TB */
1.1.1.13  root     1641:     if (tb) {
1.1.1.10  root     1642:         env->current_tb = NULL;
                   1643:         tb_reset_jump_recursive(tb);
1.1.1.8   root     1644:     }
1.1.1.13  root     1645:     spin_unlock(&interrupt_lock);
1.1.1.10  root     1646: }
                   1647: 
1.1.1.15  root     1648: #ifndef CONFIG_USER_ONLY
1.1.1.10  root     1649: /* mask must never be zero, except for A20 change call */
1.1.1.15  root     1650: static void tcg_handle_interrupt(CPUState *env, int mask)
1.1.1.10  root     1651: {
                   1652:     int old_mask;
1.1.1.8   root     1653: 
1.1.1.7   root     1654:     old_mask = env->interrupt_request;
1.1       root     1655:     env->interrupt_request |= mask;
1.1.1.10  root     1656: 
                   1657:     /*
                   1658:      * If called from iothread context, wake the target cpu in
                   1659:      * case its halted.
                   1660:      */
1.1.1.15  root     1661:     if (!qemu_cpu_is_self(env)) {
1.1.1.10  root     1662:         qemu_cpu_kick(env);
                   1663:         return;
                   1664:     }
                   1665: 
1.1.1.7   root     1666:     if (use_icount) {
                   1667:         env->icount_decr.u16.high = 0xffff;
                   1668:         if (!can_do_io(env)
1.1.1.8   root     1669:             && (mask & ~old_mask) != 0) {
1.1.1.7   root     1670:             cpu_abort(env, "Raised interrupt while not in I/O function");
                   1671:         }
                   1672:     } else {
1.1.1.10  root     1673:         cpu_unlink_tb(env);
1.1       root     1674:     }
                   1675: }
                   1676: 
1.1.1.15  root     1677: CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
                   1678: 
                   1679: #else /* CONFIG_USER_ONLY */
                   1680: 
                   1681: void cpu_interrupt(CPUState *env, int mask)
                   1682: {
                   1683:     env->interrupt_request |= mask;
                   1684:     cpu_unlink_tb(env);
                   1685: }
                   1686: #endif /* CONFIG_USER_ONLY */
                   1687: 
1.1       root     1688: void cpu_reset_interrupt(CPUState *env, int mask)
                   1689: {
                   1690:     env->interrupt_request &= ~mask;
                   1691: }
                   1692: 
1.1.1.10  root     1693: void cpu_exit(CPUState *env)
                   1694: {
                   1695:     env->exit_request = 1;
                   1696:     cpu_unlink_tb(env);
                   1697: }
                   1698: 
1.1.1.7   root     1699: const CPULogItem cpu_log_items[] = {
1.1.1.6   root     1700:     { CPU_LOG_TB_OUT_ASM, "out_asm",
1.1       root     1701:       "show generated host assembly code for each compiled TB" },
                   1702:     { CPU_LOG_TB_IN_ASM, "in_asm",
                   1703:       "show target assembly code for each compiled TB" },
1.1.1.6   root     1704:     { CPU_LOG_TB_OP, "op",
1.1.1.7   root     1705:       "show micro ops for each compiled TB" },
1.1       root     1706:     { CPU_LOG_TB_OP_OPT, "op_opt",
1.1.1.7   root     1707:       "show micro ops "
                   1708: #ifdef TARGET_I386
                   1709:       "before eflags optimization and "
1.1       root     1710: #endif
1.1.1.7   root     1711:       "after liveness analysis" },
1.1       root     1712:     { CPU_LOG_INT, "int",
                   1713:       "show interrupts/exceptions in short format" },
                   1714:     { CPU_LOG_EXEC, "exec",
                   1715:       "show trace before each executed TB (lots of logs)" },
                   1716:     { CPU_LOG_TB_CPU, "cpu",
1.1.1.6   root     1717:       "show CPU state before block translation" },
1.1       root     1718: #ifdef TARGET_I386
                   1719:     { CPU_LOG_PCALL, "pcall",
                   1720:       "show protected mode far calls/returns/exceptions" },
1.1.1.7   root     1721:     { CPU_LOG_RESET, "cpu_reset",
                   1722:       "show CPU state before CPU resets" },
1.1       root     1723: #endif
                   1724: #ifdef DEBUG_IOPORT
                   1725:     { CPU_LOG_IOPORT, "ioport",
                   1726:       "show all i/o ports accesses" },
                   1727: #endif
                   1728:     { 0, NULL, NULL },
                   1729: };
                   1730: 
1.1.1.13  root     1731: #ifndef CONFIG_USER_ONLY
                   1732: static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
                   1733:     = QLIST_HEAD_INITIALIZER(memory_client_list);
                   1734: 
                   1735: static void cpu_notify_set_memory(target_phys_addr_t start_addr,
1.1.1.14  root     1736:                                   ram_addr_t size,
1.1.1.15  root     1737:                                   ram_addr_t phys_offset,
                   1738:                                   bool log_dirty)
1.1.1.13  root     1739: {
                   1740:     CPUPhysMemoryClient *client;
                   1741:     QLIST_FOREACH(client, &memory_client_list, list) {
1.1.1.15  root     1742:         client->set_memory(client, start_addr, size, phys_offset, log_dirty);
1.1.1.13  root     1743:     }
                   1744: }
                   1745: 
                   1746: static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
1.1.1.14  root     1747:                                         target_phys_addr_t end)
1.1.1.13  root     1748: {
                   1749:     CPUPhysMemoryClient *client;
                   1750:     QLIST_FOREACH(client, &memory_client_list, list) {
                   1751:         int r = client->sync_dirty_bitmap(client, start, end);
                   1752:         if (r < 0)
                   1753:             return r;
                   1754:     }
                   1755:     return 0;
                   1756: }
                   1757: 
                   1758: static int cpu_notify_migration_log(int enable)
                   1759: {
                   1760:     CPUPhysMemoryClient *client;
                   1761:     QLIST_FOREACH(client, &memory_client_list, list) {
                   1762:         int r = client->migration_log(client, enable);
                   1763:         if (r < 0)
                   1764:             return r;
                   1765:     }
                   1766:     return 0;
                   1767: }
                   1768: 
1.1.1.15  root     1769: struct last_map {
                   1770:     target_phys_addr_t start_addr;
                   1771:     ram_addr_t size;
                   1772:     ram_addr_t phys_offset;
                   1773: };
                   1774: 
                   1775: /* The l1_phys_map provides the upper P_L1_BITs of the guest physical
                   1776:  * address.  Each intermediate table provides the next L2_BITs of guest
                   1777:  * physical address space.  The number of levels vary based on host and
                   1778:  * guest configuration, making it efficient to build the final guest
                   1779:  * physical address by seeding the L1 offset and shifting and adding in
                   1780:  * each L2 offset as we recurse through them. */
                   1781: static void phys_page_for_each_1(CPUPhysMemoryClient *client, int level,
                   1782:                                  void **lp, target_phys_addr_t addr,
                   1783:                                  struct last_map *map)
1.1.1.13  root     1784: {
                   1785:     int i;
                   1786: 
                   1787:     if (*lp == NULL) {
                   1788:         return;
                   1789:     }
                   1790:     if (level == 0) {
                   1791:         PhysPageDesc *pd = *lp;
1.1.1.15  root     1792:         addr <<= L2_BITS + TARGET_PAGE_BITS;
1.1.1.13  root     1793:         for (i = 0; i < L2_SIZE; ++i) {
                   1794:             if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
1.1.1.15  root     1795:                 target_phys_addr_t start_addr = addr | i << TARGET_PAGE_BITS;
                   1796: 
                   1797:                 if (map->size &&
                   1798:                     start_addr == map->start_addr + map->size &&
                   1799:                     pd[i].phys_offset == map->phys_offset + map->size) {
                   1800: 
                   1801:                     map->size += TARGET_PAGE_SIZE;
                   1802:                     continue;
                   1803:                 } else if (map->size) {
                   1804:                     client->set_memory(client, map->start_addr,
                   1805:                                        map->size, map->phys_offset, false);
                   1806:                 }
                   1807: 
                   1808:                 map->start_addr = start_addr;
                   1809:                 map->size = TARGET_PAGE_SIZE;
                   1810:                 map->phys_offset = pd[i].phys_offset;
1.1.1.13  root     1811:             }
                   1812:         }
                   1813:     } else {
                   1814:         void **pp = *lp;
                   1815:         for (i = 0; i < L2_SIZE; ++i) {
1.1.1.15  root     1816:             phys_page_for_each_1(client, level - 1, pp + i,
                   1817:                                  (addr << L2_BITS) | i, map);
1.1.1.13  root     1818:         }
                   1819:     }
                   1820: }
                   1821: 
                   1822: static void phys_page_for_each(CPUPhysMemoryClient *client)
                   1823: {
                   1824:     int i;
1.1.1.15  root     1825:     struct last_map map = { };
                   1826: 
1.1.1.13  root     1827:     for (i = 0; i < P_L1_SIZE; ++i) {
                   1828:         phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
1.1.1.15  root     1829:                              l1_phys_map + i, i, &map);
                   1830:     }
                   1831:     if (map.size) {
                   1832:         client->set_memory(client, map.start_addr, map.size, map.phys_offset,
                   1833:                            false);
1.1.1.13  root     1834:     }
                   1835: }
                   1836: 
                   1837: void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
                   1838: {
                   1839:     QLIST_INSERT_HEAD(&memory_client_list, client, list);
                   1840:     phys_page_for_each(client);
                   1841: }
                   1842: 
                   1843: void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
                   1844: {
                   1845:     QLIST_REMOVE(client, list);
                   1846: }
                   1847: #endif
                   1848: 
1.1       root     1849: static int cmp1(const char *s1, int n, const char *s2)
                   1850: {
                   1851:     if (strlen(s2) != n)
                   1852:         return 0;
                   1853:     return memcmp(s1, s2, n) == 0;
                   1854: }
1.1.1.6   root     1855: 
1.1       root     1856: /* takes a comma separated list of log masks. Return 0 if error. */
                   1857: int cpu_str_to_log_mask(const char *str)
                   1858: {
1.1.1.7   root     1859:     const CPULogItem *item;
1.1       root     1860:     int mask;
                   1861:     const char *p, *p1;
                   1862: 
                   1863:     p = str;
                   1864:     mask = 0;
                   1865:     for(;;) {
                   1866:         p1 = strchr(p, ',');
                   1867:         if (!p1)
                   1868:             p1 = p + strlen(p);
1.1.1.14  root     1869:         if(cmp1(p,p1-p,"all")) {
                   1870:             for(item = cpu_log_items; item->mask != 0; item++) {
                   1871:                 mask |= item->mask;
                   1872:             }
                   1873:         } else {
                   1874:             for(item = cpu_log_items; item->mask != 0; item++) {
                   1875:                 if (cmp1(p, p1 - p, item->name))
                   1876:                     goto found;
                   1877:             }
                   1878:             return 0;
1.1       root     1879:         }
                   1880:     found:
                   1881:         mask |= item->mask;
                   1882:         if (*p1 != ',')
                   1883:             break;
                   1884:         p = p1 + 1;
                   1885:     }
                   1886:     return mask;
                   1887: }
                   1888: 
                   1889: void cpu_abort(CPUState *env, const char *fmt, ...)
                   1890: {
                   1891:     va_list ap;
1.1.1.6   root     1892:     va_list ap2;
1.1       root     1893: 
                   1894:     va_start(ap, fmt);
1.1.1.6   root     1895:     va_copy(ap2, ap);
1.1       root     1896:     fprintf(stderr, "qemu: fatal: ");
                   1897:     vfprintf(stderr, fmt, ap);
                   1898:     fprintf(stderr, "\n");
                   1899: #ifdef TARGET_I386
                   1900:     cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
                   1901: #else
                   1902:     cpu_dump_state(env, stderr, fprintf, 0);
                   1903: #endif
1.1.1.7   root     1904:     if (qemu_log_enabled()) {
                   1905:         qemu_log("qemu: fatal: ");
                   1906:         qemu_log_vprintf(fmt, ap2);
                   1907:         qemu_log("\n");
1.1.1.6   root     1908: #ifdef TARGET_I386
1.1.1.7   root     1909:         log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1.1.1.6   root     1910: #else
1.1.1.7   root     1911:         log_cpu_state(env, 0);
1.1.1.6   root     1912: #endif
1.1.1.7   root     1913:         qemu_log_flush();
                   1914:         qemu_log_close();
1.1.1.6   root     1915:     }
                   1916:     va_end(ap2);
1.1       root     1917:     va_end(ap);
1.1.1.13  root     1918: #if defined(CONFIG_USER_ONLY)
                   1919:     {
                   1920:         struct sigaction act;
                   1921:         sigfillset(&act.sa_mask);
                   1922:         act.sa_handler = SIG_DFL;
                   1923:         sigaction(SIGABRT, &act, NULL);
                   1924:     }
                   1925: #endif
1.1       root     1926:     abort();
                   1927: }
                   1928: 
1.1.1.6   root     1929: CPUState *cpu_copy(CPUState *env)
                   1930: {
                   1931:     CPUState *new_env = cpu_init(env->cpu_model_str);
                   1932:     CPUState *next_cpu = new_env->next_cpu;
                   1933:     int cpu_index = new_env->cpu_index;
1.1.1.7   root     1934: #if defined(TARGET_HAS_ICE)
                   1935:     CPUBreakpoint *bp;
                   1936:     CPUWatchpoint *wp;
                   1937: #endif
                   1938: 
1.1.1.6   root     1939:     memcpy(new_env, env, sizeof(CPUState));
1.1.1.7   root     1940: 
                   1941:     /* Preserve chaining and index. */
1.1.1.6   root     1942:     new_env->next_cpu = next_cpu;
                   1943:     new_env->cpu_index = cpu_index;
1.1.1.7   root     1944: 
                   1945:     /* Clone all break/watchpoints.
                   1946:        Note: Once we support ptrace with hw-debug register access, make sure
                   1947:        BP_CPU break/watchpoints are handled correctly on clone. */
1.1.1.11  root     1948:     QTAILQ_INIT(&env->breakpoints);
                   1949:     QTAILQ_INIT(&env->watchpoints);
1.1.1.7   root     1950: #if defined(TARGET_HAS_ICE)
1.1.1.11  root     1951:     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1.1.1.7   root     1952:         cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
                   1953:     }
1.1.1.11  root     1954:     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1.1.1.7   root     1955:         cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
                   1956:                               wp->flags, NULL);
                   1957:     }
                   1958: #endif
                   1959: 
1.1.1.6   root     1960:     return new_env;
                   1961: }
                   1962: 
1.1       root     1963: #if !defined(CONFIG_USER_ONLY)
                   1964: 
1.1.1.7   root     1965: static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
                   1966: {
                   1967:     unsigned int i;
                   1968: 
                   1969:     /* Discard jump cache entries for any tb which might potentially
                   1970:        overlap the flushed page.  */
                   1971:     i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
                   1972:     memset (&env->tb_jmp_cache[i], 0, 
1.1.1.14  root     1973:             TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1.1.1.7   root     1974: 
                   1975:     i = tb_jmp_cache_hash_page(addr);
                   1976:     memset (&env->tb_jmp_cache[i], 0, 
1.1.1.14  root     1977:             TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1.1.1.7   root     1978: }
                   1979: 
1.1.1.10  root     1980: static CPUTLBEntry s_cputlb_empty_entry = {
                   1981:     .addr_read  = -1,
                   1982:     .addr_write = -1,
                   1983:     .addr_code  = -1,
                   1984:     .addend     = -1,
                   1985: };
                   1986: 
1.1       root     1987: /* NOTE: if flush_global is true, also flush global entries (not
                   1988:    implemented yet) */
                   1989: void tlb_flush(CPUState *env, int flush_global)
                   1990: {
                   1991:     int i;
                   1992: 
                   1993: #if defined(DEBUG_TLB)
                   1994:     printf("tlb_flush:\n");
                   1995: #endif
                   1996:     /* must reset current TB so that interrupts cannot modify the
                   1997:        links while we are modifying them */
                   1998:     env->current_tb = NULL;
                   1999: 
                   2000:     for(i = 0; i < CPU_TLB_SIZE; i++) {
1.1.1.10  root     2001:         int mmu_idx;
                   2002:         for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
                   2003:             env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
                   2004:         }
1.1       root     2005:     }
                   2006: 
1.1.1.2   root     2007:     memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1.1       root     2008: 
1.1.1.13  root     2009:     env->tlb_flush_addr = -1;
                   2010:     env->tlb_flush_mask = 0;
1.1       root     2011:     tlb_flush_count++;
                   2012: }
                   2013: 
                   2014: static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
                   2015: {
1.1.1.6   root     2016:     if (addr == (tlb_entry->addr_read &
1.1.1.2   root     2017:                  (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1.1.1.6   root     2018:         addr == (tlb_entry->addr_write &
1.1.1.2   root     2019:                  (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1.1.1.6   root     2020:         addr == (tlb_entry->addr_code &
1.1.1.2   root     2021:                  (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1.1.1.10  root     2022:         *tlb_entry = s_cputlb_empty_entry;
1.1.1.2   root     2023:     }
1.1       root     2024: }
                   2025: 
                   2026: void tlb_flush_page(CPUState *env, target_ulong addr)
                   2027: {
1.1.1.2   root     2028:     int i;
1.1.1.10  root     2029:     int mmu_idx;
1.1       root     2030: 
                   2031: #if defined(DEBUG_TLB)
                   2032:     printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
                   2033: #endif
1.1.1.13  root     2034:     /* Check if we need to flush due to large pages.  */
                   2035:     if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
                   2036: #if defined(DEBUG_TLB)
                   2037:         printf("tlb_flush_page: forced full flush ("
                   2038:                TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
                   2039:                env->tlb_flush_addr, env->tlb_flush_mask);
                   2040: #endif
                   2041:         tlb_flush(env, 1);
                   2042:         return;
                   2043:     }
1.1       root     2044:     /* must reset current TB so that interrupts cannot modify the
                   2045:        links while we are modifying them */
                   2046:     env->current_tb = NULL;
                   2047: 
                   2048:     addr &= TARGET_PAGE_MASK;
                   2049:     i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1.1.1.10  root     2050:     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
                   2051:         tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
1.1       root     2052: 
1.1.1.7   root     2053:     tlb_flush_jmp_cache(env, addr);
1.1       root     2054: }
                   2055: 
                   2056: /* update the TLBs so that writes to code in the virtual page 'addr'
                   2057:    can be detected */
1.1.1.2   root     2058: static void tlb_protect_code(ram_addr_t ram_addr)
1.1       root     2059: {
1.1.1.6   root     2060:     cpu_physical_memory_reset_dirty(ram_addr,
1.1.1.2   root     2061:                                     ram_addr + TARGET_PAGE_SIZE,
                   2062:                                     CODE_DIRTY_FLAG);
1.1       root     2063: }
                   2064: 
                   2065: /* update the TLB so that writes in physical page 'phys_addr' are no longer
                   2066:    tested for self modifying code */
1.1.1.6   root     2067: static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1.1       root     2068:                                     target_ulong vaddr)
                   2069: {
1.1.1.13  root     2070:     cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
1.1       root     2071: }
                   2072: 
1.1.1.6   root     2073: static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1.1       root     2074:                                          unsigned long start, unsigned long length)
                   2075: {
                   2076:     unsigned long addr;
1.1.1.2   root     2077:     if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
                   2078:         addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1.1       root     2079:         if ((addr - start) < length) {
1.1.1.7   root     2080:             tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1.1       root     2081:         }
                   2082:     }
                   2083: }
                   2084: 
1.1.1.10  root     2085: /* Note: start and end must be within the same ram block.  */
1.1       root     2086: void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
                   2087:                                      int dirty_flags)
                   2088: {
                   2089:     CPUState *env;
                   2090:     unsigned long length, start1;
1.1.1.13  root     2091:     int i;
1.1       root     2092: 
                   2093:     start &= TARGET_PAGE_MASK;
                   2094:     end = TARGET_PAGE_ALIGN(end);
                   2095: 
                   2096:     length = end - start;
                   2097:     if (length == 0)
                   2098:         return;
1.1.1.13  root     2099:     cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
1.1       root     2100: 
                   2101:     /* we modify the TLB cache so that the dirty bit will be set again
                   2102:        when accessing the range */
1.1.1.14  root     2103:     start1 = (unsigned long)qemu_safe_ram_ptr(start);
1.1.1.15  root     2104:     /* Check that we don't span multiple blocks - this breaks the
1.1.1.10  root     2105:        address comparisons below.  */
1.1.1.14  root     2106:     if ((unsigned long)qemu_safe_ram_ptr(end - 1) - start1
1.1.1.10  root     2107:             != (end - 1) - start) {
                   2108:         abort();
                   2109:     }
                   2110: 
1.1.1.2   root     2111:     for(env = first_cpu; env != NULL; env = env->next_cpu) {
1.1.1.10  root     2112:         int mmu_idx;
                   2113:         for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
                   2114:             for(i = 0; i < CPU_TLB_SIZE; i++)
                   2115:                 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
                   2116:                                       start1, length);
                   2117:         }
1.1.1.2   root     2118:     }
1.1.1.7   root     2119: }
1.1       root     2120: 
1.1.1.7   root     2121: int cpu_physical_memory_set_dirty_tracking(int enable)
                   2122: {
1.1.1.13  root     2123:     int ret = 0;
1.1.1.7   root     2124:     in_migration = enable;
1.1.1.13  root     2125:     ret = cpu_notify_migration_log(!!enable);
                   2126:     return ret;
1.1.1.7   root     2127: }
1.1       root     2128: 
1.1.1.7   root     2129: int cpu_physical_memory_get_dirty_tracking(void)
                   2130: {
                   2131:     return in_migration;
                   2132: }
                   2133: 
1.1.1.10  root     2134: int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
                   2135:                                    target_phys_addr_t end_addr)
1.1.1.7   root     2136: {
1.1.1.13  root     2137:     int ret;
1.1.1.10  root     2138: 
1.1.1.13  root     2139:     ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
1.1.1.10  root     2140:     return ret;
1.1       root     2141: }
                   2142: 
1.1.1.15  root     2143: int cpu_physical_log_start(target_phys_addr_t start_addr,
                   2144:                            ram_addr_t size)
                   2145: {
                   2146:     CPUPhysMemoryClient *client;
                   2147:     QLIST_FOREACH(client, &memory_client_list, list) {
                   2148:         if (client->log_start) {
                   2149:             int r = client->log_start(client, start_addr, size);
                   2150:             if (r < 0) {
                   2151:                 return r;
                   2152:             }
                   2153:         }
                   2154:     }
                   2155:     return 0;
                   2156: }
                   2157: 
                   2158: int cpu_physical_log_stop(target_phys_addr_t start_addr,
                   2159:                           ram_addr_t size)
                   2160: {
                   2161:     CPUPhysMemoryClient *client;
                   2162:     QLIST_FOREACH(client, &memory_client_list, list) {
                   2163:         if (client->log_stop) {
                   2164:             int r = client->log_stop(client, start_addr, size);
                   2165:             if (r < 0) {
                   2166:                 return r;
                   2167:             }
                   2168:         }
                   2169:     }
                   2170:     return 0;
                   2171: }
                   2172: 
1.1       root     2173: static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
                   2174: {
                   2175:     ram_addr_t ram_addr;
1.1.1.10  root     2176:     void *p;
1.1       root     2177: 
1.1.1.2   root     2178:     if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1.1.1.10  root     2179:         p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
                   2180:             + tlb_entry->addend);
1.1.1.14  root     2181:         ram_addr = qemu_ram_addr_from_host_nofail(p);
1.1       root     2182:         if (!cpu_physical_memory_is_dirty(ram_addr)) {
1.1.1.7   root     2183:             tlb_entry->addr_write |= TLB_NOTDIRTY;
1.1       root     2184:         }
                   2185:     }
                   2186: }
                   2187: 
                   2188: /* update the TLB according to the current state of the dirty bits */
                   2189: void cpu_tlb_update_dirty(CPUState *env)
                   2190: {
                   2191:     int i;
1.1.1.10  root     2192:     int mmu_idx;
                   2193:     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
                   2194:         for(i = 0; i < CPU_TLB_SIZE; i++)
                   2195:             tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
                   2196:     }
1.1       root     2197: }
                   2198: 
1.1.1.7   root     2199: static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1.1       root     2200: {
1.1.1.7   root     2201:     if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
                   2202:         tlb_entry->addr_write = vaddr;
1.1       root     2203: }
                   2204: 
1.1.1.7   root     2205: /* update the TLB corresponding to virtual page vaddr
                   2206:    so that it is no longer dirty */
                   2207: static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
1.1       root     2208: {
                   2209:     int i;
1.1.1.10  root     2210:     int mmu_idx;
1.1       root     2211: 
1.1.1.7   root     2212:     vaddr &= TARGET_PAGE_MASK;
1.1       root     2213:     i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1.1.1.10  root     2214:     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
                   2215:         tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
1.1       root     2216: }
                   2217: 
1.1.1.13  root     2218: /* Our TLB does not support large pages, so remember the area covered by
                   2219:    large pages and trigger a full TLB flush if these are invalidated.  */
                   2220: static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
                   2221:                                target_ulong size)
                   2222: {
                   2223:     target_ulong mask = ~(size - 1);
                   2224: 
                   2225:     if (env->tlb_flush_addr == (target_ulong)-1) {
                   2226:         env->tlb_flush_addr = vaddr & mask;
                   2227:         env->tlb_flush_mask = mask;
                   2228:         return;
                   2229:     }
                   2230:     /* Extend the existing region to include the new page.
                   2231:        This is a compromise between unnecessary flushes and the cost
                   2232:        of maintaining a full variable size TLB.  */
                   2233:     mask &= env->tlb_flush_mask;
                   2234:     while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
                   2235:         mask <<= 1;
                   2236:     }
                   2237:     env->tlb_flush_addr &= mask;
                   2238:     env->tlb_flush_mask = mask;
                   2239: }
                   2240: 
                   2241: /* Add a new TLB entry. At most one entry for a given virtual address
                   2242:    is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
                   2243:    supplied size is only used by tlb_flush_page.  */
                   2244: void tlb_set_page(CPUState *env, target_ulong vaddr,
                   2245:                   target_phys_addr_t paddr, int prot,
                   2246:                   int mmu_idx, target_ulong size)
1.1       root     2247: {
                   2248:     PhysPageDesc *p;
                   2249:     unsigned long pd;
                   2250:     unsigned int index;
                   2251:     target_ulong address;
1.1.1.7   root     2252:     target_ulong code_address;
1.1.1.13  root     2253:     unsigned long addend;
1.1.1.2   root     2254:     CPUTLBEntry *te;
1.1.1.7   root     2255:     CPUWatchpoint *wp;
                   2256:     target_phys_addr_t iotlb;
1.1       root     2257: 
1.1.1.13  root     2258:     assert(size >= TARGET_PAGE_SIZE);
                   2259:     if (size != TARGET_PAGE_SIZE) {
                   2260:         tlb_add_large_page(env, vaddr, size);
                   2261:     }
1.1       root     2262:     p = phys_page_find(paddr >> TARGET_PAGE_BITS);
                   2263:     if (!p) {
                   2264:         pd = IO_MEM_UNASSIGNED;
                   2265:     } else {
                   2266:         pd = p->phys_offset;
                   2267:     }
                   2268: #if defined(DEBUG_TLB)
1.1.1.14  root     2269:     printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
                   2270:            " prot=%x idx=%d pd=0x%08lx\n",
                   2271:            vaddr, paddr, prot, mmu_idx, pd);
1.1       root     2272: #endif
                   2273: 
1.1.1.7   root     2274:     address = vaddr;
                   2275:     if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
                   2276:         /* IO memory case (romd handled later) */
                   2277:         address |= TLB_MMIO;
                   2278:     }
1.1.1.10  root     2279:     addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
1.1.1.7   root     2280:     if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
                   2281:         /* Normal RAM.  */
                   2282:         iotlb = pd & TARGET_PAGE_MASK;
                   2283:         if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
                   2284:             iotlb |= IO_MEM_NOTDIRTY;
                   2285:         else
                   2286:             iotlb |= IO_MEM_ROM;
                   2287:     } else {
1.1.1.10  root     2288:         /* IO handlers are currently passed a physical address.
1.1.1.7   root     2289:            It would be nice to pass an offset from the base address
                   2290:            of that region.  This would avoid having to special case RAM,
                   2291:            and avoid full address decoding in every device.
                   2292:            We can't use the high bits of pd for this because
                   2293:            IO_MEM_ROMD uses these as a ram address.  */
                   2294:         iotlb = (pd & ~TARGET_PAGE_MASK);
                   2295:         if (p) {
                   2296:             iotlb += p->region_offset;
1.1       root     2297:         } else {
1.1.1.7   root     2298:             iotlb += paddr;
1.1       root     2299:         }
1.1.1.7   root     2300:     }
1.1.1.6   root     2301: 
1.1.1.7   root     2302:     code_address = address;
                   2303:     /* Make accesses to pages with watchpoints go via the
                   2304:        watchpoint trap routines.  */
1.1.1.11  root     2305:     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1.1.1.7   root     2306:         if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
1.1.1.13  root     2307:             /* Avoid trapping reads of pages with a write breakpoint. */
                   2308:             if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
                   2309:                 iotlb = io_mem_watch + paddr;
                   2310:                 address |= TLB_MMIO;
                   2311:                 break;
                   2312:             }
1.1.1.6   root     2313:         }
1.1.1.7   root     2314:     }
1.1.1.6   root     2315: 
1.1.1.7   root     2316:     index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
                   2317:     env->iotlb[mmu_idx][index] = iotlb - vaddr;
                   2318:     te = &env->tlb_table[mmu_idx][index];
                   2319:     te->addend = addend - vaddr;
                   2320:     if (prot & PAGE_READ) {
                   2321:         te->addr_read = address;
                   2322:     } else {
                   2323:         te->addr_read = -1;
1.1       root     2324:     }
                   2325: 
1.1.1.7   root     2326:     if (prot & PAGE_EXEC) {
                   2327:         te->addr_code = code_address;
                   2328:     } else {
                   2329:         te->addr_code = -1;
                   2330:     }
                   2331:     if (prot & PAGE_WRITE) {
                   2332:         if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
                   2333:             (pd & IO_MEM_ROMD)) {
                   2334:             /* Write access calls the I/O callback.  */
                   2335:             te->addr_write = address | TLB_MMIO;
                   2336:         } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
                   2337:                    !cpu_physical_memory_is_dirty(pd)) {
                   2338:             te->addr_write = address | TLB_NOTDIRTY;
                   2339:         } else {
                   2340:             te->addr_write = address;
1.1       root     2341:         }
1.1.1.7   root     2342:     } else {
                   2343:         te->addr_write = -1;
1.1       root     2344:     }
                   2345: }
                   2346: 
                   2347: #else
                   2348: 
                   2349: void tlb_flush(CPUState *env, int flush_global)
                   2350: {
                   2351: }
                   2352: 
                   2353: void tlb_flush_page(CPUState *env, target_ulong addr)
                   2354: {
                   2355: }
                   2356: 
1.1.1.10  root     2357: /*
                   2358:  * Walks guest process memory "regions" one by one
                   2359:  * and calls callback function 'fn' for each region.
                   2360:  */
1.1.1.13  root     2361: 
                   2362: struct walk_memory_regions_data
                   2363: {
                   2364:     walk_memory_regions_fn fn;
                   2365:     void *priv;
                   2366:     unsigned long start;
                   2367:     int prot;
                   2368: };
                   2369: 
                   2370: static int walk_memory_regions_end(struct walk_memory_regions_data *data,
                   2371:                                    abi_ulong end, int new_prot)
1.1       root     2372: {
1.1.1.13  root     2373:     if (data->start != -1ul) {
                   2374:         int rc = data->fn(data->priv, data->start, end, data->prot);
                   2375:         if (rc != 0) {
                   2376:             return rc;
                   2377:         }
                   2378:     }
                   2379: 
                   2380:     data->start = (new_prot ? end : -1ul);
                   2381:     data->prot = new_prot;
                   2382: 
                   2383:     return 0;
                   2384: }
                   2385: 
                   2386: static int walk_memory_regions_1(struct walk_memory_regions_data *data,
                   2387:                                  abi_ulong base, int level, void **lp)
                   2388: {
                   2389:     abi_ulong pa;
                   2390:     int i, rc;
                   2391: 
                   2392:     if (*lp == NULL) {
                   2393:         return walk_memory_regions_end(data, base, 0);
                   2394:     }
                   2395: 
                   2396:     if (level == 0) {
                   2397:         PageDesc *pd = *lp;
                   2398:         for (i = 0; i < L2_SIZE; ++i) {
                   2399:             int prot = pd[i].flags;
                   2400: 
                   2401:             pa = base | (i << TARGET_PAGE_BITS);
                   2402:             if (prot != data->prot) {
                   2403:                 rc = walk_memory_regions_end(data, pa, prot);
                   2404:                 if (rc != 0) {
                   2405:                     return rc;
1.1       root     2406:                 }
                   2407:             }
1.1.1.13  root     2408:         }
                   2409:     } else {
                   2410:         void **pp = *lp;
                   2411:         for (i = 0; i < L2_SIZE; ++i) {
                   2412:             pa = base | ((abi_ulong)i <<
                   2413:                 (TARGET_PAGE_BITS + L2_BITS * level));
                   2414:             rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
                   2415:             if (rc != 0) {
                   2416:                 return rc;
                   2417:             }
1.1       root     2418:         }
                   2419:     }
1.1.1.13  root     2420: 
                   2421:     return 0;
1.1.1.10  root     2422: }
                   2423: 
1.1.1.13  root     2424: int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
                   2425: {
                   2426:     struct walk_memory_regions_data data;
                   2427:     unsigned long i;
                   2428: 
                   2429:     data.fn = fn;
                   2430:     data.priv = priv;
                   2431:     data.start = -1ul;
                   2432:     data.prot = 0;
                   2433: 
                   2434:     for (i = 0; i < V_L1_SIZE; i++) {
                   2435:         int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
                   2436:                                        V_L1_SHIFT / L2_BITS - 1, l1_map + i);
                   2437:         if (rc != 0) {
                   2438:             return rc;
                   2439:         }
                   2440:     }
                   2441: 
                   2442:     return walk_memory_regions_end(&data, 0, 0);
                   2443: }
                   2444: 
                   2445: static int dump_region(void *priv, abi_ulong start,
                   2446:     abi_ulong end, unsigned long prot)
1.1.1.10  root     2447: {
                   2448:     FILE *f = (FILE *)priv;
                   2449: 
1.1.1.13  root     2450:     (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
                   2451:         " "TARGET_ABI_FMT_lx" %c%c%c\n",
1.1.1.10  root     2452:         start, end, end - start,
                   2453:         ((prot & PAGE_READ) ? 'r' : '-'),
                   2454:         ((prot & PAGE_WRITE) ? 'w' : '-'),
                   2455:         ((prot & PAGE_EXEC) ? 'x' : '-'));
                   2456: 
                   2457:     return (0);
                   2458: }
                   2459: 
                   2460: /* dump memory mappings */
                   2461: void page_dump(FILE *f)
                   2462: {
                   2463:     (void) fprintf(f, "%-8s %-8s %-8s %s\n",
                   2464:             "start", "end", "size", "prot");
                   2465:     walk_memory_regions(f, dump_region);
1.1       root     2466: }
                   2467: 
1.1.1.3   root     2468: int page_get_flags(target_ulong address)
1.1       root     2469: {
                   2470:     PageDesc *p;
                   2471: 
                   2472:     p = page_find(address >> TARGET_PAGE_BITS);
                   2473:     if (!p)
                   2474:         return 0;
                   2475:     return p->flags;
                   2476: }
                   2477: 
1.1.1.13  root     2478: /* Modify the flags of a page and invalidate the code if necessary.
                   2479:    The flag PAGE_WRITE_ORG is positioned automatically depending
                   2480:    on PAGE_WRITE.  The mmap_lock should already be held.  */
1.1.1.3   root     2481: void page_set_flags(target_ulong start, target_ulong end, int flags)
1.1       root     2482: {
1.1.1.13  root     2483:     target_ulong addr, len;
                   2484: 
                   2485:     /* This function should never be called with addresses outside the
                   2486:        guest address space.  If this assert fires, it probably indicates
                   2487:        a missing call to h2g_valid.  */
                   2488: #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
                   2489:     assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
                   2490: #endif
                   2491:     assert(start < end);
1.1       root     2492: 
                   2493:     start = start & TARGET_PAGE_MASK;
                   2494:     end = TARGET_PAGE_ALIGN(end);
1.1.1.13  root     2495: 
                   2496:     if (flags & PAGE_WRITE) {
1.1       root     2497:         flags |= PAGE_WRITE_ORG;
1.1.1.13  root     2498:     }
                   2499: 
                   2500:     for (addr = start, len = end - start;
                   2501:          len != 0;
                   2502:          len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
                   2503:         PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
                   2504: 
                   2505:         /* If the write protection bit is set, then we invalidate
                   2506:            the code inside.  */
1.1.1.6   root     2507:         if (!(p->flags & PAGE_WRITE) &&
1.1       root     2508:             (flags & PAGE_WRITE) &&
                   2509:             p->first_tb) {
                   2510:             tb_invalidate_phys_page(addr, 0, NULL);
                   2511:         }
                   2512:         p->flags = flags;
                   2513:     }
                   2514: }
                   2515: 
1.1.1.6   root     2516: int page_check_range(target_ulong start, target_ulong len, int flags)
                   2517: {
                   2518:     PageDesc *p;
                   2519:     target_ulong end;
                   2520:     target_ulong addr;
                   2521: 
1.1.1.13  root     2522:     /* This function should never be called with addresses outside the
                   2523:        guest address space.  If this assert fires, it probably indicates
                   2524:        a missing call to h2g_valid.  */
                   2525: #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
                   2526:     assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
                   2527: #endif
                   2528: 
                   2529:     if (len == 0) {
                   2530:         return 0;
                   2531:     }
                   2532:     if (start + len - 1 < start) {
                   2533:         /* We've wrapped around.  */
1.1.1.7   root     2534:         return -1;
1.1.1.13  root     2535:     }
1.1.1.7   root     2536: 
1.1.1.6   root     2537:     end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
                   2538:     start = start & TARGET_PAGE_MASK;
                   2539: 
1.1.1.13  root     2540:     for (addr = start, len = end - start;
                   2541:          len != 0;
                   2542:          len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1.1.1.6   root     2543:         p = page_find(addr >> TARGET_PAGE_BITS);
                   2544:         if( !p )
                   2545:             return -1;
                   2546:         if( !(p->flags & PAGE_VALID) )
                   2547:             return -1;
                   2548: 
                   2549:         if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
                   2550:             return -1;
                   2551:         if (flags & PAGE_WRITE) {
                   2552:             if (!(p->flags & PAGE_WRITE_ORG))
                   2553:                 return -1;
                   2554:             /* unprotect the page if it was put read-only because it
                   2555:                contains translated code */
                   2556:             if (!(p->flags & PAGE_WRITE)) {
                   2557:                 if (!page_unprotect(addr, 0, NULL))
                   2558:                     return -1;
                   2559:             }
                   2560:             return 0;
                   2561:         }
                   2562:     }
                   2563:     return 0;
                   2564: }
                   2565: 
1.1       root     2566: /* called from signal handler: invalidate the code and unprotect the
1.1.1.10  root     2567:    page. Return TRUE if the fault was successfully handled. */
1.1.1.3   root     2568: int page_unprotect(target_ulong address, unsigned long pc, void *puc)
1.1       root     2569: {
1.1.1.13  root     2570:     unsigned int prot;
                   2571:     PageDesc *p;
1.1.1.3   root     2572:     target_ulong host_start, host_end, addr;
1.1       root     2573: 
1.1.1.7   root     2574:     /* Technically this isn't safe inside a signal handler.  However we
                   2575:        know this only ever happens in a synchronous SEGV handler, so in
                   2576:        practice it seems to be ok.  */
                   2577:     mmap_lock();
                   2578: 
1.1.1.13  root     2579:     p = page_find(address >> TARGET_PAGE_BITS);
                   2580:     if (!p) {
1.1.1.7   root     2581:         mmap_unlock();
1.1       root     2582:         return 0;
1.1.1.7   root     2583:     }
1.1.1.13  root     2584: 
1.1       root     2585:     /* if the page was really writable, then we change its
                   2586:        protection back to writable */
1.1.1.13  root     2587:     if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
                   2588:         host_start = address & qemu_host_page_mask;
                   2589:         host_end = host_start + qemu_host_page_size;
                   2590: 
                   2591:         prot = 0;
                   2592:         for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
                   2593:             p = page_find(addr >> TARGET_PAGE_BITS);
                   2594:             p->flags |= PAGE_WRITE;
                   2595:             prot |= p->flags;
                   2596: 
1.1       root     2597:             /* and since the content will be modified, we must invalidate
                   2598:                the corresponding translated code. */
1.1.1.13  root     2599:             tb_invalidate_phys_page(addr, pc, puc);
1.1       root     2600: #ifdef DEBUG_TB_CHECK
1.1.1.13  root     2601:             tb_invalidate_check(addr);
1.1       root     2602: #endif
                   2603:         }
1.1.1.13  root     2604:         mprotect((void *)g2h(host_start), qemu_host_page_size,
                   2605:                  prot & PAGE_BITS);
                   2606: 
                   2607:         mmap_unlock();
                   2608:         return 1;
1.1       root     2609:     }
1.1.1.7   root     2610:     mmap_unlock();
1.1       root     2611:     return 0;
                   2612: }
                   2613: 
1.1.1.2   root     2614: static inline void tlb_set_dirty(CPUState *env,
                   2615:                                  unsigned long addr, target_ulong vaddr)
1.1       root     2616: {
                   2617: }
                   2618: #endif /* defined(CONFIG_USER_ONLY) */
                   2619: 
1.1.1.7   root     2620: #if !defined(CONFIG_USER_ONLY)
                   2621: 
1.1.1.13  root     2622: #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
                   2623: typedef struct subpage_t {
                   2624:     target_phys_addr_t base;
                   2625:     ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
                   2626:     ram_addr_t region_offset[TARGET_PAGE_SIZE];
                   2627: } subpage_t;
                   2628: 
1.1.1.6   root     2629: static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1.1.1.7   root     2630:                              ram_addr_t memory, ram_addr_t region_offset);
1.1.1.13  root     2631: static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
                   2632:                                 ram_addr_t orig_memory,
                   2633:                                 ram_addr_t region_offset);
1.1.1.6   root     2634: #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
                   2635:                       need_subpage)                                     \
                   2636:     do {                                                                \
                   2637:         if (addr > start_addr)                                          \
                   2638:             start_addr2 = 0;                                            \
                   2639:         else {                                                          \
                   2640:             start_addr2 = start_addr & ~TARGET_PAGE_MASK;               \
                   2641:             if (start_addr2 > 0)                                        \
                   2642:                 need_subpage = 1;                                       \
                   2643:         }                                                               \
                   2644:                                                                         \
                   2645:         if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE)        \
                   2646:             end_addr2 = TARGET_PAGE_SIZE - 1;                           \
                   2647:         else {                                                          \
                   2648:             end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
                   2649:             if (end_addr2 < TARGET_PAGE_SIZE - 1)                       \
                   2650:                 need_subpage = 1;                                       \
                   2651:         }                                                               \
                   2652:     } while (0)
                   2653: 
1.1.1.11  root     2654: /* register physical memory.
                   2655:    For RAM, 'size' must be a multiple of the target page size.
                   2656:    If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
1.1.1.7   root     2657:    io memory page.  The address used when calling the IO function is
                   2658:    the offset from the start of the region, plus region_offset.  Both
1.1.1.10  root     2659:    start_addr and region_offset are rounded down to a page boundary
1.1.1.7   root     2660:    before calculating this offset.  This should not be a problem unless
                   2661:    the low bits of start_addr and region_offset differ.  */
1.1.1.15  root     2662: void cpu_register_physical_memory_log(target_phys_addr_t start_addr,
1.1.1.7   root     2663:                                          ram_addr_t size,
                   2664:                                          ram_addr_t phys_offset,
1.1.1.15  root     2665:                                          ram_addr_t region_offset,
                   2666:                                          bool log_dirty)
1.1       root     2667: {
                   2668:     target_phys_addr_t addr, end_addr;
                   2669:     PhysPageDesc *p;
1.1.1.4   root     2670:     CPUState *env;
1.1.1.7   root     2671:     ram_addr_t orig_size = size;
1.1.1.13  root     2672:     subpage_t *subpage;
1.1       root     2673: 
1.1.1.15  root     2674:     assert(size);
                   2675:     cpu_notify_set_memory(start_addr, size, phys_offset, log_dirty);
1.1.1.7   root     2676: 
                   2677:     if (phys_offset == IO_MEM_UNASSIGNED) {
                   2678:         region_offset = start_addr;
                   2679:     }
                   2680:     region_offset &= TARGET_PAGE_MASK;
1.1       root     2681:     size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
1.1.1.6   root     2682:     end_addr = start_addr + (target_phys_addr_t)size;
1.1.1.15  root     2683: 
                   2684:     addr = start_addr;
                   2685:     do {
1.1.1.6   root     2686:         p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   2687:         if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
1.1.1.7   root     2688:             ram_addr_t orig_memory = p->phys_offset;
1.1.1.6   root     2689:             target_phys_addr_t start_addr2, end_addr2;
                   2690:             int need_subpage = 0;
                   2691: 
                   2692:             CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
                   2693:                           need_subpage);
1.1.1.13  root     2694:             if (need_subpage) {
1.1.1.6   root     2695:                 if (!(orig_memory & IO_MEM_SUBPAGE)) {
                   2696:                     subpage = subpage_init((addr & TARGET_PAGE_MASK),
1.1.1.7   root     2697:                                            &p->phys_offset, orig_memory,
                   2698:                                            p->region_offset);
1.1.1.6   root     2699:                 } else {
                   2700:                     subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
                   2701:                                             >> IO_MEM_SHIFT];
                   2702:                 }
1.1.1.7   root     2703:                 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
                   2704:                                  region_offset);
                   2705:                 p->region_offset = 0;
1.1.1.6   root     2706:             } else {
                   2707:                 p->phys_offset = phys_offset;
                   2708:                 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
                   2709:                     (phys_offset & IO_MEM_ROMD))
                   2710:                     phys_offset += TARGET_PAGE_SIZE;
                   2711:             }
                   2712:         } else {
                   2713:             p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
                   2714:             p->phys_offset = phys_offset;
1.1.1.7   root     2715:             p->region_offset = region_offset;
1.1.1.6   root     2716:             if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
1.1.1.7   root     2717:                 (phys_offset & IO_MEM_ROMD)) {
1.1.1.6   root     2718:                 phys_offset += TARGET_PAGE_SIZE;
1.1.1.7   root     2719:             } else {
1.1.1.6   root     2720:                 target_phys_addr_t start_addr2, end_addr2;
                   2721:                 int need_subpage = 0;
                   2722: 
                   2723:                 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
                   2724:                               end_addr2, need_subpage);
                   2725: 
1.1.1.13  root     2726:                 if (need_subpage) {
1.1.1.6   root     2727:                     subpage = subpage_init((addr & TARGET_PAGE_MASK),
1.1.1.7   root     2728:                                            &p->phys_offset, IO_MEM_UNASSIGNED,
                   2729:                                            addr & TARGET_PAGE_MASK);
1.1.1.6   root     2730:                     subpage_register(subpage, start_addr2, end_addr2,
1.1.1.7   root     2731:                                      phys_offset, region_offset);
                   2732:                     p->region_offset = 0;
1.1.1.6   root     2733:                 }
                   2734:             }
                   2735:         }
1.1.1.7   root     2736:         region_offset += TARGET_PAGE_SIZE;
1.1.1.15  root     2737:         addr += TARGET_PAGE_SIZE;
                   2738:     } while (addr != end_addr);
1.1.1.6   root     2739: 
1.1.1.4   root     2740:     /* since each CPU stores ram addresses in its TLB cache, we must
                   2741:        reset the modified entries */
                   2742:     /* XXX: slow ! */
                   2743:     for(env = first_cpu; env != NULL; env = env->next_cpu) {
                   2744:         tlb_flush(env, 1);
                   2745:     }
1.1       root     2746: }
                   2747: 
1.1.1.5   root     2748: /* XXX: temporary until new memory mapping API */
1.1.1.7   root     2749: ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
1.1.1.5   root     2750: {
                   2751:     PhysPageDesc *p;
                   2752: 
                   2753:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   2754:     if (!p)
                   2755:         return IO_MEM_UNASSIGNED;
                   2756:     return p->phys_offset;
                   2757: }
                   2758: 
1.1.1.7   root     2759: void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
                   2760: {
                   2761:     if (kvm_enabled())
                   2762:         kvm_coalesce_mmio_region(addr, size);
                   2763: }
                   2764: 
                   2765: void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
                   2766: {
                   2767:     if (kvm_enabled())
                   2768:         kvm_uncoalesce_mmio_region(addr, size);
                   2769: }
                   2770: 
1.1.1.13  root     2771: void qemu_flush_coalesced_mmio_buffer(void)
1.1.1.10  root     2772: {
1.1.1.13  root     2773:     if (kvm_enabled())
                   2774:         kvm_flush_coalesced_mmio_buffer();
                   2775: }
                   2776: 
                   2777: #if defined(__linux__) && !defined(TARGET_S390X)
                   2778: 
                   2779: #include <sys/vfs.h>
                   2780: 
                   2781: #define HUGETLBFS_MAGIC       0x958458f6
                   2782: 
                   2783: static long gethugepagesize(const char *path)
                   2784: {
                   2785:     struct statfs fs;
                   2786:     int ret;
                   2787: 
                   2788:     do {
1.1.1.14  root     2789:         ret = statfs(path, &fs);
1.1.1.13  root     2790:     } while (ret != 0 && errno == EINTR);
                   2791: 
                   2792:     if (ret != 0) {
1.1.1.14  root     2793:         perror(path);
                   2794:         return 0;
1.1.1.13  root     2795:     }
                   2796: 
                   2797:     if (fs.f_type != HUGETLBFS_MAGIC)
1.1.1.14  root     2798:         fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
1.1.1.13  root     2799: 
                   2800:     return fs.f_bsize;
                   2801: }
                   2802: 
                   2803: static void *file_ram_alloc(RAMBlock *block,
                   2804:                             ram_addr_t memory,
                   2805:                             const char *path)
                   2806: {
                   2807:     char *filename;
                   2808:     void *area;
                   2809:     int fd;
                   2810: #ifdef MAP_POPULATE
                   2811:     int flags;
                   2812: #endif
                   2813:     unsigned long hpagesize;
                   2814: 
                   2815:     hpagesize = gethugepagesize(path);
                   2816:     if (!hpagesize) {
1.1.1.14  root     2817:         return NULL;
1.1.1.13  root     2818:     }
                   2819: 
                   2820:     if (memory < hpagesize) {
                   2821:         return NULL;
                   2822:     }
                   2823: 
                   2824:     if (kvm_enabled() && !kvm_has_sync_mmu()) {
                   2825:         fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
                   2826:         return NULL;
                   2827:     }
                   2828: 
                   2829:     if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
1.1.1.14  root     2830:         return NULL;
1.1.1.13  root     2831:     }
                   2832: 
                   2833:     fd = mkstemp(filename);
                   2834:     if (fd < 0) {
1.1.1.14  root     2835:         perror("unable to create backing store for hugepages");
                   2836:         free(filename);
                   2837:         return NULL;
1.1.1.13  root     2838:     }
                   2839:     unlink(filename);
                   2840:     free(filename);
                   2841: 
                   2842:     memory = (memory+hpagesize-1) & ~(hpagesize-1);
                   2843: 
                   2844:     /*
                   2845:      * ftruncate is not supported by hugetlbfs in older
                   2846:      * hosts, so don't bother bailing out on errors.
                   2847:      * If anything goes wrong with it under other filesystems,
                   2848:      * mmap will fail.
                   2849:      */
                   2850:     if (ftruncate(fd, memory))
1.1.1.14  root     2851:         perror("ftruncate");
1.1.1.13  root     2852: 
                   2853: #ifdef MAP_POPULATE
                   2854:     /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
                   2855:      * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
                   2856:      * to sidestep this quirk.
                   2857:      */
                   2858:     flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
                   2859:     area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
                   2860: #else
                   2861:     area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
                   2862: #endif
                   2863:     if (area == MAP_FAILED) {
1.1.1.14  root     2864:         perror("file_ram_alloc: can't mmap RAM pages");
                   2865:         close(fd);
                   2866:         return (NULL);
1.1.1.13  root     2867:     }
                   2868:     block->fd = fd;
                   2869:     return area;
                   2870: }
                   2871: #endif
                   2872: 
                   2873: static ram_addr_t find_ram_offset(ram_addr_t size)
                   2874: {
                   2875:     RAMBlock *block, *next_block;
1.1.1.16! root     2876:     ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1.1.1.13  root     2877: 
                   2878:     if (QLIST_EMPTY(&ram_list.blocks))
                   2879:         return 0;
                   2880: 
                   2881:     QLIST_FOREACH(block, &ram_list.blocks, next) {
1.1.1.16! root     2882:         ram_addr_t end, next = RAM_ADDR_MAX;
1.1.1.13  root     2883: 
                   2884:         end = block->offset + block->length;
                   2885: 
                   2886:         QLIST_FOREACH(next_block, &ram_list.blocks, next) {
                   2887:             if (next_block->offset >= end) {
                   2888:                 next = MIN(next, next_block->offset);
                   2889:             }
                   2890:         }
                   2891:         if (next - end >= size && next - end < mingap) {
1.1.1.16! root     2892:             offset = end;
1.1.1.13  root     2893:             mingap = next - end;
                   2894:         }
                   2895:     }
1.1.1.16! root     2896: 
        !          2897:     if (offset == RAM_ADDR_MAX) {
        !          2898:         fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
        !          2899:                 (uint64_t)size);
        !          2900:         abort();
        !          2901:     }
        !          2902: 
1.1.1.13  root     2903:     return offset;
                   2904: }
                   2905: 
                   2906: static ram_addr_t last_ram_offset(void)
                   2907: {
                   2908:     RAMBlock *block;
                   2909:     ram_addr_t last = 0;
                   2910: 
                   2911:     QLIST_FOREACH(block, &ram_list.blocks, next)
                   2912:         last = MAX(last, block->offset + block->length);
                   2913: 
                   2914:     return last;
                   2915: }
                   2916: 
                   2917: ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
1.1.1.14  root     2918:                                    ram_addr_t size, void *host)
1.1.1.13  root     2919: {
                   2920:     RAMBlock *new_block, *block;
                   2921: 
                   2922:     size = TARGET_PAGE_ALIGN(size);
1.1.1.16! root     2923:     new_block = g_malloc0(sizeof(*new_block));
1.1.1.13  root     2924: 
                   2925:     if (dev && dev->parent_bus && dev->parent_bus->info->get_dev_path) {
                   2926:         char *id = dev->parent_bus->info->get_dev_path(dev);
                   2927:         if (id) {
                   2928:             snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1.1.1.16! root     2929:             g_free(id);
1.1.1.13  root     2930:         }
                   2931:     }
                   2932:     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
                   2933: 
                   2934:     QLIST_FOREACH(block, &ram_list.blocks, next) {
                   2935:         if (!strcmp(block->idstr, new_block->idstr)) {
                   2936:             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
                   2937:                     new_block->idstr);
                   2938:             abort();
                   2939:         }
                   2940:     }
                   2941: 
1.1.1.15  root     2942:     new_block->offset = find_ram_offset(size);
1.1.1.14  root     2943:     if (host) {
                   2944:         new_block->host = host;
1.1.1.15  root     2945:         new_block->flags |= RAM_PREALLOC_MASK;
1.1.1.14  root     2946:     } else {
                   2947:         if (mem_path) {
1.1.1.13  root     2948: #if defined (__linux__) && !defined(TARGET_S390X)
1.1.1.14  root     2949:             new_block->host = file_ram_alloc(new_block, size, mem_path);
                   2950:             if (!new_block->host) {
                   2951:                 new_block->host = qemu_vmalloc(size);
                   2952:                 qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
                   2953:             }
1.1.1.13  root     2954: #else
1.1.1.14  root     2955:             fprintf(stderr, "-mem-path option unsupported\n");
                   2956:             exit(1);
1.1.1.13  root     2957: #endif
1.1.1.14  root     2958:         } else {
1.1.1.11  root     2959: #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1.1.1.15  root     2960:             /* S390 KVM requires the topmost vma of the RAM to be smaller than
                   2961:                an system defined value, which is at least 256GB. Larger systems
                   2962:                have larger values. We put the guest between the end of data
                   2963:                segment (system break) and this value. We use 32GB as a base to
                   2964:                have enough room for the system break to grow. */
                   2965:             new_block->host = mmap((void*)0x800000000, size,
1.1.1.14  root     2966:                                    PROT_EXEC|PROT_READ|PROT_WRITE,
1.1.1.15  root     2967:                                    MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
                   2968:             if (new_block->host == MAP_FAILED) {
                   2969:                 fprintf(stderr, "Allocating RAM failed\n");
                   2970:                 abort();
                   2971:             }
1.1.1.11  root     2972: #else
1.1.1.15  root     2973:             if (xen_enabled()) {
                   2974:                 xen_ram_alloc(new_block->offset, size);
                   2975:             } else {
                   2976:                 new_block->host = qemu_vmalloc(size);
                   2977:             }
1.1.1.11  root     2978: #endif
1.1.1.14  root     2979:             qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
                   2980:         }
1.1.1.13  root     2981:     }
1.1.1.10  root     2982:     new_block->length = size;
                   2983: 
1.1.1.13  root     2984:     QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
1.1.1.10  root     2985: 
1.1.1.16! root     2986:     ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1.1.1.13  root     2987:                                        last_ram_offset() >> TARGET_PAGE_BITS);
                   2988:     memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1.1.1.10  root     2989:            0xff, size >> TARGET_PAGE_BITS);
                   2990: 
1.1.1.9   root     2991:     if (kvm_enabled())
1.1.1.10  root     2992:         kvm_setup_guest_memory(new_block->host, size);
1.1.1.9   root     2993: 
1.1.1.10  root     2994:     return new_block->offset;
1.1.1.6   root     2995: }
                   2996: 
1.1.1.14  root     2997: ram_addr_t qemu_ram_alloc(DeviceState *dev, const char *name, ram_addr_t size)
                   2998: {
                   2999:     return qemu_ram_alloc_from_ptr(dev, name, size, NULL);
                   3000: }
                   3001: 
1.1.1.15  root     3002: void qemu_ram_free_from_ptr(ram_addr_t addr)
                   3003: {
                   3004:     RAMBlock *block;
                   3005: 
                   3006:     QLIST_FOREACH(block, &ram_list.blocks, next) {
                   3007:         if (addr == block->offset) {
                   3008:             QLIST_REMOVE(block, next);
1.1.1.16! root     3009:             g_free(block);
1.1.1.15  root     3010:             return;
                   3011:         }
                   3012:     }
                   3013: }
                   3014: 
1.1.1.6   root     3015: void qemu_ram_free(ram_addr_t addr)
                   3016: {
1.1.1.13  root     3017:     RAMBlock *block;
                   3018: 
                   3019:     QLIST_FOREACH(block, &ram_list.blocks, next) {
                   3020:         if (addr == block->offset) {
                   3021:             QLIST_REMOVE(block, next);
1.1.1.15  root     3022:             if (block->flags & RAM_PREALLOC_MASK) {
                   3023:                 ;
                   3024:             } else if (mem_path) {
1.1.1.13  root     3025: #if defined (__linux__) && !defined(TARGET_S390X)
                   3026:                 if (block->fd) {
                   3027:                     munmap(block->host, block->length);
                   3028:                     close(block->fd);
                   3029:                 } else {
                   3030:                     qemu_vfree(block->host);
                   3031:                 }
1.1.1.15  root     3032: #else
                   3033:                 abort();
1.1.1.13  root     3034: #endif
                   3035:             } else {
                   3036: #if defined(TARGET_S390X) && defined(CONFIG_KVM)
                   3037:                 munmap(block->host, block->length);
                   3038: #else
1.1.1.15  root     3039:                 if (xen_enabled()) {
                   3040:                     xen_invalidate_map_cache_entry(block->host);
                   3041:                 } else {
                   3042:                     qemu_vfree(block->host);
                   3043:                 }
1.1.1.13  root     3044: #endif
                   3045:             }
1.1.1.16! root     3046:             g_free(block);
1.1.1.13  root     3047:             return;
                   3048:         }
                   3049:     }
                   3050: 
1.1.1.10  root     3051: }
                   3052: 
1.1.1.15  root     3053: #ifndef _WIN32
                   3054: void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
                   3055: {
                   3056:     RAMBlock *block;
                   3057:     ram_addr_t offset;
                   3058:     int flags;
                   3059:     void *area, *vaddr;
                   3060: 
                   3061:     QLIST_FOREACH(block, &ram_list.blocks, next) {
                   3062:         offset = addr - block->offset;
                   3063:         if (offset < block->length) {
                   3064:             vaddr = block->host + offset;
                   3065:             if (block->flags & RAM_PREALLOC_MASK) {
                   3066:                 ;
                   3067:             } else {
                   3068:                 flags = MAP_FIXED;
                   3069:                 munmap(vaddr, length);
                   3070:                 if (mem_path) {
                   3071: #if defined(__linux__) && !defined(TARGET_S390X)
                   3072:                     if (block->fd) {
                   3073: #ifdef MAP_POPULATE
                   3074:                         flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
                   3075:                             MAP_PRIVATE;
                   3076: #else
                   3077:                         flags |= MAP_PRIVATE;
                   3078: #endif
                   3079:                         area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
                   3080:                                     flags, block->fd, offset);
                   3081:                     } else {
                   3082:                         flags |= MAP_PRIVATE | MAP_ANONYMOUS;
                   3083:                         area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
                   3084:                                     flags, -1, 0);
                   3085:                     }
                   3086: #else
                   3087:                     abort();
                   3088: #endif
                   3089:                 } else {
                   3090: #if defined(TARGET_S390X) && defined(CONFIG_KVM)
                   3091:                     flags |= MAP_SHARED | MAP_ANONYMOUS;
                   3092:                     area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
                   3093:                                 flags, -1, 0);
                   3094: #else
                   3095:                     flags |= MAP_PRIVATE | MAP_ANONYMOUS;
                   3096:                     area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
                   3097:                                 flags, -1, 0);
                   3098: #endif
                   3099:                 }
                   3100:                 if (area != vaddr) {
1.1.1.16! root     3101:                     fprintf(stderr, "Could not remap addr: "
        !          3102:                             RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1.1.1.15  root     3103:                             length, addr);
                   3104:                     exit(1);
                   3105:                 }
                   3106:                 qemu_madvise(vaddr, length, QEMU_MADV_MERGEABLE);
                   3107:             }
                   3108:             return;
                   3109:         }
                   3110:     }
                   3111: }
                   3112: #endif /* !_WIN32 */
                   3113: 
1.1.1.10  root     3114: /* Return a host pointer to ram allocated with qemu_ram_alloc.
                   3115:    With the exception of the softmmu code in this file, this should
                   3116:    only be used for local memory (e.g. video ram) that the device owns,
                   3117:    and knows it isn't going to access beyond the end of the block.
                   3118: 
                   3119:    It should not be used for general purpose DMA.
                   3120:    Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
                   3121:  */
                   3122: void *qemu_get_ram_ptr(ram_addr_t addr)
                   3123: {
                   3124:     RAMBlock *block;
                   3125: 
1.1.1.13  root     3126:     QLIST_FOREACH(block, &ram_list.blocks, next) {
                   3127:         if (addr - block->offset < block->length) {
1.1.1.15  root     3128:             /* Move this entry to to start of the list.  */
                   3129:             if (block != QLIST_FIRST(&ram_list.blocks)) {
                   3130:                 QLIST_REMOVE(block, next);
                   3131:                 QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
                   3132:             }
                   3133:             if (xen_enabled()) {
                   3134:                 /* We need to check if the requested address is in the RAM
                   3135:                  * because we don't want to map the entire memory in QEMU.
                   3136:                  * In that case just map until the end of the page.
                   3137:                  */
                   3138:                 if (block->offset == 0) {
                   3139:                     return xen_map_cache(addr, 0, 0);
                   3140:                 } else if (block->host == NULL) {
                   3141:                     block->host =
                   3142:                         xen_map_cache(block->offset, block->length, 1);
                   3143:                 }
                   3144:             }
1.1.1.13  root     3145:             return block->host + (addr - block->offset);
                   3146:         }
1.1.1.10  root     3147:     }
1.1.1.13  root     3148: 
                   3149:     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
                   3150:     abort();
                   3151: 
                   3152:     return NULL;
1.1.1.10  root     3153: }
                   3154: 
1.1.1.14  root     3155: /* Return a host pointer to ram allocated with qemu_ram_alloc.
                   3156:  * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
                   3157:  */
                   3158: void *qemu_safe_ram_ptr(ram_addr_t addr)
                   3159: {
                   3160:     RAMBlock *block;
                   3161: 
                   3162:     QLIST_FOREACH(block, &ram_list.blocks, next) {
                   3163:         if (addr - block->offset < block->length) {
1.1.1.15  root     3164:             if (xen_enabled()) {
                   3165:                 /* We need to check if the requested address is in the RAM
                   3166:                  * because we don't want to map the entire memory in QEMU.
                   3167:                  * In that case just map until the end of the page.
                   3168:                  */
                   3169:                 if (block->offset == 0) {
                   3170:                     return xen_map_cache(addr, 0, 0);
                   3171:                 } else if (block->host == NULL) {
                   3172:                     block->host =
                   3173:                         xen_map_cache(block->offset, block->length, 1);
                   3174:                 }
                   3175:             }
1.1.1.14  root     3176:             return block->host + (addr - block->offset);
                   3177:         }
                   3178:     }
                   3179: 
                   3180:     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
                   3181:     abort();
                   3182: 
                   3183:     return NULL;
                   3184: }
                   3185: 
1.1.1.15  root     3186: /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
                   3187:  * but takes a size argument */
                   3188: void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
                   3189: {
                   3190:     if (*size == 0) {
                   3191:         return NULL;
                   3192:     }
                   3193:     if (xen_enabled()) {
                   3194:         return xen_map_cache(addr, *size, 1);
                   3195:     } else {
                   3196:         RAMBlock *block;
                   3197: 
                   3198:         QLIST_FOREACH(block, &ram_list.blocks, next) {
                   3199:             if (addr - block->offset < block->length) {
                   3200:                 if (addr - block->offset + *size > block->length)
                   3201:                     *size = block->length - addr + block->offset;
                   3202:                 return block->host + (addr - block->offset);
                   3203:             }
                   3204:         }
                   3205: 
                   3206:         fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
                   3207:         abort();
                   3208:     }
                   3209: }
                   3210: 
                   3211: void qemu_put_ram_ptr(void *addr)
                   3212: {
                   3213:     trace_qemu_put_ram_ptr(addr);
                   3214: }
                   3215: 
1.1.1.14  root     3216: int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1.1.1.10  root     3217: {
                   3218:     RAMBlock *block;
                   3219:     uint8_t *host = ptr;
                   3220: 
1.1.1.15  root     3221:     if (xen_enabled()) {
                   3222:         *ram_addr = xen_ram_addr_from_mapcache(ptr);
                   3223:         return 0;
                   3224:     }
                   3225: 
1.1.1.13  root     3226:     QLIST_FOREACH(block, &ram_list.blocks, next) {
1.1.1.15  root     3227:         /* This case append when the block is not mapped. */
                   3228:         if (block->host == NULL) {
                   3229:             continue;
                   3230:         }
1.1.1.13  root     3231:         if (host - block->host < block->length) {
1.1.1.14  root     3232:             *ram_addr = block->offset + (host - block->host);
                   3233:             return 0;
1.1.1.13  root     3234:         }
1.1.1.10  root     3235:     }
1.1.1.15  root     3236: 
1.1.1.14  root     3237:     return -1;
                   3238: }
1.1.1.13  root     3239: 
1.1.1.14  root     3240: /* Some of the softmmu routines need to translate from a host pointer
                   3241:    (typically a TLB entry) back to a ram offset.  */
                   3242: ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
                   3243: {
                   3244:     ram_addr_t ram_addr;
1.1.1.13  root     3245: 
1.1.1.14  root     3246:     if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
                   3247:         fprintf(stderr, "Bad ram pointer %p\n", ptr);
                   3248:         abort();
                   3249:     }
                   3250:     return ram_addr;
1.1.1.6   root     3251: }
                   3252: 
1.1       root     3253: static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
                   3254: {
1.1.1.5   root     3255: #ifdef DEBUG_UNASSIGNED
1.1.1.6   root     3256:     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
                   3257: #endif
1.1.1.15  root     3258: #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
                   3259:     cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, 1);
1.1.1.7   root     3260: #endif
                   3261:     return 0;
                   3262: }
                   3263: 
                   3264: static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
                   3265: {
                   3266: #ifdef DEBUG_UNASSIGNED
                   3267:     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
                   3268: #endif
1.1.1.15  root     3269: #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
                   3270:     cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, 2);
1.1.1.7   root     3271: #endif
                   3272:     return 0;
                   3273: }
                   3274: 
                   3275: static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
                   3276: {
                   3277: #ifdef DEBUG_UNASSIGNED
                   3278:     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
                   3279: #endif
1.1.1.15  root     3280: #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
                   3281:     cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, 4);
1.1.1.5   root     3282: #endif
1.1       root     3283:     return 0;
                   3284: }
                   3285: 
                   3286: static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
                   3287: {
1.1.1.5   root     3288: #ifdef DEBUG_UNASSIGNED
1.1.1.6   root     3289:     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
                   3290: #endif
1.1.1.15  root     3291: #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
                   3292:     cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, 1);
1.1.1.7   root     3293: #endif
                   3294: }
                   3295: 
                   3296: static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
                   3297: {
                   3298: #ifdef DEBUG_UNASSIGNED
                   3299:     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
                   3300: #endif
1.1.1.15  root     3301: #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
                   3302:     cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, 2);
1.1.1.7   root     3303: #endif
                   3304: }
                   3305: 
                   3306: static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
                   3307: {
                   3308: #ifdef DEBUG_UNASSIGNED
                   3309:     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
                   3310: #endif
1.1.1.15  root     3311: #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
                   3312:     cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, 4);
1.1.1.5   root     3313: #endif
1.1       root     3314: }
                   3315: 
1.1.1.11  root     3316: static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
1.1       root     3317:     unassigned_mem_readb,
1.1.1.7   root     3318:     unassigned_mem_readw,
                   3319:     unassigned_mem_readl,
1.1       root     3320: };
                   3321: 
1.1.1.11  root     3322: static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
1.1       root     3323:     unassigned_mem_writeb,
1.1.1.7   root     3324:     unassigned_mem_writew,
                   3325:     unassigned_mem_writel,
1.1       root     3326: };
                   3327: 
1.1.1.7   root     3328: static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
                   3329:                                 uint32_t val)
1.1       root     3330: {
                   3331:     int dirty_flags;
1.1.1.13  root     3332:     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1.1       root     3333:     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
                   3334: #if !defined(CONFIG_USER_ONLY)
                   3335:         tb_invalidate_phys_page_fast(ram_addr, 1);
1.1.1.13  root     3336:         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1.1       root     3337: #endif
                   3338:     }
1.1.1.10  root     3339:     stb_p(qemu_get_ram_ptr(ram_addr), val);
1.1       root     3340:     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1.1.1.13  root     3341:     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1.1       root     3342:     /* we remove the notdirty callback only if the code has been
                   3343:        flushed */
                   3344:     if (dirty_flags == 0xff)
1.1.1.7   root     3345:         tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1.1       root     3346: }
                   3347: 
1.1.1.7   root     3348: static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
                   3349:                                 uint32_t val)
1.1       root     3350: {
                   3351:     int dirty_flags;
1.1.1.13  root     3352:     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1.1       root     3353:     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
                   3354: #if !defined(CONFIG_USER_ONLY)
                   3355:         tb_invalidate_phys_page_fast(ram_addr, 2);
1.1.1.13  root     3356:         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1.1       root     3357: #endif
                   3358:     }
1.1.1.10  root     3359:     stw_p(qemu_get_ram_ptr(ram_addr), val);
1.1       root     3360:     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1.1.1.13  root     3361:     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1.1       root     3362:     /* we remove the notdirty callback only if the code has been
                   3363:        flushed */
                   3364:     if (dirty_flags == 0xff)
1.1.1.7   root     3365:         tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1.1       root     3366: }
                   3367: 
1.1.1.7   root     3368: static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
                   3369:                                 uint32_t val)
1.1       root     3370: {
                   3371:     int dirty_flags;
1.1.1.13  root     3372:     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1.1       root     3373:     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
                   3374: #if !defined(CONFIG_USER_ONLY)
                   3375:         tb_invalidate_phys_page_fast(ram_addr, 4);
1.1.1.13  root     3376:         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1.1       root     3377: #endif
                   3378:     }
1.1.1.10  root     3379:     stl_p(qemu_get_ram_ptr(ram_addr), val);
1.1       root     3380:     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1.1.1.13  root     3381:     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1.1       root     3382:     /* we remove the notdirty callback only if the code has been
                   3383:        flushed */
                   3384:     if (dirty_flags == 0xff)
1.1.1.7   root     3385:         tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1.1       root     3386: }
                   3387: 
1.1.1.11  root     3388: static CPUReadMemoryFunc * const error_mem_read[3] = {
1.1       root     3389:     NULL, /* never used */
                   3390:     NULL, /* never used */
                   3391:     NULL, /* never used */
                   3392: };
                   3393: 
1.1.1.11  root     3394: static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
1.1       root     3395:     notdirty_mem_writeb,
                   3396:     notdirty_mem_writew,
                   3397:     notdirty_mem_writel,
                   3398: };
                   3399: 
1.1.1.7   root     3400: /* Generate a debug exception if a watchpoint has been hit.  */
                   3401: static void check_watchpoint(int offset, int len_mask, int flags)
                   3402: {
                   3403:     CPUState *env = cpu_single_env;
                   3404:     target_ulong pc, cs_base;
                   3405:     TranslationBlock *tb;
                   3406:     target_ulong vaddr;
                   3407:     CPUWatchpoint *wp;
                   3408:     int cpu_flags;
                   3409: 
                   3410:     if (env->watchpoint_hit) {
                   3411:         /* We re-entered the check after replacing the TB. Now raise
                   3412:          * the debug interrupt so that is will trigger after the
                   3413:          * current instruction. */
                   3414:         cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
                   3415:         return;
                   3416:     }
                   3417:     vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1.1.1.11  root     3418:     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1.1.1.7   root     3419:         if ((vaddr == (wp->vaddr & len_mask) ||
                   3420:              (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
                   3421:             wp->flags |= BP_WATCHPOINT_HIT;
                   3422:             if (!env->watchpoint_hit) {
                   3423:                 env->watchpoint_hit = wp;
                   3424:                 tb = tb_find_pc(env->mem_io_pc);
                   3425:                 if (!tb) {
                   3426:                     cpu_abort(env, "check_watchpoint: could not find TB for "
                   3427:                               "pc=%p", (void *)env->mem_io_pc);
                   3428:                 }
1.1.1.15  root     3429:                 cpu_restore_state(tb, env, env->mem_io_pc);
1.1.1.7   root     3430:                 tb_phys_invalidate(tb, -1);
                   3431:                 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
                   3432:                     env->exception_index = EXCP_DEBUG;
                   3433:                 } else {
                   3434:                     cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
                   3435:                     tb_gen_code(env, pc, cs_base, cpu_flags, 1);
                   3436:                 }
                   3437:                 cpu_resume_from_signal(env, NULL);
                   3438:             }
                   3439:         } else {
                   3440:             wp->flags &= ~BP_WATCHPOINT_HIT;
                   3441:         }
                   3442:     }
                   3443: }
                   3444: 
1.1.1.6   root     3445: /* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
                   3446:    so these check for a hit then pass through to the normal out-of-line
                   3447:    phys routines.  */
                   3448: static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
                   3449: {
1.1.1.7   root     3450:     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
1.1.1.6   root     3451:     return ldub_phys(addr);
                   3452: }
                   3453: 
                   3454: static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
                   3455: {
1.1.1.7   root     3456:     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
1.1.1.6   root     3457:     return lduw_phys(addr);
                   3458: }
                   3459: 
                   3460: static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
                   3461: {
1.1.1.7   root     3462:     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
1.1.1.6   root     3463:     return ldl_phys(addr);
                   3464: }
                   3465: 
                   3466: static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
                   3467:                              uint32_t val)
                   3468: {
1.1.1.7   root     3469:     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
1.1.1.6   root     3470:     stb_phys(addr, val);
                   3471: }
                   3472: 
                   3473: static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
                   3474:                              uint32_t val)
                   3475: {
1.1.1.7   root     3476:     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
1.1.1.6   root     3477:     stw_phys(addr, val);
                   3478: }
                   3479: 
                   3480: static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
                   3481:                              uint32_t val)
                   3482: {
1.1.1.7   root     3483:     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
1.1.1.6   root     3484:     stl_phys(addr, val);
                   3485: }
                   3486: 
1.1.1.11  root     3487: static CPUReadMemoryFunc * const watch_mem_read[3] = {
1.1.1.6   root     3488:     watch_mem_readb,
                   3489:     watch_mem_readw,
                   3490:     watch_mem_readl,
                   3491: };
                   3492: 
1.1.1.11  root     3493: static CPUWriteMemoryFunc * const watch_mem_write[3] = {
1.1.1.6   root     3494:     watch_mem_writeb,
                   3495:     watch_mem_writew,
                   3496:     watch_mem_writel,
                   3497: };
                   3498: 
1.1.1.13  root     3499: static inline uint32_t subpage_readlen (subpage_t *mmio,
                   3500:                                         target_phys_addr_t addr,
                   3501:                                         unsigned int len)
1.1.1.6   root     3502: {
1.1.1.13  root     3503:     unsigned int idx = SUBPAGE_IDX(addr);
1.1.1.6   root     3504: #if defined(DEBUG_SUBPAGE)
                   3505:     printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
                   3506:            mmio, len, addr, idx);
                   3507: #endif
                   3508: 
1.1.1.13  root     3509:     addr += mmio->region_offset[idx];
                   3510:     idx = mmio->sub_io_index[idx];
                   3511:     return io_mem_read[idx][len](io_mem_opaque[idx], addr);
1.1.1.6   root     3512: }
                   3513: 
                   3514: static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
1.1.1.13  root     3515:                                      uint32_t value, unsigned int len)
1.1.1.6   root     3516: {
1.1.1.13  root     3517:     unsigned int idx = SUBPAGE_IDX(addr);
1.1.1.6   root     3518: #if defined(DEBUG_SUBPAGE)
1.1.1.13  root     3519:     printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n",
                   3520:            __func__, mmio, len, addr, idx, value);
1.1.1.6   root     3521: #endif
1.1.1.13  root     3522: 
                   3523:     addr += mmio->region_offset[idx];
                   3524:     idx = mmio->sub_io_index[idx];
                   3525:     io_mem_write[idx][len](io_mem_opaque[idx], addr, value);
1.1.1.6   root     3526: }
                   3527: 
                   3528: static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
                   3529: {
                   3530:     return subpage_readlen(opaque, addr, 0);
                   3531: }
                   3532: 
                   3533: static void subpage_writeb (void *opaque, target_phys_addr_t addr,
                   3534:                             uint32_t value)
                   3535: {
                   3536:     subpage_writelen(opaque, addr, value, 0);
                   3537: }
                   3538: 
                   3539: static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
                   3540: {
                   3541:     return subpage_readlen(opaque, addr, 1);
                   3542: }
                   3543: 
                   3544: static void subpage_writew (void *opaque, target_phys_addr_t addr,
                   3545:                             uint32_t value)
                   3546: {
                   3547:     subpage_writelen(opaque, addr, value, 1);
                   3548: }
                   3549: 
                   3550: static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
                   3551: {
                   3552:     return subpage_readlen(opaque, addr, 2);
                   3553: }
                   3554: 
1.1.1.13  root     3555: static void subpage_writel (void *opaque, target_phys_addr_t addr,
                   3556:                             uint32_t value)
1.1.1.6   root     3557: {
                   3558:     subpage_writelen(opaque, addr, value, 2);
                   3559: }
                   3560: 
1.1.1.11  root     3561: static CPUReadMemoryFunc * const subpage_read[] = {
1.1.1.6   root     3562:     &subpage_readb,
                   3563:     &subpage_readw,
                   3564:     &subpage_readl,
                   3565: };
                   3566: 
1.1.1.11  root     3567: static CPUWriteMemoryFunc * const subpage_write[] = {
1.1.1.6   root     3568:     &subpage_writeb,
                   3569:     &subpage_writew,
                   3570:     &subpage_writel,
                   3571: };
                   3572: 
1.1.1.16! root     3573: static uint32_t subpage_ram_readb(void *opaque, target_phys_addr_t addr)
        !          3574: {
        !          3575:     ram_addr_t raddr = addr;
        !          3576:     void *ptr = qemu_get_ram_ptr(raddr);
        !          3577:     return ldub_p(ptr);
        !          3578: }
        !          3579: 
        !          3580: static void subpage_ram_writeb(void *opaque, target_phys_addr_t addr,
        !          3581:                                uint32_t value)
        !          3582: {
        !          3583:     ram_addr_t raddr = addr;
        !          3584:     void *ptr = qemu_get_ram_ptr(raddr);
        !          3585:     stb_p(ptr, value);
        !          3586: }
        !          3587: 
        !          3588: static uint32_t subpage_ram_readw(void *opaque, target_phys_addr_t addr)
        !          3589: {
        !          3590:     ram_addr_t raddr = addr;
        !          3591:     void *ptr = qemu_get_ram_ptr(raddr);
        !          3592:     return lduw_p(ptr);
        !          3593: }
        !          3594: 
        !          3595: static void subpage_ram_writew(void *opaque, target_phys_addr_t addr,
        !          3596:                                uint32_t value)
        !          3597: {
        !          3598:     ram_addr_t raddr = addr;
        !          3599:     void *ptr = qemu_get_ram_ptr(raddr);
        !          3600:     stw_p(ptr, value);
        !          3601: }
        !          3602: 
        !          3603: static uint32_t subpage_ram_readl(void *opaque, target_phys_addr_t addr)
        !          3604: {
        !          3605:     ram_addr_t raddr = addr;
        !          3606:     void *ptr = qemu_get_ram_ptr(raddr);
        !          3607:     return ldl_p(ptr);
        !          3608: }
        !          3609: 
        !          3610: static void subpage_ram_writel(void *opaque, target_phys_addr_t addr,
        !          3611:                                uint32_t value)
        !          3612: {
        !          3613:     ram_addr_t raddr = addr;
        !          3614:     void *ptr = qemu_get_ram_ptr(raddr);
        !          3615:     stl_p(ptr, value);
        !          3616: }
        !          3617: 
        !          3618: static CPUReadMemoryFunc * const subpage_ram_read[] = {
        !          3619:     &subpage_ram_readb,
        !          3620:     &subpage_ram_readw,
        !          3621:     &subpage_ram_readl,
        !          3622: };
        !          3623: 
        !          3624: static CPUWriteMemoryFunc * const subpage_ram_write[] = {
        !          3625:     &subpage_ram_writeb,
        !          3626:     &subpage_ram_writew,
        !          3627:     &subpage_ram_writel,
        !          3628: };
        !          3629: 
1.1.1.6   root     3630: static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1.1.1.7   root     3631:                              ram_addr_t memory, ram_addr_t region_offset)
1.1.1.6   root     3632: {
                   3633:     int idx, eidx;
                   3634: 
                   3635:     if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
                   3636:         return -1;
                   3637:     idx = SUBPAGE_IDX(start);
                   3638:     eidx = SUBPAGE_IDX(end);
                   3639: #if defined(DEBUG_SUBPAGE)
1.1.1.10  root     3640:     printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1.1.1.6   root     3641:            mmio, start, end, idx, eidx, memory);
                   3642: #endif
1.1.1.16! root     3643:     if ((memory & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
        !          3644:         memory = IO_MEM_SUBPAGE_RAM;
        !          3645:     }
1.1.1.13  root     3646:     memory = (memory >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
1.1.1.6   root     3647:     for (; idx <= eidx; idx++) {
1.1.1.13  root     3648:         mmio->sub_io_index[idx] = memory;
                   3649:         mmio->region_offset[idx] = region_offset;
1.1.1.6   root     3650:     }
                   3651: 
                   3652:     return 0;
                   3653: }
                   3654: 
1.1.1.13  root     3655: static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
                   3656:                                 ram_addr_t orig_memory,
                   3657:                                 ram_addr_t region_offset)
1.1.1.6   root     3658: {
                   3659:     subpage_t *mmio;
                   3660:     int subpage_memory;
                   3661: 
1.1.1.16! root     3662:     mmio = g_malloc0(sizeof(subpage_t));
1.1.1.7   root     3663: 
                   3664:     mmio->base = base;
1.1.1.14  root     3665:     subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio,
                   3666:                                             DEVICE_NATIVE_ENDIAN);
1.1.1.6   root     3667: #if defined(DEBUG_SUBPAGE)
1.1.1.7   root     3668:     printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
                   3669:            mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1.1.1.6   root     3670: #endif
1.1.1.7   root     3671:     *phys = subpage_memory | IO_MEM_SUBPAGE;
1.1.1.13  root     3672:     subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset);
1.1.1.6   root     3673: 
                   3674:     return mmio;
                   3675: }
                   3676: 
1.1.1.7   root     3677: static int get_free_io_mem_idx(void)
                   3678: {
                   3679:     int i;
                   3680: 
                   3681:     for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
                   3682:         if (!io_mem_used[i]) {
                   3683:             io_mem_used[i] = 1;
                   3684:             return i;
                   3685:         }
1.1.1.13  root     3686:     fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
1.1.1.7   root     3687:     return -1;
                   3688: }
                   3689: 
1.1.1.14  root     3690: /*
                   3691:  * Usually, devices operate in little endian mode. There are devices out
                   3692:  * there that operate in big endian too. Each device gets byte swapped
                   3693:  * mmio if plugged onto a CPU that does the other endianness.
                   3694:  *
                   3695:  * CPU          Device           swap?
                   3696:  *
                   3697:  * little       little           no
                   3698:  * little       big              yes
                   3699:  * big          little           yes
                   3700:  * big          big              no
                   3701:  */
                   3702: 
                   3703: typedef struct SwapEndianContainer {
                   3704:     CPUReadMemoryFunc *read[3];
                   3705:     CPUWriteMemoryFunc *write[3];
                   3706:     void *opaque;
                   3707: } SwapEndianContainer;
                   3708: 
                   3709: static uint32_t swapendian_mem_readb (void *opaque, target_phys_addr_t addr)
                   3710: {
                   3711:     uint32_t val;
                   3712:     SwapEndianContainer *c = opaque;
                   3713:     val = c->read[0](c->opaque, addr);
                   3714:     return val;
                   3715: }
                   3716: 
                   3717: static uint32_t swapendian_mem_readw(void *opaque, target_phys_addr_t addr)
                   3718: {
                   3719:     uint32_t val;
                   3720:     SwapEndianContainer *c = opaque;
                   3721:     val = bswap16(c->read[1](c->opaque, addr));
                   3722:     return val;
                   3723: }
                   3724: 
                   3725: static uint32_t swapendian_mem_readl(void *opaque, target_phys_addr_t addr)
                   3726: {
                   3727:     uint32_t val;
                   3728:     SwapEndianContainer *c = opaque;
                   3729:     val = bswap32(c->read[2](c->opaque, addr));
                   3730:     return val;
                   3731: }
                   3732: 
                   3733: static CPUReadMemoryFunc * const swapendian_readfn[3]={
                   3734:     swapendian_mem_readb,
                   3735:     swapendian_mem_readw,
                   3736:     swapendian_mem_readl
                   3737: };
                   3738: 
                   3739: static void swapendian_mem_writeb(void *opaque, target_phys_addr_t addr,
                   3740:                                   uint32_t val)
                   3741: {
                   3742:     SwapEndianContainer *c = opaque;
                   3743:     c->write[0](c->opaque, addr, val);
                   3744: }
                   3745: 
                   3746: static void swapendian_mem_writew(void *opaque, target_phys_addr_t addr,
                   3747:                                   uint32_t val)
                   3748: {
                   3749:     SwapEndianContainer *c = opaque;
                   3750:     c->write[1](c->opaque, addr, bswap16(val));
                   3751: }
                   3752: 
                   3753: static void swapendian_mem_writel(void *opaque, target_phys_addr_t addr,
                   3754:                                   uint32_t val)
                   3755: {
                   3756:     SwapEndianContainer *c = opaque;
                   3757:     c->write[2](c->opaque, addr, bswap32(val));
                   3758: }
                   3759: 
                   3760: static CPUWriteMemoryFunc * const swapendian_writefn[3]={
                   3761:     swapendian_mem_writeb,
                   3762:     swapendian_mem_writew,
                   3763:     swapendian_mem_writel
                   3764: };
                   3765: 
                   3766: static void swapendian_init(int io_index)
                   3767: {
1.1.1.16! root     3768:     SwapEndianContainer *c = g_malloc(sizeof(SwapEndianContainer));
1.1.1.14  root     3769:     int i;
                   3770: 
                   3771:     /* Swap mmio for big endian targets */
                   3772:     c->opaque = io_mem_opaque[io_index];
                   3773:     for (i = 0; i < 3; i++) {
                   3774:         c->read[i] = io_mem_read[io_index][i];
                   3775:         c->write[i] = io_mem_write[io_index][i];
                   3776: 
                   3777:         io_mem_read[io_index][i] = swapendian_readfn[i];
                   3778:         io_mem_write[io_index][i] = swapendian_writefn[i];
                   3779:     }
                   3780:     io_mem_opaque[io_index] = c;
                   3781: }
                   3782: 
                   3783: static void swapendian_del(int io_index)
                   3784: {
                   3785:     if (io_mem_read[io_index][0] == swapendian_readfn[0]) {
1.1.1.16! root     3786:         g_free(io_mem_opaque[io_index]);
1.1.1.14  root     3787:     }
                   3788: }
                   3789: 
1.1       root     3790: /* mem_read and mem_write are arrays of functions containing the
                   3791:    function to access byte (index 0), word (index 1) and dword (index
1.1.1.10  root     3792:    2). Functions can be omitted with a NULL function pointer.
1.1.1.6   root     3793:    If io_index is non zero, the corresponding io zone is
                   3794:    modified. If it is zero, a new io zone is allocated. The return
                   3795:    value can be used with cpu_register_physical_memory(). (-1) is
                   3796:    returned if error. */
1.1.1.10  root     3797: static int cpu_register_io_memory_fixed(int io_index,
1.1.1.11  root     3798:                                         CPUReadMemoryFunc * const *mem_read,
                   3799:                                         CPUWriteMemoryFunc * const *mem_write,
1.1.1.14  root     3800:                                         void *opaque, enum device_endian endian)
1.1       root     3801: {
1.1.1.13  root     3802:     int i;
1.1       root     3803: 
                   3804:     if (io_index <= 0) {
1.1.1.7   root     3805:         io_index = get_free_io_mem_idx();
                   3806:         if (io_index == -1)
                   3807:             return io_index;
1.1       root     3808:     } else {
1.1.1.10  root     3809:         io_index >>= IO_MEM_SHIFT;
1.1       root     3810:         if (io_index >= IO_MEM_NB_ENTRIES)
                   3811:             return -1;
                   3812:     }
1.1.1.2   root     3813: 
1.1.1.13  root     3814:     for (i = 0; i < 3; ++i) {
                   3815:         io_mem_read[io_index][i]
                   3816:             = (mem_read[i] ? mem_read[i] : unassigned_mem_read[i]);
                   3817:     }
                   3818:     for (i = 0; i < 3; ++i) {
                   3819:         io_mem_write[io_index][i]
                   3820:             = (mem_write[i] ? mem_write[i] : unassigned_mem_write[i]);
1.1       root     3821:     }
                   3822:     io_mem_opaque[io_index] = opaque;
1.1.1.13  root     3823: 
1.1.1.14  root     3824:     switch (endian) {
                   3825:     case DEVICE_BIG_ENDIAN:
                   3826: #ifndef TARGET_WORDS_BIGENDIAN
                   3827:         swapendian_init(io_index);
                   3828: #endif
                   3829:         break;
                   3830:     case DEVICE_LITTLE_ENDIAN:
                   3831: #ifdef TARGET_WORDS_BIGENDIAN
                   3832:         swapendian_init(io_index);
                   3833: #endif
                   3834:         break;
                   3835:     case DEVICE_NATIVE_ENDIAN:
                   3836:     default:
                   3837:         break;
                   3838:     }
                   3839: 
1.1.1.13  root     3840:     return (io_index << IO_MEM_SHIFT);
1.1       root     3841: }
                   3842: 
1.1.1.11  root     3843: int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
                   3844:                            CPUWriteMemoryFunc * const *mem_write,
1.1.1.14  root     3845:                            void *opaque, enum device_endian endian)
1.1.1.10  root     3846: {
1.1.1.14  root     3847:     return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque, endian);
1.1.1.10  root     3848: }
                   3849: 
1.1.1.7   root     3850: void cpu_unregister_io_memory(int io_table_address)
                   3851: {
                   3852:     int i;
                   3853:     int io_index = io_table_address >> IO_MEM_SHIFT;
                   3854: 
1.1.1.14  root     3855:     swapendian_del(io_index);
                   3856: 
1.1.1.7   root     3857:     for (i=0;i < 3; i++) {
                   3858:         io_mem_read[io_index][i] = unassigned_mem_read[i];
                   3859:         io_mem_write[io_index][i] = unassigned_mem_write[i];
                   3860:     }
                   3861:     io_mem_opaque[io_index] = NULL;
                   3862:     io_mem_used[io_index] = 0;
                   3863: }
                   3864: 
1.1.1.10  root     3865: static void io_mem_init(void)
1.1       root     3866: {
1.1.1.10  root     3867:     int i;
1.1       root     3868: 
1.1.1.14  root     3869:     cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read,
                   3870:                                  unassigned_mem_write, NULL,
                   3871:                                  DEVICE_NATIVE_ENDIAN);
                   3872:     cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read,
                   3873:                                  unassigned_mem_write, NULL,
                   3874:                                  DEVICE_NATIVE_ENDIAN);
                   3875:     cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read,
                   3876:                                  notdirty_mem_write, NULL,
                   3877:                                  DEVICE_NATIVE_ENDIAN);
1.1.1.16! root     3878:     cpu_register_io_memory_fixed(IO_MEM_SUBPAGE_RAM, subpage_ram_read,
        !          3879:                                  subpage_ram_write, NULL,
        !          3880:                                  DEVICE_NATIVE_ENDIAN);
1.1.1.10  root     3881:     for (i=0; i<5; i++)
                   3882:         io_mem_used[i] = 1;
                   3883: 
                   3884:     io_mem_watch = cpu_register_io_memory(watch_mem_read,
1.1.1.14  root     3885:                                           watch_mem_write, NULL,
                   3886:                                           DEVICE_NATIVE_ENDIAN);
1.1       root     3887: }
                   3888: 
1.1.1.16! root     3889: static void memory_map_init(void)
        !          3890: {
        !          3891:     system_memory = g_malloc(sizeof(*system_memory));
        !          3892:     memory_region_init(system_memory, "system", INT64_MAX);
        !          3893:     set_system_memory_map(system_memory);
        !          3894: 
        !          3895:     system_io = g_malloc(sizeof(*system_io));
        !          3896:     memory_region_init(system_io, "io", 65536);
        !          3897:     set_system_io_map(system_io);
        !          3898: }
        !          3899: 
        !          3900: MemoryRegion *get_system_memory(void)
        !          3901: {
        !          3902:     return system_memory;
        !          3903: }
        !          3904: 
        !          3905: MemoryRegion *get_system_io(void)
        !          3906: {
        !          3907:     return system_io;
        !          3908: }
        !          3909: 
1.1.1.7   root     3910: #endif /* !defined(CONFIG_USER_ONLY) */
                   3911: 
1.1       root     3912: /* physical memory access (slow version, mainly for debug) */
                   3913: #if defined(CONFIG_USER_ONLY)
1.1.1.13  root     3914: int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
                   3915:                         uint8_t *buf, int len, int is_write)
1.1       root     3916: {
                   3917:     int l, flags;
                   3918:     target_ulong page;
1.1.1.3   root     3919:     void * p;
1.1       root     3920: 
                   3921:     while (len > 0) {
                   3922:         page = addr & TARGET_PAGE_MASK;
                   3923:         l = (page + TARGET_PAGE_SIZE) - addr;
                   3924:         if (l > len)
                   3925:             l = len;
                   3926:         flags = page_get_flags(page);
                   3927:         if (!(flags & PAGE_VALID))
1.1.1.13  root     3928:             return -1;
1.1       root     3929:         if (is_write) {
                   3930:             if (!(flags & PAGE_WRITE))
1.1.1.13  root     3931:                 return -1;
1.1.1.6   root     3932:             /* XXX: this code should not depend on lock_user */
1.1.1.7   root     3933:             if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1.1.1.13  root     3934:                 return -1;
1.1.1.7   root     3935:             memcpy(p, buf, l);
                   3936:             unlock_user(p, addr, l);
1.1       root     3937:         } else {
                   3938:             if (!(flags & PAGE_READ))
1.1.1.13  root     3939:                 return -1;
1.1.1.6   root     3940:             /* XXX: this code should not depend on lock_user */
1.1.1.7   root     3941:             if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1.1.1.13  root     3942:                 return -1;
1.1.1.7   root     3943:             memcpy(buf, p, l);
1.1.1.3   root     3944:             unlock_user(p, addr, 0);
1.1       root     3945:         }
                   3946:         len -= l;
                   3947:         buf += l;
                   3948:         addr += l;
                   3949:     }
1.1.1.13  root     3950:     return 0;
1.1       root     3951: }
                   3952: 
                   3953: #else
1.1.1.6   root     3954: void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
1.1       root     3955:                             int len, int is_write)
                   3956: {
                   3957:     int l, io_index;
                   3958:     uint8_t *ptr;
                   3959:     uint32_t val;
                   3960:     target_phys_addr_t page;
1.1.1.16! root     3961:     ram_addr_t pd;
1.1       root     3962:     PhysPageDesc *p;
1.1.1.6   root     3963: 
1.1       root     3964:     while (len > 0) {
                   3965:         page = addr & TARGET_PAGE_MASK;
                   3966:         l = (page + TARGET_PAGE_SIZE) - addr;
                   3967:         if (l > len)
                   3968:             l = len;
                   3969:         p = phys_page_find(page >> TARGET_PAGE_BITS);
                   3970:         if (!p) {
                   3971:             pd = IO_MEM_UNASSIGNED;
                   3972:         } else {
                   3973:             pd = p->phys_offset;
                   3974:         }
1.1.1.6   root     3975: 
1.1       root     3976:         if (is_write) {
                   3977:             if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
1.1.1.7   root     3978:                 target_phys_addr_t addr1 = addr;
1.1       root     3979:                 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
1.1.1.7   root     3980:                 if (p)
                   3981:                     addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
1.1.1.2   root     3982:                 /* XXX: could force cpu_single_env to NULL to avoid
                   3983:                    potential bugs */
1.1.1.7   root     3984:                 if (l >= 4 && ((addr1 & 3) == 0)) {
1.1       root     3985:                     /* 32 bit write access */
                   3986:                     val = ldl_p(buf);
1.1.1.7   root     3987:                     io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
1.1       root     3988:                     l = 4;
1.1.1.7   root     3989:                 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1.1       root     3990:                     /* 16 bit write access */
                   3991:                     val = lduw_p(buf);
1.1.1.7   root     3992:                     io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
1.1       root     3993:                     l = 2;
                   3994:                 } else {
                   3995:                     /* 8 bit write access */
                   3996:                     val = ldub_p(buf);
1.1.1.7   root     3997:                     io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
1.1       root     3998:                     l = 1;
                   3999:                 }
                   4000:             } else {
1.1.1.16! root     4001:                 ram_addr_t addr1;
1.1       root     4002:                 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
                   4003:                 /* RAM case */
1.1.1.10  root     4004:                 ptr = qemu_get_ram_ptr(addr1);
1.1       root     4005:                 memcpy(ptr, buf, l);
                   4006:                 if (!cpu_physical_memory_is_dirty(addr1)) {
                   4007:                     /* invalidate code */
                   4008:                     tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
                   4009:                     /* set dirty bit */
1.1.1.13  root     4010:                     cpu_physical_memory_set_dirty_flags(
                   4011:                         addr1, (0xff & ~CODE_DIRTY_FLAG));
1.1       root     4012:                 }
1.1.1.15  root     4013:                 qemu_put_ram_ptr(ptr);
1.1       root     4014:             }
                   4015:         } else {
1.1.1.6   root     4016:             if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
1.1.1.4   root     4017:                 !(pd & IO_MEM_ROMD)) {
1.1.1.7   root     4018:                 target_phys_addr_t addr1 = addr;
1.1       root     4019:                 /* I/O case */
                   4020:                 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
1.1.1.7   root     4021:                 if (p)
                   4022:                     addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
                   4023:                 if (l >= 4 && ((addr1 & 3) == 0)) {
1.1       root     4024:                     /* 32 bit read access */
1.1.1.7   root     4025:                     val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
1.1       root     4026:                     stl_p(buf, val);
                   4027:                     l = 4;
1.1.1.7   root     4028:                 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1.1       root     4029:                     /* 16 bit read access */
1.1.1.7   root     4030:                     val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
1.1       root     4031:                     stw_p(buf, val);
                   4032:                     l = 2;
                   4033:                 } else {
                   4034:                     /* 8 bit read access */
1.1.1.7   root     4035:                     val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
1.1       root     4036:                     stb_p(buf, val);
                   4037:                     l = 1;
                   4038:                 }
                   4039:             } else {
                   4040:                 /* RAM case */
1.1.1.15  root     4041:                 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
                   4042:                 memcpy(buf, ptr + (addr & ~TARGET_PAGE_MASK), l);
                   4043:                 qemu_put_ram_ptr(ptr);
1.1       root     4044:             }
                   4045:         }
                   4046:         len -= l;
                   4047:         buf += l;
                   4048:         addr += l;
                   4049:     }
                   4050: }
                   4051: 
1.1.1.3   root     4052: /* used for ROM loading : can write in RAM and ROM */
1.1.1.6   root     4053: void cpu_physical_memory_write_rom(target_phys_addr_t addr,
1.1.1.3   root     4054:                                    const uint8_t *buf, int len)
                   4055: {
                   4056:     int l;
                   4057:     uint8_t *ptr;
                   4058:     target_phys_addr_t page;
                   4059:     unsigned long pd;
                   4060:     PhysPageDesc *p;
1.1.1.6   root     4061: 
1.1.1.3   root     4062:     while (len > 0) {
                   4063:         page = addr & TARGET_PAGE_MASK;
                   4064:         l = (page + TARGET_PAGE_SIZE) - addr;
                   4065:         if (l > len)
                   4066:             l = len;
                   4067:         p = phys_page_find(page >> TARGET_PAGE_BITS);
                   4068:         if (!p) {
                   4069:             pd = IO_MEM_UNASSIGNED;
                   4070:         } else {
                   4071:             pd = p->phys_offset;
                   4072:         }
1.1.1.6   root     4073: 
1.1.1.3   root     4074:         if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
1.1.1.4   root     4075:             (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
                   4076:             !(pd & IO_MEM_ROMD)) {
1.1.1.3   root     4077:             /* do nothing */
                   4078:         } else {
                   4079:             unsigned long addr1;
                   4080:             addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
                   4081:             /* ROM/RAM case */
1.1.1.10  root     4082:             ptr = qemu_get_ram_ptr(addr1);
1.1.1.3   root     4083:             memcpy(ptr, buf, l);
1.1.1.15  root     4084:             qemu_put_ram_ptr(ptr);
1.1.1.3   root     4085:         }
                   4086:         len -= l;
                   4087:         buf += l;
                   4088:         addr += l;
                   4089:     }
                   4090: }
                   4091: 
1.1.1.7   root     4092: typedef struct {
                   4093:     void *buffer;
                   4094:     target_phys_addr_t addr;
                   4095:     target_phys_addr_t len;
                   4096: } BounceBuffer;
                   4097: 
                   4098: static BounceBuffer bounce;
                   4099: 
                   4100: typedef struct MapClient {
                   4101:     void *opaque;
                   4102:     void (*callback)(void *opaque);
1.1.1.11  root     4103:     QLIST_ENTRY(MapClient) link;
1.1.1.7   root     4104: } MapClient;
                   4105: 
1.1.1.11  root     4106: static QLIST_HEAD(map_client_list, MapClient) map_client_list
                   4107:     = QLIST_HEAD_INITIALIZER(map_client_list);
1.1.1.7   root     4108: 
                   4109: void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
                   4110: {
1.1.1.16! root     4111:     MapClient *client = g_malloc(sizeof(*client));
1.1.1.7   root     4112: 
                   4113:     client->opaque = opaque;
                   4114:     client->callback = callback;
1.1.1.11  root     4115:     QLIST_INSERT_HEAD(&map_client_list, client, link);
1.1.1.7   root     4116:     return client;
                   4117: }
                   4118: 
                   4119: void cpu_unregister_map_client(void *_client)
                   4120: {
                   4121:     MapClient *client = (MapClient *)_client;
                   4122: 
1.1.1.11  root     4123:     QLIST_REMOVE(client, link);
1.1.1.16! root     4124:     g_free(client);
1.1.1.7   root     4125: }
                   4126: 
                   4127: static void cpu_notify_map_clients(void)
                   4128: {
                   4129:     MapClient *client;
                   4130: 
1.1.1.11  root     4131:     while (!QLIST_EMPTY(&map_client_list)) {
                   4132:         client = QLIST_FIRST(&map_client_list);
1.1.1.7   root     4133:         client->callback(client->opaque);
1.1.1.10  root     4134:         cpu_unregister_map_client(client);
1.1.1.7   root     4135:     }
                   4136: }
                   4137: 
                   4138: /* Map a physical memory region into a host virtual address.
                   4139:  * May map a subset of the requested range, given by and returned in *plen.
                   4140:  * May return NULL if resources needed to perform the mapping are exhausted.
                   4141:  * Use only for reads OR writes - not for read-modify-write operations.
                   4142:  * Use cpu_register_map_client() to know when retrying the map operation is
                   4143:  * likely to succeed.
                   4144:  */
                   4145: void *cpu_physical_memory_map(target_phys_addr_t addr,
                   4146:                               target_phys_addr_t *plen,
                   4147:                               int is_write)
                   4148: {
                   4149:     target_phys_addr_t len = *plen;
1.1.1.15  root     4150:     target_phys_addr_t todo = 0;
1.1.1.7   root     4151:     int l;
                   4152:     target_phys_addr_t page;
                   4153:     unsigned long pd;
                   4154:     PhysPageDesc *p;
1.1.1.16! root     4155:     ram_addr_t raddr = RAM_ADDR_MAX;
1.1.1.15  root     4156:     ram_addr_t rlen;
                   4157:     void *ret;
1.1.1.7   root     4158: 
                   4159:     while (len > 0) {
                   4160:         page = addr & TARGET_PAGE_MASK;
                   4161:         l = (page + TARGET_PAGE_SIZE) - addr;
                   4162:         if (l > len)
                   4163:             l = len;
                   4164:         p = phys_page_find(page >> TARGET_PAGE_BITS);
                   4165:         if (!p) {
                   4166:             pd = IO_MEM_UNASSIGNED;
                   4167:         } else {
                   4168:             pd = p->phys_offset;
                   4169:         }
                   4170: 
                   4171:         if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
1.1.1.15  root     4172:             if (todo || bounce.buffer) {
1.1.1.7   root     4173:                 break;
                   4174:             }
                   4175:             bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
                   4176:             bounce.addr = addr;
                   4177:             bounce.len = l;
                   4178:             if (!is_write) {
1.1.1.15  root     4179:                 cpu_physical_memory_read(addr, bounce.buffer, l);
1.1.1.7   root     4180:             }
1.1.1.15  root     4181: 
                   4182:             *plen = l;
                   4183:             return bounce.buffer;
1.1.1.7   root     4184:         }
1.1.1.15  root     4185:         if (!todo) {
                   4186:             raddr = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
1.1.1.7   root     4187:         }
                   4188: 
                   4189:         len -= l;
                   4190:         addr += l;
1.1.1.15  root     4191:         todo += l;
1.1.1.7   root     4192:     }
1.1.1.15  root     4193:     rlen = todo;
                   4194:     ret = qemu_ram_ptr_length(raddr, &rlen);
                   4195:     *plen = rlen;
1.1.1.7   root     4196:     return ret;
                   4197: }
                   4198: 
                   4199: /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
                   4200:  * Will also mark the memory as dirty if is_write == 1.  access_len gives
                   4201:  * the amount of memory that was actually read or written by the caller.
                   4202:  */
                   4203: void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
                   4204:                                int is_write, target_phys_addr_t access_len)
                   4205: {
                   4206:     if (buffer != bounce.buffer) {
                   4207:         if (is_write) {
1.1.1.14  root     4208:             ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
1.1.1.7   root     4209:             while (access_len) {
                   4210:                 unsigned l;
                   4211:                 l = TARGET_PAGE_SIZE;
                   4212:                 if (l > access_len)
                   4213:                     l = access_len;
                   4214:                 if (!cpu_physical_memory_is_dirty(addr1)) {
                   4215:                     /* invalidate code */
                   4216:                     tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
                   4217:                     /* set dirty bit */
1.1.1.13  root     4218:                     cpu_physical_memory_set_dirty_flags(
                   4219:                         addr1, (0xff & ~CODE_DIRTY_FLAG));
1.1.1.7   root     4220:                 }
                   4221:                 addr1 += l;
                   4222:                 access_len -= l;
                   4223:             }
                   4224:         }
1.1.1.15  root     4225:         if (xen_enabled()) {
                   4226:             xen_invalidate_map_cache_entry(buffer);
                   4227:         }
1.1.1.7   root     4228:         return;
                   4229:     }
                   4230:     if (is_write) {
                   4231:         cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
                   4232:     }
1.1.1.12  root     4233:     qemu_vfree(bounce.buffer);
1.1.1.7   root     4234:     bounce.buffer = NULL;
                   4235:     cpu_notify_map_clients();
                   4236: }
1.1.1.3   root     4237: 
1.1       root     4238: /* warning: addr must be aligned */
1.1.1.15  root     4239: static inline uint32_t ldl_phys_internal(target_phys_addr_t addr,
                   4240:                                          enum device_endian endian)
1.1       root     4241: {
                   4242:     int io_index;
                   4243:     uint8_t *ptr;
                   4244:     uint32_t val;
                   4245:     unsigned long pd;
                   4246:     PhysPageDesc *p;
                   4247: 
                   4248:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   4249:     if (!p) {
                   4250:         pd = IO_MEM_UNASSIGNED;
                   4251:     } else {
                   4252:         pd = p->phys_offset;
                   4253:     }
1.1.1.6   root     4254: 
                   4255:     if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
1.1.1.4   root     4256:         !(pd & IO_MEM_ROMD)) {
1.1       root     4257:         /* I/O case */
                   4258:         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
1.1.1.7   root     4259:         if (p)
                   4260:             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
1.1       root     4261:         val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
1.1.1.15  root     4262: #if defined(TARGET_WORDS_BIGENDIAN)
                   4263:         if (endian == DEVICE_LITTLE_ENDIAN) {
                   4264:             val = bswap32(val);
                   4265:         }
                   4266: #else
                   4267:         if (endian == DEVICE_BIG_ENDIAN) {
                   4268:             val = bswap32(val);
                   4269:         }
                   4270: #endif
1.1       root     4271:     } else {
                   4272:         /* RAM case */
1.1.1.10  root     4273:         ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
1.1       root     4274:             (addr & ~TARGET_PAGE_MASK);
1.1.1.15  root     4275:         switch (endian) {
                   4276:         case DEVICE_LITTLE_ENDIAN:
                   4277:             val = ldl_le_p(ptr);
                   4278:             break;
                   4279:         case DEVICE_BIG_ENDIAN:
                   4280:             val = ldl_be_p(ptr);
                   4281:             break;
                   4282:         default:
                   4283:             val = ldl_p(ptr);
                   4284:             break;
                   4285:         }
1.1       root     4286:     }
                   4287:     return val;
                   4288: }
                   4289: 
1.1.1.15  root     4290: uint32_t ldl_phys(target_phys_addr_t addr)
                   4291: {
                   4292:     return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
                   4293: }
                   4294: 
                   4295: uint32_t ldl_le_phys(target_phys_addr_t addr)
                   4296: {
                   4297:     return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
                   4298: }
                   4299: 
                   4300: uint32_t ldl_be_phys(target_phys_addr_t addr)
                   4301: {
                   4302:     return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
                   4303: }
                   4304: 
1.1.1.2   root     4305: /* warning: addr must be aligned */
1.1.1.15  root     4306: static inline uint64_t ldq_phys_internal(target_phys_addr_t addr,
                   4307:                                          enum device_endian endian)
1.1.1.2   root     4308: {
                   4309:     int io_index;
                   4310:     uint8_t *ptr;
                   4311:     uint64_t val;
                   4312:     unsigned long pd;
                   4313:     PhysPageDesc *p;
                   4314: 
                   4315:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   4316:     if (!p) {
                   4317:         pd = IO_MEM_UNASSIGNED;
                   4318:     } else {
                   4319:         pd = p->phys_offset;
                   4320:     }
1.1.1.6   root     4321: 
1.1.1.4   root     4322:     if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
                   4323:         !(pd & IO_MEM_ROMD)) {
1.1.1.2   root     4324:         /* I/O case */
                   4325:         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
1.1.1.7   root     4326:         if (p)
                   4327:             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
1.1.1.15  root     4328: 
                   4329:         /* XXX This is broken when device endian != cpu endian.
                   4330:                Fix and add "endian" variable check */
1.1.1.2   root     4331: #ifdef TARGET_WORDS_BIGENDIAN
                   4332:         val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
                   4333:         val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
                   4334: #else
                   4335:         val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
                   4336:         val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
                   4337: #endif
                   4338:     } else {
                   4339:         /* RAM case */
1.1.1.10  root     4340:         ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
1.1.1.2   root     4341:             (addr & ~TARGET_PAGE_MASK);
1.1.1.15  root     4342:         switch (endian) {
                   4343:         case DEVICE_LITTLE_ENDIAN:
                   4344:             val = ldq_le_p(ptr);
                   4345:             break;
                   4346:         case DEVICE_BIG_ENDIAN:
                   4347:             val = ldq_be_p(ptr);
                   4348:             break;
                   4349:         default:
                   4350:             val = ldq_p(ptr);
                   4351:             break;
                   4352:         }
1.1.1.2   root     4353:     }
                   4354:     return val;
                   4355: }
                   4356: 
1.1.1.15  root     4357: uint64_t ldq_phys(target_phys_addr_t addr)
                   4358: {
                   4359:     return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
                   4360: }
                   4361: 
                   4362: uint64_t ldq_le_phys(target_phys_addr_t addr)
                   4363: {
                   4364:     return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
                   4365: }
                   4366: 
                   4367: uint64_t ldq_be_phys(target_phys_addr_t addr)
                   4368: {
                   4369:     return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
                   4370: }
                   4371: 
1.1.1.2   root     4372: /* XXX: optimize */
                   4373: uint32_t ldub_phys(target_phys_addr_t addr)
                   4374: {
                   4375:     uint8_t val;
                   4376:     cpu_physical_memory_read(addr, &val, 1);
                   4377:     return val;
                   4378: }
                   4379: 
1.1.1.13  root     4380: /* warning: addr must be aligned */
1.1.1.15  root     4381: static inline uint32_t lduw_phys_internal(target_phys_addr_t addr,
                   4382:                                           enum device_endian endian)
1.1.1.2   root     4383: {
1.1.1.13  root     4384:     int io_index;
                   4385:     uint8_t *ptr;
                   4386:     uint64_t val;
                   4387:     unsigned long pd;
                   4388:     PhysPageDesc *p;
                   4389: 
                   4390:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   4391:     if (!p) {
                   4392:         pd = IO_MEM_UNASSIGNED;
                   4393:     } else {
                   4394:         pd = p->phys_offset;
                   4395:     }
                   4396: 
                   4397:     if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
                   4398:         !(pd & IO_MEM_ROMD)) {
                   4399:         /* I/O case */
                   4400:         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
                   4401:         if (p)
                   4402:             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
                   4403:         val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
1.1.1.15  root     4404: #if defined(TARGET_WORDS_BIGENDIAN)
                   4405:         if (endian == DEVICE_LITTLE_ENDIAN) {
                   4406:             val = bswap16(val);
                   4407:         }
                   4408: #else
                   4409:         if (endian == DEVICE_BIG_ENDIAN) {
                   4410:             val = bswap16(val);
                   4411:         }
                   4412: #endif
1.1.1.13  root     4413:     } else {
                   4414:         /* RAM case */
                   4415:         ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
                   4416:             (addr & ~TARGET_PAGE_MASK);
1.1.1.15  root     4417:         switch (endian) {
                   4418:         case DEVICE_LITTLE_ENDIAN:
                   4419:             val = lduw_le_p(ptr);
                   4420:             break;
                   4421:         case DEVICE_BIG_ENDIAN:
                   4422:             val = lduw_be_p(ptr);
                   4423:             break;
                   4424:         default:
                   4425:             val = lduw_p(ptr);
                   4426:             break;
                   4427:         }
1.1.1.13  root     4428:     }
                   4429:     return val;
1.1.1.2   root     4430: }
                   4431: 
1.1.1.15  root     4432: uint32_t lduw_phys(target_phys_addr_t addr)
                   4433: {
                   4434:     return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
                   4435: }
                   4436: 
                   4437: uint32_t lduw_le_phys(target_phys_addr_t addr)
                   4438: {
                   4439:     return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
                   4440: }
                   4441: 
                   4442: uint32_t lduw_be_phys(target_phys_addr_t addr)
                   4443: {
                   4444:     return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
                   4445: }
                   4446: 
1.1       root     4447: /* warning: addr must be aligned. The ram page is not masked as dirty
                   4448:    and the code inside is not invalidated. It is useful if the dirty
                   4449:    bits are used to track modified PTEs */
                   4450: void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
                   4451: {
                   4452:     int io_index;
                   4453:     uint8_t *ptr;
                   4454:     unsigned long pd;
                   4455:     PhysPageDesc *p;
                   4456: 
                   4457:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   4458:     if (!p) {
                   4459:         pd = IO_MEM_UNASSIGNED;
                   4460:     } else {
                   4461:         pd = p->phys_offset;
                   4462:     }
1.1.1.6   root     4463: 
1.1       root     4464:     if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
                   4465:         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
1.1.1.7   root     4466:         if (p)
                   4467:             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
1.1       root     4468:         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
                   4469:     } else {
1.1.1.7   root     4470:         unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
1.1.1.10  root     4471:         ptr = qemu_get_ram_ptr(addr1);
1.1       root     4472:         stl_p(ptr, val);
1.1.1.7   root     4473: 
                   4474:         if (unlikely(in_migration)) {
                   4475:             if (!cpu_physical_memory_is_dirty(addr1)) {
                   4476:                 /* invalidate code */
                   4477:                 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
                   4478:                 /* set dirty bit */
1.1.1.13  root     4479:                 cpu_physical_memory_set_dirty_flags(
                   4480:                     addr1, (0xff & ~CODE_DIRTY_FLAG));
1.1.1.7   root     4481:             }
                   4482:         }
1.1       root     4483:     }
                   4484: }
                   4485: 
1.1.1.6   root     4486: void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
                   4487: {
                   4488:     int io_index;
                   4489:     uint8_t *ptr;
                   4490:     unsigned long pd;
                   4491:     PhysPageDesc *p;
                   4492: 
                   4493:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   4494:     if (!p) {
                   4495:         pd = IO_MEM_UNASSIGNED;
                   4496:     } else {
                   4497:         pd = p->phys_offset;
                   4498:     }
                   4499: 
                   4500:     if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
                   4501:         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
1.1.1.7   root     4502:         if (p)
                   4503:             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
1.1.1.6   root     4504: #ifdef TARGET_WORDS_BIGENDIAN
                   4505:         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
                   4506:         io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
                   4507: #else
                   4508:         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
                   4509:         io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
                   4510: #endif
                   4511:     } else {
1.1.1.10  root     4512:         ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
1.1.1.6   root     4513:             (addr & ~TARGET_PAGE_MASK);
                   4514:         stq_p(ptr, val);
                   4515:     }
                   4516: }
                   4517: 
1.1       root     4518: /* warning: addr must be aligned */
1.1.1.15  root     4519: static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val,
                   4520:                                      enum device_endian endian)
1.1       root     4521: {
                   4522:     int io_index;
                   4523:     uint8_t *ptr;
                   4524:     unsigned long pd;
                   4525:     PhysPageDesc *p;
                   4526: 
                   4527:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   4528:     if (!p) {
                   4529:         pd = IO_MEM_UNASSIGNED;
                   4530:     } else {
                   4531:         pd = p->phys_offset;
                   4532:     }
1.1.1.6   root     4533: 
1.1       root     4534:     if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
                   4535:         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
1.1.1.7   root     4536:         if (p)
                   4537:             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
1.1.1.15  root     4538: #if defined(TARGET_WORDS_BIGENDIAN)
                   4539:         if (endian == DEVICE_LITTLE_ENDIAN) {
                   4540:             val = bswap32(val);
                   4541:         }
                   4542: #else
                   4543:         if (endian == DEVICE_BIG_ENDIAN) {
                   4544:             val = bswap32(val);
                   4545:         }
                   4546: #endif
1.1       root     4547:         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
                   4548:     } else {
                   4549:         unsigned long addr1;
                   4550:         addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
                   4551:         /* RAM case */
1.1.1.10  root     4552:         ptr = qemu_get_ram_ptr(addr1);
1.1.1.15  root     4553:         switch (endian) {
                   4554:         case DEVICE_LITTLE_ENDIAN:
                   4555:             stl_le_p(ptr, val);
                   4556:             break;
                   4557:         case DEVICE_BIG_ENDIAN:
                   4558:             stl_be_p(ptr, val);
                   4559:             break;
                   4560:         default:
                   4561:             stl_p(ptr, val);
                   4562:             break;
                   4563:         }
1.1       root     4564:         if (!cpu_physical_memory_is_dirty(addr1)) {
                   4565:             /* invalidate code */
                   4566:             tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
                   4567:             /* set dirty bit */
1.1.1.13  root     4568:             cpu_physical_memory_set_dirty_flags(addr1,
                   4569:                 (0xff & ~CODE_DIRTY_FLAG));
1.1       root     4570:         }
                   4571:     }
                   4572: }
                   4573: 
1.1.1.15  root     4574: void stl_phys(target_phys_addr_t addr, uint32_t val)
                   4575: {
                   4576:     stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
                   4577: }
                   4578: 
                   4579: void stl_le_phys(target_phys_addr_t addr, uint32_t val)
                   4580: {
                   4581:     stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
                   4582: }
                   4583: 
                   4584: void stl_be_phys(target_phys_addr_t addr, uint32_t val)
                   4585: {
                   4586:     stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
                   4587: }
                   4588: 
1.1.1.2   root     4589: /* XXX: optimize */
                   4590: void stb_phys(target_phys_addr_t addr, uint32_t val)
                   4591: {
                   4592:     uint8_t v = val;
                   4593:     cpu_physical_memory_write(addr, &v, 1);
                   4594: }
                   4595: 
1.1.1.13  root     4596: /* warning: addr must be aligned */
1.1.1.15  root     4597: static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val,
                   4598:                                      enum device_endian endian)
1.1.1.2   root     4599: {
1.1.1.13  root     4600:     int io_index;
                   4601:     uint8_t *ptr;
                   4602:     unsigned long pd;
                   4603:     PhysPageDesc *p;
                   4604: 
                   4605:     p = phys_page_find(addr >> TARGET_PAGE_BITS);
                   4606:     if (!p) {
                   4607:         pd = IO_MEM_UNASSIGNED;
                   4608:     } else {
                   4609:         pd = p->phys_offset;
                   4610:     }
                   4611: 
                   4612:     if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
                   4613:         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
                   4614:         if (p)
                   4615:             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
1.1.1.15  root     4616: #if defined(TARGET_WORDS_BIGENDIAN)
                   4617:         if (endian == DEVICE_LITTLE_ENDIAN) {
                   4618:             val = bswap16(val);
                   4619:         }
                   4620: #else
                   4621:         if (endian == DEVICE_BIG_ENDIAN) {
                   4622:             val = bswap16(val);
                   4623:         }
                   4624: #endif
1.1.1.13  root     4625:         io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
                   4626:     } else {
                   4627:         unsigned long addr1;
                   4628:         addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
                   4629:         /* RAM case */
                   4630:         ptr = qemu_get_ram_ptr(addr1);
1.1.1.15  root     4631:         switch (endian) {
                   4632:         case DEVICE_LITTLE_ENDIAN:
                   4633:             stw_le_p(ptr, val);
                   4634:             break;
                   4635:         case DEVICE_BIG_ENDIAN:
                   4636:             stw_be_p(ptr, val);
                   4637:             break;
                   4638:         default:
                   4639:             stw_p(ptr, val);
                   4640:             break;
                   4641:         }
1.1.1.13  root     4642:         if (!cpu_physical_memory_is_dirty(addr1)) {
                   4643:             /* invalidate code */
                   4644:             tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
                   4645:             /* set dirty bit */
                   4646:             cpu_physical_memory_set_dirty_flags(addr1,
                   4647:                 (0xff & ~CODE_DIRTY_FLAG));
                   4648:         }
                   4649:     }
1.1.1.2   root     4650: }
                   4651: 
1.1.1.15  root     4652: void stw_phys(target_phys_addr_t addr, uint32_t val)
                   4653: {
                   4654:     stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
                   4655: }
                   4656: 
                   4657: void stw_le_phys(target_phys_addr_t addr, uint32_t val)
                   4658: {
                   4659:     stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
                   4660: }
                   4661: 
                   4662: void stw_be_phys(target_phys_addr_t addr, uint32_t val)
                   4663: {
                   4664:     stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
                   4665: }
                   4666: 
1.1.1.2   root     4667: /* XXX: optimize */
                   4668: void stq_phys(target_phys_addr_t addr, uint64_t val)
                   4669: {
                   4670:     val = tswap64(val);
1.1.1.15  root     4671:     cpu_physical_memory_write(addr, &val, 8);
                   4672: }
                   4673: 
                   4674: void stq_le_phys(target_phys_addr_t addr, uint64_t val)
                   4675: {
                   4676:     val = cpu_to_le64(val);
                   4677:     cpu_physical_memory_write(addr, &val, 8);
                   4678: }
                   4679: 
                   4680: void stq_be_phys(target_phys_addr_t addr, uint64_t val)
                   4681: {
                   4682:     val = cpu_to_be64(val);
                   4683:     cpu_physical_memory_write(addr, &val, 8);
1.1.1.2   root     4684: }
                   4685: 
1.1.1.10  root     4686: /* virtual memory access for debug (includes writing to ROM) */
1.1.1.6   root     4687: int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
1.1       root     4688:                         uint8_t *buf, int len, int is_write)
                   4689: {
                   4690:     int l;
1.1.1.6   root     4691:     target_phys_addr_t phys_addr;
                   4692:     target_ulong page;
1.1       root     4693: 
                   4694:     while (len > 0) {
                   4695:         page = addr & TARGET_PAGE_MASK;
                   4696:         phys_addr = cpu_get_phys_page_debug(env, page);
                   4697:         /* if no physical page mapped, return an error */
                   4698:         if (phys_addr == -1)
                   4699:             return -1;
                   4700:         l = (page + TARGET_PAGE_SIZE) - addr;
                   4701:         if (l > len)
                   4702:             l = len;
1.1.1.10  root     4703:         phys_addr += (addr & ~TARGET_PAGE_MASK);
                   4704:         if (is_write)
                   4705:             cpu_physical_memory_write_rom(phys_addr, buf, l);
                   4706:         else
                   4707:             cpu_physical_memory_rw(phys_addr, buf, l, is_write);
1.1       root     4708:         len -= l;
                   4709:         buf += l;
                   4710:         addr += l;
                   4711:     }
                   4712:     return 0;
                   4713: }
1.1.1.13  root     4714: #endif
1.1       root     4715: 
1.1.1.7   root     4716: /* in deterministic execution mode, instructions doing device I/Os
                   4717:    must be at the end of the TB */
                   4718: void cpu_io_recompile(CPUState *env, void *retaddr)
                   4719: {
                   4720:     TranslationBlock *tb;
                   4721:     uint32_t n, cflags;
                   4722:     target_ulong pc, cs_base;
                   4723:     uint64_t flags;
                   4724: 
                   4725:     tb = tb_find_pc((unsigned long)retaddr);
                   4726:     if (!tb) {
                   4727:         cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p", 
                   4728:                   retaddr);
                   4729:     }
                   4730:     n = env->icount_decr.u16.low + tb->icount;
1.1.1.15  root     4731:     cpu_restore_state(tb, env, (unsigned long)retaddr);
1.1.1.7   root     4732:     /* Calculate how many instructions had been executed before the fault
                   4733:        occurred.  */
                   4734:     n = n - env->icount_decr.u16.low;
                   4735:     /* Generate a new TB ending on the I/O insn.  */
                   4736:     n++;
                   4737:     /* On MIPS and SH, delay slot instructions can only be restarted if
                   4738:        they were already the first instruction in the TB.  If this is not
                   4739:        the first instruction in a TB then re-execute the preceding
                   4740:        branch.  */
                   4741: #if defined(TARGET_MIPS)
                   4742:     if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
                   4743:         env->active_tc.PC -= 4;
                   4744:         env->icount_decr.u16.low++;
                   4745:         env->hflags &= ~MIPS_HFLAG_BMASK;
                   4746:     }
                   4747: #elif defined(TARGET_SH4)
                   4748:     if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
                   4749:             && n > 1) {
                   4750:         env->pc -= 2;
                   4751:         env->icount_decr.u16.low++;
                   4752:         env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
                   4753:     }
                   4754: #endif
                   4755:     /* This should never happen.  */
                   4756:     if (n > CF_COUNT_MASK)
                   4757:         cpu_abort(env, "TB too big during recompile");
                   4758: 
                   4759:     cflags = n | CF_LAST_IO;
                   4760:     pc = tb->pc;
                   4761:     cs_base = tb->cs_base;
                   4762:     flags = tb->flags;
                   4763:     tb_phys_invalidate(tb, -1);
                   4764:     /* FIXME: In theory this could raise an exception.  In practice
                   4765:        we have already translated the block once so it's probably ok.  */
                   4766:     tb_gen_code(env, pc, cs_base, flags, cflags);
                   4767:     /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
                   4768:        the first in the TB) then we end up generating a whole new TB and
                   4769:        repeating the fault, which is horribly inefficient.
                   4770:        Better would be to execute just this insn uncached, or generate a
                   4771:        second new TB.  */
                   4772:     cpu_resume_from_signal(env, NULL);
                   4773: }
                   4774: 
1.1.1.13  root     4775: #if !defined(CONFIG_USER_ONLY)
                   4776: 
1.1.1.14  root     4777: void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1.1       root     4778: {
                   4779:     int i, target_code_size, max_target_code_size;
                   4780:     int direct_jmp_count, direct_jmp2_count, cross_page;
                   4781:     TranslationBlock *tb;
1.1.1.6   root     4782: 
1.1       root     4783:     target_code_size = 0;
                   4784:     max_target_code_size = 0;
                   4785:     cross_page = 0;
                   4786:     direct_jmp_count = 0;
                   4787:     direct_jmp2_count = 0;
                   4788:     for(i = 0; i < nb_tbs; i++) {
                   4789:         tb = &tbs[i];
                   4790:         target_code_size += tb->size;
                   4791:         if (tb->size > max_target_code_size)
                   4792:             max_target_code_size = tb->size;
                   4793:         if (tb->page_addr[1] != -1)
                   4794:             cross_page++;
                   4795:         if (tb->tb_next_offset[0] != 0xffff) {
                   4796:             direct_jmp_count++;
                   4797:             if (tb->tb_next_offset[1] != 0xffff) {
                   4798:                 direct_jmp2_count++;
                   4799:             }
                   4800:         }
                   4801:     }
                   4802:     /* XXX: avoid using doubles ? */
1.1.1.7   root     4803:     cpu_fprintf(f, "Translation buffer state:\n");
1.1.1.14  root     4804:     cpu_fprintf(f, "gen code size       %td/%ld\n",
1.1.1.7   root     4805:                 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
                   4806:     cpu_fprintf(f, "TB count            %d/%d\n", 
                   4807:                 nb_tbs, code_gen_max_blocks);
1.1.1.6   root     4808:     cpu_fprintf(f, "TB avg target size  %d max=%d bytes\n",
1.1       root     4809:                 nb_tbs ? target_code_size / nb_tbs : 0,
                   4810:                 max_target_code_size);
1.1.1.14  root     4811:     cpu_fprintf(f, "TB avg host size    %td bytes (expansion ratio: %0.1f)\n",
1.1       root     4812:                 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
                   4813:                 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
1.1.1.6   root     4814:     cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
                   4815:             cross_page,
1.1       root     4816:             nb_tbs ? (cross_page * 100) / nb_tbs : 0);
                   4817:     cpu_fprintf(f, "direct jump count   %d (%d%%) (2 jumps=%d %d%%)\n",
1.1.1.6   root     4818:                 direct_jmp_count,
1.1       root     4819:                 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
                   4820:                 direct_jmp2_count,
                   4821:                 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
1.1.1.7   root     4822:     cpu_fprintf(f, "\nStatistics:\n");
1.1       root     4823:     cpu_fprintf(f, "TB flush count      %d\n", tb_flush_count);
                   4824:     cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
                   4825:     cpu_fprintf(f, "TLB flush count     %d\n", tlb_flush_count);
1.1.1.7   root     4826:     tcg_dump_info(f, cpu_fprintf);
1.1       root     4827: }
                   4828: 
                   4829: #define MMUSUFFIX _cmmu
1.1.1.16! root     4830: #undef GETPC
1.1       root     4831: #define GETPC() NULL
                   4832: #define env cpu_single_env
                   4833: #define SOFTMMU_CODE_ACCESS
                   4834: 
                   4835: #define SHIFT 0
                   4836: #include "softmmu_template.h"
                   4837: 
                   4838: #define SHIFT 1
                   4839: #include "softmmu_template.h"
                   4840: 
                   4841: #define SHIFT 2
                   4842: #include "softmmu_template.h"
                   4843: 
                   4844: #define SHIFT 3
                   4845: #include "softmmu_template.h"
                   4846: 
                   4847: #undef env
                   4848: 
                   4849: #endif

unix.superglobalmegacorp.com