![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to mips64_jit.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Dynamips Cisco Emulator] / cf|
Return to mips64_jit.c CVS log | Up to [Dynamips Cisco Emulator] / cf |
1.1 root 1: /* 1.1.1.9 ! root 2: * Cisco router simulation platform. 1.1 root 3: * Copyright (c) 2005,2006 Christophe Fillot ([email protected]) 4: * 5: * MIPS64 JIT compiler. 6: */ 7: 8: #include <stdio.h> 9: #include <stdlib.h> 10: #include <unistd.h> 11: #include <string.h> 12: #include <sys/types.h> 13: #include <sys/stat.h> 14: #include <sys/mman.h> 15: #include <signal.h> 16: #include <fcntl.h> 17: #include <assert.h> 18: 19: #include "cpu.h" 20: #include "device.h" 1.1.1.4 root 21: #include "mips64.h" 1.1.1.9 ! root 22: #include "mips64_cp0.h" 1.1 root 23: #include "mips64_exec.h" 1.1.1.9 ! root 24: #include "mips64_jit.h" 1.1 root 25: #include "insn_lookup.h" 1.1.1.9 ! root 26: #include "memory.h" 1.1 root 27: #include "ptask.h" 28: 1.1.1.9 ! root 29: #include MIPS64_ARCH_INC_FILE ! 30: 1.1 root 31: #if DEBUG_BLOCK_TIMESTAMP 32: static volatile m_uint64_t jit_jiffies = 0; 33: #endif 34: 35: /* MIPS jump instructions for block scan */ 1.1.1.9 ! root 36: struct mips64_insn_jump mips64_insn_jumps[] = { 1.1 root 37: { "b" , 0xffff0000, 0x10000000, 16, 1 }, 38: { "bal" , 0xffff0000, 0x04110000, 16, 1 }, 39: { "beq" , 0xfc000000, 0x10000000, 16, 1 }, 40: { "beql" , 0xfc000000, 0x50000000, 16, 1 }, 41: { "bgez" , 0xfc1f0000, 0x04010000, 16, 1 }, 42: { "bgezl" , 0xfc1f0000, 0x04030000, 16, 1 }, 43: { "bgezal" , 0xfc1f0000, 0x04110000, 16, 1 }, 44: { "bgezall" , 0xfc1f0000, 0x04130000, 16, 1 }, 45: { "bgtz" , 0xfc1f0000, 0x1c000000, 16, 1 }, 46: { "bgtzl" , 0xfc1f0000, 0x5c000000, 16, 1 }, 47: { "blez" , 0xfc1f0000, 0x18000000, 16, 1 }, 48: { "blezl" , 0xfc1f0000, 0x58000000, 16, 1 }, 49: { "bltz" , 0xfc1f0000, 0x04000000, 16, 1 }, 50: { "bltzl" , 0xfc1f0000, 0x04020000, 16, 1 }, 51: { "bltzal" , 0xfc1f0000, 0x04100000, 16, 1 }, 52: { "bltzall" , 0xfc1f0000, 0x04120000, 16, 1 }, 53: { "bne" , 0xfc000000, 0x14000000, 16, 1 }, 54: { "bnel" , 0xfc000000, 0x54000000, 16, 1 }, 55: { "j" , 0xfc000000, 0x08000000, 26, 0 }, 56: { NULL , 0x00000000, 0x00000000, 0, 0 }, 57: }; 58: 59: /* Instruction Lookup Table */ 60: static insn_lookup_t *ilt = NULL; 61: 62: static void *mips64_jit_get_insn(int index) 63: { 64: return(&mips64_insn_tags[index]); 65: } 66: 1.1.1.9 ! root 67: static int mips64_jit_chk_lo(struct mips64_insn_tag *tag,int value) 1.1 root 68: { 69: return((value & tag->mask) == (tag->value & 0xFFFF)); 70: } 71: 1.1.1.9 ! root 72: static int mips64_jit_chk_hi(struct mips64_insn_tag *tag,int value) 1.1 root 73: { 74: return((value & (tag->mask >> 16)) == (tag->value >> 16)); 75: } 76: 77: /* Initialize instruction lookup table */ 78: void mips64_jit_create_ilt(void) 79: { 80: int i,count; 81: 82: for(i=0,count=0;mips64_insn_tags[i].emit;i++) 83: count++; 84: 1.1.1.9 ! root 85: ilt = ilt_create(count+1, 1.1 root 86: (ilt_get_insn_cbk_t)mips64_jit_get_insn, 87: (ilt_check_cbk_t)mips64_jit_chk_lo, 88: (ilt_check_cbk_t)mips64_jit_chk_hi); 89: } 90: 1.1.1.4 root 91: /* Initialize the JIT structure */ 92: int mips64_jit_init(cpu_mips_t *cpu) 1.1.1.2 root 93: { 1.1.1.4 root 94: insn_exec_page_t *cp; 95: u_char *cp_addr; 96: u_int area_size; 97: size_t len; 98: int i; 99: 100: /* Physical mapping for executable pages */ 101: len = 1048576 * sizeof(void *); 102: cpu->exec_phys_map = m_memalign(4096,len); 103: memset(cpu->exec_phys_map,0,len); 104: 105: /* Get area size */ 106: if (!(area_size = cpu->vm->exec_area_size)) 107: area_size = MIPS_EXEC_AREA_SIZE; 108: 109: /* Create executable page area */ 110: cpu->exec_page_area_size = area_size * 1048576; 111: cpu->exec_page_area = mmap(NULL,cpu->exec_page_area_size, 112: PROT_EXEC|PROT_READ|PROT_WRITE, 113: MAP_SHARED|MAP_ANONYMOUS,-1,(off_t)0); 114: 115: if (!cpu->exec_page_area) { 116: fprintf(stderr, 117: "mips64_jit_init: unable to create exec area (size %lu)\n", 118: (u_long)cpu->exec_page_area_size); 119: return(-1); 120: } 121: 122: /* Carve the executable page area */ 123: cpu->exec_page_count = cpu->exec_page_area_size / MIPS_JIT_BUFSIZE; 124: 125: cpu->exec_page_array = calloc(cpu->exec_page_count, 126: sizeof(insn_exec_page_t)); 1.1.1.2 root 127: 1.1.1.4 root 128: if (!cpu->exec_page_array) { 129: fprintf(stderr,"mips64_jit_init: unable to create exec page array\n"); 130: return(-1); 131: } 132: 133: for(i=0,cp_addr=cpu->exec_page_area;i<cpu->exec_page_count;i++) { 134: cp = &cpu->exec_page_array[i]; 135: 136: cp->ptr = cp_addr; 137: cp_addr += MIPS_JIT_BUFSIZE; 1.1.1.2 root 138: 1.1.1.4 root 139: cp->next = cpu->exec_page_free_list; 140: cpu->exec_page_free_list = cp; 1.1.1.2 root 141: } 1.1.1.4 root 142: 143: printf("CPU%u: carved JIT exec zone of %lu Mb into %lu pages of %u Kb.\n", 1.1.1.9 ! root 144: cpu->gen->id, ! 145: (u_long)(cpu->exec_page_area_size / 1048576), 1.1.1.4 root 146: (u_long)cpu->exec_page_count,MIPS_JIT_BUFSIZE / 1024); 147: return(0); 1.1.1.2 root 148: } 149: 1.1.1.4 root 150: /* Flush the JIT */ 151: u_int mips64_jit_flush(cpu_mips_t *cpu,u_int threshold) 1.1.1.2 root 152: { 1.1.1.9 ! root 153: mips64_jit_tcb_t *p,*next; 1.1.1.4 root 154: u_int count = 0; 155: 156: if (!threshold) 157: threshold = (u_int)(-1); /* UINT_MAX not defined everywhere */ 1.1.1.2 root 158: 1.1.1.9 ! root 159: for(p=cpu->tcb_list;p;p=next) { 1.1.1.4 root 160: next = p->next; 161: 162: if (p->acc_count <= threshold) { 163: cpu->exec_phys_map[p->phys_page] = NULL; 1.1.1.9 ! root 164: mips64_jit_tcb_free(cpu,p,TRUE); 1.1.1.4 root 165: count++; 166: } 1.1.1.2 root 167: } 1.1.1.4 root 168: 169: cpu->compiled_pages -= count; 170: return(count); 171: } 172: 173: /* Shutdown the JIT */ 174: void mips64_jit_shutdown(cpu_mips_t *cpu) 175: { 1.1.1.9 ! root 176: mips64_jit_tcb_t *p,*next; 1.1.1.4 root 177: 178: /* Flush the JIT */ 179: mips64_jit_flush(cpu,0); 180: 181: /* Free the instruction blocks */ 1.1.1.9 ! root 182: for(p=cpu->tcb_free_list;p;p=next) { 1.1.1.4 root 183: next = p->next; 184: free(p); 185: } 186: 187: /* Unmap the executable page area */ 188: if (cpu->exec_page_area) 189: munmap(cpu->exec_page_area,cpu->exec_page_area_size); 190: 191: /* Free the exec page array */ 192: free(cpu->exec_page_array); 193: 194: /* Free physical mapping for executable pages */ 195: free(cpu->exec_phys_map); 1.1.1.2 root 196: } 197: 1.1.1.4 root 198: /* Allocate an exec page */ 199: static inline insn_exec_page_t *exec_page_alloc(cpu_mips_t *cpu) 1.1.1.2 root 200: { 1.1.1.4 root 201: insn_exec_page_t *p; 202: u_int count; 1.1.1.2 root 203: 1.1.1.4 root 204: /* If the free list is empty, flush JIT */ 205: if (unlikely(!cpu->exec_page_free_list)) 206: { 207: if (cpu->jit_flush_method) { 1.1.1.9 ! root 208: cpu_log(cpu->gen, ! 209: "JIT","flushing data structures (compiled pages=%u)\n", 1.1.1.4 root 210: cpu->compiled_pages); 211: mips64_jit_flush(cpu,0); 212: } else { 213: count = mips64_jit_flush(cpu,100); 1.1.1.9 ! root 214: cpu_log(cpu->gen,"JIT","partial JIT flush (count=%u)\n",count); 1.1.1.4 root 215: 216: if (!cpu->exec_page_free_list) 217: mips64_jit_flush(cpu,0); 1.1.1.2 root 218: } 1.1.1.4 root 219: 220: /* Use both methods alternatively */ 221: cpu->jit_flush_method = 1 - cpu->jit_flush_method; 222: } 223: 224: if (unlikely(!(p = cpu->exec_page_free_list))) 225: return NULL; 226: 227: cpu->exec_page_free_list = p->next; 228: cpu->exec_page_alloc++; 229: return p; 230: } 231: 232: /* Free an exec page and returns it to the pool */ 233: static inline void exec_page_free(cpu_mips_t *cpu,insn_exec_page_t *p) 234: { 235: if (p) { 236: p->next = cpu->exec_page_free_list; 237: cpu->exec_page_free_list = p; 238: cpu->exec_page_alloc--; 1.1.1.2 root 239: } 240: } 241: 1.1 root 242: /* Find the JIT code emitter for the specified MIPS instruction */ 1.1.1.9 ! root 243: static struct mips64_insn_tag *insn_tag_find(mips_insn_t ins) 1.1 root 244: { 1.1.1.9 ! root 245: struct mips64_insn_tag *tag = NULL; 1.1 root 246: int index; 247: 248: index = ilt_lookup(ilt,ins); 249: tag = mips64_jit_get_insn(index); 250: return tag; 251: } 252: 253: /* Check if the specified MIPS instruction is a jump */ 1.1.1.9 ! root 254: static struct mips64_insn_jump *insn_jump_find(mips_insn_t ins) 1.1 root 255: { 1.1.1.9 ! root 256: struct mips64_insn_jump *jump = NULL; 1.1 root 257: int i; 258: 259: for(i=0;mips64_insn_jumps[i].name;i++) 260: if ((ins & mips64_insn_jumps[i].mask) == mips64_insn_jumps[i].value) { 261: jump = &mips64_insn_jumps[i]; 262: break; 263: } 264: 265: return(jump); 266: } 267: 268: /* Fetch a MIPS instruction */ 1.1.1.9 ! root 269: static forced_inline mips_insn_t insn_fetch(mips64_jit_tcb_t *b) 1.1 root 270: { 271: return(vmtoh32(b->mips_code[b->mips_trans_pos])); 272: } 273: 274: /* Emit a breakpoint if necessary */ 1.1.1.4 root 275: #if BREAKPOINT_ENABLE 1.1.1.9 ! root 276: static void insn_emit_breakpoint(cpu_mips_t *cpu,mips64_jit_tcb_t *b) 1.1 root 277: { 278: m_uint64_t pc; 279: int i; 280: 281: pc = b->start_pc+((b->mips_trans_pos-1)<<2); 282: 283: for(i=0;i<MIPS64_MAX_BREAKPOINTS;i++) 284: if (pc == cpu->breakpoints[i]) { 285: mips64_emit_breakpoint(b); 286: break; 287: } 288: } 1.1.1.4 root 289: #endif /* BREAKPOINT_ENABLE */ 1.1 root 290: 1.1.1.9 ! root 291: /* Get host pointer for the physical address */ ! 292: static inline void *physmem_get_hptr(vm_instance_t *vm,m_uint64_t paddr, ! 293: u_int op_size,u_int op_type, ! 294: m_uint64_t *data) ! 295: { ! 296: struct vdevice *dev; ! 297: m_uint32_t offset; ! 298: void *ptr; ! 299: int cow; ! 300: ! 301: if (!(dev = dev_lookup(vm,paddr,FALSE))) ! 302: return NULL; ! 303: ! 304: if (dev->flags & VDEVICE_FLAG_SPARSE) { ! 305: ptr = (void *)dev_sparse_get_host_addr(vm,dev,paddr,op_type,&cow); ! 306: if (!ptr) return NULL; ! 307: ! 308: return(ptr + (paddr & VM_PAGE_IMASK)); ! 309: } ! 310: ! 311: if ((dev->host_addr != 0) && !(dev->flags & VDEVICE_FLAG_NO_MTS_MMAP)) ! 312: return((void *)dev->host_addr + (paddr - dev->phys_addr)); ! 313: ! 314: if (op_size == 0) ! 315: return NULL; ! 316: ! 317: offset = paddr - dev->phys_addr; ! 318: return(dev->handler(vm->boot_cpu,dev,offset,op_size,op_type,data)); ! 319: } ! 320: 1.1 root 321: /* Fetch a MIPS instruction and emit corresponding translated code */ 1.1.1.9 ! root 322: struct mips64_insn_tag *mips64_jit_fetch_and_emit(cpu_mips_t *cpu, ! 323: mips64_jit_tcb_t *block, ! 324: int delay_slot) 1.1 root 325: { 1.1.1.9 ! root 326: struct mips64_insn_tag *tag; 1.1 root 327: mips_insn_t code; 328: 329: code = insn_fetch(block); 330: tag = insn_tag_find(code); 331: assert(tag); 1.1.1.9 ! root 332: ! 333: /* Branch-delay slot is in another page: slow exec */ ! 334: if ((block->mips_trans_pos == (MIPS_INSN_PER_PAGE-1)) && !tag->delay_slot) { ! 335: block->jit_insn_ptr[block->mips_trans_pos] = block->jit_ptr; ! 336: ! 337: mips64_set_pc(block,block->start_pc + (block->mips_trans_pos << 2)); ! 338: mips64_emit_single_step(block,code); ! 339: mips64_jit_tcb_push_epilog(block); ! 340: block->mips_trans_pos++; ! 341: return tag; ! 342: } ! 343: 1.1 root 344: if (delay_slot && !tag->delay_slot) { 1.1.1.4 root 345: mips64_emit_invalid_delay_slot(block); 1.1 root 346: return NULL; 347: } 348: 1.1.1.9 ! root 349: if (!delay_slot) 1.1 root 350: block->jit_insn_ptr[block->mips_trans_pos] = block->jit_ptr; 351: 352: if (delay_slot != 2) 353: block->mips_trans_pos++; 354: 1.1.1.8 root 355: #if DEBUG_INSN_PERF_CNT 1.1.1.4 root 356: mips64_inc_perf_counter(block); 357: #endif 1.1 root 358: 1.1.1.4 root 359: if (!delay_slot) { 360: /* Check for IRQs + Increment count register before jumps */ 361: if (!tag->delay_slot) { 362: mips64_inc_cp0_count_reg(block); 363: mips64_check_pending_irq(block); 364: } 365: } 1.1 root 366: 1.1.1.3 root 367: #if BREAKPOINT_ENABLE 368: if (cpu->breakpoints_enabled) 369: insn_emit_breakpoint(cpu,block); 370: #endif 371: 1.1 root 372: tag->emit(cpu,block,code); 373: return tag; 374: } 375: 376: /* Add end of JIT block */ 1.1.1.9 ! root 377: static void mips64_jit_tcb_add_end(mips64_jit_tcb_t *b) 1.1 root 378: { 379: mips64_set_pc(b,b->start_pc+(b->mips_trans_pos<<2)); 1.1.1.9 ! root 380: mips64_jit_tcb_push_epilog(b); 1.1 root 381: } 382: 383: /* Record a patch to apply in a compiled block */ 1.1.1.9 ! root 384: int mips64_jit_tcb_record_patch(mips64_jit_tcb_t *block,u_char *jit_ptr, ! 385: m_uint64_t vaddr) 1.1 root 386: { 1.1.1.9 ! root 387: struct mips64_jit_patch_table *ipt = block->patch_table; ! 388: struct mips64_insn_patch *patch; 1.1 root 389: 390: /* pc must be 32-bit aligned */ 391: if (vaddr & 0x03) { 392: fprintf(stderr,"Block 0x%8.8llx: trying to record an invalid PC " 393: "(0x%8.8llx) - mips_trans_pos=%d.\n", 394: block->start_pc,vaddr,block->mips_trans_pos); 395: return(-1); 396: } 397: 1.1.1.9 ! root 398: if (!ipt || (ipt->cur_patch >= MIPS64_INSN_PATCH_TABLE_SIZE)) 1.1 root 399: { 400: /* full table or no table, create a new one */ 401: ipt = malloc(sizeof(*ipt)); 402: if (!ipt) { 1.1.1.9 ! root 403: fprintf(stderr,"Block 0x%8.8llx: unable to create patch table.\n", ! 404: block->start_pc); 1.1 root 405: return(-1); 406: } 407: 408: memset(ipt,0,sizeof(*ipt)); 409: ipt->next = block->patch_table; 410: block->patch_table = ipt; 411: } 412: 413: #if DEBUG_BLOCK_PATCH 414: printf("Block 0x%8.8llx: recording patch [JIT:%p->mips:0x%8.8llx], " 415: "MTP=%d\n",block->start_pc,jit_ptr,vaddr,block->mips_trans_pos); 416: #endif 417: 418: patch = &ipt->patches[ipt->cur_patch]; 419: patch->jit_insn = jit_ptr; 420: patch->mips_pc = vaddr; 421: ipt->cur_patch++; 422: return(0); 423: } 424: 425: /* Apply all patches */ 1.1.1.9 ! root 426: static int mips64_jit_tcb_apply_patches(cpu_mips_t *cpu, ! 427: mips64_jit_tcb_t *block) 1.1 root 428: { 1.1.1.9 ! root 429: struct mips64_jit_patch_table *ipt; ! 430: struct mips64_insn_patch *patch; 1.1 root 431: u_char *jit_dst; 432: int i; 433: 434: for(ipt=block->patch_table;ipt;ipt=ipt->next) 435: for(i=0;i<ipt->cur_patch;i++) 436: { 437: patch = &ipt->patches[i]; 1.1.1.9 ! root 438: jit_dst = mips64_jit_tcb_get_host_ptr(block,patch->mips_pc); 1.1 root 439: 1.1.1.4 root 440: if (jit_dst) { 1.1 root 441: #if DEBUG_BLOCK_PATCH 1.1.1.4 root 442: printf("Block 0x%8.8llx: applying patch " 443: "[JIT:%p->mips:0x%8.8llx=JIT:%p]\n", 444: block->start_pc,patch->jit_insn,patch->mips_pc,jit_dst); 1.1 root 445: #endif 1.1.1.9 ! root 446: mips64_jit_tcb_set_patch(patch->jit_insn,jit_dst); 1.1 root 447: } 448: } 449: 450: return(0); 451: } 452: 1.1.1.4 root 453: /* Free the patch table */ 1.1.1.9 ! root 454: static void mips64_jit_tcb_free_patches(mips64_jit_tcb_t *block) 1.1.1.4 root 455: { 1.1.1.9 ! root 456: struct mips64_jit_patch_table *p,*next; 1.1.1.4 root 457: 458: for(p=block->patch_table;p;p=next) { 459: next = p->next; 460: free(p); 461: } 462: 463: block->patch_table = NULL; 464: } 465: 1.1 root 466: /* Adjust the JIT buffer if its size is not sufficient */ 1.1.1.9 ! root 467: static int mips64_jit_tcb_adjust_buffer(cpu_mips_t *cpu, ! 468: mips64_jit_tcb_t *block) 1.1 root 469: { 1.1.1.4 root 470: insn_exec_page_t *new_buffer; 1.1 root 471: 1.1.1.4 root 472: if ((block->jit_ptr - block->jit_buffer->ptr) <= (MIPS_JIT_BUFSIZE - 512)) 1.1 root 473: return(0); 474: 475: #if DEBUG_BLOCK_CHUNK 476: printf("Block 0x%llx: adjusting JIT buffer...\n",block->start_pc); 477: #endif 478: 1.1.1.9 ! root 479: if (block->jit_chunk_pos >= MIPS_JIT_MAX_CHUNKS) { 1.1 root 480: fprintf(stderr,"Block 0x%llx: too many JIT chunks.\n",block->start_pc); 481: return(-1); 482: } 483: 1.1.1.4 root 484: if (!(new_buffer = exec_page_alloc(cpu))) 1.1 root 485: return(-1); 486: 1.1.1.4 root 487: /* record the new exec page */ 488: block->jit_chunks[block->jit_chunk_pos++] = block->jit_buffer; 489: block->jit_buffer = new_buffer; 490: 491: /* jump to the new exec page (link) */ 1.1.1.9 ! root 492: mips64_jit_tcb_set_jump(block->jit_ptr,new_buffer->ptr); 1.1.1.4 root 493: block->jit_ptr = new_buffer->ptr; 1.1 root 494: return(0); 495: } 496: 1.1.1.4 root 497: /* Allocate an instruction block */ 1.1.1.9 ! root 498: static inline mips64_jit_tcb_t *mips64_jit_tcb_alloc(cpu_mips_t *cpu) 1.1 root 499: { 1.1.1.9 ! root 500: mips64_jit_tcb_t *p; 1.1 root 501: 1.1.1.9 ! root 502: if (cpu->tcb_free_list) { ! 503: p = cpu->tcb_free_list; ! 504: cpu->tcb_free_list = p->next; 1.1.1.4 root 505: } else { 506: if (!(p = malloc(sizeof(*p)))) 507: return NULL; 508: } 1.1 root 509: 1.1.1.4 root 510: memset(p,0,sizeof(*p)); 511: return p; 512: } 1.1 root 513: 1.1.1.4 root 514: /* Free an instruction block */ 1.1.1.9 ! root 515: void mips64_jit_tcb_free(cpu_mips_t *cpu,mips64_jit_tcb_t *block, ! 516: int list_removal) 1.1.1.4 root 517: { 518: int i; 1.1 root 519: 1.1.1.4 root 520: if (block) { 521: if (list_removal) { 522: /* Remove the block from the linked list */ 523: if (block->next) 524: block->next->prev = block->prev; 525: else 1.1.1.9 ! root 526: cpu->tcb_last = block->prev; 1.1.1.4 root 527: 528: if (block->prev) 529: block->prev->next = block->next; 530: else 1.1.1.9 ! root 531: cpu->tcb_list = block->next; 1.1 root 532: } 533: 1.1.1.4 root 534: /* Free the patch tables */ 1.1.1.9 ! root 535: mips64_jit_tcb_free_patches(block); 1.1 root 536: 1.1.1.4 root 537: /* Free code pages */ 1.1.1.9 ! root 538: for(i=0;i<MIPS_JIT_MAX_CHUNKS;i++) 1.1.1.4 root 539: exec_page_free(cpu,block->jit_chunks[i]); 1.1 root 540: 1.1.1.4 root 541: /* Free the current JIT buffer */ 542: exec_page_free(cpu,block->jit_buffer); 1.1 root 543: 1.1.1.4 root 544: /* Free the MIPS-to-native code mapping */ 545: free(block->jit_insn_ptr); 1.1 root 546: 1.1.1.4 root 547: /* Make the block return to the free list */ 1.1.1.9 ! root 548: block->next = cpu->tcb_free_list; ! 549: cpu->tcb_free_list = block; 1.1 root 550: } 1.1.1.4 root 551: } 1.1 root 552: 1.1.1.4 root 553: /* Create an instruction block */ 1.1.1.9 ! root 554: static mips64_jit_tcb_t *mips64_jit_tcb_create(cpu_mips_t *cpu, ! 555: m_uint64_t vaddr) 1.1.1.4 root 556: { 1.1.1.9 ! root 557: mips64_jit_tcb_t *block = NULL; 1.1 root 558: 1.1.1.9 ! root 559: if (!(block = mips64_jit_tcb_alloc(cpu))) 1.1.1.4 root 560: goto err_block_alloc; 561: 562: block->start_pc = vaddr; 1.1 root 563: 1.1.1.4 root 564: /* Allocate the first JIT buffer */ 565: if (!(block->jit_buffer = exec_page_alloc(cpu))) 566: goto err_jit_alloc; 1.1 root 567: 1.1.1.4 root 568: block->jit_ptr = block->jit_buffer->ptr; 569: block->mips_code = cpu->mem_op_lookup(cpu,block->start_pc); 570: 571: if (!block->mips_code) { 572: fprintf(stderr,"%% No memory map for code execution at 0x%llx\n", 573: block->start_pc); 574: goto err_lookup; 575: } 576: 577: #if DEBUG_BLOCK_TIMESTAMP 578: block->tm_first_use = block->tm_last_use = jit_jiffies; 1.1 root 579: #endif 1.1.1.4 root 580: return block; 1.1 root 581: 1.1.1.4 root 582: err_lookup: 583: err_jit_alloc: 1.1.1.9 ! root 584: mips64_jit_tcb_free(cpu,block,FALSE); 1.1.1.4 root 585: err_block_alloc: 586: fprintf(stderr,"%% Unable to create instruction block for vaddr=0x%llx\n", 587: vaddr); 588: return NULL; 1.1 root 589: } 590: 1.1.1.4 root 591: /* Compile a MIPS instruction page */ 1.1.1.9 ! root 592: static inline ! 593: mips64_jit_tcb_t *mips64_jit_tcb_compile(cpu_mips_t *cpu,m_uint64_t vaddr) 1.1 root 594: { 1.1.1.9 ! root 595: mips64_jit_tcb_t *block; ! 596: struct mips64_insn_tag *tag; 1.1.1.4 root 597: m_uint64_t page_addr; 598: size_t len; 599: 600: page_addr = vaddr & ~(m_uint64_t)MIPS_MIN_PAGE_IMASK; 601: 1.1.1.9 ! root 602: if (unlikely(!(block = mips64_jit_tcb_create(cpu,page_addr)))) { 1.1.1.4 root 603: fprintf(stderr,"insn_page_compile: unable to create JIT block.\n"); 604: return NULL; 605: } 606: 607: /* Allocate the array used to convert MIPS code ptr to native code ptr */ 608: len = MIPS_MIN_PAGE_SIZE / sizeof(mips_insn_t); 609: 610: if (!(block->jit_insn_ptr = calloc(len,sizeof(u_char *)))) { 611: fprintf(stderr,"insn_page_compile: unable to create JIT mappings.\n"); 612: goto error; 613: } 1.1 root 614: 1.1.1.4 root 615: /* Emit native code for each instruction */ 1.1 root 616: block->mips_trans_pos = 0; 617: 1.1.1.9 ! root 618: while(block->mips_trans_pos < MIPS_INSN_PER_PAGE) 1.1 root 619: { 1.1.1.9 ! root 620: if (unlikely(!(tag = mips64_jit_fetch_and_emit(cpu,block,0)))) { 1.1.1.4 root 621: fprintf(stderr,"insn_page_compile: unable to fetch instruction.\n"); 622: goto error; 623: } 1.1 root 624: 625: #if DEBUG_BLOCK_COMPILE 1.1.1.4 root 626: printf("Page 0x%8.8llx: emitted tag 0x%8.8x/0x%8.8x\n", 1.1 root 627: block->start_pc,tag->mask,tag->value); 628: #endif 629: 1.1.1.9 ! root 630: mips64_jit_tcb_adjust_buffer(cpu,block); 1.1 root 631: } 632: 1.1.1.9 ! root 633: mips64_jit_tcb_add_end(block); ! 634: mips64_jit_tcb_apply_patches(cpu,block); ! 635: mips64_jit_tcb_free_patches(block); 1.1 root 636: 1.1.1.4 root 637: /* Add the block to the linked list */ 1.1.1.9 ! root 638: block->next = cpu->tcb_list; 1.1.1.4 root 639: block->prev = NULL; 640: 1.1.1.9 ! root 641: if (cpu->tcb_list) ! 642: cpu->tcb_list->prev = block; 1.1.1.4 root 643: else 1.1.1.9 ! root 644: cpu->tcb_last = block; 1.1 root 645: 1.1.1.9 ! root 646: cpu->tcb_list = block; 1.1.1.2 root 647: 1.1.1.4 root 648: cpu->compiled_pages++; 1.1 root 649: return block; 1.1.1.4 root 650: 651: error: 1.1.1.9 ! root 652: mips64_jit_tcb_free(cpu,block,FALSE); 1.1.1.4 root 653: return NULL; 1.1 root 654: } 655: 656: /* Run a compiled MIPS instruction block */ 1.1.1.9 ! root 657: static forced_inline ! 658: void mips64_jit_tcb_run(cpu_mips_t *cpu,mips64_jit_tcb_t *block) 1.1 root 659: { 660: #if DEBUG_SYM_TREE 661: struct symbol *sym = NULL; 662: int mark = FALSE; 663: #endif 664: 665: if (unlikely(cpu->pc & 0x03)) { 1.1.1.9 ! root 666: fprintf(stderr,"mips64_jit_tcb_run: Invalid PC 0x%llx.\n",cpu->pc); ! 667: mips64_dump_regs(cpu->gen); ! 668: mips64_tlb_dump(cpu->gen); ! 669: cpu_stop(cpu->gen); 1.1.1.4 root 670: return; 1.1 root 671: } 672: 673: #if DEBUG_SYM_TREE 1.1.1.4 root 674: if (cpu->sym_trace && cpu->sym_tree) 1.1 root 675: { 1.1.1.4 root 676: if ((sym = mips64_sym_lookup(cpu,cpu->pc)) != NULL) { 1.1.1.9 ! root 677: cpu_log(cpu,"mips64_jit_tcb_run(start)", 1.1.1.4 root 678: "%s (PC=0x%llx) RA = 0x%llx\na0=0x%llx, " 1.1 root 679: "a1=0x%llx, a2=0x%llx, a3=0x%llx\n", 680: sym->name, cpu->pc, cpu->gpr[MIPS_GPR_RA], 681: cpu->gpr[MIPS_GPR_A0], cpu->gpr[MIPS_GPR_A1], 682: cpu->gpr[MIPS_GPR_A2], cpu->gpr[MIPS_GPR_A3]); 683: mark = TRUE; 684: } 685: } 686: #endif 687: 688: /* Execute JIT compiled code */ 1.1.1.9 ! root 689: mips64_jit_tcb_exec(cpu,block); 1.1 root 690: 691: #if DEBUG_SYM_TREE 692: if (mark) { 1.1.1.9 ! root 693: cpu_log(cpu,"mips64_jit_tcb_run(end)","%s, v0 = 0x%llx\n", 1.1 root 694: sym->name,cpu->gpr[MIPS_GPR_V0]); 695: } 696: #endif 697: } 698: 699: /* Check if the specified address belongs to the specified block */ 1.1.1.9 ! root 700: int mips64_jit_tcb_local_addr(mips64_jit_tcb_t *block,m_uint64_t vaddr, ! 701: u_char **jit_addr) 1.1 root 702: { 1.1.1.4 root 703: if ((vaddr >= block->start_pc) && 704: ((vaddr - block->start_pc) < MIPS_MIN_PAGE_SIZE)) 705: { 1.1.1.9 ! root 706: *jit_addr = mips64_jit_tcb_get_host_ptr(block,vaddr); 1.1 root 707: return(1); 708: } 1.1.1.4 root 709: 1.1 root 710: return(0); 711: } 712: 1.1.1.5 root 713: /* Check if PC register matches the compiled block virtual address */ 1.1.1.9 ! root 714: static forced_inline ! 715: int mips64_jit_tcb_match(cpu_mips_t *cpu,mips64_jit_tcb_t *block) 1.1.1.5 root 716: { 717: m_uint64_t vpage; 718: 719: vpage = cpu->pc & ~(m_uint64_t)MIPS_MIN_PAGE_IMASK; 720: return(block->start_pc == vpage); 721: } 722: 1.1.1.9 ! root 723: /* Execute compiled MIPS code */ ! 724: void *mips64_jit_run_cpu(cpu_gen_t *gen) ! 725: { ! 726: cpu_mips_t *cpu = CPU_MIPS64(gen); 1.1.1.4 root 727: pthread_t timer_irq_thread; 1.1.1.9 ! root 728: mips64_jit_tcb_t *block; 1.1.1.4 root 729: m_uint32_t phys_page; 730: int timer_irq_check = 0; 1.1 root 731: 1.1.1.4 root 732: if (pthread_create(&timer_irq_thread,NULL, 733: (void *)mips64_timer_irq_run,cpu)) 734: { 1.1.1.7 root 735: fprintf(stderr, 736: "VM '%s': unable to create Timer IRQ thread for CPU%u.\n", 1.1.1.9 ! root 737: cpu->vm->name,gen->id); ! 738: cpu_stop(cpu->gen); 1.1.1.4 root 739: return NULL; 740: } 741: 1.1.1.9 ! root 742: gen->cpu_thread_running = TRUE; 1.1.1.6 root 743: 1.1.1.4 root 744: start_cpu: 1.1.1.9 ! root 745: gen->idle_count = 0; 1.1.1.6 root 746: 1.1 root 747: for(;;) { 1.1.1.9 ! root 748: if (unlikely(gen->state != CPU_STATE_RUNNING)) 1.1 root 749: break; 750: 1.1.1.8 root 751: #if DEBUG_BLOCK_PERF_CNT 752: cpu->perf_counter++; 753: #endif 1.1.1.4 root 754: /* Handle virtual idle loop */ 755: if (unlikely(cpu->pc == cpu->idle_pc)) { 1.1.1.9 ! root 756: if (++gen->idle_count == gen->idle_max) { ! 757: cpu_idle_loop(gen); ! 758: gen->idle_count = 0; 1.1.1.4 root 759: } 760: } 761: 762: /* Handle the virtual CPU clock */ 763: if (++timer_irq_check == cpu->timer_irq_check_itv) { 764: timer_irq_check = 0; 765: 766: if (cpu->timer_irq_pending && !cpu->irq_disable) { 767: mips64_trigger_timer_irq(cpu); 768: mips64_trigger_irq(cpu); 769: cpu->timer_irq_pending--; 770: } 771: } 1.1 root 772: 1.1.1.4 root 773: /* Get the physical page address corresponding to PC register */ 774: if (unlikely(cpu->translate(cpu,cpu->pc,&phys_page))) { 775: fprintf(stderr,"VM '%s': no physical page for CPU%u PC=0x%llx\n", 1.1.1.9 ! root 776: cpu->vm->name,gen->id,cpu->pc); ! 777: cpu_stop(gen); 1.1.1.4 root 778: break; 779: } 1.1.1.2 root 780: 1.1.1.4 root 781: block = cpu->exec_phys_map[phys_page]; 782: 783: /* No block found, compile the page */ 1.1.1.9 ! root 784: if (unlikely(!block) || unlikely(!mips64_jit_tcb_match(cpu,block))) 1.1.1.5 root 785: { 786: if (block != NULL) { 1.1.1.9 ! root 787: mips64_jit_tcb_free(cpu,block,TRUE); 1.1.1.5 root 788: cpu->exec_phys_map[phys_page] = NULL; 789: } 790: 1.1.1.9 ! root 791: block = mips64_jit_tcb_compile(cpu,cpu->pc); 1.1.1.4 root 792: if (unlikely(!block)) { 793: fprintf(stderr, 794: "VM '%s': unable to compile block for CPU%u PC=0x%llx\n", 1.1.1.9 ! root 795: cpu->vm->name,gen->id,cpu->pc); ! 796: cpu_stop(gen); 1.1.1.4 root 797: break; 1.1 root 798: } 1.1.1.2 root 799: 1.1.1.4 root 800: block->phys_page = phys_page; 801: cpu->exec_phys_map[phys_page] = block; 1.1 root 802: } 803: 804: #if DEBUG_BLOCK_TIMESTAMP 805: block->tm_last_use = jit_jiffies++; 806: #endif 807: block->acc_count++; 1.1.1.9 ! root 808: mips64_jit_tcb_run(cpu,block); 1.1 root 809: } 1.1.1.4 root 810: 1.1 root 811: if (!cpu->pc) { 1.1.1.9 ! root 812: cpu_stop(gen); ! 813: cpu_log(gen,"JIT","PC=0, halting CPU.\n"); 1.1 root 814: } 815: 1.1.1.4 root 816: /* Check regularly if the CPU has been restarted */ 1.1.1.9 ! root 817: while(gen->cpu_thread_running) { ! 818: gen->seq_state++; 1.1.1.4 root 819: 1.1.1.9 ! root 820: switch(gen->state) { ! 821: case CPU_STATE_RUNNING: ! 822: gen->state = CPU_STATE_RUNNING; 1.1.1.4 root 823: goto start_cpu; 824: 1.1.1.9 ! root 825: case CPU_STATE_HALTED: ! 826: gen->cpu_thread_running = FALSE; 1.1.1.4 root 827: pthread_join(timer_irq_thread,NULL); 828: break; 829: } 1.1 root 830: 1.1.1.4 root 831: /* CPU is paused */ 1.1 root 832: usleep(200000); 1.1.1.4 root 833: } 1.1 root 834: 835: return NULL; 836: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.