![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to rc-x86-chain.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [The Machine Emulator] / tme / libtme / host / x86|
Return to rc-x86-chain.c CVS log | Up to [The Machine Emulator] / tme / libtme / host / x86 |
1.1 ! root 1: /* $Id: rc-x86-chain.c,v 1.3 2010/02/15 22:15:59 fredette Exp $ */ ! 2: ! 3: /* libtme/host/x86/rc-x86-chain.c - x86 host recode chain support: */ ! 4: ! 5: /* ! 6: * Copyright (c) 2009 Matt Fredette ! 7: * All rights reserved. ! 8: * ! 9: * Redistribution and use in source and binary forms, with or without ! 10: * modification, are permitted provided that the following conditions ! 11: * are met: ! 12: * 1. Redistributions of source code must retain the above copyright ! 13: * notice, this list of conditions and the following disclaimer. ! 14: * 2. Redistributions in binary form must reproduce the above copyright ! 15: * notice, this list of conditions and the following disclaimer in the ! 16: * documentation and/or other materials provided with the distribution. ! 17: * 3. All advertising materials mentioning features or use of this software ! 18: * must display the following acknowledgement: ! 19: * This product includes software developed by Matt Fredette. ! 20: * 4. The name of the author may not be used to endorse or promote products ! 21: * derived from this software without specific prior written permission. ! 22: * ! 23: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR ! 24: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ! 25: * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ! 26: * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, ! 27: * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ! 28: * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR ! 29: * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) ! 30: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, ! 31: * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ! 32: * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE ! 33: * POSSIBILITY OF SUCH DAMAGE. ! 34: */ ! 35: ! 36: _TME_RCSID("$Id: rc-x86-chain.c,v 1.3 2010/02/15 22:15:59 fredette Exp $"); ! 37: ! 38: /* macros: */ ! 39: ! 40: /* the x86 register for the pointer to the token for the current ! 41: instruction TLB entry: */ ! 42: #define TME_RECODE_X86_REG_CHAIN_ITLB_CURRENT_TOKEN (TME_RECODE_X86_REG_TLB) ! 43: ! 44: /* the x86 register for the chain return address: */ ! 45: #define TME_RECODE_X86_REG_CHAIN_RETURN_ADDRESS (TME_RECODE_X86_REG_TLB) ! 46: ! 47: /* the x86 register for the chain return address stack pointer: */ ! 48: #define TME_RECODE_X86_REG_CHAIN_RAS_POINTER (TME_RECODE_X86_REG_TLB_SCRATCH) ! 49: ! 50: /* the x86 register for the fixup address: */ ! 51: #define TME_RECODE_X86_REG_CHAIN_FIXUP_ADDRESS (TME_RECODE_X86_REG_TLB_SCRATCH) ! 52: ! 53: /* the x86 register for the guest instructions source address: */ ! 54: /* NB: we use the a register to take advantage of the fact that there ! 55: is a special cmp $imm, %a form that will save a byte in each ! 56: instruction thunk: */ ! 57: #define TME_RECODE_X86_REG_CHAIN_GUEST_SRC (TME_RECODE_X86_REG_A) ! 58: ! 59: /* this gives the one or two opcode bytes for a cmp $imm32, ! 60: %TME_RECODE_X86_REG_CHAIN_GUEST_SRC: */ ! 61: #if TME_RECODE_X86_REG_CHAIN_GUEST_SRC != TME_RECODE_X86_REG_A ! 62: #error "TME_RECODE_X86_REG_CHAIN_GUEST_SRC changed" ! 63: #endif ! 64: #define TME_RECODE_X86_INSN_CMP_Iz_REG_CHAIN_GUEST_SRC \ ! 65: (TME_RECODE_X86_REX_B(TME_RECODE_SIZE_HOST, \ ! 66: TME_RECODE_X86_REG_CHAIN_GUEST_SRC) \ ! 67: + ((TME_RECODE_X86_OPCODE_BINOP_CMP \ ! 68: + TME_RECODE_X86_OPCODE_BINOP_Iz_A) \ ! 69: << (8 * (TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32)))) ! 70: ! 71: /* this gives the one opcode byte for a movl $imm32, %TME_RECODE_X86_REG_TLB_SCRATCH: */ ! 72: #if TME_RECODE_X86_REG(TME_RECODE_X86_REG_TLB_SCRATCH) != TME_RECODE_X86_REG_TLB_SCRATCH ! 73: #error "TME_RECODE_X86_REG_TLB_SCRATCH changed" ! 74: #endif ! 75: #define TME_RECODE_X86_INSN_MOVL_IMM32_REG_TLB_SCRATCH \ ! 76: TME_RECODE_X86_OPCODE_MOV_Iv_Gv(TME_RECODE_X86_REG_TLB_SCRATCH) ! 77: ! 78: /* this gives the three opcode bytes for a lea disp32(%ip), ! 79: %TME_RECODE_X86_REG_TLB_SCRATCH: */ ! 80: #define TME_RECODE_X86_INSN_LEA_DISP32_IP_REG_TLB_SCRATCH \ ! 81: (TME_RECODE_X86_REX_R(TME_RECODE_SIZE_HOST, \ ! 82: TME_RECODE_X86_REG_TLB_SCRATCH) \ ! 83: + (TME_RECODE_X86_OPCODE_LEA \ ! 84: << 8) \ ! 85: + (TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA(TME_RECODE_X86_EA_BASE_IP),\ ! 86: TME_RECODE_X86_REG(TME_RECODE_X86_REG_TLB_SCRATCH))\ ! 87: << 16)) ! 88: ! 89: /* this gives the two opcode bytes for a movq $imm64, ! 90: %TME_RECODE_X86_REG_TLB_SCRATCH: */ ! 91: #define TME_RECODE_X86_INSN_MOVQ_IMM64_REG_TLB_SCRATCH \ ! 92: (TME_RECODE_X86_REX_R(TME_RECODE_SIZE_HOST, \ ! 93: TME_RECODE_X86_REG_TLB_SCRATCH) \ ! 94: + (TME_RECODE_X86_OPCODE_MOV_Iv_Gv(TME_RECODE_X86_REG_TLB_SCRATCH)\ ! 95: << 8)) ! 96: ! 97: /* this gives the three opcode bytes for a testb $imm8, addr32: */ ! 98: #define TME_RECODE_X86_INSN_TESTB_IMM8_ADDR32 \ ! 99: (TME_RECODE_X86_OPCODE_GRP3_Eb \ ! 100: + (TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA(TME_RECODE_X86_EA_BASE_SIB),\ ! 101: TME_RECODE_X86_OPCODE_GRP3_TEST)\ ! 102: << 8) \ ! 103: + (TME_RECODE_X86_SIB(TME_RECODE_X86_SIB_BASE_NONE, \ ! 104: TME_RECODE_X86_SIB_INDEX_NONE, \ ! 105: 1) \ ! 106: << 16)) ! 107: ! 108: /* this gives the two opcode bytes for a testb $imm8, disp32(%ip): */ ! 109: #define TME_RECODE_X86_INSN_TESTB_IMM8_DISP32_IP \ ! 110: (TME_RECODE_X86_OPCODE_GRP3_Eb \ ! 111: + (TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA(TME_RECODE_X86_EA_BASE_IP),\ ! 112: TME_RECODE_X86_OPCODE_GRP3_TEST)\ ! 113: << 8)) ! 114: ! 115: /* this gives the two opcode bytes for a testb $imm8, ! 116: disp32(%TME_RECODE_X86_REG_CHAIN_GUEST_SRC): */ ! 117: #if TME_RECODE_X86_REG(TME_RECODE_X86_REG_CHAIN_GUEST_SRC) != TME_RECODE_X86_REG_CHAIN_GUEST_SRC ! 118: #error "TME_RECODE_X86_REG_CHAIN_GUEST_SRC changed" ! 119: #endif ! 120: #define TME_RECODE_X86_INSN_TESTB_IMM8_DISP32_REG_CHAIN_GUEST_SRC \ ! 121: (TME_RECODE_X86_OPCODE_GRP3_Eb \ ! 122: + (TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA_DISP32(TME_RECODE_X86_REG_CHAIN_GUEST_SRC),\ ! 123: TME_RECODE_X86_OPCODE_GRP3_TEST)\ ! 124: << 8)) ! 125: ! 126: /* this is the size of a jcc epilogue instruction: */ ! 127: #define TME_RECODE_X86_CHAIN_IN_SIZE_JCC_EPILOGUE \ ! 128: (6 /* jcc epilogue */) ! 129: ! 130: /* this is the size of the chain in far instructions on an ia32 host: */ ! 131: #define TME_RECODE_IA32_CHAIN_IN_SIZE_FAR \ ! 132: (5 /* cmpl $imm32, %eax */ \ ! 133: + TME_RECODE_X86_CHAIN_IN_SIZE_JCC_EPILOGUE \ ! 134: + 7 /* testb $imm8, addr32 */ \ ! 135: + TME_RECODE_X86_CHAIN_IN_SIZE_JCC_EPILOGUE) ! 136: ! 137: /* the size of cmpq $imm32, %TME_RECODE_X86_REG_CHAIN_GUEST_SRC ; jnz ! 138: epilogue */ ! 139: #define TME_RECODE_X86_64_CHAIN_IN_SIZE_CMPQ_IMM32_JNZ \ ! 140: (6 /* cmpq $imm32, %rax */ \ ! 141: + TME_RECODE_X86_CHAIN_IN_SIZE_JCC_EPILOGUE) ! 142: ! 143: /* the size of cmpq %reg, %reg ; jnz epilogue */ ! 144: #define TME_RECODE_X86_64_CHAIN_IN_SIZE_CMP_JNZ \ ! 145: (3 /* cmpq %reg, %reg */ \ ! 146: + TME_RECODE_X86_CHAIN_IN_SIZE_JCC_EPILOGUE) ! 147: ! 148: /* the size of movl $imm32, %reg ; cmpq %reg, %reg ; jnz epilogue */ ! 149: #define TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVL_CMP_JNZ \ ! 150: (5 /* movl $imm32, %reg */ \ ! 151: + TME_RECODE_X86_64_CHAIN_IN_SIZE_CMP_JNZ) ! 152: ! 153: /* the size of lea disp32(%ip), %reg ; cmpq %reg, %reg ; jnz epilogue */ ! 154: #define TME_RECODE_X86_64_CHAIN_IN_SIZE_LEA_IP_CMP_JNZ \ ! 155: (7 /* leaq disp32(%ip), %reg */ \ ! 156: + TME_RECODE_X86_64_CHAIN_IN_SIZE_CMP_JNZ) ! 157: ! 158: /* the size of movq $imm64, %reg ; cmpq %reg, %reg ; jnz epilogue */ ! 159: #define TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVQ_CMP_JNZ \ ! 160: (10 /* movq $imm64, %reg */ \ ! 161: + TME_RECODE_X86_64_CHAIN_IN_SIZE_CMP_JNZ) ! 162: ! 163: /* the size of testb $imm8, addr32 ; jz epilogue */ ! 164: #define TME_RECODE_X86_64_CHAIN_IN_SIZE_TESTB_ADDR32_JZ \ ! 165: (8 /* testb $imm8, addr32 */ \ ! 166: + TME_RECODE_X86_CHAIN_IN_SIZE_JCC_EPILOGUE) ! 167: ! 168: /* the size of testb $imm8, base32(%reg) ; jz epilogue */ ! 169: #define TME_RECODE_X86_64_CHAIN_IN_SIZE_TESTB_BASE_JZ \ ! 170: (7 /* testb $imm8, addr32 */ \ ! 171: + TME_RECODE_X86_CHAIN_IN_SIZE_JCC_EPILOGUE) ! 172: ! 173: /* the size of movq $imm64, %reg ; testb $imm32, (%reg) ; jz epilogue */ ! 174: #define TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVQ_TESTB_JZ \ ! 175: (10 /* movq $imm64, %reg */ \ ! 176: + 3 /* testb $imm32, (%reg) */ \ ! 177: + TME_RECODE_X86_CHAIN_IN_SIZE_JCC_EPILOGUE) ! 178: ! 179: /* this gives a chain subs: */ ! 180: #if (TME_RECODE_CHAIN_INFO_CONDITIONAL | TME_RECODE_CHAIN_INFO_FAR) != 3 ! 181: #error "TME_RECODE_CHAIN_INFO_ values changed" ! 182: #endif ! 183: #if (TME_RECODE_CHAIN_INFO_JUMP != 4 || TME_RECODE_CHAIN_INFO_RETURN != 8) ! 184: #error "TME_RECODE_CHAIN_INFO_ values changed" ! 185: #endif ! 186: #define TME_RECODE_X86_CHAIN_SUBS(chain_thunk, chain_info)\ ! 187: ((chain_thunk)->tme_recode_x86_chain_thunk_subs \ ! 188: [(chain_info) \ ! 189: & (TME_RECODE_CHAIN_INFO_CONDITIONAL \ ! 190: | TME_RECODE_CHAIN_INFO_FAR \ ! 191: | TME_RECODE_CHAIN_INFO_JUMP \ ! 192: | TME_RECODE_CHAIN_INFO_RETURN)]) ! 193: ! 194: /* these give a chain fixup target for the regular target or alternate ! 195: target: */ ! 196: #if TME_RECODE_CHAIN_INFO_FAR != 2 || TME_RECODE_CHAIN_INFO_ALTERNATE_FAR == 0 ! 197: #error "TME_RECODE_CHAIN_INFO_ values changed" ! 198: #endif ! 199: #define TME_RECODE_X86_CHAIN_FIXUP_TARGET(ic, chain_info) \ ! 200: ((ic)->tme_recode_x86_ic_chain_fixup_target \ ! 201: [((chain_info) & TME_RECODE_CHAIN_INFO_FAR) \ ! 202: + (((chain_info) / TME_RECODE_CHAIN_INFO_CALL) & 1)]) ! 203: #define TME_RECODE_X86_CHAIN_FIXUP_TARGET_ALTERNATE(ic, chain_info) \ ! 204: ((ic)->tme_recode_x86_ic_chain_fixup_target \ ! 205: [4 \ ! 206: + (((chain_info) / TME_RECODE_CHAIN_INFO_ALTERNATE_FAR) & 1)]) ! 207: ! 208: /* on an ia32 host, an entry on the return address stack is the ! 209: absolute address of either an instructions thunk or the chain ! 210: epilogue. on an x86-64 host, an entry on the return address stack ! 211: is the instructions thunk offset of either an instructions thunk or ! 212: the chain epilogue: */ ! 213: #if TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32 ! 214: #define TME_RECODE_X86_CHAIN_RETURN_ADDRESS(ic, insns_thunk)\ ! 215: ((tme_recode_ras_entry_t) (insns_thunk)) ! 216: #else /* TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32 */ ! 217: #define TME_RECODE_X86_CHAIN_RETURN_ADDRESS(ic, insns_thunk)\ ! 218: tme_recode_thunk_off_to_pointer(ic, \ ! 219: insns_thunk, \ ! 220: tme_recode_ras_entry_t) ! 221: #endif /* TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32 */ ! 222: ! 223: /* this emits instructions for a chain epilogue: */ ! 224: static void ! 225: _tme_recode_x86_chain_epilogue(struct tme_recode_ic *ic) ! 226: { ! 227: tme_uint8_t *thunk_bytes; ! 228: ! 229: /* start more instructions: */ ! 230: tme_recode_x86_insns_start(ic, thunk_bytes); ! 231: ! 232: /* set the instructions thunk offset of the chain epilogue: */ ! 233: ic->tme_recode_x86_ic_chain_epilogue = tme_recode_build_to_thunk_off(ic, thunk_bytes); ! 234: ! 235: /* remove the recode carry flag stack word: */ ! 236: thunk_bytes ! 237: = _tme_recode_x86_emit_adjust_sp(thunk_bytes, ! 238: TME_BIT(TME_RECODE_SIZE_HOST ! 239: - TME_RECODE_SIZE_8)); ! 240: ! 241: /* pop all callee-saved registers: */ ! 242: if (TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32) { ! 243: _tme_recode_x86_emit_reg_pop(thunk_bytes, TME_RECODE_X86_REG_N(15)); ! 244: _tme_recode_x86_emit_reg_pop(thunk_bytes, TME_RECODE_X86_REG_N(14)); ! 245: _tme_recode_x86_emit_reg_pop(thunk_bytes, TME_RECODE_X86_REG_N(13)); ! 246: _tme_recode_x86_emit_reg_pop(thunk_bytes, TME_RECODE_X86_REG_N(12)); ! 247: } ! 248: else { ! 249: _tme_recode_x86_emit_reg_pop(thunk_bytes, TME_RECODE_X86_REG_DI); ! 250: _tme_recode_x86_emit_reg_pop(thunk_bytes, TME_RECODE_X86_REG_SI); ! 251: } ! 252: _tme_recode_x86_emit_reg_pop(thunk_bytes, TME_RECODE_X86_REG_B); ! 253: _tme_recode_x86_emit_reg_pop(thunk_bytes, TME_RECODE_X86_REG_BP); ! 254: ! 255: /* emit the return: */ ! 256: thunk_bytes[0] = TME_RECODE_X86_OPCODE_RET; ! 257: thunk_bytes++; ! 258: ! 259: /* finish these instructions: */ ! 260: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 261: ! 262: /* finish the chain epilogue: */ ! 263: tme_recode_host_thunk_finish(ic); ! 264: } ! 265: ! 266: /* this emits instructions to advance the fixup address register ! 267: past a chain conditional six-byte jcc instruction: */ ! 268: static tme_uint8_t * ! 269: _tme_recode_x86_chain_fixup_alternate(tme_uint8_t *thunk_bytes) ! 270: { ! 271: ! 272: /* NB: this is always a 32-bit instruction, even on an x86-64 host, ! 273: because we end up converting the fixup address into an ! 274: instructions thunk offset, which is only 32 bits: */ ! 275: assert (sizeof(tme_recode_thunk_off_t) == sizeof(tme_int32_t)); ! 276: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP1_Ib_Ev; ! 277: thunk_bytes[1] ! 278: = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(TME_RECODE_X86_REG_CHAIN_FIXUP_ADDRESS), ! 279: TME_RECODE_X86_OPCODE_GRP1_BINOP(TME_RECODE_X86_OPCODE_BINOP_ADD)); ! 280: thunk_bytes[2] = 2 + sizeof(tme_int32_t); ! 281: thunk_bytes += 3; ! 282: return (thunk_bytes); ! 283: } ! 284: ! 285: /* this emits instructions to make a constant for the third argument ! 286: to the chain fixup function: */ ! 287: static tme_uint8_t * ! 288: _tme_recode_x86_chain_fixup_arg2(tme_uint8_t *thunk_bytes, ! 289: tme_uint32_t chain_info) ! 290: { ! 291: ! 292: /* if this is an ia32 host: */ ! 293: if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32) { ! 294: ! 295: /* push the argument: */ ! 296: assert (chain_info < 0x80); ! 297: thunk_bytes[0] = TME_RECODE_X86_OPCODE_PUSH_Ib; ! 298: thunk_bytes[1] = chain_info; ! 299: thunk_bytes += 2; ! 300: } ! 301: ! 302: /* otherwise, this is an x86-64 host: */ ! 303: else { ! 304: ! 305: /* load the third argument register: */ ! 306: thunk_bytes[0] = TME_RECODE_X86_OPCODE_MOV_Iv_Gv(TME_RECODE_X86_REG_D); ! 307: *((tme_uint32_t *) &thunk_bytes[1]) = chain_info; ! 308: thunk_bytes += 1 + sizeof(tme_uint32_t); ! 309: } ! 310: ! 311: return (thunk_bytes); ! 312: } ! 313: ! 314: /* this emits instructions for the chain fixup targets for chain jump ! 315: far, chain call far, and chain call near: */ ! 316: static void ! 317: _tme_recode_x86_chain_fixup_target(struct tme_recode_ic *ic, ! 318: tme_uint32_t chain_info, ! 319: const tme_uint8_t *thunk_bytes_fixup_call) ! 320: { ! 321: tme_uint8_t *thunk_bytes; ! 322: ! 323: /* start more instructions: */ ! 324: tme_recode_x86_insns_start(ic, thunk_bytes); ! 325: ! 326: /* set next fixup target: */ ! 327: TME_RECODE_X86_CHAIN_FIXUP_TARGET(ic, ! 328: chain_info) ! 329: = tme_recode_build_to_thunk_off(ic, thunk_bytes); ! 330: ! 331: /* make the third argument for the chain fixup function: */ ! 332: thunk_bytes = _tme_recode_x86_chain_fixup_arg2(thunk_bytes, chain_info); ! 333: ! 334: /* jmp to the common chain fixup call point: */ ! 335: thunk_bytes ! 336: = _tme_recode_x86_emit_jmp(thunk_bytes, ! 337: TME_RECODE_X86_OPCODE_JMP_RELz, ! 338: thunk_bytes_fixup_call); ! 339: ! 340: /* finish these instructions: */ ! 341: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 342: } ! 343: ! 344: /* this makes the chain fixup targets for a new IC: */ ! 345: static void ! 346: _tme_recode_x86_chain_fixup_targets(struct tme_recode_ic *ic) ! 347: { ! 348: tme_uint8_t *thunk_bytes; ! 349: const tme_uint8_t *thunk_bytes_fixup_call; ! 350: unsigned long thunk_address0; ! 351: unsigned int stack_adjust; ! 352: ! 353: /* start more instructions: */ ! 354: tme_recode_x86_insns_start(ic, thunk_bytes); ! 355: ! 356: /* the first fixup target is for chain jump alternate near: */ ! 357: TME_RECODE_X86_CHAIN_FIXUP_TARGET_ALTERNATE(ic, ! 358: (TME_RECODE_CHAIN_INFO_JUMP ! 359: + TME_RECODE_CHAIN_INFO_ALTERNATE_NEAR)) ! 360: = tme_recode_build_to_thunk_off(ic, thunk_bytes); ! 361: ! 362: /* advance the fixup address register past the chain jump ! 363: conditional six-byte jcc instruction: */ ! 364: thunk_bytes = _tme_recode_x86_chain_fixup_alternate(thunk_bytes); ! 365: ! 366: /* the next fixup target is for chain jump near: */ ! 367: /* NB: this fixup target is also for chain return alternate near, ! 368: which is treated like a chain jump near: */ ! 369: TME_RECODE_X86_CHAIN_FIXUP_TARGET(ic, ! 370: (TME_RECODE_CHAIN_INFO_JUMP ! 371: + TME_RECODE_CHAIN_INFO_NEAR)) ! 372: = tme_recode_build_to_thunk_off(ic, thunk_bytes); ! 373: ! 374: /* make the third argument for the chain fixup function: */ ! 375: thunk_bytes ! 376: = _tme_recode_x86_chain_fixup_arg2(thunk_bytes, ! 377: TME_RECODE_CHAIN_INFO_NEAR); ! 378: ! 379: /* this is the common chain fixup call point: */ ! 380: thunk_bytes_fixup_call = thunk_bytes; ! 381: ! 382: /* get the near address of the thunk at offset zero: */ ! 383: thunk_address0 = tme_recode_thunk_off_to_pointer(ic, 0, char *) - (char *) 0; ! 384: ! 385: /* convert the fixup address (which has been already truncated to 32 ! 386: bits) into the instructions thunk offset: */ ! 387: /* NB: this is always a 32-bit instruction, because a ! 388: tme_recode_thunk_off_t is a tme_int32_t: */ ! 389: assert (sizeof(tme_recode_thunk_off_t) == sizeof(tme_int32_t)); ! 390: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP1_Iz_Ev; ! 391: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(TME_RECODE_X86_REG_CHAIN_FIXUP_ADDRESS), ! 392: TME_RECODE_X86_OPCODE_GRP1_BINOP(TME_RECODE_X86_OPCODE_BINOP_SUB)); ! 393: *((tme_uint32_t *) &thunk_bytes[2]) = thunk_address0; ! 394: thunk_bytes += 2 + sizeof(tme_uint32_t); ! 395: ! 396: /* if this is an ia32 host: */ ! 397: if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32) { ! 398: ! 399: /* push the arguments for the chain fixup function: */ ! 400: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_CHAIN_FIXUP_ADDRESS); ! 401: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_IC); ! 402: ! 403: /* we will need to remove the arguments from the stack: */ ! 404: stack_adjust = sizeof(struct tme_ic *) + sizeof(tme_recode_thunk_off_t) + sizeof(tme_uint32_t); ! 405: } ! 406: ! 407: /* otherwise, this is an x86-64 host: */ ! 408: else { ! 409: ! 410: /* copy the arguments for the guest jump chain function: */ ! 411: _tme_recode_x86_emit_reg_copy(thunk_bytes, TME_RECODE_X86_REG_IC, TME_RECODE_X86_REG_DI); ! 412: _tme_recode_x86_emit_reg_copy(thunk_bytes, TME_RECODE_X86_REG_CHAIN_FIXUP_ADDRESS, TME_RECODE_X86_REG_SI); ! 413: ! 414: /* we don't need to adjust the stack. NB that the instructions ! 415: thunk, where the stack pointer is 16-byte aligned, jumped to ! 416: us, so we don't have to align it for the x86-64 ABI: */ ! 417: stack_adjust = 0; ! 418: } ! 419: ! 420: /* call the chain fixup function: */ ! 421: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 422: _tme_recode_x86_emit_transfer_func(ic, ! 423: TME_RECODE_X86_OPCODE_CALL_RELz, ! 424: ((void (*) _TME_P((void))) ! 425: ic->tme_recode_ic_chain_fixup)); ! 426: tme_recode_x86_insns_start(ic, thunk_bytes); ! 427: ! 428: /* do any stack adjust: */ ! 429: if (stack_adjust) { ! 430: thunk_bytes = _tme_recode_x86_emit_adjust_sp(thunk_bytes, stack_adjust); ! 431: } ! 432: ! 433: /* jmp to the chain epilogue: */ ! 434: thunk_bytes ! 435: = _tme_recode_x86_emit_jmp(thunk_bytes, ! 436: TME_RECODE_X86_OPCODE_JMP_RELz, ! 437: tme_recode_thunk_off_to_pointer(ic, ! 438: ic->tme_recode_x86_ic_chain_epilogue, ! 439: tme_uint8_t *)); ! 440: ! 441: /* the next fixup target is for chain jump alternate far: */ ! 442: TME_RECODE_X86_CHAIN_FIXUP_TARGET_ALTERNATE(ic, ! 443: (TME_RECODE_CHAIN_INFO_JUMP ! 444: + TME_RECODE_CHAIN_INFO_ALTERNATE_FAR)) ! 445: = tme_recode_build_to_thunk_off(ic, thunk_bytes); ! 446: ! 447: /* advance the fixup address register past the chain jump ! 448: conditional six-byte jcc instruction: */ ! 449: thunk_bytes = _tme_recode_x86_chain_fixup_alternate(thunk_bytes); ! 450: ! 451: /* finish these instructions: */ ! 452: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 453: ! 454: /* the next fixup target is for chain jump far: */ ! 455: _tme_recode_x86_chain_fixup_target(ic, ! 456: TME_RECODE_CHAIN_INFO_FAR, ! 457: thunk_bytes_fixup_call); ! 458: ! 459: /* the next fixup target is for chain call far: */ ! 460: _tme_recode_x86_chain_fixup_target(ic, ! 461: (TME_RECODE_CHAIN_INFO_CALL ! 462: + TME_RECODE_CHAIN_INFO_FAR), ! 463: thunk_bytes_fixup_call); ! 464: ! 465: /* the next fixup target is for chain call near: */ ! 466: _tme_recode_x86_chain_fixup_target(ic, ! 467: (TME_RECODE_CHAIN_INFO_CALL ! 468: + TME_RECODE_CHAIN_INFO_NEAR), ! 469: thunk_bytes_fixup_call); ! 470: ! 471: /* finish the fixup targets thunk: */ ! 472: tme_recode_host_thunk_finish(ic); ! 473: } ! 474: ! 475: /* this emits a jmp to a chain subs: */ ! 476: static tme_uint8_t * ! 477: _tme_recode_x86_chain_subs_jmp(struct tme_recode_ic *ic, ! 478: tme_uint32_t opcode, ! 479: struct tme_recode_chain_thunk *chain_thunk, ! 480: tme_uint32_t chain_info) ! 481: { ! 482: tme_uint8_t *thunk_bytes; ! 483: tme_uint8_t *thunk_bytes_jc; ! 484: tme_recode_thunk_off_t chain_subs; ! 485: tme_uint8_t *thunk_bytes_target; ! 486: ! 487: /* if this is a conditional jmp: */ ! 488: if (opcode != TME_RECODE_X86_OPCODE_JMP_RELb ! 489: && opcode != TME_RECODE_X86_OPCODE_JMP_RELz) { ! 490: ! 491: /* define CF with the recode jump flag. NB that the recode flags ! 492: are now above the return address to the instructions thunk: */ ! 493: _tme_recode_x86_conds_testc(ic, TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8) + TME_RECODE_FLAG_JUMP); ! 494: } ! 495: ! 496: /* start more instructions: */ ! 497: tme_recode_x86_insns_start(ic, thunk_bytes); ! 498: ! 499: /* emit the jmp: */ ! 500: chain_subs = TME_RECODE_X86_CHAIN_SUBS(chain_thunk, chain_info); ! 501: if (chain_subs == 0) { ! 502: thunk_bytes_jc = thunk_bytes; ! 503: thunk_bytes_target = (tme_uint8_t *) NULL; ! 504: } ! 505: else { ! 506: thunk_bytes_jc = (tme_uint8_t *) NULL; ! 507: thunk_bytes_target = tme_recode_thunk_off_to_pointer(ic, chain_subs, tme_uint8_t *); ! 508: } ! 509: thunk_bytes = _tme_recode_x86_emit_jmp(thunk_bytes, opcode, thunk_bytes_target); ! 510: ! 511: /* finish these instructions: */ ! 512: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 513: ! 514: return (thunk_bytes_jc); ! 515: } ! 516: ! 517: /* this emits instructions to add a sign-extended byte to the chain ! 518: return address stack pointer, mask it, and then store it: */ ! 519: static void ! 520: _tme_recode_x86_chain_ras_pointer_update(struct tme_recode_ic *ic, ! 521: tme_int8_t addend) ! 522: { ! 523: tme_uint8_t *thunk_bytes; ! 524: tme_uint8_t ras_size; ! 525: ! 526: /* start these instructions: */ ! 527: tme_recode_x86_insns_start(ic, thunk_bytes); ! 528: ! 529: /* emit the add $imm8: */ ! 530: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP1_Ib_Ev; ! 531: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(TME_RECODE_X86_REG_CHAIN_RAS_POINTER), ! 532: TME_RECODE_X86_OPCODE_GRP1_BINOP(TME_RECODE_X86_OPCODE_BINOP_ADD)); ! 533: thunk_bytes[2] = addend; ! 534: thunk_bytes += 3; ! 535: ! 536: /* emit the and $imm8: */ ! 537: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP1_Ib_Ev; ! 538: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(TME_RECODE_X86_REG_CHAIN_RAS_POINTER), ! 539: TME_RECODE_X86_OPCODE_GRP1_BINOP(TME_RECODE_X86_OPCODE_BINOP_AND)); ! 540: ras_size = ic->tme_recode_ic_chain_ras_size; ! 541: assert (ras_size > 0 && (ras_size & (ras_size - 1)) == 0); ! 542: thunk_bytes[2] = ras_size - 1; ! 543: thunk_bytes += 3; ! 544: ! 545: /* store the chain return address stack pointer: */ ! 546: thunk_bytes[0] = (TME_RECODE_X86_OPCODE_BINOP_MOV + TME_RECODE_X86_OPCODE_BINOP_Gv_Ev); ! 547: thunk_bytes = _tme_recode_x86_emit_ic_modrm(thunk_bytes + 1, ! 548: ic->tme_recode_ic_chain_ras_pointer_offset, ! 549: TME_RECODE_X86_REG_CHAIN_RAS_POINTER); ! 550: ! 551: /* finish these instructions: */ ! 552: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 553: } ! 554: ! 555: /* this emits an instruction to load or store an entry on the chain ! 556: return address stack: */ ! 557: static void ! 558: _tme_recode_x86_chain_ras_ls(struct tme_recode_ic *ic, ! 559: int store) ! 560: { ! 561: tme_uint8_t *thunk_bytes; ! 562: tme_int32_t ras_offset; ! 563: ! 564: /* start more instructions: */ ! 565: tme_recode_x86_insns_start(ic, thunk_bytes); ! 566: ! 567: /* emit the movl: */ ! 568: thunk_bytes[0] ! 569: = (TME_RECODE_X86_OPCODE_BINOP_MOV ! 570: + (store ! 571: ? TME_RECODE_X86_OPCODE_BINOP_Gv_Ev ! 572: : TME_RECODE_X86_OPCODE_BINOP_Ev_Gv)); ! 573: ras_offset = ic->tme_recode_ic_chain_ras_offset; ! 574: thunk_bytes[1] ! 575: = TME_RECODE_X86_MOD_OPREG_RM((ras_offset < 0x80 ! 576: ? TME_RECODE_X86_MOD_RM_EA_DISP8(TME_RECODE_X86_EA_BASE_SIB) ! 577: : TME_RECODE_X86_MOD_RM_EA_DISP32(TME_RECODE_X86_EA_BASE_SIB)), ! 578: TME_RECODE_X86_REG(TME_RECODE_X86_REG_CHAIN_RETURN_ADDRESS)); ! 579: thunk_bytes[2] ! 580: = TME_RECODE_X86_SIB(TME_RECODE_X86_REG_IC, ! 581: TME_RECODE_X86_REG_CHAIN_RAS_POINTER, ! 582: sizeof(tme_recode_ras_entry_t)); ! 583: *((tme_uint32_t *) (thunk_bytes + 3)) = ras_offset; ! 584: thunk_bytes += 3 + (ras_offset < 0x80 ? sizeof(tme_uint8_t) : sizeof(tme_uint32_t)); ! 585: ! 586: /* finish these instructions: */ ! 587: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 588: } ! 589: ! 590: /* this emits instructions to push the chain return address stack: */ ! 591: static void ! 592: _tme_recode_x86_chain_ras_push(struct tme_recode_ic *ic) ! 593: { ! 594: tme_uint8_t *thunk_bytes; ! 595: ! 596: /* start more instructions: */ ! 597: tme_recode_x86_insns_start(ic, thunk_bytes); ! 598: ! 599: /* load the return address stack pointer: */ ! 600: thunk_bytes[0] = (TME_RECODE_X86_OPCODE_BINOP_MOV + TME_RECODE_X86_OPCODE_BINOP_Ev_Gv); ! 601: thunk_bytes = _tme_recode_x86_emit_ic_modrm(thunk_bytes + 1, ! 602: ic->tme_recode_ic_chain_ras_pointer_offset, ! 603: TME_RECODE_X86_REG_CHAIN_RAS_POINTER); ! 604: ! 605: /* finish these instructions: */ ! 606: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 607: ! 608: /* subtract one from the return address stack pointer, and store it: */ ! 609: _tme_recode_x86_chain_ras_pointer_update(ic, -1); ! 610: ! 611: /* store the return address: */ ! 612: _tme_recode_x86_chain_ras_ls(ic, TRUE); ! 613: } ! 614: ! 615: /* this emits instructions to load or store the pointer to the token ! 616: for the current recode instruction TLB entry: */ ! 617: static void ! 618: _tme_recode_x86_chain_itlb_current_token_ls(struct tme_recode_ic *ic, ! 619: int store) ! 620: { ! 621: tme_uint8_t *thunk_bytes; ! 622: ! 623: /* start more instructions: */ ! 624: tme_recode_x86_insns_start(ic, thunk_bytes); ! 625: ! 626: /* load or store the pointer to the token for the current recode ! 627: instruction TLB entry: */ ! 628: if (TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32) { ! 629: *(thunk_bytes++) ! 630: = TME_RECODE_X86_REX_B(TME_RECODE_SIZE_HOST, ! 631: TME_RECODE_X86_REG_CHAIN_ITLB_CURRENT_TOKEN); ! 632: } ! 633: thunk_bytes[0] ! 634: = (TME_RECODE_X86_OPCODE_BINOP_MOV ! 635: + (store ! 636: ? TME_RECODE_X86_OPCODE_BINOP_Gv_Ev ! 637: : TME_RECODE_X86_OPCODE_BINOP_Ev_Gv)); ! 638: thunk_bytes = _tme_recode_x86_emit_ic_modrm(thunk_bytes + 1, ! 639: ic->tme_recode_ic_itlb_current_token_offset, ! 640: TME_RECODE_X86_REG_CHAIN_ITLB_CURRENT_TOKEN); ! 641: ! 642: /* finish these instructions: */ ! 643: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 644: } ! 645: #define _tme_recode_x86_chain_itlb_current_token_load(ic) _tme_recode_x86_chain_itlb_current_token_ls(ic, FALSE) ! 646: ! 647: /* this stores the pointer to the current recode instruction TLB entry: */ ! 648: static void ! 649: _tme_recode_x86_chain_itlb_current_store(struct tme_recode_ic *ic, ! 650: const struct tme_recode_x86_tlb_type *x86_tlb_type) ! 651: { ! 652: tme_uint8_t *thunk_bytes; ! 653: ! 654: /* start more instructions: */ ! 655: tme_recode_x86_insns_start(ic, thunk_bytes); ! 656: ! 657: /* convert the pointer to the recode instruction TLB entry in ! 658: TME_RECODE_X86_REG_TLB into a pointer to the token for that entry ! 659: in TME_RECODE_X86_REG_CHAIN_ITLB_CURRENT_TOKEN: */ ! 660: #if TME_RECODE_X86_REG_CHAIN_ITLB_CURRENT_TOKEN != TME_RECODE_X86_REG_TLB ! 661: #error "TME_RECODE_X86_REG_CHAIN_ITLB_CURRENT_TOKEN changed" ! 662: #endif ! 663: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP1_Ib_Ev; ! 664: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(TME_RECODE_X86_REG_TLB), ! 665: TME_RECODE_X86_OPCODE_GRP1_BINOP(TME_RECODE_X86_OPCODE_BINOP_ADD)); ! 666: assert (x86_tlb_type->tme_recode_tlb_type.tme_recode_tlb_type_offset_token < 0x80); ! 667: thunk_bytes[2] = x86_tlb_type->tme_recode_tlb_type.tme_recode_tlb_type_offset_token; ! 668: thunk_bytes += 3; ! 669: ! 670: /* finish these instructions: */ ! 671: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 672: ! 673: /* store the pointer to the token for the current recode instruction ! 674: TLB entry: */ ! 675: _tme_recode_x86_chain_itlb_current_token_ls(ic, TRUE); ! 676: } ! 677: ! 678: /* this emits instructions for the call and jump chain subs, or for ! 679: the return chain subs: */ ! 680: static void ! 681: _tme_recode_x86_chain_subs(struct tme_recode_ic *ic, ! 682: const struct tme_recode_chain *chain, ! 683: struct tme_recode_chain_thunk *chain_thunk, ! 684: tme_uint32_t chain_info_call_or_return) ! 685: { ! 686: tme_uint8_t *thunk_bytes_jmp_to_jump_far_unconditional; ! 687: tme_uint8_t *thunk_bytes; ! 688: struct tme_recode_x86_tlb_type x86_tlb_type; ! 689: union tme_recode_reginfo reginfo; ! 690: unsigned int rex; ! 691: unsigned int reg_x86_address; ! 692: tme_uint8_t *thunk_bytes_jcc_chain_counter; ! 693: const tme_uint8_t *thunk_bytes_jmp_to_epilogue; ! 694: ! 695: /* the first chain subs is for a chain call far conditional or chain ! 696: return far conditional: */ ! 697: TME_RECODE_X86_CHAIN_SUBS(chain_thunk, ! 698: (chain_info_call_or_return ! 699: + TME_RECODE_CHAIN_INFO_FAR ! 700: + TME_RECODE_CHAIN_INFO_CONDITIONAL)) ! 701: = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next); ! 702: ! 703: /* if the recode jump flag is false, jmp to the chain subs for a ! 704: chain jump far unconditional: */ ! 705: thunk_bytes_jmp_to_jump_far_unconditional ! 706: = _tme_recode_x86_chain_subs_jmp(ic, ! 707: (TME_RECODE_X86_OPCODE_ESC_0F ! 708: + (TME_RECODE_X86_OPCODE0F_JCC(TME_RECODE_X86_COND_NOT ! 709: | TME_RECODE_X86_COND_C) ! 710: << 8)), ! 711: chain_thunk, ! 712: (TME_RECODE_CHAIN_INFO_JUMP ! 713: + TME_RECODE_CHAIN_INFO_FAR ! 714: + TME_RECODE_CHAIN_INFO_UNCONDITIONAL)); ! 715: ! 716: /* if this the chain subs for a chain return far conditional: */ ! 717: if (chain_info_call_or_return == TME_RECODE_CHAIN_INFO_RETURN) { ! 718: ! 719: /* throw away the return address to the instructions thunk: */ ! 720: tme_recode_x86_insns_start(ic, thunk_bytes); ! 721: thunk_bytes = _tme_recode_x86_emit_adjust_sp(thunk_bytes, ! 722: TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8)); ! 723: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 724: } ! 725: ! 726: /* the next chain subs is for a chain call far unconditional or a ! 727: chain return far unconditional: */ ! 728: TME_RECODE_X86_CHAIN_SUBS(chain_thunk, ! 729: (chain_info_call_or_return ! 730: + TME_RECODE_CHAIN_INFO_FAR ! 731: + TME_RECODE_CHAIN_INFO_UNCONDITIONAL)) ! 732: = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next); ! 733: ! 734: /* if this is the chain subs for a chain call far unconditional: */ ! 735: if (chain_info_call_or_return == TME_RECODE_CHAIN_INFO_CALL) { ! 736: ! 737: /* push the chain return address stack: */ ! 738: _tme_recode_x86_chain_ras_push(ic); ! 739: ! 740: /* the next chain subs is for both a chain jump far conditional ! 741: and a chain jump far unconditional: */ ! 742: TME_RECODE_X86_CHAIN_SUBS(chain_thunk, ! 743: (TME_RECODE_CHAIN_INFO_JUMP ! 744: + TME_RECODE_CHAIN_INFO_FAR ! 745: + TME_RECODE_CHAIN_INFO_CONDITIONAL)) ! 746: = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next); ! 747: TME_RECODE_X86_CHAIN_SUBS(chain_thunk, ! 748: (TME_RECODE_CHAIN_INFO_JUMP ! 749: + TME_RECODE_CHAIN_INFO_FAR ! 750: + TME_RECODE_CHAIN_INFO_UNCONDITIONAL)) ! 751: = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next); ! 752: ! 753: /* fix up the jmp to the chain subs for a chain jump far ! 754: unconditional: */ ! 755: _tme_recode_x86_fixup_jmp(thunk_bytes_jmp_to_jump_far_unconditional, ! 756: ic->tme_recode_ic_thunk_build_next); ! 757: } ! 758: ! 759: /* load the pointer to the token for the current instruction TLB ! 760: entry: */ ! 761: _tme_recode_x86_chain_itlb_current_token_load(ic); ! 762: ! 763: /* get the TLB type for the address type: */ ! 764: tme_recode_address_type_tlb_type(ic, ! 765: &chain->tme_recode_chain_address_type, ! 766: &x86_tlb_type.tme_recode_tlb_type); ! 767: ! 768: /* unbusy the token for the current instruction TLB entry: */ ! 769: #if TME_RECODE_X86_REG_CHAIN_ITLB_CURRENT_TOKEN != TME_RECODE_X86_REG_TLB ! 770: #error "TME_RECODE_X86_REG_CHAIN_ITLB_CURRENT_TOKEN changed" ! 771: #endif ! 772: _tme_recode_x86_tlb_unbusy(ic, 0); ! 773: ! 774: /* load the PC to chain to: */ ! 775: reginfo = ic->tme_recode_ic_reginfo[chain->tme_recode_chain_reg_guest]; ! 776: reginfo.tme_recode_reginfo_tags_ruses ! 777: = (TME_RECODE_REGINFO_TAGS_REG_HOST(_tme_recode_x86_tlb_reg_host_address(ic)) ! 778: + TME_RECODE_REGINFO_TAGS_VALID_SIZE(ic->tme_recode_ic_reg_size) ! 779: + TME_RECODE_REGINFO_TAGS_CLEAN); ! 780: ic->tme_recode_x86_ic_thunks_reg_guest_window_c = TME_RECODE_REG_GUEST_WINDOW_UNDEF; ! 781: tme_recode_host_reg_move(ic, ! 782: chain->tme_recode_chain_reg_guest, ! 783: reginfo.tme_recode_reginfo_all); ! 784: ic->tme_recode_x86_ic_thunks_reg_guest_window_c = TME_RECODE_REG_GUEST_WINDOW_UNDEF; ! 785: ! 786: /* find, busy, and check an instruction TLB entry: */ ! 787: _tme_recode_x86_tlb_busy(ic, ! 788: &chain->tme_recode_chain_address_type, ! 789: &x86_tlb_type); ! 790: ! 791: /* store the pointer to the current recode instruction TLB entry: */ ! 792: _tme_recode_x86_chain_itlb_current_store(ic, &x86_tlb_type); ! 793: ! 794: /* if this is the chain subs for a chain jump far conditional and ! 795: a chain jump far unconditional: */ ! 796: if (chain_info_call_or_return == TME_RECODE_CHAIN_INFO_CALL) { ! 797: ! 798: /* the next chain subs is for both a chain jump near conditional ! 799: and a chain jump near unconditional: */ ! 800: TME_RECODE_X86_CHAIN_SUBS(chain_thunk, ! 801: (TME_RECODE_CHAIN_INFO_JUMP ! 802: + TME_RECODE_CHAIN_INFO_NEAR ! 803: + TME_RECODE_CHAIN_INFO_CONDITIONAL)) ! 804: = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next); ! 805: TME_RECODE_X86_CHAIN_SUBS(chain_thunk, ! 806: (TME_RECODE_CHAIN_INFO_JUMP ! 807: + TME_RECODE_CHAIN_INFO_NEAR ! 808: + TME_RECODE_CHAIN_INFO_UNCONDITIONAL)) ! 809: = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next); ! 810: } ! 811: ! 812: /* start more instructions: */ ! 813: tme_recode_x86_insns_start(ic, thunk_bytes); ! 814: ! 815: /* subtract one from the guest chain counter: */ ! 816: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP1_Ib_Ev; ! 817: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA_DISP32(TME_RECODE_X86_REG_IC), ! 818: TME_RECODE_X86_OPCODE_GRP1_BINOP(TME_RECODE_X86_OPCODE_BINOP_SUB)); ! 819: *((tme_int32_t *) &thunk_bytes[2]) = ic->tme_recode_ic_chain_counter_offset; ! 820: thunk_bytes[2 + sizeof(tme_int32_t)] = 1; ! 821: thunk_bytes += 2 + sizeof(tme_int32_t) + 1; ! 822: ! 823: /* if this is the chain subs for a chain jump near conditional and a ! 824: chain jump near unconditional: */ ! 825: if (chain_info_call_or_return == TME_RECODE_CHAIN_INFO_CALL) { ! 826: ! 827: /* load the return address to the instructions thunk into the ! 828: fixup address register: */ ! 829: /* NB: this is always a 32-bit instruction, even on an x86-64 ! 830: host, because we end up converting the return address into an ! 831: instructions thunk offset, which is only 32 bits: */ ! 832: assert (sizeof(tme_recode_thunk_off_t) == sizeof(tme_int32_t)); ! 833: rex = TME_RECODE_X86_REX_B(TME_RECODE_SIZE_32, TME_RECODE_X86_REG_CHAIN_FIXUP_ADDRESS); ! 834: if (rex != 0) { ! 835: *(thunk_bytes++) = rex; ! 836: } ! 837: thunk_bytes[0] = TME_RECODE_X86_OPCODE_BINOP_MOV + TME_RECODE_X86_OPCODE_BINOP_Ev_Gv; ! 838: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA(TME_RECODE_X86_EA_BASE_SIB), ! 839: TME_RECODE_X86_REG_CHAIN_FIXUP_ADDRESS); ! 840: thunk_bytes[2] = TME_RECODE_X86_SIB(TME_RECODE_X86_REG_SP, TME_RECODE_X86_SIB_INDEX_NONE, 1); ! 841: thunk_bytes += 3; ! 842: } ! 843: ! 844: /* otherwise, this is the chain subs for a return far unconditional: */ ! 845: else { ! 846: ! 847: /* load the chain return address stack pointer: */ ! 848: thunk_bytes[0] = (TME_RECODE_X86_OPCODE_BINOP_MOV + TME_RECODE_X86_OPCODE_BINOP_Ev_Gv); ! 849: thunk_bytes = _tme_recode_x86_emit_ic_modrm(thunk_bytes + 1, ! 850: ic->tme_recode_ic_chain_ras_pointer_offset, ! 851: TME_RECODE_X86_REG_CHAIN_RAS_POINTER); ! 852: } ! 853: ! 854: /* if needed, copy the guest instructions source address into the ! 855: correct register: */ ! 856: reg_x86_address ! 857: = tme_recode_x86_reg_from_host[_tme_recode_x86_tlb_reg_host_address(ic)]; ! 858: if (reg_x86_address != TME_RECODE_X86_REG_CHAIN_GUEST_SRC) { ! 859: _tme_recode_x86_emit_reg_copy(thunk_bytes, reg_x86_address, TME_RECODE_X86_REG_CHAIN_GUEST_SRC); ! 860: } ! 861: ! 862: /* if the guest chain counter was zero or one, jmp to the code that ! 863: handles a guest chain counter underflow: */ ! 864: thunk_bytes_jcc_chain_counter = thunk_bytes; ! 865: thunk_bytes ! 866: = _tme_recode_x86_emit_jmp(thunk_bytes, ! 867: TME_RECODE_X86_OPCODE_JCC(TME_RECODE_X86_COND_BE), ! 868: (tme_uint8_t *) NULL); ! 869: ! 870: /* finish these instructions: */ ! 871: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 872: ! 873: /* if this is the chain subs for a chain jump near conditional and a ! 874: chain jump near unconditional: */ ! 875: if (chain_info_call_or_return == TME_RECODE_CHAIN_INFO_CALL) { ! 876: ! 877: /* define CF with the recode jump flag. NB that the recode flags ! 878: are now above the return address to the instructions thunk: */ ! 879: _tme_recode_x86_conds_testc(ic, TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8) + TME_RECODE_FLAG_JUMP); ! 880: ! 881: /* emit a return back to the instructions thunk: */ ! 882: tme_recode_x86_insns_start(ic, thunk_bytes); ! 883: thunk_bytes[0] = TME_RECODE_X86_OPCODE_RET; ! 884: thunk_bytes++; ! 885: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 886: ! 887: /* the next chain subs is for a chain call near conditional: */ ! 888: TME_RECODE_X86_CHAIN_SUBS(chain_thunk, ! 889: (TME_RECODE_CHAIN_INFO_CALL ! 890: + TME_RECODE_CHAIN_INFO_NEAR ! 891: + TME_RECODE_CHAIN_INFO_CONDITIONAL)) ! 892: = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next); ! 893: ! 894: /* if the recode jump flag is false, jmp to the chain subs for a ! 895: chain jump near conditional: */ ! 896: _tme_recode_x86_chain_subs_jmp(ic, ! 897: TME_RECODE_X86_OPCODE_JCC(TME_RECODE_X86_COND_NOT ! 898: | TME_RECODE_X86_COND_C), ! 899: chain_thunk, ! 900: (TME_RECODE_CHAIN_INFO_JUMP ! 901: + TME_RECODE_CHAIN_INFO_NEAR ! 902: + TME_RECODE_CHAIN_INFO_CONDITIONAL)); ! 903: ! 904: /* the next chain subs is for a chain call near unconditional: */ ! 905: TME_RECODE_X86_CHAIN_SUBS(chain_thunk, ! 906: (TME_RECODE_CHAIN_INFO_CALL ! 907: + TME_RECODE_CHAIN_INFO_NEAR ! 908: + TME_RECODE_CHAIN_INFO_UNCONDITIONAL)) ! 909: = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next); ! 910: ! 911: /* push the chain return address stack: */ ! 912: _tme_recode_x86_chain_ras_push(ic); ! 913: ! 914: /* jmp to the chain subs for a chain jump near unconditional: */ ! 915: _tme_recode_x86_chain_subs_jmp(ic, ! 916: TME_RECODE_X86_OPCODE_JMP_RELb, ! 917: chain_thunk, ! 918: (TME_RECODE_CHAIN_INFO_JUMP ! 919: + TME_RECODE_CHAIN_INFO_NEAR ! 920: + TME_RECODE_CHAIN_INFO_UNCONDITIONAL)); ! 921: } ! 922: ! 923: /* otherwise, this is the chain subs for a chain return far ! 924: unconditional: */ ! 925: else { ! 926: ! 927: /* load the return address: */ ! 928: _tme_recode_x86_chain_ras_ls(ic, FALSE); ! 929: ! 930: /* add one to the return address stack pointer, and store it: */ ! 931: _tme_recode_x86_chain_ras_pointer_update(ic, 1); ! 932: ! 933: /* start more instructions: */ ! 934: tme_recode_x86_insns_start(ic, thunk_bytes); ! 935: ! 936: /* if this is an x86-64 host: */ ! 937: if (TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32) { ! 938: ! 939: /* emit an %ip-relative lea instruction, to load the address of ! 940: thunk offset zero into the TLB scratch register: */ ! 941: thunk_bytes[0] = TME_RECODE_X86_REX_R(TME_RECODE_SIZE_HOST, TME_RECODE_X86_REG_TLB_SCRATCH); ! 942: thunk_bytes[1] = TME_RECODE_X86_OPCODE_LEA; ! 943: thunk_bytes[2] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA(TME_RECODE_X86_EA_BASE_IP), ! 944: TME_RECODE_X86_REG(TME_RECODE_X86_REG_TLB_SCRATCH)); ! 945: thunk_bytes += 3 + sizeof(tme_int32_t); ! 946: ((tme_int32_t *) thunk_bytes)[-1] ! 947: = (0 - (tme_int32_t) tme_recode_build_to_thunk_off(ic, thunk_bytes)); ! 948: ! 949: /* add the address of thunk offset zero into the return address register: */ ! 950: _tme_recode_x86_emit_reg_binop(thunk_bytes, ! 951: TME_RECODE_X86_OPCODE_BINOP_ADD, ! 952: TME_RECODE_X86_REG_TLB_SCRATCH, ! 953: TME_RECODE_X86_REG_CHAIN_RETURN_ADDRESS); ! 954: } ! 955: ! 956: /* emit the indirect jmp instruction with the return address ! 957: register: */ ! 958: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP5; ! 959: thunk_bytes[1] ! 960: = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(TME_RECODE_X86_REG_CHAIN_RETURN_ADDRESS), ! 961: TME_RECODE_X86_OPCODE_GRP5_JMP); ! 962: thunk_bytes += 2; ! 963: ! 964: /* unknown indirect jmps are predicted to fallthrough; placing a UD2 ! 965: instruction after an indirect jmp can stop a processor from ! 966: speculatively executing garbage fallthrough instructions: */ ! 967: thunk_bytes[0] = TME_RECODE_X86_OPCODE_ESC_0F; ! 968: thunk_bytes[1] = TME_RECODE_X86_OPCODE0F_UD2; ! 969: thunk_bytes += 2; ! 970: ! 971: /* finish these instructions: */ ! 972: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 973: } ! 974: ! 975: /* this is the code that handles an instruction TLB miss: */ ! 976: ! 977: /* fix up the assist jcc(s) in the instruction TLB busy: */ ! 978: if (x86_tlb_type.tme_recode_x86_tlb_type_assist_jmp_address_ok != NULL) { ! 979: _tme_recode_x86_fixup_jmp(x86_tlb_type.tme_recode_x86_tlb_type_assist_jmp_address_ok, ! 980: ic->tme_recode_ic_thunk_build_next); ! 981: } ! 982: _tme_recode_x86_fixup_jmp(x86_tlb_type.tme_recode_x86_tlb_type_assist_jmp, ! 983: ic->tme_recode_ic_thunk_build_next); ! 984: ! 985: /* store the pointer to the current recode instruction TLB entry: */ ! 986: /* NB: this is necessary because the instruction TLB that missed is ! 987: still busy: */ ! 988: _tme_recode_x86_chain_itlb_current_store(ic, &x86_tlb_type); ! 989: ! 990: /* start more instructions: */ ! 991: tme_recode_x86_insns_start(ic, thunk_bytes); ! 992: ! 993: /* this is the code that jmps to the chain epilogue: */ ! 994: thunk_bytes_jmp_to_epilogue = thunk_bytes; ! 995: ! 996: /* if this is not for the chain return far subs: */ ! 997: if (chain_info_call_or_return != TME_RECODE_CHAIN_INFO_RETURN) { ! 998: ! 999: /* throw away the return address to the instructions thunk: */ ! 1000: thunk_bytes = _tme_recode_x86_emit_adjust_sp(thunk_bytes, ! 1001: TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8)); ! 1002: } ! 1003: ! 1004: /* jmp to the chain epilogue: */ ! 1005: thunk_bytes ! 1006: = _tme_recode_x86_emit_jmp(thunk_bytes, ! 1007: TME_RECODE_X86_OPCODE_JMP_RELz, ! 1008: tme_recode_thunk_off_to_pointer(ic, ! 1009: ic->tme_recode_x86_ic_chain_epilogue, ! 1010: tme_uint8_t *)); ! 1011: ! 1012: /* this is the code that handles a guest chain counter underflow: */ ! 1013: _tme_recode_x86_fixup_jmp(thunk_bytes_jcc_chain_counter, thunk_bytes); ! 1014: ! 1015: /* clear the guest chain counter, since we may have wrapped it past ! 1016: zero: */ ! 1017: thunk_bytes[0] = TME_RECODE_X86_OPCODE_MOV_Iz_Ev; ! 1018: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA_DISP32(TME_RECODE_X86_REG_IC), ! 1019: 0 /* undefined */); ! 1020: *((tme_int32_t *) &thunk_bytes[2]) = ic->tme_recode_ic_chain_counter_offset; ! 1021: *((tme_uint32_t *) &thunk_bytes[6]) = 0; ! 1022: thunk_bytes += 2 + sizeof(tme_int32_t) + sizeof(tme_uint32_t); ! 1023: ! 1024: /* jmp to the code that jmps to the chain epilogue: */ ! 1025: thunk_bytes ! 1026: = _tme_recode_x86_emit_jmp(thunk_bytes, ! 1027: TME_RECODE_X86_OPCODE_JMP_RELb, ! 1028: thunk_bytes_jmp_to_epilogue); ! 1029: ! 1030: /* finish these instructions: */ ! 1031: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 1032: } ! 1033: ! 1034: /* this emits instructions for a chain out: */ ! 1035: static void ! 1036: _tme_recode_x86_chain_out(struct tme_recode_ic *ic, ! 1037: const struct tme_recode_insns_group *insns_group) ! 1038: { ! 1039: tme_uint8_t *thunk_bytes; ! 1040: tme_uint8_t *thunk_bytes_start; ! 1041: tme_uint32_t chain_info; ! 1042: ! 1043: /* the chain subs, and a chain call instruction itself, destroy the ! 1044: c register, so it won't hold the base offset of a guest register ! 1045: window any more: */ ! 1046: ic->tme_recode_x86_ic_thunks_reg_guest_window_c = TME_RECODE_REG_GUEST_WINDOW_UNDEF; ! 1047: ! 1048: /* start more instructions: */ ! 1049: tme_recode_x86_insns_start(ic, thunk_bytes); ! 1050: ! 1051: /* remember where these instructions started: */ ! 1052: thunk_bytes_start = thunk_bytes; ! 1053: ! 1054: /* get the chain information: */ ! 1055: chain_info = insns_group->tme_recode_insns_group_chain_info; ! 1056: ! 1057: /* exactly one of jump, call, and return may be set: */ ! 1058: assert ((chain_info ! 1059: & (TME_RECODE_CHAIN_INFO_JUMP ! 1060: | TME_RECODE_CHAIN_INFO_CALL ! 1061: | TME_RECODE_CHAIN_INFO_RETURN)) != 0 ! 1062: && ((chain_info ! 1063: & (TME_RECODE_CHAIN_INFO_JUMP ! 1064: | TME_RECODE_CHAIN_INFO_CALL ! 1065: | TME_RECODE_CHAIN_INFO_RETURN)) ! 1066: & ((chain_info ! 1067: & (TME_RECODE_CHAIN_INFO_JUMP ! 1068: | TME_RECODE_CHAIN_INFO_CALL ! 1069: | TME_RECODE_CHAIN_INFO_RETURN)) ! 1070: - 1)) == 0); ! 1071: ! 1072: /* there is no such thing as a chain return near: */ ! 1073: assert ((chain_info & TME_RECODE_CHAIN_INFO_RETURN) == 0 ! 1074: || (chain_info ! 1075: & (TME_RECODE_CHAIN_INFO_NEAR ! 1076: | TME_RECODE_CHAIN_INFO_FAR)) != TME_RECODE_CHAIN_INFO_NEAR); ! 1077: ! 1078: /* if this is a chain conditional, and the alternate target is far, ! 1079: force the main target to be far, since the alternate target needs ! 1080: the information generated by a chain far subs: */ ! 1081: assert (((chain_info ! 1082: & (TME_RECODE_CHAIN_INFO_ALTERNATE_NEAR ! 1083: | TME_RECODE_CHAIN_INFO_ALTERNATE_FAR)) ! 1084: == TME_RECODE_CHAIN_INFO_ALTERNATE_NEAR) ! 1085: || ((chain_info ! 1086: & (TME_RECODE_CHAIN_INFO_UNCONDITIONAL ! 1087: | TME_RECODE_CHAIN_INFO_CONDITIONAL)) ! 1088: != TME_RECODE_CHAIN_INFO_UNCONDITIONAL)); ! 1089: #if TME_RECODE_CHAIN_INFO_FAR == 0 || TME_RECODE_CHAIN_INFO_ALTERNATE_FAR <= TME_RECODE_CHAIN_INFO_FAR ! 1090: #error "TME_RECODE_CHAIN_INFO_ values changed" ! 1091: #endif ! 1092: chain_info ! 1093: |= ((chain_info ! 1094: / (TME_RECODE_CHAIN_INFO_ALTERNATE_FAR ! 1095: / TME_RECODE_CHAIN_INFO_FAR)) ! 1096: & TME_RECODE_CHAIN_INFO_FAR); ! 1097: ! 1098: /* if this is a chain call: */ ! 1099: if (chain_info & TME_RECODE_CHAIN_INFO_CALL) { ! 1100: ! 1101: /* load the return address register with a value that indicates ! 1102: the chain epilogue. this will later get fixed up to indicate ! 1103: the instructions thunk to return to: */ ! 1104: thunk_bytes[0] = TME_RECODE_X86_OPCODE_MOV_Iv_Gv(TME_RECODE_X86_REG_CHAIN_RETURN_ADDRESS); ! 1105: *((tme_uint32_t *) (thunk_bytes + 1)) ! 1106: = TME_RECODE_X86_CHAIN_RETURN_ADDRESS(ic, ic->tme_recode_x86_ic_chain_epilogue); ! 1107: thunk_bytes += 1 + sizeof(tme_uint32_t); ! 1108: } ! 1109: ! 1110: /* if this is a chain return unconditional, jump to the chain subs, ! 1111: otherwise call the chain subs: */ ! 1112: thunk_bytes[0] ! 1113: = (((chain_info ! 1114: & (TME_RECODE_CHAIN_INFO_RETURN ! 1115: + (TME_RECODE_CHAIN_INFO_UNCONDITIONAL ! 1116: | TME_RECODE_CHAIN_INFO_CONDITIONAL))) ! 1117: == (TME_RECODE_CHAIN_INFO_RETURN ! 1118: + TME_RECODE_CHAIN_INFO_UNCONDITIONAL)) ! 1119: ? TME_RECODE_X86_OPCODE_JMP_RELz ! 1120: : TME_RECODE_X86_OPCODE_CALL_RELz); ! 1121: thunk_bytes += 1 + sizeof(tme_int32_t); ! 1122: ((tme_int32_t *) thunk_bytes)[-1] ! 1123: = (TME_RECODE_X86_CHAIN_SUBS(insns_group->tme_recode_insns_group_chain_thunk, ! 1124: chain_info) ! 1125: - tme_recode_build_to_thunk_off(ic, thunk_bytes)); ! 1126: ! 1127: /* for the regular target, emit the unconditional jmp instruction, ! 1128: or the conditional jc instruction, to the chain fixup target: */ ! 1129: if ((chain_info ! 1130: & (TME_RECODE_CHAIN_INFO_UNCONDITIONAL ! 1131: | TME_RECODE_CHAIN_INFO_CONDITIONAL)) ! 1132: == TME_RECODE_CHAIN_INFO_UNCONDITIONAL) { ! 1133: thunk_bytes[0] = TME_RECODE_X86_OPCODE_JMP_RELz; ! 1134: } ! 1135: else { ! 1136: *((tme_uint16_t *) thunk_bytes) ! 1137: = (TME_RECODE_X86_OPCODE_ESC_0F ! 1138: + (TME_RECODE_X86_OPCODE0F_JCC(TME_RECODE_X86_COND_C) ! 1139: << 8)); ! 1140: thunk_bytes++; ! 1141: } ! 1142: thunk_bytes += 1 + sizeof(tme_int32_t); ! 1143: ((tme_int32_t *) thunk_bytes)[-1] ! 1144: = (TME_RECODE_X86_CHAIN_FIXUP_TARGET(ic, chain_info) ! 1145: - tme_recode_build_to_thunk_off(ic, thunk_bytes)); ! 1146: ! 1147: /* if this is a chain conditional instruction: */ ! 1148: if ((chain_info ! 1149: & (TME_RECODE_CHAIN_INFO_UNCONDITIONAL ! 1150: | TME_RECODE_CHAIN_INFO_CONDITIONAL)) ! 1151: != TME_RECODE_CHAIN_INFO_UNCONDITIONAL) { ! 1152: ! 1153: /* for the alternate target, emit the unconditional jmp ! 1154: instruction to the chain fixup target alternate: */ ! 1155: thunk_bytes[0] = TME_RECODE_X86_OPCODE_JMP_RELz; ! 1156: thunk_bytes += 1 + sizeof(tme_int32_t); ! 1157: ((tme_int32_t *) thunk_bytes)[-1] ! 1158: = (TME_RECODE_X86_CHAIN_FIXUP_TARGET_ALTERNATE(ic, chain_info) ! 1159: - tme_recode_build_to_thunk_off(ic, thunk_bytes)); ! 1160: } ! 1161: ! 1162: /* check the size of the chain out: */ ! 1163: assert ((thunk_bytes - thunk_bytes_start) ! 1164: <= TME_RECODE_X86_CHAIN_OUT_SIZE_MAX); ! 1165: ! 1166: /* finish these instructions: */ ! 1167: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 1168: } ! 1169: ! 1170: /* if the given address is reachable with an effective address using ! 1171: an address in another register, this returns the signed 32-bit ! 1172: displacement, otherwise this returns zero: */ ! 1173: static inline tme_int32_t ! 1174: _tme_recode_x86_chain_base_disp32(const tme_shared tme_uint8_t *base, ! 1175: const tme_shared tme_uint8_t *pointer) ! 1176: { ! 1177: signed long disp; ! 1178: ! 1179: disp = pointer - base; ! 1180: return (disp == (tme_int32_t) disp ! 1181: ? disp ! 1182: : 0); ! 1183: } ! 1184: ! 1185: /* if the given address is reachable with a PC-relative effective ! 1186: address, this returns the signed 32-bit displacement, otherwise ! 1187: this returns zero: */ ! 1188: static inline tme_int32_t ! 1189: _tme_recode_x86_chain_ip_disp32(const struct tme_recode_ic *ic, ! 1190: const tme_uint8_t *thunk_bytes, ! 1191: const tme_shared tme_uint8_t *pointer) ! 1192: { ! 1193: tme_recode_thunk_off_t thunk_off; ! 1194: const tme_uint8_t *ip; ! 1195: ! 1196: thunk_off = tme_recode_build_to_thunk_off(ic, thunk_bytes); ! 1197: ip = tme_recode_thunk_off_to_pointer(ic, thunk_off, const tme_uint8_t *); ! 1198: return (_tme_recode_x86_chain_base_disp32(ip, pointer)); ! 1199: } ! 1200: ! 1201: /* this emits instructions for a chain in: */ ! 1202: static void ! 1203: _tme_recode_x86_chain_in(struct tme_recode_ic *ic, ! 1204: const struct tme_recode_insns_group *insns_group) ! 1205: { ! 1206: tme_uint8_t *thunk_bytes; ! 1207: tme_uint8_t *thunk_bytes_start; ! 1208: const tme_shared tme_uint8_t *insns_group_src; ! 1209: unsigned int size_insns_group_src; ! 1210: tme_int32_t disp32; ! 1211: unsigned int rex; ! 1212: const tme_shared tme_uint8_t *insns_group_valid_byte; ! 1213: unsigned int size_insns_group_valid_byte; ! 1214: ! 1215: /* start more instructions: */ ! 1216: tme_recode_x86_insns_start(ic, thunk_bytes); ! 1217: ! 1218: /* remember where these instructions started: */ ! 1219: thunk_bytes_start = thunk_bytes; ! 1220: ! 1221: /* get the guest instructions source address: */ ! 1222: insns_group_src = insns_group->tme_recode_insns_group_src; ! 1223: ! 1224: /* if this is an ia32 host, or if this is an x86-64 host and the ! 1225: guest instructions source address fits in a sign-extended 32 ! 1226: bits: */ ! 1227: if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32 ! 1228: || (((tme_int32_t) ! 1229: (signed long) ! 1230: insns_group_src) ! 1231: == (signed long) insns_group_src)) { ! 1232: ! 1233: /* emit a cmpl $imm32, %reg or cmpq $imm32, %reg: */ ! 1234: if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32) { ! 1235: thunk_bytes[0] = (tme_uint8_t) TME_RECODE_X86_INSN_CMP_Iz_REG_CHAIN_GUEST_SRC; ! 1236: } ! 1237: else { ! 1238: *((tme_uint16_t *) thunk_bytes) = TME_RECODE_X86_INSN_CMP_Iz_REG_CHAIN_GUEST_SRC; ! 1239: } ! 1240: *((tme_int32_t *) &thunk_bytes[(TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32) + 1]) ! 1241: = (signed long) insns_group_src; ! 1242: thunk_bytes += (TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32) + 1 + sizeof(tme_int32_t); ! 1243: ! 1244: /* assume that this is an x86-64 host, and set the size of the ! 1245: instructions that check the guest instruction source address ! 1246: matches the instructions thunk: */ ! 1247: size_insns_group_src = TME_RECODE_X86_64_CHAIN_IN_SIZE_CMPQ_IMM32_JNZ; ! 1248: } ! 1249: ! 1250: /* otherwise, this is an x86-64 host and the guest instructions ! 1251: source address doesn't fit in a sign-extended 32 bits: */ ! 1252: else { ! 1253: ! 1254: /* if the guest instructions source address fits in 32 bits: */ ! 1255: if (TME_RECODE_SIZE_FITS((unsigned long) insns_group_src, ! 1256: TME_RECODE_SIZE_32)) { ! 1257: ! 1258: /* emit a movl $imm32, %reg: */ ! 1259: thunk_bytes[0] = TME_RECODE_X86_INSN_MOVL_IMM32_REG_TLB_SCRATCH; ! 1260: *((tme_uint32_t *) &thunk_bytes[1]) = (unsigned long) insns_group_src; ! 1261: thunk_bytes += 1 + sizeof(tme_uint32_t); ! 1262: ! 1263: /* set the size of the instructions that check the guest ! 1264: instruction source address matches the instructions thunk: */ ! 1265: size_insns_group_src = TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVL_CMP_JNZ; ! 1266: } ! 1267: ! 1268: /* otherwise, if the guest instructions source address is within a ! 1269: signed 32-bit displacement of the %ip of an lea ! 1270: instruction: */ ! 1271: else if ((disp32 ! 1272: = _tme_recode_x86_chain_ip_disp32(ic, ! 1273: (thunk_bytes ! 1274: + 1 /* rex */ ! 1275: + 1 /* lea */ ! 1276: + 1 /* modR/M */ ! 1277: + sizeof(disp32)), ! 1278: insns_group_src))) { ! 1279: ! 1280: /* emit an lea disp32(%ip), %reg: */ ! 1281: thunk_bytes[0] = (tme_uint8_t) TME_RECODE_X86_INSN_LEA_DISP32_IP_REG_TLB_SCRATCH; ! 1282: *((tme_uint16_t *) &thunk_bytes[1]) = (TME_RECODE_X86_INSN_LEA_DISP32_IP_REG_TLB_SCRATCH >> 8); ! 1283: *((tme_int32_t *) &thunk_bytes[3]) = disp32; ! 1284: thunk_bytes += 1 + 2 + sizeof(disp32); ! 1285: ! 1286: /* set the size of the instructions that check the guest ! 1287: instruction source address matches the instructions thunk: */ ! 1288: size_insns_group_src = TME_RECODE_X86_64_CHAIN_IN_SIZE_LEA_IP_CMP_JNZ; ! 1289: } ! 1290: ! 1291: /* otherwise, the guest instructions source address can't be ! 1292: generated: */ ! 1293: else { ! 1294: ! 1295: /* emit a movq $imm64, %reg: */ ! 1296: *((tme_uint16_t *) thunk_bytes) = TME_RECODE_X86_INSN_MOVQ_IMM64_REG_TLB_SCRATCH; ! 1297: *((unsigned long *) &thunk_bytes[2]) = (unsigned long) insns_group_src; ! 1298: thunk_bytes += 1 + 1 + TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8); ! 1299: ! 1300: /* set the size of the instructions that check the guest ! 1301: instruction source address matches the instructions thunk: */ ! 1302: size_insns_group_src = TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVQ_CMP_JNZ; ! 1303: } ! 1304: ! 1305: /* emit a cmpq %reg, %reg: */ ! 1306: rex = (TME_RECODE_X86_REX_B(TME_RECODE_SIZE_HOST, ! 1307: TME_RECODE_X86_REG_CHAIN_GUEST_SRC) ! 1308: | TME_RECODE_X86_REX_R(TME_RECODE_SIZE_HOST, ! 1309: TME_RECODE_X86_REG_TLB_SCRATCH)); ! 1310: assert (rex != 0); ! 1311: thunk_bytes[0] = rex; ! 1312: *((tme_uint16_t *) &thunk_bytes[1]) ! 1313: = ((TME_RECODE_X86_OPCODE_BINOP_CMP ! 1314: + TME_RECODE_X86_OPCODE_BINOP_Ev_Gv) ! 1315: + (TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(TME_RECODE_X86_REG_CHAIN_GUEST_SRC), ! 1316: TME_RECODE_X86_REG(TME_RECODE_X86_REG_TLB_SCRATCH)) ! 1317: << 8)); ! 1318: thunk_bytes += 3; ! 1319: } ! 1320: ! 1321: /* if the guest instructions source address doesn't match this ! 1322: instructions thunk, jump to the chain epilogue: */ ! 1323: *((tme_uint16_t *) thunk_bytes) ! 1324: = (TME_RECODE_X86_OPCODE_ESC_0F ! 1325: + (TME_RECODE_X86_OPCODE0F_JCC(TME_RECODE_X86_COND_NOT | TME_RECODE_X86_COND_Z) ! 1326: << 8)); ! 1327: thunk_bytes += 2 + sizeof(tme_int32_t); ! 1328: ((tme_int32_t *) thunk_bytes)[-1] ! 1329: = (ic->tme_recode_x86_ic_chain_epilogue ! 1330: - tme_recode_build_to_thunk_off(ic, thunk_bytes)); ! 1331: ! 1332: /* get the guest instructions valid byte address: */ ! 1333: insns_group_valid_byte = insns_group->tme_recode_insns_group_valid_byte; ! 1334: ! 1335: /* if this is an ia32 host: */ ! 1336: if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32) { ! 1337: ! 1338: /* emit the opcode and addr32 for a testb $imm8, addr32: */ ! 1339: *((tme_uint16_t *) thunk_bytes) ! 1340: = (TME_RECODE_X86_OPCODE_GRP3_Eb ! 1341: + (TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA(TME_RECODE_X86_EA_BASE_NONE), ! 1342: TME_RECODE_X86_OPCODE_GRP3_TEST) ! 1343: << 8)); ! 1344: *((tme_uint32_t *) &thunk_bytes[2]) = (unsigned long) insns_group_valid_byte; ! 1345: thunk_bytes += 2 + sizeof(tme_uint32_t); ! 1346: ! 1347: /* silence uninitialized variable warnings: */ ! 1348: size_insns_group_valid_byte = 0; ! 1349: } ! 1350: ! 1351: /* otherwise, this is an x86-64 host: */ ! 1352: else { ! 1353: ! 1354: /* assume that we can emit a testb followed by a jz, and emit the ! 1355: first opcode byte for the testb: */ ! 1356: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP3_Eb; ! 1357: thunk_bytes++; ! 1358: ! 1359: /* if the guest instructions valid byte address fits in a ! 1360: sign-extended 32 bits: */ ! 1361: if (((tme_int32_t) ! 1362: (signed long) ! 1363: insns_group_valid_byte) ! 1364: == (signed long) insns_group_valid_byte) { ! 1365: ! 1366: /* emit the modR/M, SIB, and disp32 for a testb $imm8, addr32: */ ! 1367: *((tme_uint16_t *) thunk_bytes) = TME_RECODE_X86_INSN_TESTB_IMM8_ADDR32 >> 8; ! 1368: *((tme_int32_t *) &thunk_bytes[2]) = (signed long) insns_group_valid_byte; ! 1369: thunk_bytes += 2 + sizeof(tme_int32_t); ! 1370: ! 1371: /* set the size of the instructions that check that the guest ! 1372: instructions are still valid: */ ! 1373: size_insns_group_valid_byte = TME_RECODE_X86_64_CHAIN_IN_SIZE_TESTB_ADDR32_JZ; ! 1374: } ! 1375: ! 1376: /* otherwise, if the guest instructions valid byte address is ! 1377: within a signed 32-bit displacement of the %ip of the testb ! 1378: instruction: */ ! 1379: else if ((disp32 ! 1380: = _tme_recode_x86_chain_ip_disp32(ic, ! 1381: (thunk_bytes ! 1382: + 1 /* testb */ ! 1383: + 1 /* modR/M */ ! 1384: + sizeof(disp32) ! 1385: + 1 /* imm8 */), ! 1386: insns_group_valid_byte))) { ! 1387: ! 1388: /* emit the modR/M and disp32 for a testb $imm8, disp32(%ip): */ ! 1389: thunk_bytes[0] = TME_RECODE_X86_INSN_TESTB_IMM8_DISP32_IP >> 8; ! 1390: *((tme_int32_t *) &thunk_bytes[1]) = disp32; ! 1391: thunk_bytes += 1 + sizeof(disp32); ! 1392: ! 1393: /* set the size of the instructions that check that the guest ! 1394: instructions are still valid: */ ! 1395: size_insns_group_valid_byte = TME_RECODE_X86_64_CHAIN_IN_SIZE_TESTB_BASE_JZ; ! 1396: } ! 1397: ! 1398: /* otherwise, if the guest instructions valid byte address is ! 1399: within a signed 32-bit displacement of the guest instructions ! 1400: source address: */ ! 1401: else if ((disp32 ! 1402: = _tme_recode_x86_chain_base_disp32(insns_group_src, ! 1403: insns_group_valid_byte))) { ! 1404: ! 1405: /* emit the modR/M and disp32 for a testb $imm8, disp32(%reg): */ ! 1406: thunk_bytes[0] = TME_RECODE_X86_INSN_TESTB_IMM8_DISP32_REG_CHAIN_GUEST_SRC >> 8; ! 1407: *((tme_int32_t *) &thunk_bytes[1]) = disp32; ! 1408: thunk_bytes += 1 + sizeof(disp32); ! 1409: ! 1410: /* set the size of the instructions that check that the guest ! 1411: instructions are still valid: */ ! 1412: size_insns_group_valid_byte = TME_RECODE_X86_64_CHAIN_IN_SIZE_TESTB_BASE_JZ; ! 1413: } ! 1414: ! 1415: /* otherwise, the guest instructions valid byte address can't be ! 1416: generated: */ ! 1417: else { ! 1418: ! 1419: /* emit a movq $imm64, %reg: */ ! 1420: /* NB: thunk_bytes has already been advanced by one: */ ! 1421: *((tme_uint16_t *) &thunk_bytes[0 - 1]) = TME_RECODE_X86_INSN_MOVQ_IMM64_REG_TLB_SCRATCH; ! 1422: *((unsigned long *) &thunk_bytes[2 - 1]) = (unsigned long) insns_group_valid_byte; ! 1423: thunk_bytes += -1 + 1 + 1 + TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8); ! 1424: ! 1425: /* emit the opcode and modR/M for a testb $imm8, (%reg): */ ! 1426: *((tme_uint16_t *) thunk_bytes) ! 1427: = (TME_RECODE_X86_OPCODE_GRP3_Eb ! 1428: + (TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA(TME_RECODE_X86_REG_TLB_SCRATCH), ! 1429: TME_RECODE_X86_OPCODE_GRP3_TEST) ! 1430: << 8)); ! 1431: thunk_bytes += 2; ! 1432: ! 1433: /* set the size of the instructions that check that the guest ! 1434: instructions are still valid: */ ! 1435: size_insns_group_valid_byte = TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVQ_TESTB_JZ; ! 1436: } ! 1437: } ! 1438: ! 1439: /* emit the $imm8, and if the guest instructions aren't valid, jump ! 1440: to the chain epilogue: */ ! 1441: thunk_bytes[0] = insns_group->tme_recode_insns_group_valid_mask; ! 1442: *((tme_uint16_t *) &thunk_bytes[1]) ! 1443: = (TME_RECODE_X86_OPCODE_ESC_0F ! 1444: + (TME_RECODE_X86_OPCODE0F_JCC(TME_RECODE_X86_COND_Z) ! 1445: << 8)); ! 1446: thunk_bytes += 1 + 2 + sizeof(tme_int32_t); ! 1447: ((tme_int32_t *) thunk_bytes)[-1] ! 1448: = (ic->tme_recode_x86_ic_chain_epilogue ! 1449: - tme_recode_build_to_thunk_off(ic, thunk_bytes)); ! 1450: ! 1451: /* check the size of the chain in: */ ! 1452: assert ((thunk_bytes - thunk_bytes_start) ! 1453: <= TME_RECODE_X86_CHAIN_IN_SIZE_MAX); ! 1454: ! 1455: /* check the sizes of the instructions that we need to skip for a ! 1456: chain in near: */ ! 1457: assert ((thunk_bytes - thunk_bytes_start) ! 1458: == (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32 ! 1459: ? TME_RECODE_IA32_CHAIN_IN_SIZE_FAR ! 1460: : (size_insns_group_src ! 1461: + size_insns_group_valid_byte))); ! 1462: ! 1463: /* finish these instructions: */ ! 1464: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 1465: } ! 1466: ! 1467: /* this makes the chain prologue for a new IC: */ ! 1468: static void ! 1469: _tme_recode_x86_chain_prologue(struct tme_recode_ic *ic, ! 1470: const struct tme_recode_chain *chain, ! 1471: struct tme_recode_chain_thunk *chain_thunk) ! 1472: { ! 1473: tme_uint8_t *thunk_bytes; ! 1474: unsigned int reg_x86_insns_thunk; ! 1475: unsigned long thunk_address0; ! 1476: ! 1477: /* start more instructions: */ ! 1478: tme_recode_x86_insns_start(ic, thunk_bytes); ! 1479: ! 1480: /* set the chain prologue: */ ! 1481: chain_thunk->tme_recode_x86_chain_thunk_prologue ! 1482: = tme_recode_thunk_off_to_pointer(ic, ! 1483: tme_recode_build_to_thunk_off(ic, thunk_bytes), ! 1484: void (*) _TME_P((struct tme_ic *, tme_recode_thunk_off_t))); ! 1485: ! 1486: /* push all callee-saved registers: */ ! 1487: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_BP); ! 1488: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_B); ! 1489: if (TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32) { ! 1490: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_N(12)); ! 1491: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_N(13)); ! 1492: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_N(14)); ! 1493: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_N(15)); ! 1494: } ! 1495: else { ! 1496: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_SI); ! 1497: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_DI); ! 1498: } ! 1499: ! 1500: /* the instructions thunk offset and address will be in the si register: */ ! 1501: reg_x86_insns_thunk = TME_RECODE_X86_REG_SI; ! 1502: ! 1503: /* if this is an x86-64 host: */ ! 1504: if (TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32) { ! 1505: ! 1506: /* copy the struct tme_ic * argument into the ic register: */ ! 1507: _tme_recode_x86_emit_reg_copy(thunk_bytes, TME_RECODE_X86_REG_DI, TME_RECODE_X86_REG_IC); ! 1508: ! 1509: /* the instructions thunk offset is already in the second argument register: */ ! 1510: assert (reg_x86_insns_thunk == TME_RECODE_X86_REG_SI); ! 1511: } ! 1512: ! 1513: /* otherwise, this is an ia32 host: */ ! 1514: else { ! 1515: ! 1516: /* load the struct tme_ic * argument from the stack: */ ! 1517: /* NB: the magic 5 below is for the four callee-saved registers ! 1518: that we pushed above, plus the return address: */ ! 1519: thunk_bytes[0] = (TME_RECODE_X86_OPCODE_BINOP_MOV + TME_RECODE_X86_OPCODE_BINOP_Ev_Gv); ! 1520: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA_DISP8(TME_RECODE_X86_EA_BASE_SIB), ! 1521: TME_RECODE_X86_REG_IC); ! 1522: thunk_bytes[2] = TME_RECODE_X86_SIB(TME_RECODE_X86_REG_SP, TME_RECODE_X86_SIB_INDEX_NONE, 1); ! 1523: thunk_bytes[3] = (TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8) * 5); ! 1524: thunk_bytes += 4; ! 1525: ! 1526: /* load the instructions thunk offset from the stack: */ ! 1527: /* NB: the magic 6 below is for the four callee-saved registers ! 1528: that we pushed above, plus the return address, plus the struct ! 1529: tme_ic * argument: */ ! 1530: thunk_bytes[0] = (TME_RECODE_X86_OPCODE_BINOP_MOV + TME_RECODE_X86_OPCODE_BINOP_Ev_Gv); ! 1531: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA_DISP8(TME_RECODE_X86_EA_BASE_SIB), ! 1532: reg_x86_insns_thunk); ! 1533: thunk_bytes[2] = TME_RECODE_X86_SIB(TME_RECODE_X86_REG_SP, TME_RECODE_X86_SIB_INDEX_NONE, 1); ! 1534: thunk_bytes[3] = (TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8) * 6); ! 1535: thunk_bytes += 4; ! 1536: } ! 1537: ! 1538: /* get the near address of the thunk at offset zero: */ ! 1539: thunk_address0 = tme_recode_thunk_off_to_pointer(ic, 0, char *) - (char *) 0; ! 1540: ! 1541: /* if this is an ia32 host, or if this is an x86-64 host and the ! 1542: near address of the thunk at offset zero fits in 32 bits: */ ! 1543: if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32 ! 1544: || thunk_address0 == (tme_uint32_t) thunk_address0) { ! 1545: ! 1546: /* add the near address of the thunk at offset zero to the ! 1547: instructions thunk offset: */ ! 1548: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP1_Iz_Ev; ! 1549: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(reg_x86_insns_thunk), ! 1550: TME_RECODE_X86_OPCODE_GRP1_BINOP(TME_RECODE_X86_OPCODE_BINOP_ADD)); ! 1551: *((tme_uint32_t *) &thunk_bytes[2]) = thunk_address0; ! 1552: thunk_bytes += 2 + sizeof(tme_uint32_t); ! 1553: } ! 1554: ! 1555: /* otherwise, this is an x86-64 host and the near address of the ! 1556: thunk at offset zero doesn't fit in 32 bits: */ ! 1557: else { ! 1558: ! 1559: /* zero-extend the instructions thunk offset to 64 bits: */ ! 1560: thunk_bytes[0] ! 1561: = (TME_RECODE_X86_OPCODE_BINOP_MOV ! 1562: + TME_RECODE_X86_OPCODE_BINOP_Gv_Ev); ! 1563: thunk_bytes[1] ! 1564: = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(reg_x86_insns_thunk), ! 1565: TME_RECODE_X86_REG(reg_x86_insns_thunk)); ! 1566: thunk_bytes += 2; ! 1567: ! 1568: /* load the near address of the thunk at offset zero into the TLB ! 1569: scratch register: */ ! 1570: *((tme_uint16_t *) &thunk_bytes[0]) = TME_RECODE_X86_INSN_MOVQ_IMM64_REG_TLB_SCRATCH; ! 1571: *((unsigned long *) &thunk_bytes[2]) = thunk_address0; ! 1572: thunk_bytes += 2 + TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8); ! 1573: ! 1574: /* add the TLB scratch register to the instructions thunk ! 1575: offset: */ ! 1576: _tme_recode_x86_emit_reg_binop(thunk_bytes, ! 1577: TME_RECODE_X86_OPCODE_BINOP_ADD, ! 1578: TME_RECODE_X86_REG_TLB_SCRATCH, ! 1579: reg_x86_insns_thunk); ! 1580: } ! 1581: ! 1582: /* allocate one word on the stack for the recode flag bytes. NB ! 1583: that on x86-64, this returns the stack pointer to 16-byte ! 1584: alignment as required by the ABI: */ ! 1585: thunk_bytes ! 1586: = _tme_recode_x86_emit_adjust_sp(thunk_bytes, ! 1587: (0 - TME_BIT(TME_RECODE_SIZE_HOST ! 1588: - TME_RECODE_SIZE_8))); ! 1589: ! 1590: /* call the chain jump far unconditional subs: */ ! 1591: thunk_bytes[0] = TME_RECODE_X86_OPCODE_CALL_RELz; ! 1592: thunk_bytes += 1 + sizeof(tme_int32_t); ! 1593: ((tme_int32_t *) thunk_bytes)[-1] ! 1594: = (TME_RECODE_X86_CHAIN_SUBS(chain_thunk, ! 1595: (TME_RECODE_CHAIN_INFO_JUMP ! 1596: + TME_RECODE_CHAIN_INFO_FAR ! 1597: + TME_RECODE_CHAIN_INFO_UNCONDITIONAL)) ! 1598: - tme_recode_build_to_thunk_off(ic, thunk_bytes)); ! 1599: ! 1600: /* do the indirect jmp into the instructions thunk: */ ! 1601: assert (TME_RECODE_X86_REX_B(0, reg_x86_insns_thunk) == 0); ! 1602: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP5; ! 1603: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(reg_x86_insns_thunk), ! 1604: TME_RECODE_X86_OPCODE_GRP5_JMP); ! 1605: thunk_bytes += 2; ! 1606: ! 1607: /* unknown indirect jmps are predicted to fallthrough; placing a UD2 ! 1608: instruction after an indirect jmp can stop a processor from ! 1609: speculatively executing garbage fallthrough instructions: */ ! 1610: thunk_bytes[0] = TME_RECODE_X86_OPCODE_ESC_0F; ! 1611: thunk_bytes[1] = TME_RECODE_X86_OPCODE0F_UD2; ! 1612: thunk_bytes += 2; ! 1613: ! 1614: /* finish these instructions: */ ! 1615: tme_recode_x86_insns_finish(ic, thunk_bytes); ! 1616: ! 1617: /* finish the chain prologue: */ ! 1618: tme_recode_host_thunk_finish(ic); ! 1619: } ! 1620: ! 1621: /* this returns a chain thunk: */ ! 1622: const struct tme_recode_chain_thunk * ! 1623: tme_recode_chain_thunk(struct tme_recode_ic *ic, ! 1624: const struct tme_recode_chain *chain) ! 1625: { ! 1626: struct tme_recode_chain_thunk *chain_thunk; ! 1627: ! 1628: /* if the chain fixup targets haven't been made yet: */ ! 1629: if (TME_RECODE_X86_CHAIN_FIXUP_TARGET_ALTERNATE(ic, ! 1630: (TME_RECODE_CHAIN_INFO_JUMP ! 1631: + TME_RECODE_CHAIN_INFO_ALTERNATE_NEAR)) ! 1632: == 0) { ! 1633: ! 1634: /* make the chain fixup targets: */ ! 1635: _tme_recode_x86_chain_fixup_targets(ic); ! 1636: } ! 1637: ! 1638: /* start the thunk for the chain subs: */ ! 1639: if (!tme_recode_host_thunk_start(ic)) { ! 1640: abort(); ! 1641: } ! 1642: chain_thunk = tme_new0(struct tme_recode_chain_thunk, 1); ! 1643: ! 1644: /* make the chain subs for chain calls and chain jumps: */ ! 1645: _tme_recode_x86_chain_subs(ic, ! 1646: chain, ! 1647: chain_thunk, ! 1648: TME_RECODE_CHAIN_INFO_CALL); ! 1649: ! 1650: /* make the chain subs for chain returns: */ ! 1651: _tme_recode_x86_chain_subs(ic, ! 1652: chain, ! 1653: chain_thunk, ! 1654: TME_RECODE_CHAIN_INFO_RETURN); ! 1655: ! 1656: /* finish the thunk for the chain subs: */ ! 1657: tme_recode_host_thunk_finish(ic); ! 1658: ! 1659: /* start the thunk for the chain prologue: */ ! 1660: if (!tme_recode_host_thunk_start(ic)) { ! 1661: abort(); ! 1662: } ! 1663: ! 1664: /* make the chain prologue: */ ! 1665: _tme_recode_x86_chain_prologue(ic, ! 1666: chain, ! 1667: chain_thunk); ! 1668: ! 1669: /* finish the thunk for the chain prologue: */ ! 1670: tme_recode_host_thunk_finish(ic); ! 1671: ! 1672: return (chain_thunk); ! 1673: } ! 1674: ! 1675: /* this clears the return address stack: */ ! 1676: void ! 1677: tme_recode_chain_ras_clear(const struct tme_recode_ic *recode_ic, ! 1678: struct tme_ic *ic) ! 1679: { ! 1680: tme_recode_ras_entry_t *_ras_entry; ! 1681: tme_uint32_t ras_size; ! 1682: tme_recode_ras_entry_t ras_entry; ! 1683: ! 1684: /* clear the return address stack: */ ! 1685: ras_entry ! 1686: = TME_RECODE_X86_CHAIN_RETURN_ADDRESS(recode_ic, ! 1687: recode_ic->tme_recode_x86_ic_chain_epilogue); ! 1688: _ras_entry ! 1689: = ((tme_recode_ras_entry_t *) ! 1690: (((tme_uint8_t *) ic) ! 1691: + recode_ic->tme_recode_ic_chain_ras_offset)); ! 1692: ras_size = recode_ic->tme_recode_ic_chain_ras_size; ! 1693: do { ! 1694: _ras_entry[ras_size - 1] = ras_entry; ! 1695: } while (--ras_size); ! 1696: ! 1697: /* make sure that the return address stack pointer is valid: */ ! 1698: assert (*((tme_uint32_t *) ! 1699: (((tme_uint8_t *) ic) ! 1700: + recode_ic->tme_recode_ic_chain_ras_pointer_offset)) ! 1701: < recode_ic->tme_recode_ic_chain_ras_size); ! 1702: } ! 1703: ! 1704: /* this fixes up a chain: */ ! 1705: tme_recode_thunk_off_t ! 1706: tme_recode_chain_fixup(struct tme_recode_ic *ic, ! 1707: tme_recode_thunk_off_t chain_fixup, ! 1708: tme_uint32_t chain_info, ! 1709: tme_recode_thunk_off_t insns_thunk_next, ! 1710: tme_recode_thunk_off_t insns_thunk_return) ! 1711: { ! 1712: tme_uint32_t return_imm32; ! 1713: tme_uint16_t insn_0_15_buffer; ! 1714: tme_uint32_t insn_0_15; ! 1715: const tme_uint8_t *thunk_bytes; ! 1716: tme_uint8_t opcode_buffer; ! 1717: tme_int32_t displacement; ! 1718: ! 1719: /* if this is a chain call: */ ! 1720: if (chain_info & TME_RECODE_CHAIN_INFO_CALL) { ! 1721: ! 1722: /* there must be a return instructions thunk: */ ! 1723: assert (insns_thunk_return != 0); ! 1724: ! 1725: /* a chain call emits: ! 1726: ! 1727: movl $imm32, %TME_RECODE_X86_REG_CHAIN_RETURN_ADDRESS ! 1728: call chain_subs_call_{near|far}_{unconditional|conditional} ! 1729: jmp/jnc chain_fixup_call_{near|far} ! 1730: ! 1731: on an ia32 host, the $imm32 is the absolute address of the ! 1732: instructions thunk to chain to. on an x86-64 host, the $imm32 ! 1733: is the offset of the instructions thunk to chain to. on both ! 1734: hosts, before fixup, the $imm32 indicates the chain epilogue. ! 1735: fix up the $imm32: */ ! 1736: /* NB: chain_fixup points to the jump instruction. the call ! 1737: instruction is a call rel32, which is five bytes long, and the ! 1738: $imm32 is immediately before that: */ ! 1739: return_imm32 = TME_RECODE_X86_CHAIN_RETURN_ADDRESS(ic, insns_thunk_return); ! 1740: tme_recode_thunk_off_write(ic, ! 1741: (chain_fixup ! 1742: - (5 ! 1743: + sizeof(tme_uint32_t))), ! 1744: tme_uint32_t, ! 1745: return_imm32); ! 1746: } ! 1747: ! 1748: /* if this is a chain near: */ ! 1749: if ((chain_info ! 1750: & (TME_RECODE_CHAIN_INFO_NEAR ! 1751: | TME_RECODE_CHAIN_INFO_FAR)) ! 1752: == TME_RECODE_CHAIN_INFO_NEAR) { ! 1753: ! 1754: /* if this is an ia32 host: */ ! 1755: if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32) { ! 1756: ! 1757: /* advance the next instructions thunk offset past the chain ! 1758: in far checks: */ ! 1759: insns_thunk_next += TME_RECODE_IA32_CHAIN_IN_SIZE_FAR; ! 1760: } ! 1761: ! 1762: /* otherwise, this is an x86-64 host: */ ! 1763: else { ! 1764: ! 1765: /* read the first two bytes of the first instruction of the ! 1766: chain in. this instruction starts checking that the guest ! 1767: instruction source address matches the instructions thunk: */ ! 1768: insn_0_15 = *tme_recode_thunk_off_read(ic, insns_thunk_next, tme_uint16_t, insn_0_15_buffer); ! 1769: ! 1770: /* if this instruction is a cmpq $imm32, %reg: */ ! 1771: if (insn_0_15 == (tme_uint16_t) TME_RECODE_X86_INSN_CMP_Iz_REG_CHAIN_GUEST_SRC) { ! 1772: ! 1773: /* advance the next instructions thunk offset past the cmpq ! 1774: $imm32, %reg; jnz epilogue */ ! 1775: insns_thunk_next += TME_RECODE_X86_64_CHAIN_IN_SIZE_CMPQ_IMM32_JNZ; ! 1776: } ! 1777: ! 1778: /* if this instruction is an lea disp32(%ip), %reg: */ ! 1779: if (insn_0_15 == (tme_uint16_t) TME_RECODE_X86_INSN_LEA_DISP32_IP_REG_TLB_SCRATCH) { ! 1780: ! 1781: /* advance the next instructions thunk offset past the lea ! 1782: disp32(%ip), %reg ; cmpq %reg, %reg ; jnz epilogue */ ! 1783: insns_thunk_next += TME_RECODE_X86_64_CHAIN_IN_SIZE_LEA_IP_CMP_JNZ; ! 1784: } ! 1785: ! 1786: /* if this instruction is a movq $imm64, %reg: */ ! 1787: if (insn_0_15 == (tme_uint16_t) TME_RECODE_X86_INSN_MOVQ_IMM64_REG_TLB_SCRATCH) { ! 1788: ! 1789: /* advance the next instructions thunk offset past the movq ! 1790: $imm64, %reg ; cmpq %reg, %reg ; jnz epilogue */ ! 1791: insns_thunk_next += TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVQ_CMP_JNZ; ! 1792: } ! 1793: ! 1794: /* if this instruction is a movl $imm32, %reg: */ ! 1795: if (((tme_uint8_t) insn_0_15) == TME_RECODE_X86_INSN_MOVL_IMM32_REG_TLB_SCRATCH) { ! 1796: ! 1797: /* advance the next instructions thunk offset past the movl ! 1798: $imm32, %reg ; cmpq %reg, %reg ; jnz epilogue */ ! 1799: insns_thunk_next += TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVL_CMP_JNZ; ! 1800: } ! 1801: ! 1802: /* read the first two bytes of the next instruction of the chain ! 1803: in. this instruction starts checking that the guest ! 1804: instructions are still valid: */ ! 1805: insn_0_15 = *tme_recode_thunk_off_read(ic, insns_thunk_next, tme_uint16_t, insn_0_15_buffer); ! 1806: ! 1807: /* if this instruction is a testb $imm8, addr32: */ ! 1808: if (insn_0_15 == (tme_uint16_t) TME_RECODE_X86_INSN_TESTB_IMM8_ADDR32) { ! 1809: ! 1810: /* advance the next instructions thunk offset past the testb ! 1811: $imm8, addr32 ; jz epilogue */ ! 1812: insns_thunk_next += TME_RECODE_X86_64_CHAIN_IN_SIZE_TESTB_ADDR32_JZ; ! 1813: } ! 1814: ! 1815: /* if this instruction is a testb $imm8, disp32(%ip): */ ! 1816: if (insn_0_15 == (tme_uint16_t) TME_RECODE_X86_INSN_TESTB_IMM8_DISP32_IP) { ! 1817: ! 1818: /* advance the next instructions thunk offset past the testb ! 1819: $imm8, disp32(%ip) ; jz epilogue */ ! 1820: insns_thunk_next += TME_RECODE_X86_64_CHAIN_IN_SIZE_TESTB_BASE_JZ; ! 1821: } ! 1822: ! 1823: /* if this instruction is a testb $imm8, disp32(%reg): */ ! 1824: if (insn_0_15 == (tme_uint16_t) TME_RECODE_X86_INSN_TESTB_IMM8_DISP32_REG_CHAIN_GUEST_SRC) { ! 1825: ! 1826: /* advance the next instructions thunk offset past the testb ! 1827: $imm8, disp32(%reg) ; jz epilogue */ ! 1828: insns_thunk_next += TME_RECODE_X86_64_CHAIN_IN_SIZE_TESTB_BASE_JZ; ! 1829: } ! 1830: ! 1831: /* if this instruction is a movq $imm64, %reg: */ ! 1832: if (insn_0_15 == TME_RECODE_X86_INSN_MOVQ_IMM64_REG_TLB_SCRATCH) { ! 1833: ! 1834: /* advance the next instructions thunk offset past the movq ! 1835: $imm64, %reg ; testb $imm8, (%reg) ; jz epilogue */ ! 1836: insns_thunk_next += TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVQ_TESTB_JZ; ! 1837: } ! 1838: } ! 1839: } ! 1840: ! 1841: /* all chain fixup instructions are unconditional or conditional ! 1842: jump instructions with 32-bit displacements at their ends. ! 1843: unconditional jump instructions are five bytes, and conditional ! 1844: jump instructions are six bytes because they are escaped: */ ! 1845: ! 1846: /* read the first opcode byte of the jump instruction: */ ! 1847: thunk_bytes = tme_recode_thunk_off_read(ic, chain_fixup, tme_uint8_t, opcode_buffer); ! 1848: ! 1849: /* advance the chain fixup offset to the jump instruction displacement: */ ! 1850: chain_fixup += (*thunk_bytes == TME_RECODE_X86_OPCODE_ESC_0F) + 1; ! 1851: ! 1852: /* rewrite the displacement for the instructions thunk: */ ! 1853: displacement = insns_thunk_next - (chain_fixup + sizeof(tme_int32_t)); ! 1854: tme_recode_thunk_off_write(ic, chain_fixup, tme_int32_t, displacement); ! 1855: ! 1856: /* return the next instructions thunk offset: */ ! 1857: return (insns_thunk_next); ! 1858: } ! 1859: ! 1860: #ifdef TME_RECODE_DEBUG ! 1861: #include <stdio.h> ! 1862: ! 1863: /* this host function dumps a chain thunk: */ ! 1864: void ! 1865: tme_recode_chain_thunk_dump(const struct tme_recode_ic *ic, ! 1866: const struct tme_recode_chain_thunk *chain_thunk) ! 1867: { ! 1868: tme_uint32_t chain_info; ! 1869: const char *s; ! 1870: tme_recode_thunk_off_t insns_thunk; ! 1871: ! 1872: printf(" x86 chain prologue: x/10i %p\n", ! 1873: chain_thunk->tme_recode_x86_chain_thunk_prologue); ! 1874: for (chain_info = 0; ! 1875: chain_info <= (TME_RECODE_CHAIN_INFO_UNCONDITIONAL ! 1876: | TME_RECODE_CHAIN_INFO_CONDITIONAL ! 1877: | TME_RECODE_CHAIN_INFO_NEAR ! 1878: | TME_RECODE_CHAIN_INFO_FAR ! 1879: | TME_RECODE_CHAIN_INFO_JUMP ! 1880: | TME_RECODE_CHAIN_INFO_RETURN ! 1881: | TME_RECODE_CHAIN_INFO_CALL); ! 1882: chain_info++) { ! 1883: switch (chain_info ! 1884: & (TME_RECODE_CHAIN_INFO_JUMP ! 1885: | TME_RECODE_CHAIN_INFO_RETURN ! 1886: | TME_RECODE_CHAIN_INFO_CALL)) { ! 1887: case TME_RECODE_CHAIN_INFO_JUMP: s = "jump"; break; ! 1888: case TME_RECODE_CHAIN_INFO_RETURN: ! 1889: s = "return"; ! 1890: if ((chain_info & TME_RECODE_CHAIN_INFO_FAR) == 0) { ! 1891: continue; ! 1892: } ! 1893: break; ! 1894: case TME_RECODE_CHAIN_INFO_CALL: s = "call"; break; ! 1895: default: continue; ! 1896: } ! 1897: insns_thunk = TME_RECODE_X86_CHAIN_SUBS(chain_thunk, chain_info); ! 1898: printf(" x86 chain %s %s %s subs: x/10i %p\n", ! 1899: s, ! 1900: (chain_info & TME_RECODE_CHAIN_INFO_FAR ! 1901: ? "far" ! 1902: : "near"), ! 1903: (chain_info & TME_RECODE_CHAIN_INFO_CONDITIONAL ! 1904: ? "conditional" ! 1905: : "unconditional"), ! 1906: tme_recode_thunk_off_to_pointer(ic, ! 1907: insns_thunk, ! 1908: void (*)(void))); ! 1909: } ! 1910: insns_thunk ! 1911: = TME_RECODE_X86_CHAIN_FIXUP_TARGET_ALTERNATE(ic, ! 1912: (TME_RECODE_CHAIN_INFO_JUMP ! 1913: + TME_RECODE_CHAIN_INFO_ALTERNATE_NEAR)); ! 1914: printf(" x86 chain fixup chain jump alternate near: x/10i %p\n", ! 1915: tme_recode_thunk_off_to_pointer(ic, ! 1916: insns_thunk, ! 1917: void (*)(void))); ! 1918: insns_thunk ! 1919: = TME_RECODE_X86_CHAIN_FIXUP_TARGET(ic, ! 1920: (TME_RECODE_CHAIN_INFO_JUMP ! 1921: + TME_RECODE_CHAIN_INFO_NEAR)); ! 1922: printf(" x86 chain fixup chain jump near, chain return alternate near: x/10i %p\n", ! 1923: tme_recode_thunk_off_to_pointer(ic, ! 1924: insns_thunk, ! 1925: void (*)(void))); ! 1926: insns_thunk ! 1927: = TME_RECODE_X86_CHAIN_FIXUP_TARGET_ALTERNATE(ic, ! 1928: (TME_RECODE_CHAIN_INFO_JUMP ! 1929: + TME_RECODE_CHAIN_INFO_ALTERNATE_FAR)); ! 1930: printf(" x86 chain fixup chain jump alternate far: x/3i %p\n", ! 1931: tme_recode_thunk_off_to_pointer(ic, ! 1932: insns_thunk, ! 1933: void (*)(void))); ! 1934: insns_thunk ! 1935: = TME_RECODE_X86_CHAIN_FIXUP_TARGET(ic, ! 1936: (TME_RECODE_CHAIN_INFO_JUMP ! 1937: + TME_RECODE_CHAIN_INFO_FAR)); ! 1938: printf(" x86 chain fixup chain jump far: x/2i %p\n", ! 1939: tme_recode_thunk_off_to_pointer(ic, ! 1940: insns_thunk, ! 1941: void (*)(void))); ! 1942: insns_thunk ! 1943: = TME_RECODE_X86_CHAIN_FIXUP_TARGET(ic, ! 1944: (TME_RECODE_CHAIN_INFO_CALL ! 1945: + TME_RECODE_CHAIN_INFO_FAR)); ! 1946: printf(" x86 chain fixup chain call far: x/2i %p\n", ! 1947: tme_recode_thunk_off_to_pointer(ic, ! 1948: insns_thunk, ! 1949: void (*)(void))); ! 1950: insns_thunk ! 1951: = TME_RECODE_X86_CHAIN_FIXUP_TARGET(ic, ! 1952: (TME_RECODE_CHAIN_INFO_CALL ! 1953: + TME_RECODE_CHAIN_INFO_NEAR)); ! 1954: printf(" x86 chain fixup chain call near: x/2i %p\n", ! 1955: tme_recode_thunk_off_to_pointer(ic, ! 1956: insns_thunk, ! 1957: void (*)(void))); ! 1958: } ! 1959: ! 1960: #endif /* TME_RECODE_DEBUG */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.