![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to rc-x86-chain.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [The Machine Emulator] / tme / libtme / host / x86|
Return to rc-x86-chain.c CVS log | Up to [The Machine Emulator] / tme / libtme / host / x86 |
1.1 root 1: /* $Id: rc-x86-chain.c,v 1.3 2010/02/15 22:15:59 fredette Exp $ */
2:
3: /* libtme/host/x86/rc-x86-chain.c - x86 host recode chain support: */
4:
5: /*
6: * Copyright (c) 2009 Matt Fredette
7: * All rights reserved.
8: *
9: * Redistribution and use in source and binary forms, with or without
10: * modification, are permitted provided that the following conditions
11: * are met:
12: * 1. Redistributions of source code must retain the above copyright
13: * notice, this list of conditions and the following disclaimer.
14: * 2. Redistributions in binary form must reproduce the above copyright
15: * notice, this list of conditions and the following disclaimer in the
16: * documentation and/or other materials provided with the distribution.
17: * 3. All advertising materials mentioning features or use of this software
18: * must display the following acknowledgement:
19: * This product includes software developed by Matt Fredette.
20: * 4. The name of the author may not be used to endorse or promote products
21: * derived from this software without specific prior written permission.
22: *
23: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
25: * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
26: * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
27: * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28: * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29: * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31: * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
32: * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33: * POSSIBILITY OF SUCH DAMAGE.
34: */
35:
36: _TME_RCSID("$Id: rc-x86-chain.c,v 1.3 2010/02/15 22:15:59 fredette Exp $");
37:
38: /* macros: */
39:
40: /* the x86 register for the pointer to the token for the current
41: instruction TLB entry: */
42: #define TME_RECODE_X86_REG_CHAIN_ITLB_CURRENT_TOKEN (TME_RECODE_X86_REG_TLB)
43:
44: /* the x86 register for the chain return address: */
45: #define TME_RECODE_X86_REG_CHAIN_RETURN_ADDRESS (TME_RECODE_X86_REG_TLB)
46:
47: /* the x86 register for the chain return address stack pointer: */
48: #define TME_RECODE_X86_REG_CHAIN_RAS_POINTER (TME_RECODE_X86_REG_TLB_SCRATCH)
49:
50: /* the x86 register for the fixup address: */
51: #define TME_RECODE_X86_REG_CHAIN_FIXUP_ADDRESS (TME_RECODE_X86_REG_TLB_SCRATCH)
52:
53: /* the x86 register for the guest instructions source address: */
54: /* NB: we use the a register to take advantage of the fact that there
55: is a special cmp $imm, %a form that will save a byte in each
56: instruction thunk: */
57: #define TME_RECODE_X86_REG_CHAIN_GUEST_SRC (TME_RECODE_X86_REG_A)
58:
59: /* this gives the one or two opcode bytes for a cmp $imm32,
60: %TME_RECODE_X86_REG_CHAIN_GUEST_SRC: */
61: #if TME_RECODE_X86_REG_CHAIN_GUEST_SRC != TME_RECODE_X86_REG_A
62: #error "TME_RECODE_X86_REG_CHAIN_GUEST_SRC changed"
63: #endif
64: #define TME_RECODE_X86_INSN_CMP_Iz_REG_CHAIN_GUEST_SRC \
65: (TME_RECODE_X86_REX_B(TME_RECODE_SIZE_HOST, \
66: TME_RECODE_X86_REG_CHAIN_GUEST_SRC) \
67: + ((TME_RECODE_X86_OPCODE_BINOP_CMP \
68: + TME_RECODE_X86_OPCODE_BINOP_Iz_A) \
69: << (8 * (TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32))))
70:
71: /* this gives the one opcode byte for a movl $imm32, %TME_RECODE_X86_REG_TLB_SCRATCH: */
72: #if TME_RECODE_X86_REG(TME_RECODE_X86_REG_TLB_SCRATCH) != TME_RECODE_X86_REG_TLB_SCRATCH
73: #error "TME_RECODE_X86_REG_TLB_SCRATCH changed"
74: #endif
75: #define TME_RECODE_X86_INSN_MOVL_IMM32_REG_TLB_SCRATCH \
76: TME_RECODE_X86_OPCODE_MOV_Iv_Gv(TME_RECODE_X86_REG_TLB_SCRATCH)
77:
78: /* this gives the three opcode bytes for a lea disp32(%ip),
79: %TME_RECODE_X86_REG_TLB_SCRATCH: */
80: #define TME_RECODE_X86_INSN_LEA_DISP32_IP_REG_TLB_SCRATCH \
81: (TME_RECODE_X86_REX_R(TME_RECODE_SIZE_HOST, \
82: TME_RECODE_X86_REG_TLB_SCRATCH) \
83: + (TME_RECODE_X86_OPCODE_LEA \
84: << 8) \
85: + (TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA(TME_RECODE_X86_EA_BASE_IP),\
86: TME_RECODE_X86_REG(TME_RECODE_X86_REG_TLB_SCRATCH))\
87: << 16))
88:
89: /* this gives the two opcode bytes for a movq $imm64,
90: %TME_RECODE_X86_REG_TLB_SCRATCH: */
91: #define TME_RECODE_X86_INSN_MOVQ_IMM64_REG_TLB_SCRATCH \
92: (TME_RECODE_X86_REX_R(TME_RECODE_SIZE_HOST, \
93: TME_RECODE_X86_REG_TLB_SCRATCH) \
94: + (TME_RECODE_X86_OPCODE_MOV_Iv_Gv(TME_RECODE_X86_REG_TLB_SCRATCH)\
95: << 8))
96:
97: /* this gives the three opcode bytes for a testb $imm8, addr32: */
98: #define TME_RECODE_X86_INSN_TESTB_IMM8_ADDR32 \
99: (TME_RECODE_X86_OPCODE_GRP3_Eb \
100: + (TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA(TME_RECODE_X86_EA_BASE_SIB),\
101: TME_RECODE_X86_OPCODE_GRP3_TEST)\
102: << 8) \
103: + (TME_RECODE_X86_SIB(TME_RECODE_X86_SIB_BASE_NONE, \
104: TME_RECODE_X86_SIB_INDEX_NONE, \
105: 1) \
106: << 16))
107:
108: /* this gives the two opcode bytes for a testb $imm8, disp32(%ip): */
109: #define TME_RECODE_X86_INSN_TESTB_IMM8_DISP32_IP \
110: (TME_RECODE_X86_OPCODE_GRP3_Eb \
111: + (TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA(TME_RECODE_X86_EA_BASE_IP),\
112: TME_RECODE_X86_OPCODE_GRP3_TEST)\
113: << 8))
114:
115: /* this gives the two opcode bytes for a testb $imm8,
116: disp32(%TME_RECODE_X86_REG_CHAIN_GUEST_SRC): */
117: #if TME_RECODE_X86_REG(TME_RECODE_X86_REG_CHAIN_GUEST_SRC) != TME_RECODE_X86_REG_CHAIN_GUEST_SRC
118: #error "TME_RECODE_X86_REG_CHAIN_GUEST_SRC changed"
119: #endif
120: #define TME_RECODE_X86_INSN_TESTB_IMM8_DISP32_REG_CHAIN_GUEST_SRC \
121: (TME_RECODE_X86_OPCODE_GRP3_Eb \
122: + (TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA_DISP32(TME_RECODE_X86_REG_CHAIN_GUEST_SRC),\
123: TME_RECODE_X86_OPCODE_GRP3_TEST)\
124: << 8))
125:
126: /* this is the size of a jcc epilogue instruction: */
127: #define TME_RECODE_X86_CHAIN_IN_SIZE_JCC_EPILOGUE \
128: (6 /* jcc epilogue */)
129:
130: /* this is the size of the chain in far instructions on an ia32 host: */
131: #define TME_RECODE_IA32_CHAIN_IN_SIZE_FAR \
132: (5 /* cmpl $imm32, %eax */ \
133: + TME_RECODE_X86_CHAIN_IN_SIZE_JCC_EPILOGUE \
134: + 7 /* testb $imm8, addr32 */ \
135: + TME_RECODE_X86_CHAIN_IN_SIZE_JCC_EPILOGUE)
136:
137: /* the size of cmpq $imm32, %TME_RECODE_X86_REG_CHAIN_GUEST_SRC ; jnz
138: epilogue */
139: #define TME_RECODE_X86_64_CHAIN_IN_SIZE_CMPQ_IMM32_JNZ \
140: (6 /* cmpq $imm32, %rax */ \
141: + TME_RECODE_X86_CHAIN_IN_SIZE_JCC_EPILOGUE)
142:
143: /* the size of cmpq %reg, %reg ; jnz epilogue */
144: #define TME_RECODE_X86_64_CHAIN_IN_SIZE_CMP_JNZ \
145: (3 /* cmpq %reg, %reg */ \
146: + TME_RECODE_X86_CHAIN_IN_SIZE_JCC_EPILOGUE)
147:
148: /* the size of movl $imm32, %reg ; cmpq %reg, %reg ; jnz epilogue */
149: #define TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVL_CMP_JNZ \
150: (5 /* movl $imm32, %reg */ \
151: + TME_RECODE_X86_64_CHAIN_IN_SIZE_CMP_JNZ)
152:
153: /* the size of lea disp32(%ip), %reg ; cmpq %reg, %reg ; jnz epilogue */
154: #define TME_RECODE_X86_64_CHAIN_IN_SIZE_LEA_IP_CMP_JNZ \
155: (7 /* leaq disp32(%ip), %reg */ \
156: + TME_RECODE_X86_64_CHAIN_IN_SIZE_CMP_JNZ)
157:
158: /* the size of movq $imm64, %reg ; cmpq %reg, %reg ; jnz epilogue */
159: #define TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVQ_CMP_JNZ \
160: (10 /* movq $imm64, %reg */ \
161: + TME_RECODE_X86_64_CHAIN_IN_SIZE_CMP_JNZ)
162:
163: /* the size of testb $imm8, addr32 ; jz epilogue */
164: #define TME_RECODE_X86_64_CHAIN_IN_SIZE_TESTB_ADDR32_JZ \
165: (8 /* testb $imm8, addr32 */ \
166: + TME_RECODE_X86_CHAIN_IN_SIZE_JCC_EPILOGUE)
167:
168: /* the size of testb $imm8, base32(%reg) ; jz epilogue */
169: #define TME_RECODE_X86_64_CHAIN_IN_SIZE_TESTB_BASE_JZ \
170: (7 /* testb $imm8, addr32 */ \
171: + TME_RECODE_X86_CHAIN_IN_SIZE_JCC_EPILOGUE)
172:
173: /* the size of movq $imm64, %reg ; testb $imm32, (%reg) ; jz epilogue */
174: #define TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVQ_TESTB_JZ \
175: (10 /* movq $imm64, %reg */ \
176: + 3 /* testb $imm32, (%reg) */ \
177: + TME_RECODE_X86_CHAIN_IN_SIZE_JCC_EPILOGUE)
178:
179: /* this gives a chain subs: */
180: #if (TME_RECODE_CHAIN_INFO_CONDITIONAL | TME_RECODE_CHAIN_INFO_FAR) != 3
181: #error "TME_RECODE_CHAIN_INFO_ values changed"
182: #endif
183: #if (TME_RECODE_CHAIN_INFO_JUMP != 4 || TME_RECODE_CHAIN_INFO_RETURN != 8)
184: #error "TME_RECODE_CHAIN_INFO_ values changed"
185: #endif
186: #define TME_RECODE_X86_CHAIN_SUBS(chain_thunk, chain_info)\
187: ((chain_thunk)->tme_recode_x86_chain_thunk_subs \
188: [(chain_info) \
189: & (TME_RECODE_CHAIN_INFO_CONDITIONAL \
190: | TME_RECODE_CHAIN_INFO_FAR \
191: | TME_RECODE_CHAIN_INFO_JUMP \
192: | TME_RECODE_CHAIN_INFO_RETURN)])
193:
194: /* these give a chain fixup target for the regular target or alternate
195: target: */
196: #if TME_RECODE_CHAIN_INFO_FAR != 2 || TME_RECODE_CHAIN_INFO_ALTERNATE_FAR == 0
197: #error "TME_RECODE_CHAIN_INFO_ values changed"
198: #endif
199: #define TME_RECODE_X86_CHAIN_FIXUP_TARGET(ic, chain_info) \
200: ((ic)->tme_recode_x86_ic_chain_fixup_target \
201: [((chain_info) & TME_RECODE_CHAIN_INFO_FAR) \
202: + (((chain_info) / TME_RECODE_CHAIN_INFO_CALL) & 1)])
203: #define TME_RECODE_X86_CHAIN_FIXUP_TARGET_ALTERNATE(ic, chain_info) \
204: ((ic)->tme_recode_x86_ic_chain_fixup_target \
205: [4 \
206: + (((chain_info) / TME_RECODE_CHAIN_INFO_ALTERNATE_FAR) & 1)])
207:
208: /* on an ia32 host, an entry on the return address stack is the
209: absolute address of either an instructions thunk or the chain
210: epilogue. on an x86-64 host, an entry on the return address stack
211: is the instructions thunk offset of either an instructions thunk or
212: the chain epilogue: */
213: #if TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32
214: #define TME_RECODE_X86_CHAIN_RETURN_ADDRESS(ic, insns_thunk)\
215: ((tme_recode_ras_entry_t) (insns_thunk))
216: #else /* TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32 */
217: #define TME_RECODE_X86_CHAIN_RETURN_ADDRESS(ic, insns_thunk)\
218: tme_recode_thunk_off_to_pointer(ic, \
219: insns_thunk, \
220: tme_recode_ras_entry_t)
221: #endif /* TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32 */
222:
223: /* this emits instructions for a chain epilogue: */
224: static void
225: _tme_recode_x86_chain_epilogue(struct tme_recode_ic *ic)
226: {
227: tme_uint8_t *thunk_bytes;
228:
229: /* start more instructions: */
230: tme_recode_x86_insns_start(ic, thunk_bytes);
231:
232: /* set the instructions thunk offset of the chain epilogue: */
233: ic->tme_recode_x86_ic_chain_epilogue = tme_recode_build_to_thunk_off(ic, thunk_bytes);
234:
235: /* remove the recode carry flag stack word: */
236: thunk_bytes
237: = _tme_recode_x86_emit_adjust_sp(thunk_bytes,
238: TME_BIT(TME_RECODE_SIZE_HOST
239: - TME_RECODE_SIZE_8));
240:
241: /* pop all callee-saved registers: */
242: if (TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32) {
243: _tme_recode_x86_emit_reg_pop(thunk_bytes, TME_RECODE_X86_REG_N(15));
244: _tme_recode_x86_emit_reg_pop(thunk_bytes, TME_RECODE_X86_REG_N(14));
245: _tme_recode_x86_emit_reg_pop(thunk_bytes, TME_RECODE_X86_REG_N(13));
246: _tme_recode_x86_emit_reg_pop(thunk_bytes, TME_RECODE_X86_REG_N(12));
247: }
248: else {
249: _tme_recode_x86_emit_reg_pop(thunk_bytes, TME_RECODE_X86_REG_DI);
250: _tme_recode_x86_emit_reg_pop(thunk_bytes, TME_RECODE_X86_REG_SI);
251: }
252: _tme_recode_x86_emit_reg_pop(thunk_bytes, TME_RECODE_X86_REG_B);
253: _tme_recode_x86_emit_reg_pop(thunk_bytes, TME_RECODE_X86_REG_BP);
254:
255: /* emit the return: */
256: thunk_bytes[0] = TME_RECODE_X86_OPCODE_RET;
257: thunk_bytes++;
258:
259: /* finish these instructions: */
260: tme_recode_x86_insns_finish(ic, thunk_bytes);
261:
262: /* finish the chain epilogue: */
263: tme_recode_host_thunk_finish(ic);
264: }
265:
266: /* this emits instructions to advance the fixup address register
267: past a chain conditional six-byte jcc instruction: */
268: static tme_uint8_t *
269: _tme_recode_x86_chain_fixup_alternate(tme_uint8_t *thunk_bytes)
270: {
271:
272: /* NB: this is always a 32-bit instruction, even on an x86-64 host,
273: because we end up converting the fixup address into an
274: instructions thunk offset, which is only 32 bits: */
275: assert (sizeof(tme_recode_thunk_off_t) == sizeof(tme_int32_t));
276: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP1_Ib_Ev;
277: thunk_bytes[1]
278: = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(TME_RECODE_X86_REG_CHAIN_FIXUP_ADDRESS),
279: TME_RECODE_X86_OPCODE_GRP1_BINOP(TME_RECODE_X86_OPCODE_BINOP_ADD));
280: thunk_bytes[2] = 2 + sizeof(tme_int32_t);
281: thunk_bytes += 3;
282: return (thunk_bytes);
283: }
284:
285: /* this emits instructions to make a constant for the third argument
286: to the chain fixup function: */
287: static tme_uint8_t *
288: _tme_recode_x86_chain_fixup_arg2(tme_uint8_t *thunk_bytes,
289: tme_uint32_t chain_info)
290: {
291:
292: /* if this is an ia32 host: */
293: if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32) {
294:
295: /* push the argument: */
296: assert (chain_info < 0x80);
297: thunk_bytes[0] = TME_RECODE_X86_OPCODE_PUSH_Ib;
298: thunk_bytes[1] = chain_info;
299: thunk_bytes += 2;
300: }
301:
302: /* otherwise, this is an x86-64 host: */
303: else {
304:
305: /* load the third argument register: */
306: thunk_bytes[0] = TME_RECODE_X86_OPCODE_MOV_Iv_Gv(TME_RECODE_X86_REG_D);
307: *((tme_uint32_t *) &thunk_bytes[1]) = chain_info;
308: thunk_bytes += 1 + sizeof(tme_uint32_t);
309: }
310:
311: return (thunk_bytes);
312: }
313:
314: /* this emits instructions for the chain fixup targets for chain jump
315: far, chain call far, and chain call near: */
316: static void
317: _tme_recode_x86_chain_fixup_target(struct tme_recode_ic *ic,
318: tme_uint32_t chain_info,
319: const tme_uint8_t *thunk_bytes_fixup_call)
320: {
321: tme_uint8_t *thunk_bytes;
322:
323: /* start more instructions: */
324: tme_recode_x86_insns_start(ic, thunk_bytes);
325:
326: /* set next fixup target: */
327: TME_RECODE_X86_CHAIN_FIXUP_TARGET(ic,
328: chain_info)
329: = tme_recode_build_to_thunk_off(ic, thunk_bytes);
330:
331: /* make the third argument for the chain fixup function: */
332: thunk_bytes = _tme_recode_x86_chain_fixup_arg2(thunk_bytes, chain_info);
333:
334: /* jmp to the common chain fixup call point: */
335: thunk_bytes
336: = _tme_recode_x86_emit_jmp(thunk_bytes,
337: TME_RECODE_X86_OPCODE_JMP_RELz,
338: thunk_bytes_fixup_call);
339:
340: /* finish these instructions: */
341: tme_recode_x86_insns_finish(ic, thunk_bytes);
342: }
343:
344: /* this makes the chain fixup targets for a new IC: */
345: static void
346: _tme_recode_x86_chain_fixup_targets(struct tme_recode_ic *ic)
347: {
348: tme_uint8_t *thunk_bytes;
349: const tme_uint8_t *thunk_bytes_fixup_call;
350: unsigned long thunk_address0;
351: unsigned int stack_adjust;
352:
353: /* start more instructions: */
354: tme_recode_x86_insns_start(ic, thunk_bytes);
355:
356: /* the first fixup target is for chain jump alternate near: */
357: TME_RECODE_X86_CHAIN_FIXUP_TARGET_ALTERNATE(ic,
358: (TME_RECODE_CHAIN_INFO_JUMP
359: + TME_RECODE_CHAIN_INFO_ALTERNATE_NEAR))
360: = tme_recode_build_to_thunk_off(ic, thunk_bytes);
361:
362: /* advance the fixup address register past the chain jump
363: conditional six-byte jcc instruction: */
364: thunk_bytes = _tme_recode_x86_chain_fixup_alternate(thunk_bytes);
365:
366: /* the next fixup target is for chain jump near: */
367: /* NB: this fixup target is also for chain return alternate near,
368: which is treated like a chain jump near: */
369: TME_RECODE_X86_CHAIN_FIXUP_TARGET(ic,
370: (TME_RECODE_CHAIN_INFO_JUMP
371: + TME_RECODE_CHAIN_INFO_NEAR))
372: = tme_recode_build_to_thunk_off(ic, thunk_bytes);
373:
374: /* make the third argument for the chain fixup function: */
375: thunk_bytes
376: = _tme_recode_x86_chain_fixup_arg2(thunk_bytes,
377: TME_RECODE_CHAIN_INFO_NEAR);
378:
379: /* this is the common chain fixup call point: */
380: thunk_bytes_fixup_call = thunk_bytes;
381:
382: /* get the near address of the thunk at offset zero: */
383: thunk_address0 = tme_recode_thunk_off_to_pointer(ic, 0, char *) - (char *) 0;
384:
385: /* convert the fixup address (which has been already truncated to 32
386: bits) into the instructions thunk offset: */
387: /* NB: this is always a 32-bit instruction, because a
388: tme_recode_thunk_off_t is a tme_int32_t: */
389: assert (sizeof(tme_recode_thunk_off_t) == sizeof(tme_int32_t));
390: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP1_Iz_Ev;
391: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(TME_RECODE_X86_REG_CHAIN_FIXUP_ADDRESS),
392: TME_RECODE_X86_OPCODE_GRP1_BINOP(TME_RECODE_X86_OPCODE_BINOP_SUB));
393: *((tme_uint32_t *) &thunk_bytes[2]) = thunk_address0;
394: thunk_bytes += 2 + sizeof(tme_uint32_t);
395:
396: /* if this is an ia32 host: */
397: if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32) {
398:
399: /* push the arguments for the chain fixup function: */
400: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_CHAIN_FIXUP_ADDRESS);
401: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_IC);
402:
403: /* we will need to remove the arguments from the stack: */
404: stack_adjust = sizeof(struct tme_ic *) + sizeof(tme_recode_thunk_off_t) + sizeof(tme_uint32_t);
405: }
406:
407: /* otherwise, this is an x86-64 host: */
408: else {
409:
410: /* copy the arguments for the guest jump chain function: */
411: _tme_recode_x86_emit_reg_copy(thunk_bytes, TME_RECODE_X86_REG_IC, TME_RECODE_X86_REG_DI);
412: _tme_recode_x86_emit_reg_copy(thunk_bytes, TME_RECODE_X86_REG_CHAIN_FIXUP_ADDRESS, TME_RECODE_X86_REG_SI);
413:
414: /* we don't need to adjust the stack. NB that the instructions
415: thunk, where the stack pointer is 16-byte aligned, jumped to
416: us, so we don't have to align it for the x86-64 ABI: */
417: stack_adjust = 0;
418: }
419:
420: /* call the chain fixup function: */
421: tme_recode_x86_insns_finish(ic, thunk_bytes);
422: _tme_recode_x86_emit_transfer_func(ic,
423: TME_RECODE_X86_OPCODE_CALL_RELz,
424: ((void (*) _TME_P((void)))
425: ic->tme_recode_ic_chain_fixup));
426: tme_recode_x86_insns_start(ic, thunk_bytes);
427:
428: /* do any stack adjust: */
429: if (stack_adjust) {
430: thunk_bytes = _tme_recode_x86_emit_adjust_sp(thunk_bytes, stack_adjust);
431: }
432:
433: /* jmp to the chain epilogue: */
434: thunk_bytes
435: = _tme_recode_x86_emit_jmp(thunk_bytes,
436: TME_RECODE_X86_OPCODE_JMP_RELz,
437: tme_recode_thunk_off_to_pointer(ic,
438: ic->tme_recode_x86_ic_chain_epilogue,
439: tme_uint8_t *));
440:
441: /* the next fixup target is for chain jump alternate far: */
442: TME_RECODE_X86_CHAIN_FIXUP_TARGET_ALTERNATE(ic,
443: (TME_RECODE_CHAIN_INFO_JUMP
444: + TME_RECODE_CHAIN_INFO_ALTERNATE_FAR))
445: = tme_recode_build_to_thunk_off(ic, thunk_bytes);
446:
447: /* advance the fixup address register past the chain jump
448: conditional six-byte jcc instruction: */
449: thunk_bytes = _tme_recode_x86_chain_fixup_alternate(thunk_bytes);
450:
451: /* finish these instructions: */
452: tme_recode_x86_insns_finish(ic, thunk_bytes);
453:
454: /* the next fixup target is for chain jump far: */
455: _tme_recode_x86_chain_fixup_target(ic,
456: TME_RECODE_CHAIN_INFO_FAR,
457: thunk_bytes_fixup_call);
458:
459: /* the next fixup target is for chain call far: */
460: _tme_recode_x86_chain_fixup_target(ic,
461: (TME_RECODE_CHAIN_INFO_CALL
462: + TME_RECODE_CHAIN_INFO_FAR),
463: thunk_bytes_fixup_call);
464:
465: /* the next fixup target is for chain call near: */
466: _tme_recode_x86_chain_fixup_target(ic,
467: (TME_RECODE_CHAIN_INFO_CALL
468: + TME_RECODE_CHAIN_INFO_NEAR),
469: thunk_bytes_fixup_call);
470:
471: /* finish the fixup targets thunk: */
472: tme_recode_host_thunk_finish(ic);
473: }
474:
475: /* this emits a jmp to a chain subs: */
476: static tme_uint8_t *
477: _tme_recode_x86_chain_subs_jmp(struct tme_recode_ic *ic,
478: tme_uint32_t opcode,
479: struct tme_recode_chain_thunk *chain_thunk,
480: tme_uint32_t chain_info)
481: {
482: tme_uint8_t *thunk_bytes;
483: tme_uint8_t *thunk_bytes_jc;
484: tme_recode_thunk_off_t chain_subs;
485: tme_uint8_t *thunk_bytes_target;
486:
487: /* if this is a conditional jmp: */
488: if (opcode != TME_RECODE_X86_OPCODE_JMP_RELb
489: && opcode != TME_RECODE_X86_OPCODE_JMP_RELz) {
490:
491: /* define CF with the recode jump flag. NB that the recode flags
492: are now above the return address to the instructions thunk: */
493: _tme_recode_x86_conds_testc(ic, TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8) + TME_RECODE_FLAG_JUMP);
494: }
495:
496: /* start more instructions: */
497: tme_recode_x86_insns_start(ic, thunk_bytes);
498:
499: /* emit the jmp: */
500: chain_subs = TME_RECODE_X86_CHAIN_SUBS(chain_thunk, chain_info);
501: if (chain_subs == 0) {
502: thunk_bytes_jc = thunk_bytes;
503: thunk_bytes_target = (tme_uint8_t *) NULL;
504: }
505: else {
506: thunk_bytes_jc = (tme_uint8_t *) NULL;
507: thunk_bytes_target = tme_recode_thunk_off_to_pointer(ic, chain_subs, tme_uint8_t *);
508: }
509: thunk_bytes = _tme_recode_x86_emit_jmp(thunk_bytes, opcode, thunk_bytes_target);
510:
511: /* finish these instructions: */
512: tme_recode_x86_insns_finish(ic, thunk_bytes);
513:
514: return (thunk_bytes_jc);
515: }
516:
517: /* this emits instructions to add a sign-extended byte to the chain
518: return address stack pointer, mask it, and then store it: */
519: static void
520: _tme_recode_x86_chain_ras_pointer_update(struct tme_recode_ic *ic,
521: tme_int8_t addend)
522: {
523: tme_uint8_t *thunk_bytes;
524: tme_uint8_t ras_size;
525:
526: /* start these instructions: */
527: tme_recode_x86_insns_start(ic, thunk_bytes);
528:
529: /* emit the add $imm8: */
530: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP1_Ib_Ev;
531: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(TME_RECODE_X86_REG_CHAIN_RAS_POINTER),
532: TME_RECODE_X86_OPCODE_GRP1_BINOP(TME_RECODE_X86_OPCODE_BINOP_ADD));
533: thunk_bytes[2] = addend;
534: thunk_bytes += 3;
535:
536: /* emit the and $imm8: */
537: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP1_Ib_Ev;
538: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(TME_RECODE_X86_REG_CHAIN_RAS_POINTER),
539: TME_RECODE_X86_OPCODE_GRP1_BINOP(TME_RECODE_X86_OPCODE_BINOP_AND));
540: ras_size = ic->tme_recode_ic_chain_ras_size;
541: assert (ras_size > 0 && (ras_size & (ras_size - 1)) == 0);
542: thunk_bytes[2] = ras_size - 1;
543: thunk_bytes += 3;
544:
545: /* store the chain return address stack pointer: */
546: thunk_bytes[0] = (TME_RECODE_X86_OPCODE_BINOP_MOV + TME_RECODE_X86_OPCODE_BINOP_Gv_Ev);
547: thunk_bytes = _tme_recode_x86_emit_ic_modrm(thunk_bytes + 1,
548: ic->tme_recode_ic_chain_ras_pointer_offset,
549: TME_RECODE_X86_REG_CHAIN_RAS_POINTER);
550:
551: /* finish these instructions: */
552: tme_recode_x86_insns_finish(ic, thunk_bytes);
553: }
554:
555: /* this emits an instruction to load or store an entry on the chain
556: return address stack: */
557: static void
558: _tme_recode_x86_chain_ras_ls(struct tme_recode_ic *ic,
559: int store)
560: {
561: tme_uint8_t *thunk_bytes;
562: tme_int32_t ras_offset;
563:
564: /* start more instructions: */
565: tme_recode_x86_insns_start(ic, thunk_bytes);
566:
567: /* emit the movl: */
568: thunk_bytes[0]
569: = (TME_RECODE_X86_OPCODE_BINOP_MOV
570: + (store
571: ? TME_RECODE_X86_OPCODE_BINOP_Gv_Ev
572: : TME_RECODE_X86_OPCODE_BINOP_Ev_Gv));
573: ras_offset = ic->tme_recode_ic_chain_ras_offset;
574: thunk_bytes[1]
575: = TME_RECODE_X86_MOD_OPREG_RM((ras_offset < 0x80
576: ? TME_RECODE_X86_MOD_RM_EA_DISP8(TME_RECODE_X86_EA_BASE_SIB)
577: : TME_RECODE_X86_MOD_RM_EA_DISP32(TME_RECODE_X86_EA_BASE_SIB)),
578: TME_RECODE_X86_REG(TME_RECODE_X86_REG_CHAIN_RETURN_ADDRESS));
579: thunk_bytes[2]
580: = TME_RECODE_X86_SIB(TME_RECODE_X86_REG_IC,
581: TME_RECODE_X86_REG_CHAIN_RAS_POINTER,
582: sizeof(tme_recode_ras_entry_t));
583: *((tme_uint32_t *) (thunk_bytes + 3)) = ras_offset;
584: thunk_bytes += 3 + (ras_offset < 0x80 ? sizeof(tme_uint8_t) : sizeof(tme_uint32_t));
585:
586: /* finish these instructions: */
587: tme_recode_x86_insns_finish(ic, thunk_bytes);
588: }
589:
590: /* this emits instructions to push the chain return address stack: */
591: static void
592: _tme_recode_x86_chain_ras_push(struct tme_recode_ic *ic)
593: {
594: tme_uint8_t *thunk_bytes;
595:
596: /* start more instructions: */
597: tme_recode_x86_insns_start(ic, thunk_bytes);
598:
599: /* load the return address stack pointer: */
600: thunk_bytes[0] = (TME_RECODE_X86_OPCODE_BINOP_MOV + TME_RECODE_X86_OPCODE_BINOP_Ev_Gv);
601: thunk_bytes = _tme_recode_x86_emit_ic_modrm(thunk_bytes + 1,
602: ic->tme_recode_ic_chain_ras_pointer_offset,
603: TME_RECODE_X86_REG_CHAIN_RAS_POINTER);
604:
605: /* finish these instructions: */
606: tme_recode_x86_insns_finish(ic, thunk_bytes);
607:
608: /* subtract one from the return address stack pointer, and store it: */
609: _tme_recode_x86_chain_ras_pointer_update(ic, -1);
610:
611: /* store the return address: */
612: _tme_recode_x86_chain_ras_ls(ic, TRUE);
613: }
614:
615: /* this emits instructions to load or store the pointer to the token
616: for the current recode instruction TLB entry: */
617: static void
618: _tme_recode_x86_chain_itlb_current_token_ls(struct tme_recode_ic *ic,
619: int store)
620: {
621: tme_uint8_t *thunk_bytes;
622:
623: /* start more instructions: */
624: tme_recode_x86_insns_start(ic, thunk_bytes);
625:
626: /* load or store the pointer to the token for the current recode
627: instruction TLB entry: */
628: if (TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32) {
629: *(thunk_bytes++)
630: = TME_RECODE_X86_REX_B(TME_RECODE_SIZE_HOST,
631: TME_RECODE_X86_REG_CHAIN_ITLB_CURRENT_TOKEN);
632: }
633: thunk_bytes[0]
634: = (TME_RECODE_X86_OPCODE_BINOP_MOV
635: + (store
636: ? TME_RECODE_X86_OPCODE_BINOP_Gv_Ev
637: : TME_RECODE_X86_OPCODE_BINOP_Ev_Gv));
638: thunk_bytes = _tme_recode_x86_emit_ic_modrm(thunk_bytes + 1,
639: ic->tme_recode_ic_itlb_current_token_offset,
640: TME_RECODE_X86_REG_CHAIN_ITLB_CURRENT_TOKEN);
641:
642: /* finish these instructions: */
643: tme_recode_x86_insns_finish(ic, thunk_bytes);
644: }
645: #define _tme_recode_x86_chain_itlb_current_token_load(ic) _tme_recode_x86_chain_itlb_current_token_ls(ic, FALSE)
646:
647: /* this stores the pointer to the current recode instruction TLB entry: */
648: static void
649: _tme_recode_x86_chain_itlb_current_store(struct tme_recode_ic *ic,
650: const struct tme_recode_x86_tlb_type *x86_tlb_type)
651: {
652: tme_uint8_t *thunk_bytes;
653:
654: /* start more instructions: */
655: tme_recode_x86_insns_start(ic, thunk_bytes);
656:
657: /* convert the pointer to the recode instruction TLB entry in
658: TME_RECODE_X86_REG_TLB into a pointer to the token for that entry
659: in TME_RECODE_X86_REG_CHAIN_ITLB_CURRENT_TOKEN: */
660: #if TME_RECODE_X86_REG_CHAIN_ITLB_CURRENT_TOKEN != TME_RECODE_X86_REG_TLB
661: #error "TME_RECODE_X86_REG_CHAIN_ITLB_CURRENT_TOKEN changed"
662: #endif
663: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP1_Ib_Ev;
664: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(TME_RECODE_X86_REG_TLB),
665: TME_RECODE_X86_OPCODE_GRP1_BINOP(TME_RECODE_X86_OPCODE_BINOP_ADD));
666: assert (x86_tlb_type->tme_recode_tlb_type.tme_recode_tlb_type_offset_token < 0x80);
667: thunk_bytes[2] = x86_tlb_type->tme_recode_tlb_type.tme_recode_tlb_type_offset_token;
668: thunk_bytes += 3;
669:
670: /* finish these instructions: */
671: tme_recode_x86_insns_finish(ic, thunk_bytes);
672:
673: /* store the pointer to the token for the current recode instruction
674: TLB entry: */
675: _tme_recode_x86_chain_itlb_current_token_ls(ic, TRUE);
676: }
677:
678: /* this emits instructions for the call and jump chain subs, or for
679: the return chain subs: */
680: static void
681: _tme_recode_x86_chain_subs(struct tme_recode_ic *ic,
682: const struct tme_recode_chain *chain,
683: struct tme_recode_chain_thunk *chain_thunk,
684: tme_uint32_t chain_info_call_or_return)
685: {
686: tme_uint8_t *thunk_bytes_jmp_to_jump_far_unconditional;
687: tme_uint8_t *thunk_bytes;
688: struct tme_recode_x86_tlb_type x86_tlb_type;
689: union tme_recode_reginfo reginfo;
690: unsigned int rex;
691: unsigned int reg_x86_address;
692: tme_uint8_t *thunk_bytes_jcc_chain_counter;
693: const tme_uint8_t *thunk_bytes_jmp_to_epilogue;
694:
695: /* the first chain subs is for a chain call far conditional or chain
696: return far conditional: */
697: TME_RECODE_X86_CHAIN_SUBS(chain_thunk,
698: (chain_info_call_or_return
699: + TME_RECODE_CHAIN_INFO_FAR
700: + TME_RECODE_CHAIN_INFO_CONDITIONAL))
701: = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next);
702:
703: /* if the recode jump flag is false, jmp to the chain subs for a
704: chain jump far unconditional: */
705: thunk_bytes_jmp_to_jump_far_unconditional
706: = _tme_recode_x86_chain_subs_jmp(ic,
707: (TME_RECODE_X86_OPCODE_ESC_0F
708: + (TME_RECODE_X86_OPCODE0F_JCC(TME_RECODE_X86_COND_NOT
709: | TME_RECODE_X86_COND_C)
710: << 8)),
711: chain_thunk,
712: (TME_RECODE_CHAIN_INFO_JUMP
713: + TME_RECODE_CHAIN_INFO_FAR
714: + TME_RECODE_CHAIN_INFO_UNCONDITIONAL));
715:
716: /* if this the chain subs for a chain return far conditional: */
717: if (chain_info_call_or_return == TME_RECODE_CHAIN_INFO_RETURN) {
718:
719: /* throw away the return address to the instructions thunk: */
720: tme_recode_x86_insns_start(ic, thunk_bytes);
721: thunk_bytes = _tme_recode_x86_emit_adjust_sp(thunk_bytes,
722: TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8));
723: tme_recode_x86_insns_finish(ic, thunk_bytes);
724: }
725:
726: /* the next chain subs is for a chain call far unconditional or a
727: chain return far unconditional: */
728: TME_RECODE_X86_CHAIN_SUBS(chain_thunk,
729: (chain_info_call_or_return
730: + TME_RECODE_CHAIN_INFO_FAR
731: + TME_RECODE_CHAIN_INFO_UNCONDITIONAL))
732: = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next);
733:
734: /* if this is the chain subs for a chain call far unconditional: */
735: if (chain_info_call_or_return == TME_RECODE_CHAIN_INFO_CALL) {
736:
737: /* push the chain return address stack: */
738: _tme_recode_x86_chain_ras_push(ic);
739:
740: /* the next chain subs is for both a chain jump far conditional
741: and a chain jump far unconditional: */
742: TME_RECODE_X86_CHAIN_SUBS(chain_thunk,
743: (TME_RECODE_CHAIN_INFO_JUMP
744: + TME_RECODE_CHAIN_INFO_FAR
745: + TME_RECODE_CHAIN_INFO_CONDITIONAL))
746: = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next);
747: TME_RECODE_X86_CHAIN_SUBS(chain_thunk,
748: (TME_RECODE_CHAIN_INFO_JUMP
749: + TME_RECODE_CHAIN_INFO_FAR
750: + TME_RECODE_CHAIN_INFO_UNCONDITIONAL))
751: = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next);
752:
753: /* fix up the jmp to the chain subs for a chain jump far
754: unconditional: */
755: _tme_recode_x86_fixup_jmp(thunk_bytes_jmp_to_jump_far_unconditional,
756: ic->tme_recode_ic_thunk_build_next);
757: }
758:
759: /* load the pointer to the token for the current instruction TLB
760: entry: */
761: _tme_recode_x86_chain_itlb_current_token_load(ic);
762:
763: /* get the TLB type for the address type: */
764: tme_recode_address_type_tlb_type(ic,
765: &chain->tme_recode_chain_address_type,
766: &x86_tlb_type.tme_recode_tlb_type);
767:
768: /* unbusy the token for the current instruction TLB entry: */
769: #if TME_RECODE_X86_REG_CHAIN_ITLB_CURRENT_TOKEN != TME_RECODE_X86_REG_TLB
770: #error "TME_RECODE_X86_REG_CHAIN_ITLB_CURRENT_TOKEN changed"
771: #endif
772: _tme_recode_x86_tlb_unbusy(ic, 0);
773:
774: /* load the PC to chain to: */
775: reginfo = ic->tme_recode_ic_reginfo[chain->tme_recode_chain_reg_guest];
776: reginfo.tme_recode_reginfo_tags_ruses
777: = (TME_RECODE_REGINFO_TAGS_REG_HOST(_tme_recode_x86_tlb_reg_host_address(ic))
778: + TME_RECODE_REGINFO_TAGS_VALID_SIZE(ic->tme_recode_ic_reg_size)
779: + TME_RECODE_REGINFO_TAGS_CLEAN);
780: ic->tme_recode_x86_ic_thunks_reg_guest_window_c = TME_RECODE_REG_GUEST_WINDOW_UNDEF;
781: tme_recode_host_reg_move(ic,
782: chain->tme_recode_chain_reg_guest,
783: reginfo.tme_recode_reginfo_all);
784: ic->tme_recode_x86_ic_thunks_reg_guest_window_c = TME_RECODE_REG_GUEST_WINDOW_UNDEF;
785:
786: /* find, busy, and check an instruction TLB entry: */
787: _tme_recode_x86_tlb_busy(ic,
788: &chain->tme_recode_chain_address_type,
789: &x86_tlb_type);
790:
791: /* store the pointer to the current recode instruction TLB entry: */
792: _tme_recode_x86_chain_itlb_current_store(ic, &x86_tlb_type);
793:
794: /* if this is the chain subs for a chain jump far conditional and
795: a chain jump far unconditional: */
796: if (chain_info_call_or_return == TME_RECODE_CHAIN_INFO_CALL) {
797:
798: /* the next chain subs is for both a chain jump near conditional
799: and a chain jump near unconditional: */
800: TME_RECODE_X86_CHAIN_SUBS(chain_thunk,
801: (TME_RECODE_CHAIN_INFO_JUMP
802: + TME_RECODE_CHAIN_INFO_NEAR
803: + TME_RECODE_CHAIN_INFO_CONDITIONAL))
804: = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next);
805: TME_RECODE_X86_CHAIN_SUBS(chain_thunk,
806: (TME_RECODE_CHAIN_INFO_JUMP
807: + TME_RECODE_CHAIN_INFO_NEAR
808: + TME_RECODE_CHAIN_INFO_UNCONDITIONAL))
809: = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next);
810: }
811:
812: /* start more instructions: */
813: tme_recode_x86_insns_start(ic, thunk_bytes);
814:
815: /* subtract one from the guest chain counter: */
816: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP1_Ib_Ev;
817: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA_DISP32(TME_RECODE_X86_REG_IC),
818: TME_RECODE_X86_OPCODE_GRP1_BINOP(TME_RECODE_X86_OPCODE_BINOP_SUB));
819: *((tme_int32_t *) &thunk_bytes[2]) = ic->tme_recode_ic_chain_counter_offset;
820: thunk_bytes[2 + sizeof(tme_int32_t)] = 1;
821: thunk_bytes += 2 + sizeof(tme_int32_t) + 1;
822:
823: /* if this is the chain subs for a chain jump near conditional and a
824: chain jump near unconditional: */
825: if (chain_info_call_or_return == TME_RECODE_CHAIN_INFO_CALL) {
826:
827: /* load the return address to the instructions thunk into the
828: fixup address register: */
829: /* NB: this is always a 32-bit instruction, even on an x86-64
830: host, because we end up converting the return address into an
831: instructions thunk offset, which is only 32 bits: */
832: assert (sizeof(tme_recode_thunk_off_t) == sizeof(tme_int32_t));
833: rex = TME_RECODE_X86_REX_B(TME_RECODE_SIZE_32, TME_RECODE_X86_REG_CHAIN_FIXUP_ADDRESS);
834: if (rex != 0) {
835: *(thunk_bytes++) = rex;
836: }
837: thunk_bytes[0] = TME_RECODE_X86_OPCODE_BINOP_MOV + TME_RECODE_X86_OPCODE_BINOP_Ev_Gv;
838: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA(TME_RECODE_X86_EA_BASE_SIB),
839: TME_RECODE_X86_REG_CHAIN_FIXUP_ADDRESS);
840: thunk_bytes[2] = TME_RECODE_X86_SIB(TME_RECODE_X86_REG_SP, TME_RECODE_X86_SIB_INDEX_NONE, 1);
841: thunk_bytes += 3;
842: }
843:
844: /* otherwise, this is the chain subs for a return far unconditional: */
845: else {
846:
847: /* load the chain return address stack pointer: */
848: thunk_bytes[0] = (TME_RECODE_X86_OPCODE_BINOP_MOV + TME_RECODE_X86_OPCODE_BINOP_Ev_Gv);
849: thunk_bytes = _tme_recode_x86_emit_ic_modrm(thunk_bytes + 1,
850: ic->tme_recode_ic_chain_ras_pointer_offset,
851: TME_RECODE_X86_REG_CHAIN_RAS_POINTER);
852: }
853:
854: /* if needed, copy the guest instructions source address into the
855: correct register: */
856: reg_x86_address
857: = tme_recode_x86_reg_from_host[_tme_recode_x86_tlb_reg_host_address(ic)];
858: if (reg_x86_address != TME_RECODE_X86_REG_CHAIN_GUEST_SRC) {
859: _tme_recode_x86_emit_reg_copy(thunk_bytes, reg_x86_address, TME_RECODE_X86_REG_CHAIN_GUEST_SRC);
860: }
861:
862: /* if the guest chain counter was zero or one, jmp to the code that
863: handles a guest chain counter underflow: */
864: thunk_bytes_jcc_chain_counter = thunk_bytes;
865: thunk_bytes
866: = _tme_recode_x86_emit_jmp(thunk_bytes,
867: TME_RECODE_X86_OPCODE_JCC(TME_RECODE_X86_COND_BE),
868: (tme_uint8_t *) NULL);
869:
870: /* finish these instructions: */
871: tme_recode_x86_insns_finish(ic, thunk_bytes);
872:
873: /* if this is the chain subs for a chain jump near conditional and a
874: chain jump near unconditional: */
875: if (chain_info_call_or_return == TME_RECODE_CHAIN_INFO_CALL) {
876:
877: /* define CF with the recode jump flag. NB that the recode flags
878: are now above the return address to the instructions thunk: */
879: _tme_recode_x86_conds_testc(ic, TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8) + TME_RECODE_FLAG_JUMP);
880:
881: /* emit a return back to the instructions thunk: */
882: tme_recode_x86_insns_start(ic, thunk_bytes);
883: thunk_bytes[0] = TME_RECODE_X86_OPCODE_RET;
884: thunk_bytes++;
885: tme_recode_x86_insns_finish(ic, thunk_bytes);
886:
887: /* the next chain subs is for a chain call near conditional: */
888: TME_RECODE_X86_CHAIN_SUBS(chain_thunk,
889: (TME_RECODE_CHAIN_INFO_CALL
890: + TME_RECODE_CHAIN_INFO_NEAR
891: + TME_RECODE_CHAIN_INFO_CONDITIONAL))
892: = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next);
893:
894: /* if the recode jump flag is false, jmp to the chain subs for a
895: chain jump near conditional: */
896: _tme_recode_x86_chain_subs_jmp(ic,
897: TME_RECODE_X86_OPCODE_JCC(TME_RECODE_X86_COND_NOT
898: | TME_RECODE_X86_COND_C),
899: chain_thunk,
900: (TME_RECODE_CHAIN_INFO_JUMP
901: + TME_RECODE_CHAIN_INFO_NEAR
902: + TME_RECODE_CHAIN_INFO_CONDITIONAL));
903:
904: /* the next chain subs is for a chain call near unconditional: */
905: TME_RECODE_X86_CHAIN_SUBS(chain_thunk,
906: (TME_RECODE_CHAIN_INFO_CALL
907: + TME_RECODE_CHAIN_INFO_NEAR
908: + TME_RECODE_CHAIN_INFO_UNCONDITIONAL))
909: = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next);
910:
911: /* push the chain return address stack: */
912: _tme_recode_x86_chain_ras_push(ic);
913:
914: /* jmp to the chain subs for a chain jump near unconditional: */
915: _tme_recode_x86_chain_subs_jmp(ic,
916: TME_RECODE_X86_OPCODE_JMP_RELb,
917: chain_thunk,
918: (TME_RECODE_CHAIN_INFO_JUMP
919: + TME_RECODE_CHAIN_INFO_NEAR
920: + TME_RECODE_CHAIN_INFO_UNCONDITIONAL));
921: }
922:
923: /* otherwise, this is the chain subs for a chain return far
924: unconditional: */
925: else {
926:
927: /* load the return address: */
928: _tme_recode_x86_chain_ras_ls(ic, FALSE);
929:
930: /* add one to the return address stack pointer, and store it: */
931: _tme_recode_x86_chain_ras_pointer_update(ic, 1);
932:
933: /* start more instructions: */
934: tme_recode_x86_insns_start(ic, thunk_bytes);
935:
936: /* if this is an x86-64 host: */
937: if (TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32) {
938:
939: /* emit an %ip-relative lea instruction, to load the address of
940: thunk offset zero into the TLB scratch register: */
941: thunk_bytes[0] = TME_RECODE_X86_REX_R(TME_RECODE_SIZE_HOST, TME_RECODE_X86_REG_TLB_SCRATCH);
942: thunk_bytes[1] = TME_RECODE_X86_OPCODE_LEA;
943: thunk_bytes[2] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA(TME_RECODE_X86_EA_BASE_IP),
944: TME_RECODE_X86_REG(TME_RECODE_X86_REG_TLB_SCRATCH));
945: thunk_bytes += 3 + sizeof(tme_int32_t);
946: ((tme_int32_t *) thunk_bytes)[-1]
947: = (0 - (tme_int32_t) tme_recode_build_to_thunk_off(ic, thunk_bytes));
948:
949: /* add the address of thunk offset zero into the return address register: */
950: _tme_recode_x86_emit_reg_binop(thunk_bytes,
951: TME_RECODE_X86_OPCODE_BINOP_ADD,
952: TME_RECODE_X86_REG_TLB_SCRATCH,
953: TME_RECODE_X86_REG_CHAIN_RETURN_ADDRESS);
954: }
955:
956: /* emit the indirect jmp instruction with the return address
957: register: */
958: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP5;
959: thunk_bytes[1]
960: = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(TME_RECODE_X86_REG_CHAIN_RETURN_ADDRESS),
961: TME_RECODE_X86_OPCODE_GRP5_JMP);
962: thunk_bytes += 2;
963:
964: /* unknown indirect jmps are predicted to fallthrough; placing a UD2
965: instruction after an indirect jmp can stop a processor from
966: speculatively executing garbage fallthrough instructions: */
967: thunk_bytes[0] = TME_RECODE_X86_OPCODE_ESC_0F;
968: thunk_bytes[1] = TME_RECODE_X86_OPCODE0F_UD2;
969: thunk_bytes += 2;
970:
971: /* finish these instructions: */
972: tme_recode_x86_insns_finish(ic, thunk_bytes);
973: }
974:
975: /* this is the code that handles an instruction TLB miss: */
976:
977: /* fix up the assist jcc(s) in the instruction TLB busy: */
978: if (x86_tlb_type.tme_recode_x86_tlb_type_assist_jmp_address_ok != NULL) {
979: _tme_recode_x86_fixup_jmp(x86_tlb_type.tme_recode_x86_tlb_type_assist_jmp_address_ok,
980: ic->tme_recode_ic_thunk_build_next);
981: }
982: _tme_recode_x86_fixup_jmp(x86_tlb_type.tme_recode_x86_tlb_type_assist_jmp,
983: ic->tme_recode_ic_thunk_build_next);
984:
985: /* store the pointer to the current recode instruction TLB entry: */
986: /* NB: this is necessary because the instruction TLB that missed is
987: still busy: */
988: _tme_recode_x86_chain_itlb_current_store(ic, &x86_tlb_type);
989:
990: /* start more instructions: */
991: tme_recode_x86_insns_start(ic, thunk_bytes);
992:
993: /* this is the code that jmps to the chain epilogue: */
994: thunk_bytes_jmp_to_epilogue = thunk_bytes;
995:
996: /* if this is not for the chain return far subs: */
997: if (chain_info_call_or_return != TME_RECODE_CHAIN_INFO_RETURN) {
998:
999: /* throw away the return address to the instructions thunk: */
1000: thunk_bytes = _tme_recode_x86_emit_adjust_sp(thunk_bytes,
1001: TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8));
1002: }
1003:
1004: /* jmp to the chain epilogue: */
1005: thunk_bytes
1006: = _tme_recode_x86_emit_jmp(thunk_bytes,
1007: TME_RECODE_X86_OPCODE_JMP_RELz,
1008: tme_recode_thunk_off_to_pointer(ic,
1009: ic->tme_recode_x86_ic_chain_epilogue,
1010: tme_uint8_t *));
1011:
1012: /* this is the code that handles a guest chain counter underflow: */
1013: _tme_recode_x86_fixup_jmp(thunk_bytes_jcc_chain_counter, thunk_bytes);
1014:
1015: /* clear the guest chain counter, since we may have wrapped it past
1016: zero: */
1017: thunk_bytes[0] = TME_RECODE_X86_OPCODE_MOV_Iz_Ev;
1018: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA_DISP32(TME_RECODE_X86_REG_IC),
1019: 0 /* undefined */);
1020: *((tme_int32_t *) &thunk_bytes[2]) = ic->tme_recode_ic_chain_counter_offset;
1021: *((tme_uint32_t *) &thunk_bytes[6]) = 0;
1022: thunk_bytes += 2 + sizeof(tme_int32_t) + sizeof(tme_uint32_t);
1023:
1024: /* jmp to the code that jmps to the chain epilogue: */
1025: thunk_bytes
1026: = _tme_recode_x86_emit_jmp(thunk_bytes,
1027: TME_RECODE_X86_OPCODE_JMP_RELb,
1028: thunk_bytes_jmp_to_epilogue);
1029:
1030: /* finish these instructions: */
1031: tme_recode_x86_insns_finish(ic, thunk_bytes);
1032: }
1033:
1034: /* this emits instructions for a chain out: */
1035: static void
1036: _tme_recode_x86_chain_out(struct tme_recode_ic *ic,
1037: const struct tme_recode_insns_group *insns_group)
1038: {
1039: tme_uint8_t *thunk_bytes;
1040: tme_uint8_t *thunk_bytes_start;
1041: tme_uint32_t chain_info;
1042:
1043: /* the chain subs, and a chain call instruction itself, destroy the
1044: c register, so it won't hold the base offset of a guest register
1045: window any more: */
1046: ic->tme_recode_x86_ic_thunks_reg_guest_window_c = TME_RECODE_REG_GUEST_WINDOW_UNDEF;
1047:
1048: /* start more instructions: */
1049: tme_recode_x86_insns_start(ic, thunk_bytes);
1050:
1051: /* remember where these instructions started: */
1052: thunk_bytes_start = thunk_bytes;
1053:
1054: /* get the chain information: */
1055: chain_info = insns_group->tme_recode_insns_group_chain_info;
1056:
1057: /* exactly one of jump, call, and return may be set: */
1058: assert ((chain_info
1059: & (TME_RECODE_CHAIN_INFO_JUMP
1060: | TME_RECODE_CHAIN_INFO_CALL
1061: | TME_RECODE_CHAIN_INFO_RETURN)) != 0
1062: && ((chain_info
1063: & (TME_RECODE_CHAIN_INFO_JUMP
1064: | TME_RECODE_CHAIN_INFO_CALL
1065: | TME_RECODE_CHAIN_INFO_RETURN))
1066: & ((chain_info
1067: & (TME_RECODE_CHAIN_INFO_JUMP
1068: | TME_RECODE_CHAIN_INFO_CALL
1069: | TME_RECODE_CHAIN_INFO_RETURN))
1070: - 1)) == 0);
1071:
1072: /* there is no such thing as a chain return near: */
1073: assert ((chain_info & TME_RECODE_CHAIN_INFO_RETURN) == 0
1074: || (chain_info
1075: & (TME_RECODE_CHAIN_INFO_NEAR
1076: | TME_RECODE_CHAIN_INFO_FAR)) != TME_RECODE_CHAIN_INFO_NEAR);
1077:
1078: /* if this is a chain conditional, and the alternate target is far,
1079: force the main target to be far, since the alternate target needs
1080: the information generated by a chain far subs: */
1081: assert (((chain_info
1082: & (TME_RECODE_CHAIN_INFO_ALTERNATE_NEAR
1083: | TME_RECODE_CHAIN_INFO_ALTERNATE_FAR))
1084: == TME_RECODE_CHAIN_INFO_ALTERNATE_NEAR)
1085: || ((chain_info
1086: & (TME_RECODE_CHAIN_INFO_UNCONDITIONAL
1087: | TME_RECODE_CHAIN_INFO_CONDITIONAL))
1088: != TME_RECODE_CHAIN_INFO_UNCONDITIONAL));
1089: #if TME_RECODE_CHAIN_INFO_FAR == 0 || TME_RECODE_CHAIN_INFO_ALTERNATE_FAR <= TME_RECODE_CHAIN_INFO_FAR
1090: #error "TME_RECODE_CHAIN_INFO_ values changed"
1091: #endif
1092: chain_info
1093: |= ((chain_info
1094: / (TME_RECODE_CHAIN_INFO_ALTERNATE_FAR
1095: / TME_RECODE_CHAIN_INFO_FAR))
1096: & TME_RECODE_CHAIN_INFO_FAR);
1097:
1098: /* if this is a chain call: */
1099: if (chain_info & TME_RECODE_CHAIN_INFO_CALL) {
1100:
1101: /* load the return address register with a value that indicates
1102: the chain epilogue. this will later get fixed up to indicate
1103: the instructions thunk to return to: */
1104: thunk_bytes[0] = TME_RECODE_X86_OPCODE_MOV_Iv_Gv(TME_RECODE_X86_REG_CHAIN_RETURN_ADDRESS);
1105: *((tme_uint32_t *) (thunk_bytes + 1))
1106: = TME_RECODE_X86_CHAIN_RETURN_ADDRESS(ic, ic->tme_recode_x86_ic_chain_epilogue);
1107: thunk_bytes += 1 + sizeof(tme_uint32_t);
1108: }
1109:
1110: /* if this is a chain return unconditional, jump to the chain subs,
1111: otherwise call the chain subs: */
1112: thunk_bytes[0]
1113: = (((chain_info
1114: & (TME_RECODE_CHAIN_INFO_RETURN
1115: + (TME_RECODE_CHAIN_INFO_UNCONDITIONAL
1116: | TME_RECODE_CHAIN_INFO_CONDITIONAL)))
1117: == (TME_RECODE_CHAIN_INFO_RETURN
1118: + TME_RECODE_CHAIN_INFO_UNCONDITIONAL))
1119: ? TME_RECODE_X86_OPCODE_JMP_RELz
1120: : TME_RECODE_X86_OPCODE_CALL_RELz);
1121: thunk_bytes += 1 + sizeof(tme_int32_t);
1122: ((tme_int32_t *) thunk_bytes)[-1]
1123: = (TME_RECODE_X86_CHAIN_SUBS(insns_group->tme_recode_insns_group_chain_thunk,
1124: chain_info)
1125: - tme_recode_build_to_thunk_off(ic, thunk_bytes));
1126:
1127: /* for the regular target, emit the unconditional jmp instruction,
1128: or the conditional jc instruction, to the chain fixup target: */
1129: if ((chain_info
1130: & (TME_RECODE_CHAIN_INFO_UNCONDITIONAL
1131: | TME_RECODE_CHAIN_INFO_CONDITIONAL))
1132: == TME_RECODE_CHAIN_INFO_UNCONDITIONAL) {
1133: thunk_bytes[0] = TME_RECODE_X86_OPCODE_JMP_RELz;
1134: }
1135: else {
1136: *((tme_uint16_t *) thunk_bytes)
1137: = (TME_RECODE_X86_OPCODE_ESC_0F
1138: + (TME_RECODE_X86_OPCODE0F_JCC(TME_RECODE_X86_COND_C)
1139: << 8));
1140: thunk_bytes++;
1141: }
1142: thunk_bytes += 1 + sizeof(tme_int32_t);
1143: ((tme_int32_t *) thunk_bytes)[-1]
1144: = (TME_RECODE_X86_CHAIN_FIXUP_TARGET(ic, chain_info)
1145: - tme_recode_build_to_thunk_off(ic, thunk_bytes));
1146:
1147: /* if this is a chain conditional instruction: */
1148: if ((chain_info
1149: & (TME_RECODE_CHAIN_INFO_UNCONDITIONAL
1150: | TME_RECODE_CHAIN_INFO_CONDITIONAL))
1151: != TME_RECODE_CHAIN_INFO_UNCONDITIONAL) {
1152:
1153: /* for the alternate target, emit the unconditional jmp
1154: instruction to the chain fixup target alternate: */
1155: thunk_bytes[0] = TME_RECODE_X86_OPCODE_JMP_RELz;
1156: thunk_bytes += 1 + sizeof(tme_int32_t);
1157: ((tme_int32_t *) thunk_bytes)[-1]
1158: = (TME_RECODE_X86_CHAIN_FIXUP_TARGET_ALTERNATE(ic, chain_info)
1159: - tme_recode_build_to_thunk_off(ic, thunk_bytes));
1160: }
1161:
1162: /* check the size of the chain out: */
1163: assert ((thunk_bytes - thunk_bytes_start)
1164: <= TME_RECODE_X86_CHAIN_OUT_SIZE_MAX);
1165:
1166: /* finish these instructions: */
1167: tme_recode_x86_insns_finish(ic, thunk_bytes);
1168: }
1169:
1170: /* if the given address is reachable with an effective address using
1171: an address in another register, this returns the signed 32-bit
1172: displacement, otherwise this returns zero: */
1173: static inline tme_int32_t
1174: _tme_recode_x86_chain_base_disp32(const tme_shared tme_uint8_t *base,
1175: const tme_shared tme_uint8_t *pointer)
1176: {
1177: signed long disp;
1178:
1179: disp = pointer - base;
1180: return (disp == (tme_int32_t) disp
1181: ? disp
1182: : 0);
1183: }
1184:
1185: /* if the given address is reachable with a PC-relative effective
1186: address, this returns the signed 32-bit displacement, otherwise
1187: this returns zero: */
1188: static inline tme_int32_t
1189: _tme_recode_x86_chain_ip_disp32(const struct tme_recode_ic *ic,
1190: const tme_uint8_t *thunk_bytes,
1191: const tme_shared tme_uint8_t *pointer)
1192: {
1193: tme_recode_thunk_off_t thunk_off;
1194: const tme_uint8_t *ip;
1195:
1196: thunk_off = tme_recode_build_to_thunk_off(ic, thunk_bytes);
1197: ip = tme_recode_thunk_off_to_pointer(ic, thunk_off, const tme_uint8_t *);
1198: return (_tme_recode_x86_chain_base_disp32(ip, pointer));
1199: }
1200:
1201: /* this emits instructions for a chain in: */
1202: static void
1203: _tme_recode_x86_chain_in(struct tme_recode_ic *ic,
1204: const struct tme_recode_insns_group *insns_group)
1205: {
1206: tme_uint8_t *thunk_bytes;
1207: tme_uint8_t *thunk_bytes_start;
1208: const tme_shared tme_uint8_t *insns_group_src;
1209: unsigned int size_insns_group_src;
1210: tme_int32_t disp32;
1211: unsigned int rex;
1212: const tme_shared tme_uint8_t *insns_group_valid_byte;
1213: unsigned int size_insns_group_valid_byte;
1214:
1215: /* start more instructions: */
1216: tme_recode_x86_insns_start(ic, thunk_bytes);
1217:
1218: /* remember where these instructions started: */
1219: thunk_bytes_start = thunk_bytes;
1220:
1221: /* get the guest instructions source address: */
1222: insns_group_src = insns_group->tme_recode_insns_group_src;
1223:
1224: /* if this is an ia32 host, or if this is an x86-64 host and the
1225: guest instructions source address fits in a sign-extended 32
1226: bits: */
1227: if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32
1228: || (((tme_int32_t)
1229: (signed long)
1230: insns_group_src)
1231: == (signed long) insns_group_src)) {
1232:
1233: /* emit a cmpl $imm32, %reg or cmpq $imm32, %reg: */
1234: if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32) {
1235: thunk_bytes[0] = (tme_uint8_t) TME_RECODE_X86_INSN_CMP_Iz_REG_CHAIN_GUEST_SRC;
1236: }
1237: else {
1238: *((tme_uint16_t *) thunk_bytes) = TME_RECODE_X86_INSN_CMP_Iz_REG_CHAIN_GUEST_SRC;
1239: }
1240: *((tme_int32_t *) &thunk_bytes[(TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32) + 1])
1241: = (signed long) insns_group_src;
1242: thunk_bytes += (TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32) + 1 + sizeof(tme_int32_t);
1243:
1244: /* assume that this is an x86-64 host, and set the size of the
1245: instructions that check the guest instruction source address
1246: matches the instructions thunk: */
1247: size_insns_group_src = TME_RECODE_X86_64_CHAIN_IN_SIZE_CMPQ_IMM32_JNZ;
1248: }
1249:
1250: /* otherwise, this is an x86-64 host and the guest instructions
1251: source address doesn't fit in a sign-extended 32 bits: */
1252: else {
1253:
1254: /* if the guest instructions source address fits in 32 bits: */
1255: if (TME_RECODE_SIZE_FITS((unsigned long) insns_group_src,
1256: TME_RECODE_SIZE_32)) {
1257:
1258: /* emit a movl $imm32, %reg: */
1259: thunk_bytes[0] = TME_RECODE_X86_INSN_MOVL_IMM32_REG_TLB_SCRATCH;
1260: *((tme_uint32_t *) &thunk_bytes[1]) = (unsigned long) insns_group_src;
1261: thunk_bytes += 1 + sizeof(tme_uint32_t);
1262:
1263: /* set the size of the instructions that check the guest
1264: instruction source address matches the instructions thunk: */
1265: size_insns_group_src = TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVL_CMP_JNZ;
1266: }
1267:
1268: /* otherwise, if the guest instructions source address is within a
1269: signed 32-bit displacement of the %ip of an lea
1270: instruction: */
1271: else if ((disp32
1272: = _tme_recode_x86_chain_ip_disp32(ic,
1273: (thunk_bytes
1274: + 1 /* rex */
1275: + 1 /* lea */
1276: + 1 /* modR/M */
1277: + sizeof(disp32)),
1278: insns_group_src))) {
1279:
1280: /* emit an lea disp32(%ip), %reg: */
1281: thunk_bytes[0] = (tme_uint8_t) TME_RECODE_X86_INSN_LEA_DISP32_IP_REG_TLB_SCRATCH;
1282: *((tme_uint16_t *) &thunk_bytes[1]) = (TME_RECODE_X86_INSN_LEA_DISP32_IP_REG_TLB_SCRATCH >> 8);
1283: *((tme_int32_t *) &thunk_bytes[3]) = disp32;
1284: thunk_bytes += 1 + 2 + sizeof(disp32);
1285:
1286: /* set the size of the instructions that check the guest
1287: instruction source address matches the instructions thunk: */
1288: size_insns_group_src = TME_RECODE_X86_64_CHAIN_IN_SIZE_LEA_IP_CMP_JNZ;
1289: }
1290:
1291: /* otherwise, the guest instructions source address can't be
1292: generated: */
1293: else {
1294:
1295: /* emit a movq $imm64, %reg: */
1296: *((tme_uint16_t *) thunk_bytes) = TME_RECODE_X86_INSN_MOVQ_IMM64_REG_TLB_SCRATCH;
1297: *((unsigned long *) &thunk_bytes[2]) = (unsigned long) insns_group_src;
1298: thunk_bytes += 1 + 1 + TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8);
1299:
1300: /* set the size of the instructions that check the guest
1301: instruction source address matches the instructions thunk: */
1302: size_insns_group_src = TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVQ_CMP_JNZ;
1303: }
1304:
1305: /* emit a cmpq %reg, %reg: */
1306: rex = (TME_RECODE_X86_REX_B(TME_RECODE_SIZE_HOST,
1307: TME_RECODE_X86_REG_CHAIN_GUEST_SRC)
1308: | TME_RECODE_X86_REX_R(TME_RECODE_SIZE_HOST,
1309: TME_RECODE_X86_REG_TLB_SCRATCH));
1310: assert (rex != 0);
1311: thunk_bytes[0] = rex;
1312: *((tme_uint16_t *) &thunk_bytes[1])
1313: = ((TME_RECODE_X86_OPCODE_BINOP_CMP
1314: + TME_RECODE_X86_OPCODE_BINOP_Ev_Gv)
1315: + (TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(TME_RECODE_X86_REG_CHAIN_GUEST_SRC),
1316: TME_RECODE_X86_REG(TME_RECODE_X86_REG_TLB_SCRATCH))
1317: << 8));
1318: thunk_bytes += 3;
1319: }
1320:
1321: /* if the guest instructions source address doesn't match this
1322: instructions thunk, jump to the chain epilogue: */
1323: *((tme_uint16_t *) thunk_bytes)
1324: = (TME_RECODE_X86_OPCODE_ESC_0F
1325: + (TME_RECODE_X86_OPCODE0F_JCC(TME_RECODE_X86_COND_NOT | TME_RECODE_X86_COND_Z)
1326: << 8));
1327: thunk_bytes += 2 + sizeof(tme_int32_t);
1328: ((tme_int32_t *) thunk_bytes)[-1]
1329: = (ic->tme_recode_x86_ic_chain_epilogue
1330: - tme_recode_build_to_thunk_off(ic, thunk_bytes));
1331:
1332: /* get the guest instructions valid byte address: */
1333: insns_group_valid_byte = insns_group->tme_recode_insns_group_valid_byte;
1334:
1335: /* if this is an ia32 host: */
1336: if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32) {
1337:
1338: /* emit the opcode and addr32 for a testb $imm8, addr32: */
1339: *((tme_uint16_t *) thunk_bytes)
1340: = (TME_RECODE_X86_OPCODE_GRP3_Eb
1341: + (TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA(TME_RECODE_X86_EA_BASE_NONE),
1342: TME_RECODE_X86_OPCODE_GRP3_TEST)
1343: << 8));
1344: *((tme_uint32_t *) &thunk_bytes[2]) = (unsigned long) insns_group_valid_byte;
1345: thunk_bytes += 2 + sizeof(tme_uint32_t);
1346:
1347: /* silence uninitialized variable warnings: */
1348: size_insns_group_valid_byte = 0;
1349: }
1350:
1351: /* otherwise, this is an x86-64 host: */
1352: else {
1353:
1354: /* assume that we can emit a testb followed by a jz, and emit the
1355: first opcode byte for the testb: */
1356: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP3_Eb;
1357: thunk_bytes++;
1358:
1359: /* if the guest instructions valid byte address fits in a
1360: sign-extended 32 bits: */
1361: if (((tme_int32_t)
1362: (signed long)
1363: insns_group_valid_byte)
1364: == (signed long) insns_group_valid_byte) {
1365:
1366: /* emit the modR/M, SIB, and disp32 for a testb $imm8, addr32: */
1367: *((tme_uint16_t *) thunk_bytes) = TME_RECODE_X86_INSN_TESTB_IMM8_ADDR32 >> 8;
1368: *((tme_int32_t *) &thunk_bytes[2]) = (signed long) insns_group_valid_byte;
1369: thunk_bytes += 2 + sizeof(tme_int32_t);
1370:
1371: /* set the size of the instructions that check that the guest
1372: instructions are still valid: */
1373: size_insns_group_valid_byte = TME_RECODE_X86_64_CHAIN_IN_SIZE_TESTB_ADDR32_JZ;
1374: }
1375:
1376: /* otherwise, if the guest instructions valid byte address is
1377: within a signed 32-bit displacement of the %ip of the testb
1378: instruction: */
1379: else if ((disp32
1380: = _tme_recode_x86_chain_ip_disp32(ic,
1381: (thunk_bytes
1382: + 1 /* testb */
1383: + 1 /* modR/M */
1384: + sizeof(disp32)
1385: + 1 /* imm8 */),
1386: insns_group_valid_byte))) {
1387:
1388: /* emit the modR/M and disp32 for a testb $imm8, disp32(%ip): */
1389: thunk_bytes[0] = TME_RECODE_X86_INSN_TESTB_IMM8_DISP32_IP >> 8;
1390: *((tme_int32_t *) &thunk_bytes[1]) = disp32;
1391: thunk_bytes += 1 + sizeof(disp32);
1392:
1393: /* set the size of the instructions that check that the guest
1394: instructions are still valid: */
1395: size_insns_group_valid_byte = TME_RECODE_X86_64_CHAIN_IN_SIZE_TESTB_BASE_JZ;
1396: }
1397:
1398: /* otherwise, if the guest instructions valid byte address is
1399: within a signed 32-bit displacement of the guest instructions
1400: source address: */
1401: else if ((disp32
1402: = _tme_recode_x86_chain_base_disp32(insns_group_src,
1403: insns_group_valid_byte))) {
1404:
1405: /* emit the modR/M and disp32 for a testb $imm8, disp32(%reg): */
1406: thunk_bytes[0] = TME_RECODE_X86_INSN_TESTB_IMM8_DISP32_REG_CHAIN_GUEST_SRC >> 8;
1407: *((tme_int32_t *) &thunk_bytes[1]) = disp32;
1408: thunk_bytes += 1 + sizeof(disp32);
1409:
1410: /* set the size of the instructions that check that the guest
1411: instructions are still valid: */
1412: size_insns_group_valid_byte = TME_RECODE_X86_64_CHAIN_IN_SIZE_TESTB_BASE_JZ;
1413: }
1414:
1415: /* otherwise, the guest instructions valid byte address can't be
1416: generated: */
1417: else {
1418:
1419: /* emit a movq $imm64, %reg: */
1420: /* NB: thunk_bytes has already been advanced by one: */
1421: *((tme_uint16_t *) &thunk_bytes[0 - 1]) = TME_RECODE_X86_INSN_MOVQ_IMM64_REG_TLB_SCRATCH;
1422: *((unsigned long *) &thunk_bytes[2 - 1]) = (unsigned long) insns_group_valid_byte;
1423: thunk_bytes += -1 + 1 + 1 + TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8);
1424:
1425: /* emit the opcode and modR/M for a testb $imm8, (%reg): */
1426: *((tme_uint16_t *) thunk_bytes)
1427: = (TME_RECODE_X86_OPCODE_GRP3_Eb
1428: + (TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA(TME_RECODE_X86_REG_TLB_SCRATCH),
1429: TME_RECODE_X86_OPCODE_GRP3_TEST)
1430: << 8));
1431: thunk_bytes += 2;
1432:
1433: /* set the size of the instructions that check that the guest
1434: instructions are still valid: */
1435: size_insns_group_valid_byte = TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVQ_TESTB_JZ;
1436: }
1437: }
1438:
1439: /* emit the $imm8, and if the guest instructions aren't valid, jump
1440: to the chain epilogue: */
1441: thunk_bytes[0] = insns_group->tme_recode_insns_group_valid_mask;
1442: *((tme_uint16_t *) &thunk_bytes[1])
1443: = (TME_RECODE_X86_OPCODE_ESC_0F
1444: + (TME_RECODE_X86_OPCODE0F_JCC(TME_RECODE_X86_COND_Z)
1445: << 8));
1446: thunk_bytes += 1 + 2 + sizeof(tme_int32_t);
1447: ((tme_int32_t *) thunk_bytes)[-1]
1448: = (ic->tme_recode_x86_ic_chain_epilogue
1449: - tme_recode_build_to_thunk_off(ic, thunk_bytes));
1450:
1451: /* check the size of the chain in: */
1452: assert ((thunk_bytes - thunk_bytes_start)
1453: <= TME_RECODE_X86_CHAIN_IN_SIZE_MAX);
1454:
1455: /* check the sizes of the instructions that we need to skip for a
1456: chain in near: */
1457: assert ((thunk_bytes - thunk_bytes_start)
1458: == (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32
1459: ? TME_RECODE_IA32_CHAIN_IN_SIZE_FAR
1460: : (size_insns_group_src
1461: + size_insns_group_valid_byte)));
1462:
1463: /* finish these instructions: */
1464: tme_recode_x86_insns_finish(ic, thunk_bytes);
1465: }
1466:
1467: /* this makes the chain prologue for a new IC: */
1468: static void
1469: _tme_recode_x86_chain_prologue(struct tme_recode_ic *ic,
1470: const struct tme_recode_chain *chain,
1471: struct tme_recode_chain_thunk *chain_thunk)
1472: {
1473: tme_uint8_t *thunk_bytes;
1474: unsigned int reg_x86_insns_thunk;
1475: unsigned long thunk_address0;
1476:
1477: /* start more instructions: */
1478: tme_recode_x86_insns_start(ic, thunk_bytes);
1479:
1480: /* set the chain prologue: */
1481: chain_thunk->tme_recode_x86_chain_thunk_prologue
1482: = tme_recode_thunk_off_to_pointer(ic,
1483: tme_recode_build_to_thunk_off(ic, thunk_bytes),
1484: void (*) _TME_P((struct tme_ic *, tme_recode_thunk_off_t)));
1485:
1486: /* push all callee-saved registers: */
1487: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_BP);
1488: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_B);
1489: if (TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32) {
1490: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_N(12));
1491: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_N(13));
1492: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_N(14));
1493: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_N(15));
1494: }
1495: else {
1496: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_SI);
1497: _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_DI);
1498: }
1499:
1500: /* the instructions thunk offset and address will be in the si register: */
1501: reg_x86_insns_thunk = TME_RECODE_X86_REG_SI;
1502:
1503: /* if this is an x86-64 host: */
1504: if (TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32) {
1505:
1506: /* copy the struct tme_ic * argument into the ic register: */
1507: _tme_recode_x86_emit_reg_copy(thunk_bytes, TME_RECODE_X86_REG_DI, TME_RECODE_X86_REG_IC);
1508:
1509: /* the instructions thunk offset is already in the second argument register: */
1510: assert (reg_x86_insns_thunk == TME_RECODE_X86_REG_SI);
1511: }
1512:
1513: /* otherwise, this is an ia32 host: */
1514: else {
1515:
1516: /* load the struct tme_ic * argument from the stack: */
1517: /* NB: the magic 5 below is for the four callee-saved registers
1518: that we pushed above, plus the return address: */
1519: thunk_bytes[0] = (TME_RECODE_X86_OPCODE_BINOP_MOV + TME_RECODE_X86_OPCODE_BINOP_Ev_Gv);
1520: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA_DISP8(TME_RECODE_X86_EA_BASE_SIB),
1521: TME_RECODE_X86_REG_IC);
1522: thunk_bytes[2] = TME_RECODE_X86_SIB(TME_RECODE_X86_REG_SP, TME_RECODE_X86_SIB_INDEX_NONE, 1);
1523: thunk_bytes[3] = (TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8) * 5);
1524: thunk_bytes += 4;
1525:
1526: /* load the instructions thunk offset from the stack: */
1527: /* NB: the magic 6 below is for the four callee-saved registers
1528: that we pushed above, plus the return address, plus the struct
1529: tme_ic * argument: */
1530: thunk_bytes[0] = (TME_RECODE_X86_OPCODE_BINOP_MOV + TME_RECODE_X86_OPCODE_BINOP_Ev_Gv);
1531: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_EA_DISP8(TME_RECODE_X86_EA_BASE_SIB),
1532: reg_x86_insns_thunk);
1533: thunk_bytes[2] = TME_RECODE_X86_SIB(TME_RECODE_X86_REG_SP, TME_RECODE_X86_SIB_INDEX_NONE, 1);
1534: thunk_bytes[3] = (TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8) * 6);
1535: thunk_bytes += 4;
1536: }
1537:
1538: /* get the near address of the thunk at offset zero: */
1539: thunk_address0 = tme_recode_thunk_off_to_pointer(ic, 0, char *) - (char *) 0;
1540:
1541: /* if this is an ia32 host, or if this is an x86-64 host and the
1542: near address of the thunk at offset zero fits in 32 bits: */
1543: if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32
1544: || thunk_address0 == (tme_uint32_t) thunk_address0) {
1545:
1546: /* add the near address of the thunk at offset zero to the
1547: instructions thunk offset: */
1548: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP1_Iz_Ev;
1549: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(reg_x86_insns_thunk),
1550: TME_RECODE_X86_OPCODE_GRP1_BINOP(TME_RECODE_X86_OPCODE_BINOP_ADD));
1551: *((tme_uint32_t *) &thunk_bytes[2]) = thunk_address0;
1552: thunk_bytes += 2 + sizeof(tme_uint32_t);
1553: }
1554:
1555: /* otherwise, this is an x86-64 host and the near address of the
1556: thunk at offset zero doesn't fit in 32 bits: */
1557: else {
1558:
1559: /* zero-extend the instructions thunk offset to 64 bits: */
1560: thunk_bytes[0]
1561: = (TME_RECODE_X86_OPCODE_BINOP_MOV
1562: + TME_RECODE_X86_OPCODE_BINOP_Gv_Ev);
1563: thunk_bytes[1]
1564: = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(reg_x86_insns_thunk),
1565: TME_RECODE_X86_REG(reg_x86_insns_thunk));
1566: thunk_bytes += 2;
1567:
1568: /* load the near address of the thunk at offset zero into the TLB
1569: scratch register: */
1570: *((tme_uint16_t *) &thunk_bytes[0]) = TME_RECODE_X86_INSN_MOVQ_IMM64_REG_TLB_SCRATCH;
1571: *((unsigned long *) &thunk_bytes[2]) = thunk_address0;
1572: thunk_bytes += 2 + TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8);
1573:
1574: /* add the TLB scratch register to the instructions thunk
1575: offset: */
1576: _tme_recode_x86_emit_reg_binop(thunk_bytes,
1577: TME_RECODE_X86_OPCODE_BINOP_ADD,
1578: TME_RECODE_X86_REG_TLB_SCRATCH,
1579: reg_x86_insns_thunk);
1580: }
1581:
1582: /* allocate one word on the stack for the recode flag bytes. NB
1583: that on x86-64, this returns the stack pointer to 16-byte
1584: alignment as required by the ABI: */
1585: thunk_bytes
1586: = _tme_recode_x86_emit_adjust_sp(thunk_bytes,
1587: (0 - TME_BIT(TME_RECODE_SIZE_HOST
1588: - TME_RECODE_SIZE_8)));
1589:
1590: /* call the chain jump far unconditional subs: */
1591: thunk_bytes[0] = TME_RECODE_X86_OPCODE_CALL_RELz;
1592: thunk_bytes += 1 + sizeof(tme_int32_t);
1593: ((tme_int32_t *) thunk_bytes)[-1]
1594: = (TME_RECODE_X86_CHAIN_SUBS(chain_thunk,
1595: (TME_RECODE_CHAIN_INFO_JUMP
1596: + TME_RECODE_CHAIN_INFO_FAR
1597: + TME_RECODE_CHAIN_INFO_UNCONDITIONAL))
1598: - tme_recode_build_to_thunk_off(ic, thunk_bytes));
1599:
1600: /* do the indirect jmp into the instructions thunk: */
1601: assert (TME_RECODE_X86_REX_B(0, reg_x86_insns_thunk) == 0);
1602: thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP5;
1603: thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(reg_x86_insns_thunk),
1604: TME_RECODE_X86_OPCODE_GRP5_JMP);
1605: thunk_bytes += 2;
1606:
1607: /* unknown indirect jmps are predicted to fallthrough; placing a UD2
1608: instruction after an indirect jmp can stop a processor from
1609: speculatively executing garbage fallthrough instructions: */
1610: thunk_bytes[0] = TME_RECODE_X86_OPCODE_ESC_0F;
1611: thunk_bytes[1] = TME_RECODE_X86_OPCODE0F_UD2;
1612: thunk_bytes += 2;
1613:
1614: /* finish these instructions: */
1615: tme_recode_x86_insns_finish(ic, thunk_bytes);
1616:
1617: /* finish the chain prologue: */
1618: tme_recode_host_thunk_finish(ic);
1619: }
1620:
1621: /* this returns a chain thunk: */
1622: const struct tme_recode_chain_thunk *
1623: tme_recode_chain_thunk(struct tme_recode_ic *ic,
1624: const struct tme_recode_chain *chain)
1625: {
1626: struct tme_recode_chain_thunk *chain_thunk;
1627:
1628: /* if the chain fixup targets haven't been made yet: */
1629: if (TME_RECODE_X86_CHAIN_FIXUP_TARGET_ALTERNATE(ic,
1630: (TME_RECODE_CHAIN_INFO_JUMP
1631: + TME_RECODE_CHAIN_INFO_ALTERNATE_NEAR))
1632: == 0) {
1633:
1634: /* make the chain fixup targets: */
1635: _tme_recode_x86_chain_fixup_targets(ic);
1636: }
1637:
1638: /* start the thunk for the chain subs: */
1639: if (!tme_recode_host_thunk_start(ic)) {
1640: abort();
1641: }
1642: chain_thunk = tme_new0(struct tme_recode_chain_thunk, 1);
1643:
1644: /* make the chain subs for chain calls and chain jumps: */
1645: _tme_recode_x86_chain_subs(ic,
1646: chain,
1647: chain_thunk,
1648: TME_RECODE_CHAIN_INFO_CALL);
1649:
1650: /* make the chain subs for chain returns: */
1651: _tme_recode_x86_chain_subs(ic,
1652: chain,
1653: chain_thunk,
1654: TME_RECODE_CHAIN_INFO_RETURN);
1655:
1656: /* finish the thunk for the chain subs: */
1657: tme_recode_host_thunk_finish(ic);
1658:
1659: /* start the thunk for the chain prologue: */
1660: if (!tme_recode_host_thunk_start(ic)) {
1661: abort();
1662: }
1663:
1664: /* make the chain prologue: */
1665: _tme_recode_x86_chain_prologue(ic,
1666: chain,
1667: chain_thunk);
1668:
1669: /* finish the thunk for the chain prologue: */
1670: tme_recode_host_thunk_finish(ic);
1671:
1672: return (chain_thunk);
1673: }
1674:
1675: /* this clears the return address stack: */
1676: void
1677: tme_recode_chain_ras_clear(const struct tme_recode_ic *recode_ic,
1678: struct tme_ic *ic)
1679: {
1680: tme_recode_ras_entry_t *_ras_entry;
1681: tme_uint32_t ras_size;
1682: tme_recode_ras_entry_t ras_entry;
1683:
1684: /* clear the return address stack: */
1685: ras_entry
1686: = TME_RECODE_X86_CHAIN_RETURN_ADDRESS(recode_ic,
1687: recode_ic->tme_recode_x86_ic_chain_epilogue);
1688: _ras_entry
1689: = ((tme_recode_ras_entry_t *)
1690: (((tme_uint8_t *) ic)
1691: + recode_ic->tme_recode_ic_chain_ras_offset));
1692: ras_size = recode_ic->tme_recode_ic_chain_ras_size;
1693: do {
1694: _ras_entry[ras_size - 1] = ras_entry;
1695: } while (--ras_size);
1696:
1697: /* make sure that the return address stack pointer is valid: */
1698: assert (*((tme_uint32_t *)
1699: (((tme_uint8_t *) ic)
1700: + recode_ic->tme_recode_ic_chain_ras_pointer_offset))
1701: < recode_ic->tme_recode_ic_chain_ras_size);
1702: }
1703:
1704: /* this fixes up a chain: */
1705: tme_recode_thunk_off_t
1706: tme_recode_chain_fixup(struct tme_recode_ic *ic,
1707: tme_recode_thunk_off_t chain_fixup,
1708: tme_uint32_t chain_info,
1709: tme_recode_thunk_off_t insns_thunk_next,
1710: tme_recode_thunk_off_t insns_thunk_return)
1711: {
1712: tme_uint32_t return_imm32;
1713: tme_uint16_t insn_0_15_buffer;
1714: tme_uint32_t insn_0_15;
1715: const tme_uint8_t *thunk_bytes;
1716: tme_uint8_t opcode_buffer;
1717: tme_int32_t displacement;
1718:
1719: /* if this is a chain call: */
1720: if (chain_info & TME_RECODE_CHAIN_INFO_CALL) {
1721:
1722: /* there must be a return instructions thunk: */
1723: assert (insns_thunk_return != 0);
1724:
1725: /* a chain call emits:
1726:
1727: movl $imm32, %TME_RECODE_X86_REG_CHAIN_RETURN_ADDRESS
1728: call chain_subs_call_{near|far}_{unconditional|conditional}
1729: jmp/jnc chain_fixup_call_{near|far}
1730:
1731: on an ia32 host, the $imm32 is the absolute address of the
1732: instructions thunk to chain to. on an x86-64 host, the $imm32
1733: is the offset of the instructions thunk to chain to. on both
1734: hosts, before fixup, the $imm32 indicates the chain epilogue.
1735: fix up the $imm32: */
1736: /* NB: chain_fixup points to the jump instruction. the call
1737: instruction is a call rel32, which is five bytes long, and the
1738: $imm32 is immediately before that: */
1739: return_imm32 = TME_RECODE_X86_CHAIN_RETURN_ADDRESS(ic, insns_thunk_return);
1740: tme_recode_thunk_off_write(ic,
1741: (chain_fixup
1742: - (5
1743: + sizeof(tme_uint32_t))),
1744: tme_uint32_t,
1745: return_imm32);
1746: }
1747:
1748: /* if this is a chain near: */
1749: if ((chain_info
1750: & (TME_RECODE_CHAIN_INFO_NEAR
1751: | TME_RECODE_CHAIN_INFO_FAR))
1752: == TME_RECODE_CHAIN_INFO_NEAR) {
1753:
1754: /* if this is an ia32 host: */
1755: if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32) {
1756:
1757: /* advance the next instructions thunk offset past the chain
1758: in far checks: */
1759: insns_thunk_next += TME_RECODE_IA32_CHAIN_IN_SIZE_FAR;
1760: }
1761:
1762: /* otherwise, this is an x86-64 host: */
1763: else {
1764:
1765: /* read the first two bytes of the first instruction of the
1766: chain in. this instruction starts checking that the guest
1767: instruction source address matches the instructions thunk: */
1768: insn_0_15 = *tme_recode_thunk_off_read(ic, insns_thunk_next, tme_uint16_t, insn_0_15_buffer);
1769:
1770: /* if this instruction is a cmpq $imm32, %reg: */
1771: if (insn_0_15 == (tme_uint16_t) TME_RECODE_X86_INSN_CMP_Iz_REG_CHAIN_GUEST_SRC) {
1772:
1773: /* advance the next instructions thunk offset past the cmpq
1774: $imm32, %reg; jnz epilogue */
1775: insns_thunk_next += TME_RECODE_X86_64_CHAIN_IN_SIZE_CMPQ_IMM32_JNZ;
1776: }
1777:
1778: /* if this instruction is an lea disp32(%ip), %reg: */
1779: if (insn_0_15 == (tme_uint16_t) TME_RECODE_X86_INSN_LEA_DISP32_IP_REG_TLB_SCRATCH) {
1780:
1781: /* advance the next instructions thunk offset past the lea
1782: disp32(%ip), %reg ; cmpq %reg, %reg ; jnz epilogue */
1783: insns_thunk_next += TME_RECODE_X86_64_CHAIN_IN_SIZE_LEA_IP_CMP_JNZ;
1784: }
1785:
1786: /* if this instruction is a movq $imm64, %reg: */
1787: if (insn_0_15 == (tme_uint16_t) TME_RECODE_X86_INSN_MOVQ_IMM64_REG_TLB_SCRATCH) {
1788:
1789: /* advance the next instructions thunk offset past the movq
1790: $imm64, %reg ; cmpq %reg, %reg ; jnz epilogue */
1791: insns_thunk_next += TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVQ_CMP_JNZ;
1792: }
1793:
1794: /* if this instruction is a movl $imm32, %reg: */
1795: if (((tme_uint8_t) insn_0_15) == TME_RECODE_X86_INSN_MOVL_IMM32_REG_TLB_SCRATCH) {
1796:
1797: /* advance the next instructions thunk offset past the movl
1798: $imm32, %reg ; cmpq %reg, %reg ; jnz epilogue */
1799: insns_thunk_next += TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVL_CMP_JNZ;
1800: }
1801:
1802: /* read the first two bytes of the next instruction of the chain
1803: in. this instruction starts checking that the guest
1804: instructions are still valid: */
1805: insn_0_15 = *tme_recode_thunk_off_read(ic, insns_thunk_next, tme_uint16_t, insn_0_15_buffer);
1806:
1807: /* if this instruction is a testb $imm8, addr32: */
1808: if (insn_0_15 == (tme_uint16_t) TME_RECODE_X86_INSN_TESTB_IMM8_ADDR32) {
1809:
1810: /* advance the next instructions thunk offset past the testb
1811: $imm8, addr32 ; jz epilogue */
1812: insns_thunk_next += TME_RECODE_X86_64_CHAIN_IN_SIZE_TESTB_ADDR32_JZ;
1813: }
1814:
1815: /* if this instruction is a testb $imm8, disp32(%ip): */
1816: if (insn_0_15 == (tme_uint16_t) TME_RECODE_X86_INSN_TESTB_IMM8_DISP32_IP) {
1817:
1818: /* advance the next instructions thunk offset past the testb
1819: $imm8, disp32(%ip) ; jz epilogue */
1820: insns_thunk_next += TME_RECODE_X86_64_CHAIN_IN_SIZE_TESTB_BASE_JZ;
1821: }
1822:
1823: /* if this instruction is a testb $imm8, disp32(%reg): */
1824: if (insn_0_15 == (tme_uint16_t) TME_RECODE_X86_INSN_TESTB_IMM8_DISP32_REG_CHAIN_GUEST_SRC) {
1825:
1826: /* advance the next instructions thunk offset past the testb
1827: $imm8, disp32(%reg) ; jz epilogue */
1828: insns_thunk_next += TME_RECODE_X86_64_CHAIN_IN_SIZE_TESTB_BASE_JZ;
1829: }
1830:
1831: /* if this instruction is a movq $imm64, %reg: */
1832: if (insn_0_15 == TME_RECODE_X86_INSN_MOVQ_IMM64_REG_TLB_SCRATCH) {
1833:
1834: /* advance the next instructions thunk offset past the movq
1835: $imm64, %reg ; testb $imm8, (%reg) ; jz epilogue */
1836: insns_thunk_next += TME_RECODE_X86_64_CHAIN_IN_SIZE_MOVQ_TESTB_JZ;
1837: }
1838: }
1839: }
1840:
1841: /* all chain fixup instructions are unconditional or conditional
1842: jump instructions with 32-bit displacements at their ends.
1843: unconditional jump instructions are five bytes, and conditional
1844: jump instructions are six bytes because they are escaped: */
1845:
1846: /* read the first opcode byte of the jump instruction: */
1847: thunk_bytes = tme_recode_thunk_off_read(ic, chain_fixup, tme_uint8_t, opcode_buffer);
1848:
1849: /* advance the chain fixup offset to the jump instruction displacement: */
1850: chain_fixup += (*thunk_bytes == TME_RECODE_X86_OPCODE_ESC_0F) + 1;
1851:
1852: /* rewrite the displacement for the instructions thunk: */
1853: displacement = insns_thunk_next - (chain_fixup + sizeof(tme_int32_t));
1854: tme_recode_thunk_off_write(ic, chain_fixup, tme_int32_t, displacement);
1855:
1856: /* return the next instructions thunk offset: */
1857: return (insns_thunk_next);
1858: }
1859:
1860: #ifdef TME_RECODE_DEBUG
1861: #include <stdio.h>
1862:
1863: /* this host function dumps a chain thunk: */
1864: void
1865: tme_recode_chain_thunk_dump(const struct tme_recode_ic *ic,
1866: const struct tme_recode_chain_thunk *chain_thunk)
1867: {
1868: tme_uint32_t chain_info;
1869: const char *s;
1870: tme_recode_thunk_off_t insns_thunk;
1871:
1872: printf(" x86 chain prologue: x/10i %p\n",
1873: chain_thunk->tme_recode_x86_chain_thunk_prologue);
1874: for (chain_info = 0;
1875: chain_info <= (TME_RECODE_CHAIN_INFO_UNCONDITIONAL
1876: | TME_RECODE_CHAIN_INFO_CONDITIONAL
1877: | TME_RECODE_CHAIN_INFO_NEAR
1878: | TME_RECODE_CHAIN_INFO_FAR
1879: | TME_RECODE_CHAIN_INFO_JUMP
1880: | TME_RECODE_CHAIN_INFO_RETURN
1881: | TME_RECODE_CHAIN_INFO_CALL);
1882: chain_info++) {
1883: switch (chain_info
1884: & (TME_RECODE_CHAIN_INFO_JUMP
1885: | TME_RECODE_CHAIN_INFO_RETURN
1886: | TME_RECODE_CHAIN_INFO_CALL)) {
1887: case TME_RECODE_CHAIN_INFO_JUMP: s = "jump"; break;
1888: case TME_RECODE_CHAIN_INFO_RETURN:
1889: s = "return";
1890: if ((chain_info & TME_RECODE_CHAIN_INFO_FAR) == 0) {
1891: continue;
1892: }
1893: break;
1894: case TME_RECODE_CHAIN_INFO_CALL: s = "call"; break;
1895: default: continue;
1896: }
1897: insns_thunk = TME_RECODE_X86_CHAIN_SUBS(chain_thunk, chain_info);
1898: printf(" x86 chain %s %s %s subs: x/10i %p\n",
1899: s,
1900: (chain_info & TME_RECODE_CHAIN_INFO_FAR
1901: ? "far"
1902: : "near"),
1903: (chain_info & TME_RECODE_CHAIN_INFO_CONDITIONAL
1904: ? "conditional"
1905: : "unconditional"),
1906: tme_recode_thunk_off_to_pointer(ic,
1907: insns_thunk,
1908: void (*)(void)));
1909: }
1910: insns_thunk
1911: = TME_RECODE_X86_CHAIN_FIXUP_TARGET_ALTERNATE(ic,
1912: (TME_RECODE_CHAIN_INFO_JUMP
1913: + TME_RECODE_CHAIN_INFO_ALTERNATE_NEAR));
1914: printf(" x86 chain fixup chain jump alternate near: x/10i %p\n",
1915: tme_recode_thunk_off_to_pointer(ic,
1916: insns_thunk,
1917: void (*)(void)));
1918: insns_thunk
1919: = TME_RECODE_X86_CHAIN_FIXUP_TARGET(ic,
1920: (TME_RECODE_CHAIN_INFO_JUMP
1921: + TME_RECODE_CHAIN_INFO_NEAR));
1922: printf(" x86 chain fixup chain jump near, chain return alternate near: x/10i %p\n",
1923: tme_recode_thunk_off_to_pointer(ic,
1924: insns_thunk,
1925: void (*)(void)));
1926: insns_thunk
1927: = TME_RECODE_X86_CHAIN_FIXUP_TARGET_ALTERNATE(ic,
1928: (TME_RECODE_CHAIN_INFO_JUMP
1929: + TME_RECODE_CHAIN_INFO_ALTERNATE_FAR));
1930: printf(" x86 chain fixup chain jump alternate far: x/3i %p\n",
1931: tme_recode_thunk_off_to_pointer(ic,
1932: insns_thunk,
1933: void (*)(void)));
1934: insns_thunk
1935: = TME_RECODE_X86_CHAIN_FIXUP_TARGET(ic,
1936: (TME_RECODE_CHAIN_INFO_JUMP
1937: + TME_RECODE_CHAIN_INFO_FAR));
1938: printf(" x86 chain fixup chain jump far: x/2i %p\n",
1939: tme_recode_thunk_off_to_pointer(ic,
1940: insns_thunk,
1941: void (*)(void)));
1942: insns_thunk
1943: = TME_RECODE_X86_CHAIN_FIXUP_TARGET(ic,
1944: (TME_RECODE_CHAIN_INFO_CALL
1945: + TME_RECODE_CHAIN_INFO_FAR));
1946: printf(" x86 chain fixup chain call far: x/2i %p\n",
1947: tme_recode_thunk_off_to_pointer(ic,
1948: insns_thunk,
1949: void (*)(void)));
1950: insns_thunk
1951: = TME_RECODE_X86_CHAIN_FIXUP_TARGET(ic,
1952: (TME_RECODE_CHAIN_INFO_CALL
1953: + TME_RECODE_CHAIN_INFO_NEAR));
1954: printf(" x86 chain fixup chain call near: x/2i %p\n",
1955: tme_recode_thunk_off_to_pointer(ic,
1956: insns_thunk,
1957: void (*)(void)));
1958: }
1959:
1960: #endif /* TME_RECODE_DEBUG */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.