Return to rc-x86-flags.c CVS log

Up to [The Machine Emulator] / tme / libtme / host / x86

Annotation of tme/libtme/host/x86/rc-x86-flags.c, revision 1.1

1.1     ! root        1: /* $Id: rc-x86-flags.c,v 1.5 2010/02/15 22:21:48 fredette Exp $ */
        !             2: 
        !             3: /* libtme/host/x86/rc-x86-flags.c - x86 host recode flags support: */
        !             4: 
        !             5: /*
        !             6:  * Copyright (c) 2007 Matt Fredette
        !             7:  * All rights reserved.
        !             8:  *
        !             9:  * Redistribution and use in source and binary forms, with or without
        !            10:  * modification, are permitted provided that the following conditions
        !            11:  * are met:
        !            12:  * 1. Redistributions of source code must retain the above copyright
        !            13:  *    notice, this list of conditions and the following disclaimer.
        !            14:  * 2. Redistributions in binary form must reproduce the above copyright
        !            15:  *    notice, this list of conditions and the following disclaimer in the
        !            16:  *    documentation and/or other materials provided with the distribution.
        !            17:  * 3. All advertising materials mentioning features or use of this software
        !            18:  *    must display the following acknowledgement:
        !            19:  *      This product includes software developed by Matt Fredette.
        !            20:  * 4. The name of the author may not be used to endorse or promote products
        !            21:  *    derived from this software without specific prior written permission.
        !            22:  *
        !            23:  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
        !            24:  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
        !            25:  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
        !            26:  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
        !            27:  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
        !            28:  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
        !            29:  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
        !            30:  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
        !            31:  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
        !            32:  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
        !            33:  * POSSIBILITY OF SUCH DAMAGE.
        !            34:  */
        !            35: 
        !            36: _TME_RCSID("$Id: rc-x86-flags.c,v 1.5 2010/02/15 22:21:48 fredette Exp $");
        !            37: 
        !            38: /* this gives one of the x86 registers that can be used by an x86
        !            39:    flags thunk: */
        !            40: /* an x86-64 flags thunk that needs to call a guest function must use
        !            41:    the 9, 10, and 11 registers.  on ia32 and on x86-64 otherwise, we
        !            42:    must use the a, c, and d registers: */
        !            43: #if (TME_RECODE_X86_REG_A + 1) != TME_RECODE_X86_REG_C || (TME_RECODE_X86_REG_C + 1) != TME_RECODE_X86_REG_D
        !            44: #error "TME_RECODE_X86_REG_ values changed"
        !            45: #endif
        !            46: #define _TME_RECODE_X86_FLAGS_REG(flags_group, flags_reg_index)                \
        !            47:   (((TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32                                \
        !            48:      || (flags_group)->tme_recode_flags_group_guest_func == NULL)      \
        !            49:     ? TME_RECODE_X86_REG_A                                             \
        !            50:     : TME_RECODE_X86_REG_N(9))                                         \
        !            51:    + (flags_reg_index))
        !            52: #define _TME_RECODE_X86_FLAGS_REG_COUNT   (3)
        !            53: 
        !            54: /* this adds a one-byte-opcode ModRM instruction to a thunk, that
        !            55:    addresses the stacked host flags of a particular size: */
        !            56: static tme_uint8_t *
        !            57: _tme_recode_x86_flags_host_flags(const struct tme_recode_flags_group *flags_group,
        !            58:                                 tme_uint8_t *thunk_bytes,
        !            59:                                 tme_uint32_t larger_sizes,
        !            60:                                 tme_uint8_t rex,
        !            61:                                 tme_uint16_t opcode_opreg)
        !            62: {
        !            63:   unsigned long disp;
        !            64: 
        !            65:   /* the stacked host flags are always pushed in size order, largest
        !            66:      last.  the least significant bit of larger_sizes must be set, and
        !            67:      corresponds to the size of the flags that we want.  the remaining
        !            68:      set bits are for the larger flag sizes that we must skip over.
        !            69:      calculate the displacement to the flags we want, starting from
        !            70:      the largest: */
        !            71:   assert (larger_sizes & 1);
        !            72:   larger_sizes -= 1;
        !            73:   disp = 0;
        !            74:   for (; larger_sizes != 0; larger_sizes &= (larger_sizes - 1)) {
        !            75:     disp += TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8);
        !            76:   }
        !            77: 
        !            78:   /* if there is a guest function: */
        !            79:   if (flags_group->tme_recode_flags_group_guest_func != NULL) {
        !            80: 
        !            81:     /* there may be arguments for the guest function stacked before
        !            82:        the host flags.  on ia32, all of the guest function arguments
        !            83:        are stacked.  on x86-64, the destination operand argument is
        !            84:        stacked, but only when double-host-size guests are
        !            85:        supported: */
        !            86:     disp
        !            87:       += (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32
        !            88:          ? (sizeof(struct tme_ic *)
        !            89:             + sizeof(tme_recode_uguest_t)
        !            90:             + sizeof(tme_recode_uguest_t)
        !            91:             + sizeof(tme_recode_uguest_t))
        !            92:          : TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_HOST
        !            93:          ? sizeof(tme_recode_uguest_t)
        !            94:          : 0);
        !            95:   }
        !            96: 
        !            97:   /* emit the %sp-relative reference: */
        !            98:   if (rex != 0) {
        !            99:     *(thunk_bytes++) = rex;
        !           100:   }
        !           101:   *((tme_uint16_t *) &thunk_bytes[0])
        !           102:     = (opcode_opreg
        !           103:        | (TME_RECODE_X86_MOD_OPREG_RM((disp == 0
        !           104:                                       ? TME_RECODE_X86_MOD_RM_EA(TME_RECODE_X86_EA_BASE_SIB)
        !           105:                                       : TME_RECODE_X86_MOD_RM_EA_DISP8(TME_RECODE_X86_EA_BASE_SIB)),
        !           106:                                      0) << 8));
        !           107:   thunk_bytes[2] = TME_RECODE_X86_SIB(TME_RECODE_X86_REG_SP, TME_RECODE_X86_SIB_INDEX_NONE, 1);
        !           108:   thunk_bytes += 3;
        !           109:   if (disp != 0) {
        !           110:     assert (disp < 0x80);
        !           111:     *(thunk_bytes++) = disp;
        !           112:   }
        !           113: 
        !           114:   /* done: */
        !           115:   return (thunk_bytes);
        !           116: }
        !           117: 
        !           118: /* this emits instructions to shift a destination register by some
        !           119:    count, and then add in a register and a constant.  all of these
        !           120:    three operations are optional: */
        !           121: static tme_uint8_t *
        !           122: _tme_recode_x86_flags_shift_add_add(const struct tme_recode_flags_group *flags_group,
        !           123:                                    tme_uint8_t *thunk_bytes,
        !           124:                                    unsigned int size,
        !           125:                                    unsigned int flags_reg_index_dest,
        !           126:                                    unsigned int dest_shift,
        !           127:                                    unsigned int flags_reg_index_addend,
        !           128:                                    tme_recode_uguest_t constant)
        !           129: {
        !           130:   unsigned int reg_x86_dst;
        !           131:   tme_uint8_t rex;
        !           132:   unsigned int reg_x86_base;
        !           133:   unsigned int mod_rm;
        !           134:   unsigned int size_constant;
        !           135: 
        !           136:   /* to keep things simple and instruction size down, we promote all
        !           137:      operation sizes to at least 32 bits: */
        !           138:   size = TME_MAX(size, TME_RECODE_SIZE_32);
        !           139: 
        !           140:   /* get the destination register: */
        !           141:   reg_x86_dst = _TME_RECODE_X86_FLAGS_REG(flags_group, flags_reg_index_dest);
        !           142: 
        !           143:   /* if there is a destination shift, and it's more than three (more
        !           144:      than a factor of eight) or if there is no constant and no addend
        !           145:      register: */
        !           146:   if (dest_shift != 0
        !           147:       && (dest_shift > TME_RECODE_SIZE_8
        !           148:          || (constant == 0
        !           149:              && flags_reg_index_addend >= _TME_RECODE_X86_FLAGS_REG_COUNT))) {
        !           150: 
        !           151:     /* do all of the shifting with an shl instruction: */
        !           152:     rex = TME_RECODE_X86_REX_R(size, reg_x86_dst);
        !           153:     if (rex != 0) {
        !           154:       *(thunk_bytes++) = rex;
        !           155:     }
        !           156:     thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP2_Ib_Ev;
        !           157:     thunk_bytes[1]
        !           158:       = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(reg_x86_dst),
        !           159:                                    TME_RECODE_X86_OPCODE_GRP2_SHL);
        !           160:     thunk_bytes[2] = dest_shift;
        !           161:     thunk_bytes += 3;
        !           162: 
        !           163:     /* we don't need to shift the destination any more: */
        !           164:     dest_shift = 0;
        !           165:   }
        !           166: 
        !           167:   /* make the rex prefix for the lea.  if there is an addend register,
        !           168:      that is the base register, otherwise if there is no destination
        !           169:      shift then the destination register is also the base register,
        !           170:      otherwise there is no base register.  if there is a destination
        !           171:      shift, the destination register is also the index register,
        !           172:      otherwise there is no index register: */
        !           173:   rex
        !           174:     = (TME_RECODE_X86_REX_R(size, reg_x86_dst)
        !           175:        | (flags_reg_index_addend < _TME_RECODE_X86_FLAGS_REG_COUNT
        !           176:          ? TME_RECODE_X86_REX_B(0, _TME_RECODE_X86_FLAGS_REG(flags_group, flags_reg_index_addend))
        !           177:          : dest_shift == 0
        !           178:          ? TME_RECODE_X86_REX_B(0, reg_x86_dst)
        !           179:          : 0)
        !           180:        | (dest_shift != 0
        !           181:          ? TME_RECODE_X86_REX_X(reg_x86_dst)
        !           182:          : 0));
        !           183: 
        !           184:   /* if there is an addend register or a destination shift, the R/M
        !           185:      base register indicates that there is a scale-index-base byte,
        !           186:      otherwise the destination register is the R/M base register: */
        !           187:   reg_x86_base
        !           188:     = ((flags_reg_index_addend < _TME_RECODE_X86_FLAGS_REG_COUNT
        !           189:        || dest_shift != 0)
        !           190:        ? TME_RECODE_X86_EA_BASE_SIB
        !           191:        : reg_x86_dst);
        !           192: 
        !           193:   /* if there is no constant, we don't need a displacement: */
        !           194:   if (constant == 0) {
        !           195:     mod_rm = TME_RECODE_X86_MOD_RM_EA(reg_x86_base);
        !           196:     size_constant = 0;
        !           197: 
        !           198:     /* if there is no scale-index-base byte, we don't need to emit the
        !           199:        lea instruction at all: */
        !           200:     if (reg_x86_base != TME_RECODE_X86_EA_BASE_SIB) {
        !           201:       return (thunk_bytes);
        !           202:     }
        !           203:   }
        !           204: 
        !           205:   /* otherwise, if the constant fits into a sign-extended eight bits,
        !           206:      we can use that displacement form: */
        !           207:   else if (constant < 0x7f) {
        !           208:     mod_rm = TME_RECODE_X86_MOD_RM_EA_DISP8(reg_x86_base);
        !           209:     size_constant = sizeof(tme_uint8_t);
        !           210:   }
        !           211: 
        !           212:   /* otherwise, we must use the 32-bit displacement form: */
        !           213:   else {
        !           214: 
        !           215:     /* the constant must fit into 32 bits, unless the operation size is
        !           216:        64 bits, in which case the constant must fit into 31 bits because
        !           217:        the 32-bit displacement will be sign-extended to 64 bits: */
        !           218:     assert ((constant >> 31) < (1 + (size <= TME_RECODE_SIZE_32)));
        !           219:     mod_rm = TME_RECODE_X86_MOD_RM_EA_DISP32(reg_x86_base);
        !           220:     size_constant = sizeof(tme_uint32_t);
        !           221:   }
        !           222: 
        !           223:   /* emit the lea instruction: */
        !           224:   if (rex != 0) {
        !           225:     *(thunk_bytes++) = rex;
        !           226:   }
        !           227:   thunk_bytes[0] = TME_RECODE_X86_OPCODE_LEA;
        !           228:   thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(mod_rm, TME_RECODE_X86_REG(reg_x86_dst));
        !           229:   thunk_bytes += 2;
        !           230:   if (reg_x86_base == TME_RECODE_X86_EA_BASE_SIB) {
        !           231:     *(thunk_bytes++)
        !           232:       = TME_RECODE_X86_SIB((flags_reg_index_addend < _TME_RECODE_X86_FLAGS_REG_COUNT
        !           233:                            ? _TME_RECODE_X86_FLAGS_REG(flags_group, flags_reg_index_addend)
        !           234:                            : TME_RECODE_X86_SIB_BASE_NONE),
        !           235:                           reg_x86_dst,
        !           236:                           (1 << dest_shift));
        !           237:   }
        !           238:   memcpy(thunk_bytes, &constant, size_constant);
        !           239:   thunk_bytes += size_constant;
        !           240: 
        !           241:   /* done: */
        !           242:   return (thunk_bytes);
        !           243: }
        !           244: 
        !           245: /* this updates one host-sized part of the guest flags: */
        !           246: static tme_uint8_t *
        !           247: _tme_recode_x86_flags_update(const struct tme_recode_flags_group *flags_group,
        !           248:                             tme_uint8_t *thunk_bytes,
        !           249:                             tme_recode_uguest_t flags_mask_update,
        !           250:                             unsigned int flags_reg_index_accum,
        !           251:                             tme_recode_uguest_t flag_accum_last,
        !           252:                             unsigned int flags_reg_index_tmp)
        !           253: {
        !           254:   tme_recode_uguest_t flags_defined_host;
        !           255:   tme_recode_uguest_t flags_set;
        !           256:   tme_recode_uguest_t flags_clear;
        !           257:   tme_recode_uguest_t flags_constant;
        !           258:   unsigned int byte_shift;
        !           259:   unsigned long disp;
        !           260:   unsigned int size_flags_defined;
        !           261:   unsigned int size_constant;
        !           262:   unsigned int binop;
        !           263:   unsigned int reg_x86;
        !           264:   tme_uint8_t rex;
        !           265:   
        !           266:   /* get the mask of all defined flags that we are providing, that we
        !           267:      are updating now.  if this is zero, return now: */
        !           268:   flags_defined_host = tme_recode_flags_group_flags_defined_host(flags_group, TME_RECODE_COND_UNDEF);
        !           269:   flags_defined_host &= flags_mask_update;
        !           270:   if (flags_defined_host == 0) {
        !           271:     return (thunk_bytes);
        !           272:   }
        !           273: 
        !           274:   /* get the masks of flags that are always cleared or set, that we
        !           275:      are updating now.  either or both of these may be zero: */
        !           276:   flags_clear = tme_recode_flags_group_flags_defined_host(flags_group, TME_RECODE_COND_FALSE);
        !           277:   flags_clear &= flags_mask_update;
        !           278:   flags_set = tme_recode_flags_group_flags_defined_host(flags_group, TME_RECODE_COND_MOD_NOT | TME_RECODE_COND_FALSE);
        !           279:   flags_set &= flags_mask_update;
        !           280: 
        !           281:   /* we only update whole bytes.  shift whole bytes of zero bits off
        !           282:      of all guest flags masks, and get the displacement within the
        !           283:      guest struct tme_ic of the first byte we are updating: */
        !           284:   byte_shift = _tme_recode_x86_ffs(flags_defined_host) & (((unsigned int) 0) - 8);
        !           285:   flags_defined_host >>= byte_shift;
        !           286:   flags_clear >>= byte_shift;
        !           287:   flags_set >>= byte_shift;
        !           288:   flag_accum_last >>= byte_shift;
        !           289:   disp
        !           290:     = (tme_recode_flags_reg_offset(flags_group->tme_recode_flags_group_flags_reg_size,
        !           291:                                   flags_group->tme_recode_flags_group_flags_reg)
        !           292:        + (((flags_mask_update & 1) == 0
        !           293:           ? TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8)
        !           294:           : 0)
        !           295:          + (byte_shift / 8)));
        !           296: 
        !           297:   /* get the log base two of the number of bytes in the guest flags
        !           298:      register that we are updating: */
        !           299:   size_flags_defined
        !           300:     = ((flags_defined_host <= 0xff)
        !           301:        ? TME_RECODE_SIZE_8
        !           302:        : (flags_defined_host <= 0xffff)
        !           303:        ? TME_RECODE_SIZE_16
        !           304:        : (TME_RECODE_SIZE_32
        !           305:          + ((flags_defined_host >> 31) > 1)));
        !           306: 
        !           307:   /* to keep things simple (and to avoid updating eight bit register
        !           308:      parts, which may decrease performance), we always use at least
        !           309:      32-bit immediates with register operations: */
        !           310:   size_constant = TME_MAX(size_flags_defined, TME_RECODE_SIZE_32);
        !           311: 
        !           312:   /* if the mask of all defined guest flags that we are providing,
        !           313:      that we are updating now, is all flags that are always cleared,
        !           314:      or all flags that are always set: */
        !           315:   if (flags_defined_host == flags_clear
        !           316:       || flags_defined_host == flags_set) {
        !           317: 
        !           318:     /* get the constant and binop for clearing or setting flags: */
        !           319:     if (flags_defined_host == flags_clear) {
        !           320:       flags_constant = ~flags_clear;
        !           321:       binop = TME_RECODE_X86_OPCODE_BINOP_AND;
        !           322:     }
        !           323:     else {
        !           324:       flags_constant = flags_set;
        !           325:       binop = TME_RECODE_X86_OPCODE_BINOP_OR;
        !           326:     }
        !           327: 
        !           328:     /* if this is a 64-bit operation, we have to load the constant
        !           329:        into the temporary register and then do the binop from the
        !           330:        temporary register to memory, since there aren't binop
        !           331:        instructions with 64-bit immediates: */
        !           332:     if (size_flags_defined > TME_RECODE_SIZE_32) {
        !           333: 
        !           334:       /* load the temporary register with the 64-bit immediate: */
        !           335:       reg_x86 = _TME_RECODE_X86_FLAGS_REG(flags_group, flags_reg_index_tmp);
        !           336:       thunk_bytes[0] = TME_RECODE_X86_REX_B(TME_RECODE_SIZE_HOST, reg_x86);
        !           337:       thunk_bytes[1] = TME_RECODE_X86_OPCODE_MOV_Iv_Gv(reg_x86);
        !           338:       memcpy(&thunk_bytes[2], &flags_constant, TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8));
        !           339:       thunk_bytes += 1 + 1 + TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8);
        !           340: 
        !           341:       /* update the guest flags: */
        !           342:       thunk_bytes[0]
        !           343:        = (TME_RECODE_X86_REX_B(0, TME_RECODE_X86_REG_IC)
        !           344:           | TME_RECODE_X86_REX_R(TME_RECODE_SIZE_HOST, reg_x86));
        !           345:       thunk_bytes[1] = (binop + TME_RECODE_X86_OPCODE_BINOP_Gv_Ev);
        !           346:       thunk_bytes
        !           347:        = _tme_recode_x86_emit_ic_modrm(thunk_bytes + 2,
        !           348:                                        disp,
        !           349:                                        reg_x86);
        !           350: 
        !           351:       /* done: */
        !           352:       return (thunk_bytes);
        !           353:     }
        !           354: 
        !           355:     /* otherwise, emit the binop from the constant to memory: */
        !           356:     if (size_flags_defined == TME_RECODE_SIZE_16) {
        !           357:       *(thunk_bytes++) = TME_RECODE_X86_PREFIX_OPSIZ;
        !           358:     }
        !           359: #if TME_RECODE_X86_REG_IC >= 8
        !           360: #error "TME_RECODE_X86_REG_IC changed"
        !           361: #endif
        !           362:     *(thunk_bytes++)
        !           363:       = (size_flags_defined == TME_RECODE_SIZE_8
        !           364:         ? TME_RECODE_X86_OPCODE_GRP1_Ib_Eb
        !           365:         : TME_RECODE_X86_OPCODE_GRP1_Iz_Ev);
        !           366:     thunk_bytes
        !           367:       = _tme_recode_x86_emit_ic_modrm(thunk_bytes,
        !           368:                                      disp,
        !           369:                                      TME_RECODE_X86_OPCODE_GRP1_BINOP(binop));
        !           370:     memcpy(thunk_bytes,
        !           371:           &flags_constant,
        !           372:           TME_BIT(size_flags_defined - TME_RECODE_SIZE_8));
        !           373:     thunk_bytes += TME_BIT(size_flags_defined - TME_RECODE_SIZE_8);
        !           374: 
        !           375:     /* done: */
        !           376:     return (thunk_bytes);
        !           377:   }
        !           378: 
        !           379:   /* if we're updating all of the bits in a tme_uint8_t, tme_uint16_t,
        !           380:      tme_uint32_t, or tme_uint64_t and we're not providing any
        !           381:      variable guest flags, or if this is a 64-bit operation and the
        !           382:      flags we're setting don't fit into 31 bits, we have to get the
        !           383:      always-set flags into the register that (would have held any /
        !           384:      does hold the) variable guest flags.  in the first case, this is
        !           385:      an optimization, since the guest flags don't have to be read at
        !           386:      all, and in the second case this is essential since an lea
        !           387:      displacement is a sign-extended 32 bits: */
        !           388:   if ((flags_defined_host == ((((unsigned long) 2) << (TME_BIT(size_flags_defined) - 1)) - 1)
        !           389:        && flag_accum_last == 0)
        !           390:       || (size_flags_defined > TME_RECODE_SIZE_32
        !           391:          && (flags_set >> 30) > 1)) {
        !           392: 
        !           393:     /* if there are any variable guest flags, load the always-set
        !           394:        flags into the temporary register, otherwise load them directly
        !           395:        into the register that would have held any variable guest
        !           396:        flags: */
        !           397:     reg_x86
        !           398:       = _TME_RECODE_X86_FLAGS_REG(flags_group,
        !           399:                                  (flag_accum_last != 0
        !           400:                                   ? flags_reg_index_tmp
        !           401:                                   : flags_reg_index_accum));
        !           402:     rex = TME_RECODE_X86_REX_B(size_constant, reg_x86);
        !           403:     if (rex != 0) {
        !           404:       *(thunk_bytes++) = rex;
        !           405:     }
        !           406:     thunk_bytes[0] = TME_RECODE_X86_OPCODE_MOV_Iv_Gv(reg_x86);
        !           407:     memcpy(&thunk_bytes[1], &flags_set, TME_BIT(size_constant - TME_RECODE_SIZE_8));
        !           408:     thunk_bytes += 1 + TME_BIT(size_constant - TME_RECODE_SIZE_8);
        !           409: 
        !           410:     /* if there are any variable guest flags, shift the accumulation
        !           411:        register as needed and add in the always-set flags in the
        !           412:        temporary register: */
        !           413:     if (flag_accum_last != 0) {
        !           414:       thunk_bytes
        !           415:        = _tme_recode_x86_flags_shift_add_add(flags_group,
        !           416:                                              thunk_bytes,
        !           417:                                              size_constant,
        !           418:                                              flags_reg_index_accum,
        !           419:                                              _tme_recode_x86_ffs(flag_accum_last),
        !           420:                                              flags_reg_index_tmp,
        !           421:                                              0);
        !           422:     }
        !           423: 
        !           424:     /* we just combined the always-set flags with any variable guest
        !           425:        flags.  from now on, there are no always-set flags and act like
        !           426:        there are variable guest flags: */
        !           427:     flag_accum_last = 1;
        !           428:     flags_set = 0;
        !           429:   }
        !           430: 
        !           431:   /* if we're updating all of the bits in a tme_uint8_t, tme_uint16_t,
        !           432:      tme_uint32_t, or tme_uint64_t: */
        !           433:   if (flags_defined_host == ((((unsigned long) 2) << (TME_BIT(size_flags_defined) - 1)) - 1)) {
        !           434: 
        !           435:     /* there must be some variable guest flags: */
        !           436:     assert (flag_accum_last != 0);
        !           437: 
        !           438:     /* we don't have to read the guest flags at all to preserve any
        !           439:        bits, so we don't have to add the temporary register to the
        !           440:        variable guest flags below: */
        !           441:     flags_reg_index_tmp = _TME_RECODE_X86_FLAGS_REG_COUNT;
        !           442:   }
        !           443: 
        !           444:   /* otherwise, we're not updating all of the bits in a tme_uint8_t,
        !           445:      tme_uint16_t, tme_uint32_t, or tme_uint64_t.  we have to read the
        !           446:      guest flags, mask off the flags that we are updating, and
        !           447:      preserve the rest: */
        !           448:   else {
        !           449: 
        !           450:     /* if there are any variable guest flags, load the inverse of the
        !           451:        mask of the guest flags that we are updating into the temporary
        !           452:        register, otherwise load it directly into the register that
        !           453:        would have held any variable guest flags: */
        !           454:     flags_clear = ~flags_defined_host;
        !           455:     reg_x86
        !           456:       = _TME_RECODE_X86_FLAGS_REG(flags_group,
        !           457:                                  (flag_accum_last != 0
        !           458:                                   ? flags_reg_index_tmp
        !           459:                                   : flags_reg_index_accum));
        !           460:     rex = TME_RECODE_X86_REX_B(size_constant, reg_x86);
        !           461:     if (rex != 0) {
        !           462:       *(thunk_bytes++) = rex;
        !           463:     }
        !           464:     thunk_bytes[0] = TME_RECODE_X86_OPCODE_MOV_Iv_Gv(reg_x86);
        !           465:     memcpy(&thunk_bytes[1], &flags_clear, TME_BIT(size_constant - TME_RECODE_SIZE_8));
        !           466:     thunk_bytes += 1 + TME_BIT(size_constant - TME_RECODE_SIZE_8);
        !           467: 
        !           468:     /* binary-and the part of the guest flags register that we are
        !           469:        updating into that same register: */
        !           470:     if (size_flags_defined == TME_RECODE_SIZE_16) {
        !           471:       *(thunk_bytes++) = TME_RECODE_X86_PREFIX_OPSIZ;
        !           472:     }
        !           473:     rex = TME_RECODE_X86_REX_B(size_flags_defined, reg_x86);
        !           474:     if (rex != 0) {
        !           475:       *(thunk_bytes++) = rex;
        !           476:     }
        !           477:     *(thunk_bytes++)
        !           478:       = (TME_RECODE_X86_OPCODE_BINOP_AND
        !           479:         + (size_flags_defined == TME_RECODE_SIZE_8
        !           480:            ? TME_RECODE_X86_OPCODE_BINOP_Eb_Gb
        !           481:            : TME_RECODE_X86_OPCODE_BINOP_Ev_Gv));
        !           482:     thunk_bytes
        !           483:       = _tme_recode_x86_emit_ic_modrm(thunk_bytes,
        !           484:                                      disp,
        !           485:                                      reg_x86);
        !           486: 
        !           487:     /* if there are no variable guest flags, we're only providing
        !           488:        always-set and always-clear flags, which we now have in the
        !           489:        register that would have held variable guest flags.  we don't
        !           490:        have to add in the temporary register below: */
        !           491:     if (flag_accum_last == 0) {
        !           492:       flags_reg_index_tmp = _TME_RECODE_X86_FLAGS_REG_COUNT;
        !           493:     }
        !           494:   }
        !           495: 
        !           496:   /* shift the accumulation register as needed and add in any
        !           497:      temporary register and any always-set guest flags: */
        !           498:   thunk_bytes
        !           499:     = _tme_recode_x86_flags_shift_add_add(flags_group,
        !           500:                                          thunk_bytes,
        !           501:                                          size_flags_defined,
        !           502:                                          flags_reg_index_accum,
        !           503:                                          _tme_recode_x86_ffs(flag_accum_last),
        !           504:                                          flags_reg_index_tmp,
        !           505:                                          flags_set);
        !           506: 
        !           507:   /* update the guest flags register: */
        !           508:   reg_x86 = _TME_RECODE_X86_FLAGS_REG(flags_group, flags_reg_index_accum);
        !           509:   if (size_flags_defined == TME_RECODE_SIZE_16) {
        !           510:     *(thunk_bytes++) = TME_RECODE_X86_PREFIX_OPSIZ;
        !           511:   }
        !           512:   rex = (TME_RECODE_X86_REX_R(size_flags_defined, reg_x86)
        !           513:         + TME_RECODE_X86_REX_B(0, TME_RECODE_X86_REG_IC));
        !           514:   if (rex != 0) {
        !           515:     *(thunk_bytes++) = rex;
        !           516:   }
        !           517:   *(thunk_bytes++)
        !           518:     = (TME_RECODE_X86_OPCODE_BINOP_MOV
        !           519:        + (size_flags_defined == TME_RECODE_SIZE_8
        !           520:          ? TME_RECODE_X86_OPCODE_BINOP_Gb_Eb
        !           521:          : TME_RECODE_X86_OPCODE_BINOP_Gv_Ev));
        !           522:   thunk_bytes
        !           523:     = _tme_recode_x86_emit_ic_modrm(thunk_bytes,
        !           524:                                    disp,
        !           525:                                    reg_x86);
        !           526: 
        !           527:   /* done: */
        !           528:   return (thunk_bytes);
        !           529: }
        !           530: 
        !           531: /* this returns a new host flags thunk: */
        !           532: struct tme_recode_flags_thunk *
        !           533: tme_recode_host_flags_thunk_new(struct tme_recode_ic *ic,
        !           534:                                const struct tme_recode_flags_thunk *flags_thunk_template,
        !           535:                                const struct tme_recode_flags_group *flags_group)
        !           536: {
        !           537:   struct tme_recode_flags_thunk *flags_thunk;
        !           538:   tme_uint8_t *thunk_bytes;
        !           539:   tme_recode_uguest_t flags_defined_host;
        !           540:   tme_recode_uguest_t flags_variable_host;
        !           541:   tme_uint32_t host_sizes;
        !           542:   int stack_adjust;
        !           543:   tme_uint32_t host_sizes_other;
        !           544:   tme_recode_uguest_t flags_mask_0;
        !           545:   tme_recode_uguest_t flags_mask_1;
        !           546:   unsigned int size_accum;
        !           547:   const struct tme_recode_flag *flag;
        !           548:   tme_uint32_t cond;
        !           549:   tme_uint32_t cond_mods;
        !           550:   tme_recode_uguest_t factor;
        !           551:   unsigned int subfactor;
        !           552:   unsigned int flags_reg_index_next;
        !           553:   unsigned int flags_reg_index_accum;
        !           554:   unsigned int flags_reg_drain;
        !           555:   tme_uint8_t rex;
        !           556:   unsigned int x86_reg_0;
        !           557:   unsigned int x86_reg_1;
        !           558:   tme_recode_uguest_t flags_reg_index_to_flag[_TME_RECODE_X86_FLAGS_REG_COUNT];
        !           559:   tme_uint32_t host_size_loaded;
        !           560: 
        !           561:   /* start the new flags thunk: */
        !           562:   if (!tme_recode_host_thunk_start(ic)) {
        !           563:     abort();
        !           564:   }
        !           565:   flags_thunk = tme_new(struct tme_recode_flags_thunk, 1);
        !           566:   *flags_thunk = *flags_thunk_template;
        !           567:   flags_thunk->tme_recode_x86_flags_thunk_subs_main
        !           568:     = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next);
        !           569: 
        !           570:   /* start instructions: */
        !           571:   tme_recode_x86_insns_start(ic, thunk_bytes);
        !           572: 
        !           573:   /* get the mask of all guest flags that we are providing.  this may
        !           574:      be zero: */
        !           575:   flags_defined_host = tme_recode_flags_group_flags_defined_host(flags_group, TME_RECODE_COND_UNDEF);
        !           576: 
        !           577:   /* get the mask of variable guest flags that we are providing.  this
        !           578:      may be zero: */
        !           579:   flags_variable_host
        !           580:     = (flags_defined_host
        !           581:        & ~(tme_recode_flags_group_flags_defined_host(flags_group, TME_RECODE_COND_FALSE)
        !           582:           | tme_recode_flags_group_flags_defined_host(flags_group, TME_RECODE_COND_MOD_NOT | TME_RECODE_COND_FALSE)));
        !           583: 
        !           584:   /* get the mask of sizes for which we need to provide at least one
        !           585:      variable flag in the given flags group.  this may be
        !           586:      zero: */
        !           587:   host_sizes = tme_recode_flags_group_sizes(flags_group, flags_variable_host);
        !           588: 
        !           589:   /* it's easiest to assume that we always need the instruction-size
        !           590:      flags when we need any flags at all, since the instruction-size
        !           591:      operation always happens last no matter what, and leaves its
        !           592:      flags in the flags register: */
        !           593:   if (host_sizes != 0) {
        !           594:     host_sizes |= TME_BIT(flags_group->tme_recode_flags_group_insn_size);
        !           595:   }
        !           596: 
        !           597:   /* when we need the double-host-size flags, we also need the
        !           598:      host-size flags, because we need the host-size Z flag to complete
        !           599:      the double-host-size Z flag: */
        !           600:   if (host_sizes & TME_BIT(TME_RECODE_SIZE_HOST + 1)) {
        !           601:     host_sizes |= TME_BIT(TME_RECODE_SIZE_HOST);
        !           602:   }
        !           603: 
        !           604:   /* assume that we won't need to remove anything from the stack
        !           605:      before returning to the insn thunk: */
        !           606:   stack_adjust = 0;
        !           607: 
        !           608:   /* if there are flags from multiple sizes: */
        !           609:   if (host_sizes & (host_sizes - 1)) {
        !           610: 
        !           611:     /* the flags for the instruction size are in the flags register on
        !           612:        entry into the thunk, and the others are stacked in size order,
        !           613:        with the flags for the smallest size farthest away from the
        !           614:        stack pointer.  emit a pushf to save the instruction-size flags
        !           615:        on the stack: */
        !           616:     *(thunk_bytes++) = TME_RECODE_X86_OPCODE_PUSHF;
        !           617: 
        !           618:     /* before returning to the insn thunk, we will need to remove all
        !           619:        of these flags from the stack: */
        !           620:     for (host_sizes_other = host_sizes;
        !           621:         host_sizes_other != 0;
        !           622:         host_sizes_other &= (host_sizes_other - 1)) {
        !           623:       stack_adjust += TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8);
        !           624:     }
        !           625:   }
        !           626: 
        !           627:   /* assume that the opcode-specific flags subs won't need to
        !           628:      do any stack padding: */
        !           629:   flags_thunk->tme_recode_x86_flags_thunk_stack_padding = 0;
        !           630: 
        !           631:   /* if there is a guest function: */
        !           632:   flags_thunk->tme_recode_x86_flags_thunk_has_guest_func
        !           633:     = (flags_group->tme_recode_flags_group_guest_func != NULL);
        !           634:   if (flags_thunk->tme_recode_x86_flags_thunk_has_guest_func) {
        !           635: 
        !           636:     /* if this is an ia32 host: */
        !           637:     if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32) {
        !           638: 
        !           639:       /* push the destination operand for the guest function: */
        !           640:       if (TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_HOST) {
        !           641:        _tme_recode_x86_emit_reg_push(thunk_bytes, tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_DST + 1]);
        !           642:       }
        !           643:       _tme_recode_x86_emit_reg_push(thunk_bytes, tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_DST]);
        !           644: 
        !           645:       /* push the second source operand for the guest function: */
        !           646:       if (TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_HOST) {
        !           647:        _tme_recode_x86_emit_reg_push(thunk_bytes, tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC1 + 1]);
        !           648:       }
        !           649:       _tme_recode_x86_emit_reg_push(thunk_bytes, tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC1]);
        !           650: 
        !           651:       /* push the first source operand for the guest function: */
        !           652:       if (TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_HOST) {
        !           653:        _tme_recode_x86_emit_reg_push(thunk_bytes, tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC0 + 1]);
        !           654:       }
        !           655:       _tme_recode_x86_emit_reg_push(thunk_bytes, tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC0]);
        !           656: 
        !           657:       /* push the struct tme_ic * for the guest function: */
        !           658:       _tme_recode_x86_emit_reg_push(thunk_bytes, TME_RECODE_X86_REG_IC);
        !           659: 
        !           660:       /* before returning to the insn thunk, we will need to remove
        !           661:         all of these guest function arguments from the stack: */
        !           662:       stack_adjust
        !           663:        += (sizeof(struct tme_ic *)
        !           664:            + sizeof(tme_recode_uguest_t)
        !           665:            + sizeof(tme_recode_uguest_t)
        !           666:            + sizeof(tme_recode_uguest_t));
        !           667:     }
        !           668: 
        !           669:     /* otherwise, this is an x86-64 host: */
        !           670:     else {
        !           671: 
        !           672:       /* copy the struct tme_ic * for the guest function: */
        !           673:       _tme_recode_x86_emit_reg_copy(thunk_bytes,
        !           674:                                    TME_RECODE_X86_REG_IC,
        !           675:                                    TME_RECODE_X86_REG_DI);
        !           676: 
        !           677:       /* copy the (least-significant half of) the first source operand
        !           678:         for the guest function: */
        !           679:       _tme_recode_x86_emit_reg_copy(thunk_bytes,
        !           680:                                    tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC0 + 0],
        !           681:                                    TME_RECODE_X86_REG_SI);
        !           682: 
        !           683:       /* if double-host-size guests are supported: */
        !           684:       if (TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_HOST) {
        !           685: 
        !           686:        /* copy the most-significant half of the first source operand
        !           687:           for the guest function: */
        !           688:        _tme_recode_x86_emit_reg_copy(thunk_bytes,
        !           689:                                      tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC0 + 1],
        !           690:                                      TME_RECODE_X86_REG_D);
        !           691: 
        !           692:        /* copy the second source operand for the guest function: */
        !           693:        _tme_recode_x86_emit_reg_copy(thunk_bytes,
        !           694:                                      tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC1 + 0],
        !           695:                                      TME_RECODE_X86_REG_C);
        !           696:        _tme_recode_x86_emit_reg_copy(thunk_bytes,
        !           697:                                      tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC1 + 1],
        !           698:                                      TME_RECODE_X86_REG_N(8));
        !           699: 
        !           700:        /* push the destination operand for the guest function: */
        !           701:        _tme_recode_x86_emit_reg_push(thunk_bytes, tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_DST + 1]);
        !           702:        _tme_recode_x86_emit_reg_push(thunk_bytes, tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_DST + 0]);
        !           703: 
        !           704:        /* before returning to the insn thunk, we will need to remove
        !           705:           the destination operand argument from the stack: */
        !           706:        stack_adjust += sizeof(tme_recode_uguest_t);
        !           707: 
        !           708:        /* the x86-64 ABI requires that the stack pointer be 16-byte
        !           709:           aligned immediately before a call instruction.  inside an
        !           710:           insn thunk, the stack pointer is 16-byte aligned
        !           711:           immediately before the call to a subs, which means at the
        !           712:           beginning of the subs it is only 8-byte aligned (because of
        !           713:           the return address for the subs).
        !           714: 
        !           715:           if our stack won't be 16-byte aligned before we call the
        !           716:           guest function, the subs that chained to us will need to
        !           717:           pad the stack with one extra word to get it aligned: */
        !           718:        if ((TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8) /* the subs return address */
        !           719:             + stack_adjust)
        !           720:            % (TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8) * 2)) {
        !           721:          flags_thunk->tme_recode_x86_flags_thunk_stack_padding = TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8);
        !           722:        }
        !           723:       }
        !           724: 
        !           725:       /* otherwise, double-host-size guests are not supported: */
        !           726:       else {
        !           727: 
        !           728:        /* copy the second source operand for the guest function: */
        !           729:        _tme_recode_x86_emit_reg_copy(thunk_bytes,
        !           730:                                      tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC1],
        !           731:                                      TME_RECODE_X86_REG_D);
        !           732: 
        !           733:        /* copy the destination operand for the guest function: */
        !           734:        _tme_recode_x86_emit_reg_copy(thunk_bytes,
        !           735:                                      tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_DST],
        !           736:                                      TME_RECODE_X86_REG_C);
        !           737: 
        !           738:        /* NB that when double-host-size guests are not supported, we
        !           739:           will always chain to the guest function, so we don't have
        !           740:           to worry about the ABI stack alignment like we do above: */
        !           741:       }
        !           742:     }
        !           743: 
        !           744:     /* we will also have to remove any stack padding: */
        !           745:     stack_adjust += flags_thunk->tme_recode_x86_flags_thunk_stack_padding;
        !           746:   }
        !           747: 
        !           748:   /* if any double-host-size flag is needed: */
        !           749:   if (host_sizes & TME_BIT(TME_RECODE_SIZE_HOST + 1)) {
        !           750: 
        !           751:     /* the instruction-size flags we pushed above are incomplete
        !           752:        double-host-size flags, and there are also host-size flags on
        !           753:        the stack.  load the least-significant 32 bits of the host-size
        !           754:        flags from the stack into %eax: */
        !           755:     thunk_bytes
        !           756:       = _tme_recode_x86_flags_host_flags(flags_group,
        !           757:                                         thunk_bytes,
        !           758:                                         (host_sizes >> TME_RECODE_SIZE_HOST),
        !           759:                                         TME_RECODE_X86_REX_R(TME_RECODE_SIZE_32, TME_RECODE_X86_REG_A),
        !           760:                                         ((TME_RECODE_X86_OPCODE_BINOP_MOV
        !           761:                                           + TME_RECODE_X86_OPCODE_BINOP_Ev_Gv)
        !           762:                                          + (TME_RECODE_X86_MOD_OPREG_RM(0, 
        !           763:                                                                         TME_RECODE_X86_REG(TME_RECODE_X86_REG_A))
        !           764:                                             << 8)));
        !           765: 
        !           766:     /* or a mask of all-bits-one, minus the Z flag, into %eax: */
        !           767:     thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP1_Ib_Ev;
        !           768:     thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(TME_RECODE_X86_REG_A), TME_RECODE_X86_OPCODE_GRP1_BINOP(TME_RECODE_X86_OPCODE_BINOP_OR));
        !           769: #if TME_RECODE_X86_FLAG_Z >= 0x80
        !           770: #error "TME_RECODE_X86_FLAG_Z value changed"
        !           771: #endif
        !           772:     thunk_bytes[2] = ~TME_RECODE_X86_FLAG_Z;
        !           773:     thunk_bytes += 3;
        !           774: 
        !           775:     /* and %eax into the least-significant 32 bits of the
        !           776:        double-host-size flags on the stack: */
        !           777:     thunk_bytes
        !           778:       = _tme_recode_x86_flags_host_flags(flags_group,
        !           779:                                         thunk_bytes,
        !           780:                                         (host_sizes >> (TME_RECODE_SIZE_HOST + 1)),
        !           781:                                         TME_RECODE_X86_REX_R(TME_RECODE_SIZE_32, TME_RECODE_X86_REG_A),
        !           782:                                         ((TME_RECODE_X86_OPCODE_BINOP_AND
        !           783:                                           + TME_RECODE_X86_OPCODE_BINOP_Gv_Ev)
        !           784:                                          + (TME_RECODE_X86_MOD_OPREG_RM(0,
        !           785:                                                                         TME_RECODE_X86_REG(TME_RECODE_X86_REG_A))
        !           786:                                             << 8)));
        !           787: 
        !           788:     /* initially, no host flags are loaded: */
        !           789:     host_size_loaded = TME_RECODE_SIZE_1;
        !           790:   }
        !           791: 
        !           792:   /* otherwise, no double-host-size flag is needed: */
        !           793:   else {
        !           794: 
        !           795:     /* the flags from the instruction size are always in the flags
        !           796:        register on entry into the thunk: */
        !           797:     host_size_loaded = flags_group->tme_recode_flags_group_insn_size;
        !           798:   }
        !           799: 
        !           800:   /* to keep things simple, only whole bytes are updated in the guest
        !           801:      flags register.  with only host-size or smaller guests, this
        !           802:      isn't so important since one host register will always cover the
        !           803:      entire guest flags register, and most double-host-size guests
        !           804:      will have their flags concentrated in their flags register so
        !           805:      that one host flags register will also cover all of them, even when
        !           806:      only updating whole bytes.  rarely, a double-host-size guest may
        !           807:      have flags in so many bytes of its flags register that we have to
        !           808:      split it in two halves, updated separately.  when we do this,
        !           809:      flags_mask_1 is all-bits-one covering the most significant half
        !           810:      of the guest flags register, otherwise flags_mask_1 is zero: */
        !           811:   flags_mask_1
        !           812:     = ((TME_SHIFT(tme_recode_uguest_t,
        !           813:                  _tme_recode_x86_ffs_byte_shift(flags_defined_host),
        !           814:                  >>,
        !           815:                  TME_BIT(TME_RECODE_SIZE_HOST) - 1)
        !           816:        > 1)
        !           817:        * (0 - TME_SHIFT(tme_recode_uguest_t, 1, <<, TME_BIT(TME_RECODE_SIZE_HOST))));
        !           818:   flags_mask_0 = ~flags_mask_1;
        !           819: 
        !           820:   /* if we aren't providing any variable guest flags: */
        !           821:   if (flags_variable_host == 0) {
        !           822: 
        !           823:     /* we need to clear flags register zero, which will be used as the
        !           824:        accumulated register for one or two
        !           825:        _tme_recode_x86_flags_update() calls: */
        !           826:     flags_reg_index_next = 1;
        !           827:   }
        !           828: 
        !           829:   /* otherwise, if the whole-bytes span of defined flags in any
        !           830:      host-sized part of the guest flags register is bigger than eight
        !           831:      bits: */
        !           832:   else if ((_tme_recode_x86_ffs_byte_shift(flags_defined_host & flags_mask_0)
        !           833:            | _tme_recode_x86_ffs_byte_shift(flags_defined_host & flags_mask_1))
        !           834:           > 0xff) {
        !           835: 
        !           836:     /* we need to clear all flags registers because setcc only sets
        !           837:        the least significant byte of a register.  otherwise, the
        !           838:        garbage in the other bytes would get accumulated into the
        !           839:        results: */
        !           840:     flags_reg_index_next = _TME_RECODE_X86_FLAGS_REG_COUNT;
        !           841:   }
        !           842: 
        !           843:   /* otherwise, we are providing variable guest flags, and the
        !           844:      whole-bytes span of defined guest flags in the one or two
        !           845:      host-sized parts of the guest flags register fits into eight
        !           846:      bits: */
        !           847:   else {
        !           848: 
        !           849:     /* in this case, we don't need to clear any flags registers at
        !           850:        all.  instead, we rely on setcc to clear the least significant
        !           851:        eight bits of flags registers: */
        !           852:     flags_reg_index_next = 0;
        !           853:   }
        !           854: 
        !           855:   /* if there are any flags registers to clear: */
        !           856:   if (flags_reg_index_next > 0) {
        !           857: 
        !           858:     /* if there are host flags initially loaded: */
        !           859:     if (host_size_loaded != TME_RECODE_SIZE_1) {
        !           860: 
        !           861:       /* use only flags-preserving mov instructions: */
        !           862: 
        !           863:       /* mov an immediate zero into the first flags register: */
        !           864:       x86_reg_0 = _TME_RECODE_X86_FLAGS_REG(flags_group, 0);
        !           865:       rex = TME_RECODE_X86_REX_B(TME_RECODE_SIZE_32, x86_reg_0);
        !           866:       if (rex != 0) {
        !           867:        *(thunk_bytes++) = rex;
        !           868:       }
        !           869:       thunk_bytes[0] = TME_RECODE_X86_OPCODE_MOV_Iv_Gv(x86_reg_0);
        !           870:       *((tme_uint32_t *) &thunk_bytes[1]) = 0;
        !           871:       thunk_bytes += 1 + sizeof(tme_uint32_t);
        !           872: 
        !           873:       /* use register-to-register moves to copy that zero into any
        !           874:         other flags registers: */
        !           875:       for (flags_reg_index_accum = 1;
        !           876:           flags_reg_index_accum < flags_reg_index_next;
        !           877:           flags_reg_index_accum++) {
        !           878:        x86_reg_1 = _TME_RECODE_X86_FLAGS_REG(flags_group, flags_reg_index_accum);
        !           879:        rex
        !           880:          = (TME_RECODE_X86_REX_B(TME_RECODE_SIZE_32, x86_reg_0)
        !           881:             | TME_RECODE_X86_REX_R(TME_RECODE_SIZE_32, x86_reg_1));
        !           882:        if (rex != 0) {
        !           883:          *(thunk_bytes++) = rex;
        !           884:        }
        !           885:        thunk_bytes[0] = (TME_RECODE_X86_OPCODE_BINOP_MOV + TME_RECODE_X86_OPCODE_BINOP_Ev_Gv);
        !           886:        thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(x86_reg_0), TME_RECODE_X86_REG(x86_reg_1));
        !           887:        thunk_bytes += 1 + 1;
        !           888:       }
        !           889:     }
        !           890: 
        !           891:     /* otherwise, no host flags are initially loaded: */
        !           892:     else {
        !           893: 
        !           894:       /* use xor instructions to clear all of the registers: */
        !           895:       for (flags_reg_index_accum = 0;
        !           896:           flags_reg_index_accum < flags_reg_index_next;
        !           897:           flags_reg_index_accum++) {
        !           898:        x86_reg_0 = _TME_RECODE_X86_FLAGS_REG(flags_group, flags_reg_index_accum);
        !           899:        rex
        !           900:          = (TME_RECODE_X86_REX_B(TME_RECODE_SIZE_32, x86_reg_0)
        !           901:             | TME_RECODE_X86_REX_R(TME_RECODE_SIZE_32, x86_reg_0));
        !           902:        if (rex != 0) {
        !           903:          *(thunk_bytes++) = rex;
        !           904:        }
        !           905:        thunk_bytes[0] = (TME_RECODE_X86_OPCODE_BINOP_XOR + TME_RECODE_X86_OPCODE_BINOP_Ev_Gv);
        !           906:        thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(x86_reg_0), TME_RECODE_X86_REG(x86_reg_0));
        !           907:        thunk_bytes += 2;
        !           908:       }
        !           909:     }
        !           910:   }
        !           911: 
        !           912:   /* if the whole-bytes span of variable flags in any host-sized part
        !           913:      of the guest flags register is bigger than 32 bits, we must
        !           914:      accumulate using 64-bit math, otherwise we can use 32-bit
        !           915:      math: */
        !           916:   size_accum
        !           917:     = (TME_RECODE_SIZE_32
        !           918:        + (((_tme_recode_x86_ffs_byte_shift(flags_variable_host & flags_mask_0)
        !           919:            | _tme_recode_x86_ffs_byte_shift(flags_variable_host & flags_mask_1))
        !           920:           >> 31)
        !           921:          > 1));
        !           922: 
        !           923:   /* start accumulating the guest flags into the first flags register: */
        !           924:   flags_reg_index_accum = 0;
        !           925:   flags_reg_index_next = flags_reg_index_accum;
        !           926:   memset(flags_reg_index_to_flag, 0, sizeof(flags_reg_index_to_flag));
        !           927:   flags_reg_drain = _TME_RECODE_X86_FLAGS_REG_COUNT;
        !           928: 
        !           929:   /* loop over the flags: */
        !           930:   flag = flags_group->tme_recode_flags_group_flags;
        !           931:   for (;;) {
        !           932: 
        !           933:     /* if we're not at the end of the list of flags, but
        !           934:        this is not a variable flag we're providing, skip
        !           935:        it: */
        !           936:     if (flag->tme_recode_flag_flag != 0
        !           937:        && (flag->tme_recode_flag_flag & flags_variable_host) == 0) {
        !           938:       flag++;
        !           939:       continue;
        !           940:     }
        !           941: 
        !           942:     /* unless we haven't accumulated any flags yet: */
        !           943:     if (flags_reg_index_to_flag[flags_reg_index_accum] != 0) {
        !           944: 
        !           945:       /* advance to the next register: */
        !           946:       flags_reg_index_next++;
        !           947:       if (flags_reg_index_next >= _TME_RECODE_X86_FLAGS_REG_COUNT) {
        !           948:        flags_reg_index_next = 0;
        !           949:       }
        !           950: 
        !           951:       /* always skip flags register zero, and the flags register we're
        !           952:         accumulating into (which can be nonzero when we need to
        !           953:         accumulate in two registers): */
        !           954:       if (flags_reg_index_next == 0
        !           955:          || flags_reg_index_next == flags_reg_index_accum) {
        !           956:        continue;
        !           957:       }
        !           958:     }
        !           959: 
        !           960:     /* if this next register is busy with a previous flag: */
        !           961:     if (flags_reg_index_to_flag[flags_reg_index_next]) {
        !           962: 
        !           963:       /* if flags register zero has accumulated one or more flags from
        !           964:         the most significant half of a double-host-size guest flags
        !           965:         register, and the flag that is busy in the next register is
        !           966:         from the least significant half: */
        !           967:       if ((flags_reg_index_to_flag[flags_reg_index_accum] & flags_mask_1)
        !           968:          && (flags_reg_index_to_flag[flags_reg_index_next] & flags_mask_0)) {
        !           969: 
        !           970:        /* this next register, that we originally wanted to store
        !           971:           another single flag into, is now actually the second flags
        !           972:           register that we will accumulate flags into, and already
        !           973:           has its first flag in it.  we loop to select another next
        !           974:           register for the next flag: */
        !           975:        flags_reg_index_accum = flags_reg_index_next;
        !           976:        continue;
        !           977:       }
        !           978: 
        !           979:       /* otherwise, we need to shift the accumulating register up
        !           980:         and OR in the previous flag in this next register.  if the
        !           981:         factor between the last flag in the accumulating register
        !           982:         and this previous flag is less than or equal to eight, we
        !           983:         can do the shift and or in one lea instruction, otherwise
        !           984:         we need to shift the accumulating register by itself with
        !           985:         leas first: */
        !           986:       for (factor = flags_reg_index_to_flag[flags_reg_index_accum] / flags_reg_index_to_flag[flags_reg_index_next];
        !           987:           factor > 1;
        !           988:           factor /= 8) {
        !           989: 
        !           990:        /* get the factor to multiply the index register (the
        !           991:           accumulating register) by, and the base to add to that,
        !           992:           which is none if this won't be our last shift: */
        !           993:        if (factor > 8) {
        !           994:          subfactor = 8;
        !           995:          x86_reg_0 = TME_RECODE_X86_SIB_BASE_NONE;
        !           996:        }
        !           997:        else {
        !           998:          subfactor = factor;
        !           999:          x86_reg_0 = _TME_RECODE_X86_FLAGS_REG(flags_group, flags_reg_index_next);
        !          1000:        }
        !          1001: 
        !          1002:        /* emit the lea instruction to do the shift and add: */
        !          1003:        x86_reg_1 = _TME_RECODE_X86_FLAGS_REG(flags_group, flags_reg_index_accum);
        !          1004:        rex
        !          1005:          = (TME_RECODE_X86_REX_W(size_accum)
        !          1006:             | TME_RECODE_X86_REX_R(size_accum, x86_reg_1)
        !          1007:             | TME_RECODE_X86_REX_X(x86_reg_1)
        !          1008:             | TME_RECODE_X86_REX_B(size_accum, x86_reg_0));
        !          1009:        if (rex != 0) {
        !          1010:          *(thunk_bytes++) = rex;
        !          1011:        }
        !          1012:        thunk_bytes[0] = TME_RECODE_X86_OPCODE_LEA;
        !          1013:        thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_EA_BASE_SIB, TME_RECODE_X86_REG(x86_reg_1));
        !          1014:        thunk_bytes[2] = TME_RECODE_X86_SIB(x86_reg_0, x86_reg_1, subfactor);
        !          1015:        thunk_bytes += 3;
        !          1016:       }
        !          1017: 
        !          1018:       /* update the last flag accumulated: */
        !          1019:       flags_reg_index_to_flag[flags_reg_index_accum] = flags_reg_index_to_flag[flags_reg_index_next];
        !          1020:       
        !          1021:       /* this next register is no longer busy: */
        !          1022:       flags_reg_index_to_flag[flags_reg_index_next] = 0;
        !          1023:     }
        !          1024: 
        !          1025:     /* if there are no more flags: */
        !          1026:     if (flag->tme_recode_flag_flag == 0) {
        !          1027: 
        !          1028:       /* keep looping until we've accumulated all of the flags still
        !          1029:         in flags registers: */
        !          1030:       if (flags_reg_drain--) {
        !          1031:        continue;
        !          1032:       }
        !          1033: 
        !          1034:       /* otherwise, stop now: */
        !          1035:       break;
        !          1036:     }
        !          1037: 
        !          1038:     /* get the condition and remove any modifiers: */
        !          1039:     cond = flag->tme_recode_flag_cond;
        !          1040:     cond_mods = (cond & TME_RECODE_COND_MOD_NOT);
        !          1041:     cond -= cond_mods;
        !          1042: 
        !          1043:     /* if the x86 flags for this size aren't loaded: */
        !          1044:     assert (flag->tme_recode_flag_size >= TME_RECODE_SIZE_8);
        !          1045:     if (host_size_loaded == TME_RECODE_SIZE_1
        !          1046:        || (cond != TME_RECODE_COND_PE
        !          1047:            && host_size_loaded != flag->tme_recode_flag_size)) {
        !          1048: 
        !          1049:       /* push the flags for this size: */
        !          1050:       thunk_bytes
        !          1051:        = _tme_recode_x86_flags_host_flags(flags_group,
        !          1052:                                           thunk_bytes,
        !          1053:                                           (host_sizes >> flag->tme_recode_flag_size),
        !          1054:                                           0,
        !          1055:                                           (TME_RECODE_X86_OPCODE_GRP5
        !          1056:                                            + (TME_RECODE_X86_MOD_OPREG_RM(0,
        !          1057:                                                                           TME_RECODE_X86_OPCODE_GRP5_PUSH)
        !          1058:                                               << 8)));
        !          1059: 
        !          1060:       /* popf the flags for this size: */
        !          1061:       *(thunk_bytes++) = TME_RECODE_X86_OPCODE_POPF;
        !          1062: 
        !          1063:       /* the x86 flags for this size are now loaded: */
        !          1064:       host_size_loaded = flag->tme_recode_flag_size;
        !          1065:     }
        !          1066: 
        !          1067:     /* emit the setcc instruction for this condition: */
        !          1068:     switch (cond) {
        !          1069:     default: assert(FALSE);
        !          1070:     case TME_RECODE_COND_N: cond = TME_RECODE_X86_COND_S; break;
        !          1071:     case TME_RECODE_COND_C: cond = TME_RECODE_X86_COND_C; break;
        !          1072:     case TME_RECODE_COND_V: cond = TME_RECODE_X86_COND_O; break;
        !          1073:     case TME_RECODE_COND_Z: cond = TME_RECODE_X86_COND_Z; break;
        !          1074:     case TME_RECODE_COND_PE: cond = TME_RECODE_X86_COND_PE; break;
        !          1075:     case TME_RECODE_COND_BE: cond = TME_RECODE_X86_COND_BE; break;
        !          1076:     case TME_RECODE_COND_LE: cond = TME_RECODE_X86_COND_LE; break;
        !          1077:     case TME_RECODE_COND_L: cond = TME_RECODE_X86_COND_L; break;
        !          1078:     }
        !          1079:     if (cond_mods & TME_RECODE_COND_MOD_NOT) {
        !          1080:       cond += TME_RECODE_X86_COND_NOT;
        !          1081:     }
        !          1082:     x86_reg_0 = _TME_RECODE_X86_FLAGS_REG(flags_group, flags_reg_index_next);
        !          1083:     rex = TME_RECODE_X86_REX_B(TME_RECODE_SIZE_8, x86_reg_0);
        !          1084:     if (rex != 0) {
        !          1085:       *(thunk_bytes++) = rex;
        !          1086:     }
        !          1087:     thunk_bytes[0] = TME_RECODE_X86_OPCODE_ESC_0F;
        !          1088:     thunk_bytes[1] = TME_RECODE_X86_OPCODE0F_SETCC(cond);
        !          1089:     thunk_bytes[2] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(x86_reg_0), 0);
        !          1090:     thunk_bytes += 3;
        !          1091: 
        !          1092:     /* this register now contains this flag: */
        !          1093:     flags_reg_index_to_flag[flags_reg_index_next] = flag->tme_recode_flag_flag;
        !          1094: 
        !          1095:     /* continue with the next flag: */
        !          1096:     flag++;
        !          1097:   }
        !          1098: 
        !          1099:   /* we always try to update at least one part of the guest flags
        !          1100:      register.  if we are providing variable flags, and they're all in
        !          1101:      the most-significant host-sized half of a double-host-size guest
        !          1102:      flags register, update that half of the guest flags register now,
        !          1103:      otherwise update the least-significant half (which may mean the
        !          1104:      whole guest flags register): */
        !          1105:   thunk_bytes
        !          1106:     = _tme_recode_x86_flags_update(flags_group,
        !          1107:                                   thunk_bytes,
        !          1108:                                   ((flags_reg_index_to_flag[flags_reg_index_accum]
        !          1109:                                     & flags_mask_1)
        !          1110:                                    ? flags_mask_1
        !          1111:                                    : flags_mask_0),
        !          1112:                                   flags_reg_index_accum,
        !          1113:                                   flags_reg_index_to_flag[flags_reg_index_accum],
        !          1114:                                   flags_reg_index_next);
        !          1115: 
        !          1116:   /* if we are providing variable flags and this is a double-host-size
        !          1117:      guest flags register and there are variable flags in both halves,
        !          1118:      we've only updated the least significant half above.  or, if all
        !          1119:      of the variable flags are in the most-significant half, we've
        !          1120:      only updated the most-significant half above.  we need to update
        !          1121:      the other half of the guest flags register: */
        !          1122:   if (flags_reg_index_accum != 0
        !          1123:       || (flags_reg_index_to_flag[flags_reg_index_accum]
        !          1124:          & flags_mask_1)) {
        !          1125: 
        !          1126:     /* we've already updated the half of the guest flags register that
        !          1127:        was being accumulated in flags_reg_index_accum, so we can now free
        !          1128:        it up: */
        !          1129:     flags_reg_index_to_flag[flags_reg_index_accum] = 0;
        !          1130: 
        !          1131:     /* if we haven't already updated the half of the guest flags
        !          1132:        variable that was being accumulated in flags register zero, we'll
        !          1133:        do it now: */
        !          1134:     flags_reg_index_accum = 0;
        !          1135: 
        !          1136:     /* update the other half of the guest flags register: */
        !          1137:     thunk_bytes
        !          1138:       = _tme_recode_x86_flags_update(flags_group,
        !          1139:                                     thunk_bytes,
        !          1140:                                     ((flags_reg_index_to_flag[flags_reg_index_accum]
        !          1141:                                       & flags_mask_1)
        !          1142:                                      ? flags_mask_1
        !          1143:                                      : flags_mask_0),
        !          1144:                                     flags_reg_index_accum,
        !          1145:                                     flags_reg_index_to_flag[flags_reg_index_accum],
        !          1146:                                     flags_reg_index_next);
        !          1147:   }
        !          1148: 
        !          1149:   /* if there is a guest function: */
        !          1150:   if (flags_thunk->tme_recode_x86_flags_thunk_has_guest_func) {
        !          1151: 
        !          1152:     /* if this is an x86-64 host, and double-host-size guests are not
        !          1153:        supported, and we have things to remove from the stack: */
        !          1154:     if (TME_RECODE_SIZE_HOST > TME_RECODE_SIZE_32
        !          1155:        && TME_RECODE_SIZE_GUEST_MAX <= TME_RECODE_SIZE_HOST
        !          1156:        && stack_adjust != 0) {
        !          1157: 
        !          1158:       /* we can chain to the guest function, so adjust the stack
        !          1159:         pointer now: */
        !          1160:       thunk_bytes = _tme_recode_x86_emit_adjust_sp(thunk_bytes, stack_adjust);
        !          1161:       stack_adjust = 0;
        !          1162:     }
        !          1163: 
        !          1164:     /* jmp or call the guest function: */
        !          1165:     tme_recode_x86_insns_finish(ic, thunk_bytes);
        !          1166:     _tme_recode_x86_emit_transfer_func(ic,
        !          1167:                                       (stack_adjust == 0
        !          1168:                                        ? TME_RECODE_X86_OPCODE_JMP_RELz
        !          1169:                                        : TME_RECODE_X86_OPCODE_CALL_RELz),
        !          1170:                                       ((void (*) _TME_P((void))) 
        !          1171:                                        flags_group->tme_recode_flags_group_guest_func));
        !          1172:     tme_recode_x86_insns_start(ic, thunk_bytes);
        !          1173:   }
        !          1174: 
        !          1175:   /* if we still have to adjust the stack, or if there is no guest
        !          1176:      function: */
        !          1177:   if (stack_adjust != 0
        !          1178:       || !flags_thunk->tme_recode_x86_flags_thunk_has_guest_func) {
        !          1179: 
        !          1180:     /* do any stack adjust: */
        !          1181:     if (stack_adjust != 0) {
        !          1182:       thunk_bytes = _tme_recode_x86_emit_adjust_sp(thunk_bytes, stack_adjust);
        !          1183:     }
        !          1184: 
        !          1185:     /* return to the instruction thunk: */
        !          1186:     *(thunk_bytes++) = TME_RECODE_X86_OPCODE_RET;
        !          1187:   }
        !          1188: 
        !          1189:   /* finish these instructions: */
        !          1190:   tme_recode_x86_insns_finish(ic, thunk_bytes);
        !          1191: 
        !          1192:   /* finish this main flags subs thunk: */
        !          1193:   tme_recode_host_thunk_finish(ic);
        !          1194: 
        !          1195:   /* add this flags group to this new flags thunk: */
        !          1196:   return (tme_recode_host_flags_thunk_add(ic, flags_thunk, flags_group));
        !          1197: }
        !          1198: 
        !          1199: /* this emits a jmp to chain to a subs, and finishes a thunk: */
        !          1200: static void
        !          1201: _tme_recode_x86_flags_thunk_chain(struct tme_recode_ic *ic,
        !          1202:                                  tme_recode_thunk_off_t subs_next)
        !          1203: {
        !          1204:   tme_uint8_t *thunk_bytes;
        !          1205: 
        !          1206:   /* start another instruction: */
        !          1207:   tme_recode_x86_insns_start(ic, thunk_bytes);
        !          1208: 
        !          1209:   /* emit a jmp to the subs: */
        !          1210:   thunk_bytes[0] = TME_RECODE_X86_OPCODE_JMP_RELz;
        !          1211:   thunk_bytes += 1 + sizeof(tme_int32_t);
        !          1212:   ((tme_int32_t *) thunk_bytes)[-1]
        !          1213:     = (subs_next
        !          1214:        - tme_recode_build_to_thunk_off(ic, thunk_bytes));
        !          1215: 
        !          1216:   /* finish this instruction: */
        !          1217:   tme_recode_x86_insns_finish(ic, thunk_bytes);
        !          1218: 
        !          1219:   /* finish this code thunk: */
        !          1220:   tme_recode_host_thunk_finish(ic);
        !          1221: }
        !          1222: 
        !          1223: /* this emits instructions to do a binop at multiple sizes: */
        !          1224: static void
        !          1225: _tme_recode_x86_flags_thunk_ops(struct tme_recode_flags_thunk *flags_thunk,
        !          1226:                                unsigned int binop,
        !          1227:                                unsigned int reg_host_src,
        !          1228:                                tme_uint32_t host_sizes,
        !          1229:                                struct tme_recode_ic *ic)
        !          1230: {
        !          1231:   tme_uint8_t *thunk_bytes;
        !          1232:   int stack_adjust;
        !          1233:   unsigned int size;
        !          1234:   unsigned int reg_x86_src;
        !          1235:   unsigned int reg_x86_dst;
        !          1236:   unsigned int reg_x86_op_src;
        !          1237:   unsigned int reg_x86_op_dst;
        !          1238:   tme_uint8_t rex;
        !          1239: 
        !          1240:   /* start more instructions: */
        !          1241:   tme_recode_x86_insns_start(ic, thunk_bytes);
        !          1242: 
        !          1243:   /* do any stack padding needed for the host ABI: */
        !          1244:   stack_adjust = flags_thunk->tme_recode_x86_flags_thunk_stack_padding;
        !          1245:   if (stack_adjust != 0) {
        !          1246:     thunk_bytes = _tme_recode_x86_emit_adjust_sp(thunk_bytes, stack_adjust);
        !          1247:   }
        !          1248: 
        !          1249:   /* we need to track the stack adjust here for binops that test the
        !          1250:      carry.  include the subs return address in the stack adjust: */
        !          1251:   stack_adjust += TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8);
        !          1252: 
        !          1253:   /* loop over all of the sizes: */
        !          1254:   for (; host_sizes != 0; ) {
        !          1255: 
        !          1256:     /* get this size, and remove it from the mask: */
        !          1257:     size = _tme_recode_x86_ffs(host_sizes);
        !          1258:     host_sizes &= (host_sizes - 1);
        !          1259: 
        !          1260:     /* if this is a double-host size operation: */
        !          1261:     if (TME_RECODE_SIZE_IS_DOUBLE_HOST(size)) {
        !          1262: 
        !          1263:       /* force a host-size operation on the most-significant halves of
        !          1264:         the host registers: */
        !          1265:       size = TME_RECODE_SIZE_HOST;
        !          1266:       reg_x86_src = tme_recode_x86_reg_from_host[reg_host_src + 1];
        !          1267:       reg_x86_dst = tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_DST + 1];
        !          1268: 
        !          1269:       /* the most-significant half of a double-host-size additive
        !          1270:         operation uses the carry from the least-significant half: */
        !          1271:       if (binop == TME_RECODE_X86_OPCODE_BINOP_ADD) {
        !          1272:        binop = TME_RECODE_X86_OPCODE_BINOP_ADC;
        !          1273:       }
        !          1274:       else if (binop == TME_RECODE_X86_OPCODE_BINOP_SUB) {
        !          1275:        binop = TME_RECODE_X86_OPCODE_BINOP_SBB;
        !          1276:       }
        !          1277:     }
        !          1278: 
        !          1279:     /* otherwise, this is not a double-host-size operation: */
        !          1280:     else {
        !          1281: 
        !          1282:       /* operate on the (least-significant halves of) the host
        !          1283:         registers: */
        !          1284:       reg_x86_src = tme_recode_x86_reg_from_host[reg_host_src];
        !          1285:       reg_x86_dst = tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_DST];
        !          1286: 
        !          1287:       /* if this is an additive instruction that needs the carry: */
        !          1288:       if (binop == TME_RECODE_X86_OPCODE_BINOP_ADC
        !          1289:          || binop == TME_RECODE_X86_OPCODE_BINOP_SBB) {
        !          1290: 
        !          1291:        /* emit the instruction to define CF with the recode carry flag: */
        !          1292:        tme_recode_x86_insns_finish(ic, thunk_bytes);
        !          1293:        _tme_recode_x86_conds_testc(ic, stack_adjust);
        !          1294:        tme_recode_x86_insns_start(ic, thunk_bytes);
        !          1295:       }
        !          1296:     }
        !          1297: 
        !          1298:     /* if this is a test instruction, the source register is the same
        !          1299:        as the destination: */
        !          1300:     if (binop == TME_RECODE_X86_OPCODE_BINOP_TEST) {
        !          1301:       reg_x86_src = reg_x86_dst;
        !          1302:     }
        !          1303: 
        !          1304:     /* assume that we will be able to do the operation on the x86
        !          1305:        registers as loaded: */
        !          1306:     reg_x86_op_src = reg_x86_src;
        !          1307:     reg_x86_op_dst = reg_x86_dst;
        !          1308: 
        !          1309:     /* if this is an ia32 host, and this is an eight-bit operation: */
        !          1310:     if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32
        !          1311:        && size == TME_RECODE_SIZE_8) {
        !          1312: 
        !          1313:       /* eight-bit operations aren't possible with the normal source
        !          1314:         and destination x86 registers, since they don't have
        !          1315:         eight-bit encodings: */
        !          1316: 
        !          1317:       /* if this is a test instruction: */
        !          1318:       if (binop == TME_RECODE_X86_OPCODE_BINOP_TEST) {
        !          1319: 
        !          1320:        /* emit a testl %reg, $0xff: */
        !          1321:        thunk_bytes[0] = TME_RECODE_X86_OPCODE_GRP3_Ev;
        !          1322:        thunk_bytes[1] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(reg_x86_op_dst),
        !          1323:                                                     TME_RECODE_X86_OPCODE_GRP3_TEST);
        !          1324:        *((tme_uint32_t *) &thunk_bytes[2]) = 0xff;
        !          1325:        thunk_bytes += 2 + sizeof(tme_uint32_t);
        !          1326: 
        !          1327:        /* suppress the binop instruction below: */
        !          1328:        size = TME_RECODE_SIZE_1;
        !          1329:       }
        !          1330: 
        !          1331:       /* otherwise, this is not a test instruction: */
        !          1332:       else {
        !          1333: 
        !          1334:        /* we need to get the (least-significant halves of the)
        !          1335:           destination and second source operands into the two halves
        !          1336:           of the first source operand host register: */
        !          1337:        reg_x86_op_src = tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC0 + 1];
        !          1338:        reg_x86_op_dst = tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC0 + 0];
        !          1339: 
        !          1340:        /* mov the (least-significant half of the) second source
        !          1341:           operand into place: */
        !          1342:        _tme_recode_x86_emit_reg_copy(thunk_bytes,
        !          1343:                                      reg_x86_src,
        !          1344:                                      reg_x86_op_src);
        !          1345: 
        !          1346:        /* the (least significant half of the) destination operand has
        !          1347:           already been copied to the (least significant half of the)
        !          1348:           first source operand host register, unless there is no
        !          1349:           guest function and this is the only operation size: */
        !          1350:        if (!flags_thunk->tme_recode_x86_flags_thunk_has_guest_func
        !          1351:            && host_sizes == 0) {
        !          1352: 
        !          1353:          /* otherwise, mov the (least-significant half of the)
        !          1354:             destination operand into place: */
        !          1355:          _tme_recode_x86_emit_reg_copy(thunk_bytes,
        !          1356:                                        reg_x86_dst,
        !          1357:                                        reg_x86_op_dst);
        !          1358:        }
        !          1359:       }
        !          1360:     }
        !          1361: 
        !          1362:     /* unless the binop has been suppressed: */
        !          1363:     if (size != TME_RECODE_SIZE_1) {
        !          1364: 
        !          1365:       /* if this is a 16-bit operation: */
        !          1366:       if (size == TME_RECODE_SIZE_16) {
        !          1367: 
        !          1368:        /* emit the OPSIZ prefix: */
        !          1369:        thunk_bytes[0] = TME_RECODE_X86_PREFIX_OPSIZ;
        !          1370:        thunk_bytes++;
        !          1371:       }
        !          1372: 
        !          1373:       /* emit any rex prefix: */
        !          1374:       /* NB: we must use the Gb_Eb/Gv_Ev forms here since those are the
        !          1375:         only ones that the test instruction supports: */
        !          1376:       rex
        !          1377:        = (TME_RECODE_X86_REX_R(size, reg_x86_op_src)
        !          1378:           | TME_RECODE_X86_REX_B(size, reg_x86_op_dst));
        !          1379:       if (rex != 0) {
        !          1380:        thunk_bytes[0] = rex;
        !          1381:        thunk_bytes++;
        !          1382:       }
        !          1383: 
        !          1384:       /* emit the opcode: */
        !          1385:       thunk_bytes[0]
        !          1386:        = (binop
        !          1387:           + (size == TME_RECODE_SIZE_8
        !          1388:              ? TME_RECODE_X86_OPCODE_BINOP_Gb_Eb
        !          1389:              : TME_RECODE_X86_OPCODE_BINOP_Gv_Ev));
        !          1390:       thunk_bytes++;
        !          1391: 
        !          1392:       /* emit the mod R/M byte: */
        !          1393:       thunk_bytes[0] = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(reg_x86_op_dst),
        !          1394:                                                   TME_RECODE_X86_REG(reg_x86_op_src));
        !          1395:       thunk_bytes++;
        !          1396:     }
        !          1397: 
        !          1398:     /* if we have more sizes to do: */
        !          1399:     if (host_sizes != 0) {
        !          1400: 
        !          1401:       /* push these flags: */
        !          1402:       thunk_bytes[0] = TME_RECODE_X86_OPCODE_PUSHF;
        !          1403:       thunk_bytes++;
        !          1404:       stack_adjust += TME_BIT(TME_RECODE_SIZE_HOST - TME_RECODE_SIZE_8);
        !          1405:     }
        !          1406: 
        !          1407:     /* if this is an ia32 host, and this was an eight-bit operation: */
        !          1408:     if (TME_RECODE_SIZE_HOST == TME_RECODE_SIZE_32
        !          1409:        && size == TME_RECODE_SIZE_8) {
        !          1410: 
        !          1411:       /* if double-host-size guests are supported, and the last size
        !          1412:         is double-host-size, and there is a guest function: */
        !          1413:       if (TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_HOST
        !          1414:          && (host_sizes & TME_BIT(TME_RECODE_SIZE_GUEST_MAX))
        !          1415:          && flags_thunk->tme_recode_x86_flags_thunk_has_guest_func) {
        !          1416: 
        !          1417:        /* copy the most-significant half of the destination operand
        !          1418:           back into the most-significant half of the first source
        !          1419:           operand, to be passed to the guest function: */
        !          1420:        _tme_recode_x86_emit_reg_copy(thunk_bytes,
        !          1421:                                      tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_DST + 1],
        !          1422:                                      reg_x86_op_src);
        !          1423:       }
        !          1424: 
        !          1425:       /* if we have a guest function, or if we have at least two more
        !          1426:         sizes to do: */
        !          1427:       if (flags_thunk->tme_recode_x86_flags_thunk_has_guest_func
        !          1428:          || (host_sizes & (host_sizes - 1))) {
        !          1429: 
        !          1430:        /* copy the (least-significant half of) the destination
        !          1431:           operand back into the (least-significant half of) the first
        !          1432:           source operand, so it can be passed to the guest function
        !          1433:           and/or used to reload the destination operand later.  if we
        !          1434:           have no more sizes to do, this must be an xchg to put the
        !          1435:           destination value in the destination operand: */
        !          1436:        thunk_bytes[0]
        !          1437:          = ((host_sizes == 0
        !          1438:              ? TME_RECODE_X86_OPCODE_BINOP_XCHG
        !          1439:              : TME_RECODE_X86_OPCODE_BINOP_MOV)
        !          1440:             + TME_RECODE_X86_OPCODE_BINOP_Gv_Ev);
        !          1441:        thunk_bytes[1]
        !          1442:          = TME_RECODE_X86_MOD_OPREG_RM(TME_RECODE_X86_MOD_RM_REG(reg_x86_op_dst),
        !          1443:                                        reg_x86_dst);
        !          1444:        thunk_bytes += 2;
        !          1445:       }
        !          1446: 
        !          1447:       /* otherwise, if we have no more sizes to do: */
        !          1448:       else if (host_sizes == 0) {
        !          1449: 
        !          1450:        /* copy the destination operand back into place: */
        !          1451:        _tme_recode_x86_emit_reg_copy(thunk_bytes,
        !          1452:                                      reg_x86_op_dst,
        !          1453:                                      reg_x86_dst);
        !          1454:       }
        !          1455:     }
        !          1456: 
        !          1457:     /* otherwise, this is not an ia32 host or this was not
        !          1458:        an eight-bit operation: */
        !          1459:     else {
        !          1460: 
        !          1461:       /* if this was not a test instruction, and we have at least one
        !          1462:         more size to do, and it isn't the double-host-size size: */
        !          1463:       if (binop != TME_RECODE_X86_OPCODE_BINOP_TEST
        !          1464:          && host_sizes != 0
        !          1465:          && (TME_RECODE_SIZE_GUEST_MAX == TME_RECODE_SIZE_HOST
        !          1466:              || host_sizes != TME_BIT(TME_RECODE_SIZE_GUEST_MAX))) {
        !          1467: 
        !          1468:        /* reload the (least-significant half of the) destination
        !          1469:           operand: */
        !          1470:        _tme_recode_x86_emit_reg_copy(thunk_bytes,
        !          1471:                                      tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC0 + 0],
        !          1472:                                      reg_x86_dst);
        !          1473:       }
        !          1474:     }
        !          1475:   }
        !          1476: 
        !          1477:   /* finish these instructions: */
        !          1478:   tme_recode_x86_insns_finish(ic, thunk_bytes);
        !          1479: }
        !          1480: 
        !          1481: /* this adds another flags group to an existing flags thunk: */
        !          1482: struct tme_recode_flags_thunk *
        !          1483: tme_recode_host_flags_thunk_add(struct tme_recode_ic *ic,
        !          1484:                                struct tme_recode_flags_thunk *flags_thunk,
        !          1485:                                const struct tme_recode_flags_group *flags_group)
        !          1486: {
        !          1487:   tme_recode_uguest_t flags_variable_host;
        !          1488:   tme_uint32_t host_sizes;
        !          1489:   int multiple_ops;
        !          1490:   tme_recode_thunk_off_t *_subs;
        !          1491:   tme_recode_thunk_off_t subs_test;
        !          1492:   unsigned int size_other;
        !          1493:   unsigned int opcode;
        !          1494:   struct tme_recode_insn insn;
        !          1495:   struct tme_recode_insn insn_not;
        !          1496:   unsigned int class;
        !          1497:   tme_uint8_t *thunk_bytes;
        !          1498: 
        !          1499:   /* get the mask of variable guest flags that we are providing.  this
        !          1500:      may be zero: */
        !          1501:   flags_variable_host
        !          1502:     = (tme_recode_flags_group_flags_defined_host(flags_group, TME_RECODE_COND_UNDEF)
        !          1503:        & ~(tme_recode_flags_group_flags_defined_host(flags_group, TME_RECODE_COND_FALSE)
        !          1504:           | tme_recode_flags_group_flags_defined_host(flags_group, TME_RECODE_COND_MOD_NOT | TME_RECODE_COND_FALSE)));
        !          1505: 
        !          1506:   /* get the mask of sizes for which we need to provide at least one
        !          1507:      variable flag in the given flags group.  this may
        !          1508:      be zero: */
        !          1509:   host_sizes = tme_recode_flags_group_sizes(flags_group, flags_variable_host);
        !          1510: 
        !          1511:   /* it's easiest to assume that we always need the instruction-size
        !          1512:      flags when we need any flags at all, since the instruction-size
        !          1513:      operation always happens last no matter what, and leaves its flags
        !          1514:      in the flags register: */
        !          1515:   if (host_sizes != 0) {
        !          1516:     host_sizes |= TME_BIT(flags_group->tme_recode_flags_group_insn_size);
        !          1517:   }
        !          1518: 
        !          1519:   /* when we need the double-host-size flags, we also need
        !          1520:      the host-size flags, because we need the host-size Z
        !          1521:      flag to complete the double-host-size Z flag: */
        !          1522:   if (host_sizes & TME_BIT(TME_RECODE_SIZE_HOST + 1)) {
        !          1523:     host_sizes |= TME_BIT(TME_RECODE_SIZE_HOST);
        !          1524:   }
        !          1525: 
        !          1526:   /* we have to do multiple operations if we need flags for multiple
        !          1527:      sizes, not counting the double-host-size: */
        !          1528:   multiple_ops
        !          1529:     = (((host_sizes
        !          1530:         & (TME_BIT(TME_RECODE_SIZE_HOST + 1) - 1))
        !          1531:        & ((host_sizes
        !          1532:            & (TME_BIT(TME_RECODE_SIZE_HOST + 1) - 1))
        !          1533:           - 1))
        !          1534:        != 0);
        !          1535: 
        !          1536:   /* dispatch on the flags group instruction class: */
        !          1537:   switch (flags_group->tme_recode_flags_group_insn_class) {
        !          1538:   default: abort();
        !          1539: 
        !          1540:     /* these instruction classes never chain to the test subs: */
        !          1541:   case TME_RECODE_INSN_CLASS_ADDITIVE:
        !          1542:     subs_test = 0;
        !          1543:     break;
        !          1544: 
        !          1545:     /* these instruction classes always chain to the test subs: */
        !          1546:   case TME_RECODE_INSN_CLASS_SHIFT:
        !          1547:   case TME_RECODE_INSN_CLASS_EXT:
        !          1548:     subs_test = 1;
        !          1549:     break;
        !          1550: 
        !          1551:     /* these instruction classes chain to the test subs only if flags
        !          1552:        for multiple sizes are needed: */
        !          1553:   case TME_RECODE_INSN_CLASS_LOGICAL:
        !          1554:     subs_test = multiple_ops;
        !          1555:     break;
        !          1556:   }
        !          1557: 
        !          1558:   /* if this flags group chains to the test subs: */
        !          1559:   if (subs_test != 0) {
        !          1560:     
        !          1561:     /* if we haven't already created the test subs: */
        !          1562:     _subs
        !          1563:       = &(flags_thunk->tme_recode_x86_flags_thunk_sizes
        !          1564:          [flags_group->tme_recode_flags_group_insn_size - TME_RECODE_SIZE_8]
        !          1565:          .tme_recode_x86_flags_thunk_size_subs_test);
        !          1566:     subs_test = *_subs;
        !          1567:     if (subs_test == 0) {
        !          1568: 
        !          1569:       /* make the test subs, which chains to the main flags subs: */
        !          1570:       subs_test = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next);
        !          1571:       assert (subs_test != 0);
        !          1572:       *_subs = subs_test;
        !          1573:       _tme_recode_x86_flags_thunk_ops(flags_thunk,
        !          1574:                                      TME_RECODE_X86_OPCODE_BINOP_TEST,
        !          1575:                                      TME_RECODE_X86_REG_HOST_SUBS_SRC1,
        !          1576:                                      host_sizes,
        !          1577:                                      ic);
        !          1578:       _tme_recode_x86_flags_thunk_chain(ic,
        !          1579:                                        flags_thunk->tme_recode_x86_flags_thunk_subs_main);
        !          1580:     }
        !          1581:   }
        !          1582: 
        !          1583:   /* loop over all sizes starting from eight bits: */
        !          1584:   for (size_other = TME_RECODE_SIZE_8;
        !          1585:        size_other <= TME_RECODE_SIZE_GUEST_MAX;
        !          1586:        size_other++) {
        !          1587: 
        !          1588:     /* loop over all integer opcodes: */
        !          1589:     for (opcode = 0;
        !          1590:         opcode < TME_RECODE_OPCODES_INTEGER;
        !          1591:         opcode++) {
        !          1592: 
        !          1593:       /* initialize the machine-independent parts of the recode IC
        !          1594:         such that guest registers zero and one are loaded in the x86
        !          1595:         subs destination operand and second source operand host
        !          1596:         registers, respectively: */
        !          1597:       ic->tme_recode_ic_reg_host_reserve_next = 0;
        !          1598:       ic->tme_recode_ic_reg_host_to_ruses
        !          1599:        [TME_RECODE_X86_REG_HOST_SUBS_DST]
        !          1600:        = TME_RECODE_REGINFO_RUSES(1);
        !          1601:       ic->tme_recode_ic_reg_host_to_ruses
        !          1602:        [TME_RECODE_X86_REG_HOST_SUBS_SRC1]
        !          1603:        = TME_RECODE_REGINFO_RUSES(1);
        !          1604:       ic->tme_recode_ic_reg_host_to_reg_guest
        !          1605:        [TME_RECODE_X86_REG_HOST_SUBS_DST] = TME_RECODE_REG_GUEST(0);
        !          1606:       ic->tme_recode_ic_reg_host_to_reg_guest
        !          1607:        [TME_RECODE_X86_REG_HOST_SUBS_SRC1] = TME_RECODE_REG_GUEST(1);
        !          1608:       (ic->tme_recode_ic_reginfo
        !          1609:        + TME_RECODE_REG_GUEST(0))->tme_recode_reginfo_tags_ruses
        !          1610:        = (TME_RECODE_REGINFO_TAGS_VALID
        !          1611:           | TME_RECODE_REGINFO_TAGS_VALID_SIZE(TME_RECODE_SIZE_GUEST_MAX)
        !          1612:           | TME_RECODE_REGINFO_TAGS_CLEAN
        !          1613:           | TME_RECODE_REGINFO_TAGS_REG_HOST(TME_RECODE_X86_REG_HOST_SUBS_DST));
        !          1614:       (ic->tme_recode_ic_reginfo
        !          1615:        + TME_RECODE_REG_GUEST(1))->tme_recode_reginfo_tags_ruses
        !          1616:        = (TME_RECODE_REGINFO_TAGS_VALID
        !          1617:           | TME_RECODE_REGINFO_TAGS_VALID_SIZE(TME_RECODE_SIZE_GUEST_MAX)
        !          1618:           | TME_RECODE_REGINFO_TAGS_CLEAN
        !          1619:           | TME_RECODE_REGINFO_TAGS_REG_HOST(TME_RECODE_X86_REG_HOST_SUBS_SRC1));
        !          1620: 
        !          1621:       /* if this isn't a double-host-size guest, initialize the
        !          1622:         machine-independent parts of the recode ic such that guest
        !          1623:         register two is loaded in the a register, which can be
        !          1624:         destroyed by a subs: */
        !          1625:       if (!TME_RECODE_SIZE_IS_DOUBLE_HOST(ic->tme_recode_ic_reg_size)) {
        !          1626: #if (TME_RECODE_X86_REG_HOST_SUBS_SRC1 + 1) != TME_RECODE_X86_REG_HOST_FREE_CALL
        !          1627: #error "TME_RECODE_X86_REG_HOST_ values changed"
        !          1628: #endif
        !          1629:        assert (tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC1 + 1] == TME_RECODE_X86_REG_A);
        !          1630:        ic->tme_recode_ic_reg_host_to_ruses
        !          1631:          [TME_RECODE_X86_REG_HOST_SUBS_SRC1 + 1]
        !          1632:          = TME_RECODE_REGINFO_RUSES(1);
        !          1633:        ic->tme_recode_ic_reg_host_to_reg_guest
        !          1634:          [TME_RECODE_X86_REG_HOST_SUBS_SRC1 + 1] = TME_RECODE_REG_GUEST(2);
        !          1635:        (ic->tme_recode_ic_reginfo
        !          1636:         + TME_RECODE_REG_GUEST(2))->tme_recode_reginfo_tags_ruses
        !          1637:          = (TME_RECODE_REGINFO_TAGS_VALID
        !          1638:             | TME_RECODE_REGINFO_TAGS_VALID_SIZE(TME_RECODE_SIZE_GUEST_MAX)
        !          1639:             | TME_RECODE_REGINFO_TAGS_CLEAN
        !          1640:             | TME_RECODE_REGINFO_TAGS_REG_HOST(TME_RECODE_X86_REG_HOST_SUBS_SRC1 + 1));
        !          1641:       }
        !          1642: 
        !          1643:       /* initialize the instruction, assuming that we'll emit for only
        !          1644:         one size (double-host-size if this is a double-host-size flags
        !          1645:         group, otherwise host-size), and assuming that this
        !          1646:         instruction will operate with guest register zero on the left
        !          1647:         and guest register one on the right, with the result going
        !          1648:         back into guest register zero: */
        !          1649:       insn.tme_recode_insn_opcode = opcode;
        !          1650:       insn.tme_recode_insn_size
        !          1651:        = (TME_RECODE_SIZE_HOST
        !          1652:           + (TME_RECODE_SIZE_IS_DOUBLE_HOST(flags_group->tme_recode_flags_group_insn_size) != 0));
        !          1653:       insn.tme_recode_insn_operand_src[0] = TME_RECODE_REG_GUEST(0);
        !          1654:       insn.tme_recode_insn_operand_src[1] = TME_RECODE_REG_GUEST(1);
        !          1655:       insn.tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(0);
        !          1656:       insn.tme_recode_insn_flags_thunk = NULL;
        !          1657: 
        !          1658:       /* assume that this opcode, for this flags group
        !          1659:         instruction size, only has one subs: */
        !          1660:       _subs
        !          1661:        = &(flags_thunk->tme_recode_x86_flags_thunk_sizes
        !          1662:            [flags_group->tme_recode_flags_group_insn_size - TME_RECODE_SIZE_8]
        !          1663:            .tme_recode_x86_flags_thunk_size_subs[opcode]);
        !          1664: 
        !          1665:       /* dispatch on the opcode to get its instruction class and to
        !          1666:         override any defaults: */
        !          1667:       switch (opcode) {
        !          1668:       default: assert(FALSE);
        !          1669:       case TME_RECODE_OPCODE_ANDN:
        !          1670:       case TME_RECODE_OPCODE_ORN:
        !          1671:       case TME_RECODE_OPCODE_XORN:
        !          1672:       case TME_RECODE_OPCODE_AND:
        !          1673:       case TME_RECODE_OPCODE_OR:
        !          1674:       case TME_RECODE_OPCODE_XOR:
        !          1675:        class = TME_RECODE_INSN_CLASS_LOGICAL;
        !          1676:        break;
        !          1677:       case TME_RECODE_OPCODE_SUB:
        !          1678:       case TME_RECODE_OPCODE_SUBC:
        !          1679:       case TME_RECODE_OPCODE_ADD:
        !          1680:       case TME_RECODE_OPCODE_ADDC:
        !          1681:        class = TME_RECODE_INSN_CLASS_ADDITIVE;
        !          1682:        break;
        !          1683:       case TME_RECODE_OPCODE_SHRL:
        !          1684:       case TME_RECODE_OPCODE_SHLL:
        !          1685:       case TME_RECODE_OPCODE_SHRA:
        !          1686:        class = TME_RECODE_INSN_CLASS_SHIFT;
        !          1687:        break;
        !          1688:       case TME_RECODE_OPCODE_EXTZ:
        !          1689:       case TME_RECODE_OPCODE_EXTS:
        !          1690:        class = TME_RECODE_INSN_CLASS_EXT;
        !          1691: #if (TME_RECODE_OPCODE_EXTS - TME_RECODE_OPCODE_EXTZ) != 1
        !          1692: #error "TME_RECODE_OPCODE_ values changed"
        !          1693: #endif
        !          1694:        _subs
        !          1695:          = &(flags_thunk->tme_recode_x86_flags_thunk_sizes
        !          1696:              [flags_group->tme_recode_flags_group_insn_size - TME_RECODE_SIZE_8]
        !          1697:              .tme_recode_x86_flags_thunk_size_subs_ext
        !          1698:              [size_other - TME_RECODE_SIZE_8]
        !          1699:              [opcode - TME_RECODE_OPCODE_EXTZ]);
        !          1700:        insn.tme_recode_insn_imm_uguest = size_other;
        !          1701:        break;
        !          1702:       }
        !          1703: 
        !          1704:       /* skip this opcode if it's in the wrong class: */
        !          1705:       if (class != flags_group->tme_recode_flags_group_insn_class) {
        !          1706:        continue;
        !          1707:       }
        !          1708: 
        !          1709:       /* skip the other size if it's not the result size of the
        !          1710:         instructions in this flags group, unless this is the
        !          1711:         extension class, in which case we skip other sizes that are
        !          1712:         greater than or equal to the result size: */
        !          1713:       if (class == TME_RECODE_INSN_CLASS_EXT
        !          1714:          ? size_other >= flags_group->tme_recode_flags_group_insn_size
        !          1715:          : size_other != flags_group->tme_recode_flags_group_insn_size) {
        !          1716:        continue;
        !          1717:       }
        !          1718: 
        !          1719:       /* skip this opcode if we've already created its subs: */
        !          1720:       if (*_subs != 0) {
        !          1721:        continue;
        !          1722:       }
        !          1723: 
        !          1724:       /* set the thunk offset of this subs: */
        !          1725:       *_subs = tme_recode_build_to_thunk_off(ic, ic->tme_recode_ic_thunk_build_next);
        !          1726:       assert (*_subs != 0);
        !          1727: 
        !          1728:       /* if there is a guest function, or if this class doesn't chain
        !          1729:         to the test subs and we have do to its instructions at
        !          1730:         multiple sizes, not counting the double-host-size: */
        !          1731:       if (flags_thunk->tme_recode_x86_flags_thunk_has_guest_func
        !          1732:          || (subs_test == 0
        !          1733:              && multiple_ops)) {
        !          1734: 
        !          1735:        /* start more instructions: */
        !          1736:        tme_recode_x86_insns_start(ic, thunk_bytes);
        !          1737: 
        !          1738:        /* copy the first source operand, currently in the destination
        !          1739:           operand host register, to the first source operand host
        !          1740:           register: */
        !          1741:        if (TME_RECODE_SIZE_IS_DOUBLE_HOST(flags_group->tme_recode_flags_group_insn_size)) {
        !          1742:          _tme_recode_x86_emit_reg_copy(thunk_bytes,
        !          1743:                                        tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_DST + 1],
        !          1744:                                        tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC0 + 1]);
        !          1745:        }
        !          1746:        _tme_recode_x86_emit_reg_copy(thunk_bytes,
        !          1747:                                      tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_DST],
        !          1748:                                      tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC0]);
        !          1749: 
        !          1750:        /* finish these instructions: */
        !          1751:        tme_recode_x86_insns_finish(ic, thunk_bytes);
        !          1752:       }
        !          1753: 
        !          1754:       /* if this is a logical noncommutative instruction: */
        !          1755:       if ((1 << opcode)
        !          1756:          & ((1 << TME_RECODE_OPCODE_ANDN)
        !          1757:             | (1 << TME_RECODE_OPCODE_ORN)
        !          1758:             | (1 << TME_RECODE_OPCODE_XORN))) {
        !          1759: 
        !          1760:        /* we have to invert the second source operand and convert
        !          1761:           this instruction into its commutative version.  for speed,
        !          1762:           we do at least a 32-bit not: */
        !          1763:        insn_not.tme_recode_insn_opcode = opcode;
        !          1764:        insn_not.tme_recode_insn_size = TME_MAX(insn.tme_recode_insn_size, TME_RECODE_SIZE_32);
        !          1765: 
        !          1766:        /* NB that the a register can't be allocated to any guest
        !          1767:           register at this point, since it can be destroyed by a
        !          1768:           subs.  since the a register is the second of a
        !          1769:           double-host-size pair, if this is a double-host-size guest,
        !          1770:           the first register of the pair is also free: */
        !          1771: #if (TME_RECODE_X86_REG_HOST_SUBS_SRC1 + 1) < TME_RECODE_X86_REG_HOST_FREE_CALL
        !          1772: #error "TME_RECODE_X86_REG_HOST_ values changed"
        !          1773: #endif
        !          1774:        assert (tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC1 + 1] == TME_RECODE_X86_REG_A);
        !          1775: 
        !          1776:        /* if this is a double-host-size instruction: */
        !          1777:        if (TME_RECODE_SIZE_IS_DOUBLE_HOST(flags_group->tme_recode_flags_group_insn_size)) {
        !          1778: 
        !          1779:          /* we invert the double-host-size second source operand in
        !          1780:             place, since both host registers can't be allocated to
        !          1781:             any guest register: */
        !          1782:          insn_not.tme_recode_insn_operand_dst = TME_RECODE_X86_REG_HOST_SUBS_SRC1;
        !          1783:        }
        !          1784: 
        !          1785:        /* otherwise, this is not a double-host-size instruction: */
        !          1786:        else {
        !          1787: 
        !          1788:          /* copy the second source operand into the free a register: */
        !          1789:          tme_recode_x86_insns_start(ic, thunk_bytes);
        !          1790:          _tme_recode_x86_emit_reg_copy(thunk_bytes,
        !          1791:                                        tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC1],
        !          1792:                                        tme_recode_x86_reg_from_host[TME_RECODE_X86_REG_HOST_SUBS_SRC1 + 1]);
        !          1793:          tme_recode_x86_insns_finish(ic, thunk_bytes);
        !          1794: 
        !          1795:          /* we invert the host-size or smaller second source operand in
        !          1796:             the a register: */
        !          1797:          insn_not.tme_recode_insn_operand_dst = TME_RECODE_X86_REG_HOST_SUBS_SRC1 + 1;
        !          1798: 
        !          1799:          /* rewrite the second source operand to be the guest
        !          1800:             register we "allocated" to the a register: */
        !          1801:          insn.tme_recode_insn_operand_src[1] = TME_RECODE_REG_GUEST(2);
        !          1802:        }
        !          1803: 
        !          1804:        /* invert the second source operand: */
        !          1805:        _tme_recode_x86_insn_negate(ic, &insn_not);
        !          1806: 
        !          1807:        /* rewrite the opcode to a plain logical: */
        !          1808:        insn.tme_recode_insn_opcode = TME_RECODE_OPCODE_LOGICALN_TO_PLAIN(opcode);
        !          1809:       }
        !          1810: 
        !          1811:       if (subs_test == 0) {
        !          1812: 
        !          1813:        /* make the binops subs.  NB that we have to load the second
        !          1814:           source operand, since this is not necessarily
        !          1815:           TME_RECODE_X86_REG_HOST_SUBS_SRC1: */
        !          1816:        _tme_recode_x86_flags_thunk_ops(flags_thunk,
        !          1817:                                        TME_RECODE_X86_OPCODE_TO_BINOP(insn.tme_recode_insn_opcode),
        !          1818:                                        tme_recode_regs_src_any(ic,
        !          1819:                                                                &insn,
        !          1820:                                                                insn.tme_recode_insn_operand_src[1]),
        !          1821:                                        host_sizes,
        !          1822:                                        ic);
        !          1823:       }
        !          1824: 
        !          1825:       else {
        !          1826: 
        !          1827:        /* emit an instruction normally: */
        !          1828:        _tme_recode_x86_insn_emit(ic, &insn);
        !          1829:       }
        !          1830: 
        !          1831:       /* if this is a double-host-size logical noncommutative
        !          1832:         instruction and there is a guest function: */
        !          1833:       if (((1 << opcode)
        !          1834:           & ((1 << TME_RECODE_OPCODE_ANDN)
        !          1835:              | (1 << TME_RECODE_OPCODE_ORN)
        !          1836:              | (1 << TME_RECODE_OPCODE_XORN)))
        !          1837:          && TME_RECODE_SIZE_IS_DOUBLE_HOST(flags_group->tme_recode_flags_group_insn_size)
        !          1838:          && flags_thunk->tme_recode_x86_flags_thunk_has_guest_func) {
        !          1839: 
        !          1840:        /* we have to undo the inversion of the double-host-size
        !          1841:           second source operand in place, because we have to pass it
        !          1842:           to the guest function: */
        !          1843: 
        !          1844:        /* if there isn't a test subs: */
        !          1845:        if (subs_test == 0) {
        !          1846: 
        !          1847:          /* push the double-host-size flags: */
        !          1848:          tme_recode_x86_insns_start(ic, thunk_bytes);
        !          1849:          thunk_bytes[0] = TME_RECODE_X86_OPCODE_PUSHF;
        !          1850:          thunk_bytes++;
        !          1851:          tme_recode_x86_insns_finish(ic, thunk_bytes);
        !          1852:        }
        !          1853: 
        !          1854:        /* undo the inversion by redoing it: */
        !          1855:        _tme_recode_x86_insn_negate(ic, &insn_not);
        !          1856: 
        !          1857:        /* if there isn't a test subs: */
        !          1858:        if (subs_test == 0) {
        !          1859: 
        !          1860:          /* pop the double-host-size flags: */
        !          1861:          tme_recode_x86_insns_start(ic, thunk_bytes);
        !          1862:          thunk_bytes[0] = TME_RECODE_X86_OPCODE_PUSHF;
        !          1863:          thunk_bytes++;
        !          1864:          tme_recode_x86_insns_finish(ic, thunk_bytes);
        !          1865:        }
        !          1866:       }
        !          1867: 
        !          1868:       /* chain to either the test subs or the main flags subs: */
        !          1869:       _tme_recode_x86_flags_thunk_chain(ic,
        !          1870:                                        (subs_test != 0
        !          1871:                                         ? subs_test
        !          1872:                                         : flags_thunk->tme_recode_x86_flags_thunk_subs_main));
        !          1873:     }
        !          1874:   }
        !          1875: 
        !          1876:   /* return the flags thunk: */
        !          1877:   return (flags_thunk);
        !          1878: }
        !          1879: 
        !          1880: #ifdef TME_RECODE_DEBUG
        !          1881: #include <stdio.h>
        !          1882: 
        !          1883: /* this host function dumps a flags thunk: */
        !          1884: void
        !          1885: tme_recode_host_flags_thunk_dump(const struct tme_recode_ic *ic,
        !          1886:                                 const struct tme_recode_flags_thunk *flags_thunk)
        !          1887: {
        !          1888:   unsigned int insn_size;
        !          1889:   unsigned int opcode;
        !          1890:   tme_recode_thunk_off_t thunk_off;
        !          1891:   unsigned int insn_size_other;
        !          1892: 
        !          1893:   printf("  x86 stack padding: %d\n", flags_thunk->tme_recode_x86_flags_thunk_stack_padding);
        !          1894:   printf("  x86 has guest func: %s\n",
        !          1895:         (flags_thunk->tme_recode_x86_flags_thunk_has_guest_func
        !          1896:          ? "yes"
        !          1897:          : "no"));
        !          1898:   printf("  x86 main flags subs: x/15i %p\n",
        !          1899:         tme_recode_thunk_off_to_pointer(ic,
        !          1900:                                         flags_thunk->tme_recode_x86_flags_thunk_subs_main,
        !          1901:                                         void (*)(void)));
        !          1902: 
        !          1903:   for (insn_size = TME_RECODE_SIZE_8;
        !          1904:        insn_size <= TME_RECODE_SIZE_GUEST_MAX;
        !          1905:        insn_size++) {
        !          1906:     for (opcode = 0;
        !          1907:         opcode < TME_RECODE_OPCODES_INTEGER;
        !          1908:         opcode++) {
        !          1909:       thunk_off = flags_thunk
        !          1910:        ->tme_recode_x86_flags_thunk_sizes[insn_size - TME_RECODE_SIZE_8]
        !          1911:        .tme_recode_x86_flags_thunk_size_subs[opcode];
        !          1912:       if (thunk_off != 0) {
        !          1913:        printf("  x86 %s%u flags subs: x/4i %p\n",
        !          1914:               tme_recode_opcode_dump(opcode),
        !          1915:               (1 << insn_size),
        !          1916:               tme_recode_thunk_off_to_pointer(ic,
        !          1917:                                               thunk_off,
        !          1918:                                               void (*)(void)));
        !          1919:       }
        !          1920:     }
        !          1921: #if (TME_RECODE_OPCODE_EXTZ + 1) != TME_RECODE_OPCODE_EXTS
        !          1922: #error "TME_RECODE_OPCODE_EXTS or TME_RECODE_OPCODE_EXTZ changed"
        !          1923: #endif
        !          1924:     for (opcode = TME_RECODE_OPCODE_EXTZ;
        !          1925:         opcode <= TME_RECODE_OPCODE_EXTS;
        !          1926:         opcode++) {
        !          1927:       for (insn_size_other = TME_RECODE_SIZE_8;
        !          1928:           insn_size_other < insn_size;
        !          1929:           insn_size_other++) {
        !          1930:        thunk_off = flags_thunk
        !          1931:          ->tme_recode_x86_flags_thunk_sizes[insn_size - TME_RECODE_SIZE_8]
        !          1932:          .tme_recode_x86_flags_thunk_size_subs_ext[insn_size_other - TME_RECODE_SIZE_8]
        !          1933:          [opcode - TME_RECODE_OPCODE_EXTZ];
        !          1934:        if (thunk_off != 0) {
        !          1935:          printf("  x86 ext%u%c%u flags subs: x/4i %p\n",
        !          1936:                 (1 << insn_size_other),
        !          1937:                 (opcode == TME_RECODE_OPCODE_EXTZ ? 'z' : 's'),
        !          1938:                 (1 << insn_size),
        !          1939:                 tme_recode_thunk_off_to_pointer(ic,
        !          1940:                                                 thunk_off,
        !          1941:                                                 void (*)(void)));
        !          1942:        }
        !          1943:       }
        !          1944:     }
        !          1945:   }
        !          1946: }
        !          1947: 
        !          1948: #endif /* TME_RECODE_DEBUG */

This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.