Annotation of tme/ic/sparc/sparc-rc-insns.c, revision 1.1.1.1

1.1       root        1: /* $Id: sparc-rc-insns.c,v 1.8 2010/06/05 17:00:47 fredette Exp $ */
                      2: 
                      3: /* ic/sparc/sparc-rc-insns.c - SPARC recode instruction support: */
                      4: 
                      5: /*
                      6:  * Copyright (c) 2008 Matt Fredette
                      7:  * All rights reserved.
                      8:  *
                      9:  * Redistribution and use in source and binary forms, with or without
                     10:  * modification, are permitted provided that the following conditions
                     11:  * are met:
                     12:  * 1. Redistributions of source code must retain the above copyright
                     13:  *    notice, this list of conditions and the following disclaimer.
                     14:  * 2. Redistributions in binary form must reproduce the above copyright
                     15:  *    notice, this list of conditions and the following disclaimer in the
                     16:  *    documentation and/or other materials provided with the distribution.
                     17:  * 3. All advertising materials mentioning features or use of this software
                     18:  *    must display the following acknowledgement:
                     19:  *      This product includes software developed by Matt Fredette.
                     20:  * 4. The name of the author may not be used to endorse or promote products
                     21:  *    derived from this software without specific prior written permission.
                     22:  *
                     23:  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
                     24:  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
                     25:  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
                     26:  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
                     27:  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
                     28:  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
                     29:  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
                     30:  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
                     31:  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
                     32:  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
                     33:  * POSSIBILITY OF SUCH DAMAGE.
                     34:  */
                     35: 
                     36: #if TME_SPARC_RECODE_SIZE(ic) == TME_RECODE_SIZE_32
                     37: 
                     38: _TME_RCSID("$Id: sparc-rc-insns.c,v 1.8 2010/06/05 17:00:47 fredette Exp $");
                     39: 
                     40: /* macros: */
                     41: 
                     42: /* the sparc instruction recoder is mostly driven by the 32-bit
                     43:    control value kept in the sparc_recode variable.  this value
                     44:    includes various fields and flags, and is initially taken from the
                     45:    sparc instruction's entry in _tme_sparc_recode_insn_opmap[].  this
                     46:    entry provides only the least significant 16 bits; other
                     47:    more-significant bits may be added as recoding proceeds: */
                     48: 
                     49: /* the opcode for the recode instruction: */
                     50: #define TME_SPARC_RECODE_INSN_OPCODE(x)                ((tme_uint8_t) (x))
                     51: #define TME_SPARC_RECODE_INSN_OPCODE_MASK      (0xff)
                     52: 
                     53: /* if set, the PC and PC_next registers must be valid for this sparc
                     54:    instruction, either because the sparc instruction itself needs them
                     55:    to be valid, or because the situation calls for them to be valid
                     56:    (for example, when TME_SPARC_RECODE_INSN_LAST is also set): */
                     57: #define TME_SPARC_RECODE_INSN_UPDATE_PCS       (1 << 8)
                     58: 
                     59: /* if set, this sparc instruction will be the last one recoded in this
                     60:    instructions thunk: */
                     61: #define TME_SPARC_RECODE_INSN_LAST             (1 << 9)
                     62: 
                     63: /* when TME_SPARC_RECODE_INSN_UPDATE_INSN is set, these flags are set
                     64:    if this sparc instruction doesn't take rs2/simm13 and/or rs1 source
                     65:    operands: */
                     66: #define TME_SPARC_RECODE_INSN_NO_RS2           (1 << 10)
                     67: #define TME_SPARC_RECODE_INSN_NO_RS1           (1 << 11)
                     68: 
                     69: /* when the recode opcode is TME_RECODE_OPCODE_GUEST, this flag is set
                     70:    if this sparc instruction doesn't write any sparc register visible
                     71:    to recode: */
                     72: #define TME_SPARC_RECODE_INSN_NO_RD            (1 << 12)
                     73: 
                     74: /* if set, the TME_SPARC_IREG_INSN register must be valid for this
                     75:    sparc instruction, usually because it uses a generic assist
                     76:    function that needs the instruction word: */
                     77: #define TME_SPARC_RECODE_INSN_UPDATE_INSN      (1 << 13)
                     78: 
                     79: /* if set, this is a shift instruction: */
                     80: #define TME_SPARC_RECODE_INSN_SHIFT            (1 << 14)
                     81: 
                     82: /* when TME_SPARC_RECODE_INSN_UPDATE_INSN is clear, this is set if
                     83:    this sparc instruction needs TME_RECODE_FLAG_CARRY set with the
                     84:    %icc.C flag: */
                     85: #define TME_SPARC_RECODE_INSN_DEFC             (TME_SPARC_RECODE_INSN_NO_RS2)
                     86: 
                     87: /* when TME_SPARC_RECODE_INSN_UPDATE_INSN is clear, this is set if
                     88:    this sparc instruction updates the sparc condition codes: */
                     89: #define TME_SPARC_RECODE_INSN_CC               (TME_SPARC_RECODE_INSN_NO_RS1)
                     90: 
                     91: /* NB: the following sparc_recode flags cannot be used in the 16-bit
                     92:    _tme_sparc_recode_insn_opmap[]: */
                     93: 
                     94: /* if set, this suppresses the normal recoding of the sparc
                     95:    instruction itself (although any TME_SPARC_RECODE_INSN_UPDATE_PCS
                     96:    will still be done): */
                     97: #define TME_SPARC_RECODE_INSN_NONE             (1 << 16)
                     98: 
                     99: /* if set, this is a v9 32-bit right shift: */
                    100: #define TME_SPARC_RECODE_INSN_SHIFT_RIGHT      (1 << 17)
                    101: 
                    102: /* if set, after the normal recoding of the sparc instruction, a
                    103:    recode endif will be emitted: */
                    104: #define TME_SPARC_RECODE_INSN_NEED_ENDIF       (1 << 18)
                    105: 
                    106: /* when TME_SPARC_RECODE_INSN_LAST is set, this is any recode chain
                    107:    information for the instructions thunk: */
                    108: #define TME_SPARC_RECODE_CHAIN_INFO(x)         ((x) << 19)
                    109: #if (TME_RECODE_CHAIN_INFO_UNCONDITIONAL | TME_RECODE_CHAIN_INFO_CONDITIONAL | TME_RECODE_CHAIN_INFO_NEAR | TME_RECODE_CHAIN_INFO_FAR | TME_RECODE_CHAIN_INFO_JUMP | TME_RECODE_CHAIN_INFO_RETURN | TME_RECODE_CHAIN_INFO_CALL) > 0x1f
                    110: #error "TME_RECODE_CHAIN_INFO_ values changed"
                    111: #endif
                    112: 
                    113: /* these are composite values used in
                    114:    _tme_sparc_recode_insn_opmap[]: */
                    115: 
                    116: /* this sparc instruction needs an assist: */
                    117: #define TME_SPARC_RECODE_INSN_ASSIST           \
                    118:   (TME_RECODE_OPCODE_GUEST                     \
                    119:    | TME_SPARC_RECODE_INSN_UPDATE_INSN)
                    120: 
                    121: /* this sparc instruction needs an assist, and can trap: */
                    122: #define TME_SPARC_RECODE_INSN_ASSIST_CANTRAP   \
                    123:   (TME_SPARC_RECODE_INSN_ASSIST                        \
                    124:    | TME_SPARC_RECODE_INSN_UPDATE_PCS)
                    125: 
                    126: /* this sparc instruction needs an assist, can trap, and the assist
                    127:    function does not need source operands from the rs1 and rs2/simm13
                    128:    fields: */
                    129: #define TME_SPARC_RECODE_INSN_ASSIST_FULL      \
                    130:   (TME_SPARC_RECODE_INSN_ASSIST_CANTRAP                \
                    131:    | TME_SPARC_RECODE_INSN_NO_RS2              \
                    132:    | TME_SPARC_RECODE_INSN_NO_RS1)
                    133: 
                    134: /* this sparc instruction is undefined: */
                    135: #define TME_SPARC_RECODE_INSN_UNDEF            TME_SPARC_RECODE_INSN_ASSIST_FULL
                    136: 
                    137: /* this sparc instruction is a load: */
                    138: #define TME_SPARC_RECODE_INSN_LD               \
                    139:   (TME_SPARC_RECODE_INSN_UPDATE_PCS            \
                    140:    | TME_RECODE_OPCODE_RW)
                    141: 
                    142: /* this sparc instruction is a store: */
                    143: #define TME_SPARC_RECODE_INSN_ST               \
                    144:   (TME_SPARC_RECODE_INSN_UPDATE_PCS            \
                    145:    | TME_RECODE_OPCODE_RW)
                    146: 
                    147: /* this sparc instruction is a load alternate: */
                    148: #define TME_SPARC_RECODE_INSN_LDA              \
                    149:   (TME_SPARC_VERSION(ic) < 9                   \
                    150:    ? TME_SPARC_RECODE_INSN_ASSIST_CANTRAP      \
                    151:    : TME_SPARC_RECODE_INSN_LD)
                    152: 
                    153: /* rename various things by the architecture size: */
                    154: #define tme_sparc_idle_type_pc _TME_SPARC_RECODE_SIZE(tme_sparc_idle_type_pc,/**/)
                    155: #define _tme_sparc_recode_insn_opmap _TME_SPARC_RECODE_SIZE(_tme_sparc,_recode_insn_opmap)
                    156: #define _tme_sparc_execute_opmap _TME_SPARC_RECODE_SIZE(_tme_sparc,_execute_opmap)
                    157: #define tme_sparc_recode_insn_current _TME_SPARC_RECODE_SIZE(tme_sparc,_recode_insn_current)
                    158: #define tme_sparc_recode_insn_assist_redispatch _TME_SPARC_RECODE_SIZE(tme_sparc,_recode_insn_assist_redispatch)
                    159: #define _tme_sparc_recode_insn_assist _TME_SPARC_RECODE_SIZE(_tme_sparc,_recode_insn_assist)
                    160: #define _tme_sparc_recode_insn_assist_full _TME_SPARC_RECODE_SIZE(_tme_sparc,_recode_insn_assist_full)
                    161: #define _tme_sparc_recode_insn_assist_jmpl _TME_SPARC_RECODE_SIZE(_tme_sparc,_recode_insn_assist_jmpl)
                    162: #define _tme_sparc_recode_insn_assist_unimpl _TME_SPARC_RECODE_SIZE(_tme_sparc,_recode_insn_assist_unimpl)
                    163: #define _tme_sparc_recode_insns_total _TME_SPARC_RECODE_SIZE(_tme_sparc,_recode_insns_total)
                    164: #define _tme_sparc_recode_insn_branch_delay_bad _TME_SPARC_RECODE_SIZE(_tme_sparc,_recode_insn_branch_delay_bad)
                    165: #define _tme_sparc_recode_recode _TME_SPARC_RECODE_SIZE(_tme_sparc,_recode_recode)
                    166: 
                    167: #endif /* TME_SPARC_RECODE_SIZE(ic) == TME_RECODE_SIZE_32 */
                    168: 
                    169: /* this maps format three opcodes into sparc recode controls: */
                    170: static const tme_uint16_t _tme_sparc_recode_insn_opmap[128] = {
                    171: 
                    172:   /* op=2: arithmetic, logical, shift, and remaining: */
                    173: 
                    174:   /* 000000 */ TME_RECODE_OPCODE_ADD,
                    175:   /* 000001 */ TME_RECODE_OPCODE_AND,
                    176:   /* 000010 */ TME_RECODE_OPCODE_OR,
                    177:   /* 000011 */ TME_RECODE_OPCODE_XOR,
                    178:   /* 000100 */ TME_RECODE_OPCODE_SUB,
                    179:   /* 000101 */ TME_RECODE_OPCODE_ANDN,
                    180:   /* 000110 */ TME_RECODE_OPCODE_ORN,
                    181:   /* 000111 */ TME_RECODE_OPCODE_XORN,
                    182:   /* 001000 */ TME_RECODE_OPCODE_ADDC | TME_SPARC_RECODE_INSN_DEFC,
                    183: #if TME_SPARC_VERSION(ic) < 9
                    184:   /* 001001 */ TME_SPARC_RECODE_INSN_UNDEF,
                    185: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    186:   /* 101001 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* v9: mulx */
                    187: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    188:   /* 001010 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* umul */
                    189:   /* 001011 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* smul */
                    190:   /* 001100 */ TME_RECODE_OPCODE_SUBC | TME_SPARC_RECODE_INSN_DEFC,
                    191: #if TME_SPARC_VERSION(ic) < 9
                    192:   /* 001101 */ TME_SPARC_RECODE_INSN_UNDEF,
                    193: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    194:   /* 001101 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* v9: udivx */
                    195: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    196:   /* 001110 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* udiv */
                    197:   /* 001111 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* sdiv */
                    198:   /* 010000 */ TME_SPARC_RECODE_INSN_CC | TME_RECODE_OPCODE_ADD,
                    199:   /* 010001 */ TME_SPARC_RECODE_INSN_CC | TME_RECODE_OPCODE_AND,
                    200:   /* 010010 */ TME_SPARC_RECODE_INSN_CC | TME_RECODE_OPCODE_OR,
                    201:   /* 010011 */ TME_SPARC_RECODE_INSN_CC | TME_RECODE_OPCODE_XOR,
                    202:   /* 010100 */ TME_SPARC_RECODE_INSN_CC | TME_RECODE_OPCODE_SUB,
                    203:   /* 010101 */ TME_SPARC_RECODE_INSN_CC | TME_RECODE_OPCODE_ANDN,
                    204:   /* 010110 */ TME_SPARC_RECODE_INSN_CC | TME_RECODE_OPCODE_ORN,
                    205:   /* 010111 */ TME_SPARC_RECODE_INSN_CC | TME_RECODE_OPCODE_XORN,
                    206:   /* 011000 */ TME_SPARC_RECODE_INSN_CC | TME_RECODE_OPCODE_ADDC | TME_SPARC_RECODE_INSN_DEFC,
                    207:   /* 011001 */ TME_SPARC_RECODE_INSN_UNDEF,
                    208:   /* 011010 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* umulcc */
                    209:   /* 011011 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* smulcc */
                    210:   /* 011100 */ TME_SPARC_RECODE_INSN_CC | TME_RECODE_OPCODE_SUBC | TME_SPARC_RECODE_INSN_DEFC,
                    211:   /* 011101 */ TME_SPARC_RECODE_INSN_UNDEF,
                    212:   /* 011110 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* udivcc */
                    213:   /* 011111 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* sdivcc */
                    214:   /* 100000 */ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP, /* taddcc */
                    215:   /* 100001 */ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP, /* tsubcc */
                    216:   /* 100010 */ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP, /* taddcctv */
                    217:   /* 100011 */ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP, /* tsubcctv */
                    218:   /* 100100 */ TME_SPARC_RECODE_INSN_ASSIST, /* mulscc */
                    219:   /* 100101 */ TME_RECODE_OPCODE_SHLL | TME_SPARC_RECODE_INSN_SHIFT,
                    220:   /* 100110 */ TME_RECODE_OPCODE_SHRL | TME_SPARC_RECODE_INSN_SHIFT,
                    221:   /* 100111 */ TME_RECODE_OPCODE_SHRA | TME_SPARC_RECODE_INSN_SHIFT,
                    222:   /* 101000 */ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP, /* rdasr */
                    223: #if TME_SPARC_VERSION(ic) < 9
                    224:   /* 101001 */ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP, /* rdpsr */
                    225: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    226:   /* 101001 */ TME_SPARC_RECODE_INSN_UNDEF,
                    227: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    228:   /* 101010 */ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP, /* rdwim (v9: rdpr) */
                    229:   /* 101011 */ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP, /* rdtbr (v9: flushw) */
                    230: #if TME_SPARC_VERSION(ic) < 9
                    231:   /* 101100 */ TME_SPARC_RECODE_INSN_UNDEF,
                    232:   /* 101101 */ TME_SPARC_RECODE_INSN_UNDEF,
                    233:   /* 101110 */ TME_SPARC_RECODE_INSN_UNDEF,
                    234:   /* 101111 */ TME_SPARC_RECODE_INSN_UNDEF,
                    235: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    236:   /* 101100 */ 0, /* v9: movcc (handled specially) */
                    237:   /* 101101 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* v9: sdivx */
                    238:   /* 101110 */ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP, /* v9: popc */
                    239:   /* 101111 */ 0, /* v9: movr (handled specially) */
                    240: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    241:   /* 110000 */ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP | TME_SPARC_RECODE_INSN_NO_RD, /* wrasr */
                    242: #if TME_SPARC_VERSION(ic) < 9
                    243:   /* 110001 */ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP | TME_SPARC_RECODE_INSN_NO_RD, /* wrpsr */
                    244: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    245:   /* 110001 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* v9: saved/restored */
                    246: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    247:   /* 110010 */ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP | TME_SPARC_RECODE_INSN_NO_RD, /* wrwim (v9: wrpr) */
                    248: #if TME_SPARC_VERSION(ic) < 9
                    249:   /* 110011 */ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP | TME_SPARC_RECODE_INSN_NO_RD, /* wrtbr */
                    250: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    251:   /* 110011 */ TME_SPARC_RECODE_INSN_UNDEF,
                    252: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    253:   /* 110100 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* fpop1 */
                    254:   /* 110101 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* fpop2 */
                    255:   /* 110110 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* cpop1 (v9: impdep1) */
                    256:   /* 110111 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* cpop2 (v9: impdep2) */
                    257:   /* 111000 */ 0, /* jmpl (handled specially) */
                    258:   /* 111001 */ TME_SPARC_RECODE_INSN_ASSIST_FULL | TME_SPARC_RECODE_INSN_LAST, /* rett (v9: return) */
                    259:   /* 111010 */ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP | TME_SPARC_RECODE_INSN_NO_RD, /* ticc */
                    260:   /* 111011 */ TME_SPARC_RECODE_INSN_ASSIST | TME_SPARC_RECODE_INSN_NO_RD, /* flush */
                    261:   /* 111100 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* save */
                    262:   /* 111101 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* restore */
                    263: #if TME_SPARC_VERSION(ic) < 9
                    264:   /* 111110 */ TME_SPARC_RECODE_INSN_UNDEF,
                    265: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    266:   /* 111110 */ TME_SPARC_RECODE_INSN_ASSIST_FULL | TME_SPARC_RECODE_INSN_LAST, /* v9: done/retry */
                    267: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    268:   /* 111111 */ TME_SPARC_RECODE_INSN_UNDEF,
                    269: 
                    270:   /* op=3: memory instructions: */
                    271: 
                    272:   /* 000000 */ TME_SPARC_RECODE_INSN_LD, /* ld (v9: lduw) */
                    273:   /* 000001 */ TME_SPARC_RECODE_INSN_LD, /* ldub */
                    274:   /* 000010 */ TME_SPARC_RECODE_INSN_LD, /* lduh */
                    275:   /* 000011 */ TME_SPARC_RECODE_INSN_ASSIST_FULL,  /* ldd */
                    276:   /* 000100 */ TME_SPARC_RECODE_INSN_ST, /* st (v9: stw) */
                    277:   /* 000101 */ TME_SPARC_RECODE_INSN_ST, /* stb */
                    278:   /* 000110 */ TME_SPARC_RECODE_INSN_ST, /* sth */
                    279:   /* 000111 */ TME_SPARC_RECODE_INSN_ASSIST_FULL | TME_SPARC_RECODE_INSN_NO_RD,  /* std */
                    280: #if TME_SPARC_VERSION(ic) < 9
                    281:   /* 001000 */ TME_SPARC_RECODE_INSN_UNDEF,
                    282: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    283:   /* 001000 */ TME_SPARC_RECODE_INSN_LD, /* v9: ldsw */
                    284: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    285:   /* 001001 */ TME_SPARC_RECODE_INSN_LD, /* ldsb */
                    286:   /* 001010 */ TME_SPARC_RECODE_INSN_LD, /* ldsh */
                    287: #if TME_SPARC_VERSION(ic) < 9
                    288:   /* 001011 */ TME_SPARC_RECODE_INSN_UNDEF,
                    289: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    290:   /* 001011 */ TME_SPARC_RECODE_INSN_LD, /* v9: ldx */
                    291: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    292:   /* 001100 */ TME_SPARC_RECODE_INSN_UNDEF,
                    293:   /* 001101 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* ldstub */
                    294: #if TME_SPARC_VERSION(ic) < 9
                    295:   /* 001110 */ TME_SPARC_RECODE_INSN_UNDEF,
                    296: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    297:   /* 001110 */ TME_SPARC_RECODE_INSN_ST, /* v9: stx */
                    298: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    299:   /* 001111 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* swap */
                    300:   /* 010000 */ TME_SPARC_RECODE_INSN_LDA, /* lda (v9: lduwa) */
                    301:   /* 010001 */ TME_SPARC_RECODE_INSN_LDA, /* lduba */
                    302:   /* 010010 */ TME_SPARC_RECODE_INSN_LDA, /* lduha */
                    303:   /* 010011 */ TME_SPARC_RECODE_INSN_ASSIST_FULL,  /* ldda */
                    304:   /* 010100 */ TME_SPARC_RECODE_INSN_ASSIST_FULL | TME_SPARC_RECODE_INSN_NO_RD, /* sta (v9: stwa) */
                    305:   /* 010101 */ TME_SPARC_RECODE_INSN_ASSIST_FULL | TME_SPARC_RECODE_INSN_NO_RD, /* stba */
                    306:   /* 010110 */ TME_SPARC_RECODE_INSN_ASSIST_FULL | TME_SPARC_RECODE_INSN_NO_RD, /* stha */
                    307:   /* 010111 */ TME_SPARC_RECODE_INSN_ASSIST_FULL | TME_SPARC_RECODE_INSN_NO_RD, /* stda */
                    308: #if TME_SPARC_VERSION(ic) < 9
                    309:   /* 011000 */ TME_SPARC_RECODE_INSN_UNDEF,
                    310: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    311:   /* 011000 */ TME_SPARC_RECODE_INSN_LDA, /* v9: ldswa*/
                    312: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    313:   /* 011001 */ TME_SPARC_RECODE_INSN_LDA, /* ldsba */
                    314:   /* 011010 */ TME_SPARC_RECODE_INSN_LDA, /* ldsha */
                    315: #if TME_SPARC_VERSION(ic) < 9
                    316:   /* 011011 */ TME_SPARC_RECODE_INSN_UNDEF,
                    317: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    318:   /* 001011 */ TME_SPARC_RECODE_INSN_LDA, /* v9: ldxa*/
                    319: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    320:   /* 011100 */ TME_SPARC_RECODE_INSN_UNDEF,
                    321:   /* 011101 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* ldstuba */
                    322: #if TME_SPARC_VERSION(ic) < 9
                    323:   /* 011110 */ TME_SPARC_RECODE_INSN_UNDEF,
                    324: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    325:   /* 001110 */ TME_SPARC_RECODE_INSN_ASSIST_FULL | TME_SPARC_RECODE_INSN_NO_RD, /* v9: stxa */
                    326: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    327:   /* 011111 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* swapa */
                    328:   /* 100000 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* ldf */
                    329:   /* 100001 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* ldfsr */
                    330: #if TME_SPARC_VERSION(ic) < 9
                    331:   /* 100010 */ TME_SPARC_RECODE_INSN_UNDEF,
                    332: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    333:   /* 100010 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* v9: ldqf */
                    334: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    335:   /* 100011 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* lddf */
                    336:   /* 100100 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* stf */
                    337:   /* 100101 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* stfsr */
                    338:   /* 100110 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* stdfq */
                    339:   /* 100111 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* stdf */
                    340:   /* 101000 */ TME_SPARC_RECODE_INSN_UNDEF,
                    341:   /* 101001 */ TME_SPARC_RECODE_INSN_UNDEF,
                    342:   /* 101010 */ TME_SPARC_RECODE_INSN_UNDEF,
                    343:   /* 101011 */ TME_SPARC_RECODE_INSN_UNDEF,
                    344:   /* 101100 */ TME_SPARC_RECODE_INSN_UNDEF,
                    345: #if TME_SPARC_VERSION(ic) < 9
                    346:   /* 101101 */ TME_SPARC_RECODE_INSN_UNDEF,
                    347: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    348:   /* 101101 */ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP | TME_SPARC_RECODE_INSN_NO_RD, /* v9: prefetch */
                    349: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    350:   /* 101110 */ TME_SPARC_RECODE_INSN_UNDEF,
                    351:   /* 101111 */ TME_SPARC_RECODE_INSN_UNDEF,
                    352:   /* 110000 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* ldc (v9: ldfa) */
                    353: #if TME_SPARC_VERSION(ic) < 9
                    354:   /* 110001 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* ldcsr */
                    355:   /* 110010 */ TME_SPARC_RECODE_INSN_UNDEF,
                    356: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    357:   /* 110001 */ TME_SPARC_RECODE_INSN_UNDEF,
                    358:   /* 110010 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* v9: ldqfa */
                    359: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    360:   /* 110011 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* lddc (v9: lddfa) */
                    361:   /* 110100 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* stc (v9: stfa) */
                    362: #if TME_SPARC_VERSION(ic) < 9
                    363:   /* 110101 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* stcsr */
                    364: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    365:   /* 110101 */ TME_SPARC_RECODE_INSN_UNDEF,
                    366: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    367:   /* 110110 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* stdcq (v9: stqfa) */
                    368:   /* 110111 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* stdc (v9: stdfa) */
                    369:   /* 111000 */ TME_SPARC_RECODE_INSN_UNDEF,
                    370:   /* 111001 */ TME_SPARC_RECODE_INSN_UNDEF,
                    371:   /* 111010 */ TME_SPARC_RECODE_INSN_UNDEF,
                    372:   /* 111011 */ TME_SPARC_RECODE_INSN_UNDEF,
                    373: #if TME_SPARC_VERSION(ic) < 9
                    374:   /* 111100 */ TME_SPARC_RECODE_INSN_UNDEF,
                    375:   /* 111101 */ TME_SPARC_RECODE_INSN_UNDEF,
                    376:   /* 111110 */ TME_SPARC_RECODE_INSN_UNDEF,
                    377: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    378:   /* 111100 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* v9: casa */
                    379:   /* 111101 */ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP | TME_SPARC_RECODE_INSN_NO_RD, /* v9: prefetch */
                    380:   /* 111110 */ TME_SPARC_RECODE_INSN_ASSIST_FULL, /* v9: casxa */
                    381: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    382:   /* 111111 */ TME_SPARC_RECODE_INSN_UNDEF,
                    383: };
                    384: 
                    385: /* this returns the current instruction: */
                    386: tme_uint32_t
                    387: tme_sparc_recode_insn_current(const struct tme_sparc *ic)
                    388: {
                    389:   const struct tme_token *itlb_current_token;
                    390:   const struct _TME_SPARC_RECODE_SIZE(tme_recode_tlb_c16_a,/**/) *recode_itlb_current;
                    391:   const struct tme_sparc_tlb *itlb_current;
                    392:   tme_uint32_t page_offset;
                    393:   const tme_shared tme_uint32_t *_sparc_insn;
                    394:   tme_uint32_t sparc_insn;
                    395: 
                    396:   /* in a cooperative threading system, the current ITLB entry must be
                    397:      valid.  in a noncooperative threading system, some master may be
                    398:      in the process of invalidating the current ITLB entry: */
                    399:   itlb_current_token = ic->_tme_sparc_itlb_current_token;
                    400:   assert (!TME_THREADS_COOPERATIVE || tme_token_is_valid(itlb_current_token));
                    401: 
                    402:   /* get the current recode ITLB entry: */
                    403:   recode_itlb_current
                    404:     = ((struct _TME_SPARC_RECODE_SIZE(tme_recode_tlb_c16_a,/**/) *)
                    405:        (((const char *) itlb_current_token)
                    406:        - (((char *) &(((struct _TME_SPARC_RECODE_SIZE(tme_recode_tlb_c16_a,/**/) *) 0)->_TME_SPARC_RECODE_SIZE(tme_recode_tlb_c16_a,_token)))
                    407:           - (char *) 0)));
                    408: 
                    409:   /* the current ITLB entry must cover the current context,
                    410:      instruction ASI and PC, and allow fast reading: */
                    411:   itlb_current = &ic->tme_sparc_tlbs[(recode_itlb_current - &ic->_TME_SPARC_RECODE_SIZE(tme_sparc_recode_tlb,s)[0])];
                    412:   assert ((itlb_current->tme_sparc_tlb_context > ic->tme_sparc_memory_context_max
                    413:           || itlb_current->tme_sparc_tlb_context == ic->tme_sparc_memory_context_default)
                    414:          && TME_SPARC_TLB_ASI_MASK_OK(itlb_current, ic->tme_sparc_asi_mask_insn)
                    415:          && itlb_current->tme_sparc_tlb_addr_first <= ic->tme_sparc_ireg(TME_SPARC_IREG_PC)
                    416:          && ic->tme_sparc_ireg(TME_SPARC_IREG_PC) <= itlb_current->tme_sparc_tlb_addr_last
                    417:          && itlb_current->tme_sparc_tlb_emulator_off_read != TME_EMULATOR_OFF_UNDEF);
                    418: 
                    419:   /* fetch the instruction: */
                    420:   page_offset = recode_itlb_current->_TME_SPARC_RECODE_SIZE(tme_recode_tlb_c16_a,_page);
                    421:   page_offset ^= (tme_uint32_t) ic->tme_sparc_ireg(TME_SPARC_IREG_PC);
                    422:   assert ((page_offset >> ic->tme_sparc_tlb_page_size_log2) == 0);
                    423:   _sparc_insn
                    424:     = ((const tme_shared tme_uint32_t *)
                    425:        (recode_itlb_current->_TME_SPARC_RECODE_SIZE(tme_recode_tlb_c16_a,_memory)
                    426:        + page_offset));
                    427:   sparc_insn
                    428:     = tme_memory_bus_read32(_sparc_insn,
                    429:                            (tme_rwlock_t *) NULL,
                    430:                            sizeof(tme_uint32_t),
                    431:                            sizeof(tme_sparc_ireg_t));
                    432:   sparc_insn = tme_betoh_u32(sparc_insn);
                    433: 
                    434:   return (sparc_insn);
                    435: }
                    436: 
                    437: /* this checks for conditions that require a redispatch out of recode
                    438:    instruction thunks: */
                    439: tme_recode_uguest_t
                    440: tme_sparc_recode_insn_assist_redispatch(struct tme_sparc *ic)
                    441: {
                    442:   int redispatch_required;
                    443: 
                    444:   /* if the instruction TLB entry for this PC is invalid, or doesn't
                    445:      cover the PC and context and ASI, or doesn't allow fast reading,
                    446:      or isn't covered by any cacheable, or the page containing this PC
                    447:      isn't still cache-valid (i.e., its contents have changed since we
                    448:      cached recoded instructions for it), a redispatch is required: */
                    449:   /* NB: we check for the current PC, even though it's for an
                    450:      instruction that has already completed.  
                    451: 
                    452:      if PC and PC_next are on the same page, this does the right
                    453:      thing.  if PC and PC_next are on different pages, and the PC page
                    454:      requires a redispatch but the PC_next page does not, this will
                    455:      cause a needless redispatch.  if PC and PC_next are on different
                    456:      pages, and the PC page doesn't require a redispatch but the
                    457:      PC_next page does, that will get handled by either
                    458:      _TME_SPARC_EXECUTE_NAME()/tme_sparc_recode() or the host chain
                    459:      out/chain in, because the current instructions thunk can't
                    460:      cover both PC and PC_next.
                    461: 
                    462:      so, checking for the current PC never does the wrong thing, and
                    463:      is useful for at least _tme_sparc_recode_ls_assist_ld(), which
                    464:      must redispatch after executing a ld instruction that might be
                    465:      different from the ld instruction at the time the instructions
                    466:      thunk was recoded - because if the destination register has been
                    467:      changed in the new instruction, if
                    468:      _tme_sparc_recode_ls_assist_ld() did return to the instructions
                    469:      thunk, it would incorrectly take the return value for the
                    470:      destination register in the original instruction: */
                    471:   redispatch_required
                    472:     = (_tme_sparc_recode_chain_src_key(ic,
                    473:                                       ic->tme_sparc_ireg(TME_SPARC_IREG_PC))
                    474:        == TME_SPARC_RECODE_SRC_KEY_UNDEF);
                    475: 
                    476:   /* if the instruction burst has been set to zero by a
                    477:      TME_BUS_CYCLE_SYNCHRONOUS_EVENT, a redispatch is required: */
                    478:   redispatch_required |= (ic->_tme_sparc_instruction_burst_remaining == 0);
                    479: 
                    480:   /* if a redispatch is required: */
                    481:   if (__tme_predict_false(redispatch_required)) {
                    482: 
                    483:     /* do the redispatch: */
                    484:     tme_sparc_redispatch(ic);
                    485:   }
                    486: 
                    487:   return (0);
                    488: }
                    489: 
                    490: /* the assist functions for format three instructions reference the
                    491:    opmap in struct tme_sparc: */
                    492: 
                    493: /* the assist function for TME_SPARC_RECODE_INSN_ASSIST and
                    494:    TME_SPARC_RECODE_INSN_ASSIST_CANTRAP instructions: */
                    495: static tme_recode_uguest_t
                    496: _tme_sparc_recode_insn_assist(struct tme_ic *_ic,
                    497:                              tme_recode_uguest_t rs1,
                    498:                              tme_recode_uguest_t rs2)
                    499: {
                    500:   struct tme_sparc *ic;
                    501:   tme_uint32_t sparc_insn;
                    502:   tme_uint32_t sparc_opcode;
                    503: #if TME_RECODE_SIZE_GUEST_MAX > TME_SPARC_RECODE_SIZE(ic)
                    504:   tme_sparc_ireg_t rs1_buffer;
                    505:   tme_sparc_ireg_t rs2_buffer;
                    506: #endif /* TME_RECODE_SIZE_GUEST_MAX > TME_SPARC_RECODE_SIZE(ic) */
                    507:   tme_sparc_ireg_t *_rs1;
                    508:   tme_sparc_ireg_t *_rs2;
                    509:   tme_sparc_ireg_t rd;
                    510: 
                    511:   /* recover our ic: */
                    512:   ic = (struct tme_sparc *) _ic;
                    513: 
                    514:   TME_SPARC_STAT(ic, tme_sparc_stats_recode_assist);
                    515: 
                    516:   /* set PC_next_next from PC_next: */
                    517:   /* NB that this sets PC_next_next to garbage unless this instruction
                    518:      was a TME_SPARC_RECODE_INSN_ASSIST_CANTRAP: */
                    519:   ic->tme_sparc_ireg(TME_SPARC_IREG_PC_NEXT_NEXT)
                    520:     = (ic->tme_sparc_ireg(TME_SPARC_IREG_PC_NEXT)
                    521:        + sizeof(tme_uint32_t));
                    522: 
                    523:   /* copy the instruction from the internal register into its normal
                    524:      position: */
                    525:   sparc_insn = ic->tme_sparc_ireg(TME_SPARC_IREG_INSN);
                    526:   ic->_tme_sparc_insn = sparc_insn;
                    527: 
                    528:   /* form the opcode index: */
                    529:   sparc_opcode = TME_FIELD_MASK_EXTRACTU(sparc_insn, (0x3f << 19));
                    530:   sparc_opcode += ((sparc_insn >> (30 - 6)) & 0x40);
                    531:   TME_SPARC_STAT(ic, tme_sparc_stats_recode_assist_opcode[sparc_opcode]);
                    532: 
                    533:   /* run the instruction: */
                    534: #if TME_RECODE_SIZE_GUEST_MAX > TME_SPARC_RECODE_SIZE(ic)
                    535:   rs1_buffer = rs1;
                    536:   rs2_buffer = rs2;
                    537:   _rs1 = &rs1_buffer;
                    538:   _rs2 = &rs2_buffer;
                    539: #else  /* TME_RECODE_SIZE_GUEST_MAX == TME_SPARC_RECODE_SIZE(ic) */
                    540:   _rs1 = &rs1;
                    541:   _rs2 = &rs2;
                    542: #endif /* TME_RECODE_SIZE_GUEST_MAX == TME_SPARC_RECODE_SIZE(ic) */
                    543: #ifdef _TME_SPARC_RECODE_VERIFY
                    544:   rd = 0;
                    545: #endif /* _TME_SPARC_RECODE_VERIFY */
                    546:   (*ic->_tme_sparc_execute_opmap[sparc_opcode])
                    547:     (ic, _rs1, _rs2, &rd);
                    548: 
                    549:   /* return the result: */
                    550:   return (rd);
                    551: }
                    552: 
                    553: /* the assist function for TME_SPARC_RECODE_INSN_ASSIST_FULL
                    554:    instructions: */
                    555: static tme_recode_uguest_t
                    556: _tme_sparc_recode_insn_assist_full(struct tme_ic *_ic,
                    557:                                   tme_recode_uguest_t _rs1,
                    558:                                   tme_recode_uguest_t _rs2)
                    559: {
                    560:   struct tme_sparc *ic;
                    561:   tme_uint32_t sparc_insn;
                    562:   unsigned int reg_rs1;
                    563:   unsigned int reg_rs2;
                    564:   unsigned int reg_rd;
                    565:   tme_uint32_t sparc_opcode;
                    566: 
                    567:   /* recover our ic: */
                    568:   ic = (struct tme_sparc *) _ic;
                    569: 
                    570:   TME_SPARC_STAT(ic, tme_sparc_stats_recode_assist_full);
                    571: 
                    572:   /* set PC_next_next from PC_next: */
                    573:   ic->tme_sparc_ireg(TME_SPARC_IREG_PC_NEXT_NEXT)
                    574:     = (ic->tme_sparc_ireg(TME_SPARC_IREG_PC_NEXT)
                    575:        + sizeof(tme_uint32_t));
                    576: 
                    577:   /* copy the instruction from the internal register into its normal
                    578:      position: */
                    579:   sparc_insn = ic->tme_sparc_ireg(TME_SPARC_IREG_INSN);
                    580:   ic->_tme_sparc_insn = sparc_insn;
                    581: 
                    582:   /* if the i bit is zero: */
                    583:   if (__tme_predict_true((sparc_insn & TME_BIT(13)) == 0)) {
                    584: 
                    585:     /* decode rs2: */
                    586:     reg_rs2 = TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RS2);
                    587:     TME_SPARC_REG_INDEX(ic, reg_rs2);
                    588:   }
                    589: 
                    590:   /* otherwise, the i bit is one: */
                    591:   else {
                    592: 
                    593:     /* decode simm13: */
                    594:     ic->tme_sparc_ireg(TME_SPARC_IREG_TMP(0)) = TME_FIELD_MASK_EXTRACTS(sparc_insn, (tme_sparc_ireg_t) 0x1fff);
                    595:     reg_rs2 = TME_SPARC_IREG_TMP(0);
                    596:   }
                    597: 
                    598:   /* decode rs1: */
                    599:   reg_rs1 = TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RS1);
                    600:   TME_SPARC_REG_INDEX(ic, reg_rs1);
                    601: 
                    602:   /* decode rd: */
                    603:   reg_rd = TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RD);
                    604:   TME_SPARC_REG_INDEX(ic, reg_rd);
                    605: 
                    606:   /* form the opcode index: */
                    607:   sparc_opcode = TME_FIELD_MASK_EXTRACTU(sparc_insn, (0x3f << 19));
                    608:   sparc_opcode += ((sparc_insn >> (30 - 6)) & 0x40);
                    609:   TME_SPARC_STAT(ic, tme_sparc_stats_recode_assist_opcode[sparc_opcode]);
                    610: 
                    611:   /* run the instruction: */
                    612:   (*ic->_tme_sparc_execute_opmap[sparc_opcode])
                    613:     (ic,
                    614:      &ic->tme_sparc_ireg(reg_rs1),
                    615:      &ic->tme_sparc_ireg(reg_rs2),
                    616:      &ic->tme_sparc_ireg(reg_rd));
                    617: 
                    618:   /* set %g0 to zero, in case the instruction changed it: */
                    619:   ic->tme_sparc_ireg(TME_SPARC_G0_OFFSET(ic) + TME_SPARC_IREG_G0) = 0;
                    620: 
                    621:   /* return no result */
                    622:   return (tme_sparc_recode_insn_assist_redispatch(ic));
                    623: 
                    624:   /* unused: */
                    625:   _rs1 = 0;
                    626:   _rs2 = 0;
                    627: }
                    628: 
                    629: /* the assist function for jmpl instructions: */
                    630: static tme_recode_uguest_t
                    631: _tme_sparc_recode_insn_assist_jmpl(struct tme_ic *_ic,
                    632:                                   tme_recode_uguest_t rs1,
                    633:                                   tme_recode_uguest_t rs2)
                    634: {
                    635:   struct tme_sparc *ic;
                    636:   tme_sparc_ireg_t pc_next_next;
                    637:   tme_uint32_t ls_faults;
                    638: 
                    639:   /* recover our ic: */
                    640:   ic = (struct tme_sparc *) _ic;
                    641: 
                    642:   TME_SPARC_STAT(ic, tme_sparc_stats_recode_assist_opcode[0x38 /* jmpl */]);
                    643: 
                    644:   /* get the target address: */
                    645:   pc_next_next = ((tme_sparc_ireg_t) rs1) + (tme_sparc_ireg_t) rs2;
                    646: 
                    647:   /* assume that the target address will not fault: */
                    648:   ls_faults = TME_SPARC_LS_FAULT_NONE;
                    649: 
                    650: #if TME_SPARC_VERSION(ic) >= 9
                    651: 
                    652:   /* mask the target address: */
                    653:   pc_next_next &= ic->tme_sparc_address_mask;
                    654: 
                    655:   /* if the target address is in a virtual address hole: */
                    656:   if (__tme_predict_false((pc_next_next
                    657:                            + ic->tme_sparc64_ireg_va_hole_start)
                    658:                           > ((ic->tme_sparc64_ireg_va_hole_start * 2) - 1))) {
                    659: 
                    660:     /* the target address is out of range: */
                    661:     ls_faults += TME_SPARC64_LS_FAULT_VA_RANGE_NNPC;
                    662:   }
                    663: 
                    664: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    665: 
                    666:   /* if the target address is not 32-bit aligned: */
                    667:   if (__tme_predict_false((((tme_uint32_t) pc_next_next) % sizeof(tme_uint32_t)) != 0)) {
                    668: 
                    669:     /* the target address is misaligned: */
                    670:     ls_faults += TME_SPARC_LS_FAULT_ADDRESS_NOT_ALIGNED;
                    671:   }
                    672: 
                    673:   /* if the target address has faulted: */
                    674:   if (__tme_predict_false(ls_faults != TME_SPARC_LS_FAULT_NONE)) {
                    675: 
                    676:     /* set PC_next from PC: */
                    677:     /* NB: we don't need to set PC_next_next, because we're
                    678:        faulting: */
                    679:     ic->tme_sparc_ireg(TME_SPARC_IREG_PC_NEXT)
                    680:       = (ic->tme_sparc_ireg(TME_SPARC_IREG_PC)
                    681:         + sizeof(tme_uint32_t));
                    682: 
                    683:     /* in case a CPU-specific trap function needs it, store a dummy
                    684:        jmpl instruction: */
                    685:     ic->_tme_sparc_insn
                    686:       = ((((tme_uint32_t) 2) << 30)
                    687:         + (0x38 << 19) /* jmpl */
                    688:         );
                    689: 
                    690:     /* fault: */
                    691:     tme_sparc_nnpc_trap(ic, ls_faults);
                    692:   }
                    693: 
                    694:   /* return the target address: */
                    695:   return (pc_next_next);
                    696: }
                    697: 
                    698: /* the assist function for unimplemented instructions: */
                    699: static tme_recode_uguest_t
                    700: _tme_sparc_recode_insn_assist_unimpl(struct tme_ic *_ic,
                    701:                                     tme_recode_uguest_t _rs1,
                    702:                                     tme_recode_uguest_t _rs2)
                    703: {
                    704:   struct tme_sparc *ic;
                    705: 
                    706:   /* recover our ic: */
                    707:   ic = (struct tme_sparc *) _ic;
                    708: 
                    709:   /* set PC_next_next from PC_next: */
                    710:   ic->tme_sparc_ireg(TME_SPARC_IREG_PC_NEXT_NEXT)
                    711:     = (ic->tme_sparc_ireg(TME_SPARC_IREG_PC_NEXT)
                    712:        + sizeof(tme_uint32_t));
                    713: 
                    714:   /* trap: */
                    715: #if TME_SPARC_VERSION(ic) < 9
                    716:   tme_sparc32_trap(ic, TME_SPARC32_TRAP_illegal_instruction);
                    717: #else  /* TME_SPARC_VERSION(ic) >= 9 */
                    718:   tme_sparc64_trap(ic, TME_SPARC64_TRAP_illegal_instruction);
                    719: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                    720: 
                    721:   /* return no result */
                    722:   return (0);
                    723: 
                    724:   /* unused: */
                    725:   _rs1 = 0;
                    726:   _rs2 = 0;
                    727: }
                    728: 
                    729: #ifdef _TME_SPARC_STATS
                    730: static tme_recode_uguest_t
                    731: _tme_sparc_recode_insns_total(struct tme_ic *_ic,
                    732:                              tme_recode_uguest_t _rs1,
                    733:                              tme_recode_uguest_t _rs2)
                    734: {
                    735:   TME_SPARC_STAT_N(((struct tme_sparc *) _ic),
                    736:                   tme_sparc_stats_recode_insns_total, 
                    737:                   (tme_uint32_t) _rs2);
                    738:   return (0);
                    739: }
                    740: #endif /* _TME_SPARC_STATS */
                    741: 
                    742: /* this returns nonzero if the given instruction can't be recoded in a
                    743:    branch delay slot.  in general, an instruction can't be recoded in
                    744:    a branch delay slot if it may set PC_next_next without doing a
                    745:    redispatch, or if it may use a recode if/endif: */
                    746: static int
                    747: _tme_sparc_recode_insn_branch_delay_bad(tme_uint32_t sparc_insn)
                    748: {
                    749:   unsigned int sparc_opcode;
                    750: 
                    751:   /* if this is a format three instruction, and op is three: */
                    752:   if (sparc_insn >= 0xc0000000) {
                    753: 
                    754:     /* all of the instructions with op equal to three can be recoded
                    755:        in a branch delay slot: */
                    756:     return (FALSE);
                    757:   }
                    758: 
                    759:   /* otherwise, if this is a format three instruction, and op is
                    760:      two: */
                    761:   else if (sparc_insn >= 0x80000000) {
                    762: 
                    763:     /* form the opcode index: */
                    764:     sparc_opcode = TME_FIELD_MASK_EXTRACTU(sparc_insn, (0x3f << 19));
                    765: 
                    766:     /* instructions that may set PC_next_next without doing a
                    767:        redispatch, and instructions that may use a recode if/endif
                    768:        can't be recoded in a branch delay slot: */
                    769:     return (sparc_opcode == 0x38               /* jmpl */
                    770:            || (TME_SPARC_VERSION(ic) >= 9
                    771:                && (sparc_opcode == 0x2c        /* v9: movcc */
                    772:                    || sparc_opcode == 0x2f))); /* v9: movr */
                    773:   }
                    774: 
                    775:   /* otherwise, if this is a format two instruction: */
                    776:   else if (__tme_predict_true(sparc_insn < 0x40000000)) {
                    777: 
                    778:     /* except for sethi instructions, format two instructions can't be
                    779:        recoded in a branch delay slot - the other implemented format
                    780:        two instructions are all branches that set PC_next_next: */
                    781:     return ((sparc_insn & (0x7 << 22)) != (0x4 << 22));
                    782:   }
                    783: 
                    784:   /* otherwise, this is a format three instruction: */
                    785:   else {
                    786: 
                    787:     /* we can't recode call instructions in branch delay slots, since
                    788:        they set PC_next_next: */
                    789:     return (TRUE);
                    790:   }
                    791: }
                    792: 
                    793: /* the sparc instruction recoder: */
                    794: static tme_recode_thunk_off_t
                    795: _tme_sparc_recode_recode(struct tme_sparc *ic,
                    796:                         const struct tme_sparc_tlb *itlb)
                    797: {
                    798:   tme_uint32_t page_size;
                    799:   unsigned int reg_guest_pc;
                    800:   unsigned int recode_size_address;
                    801:   struct tme_recode_insn *recode_insn;
                    802:   struct tme_recode_insn *recode_insns_last_start;
                    803:   tme_sparc_ireg_t pc_next;
                    804:   tme_uint32_t sparc_insn_next;
                    805:   tme_uint32_t sparc_insn_count;
                    806:   tme_uint32_t pc_advance;
                    807:   tme_uint32_t sparc_recode;
                    808:   tme_sparc_ireg_t itlb_addr_last;
                    809:   tme_recode_uguest_t (*sparc_assist) _TME_P((struct tme_ic *, tme_recode_uguest_t, tme_recode_uguest_t));
                    810:   tme_uint32_t sparc_insn;
                    811:   tme_uint32_t sparc_opcode;
                    812:   int recode_operand;
                    813:   tme_sparc_ireg_t imm;
                    814:   unsigned int shift_count_mask;
                    815:   unsigned int cond;
                    816:   const struct tme_recode_conds_thunk *conds_thunk;
                    817:   tme_int32_t branch_displacement_raw;
                    818:   tme_sparc_ireg_t branch_displacement;
                    819:   const struct tme_recode_flags_thunk *flags_thunk;
                    820:   int recode_operand_address;
                    821:   tme_uint32_t chain_info;
                    822: 
                    823:   /* get the page size: */
                    824:   page_size = (((tme_uint32_t) 1) << ic->tme_sparc_tlb_page_size_log2);
                    825: 
                    826:   /* on entry to this function, PC has already been updated to be the
                    827:      first instruction.  on entry to an instructions thunk, PC hasn't
                    828:      been updated yet, and PC_next is the first instruction: */
                    829:   reg_guest_pc = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC_NEXT);
                    830: 
                    831:   /* assume that addresses are guest size: */
                    832:   recode_size_address = TME_SPARC_RECODE_SIZE(ic);
                    833: 
                    834:   /* if this is a v9 CPU and PSTATE.AM is set: */
                    835:   if (TME_SPARC_VERSION(ic) >= 9
                    836:       && (ic->tme_sparc64_ireg_pstate
                    837:          & TME_SPARC64_PSTATE_AM)) {
                    838: 
                    839:     /* addresses are only 32 bits wide: */
                    840:     recode_size_address = TME_RECODE_SIZE_32;
                    841:   }
                    842: 
                    843:   /* start at the beginning of the recode instructions buffer: */
                    844:   recode_insn = &ic->tme_sparc_recode_insns[0];
                    845: 
                    846:   /* calculate the worst-case position in the recode instructions
                    847:      buffer, past which we may not be able to recode another sparc
                    848:      instruction.  this is actually an upper bound on the worst case,
                    849:      found by counting the number of "recode_insn++" statements in
                    850:      this function, and multiplying that by two so that branch
                    851:      instructions can loop to recode a branch delay slot: */
                    852:   recode_insns_last_start = &ic->tme_sparc_recode_insns[TME_ARRAY_ELS(ic->tme_sparc_recode_insns) - (35 * 2)];
                    853: 
                    854:   /* fetch the first instruction as the next instruction: */
                    855:   pc_next = ic->tme_sparc_ireg(TME_SPARC_IREG_PC);
                    856:   sparc_insn_next
                    857:     = tme_memory_bus_read32(((const tme_shared tme_uint32_t *)
                    858:                             (itlb->tme_sparc_tlb_emulator_off_read
                    859:                              + pc_next)),
                    860:                            itlb->tme_sparc_tlb_bus_rwlock,
                    861:                            sizeof(tme_uint32_t),
                    862:                            sizeof(tme_sparc_ireg_t));
                    863:   sparc_insn_next = tme_betoh_u32(sparc_insn_next);
                    864:   sparc_insn_count = 0;
                    865: 
                    866:   /* the next instruction will be the first instruction: */
                    867:   pc_advance = 0 - (tme_uint32_t) sizeof(sparc_insn_next);
                    868: 
                    869:   /* clear the recode information: */
                    870:   sparc_recode = 0;
                    871: 
                    872:   /* if PC is at the last instruction address before tme_sparc_ireg_t
                    873:      wrapping: */
                    874:   /* NB: we don't consider the v9 address mask here, because we only
                    875:      need to protect against the pc_next local variable itself
                    876:      wrapping its tme_sparc_ireg_t.  if the v9 address mask is set for
                    877:      32 bits and PC is 0xfffffffc, pc_next will not wrap to zero -
                    878:      instead it will advance to 0x100000000, but we also won't
                    879:      continue to recode because that address is on a different
                    880:      page: */
                    881:   assert (TME_SPARC_VERSION(ic) < 9
                    882:          || (pc_next < ic->tme_sparc_address_mask));
                    883:   if (__tme_predict_false(pc_next
                    884:                          == (tme_sparc_ireg_t) (0 - (tme_sparc_ireg_t) sizeof(tme_uint32_t)))) {
                    885: 
                    886:     /* recode is impossible here: */
                    887:     return (0);
                    888:   }
                    889: 
                    890:   /* stop recoding after the last instruction on the page, or right
                    891:      before the last instruction before tme_sparc_ireg_t wrapping,
                    892:      whichever comes first: */
                    893:   itlb_addr_last
                    894:     = ((pc_next
                    895:        | (page_size - 1))
                    896:        - (sizeof(tme_uint32_t) - 1));
                    897:   if (__tme_predict_false(itlb_addr_last
                    898:                          == (tme_sparc_ireg_t) (0 - (tme_sparc_ireg_t) sizeof(tme_uint32_t)))) {
                    899:     itlb_addr_last -= sizeof(tme_uint32_t);
                    900:   }
                    901:   assert (itlb_addr_last >= pc_next);
                    902:   assert (itlb->tme_sparc_tlb_addr_last >= itlb_addr_last);
                    903: 
                    904:   /* loop recoding instructions: */
                    905:   for (;;) {
                    906: 
                    907:     /* reset the assist function: */
                    908:     sparc_assist = _tme_sparc_recode_insn_assist;
                    909: 
                    910:     /* make the next instruction the current instruction: */
                    911:     pc_advance += sizeof(sparc_insn);
                    912:     sparc_insn = sparc_insn_next;
                    913:     sparc_insn_count++;
                    914: 
                    915:     /* advance the PC of the next instruction: */
                    916:     pc_next += sizeof(sparc_insn);
                    917: 
                    918:     /* if the PC of the next instruction is still in this
                    919:        page: */
                    920:     if (__tme_predict_true(pc_next <= itlb_addr_last)) {
                    921: 
                    922:       /* fetch the next instruction: */
                    923:       sparc_insn_next
                    924:        = tme_memory_bus_read32(((const tme_shared tme_uint32_t *)
                    925:                                 (itlb->tme_sparc_tlb_emulator_off_read
                    926:                                  + pc_next)),
                    927:                                itlb->tme_sparc_tlb_bus_rwlock,
                    928:                                sizeof(tme_uint32_t),
                    929:                                sizeof(tme_sparc_ireg_t));
                    930:       sparc_insn_next = tme_betoh_u32(sparc_insn_next);
                    931:     }
                    932: 
                    933:     /* otherwise, the instruction TLB entry doesn't cover the PC of
                    934:        the next instruction: */
                    935:     else {
                    936: 
                    937:       /* the current instruction is the last instruction, and we need
                    938:         to update the PC registers: */
                    939:       sparc_recode |= TME_SPARC_RECODE_INSN_LAST | TME_SPARC_RECODE_INSN_UPDATE_PCS;
                    940: 
                    941:       /* in case the current instruction is a control transfer
                    942:         instruction, poison the next instruction to look like another
                    943:         control transfer instruction.  we use a call instruction: */
                    944:       sparc_insn_next = 0x40000000;
                    945:     }
                    946: 
                    947:     /* if this is a format three instruction (op is two or three): */
                    948:     if (__tme_predict_true(sparc_insn >= 0x80000000)) {
                    949: 
                    950:       /* form the opcode index: */
                    951:       sparc_opcode = TME_FIELD_MASK_EXTRACTU(sparc_insn, (0x3f << 19));
                    952:       sparc_opcode += ((sparc_insn >> (30 - 6)) & 0x40);
                    953: 
                    954:       /* add in the recode information for this instruction: */
                    955:       sparc_recode |= _tme_sparc_recode_insn_opmap[sparc_opcode];
                    956: 
                    957:       /* if this is a jmpl instruction: */
                    958:       if (__tme_predict_false(sparc_opcode == 0x38)) {
                    959: 
                    960:        /* if the next instruction can't be recoded when it's in a
                    961:           branch delay slot (usually because it's also a control
                    962:           transfer instruction): */
                    963:        if (__tme_predict_false(_tme_sparc_recode_insn_branch_delay_bad(sparc_insn_next))) {
                    964: 
                    965:          /* if we haven't recoded any instructions yet, recode is
                    966:             impossible here: */
                    967:          if (recode_insn == &ic->tme_sparc_recode_insns[0]) {
                    968:            return (0);
                    969:          }
                    970:          
                    971:          /* stop recoding immediately, leaving the updated PC
                    972:             pointing to the instruction before the jmpl: */
                    973:          assert (pc_advance >= sizeof(sparc_insn));
                    974:          pc_advance -= sizeof(sparc_insn);
                    975:          sparc_insn_count--;
                    976:          sparc_recode
                    977:            |= (TME_SPARC_RECODE_INSN_UPDATE_PCS
                    978:                | TME_SPARC_RECODE_INSN_NONE
                    979:                | TME_SPARC_RECODE_INSN_LAST);
                    980:        }
                    981: 
                    982:        /* otherwise, the next instruction can be recoded when it's in
                    983:           a branch delay slot: */
                    984:        else {
                    985: 
                    986:          /* if PC doesn't already point to the jmpl instruction: */
                    987:          if (pc_advance != 0
                    988:              || reg_guest_pc != TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC)) {
                    989: 
                    990:            /* advance PC to the jmpl instruction: */
                    991:            recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                    992:            recode_insn->tme_recode_insn_size = recode_size_address;
                    993:            recode_insn->tme_recode_insn_operand_src[0] = reg_guest_pc;
                    994:            recode_insn->tme_recode_insn_operand_src[1]
                    995:              = (pc_advance == 0
                    996:                 ? TME_RECODE_OPERAND_ZERO
                    997:                 : TME_RECODE_OPERAND_IMM);
                    998:            recode_insn->tme_recode_insn_imm_uguest = pc_advance;
                    999:            recode_insn->tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC);
                   1000:            recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1001:            recode_insn++;
                   1002:          }
                   1003: 
                   1004:          /* if rs1 is %i7 or %o7, assume that this jmpl is doing a
                   1005:             return, otherwise assume that this jmpl is doing a
                   1006:             call.  either way, the branch delay slot will be the
                   1007:             last instruction: */
                   1008:          recode_operand = TME_RECODE_REG_GUEST(TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RS1));
                   1009:          sparc_recode
                   1010:            |= ((((recode_operand
                   1011:                   - TME_RECODE_REG_GUEST(TME_SPARC_IREG_G0))
                   1012:                  & 15 /* %o7 */)
                   1013:                 == 15 /* %o7 */)
                   1014:                ? (TME_SPARC_RECODE_INSN_LAST
                   1015:                   + TME_SPARC_RECODE_CHAIN_INFO(TME_RECODE_CHAIN_INFO_RETURN
                   1016:                                                 + TME_RECODE_CHAIN_INFO_FAR
                   1017:                                                 + TME_RECODE_CHAIN_INFO_UNCONDITIONAL))
                   1018:                : (TME_SPARC_RECODE_INSN_LAST
                   1019:                   + TME_SPARC_RECODE_CHAIN_INFO(TME_RECODE_CHAIN_INFO_CALL
                   1020:                                                 + TME_RECODE_CHAIN_INFO_FAR
                   1021:                                                 + TME_RECODE_CHAIN_INFO_UNCONDITIONAL)));
                   1022: 
                   1023:          /* call the jmpl assist function.  if this doesn't trap, it
                   1024:             returns the target address for PC_next_next, which we put
                   1025:             directly into PC_next: */
                   1026:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_GUEST;
                   1027:          recode_insn->tme_recode_insn_size = recode_size_address;
                   1028:          recode_insn->tme_recode_insn_operand_src[0]
                   1029:            = (recode_operand == TME_RECODE_REG_GUEST(TME_SPARC_IREG_G0)
                   1030:               ? TME_RECODE_OPERAND_ZERO
                   1031:               : recode_operand);
                   1032:          if (__tme_predict_true(sparc_insn & TME_BIT(13))) {
                   1033:            recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_OPERAND_IMM;
                   1034:            recode_insn->tme_recode_insn_imm_uguest
                   1035:              = TME_FIELD_MASK_EXTRACTS(sparc_insn, (tme_sparc_ireg_t) 0x1fff);
                   1036:          }
                   1037:          else {
                   1038:            recode_insn->tme_recode_insn_operand_src[1]
                   1039:              = TME_RECODE_REG_GUEST(TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RS2));
                   1040:          }
                   1041:          recode_insn->tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC_NEXT);
                   1042:          recode_insn->tme_recode_insn_guest_func = _tme_sparc_recode_insn_assist_jmpl;
                   1043:          recode_insn++;
                   1044: 
                   1045:          /* assume that rd is %g0: */
                   1046:          reg_guest_pc = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC);
                   1047: 
                   1048:          /* decode rd: */
                   1049:          recode_operand = TME_RECODE_REG_GUEST(TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RD));
                   1050: 
                   1051:          /* if rd is not %g0: */
                   1052:          if (recode_operand != TME_RECODE_REG_GUEST(TME_SPARC_IREG_G0)) {
                   1053: 
                   1054:            /* we will soon be able to find the PC of the jmpl in rd: */
                   1055:            reg_guest_pc = recode_operand;
                   1056: 
                   1057:            /* write the PC of the jmpl into the destination register: */
                   1058:            /* NB: we don't use recode_size_address here because we need to
                   1059:               update the entire destination register: */
                   1060:            recode_insn->tme_recode_insn_operand_dst = recode_operand;
                   1061:            recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                   1062:            recode_insn->tme_recode_insn_size = TME_SPARC_RECODE_SIZE(ic);
                   1063:            recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_OPERAND_ZERO;
                   1064:            recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC);
                   1065:            recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1066:            recode_insn++;
                   1067:          }
                   1068: 
                   1069:          /* advance PC to the branch delay slot: */
                   1070:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                   1071:          recode_insn->tme_recode_insn_size = recode_size_address;
                   1072:          recode_insn->tme_recode_insn_operand_src[0] = reg_guest_pc;
                   1073:          recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_OPERAND_IMM;
                   1074:          recode_insn->tme_recode_insn_imm_uguest = sizeof(sparc_insn);
                   1075:          recode_insn->tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC);
                   1076:          recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1077:          recode_insn++;
                   1078:          pc_advance = 0 - (tme_uint32_t) sizeof(sparc_insn);
                   1079:          reg_guest_pc = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC);
                   1080: 
                   1081:          /* loop now to fetch and recode the instruction in the branch
                   1082:             delay slot: */
                   1083:          continue;
                   1084:        }
                   1085:       }
                   1086: 
                   1087:       /* otherwise, if this is a v9 movcc instruction: */
                   1088:       else if (__tme_predict_false(TME_SPARC_VERSION(ic) >= 9
                   1089:                                   && sparc_opcode == 0x2c)) {
                   1090: 
                   1091:        /* if cc2 is clear, or if cc1 is set, this is not an integer
                   1092:           movcc: */
                   1093:        if (__tme_predict_false((sparc_insn
                   1094:                                 & (TME_BIT(18)
                   1095:                                    | TME_BIT(11)))
                   1096:                                != TME_BIT(18))) {
                   1097: 
                   1098:          /* this instruction needs a full assist: */
                   1099:          sparc_recode |= TME_SPARC_RECODE_INSN_ASSIST_FULL;
                   1100:        }
                   1101: 
                   1102:        /* otherwise, this is an integer movcc: */
                   1103:        else {
                   1104: 
                   1105:          /* get the condition field: */
                   1106:          cond = TME_FIELD_MASK_EXTRACTU(sparc_insn, (0xf << 14));
                   1107: 
                   1108:          /* get the conditions thunk to test: */
                   1109:          conds_thunk
                   1110:            = ((sparc_insn & TME_BIT(12))
                   1111:               ? ic->tme_sparc_recode_conds_thunk_xcc
                   1112:               : ic->tme_sparc_recode_conds_thunk_icc);
                   1113: 
                   1114:          /* define the first recode flag with the condition: */
                   1115:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_DEFC;
                   1116:          recode_insn->tme_recode_insn_operand_src[0] = (cond % TME_SPARC_COND_NOT);
                   1117:          recode_insn->tme_recode_insn_operand_src[1] = 0 - (int) (cond / TME_SPARC_COND_NOT);
                   1118:          recode_insn->tme_recode_insn_operand_dst = TME_RECODE_FLAG(0);
                   1119:          recode_insn->tme_recode_insn_conds_thunk = conds_thunk;
                   1120:          recode_insn++;
                   1121: 
                   1122:          /* emit the if to test the first recode carry flag: */
                   1123:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_IF;
                   1124:          recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_FLAG(0);
                   1125:          recode_insn++;
                   1126: 
                   1127:          /* move the source operand to the destination register: */
                   1128:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                   1129:          recode_insn->tme_recode_insn_size = TME_SPARC_RECODE_SIZE(ic);
                   1130:          recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_OPERAND_ZERO;
                   1131:          if (sparc_insn & TME_BIT(13)) {
                   1132:            recode_insn->tme_recode_insn_imm_uguest
                   1133:              = TME_FIELD_MASK_EXTRACTS(sparc_insn, (tme_sparc_ireg_t) 0x7ff);
                   1134:            recode_operand = TME_RECODE_OPERAND_IMM;
                   1135:          }
                   1136:          else {
                   1137:            recode_operand = TME_RECODE_REG_GUEST(TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RS2));
                   1138:          }
                   1139:          recode_insn->tme_recode_insn_operand_src[1] = recode_operand;
                   1140:          recode_insn->tme_recode_insn_operand_dst
                   1141:            = TME_RECODE_REG_GUEST(TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RD));
                   1142:          recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1143:          recode_insn++;
                   1144: 
                   1145:          /* emit the endif: */
                   1146:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ENDIF;
                   1147:          recode_insn++;
                   1148: 
                   1149:          /* suppress any instructions below: */
                   1150:          sparc_recode |= TME_SPARC_RECODE_INSN_NONE;
                   1151:        }
                   1152:       }
                   1153: 
                   1154:       /* otherwise, if this is a v9 movr instruction: */
                   1155:       else if (__tme_predict_false(TME_SPARC_VERSION(ic) >= 9
                   1156:                                   && sparc_opcode == 0x2f)) {
                   1157: 
                   1158:        /* get the condition field: */
                   1159:        cond = TME_FIELD_MASK_EXTRACTU(sparc_insn, (0x7 << 10));
                   1160: 
                   1161:        /* this shifts the "not" bit in the condition up from bit
                   1162:           two to bit three, to match the other branches: */
                   1163:        cond = (cond + 4) & (TME_SPARC_COND_NOT | 3);
                   1164: 
                   1165:        /* if this is an unimplemented movr: */
                   1166:        if (__tme_predict_false(!TME_SPARC_COND_IS_CONDITIONAL(cond))) {
                   1167: 
                   1168:          /* this instruction needs a full assist: */
                   1169:          sparc_recode |= TME_SPARC_RECODE_INSN_ASSIST_FULL;
                   1170:        }
                   1171: 
                   1172:        /* otherwise, this is an implemented movr: */
                   1173:        else {
                   1174: 
                   1175:          /* test the register and set the internal "rcc" flags: */
                   1176:          recode_operand = TME_RECODE_REG_GUEST(TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RS1));
                   1177:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_AND;
                   1178:          recode_insn->tme_recode_insn_size = TME_SPARC_RECODE_SIZE(ic);
                   1179:          recode_insn->tme_recode_insn_operand_src[0] = recode_operand;
                   1180:          recode_insn->tme_recode_insn_operand_src[1] = recode_operand;
                   1181:          recode_insn->tme_recode_insn_operand_dst = TME_RECODE_OPERAND_NULL;
                   1182:          recode_insn->tme_recode_insn_flags_thunk = ic->tme_sparc_recode_flags_thunk_rcc;
                   1183:          recode_insn++;
                   1184: 
                   1185:          /* define the first recode flag with the condition: */
                   1186:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_DEFC;
                   1187:          recode_insn->tme_recode_insn_operand_src[0] = (cond % TME_SPARC_COND_NOT);
                   1188:          recode_insn->tme_recode_insn_operand_src[1] = 0 - (int) (cond / TME_SPARC_COND_NOT);
                   1189:          recode_insn->tme_recode_insn_operand_dst = TME_RECODE_FLAG(0);
                   1190:          recode_insn->tme_recode_insn_conds_thunk = ic->tme_sparc_recode_conds_thunk_rcc;
                   1191:          recode_insn++;
                   1192: 
                   1193:          /* emit the if to test the first recode carry flag: */
                   1194:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_IF;
                   1195:          recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_FLAG(0);
                   1196:          recode_insn++;
                   1197: 
                   1198:          /* move the source operand to the destination register: */
                   1199:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                   1200:          recode_insn->tme_recode_insn_size = TME_SPARC_RECODE_SIZE(ic);
                   1201:          recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_OPERAND_ZERO;
                   1202:          if (sparc_insn & TME_BIT(13)) {
                   1203:            recode_insn->tme_recode_insn_imm_uguest
                   1204:              = TME_FIELD_MASK_EXTRACTS(sparc_insn, (tme_sparc_ireg_t) 0x3ff);
                   1205:            recode_operand = TME_RECODE_OPERAND_IMM;
                   1206:          }
                   1207:          else {
                   1208:            recode_operand = TME_RECODE_REG_GUEST(TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RS2));
                   1209:          }
                   1210:          recode_insn->tme_recode_insn_operand_src[1] = recode_operand;
                   1211:          recode_insn->tme_recode_insn_operand_dst
                   1212:            = TME_RECODE_REG_GUEST(TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RD));
                   1213:          recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1214:          recode_insn++;
                   1215: 
                   1216:          /* emit the endif: */
                   1217:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ENDIF;
                   1218:          recode_insn++;
                   1219: 
                   1220:          /* suppress any instructions below: */
                   1221:          sparc_recode |= TME_SPARC_RECODE_INSN_NONE;
                   1222:        }
                   1223:       }
                   1224: 
                   1225:       /* otherwise, if this is a v9 ld*a instruction: */
                   1226:       /* NB: all of the simple ld and st instructions that we handle
                   1227:         here, have bit 21 (bit 2 in the opcode) set for a st, and
                   1228:         clear for a ld: */
                   1229:       else if (__tme_predict_false(TME_SPARC_VERSION(ic) >= 9
                   1230:                                   && TME_SPARC_RECODE_INSN_OPCODE(sparc_recode) == TME_RECODE_OPCODE_RW
                   1231:                                   && ((sparc_opcode
                   1232:                                        & (TME_BIT(4)
                   1233:                                           + TME_BIT(2)))
                   1234:                                       == (TME_BIT(4)
                   1235:                                           + !TME_BIT(2))))) {
                   1236: 
                   1237:        /* if the i bit is clear, and the immediate ASI is not
                   1238:           ASI_PRIMARY_NOFAULT or ASI_PRIMARY_NOFAULT_LITTLE: */
                   1239:        if ((sparc_insn & TME_BIT(13)) == 0
                   1240:            && (TME_FIELD_MASK_EXTRACTU(sparc_insn, (0xff << 5))
                   1241:                != (TME_SPARC64_ASI_FLAG_UNRESTRICTED
                   1242:                    + ((ic->tme_sparc64_ireg_pstate & TME_SPARC64_PSTATE_CLE)
                   1243:                       ? TME_SPARC64_ASI_FLAG_LITTLE
                   1244:                       : !TME_SPARC64_ASI_FLAG_LITTLE)
                   1245:                    + TME_SPARC64_ASI_FLAG_NO_FAULT))) {
                   1246: 
                   1247:          /* this instruction can't be a simple ld instruction: */
                   1248:          sparc_recode
                   1249:            ^= (TME_SPARC_RECODE_INSN_LDA
                   1250:                ^ TME_SPARC_RECODE_INSN_ASSIST_CANTRAP);
                   1251:        }
                   1252:       }
                   1253: 
                   1254:       /* if this instruction needs a full assist: */
                   1255:       if ((sparc_recode
                   1256:           & (TME_SPARC_RECODE_INSN_OPCODE_MASK
                   1257:              | TME_SPARC_RECODE_INSN_NO_RS2
                   1258:              | TME_SPARC_RECODE_INSN_NO_RS1))
                   1259:          == (TME_RECODE_OPCODE_GUEST
                   1260:              | TME_SPARC_RECODE_INSN_NO_RS2
                   1261:              | TME_SPARC_RECODE_INSN_NO_RS1)) {
                   1262: 
                   1263:        /* set the full assist function: */
                   1264:        sparc_assist = _tme_sparc_recode_insn_assist_full;
                   1265:       }
                   1266:     }
                   1267: 
                   1268:     /* otherwise, if this is a format two instruction: */
                   1269:     else if (__tme_predict_true(sparc_insn < 0x40000000)) {
                   1270: 
                   1271:       /* dispatch on op2: */
                   1272:       switch (TME_FIELD_MASK_EXTRACTU(sparc_insn, (0x7 << 22))) {
                   1273: 
                   1274:       default:
                   1275:       case 0: /* UNIMP: */
                   1276:        sparc_recode |= TME_SPARC_RECODE_INSN_ASSIST_FULL | TME_SPARC_RECODE_INSN_LAST;
                   1277:        sparc_assist = _tme_sparc_recode_insn_assist_unimpl;
                   1278:        break;
                   1279: 
                   1280: #if TME_SPARC_VERSION(ic) >= 9
                   1281:       case 1: /* BPcc */
                   1282:       case 3: /* BPr */
                   1283: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                   1284:       case 2: /* Bicc: */
                   1285: 
                   1286:        /* if the next instruction can't be recoded when it's in a
                   1287:           branch delay slot (usually because it's also a control
                   1288:           transfer instruction): */
                   1289:        if (__tme_predict_false(_tme_sparc_recode_insn_branch_delay_bad(sparc_insn_next))) {
                   1290: 
                   1291:          /* if we haven't recoded any instructions yet, recode is
                   1292:             impossible here: */
                   1293:          if (recode_insn == &ic->tme_sparc_recode_insns[0]) {
                   1294:            return (0);
                   1295:          }
                   1296:          
                   1297:          /* stop recoding immediately, leaving the updated PC
                   1298:             pointing to the instruction before the jmpl: */
                   1299:          assert (pc_advance >= sizeof(sparc_insn));
                   1300:          pc_advance -= sizeof(sparc_insn);
                   1301:          sparc_insn_count--;
                   1302:          sparc_recode
                   1303:            |= (TME_SPARC_RECODE_INSN_UPDATE_PCS
                   1304:                | TME_SPARC_RECODE_INSN_NONE
                   1305:                | TME_SPARC_RECODE_INSN_LAST);
                   1306:          break;
                   1307:        }
                   1308: 
                   1309:        /* get the condition field: */
                   1310:        cond = TME_FIELD_MASK_EXTRACTU(sparc_insn, (0xf << 25));
                   1311: 
                   1312:        /* assume that this branch instruction tests a condition in
                   1313:           the icc flags: */
                   1314:        conds_thunk = ic->tme_sparc_recode_conds_thunk_icc;
                   1315: 
                   1316:        /* if this is a BPr or a BPcc: */
                   1317:        if (TME_SPARC_VERSION(ic) >= 9
                   1318:            && (sparc_insn & (0x1 << 22))) {
                   1319: 
                   1320:          /* if this is a BPr: */
                   1321:          if (sparc_insn & (0x2 << 22)) {
                   1322: 
                   1323:            /* if this is an unimplemented BPr instruction: */
                   1324:            if (__tme_predict_false((sparc_insn & (1 << 28)) != 0
                   1325:                                    || (sparc_insn & (3 << 25)) == 0)) {
                   1326: 
                   1327:              /* do a full assist and stop recoding: */
                   1328:              sparc_recode |= TME_SPARC_RECODE_INSN_ASSIST_FULL | TME_SPARC_RECODE_INSN_LAST;
                   1329:              sparc_assist = _tme_sparc_recode_insn_assist_unimpl;
                   1330:              break;
                   1331:            }
                   1332: 
                   1333:            /* test the register and set the internal "rcc" flags: */
                   1334:            recode_operand = TME_RECODE_REG_GUEST(TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RS1));
                   1335:            recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_AND;
                   1336:            recode_insn->tme_recode_insn_size = TME_SPARC_RECODE_SIZE(ic);
                   1337:            recode_insn->tme_recode_insn_operand_src[0] = recode_operand;
                   1338:            recode_insn->tme_recode_insn_operand_src[1] = recode_operand;
                   1339:            recode_insn->tme_recode_insn_operand_dst = TME_RECODE_OPERAND_NULL;
                   1340:            recode_insn->tme_recode_insn_flags_thunk = ic->tme_sparc_recode_flags_thunk_rcc;
                   1341:            recode_insn++;
                   1342: 
                   1343:            /* this shifts the "not" bit in the condition up from bit
                   1344:               two to bit three, to match the other branches: */
                   1345:            cond = (cond + 4) & (TME_SPARC_COND_NOT | 3);
                   1346: 
                   1347:            /* we need to test the condition in the internal "rcc" flags: */
                   1348:            conds_thunk = ic->tme_sparc_recode_conds_thunk_rcc;
                   1349: 
                   1350:            /* get the raw branch displacement: */
                   1351:            branch_displacement_raw
                   1352:              = ((tme_int16_t)
                   1353:                 (((sparc_insn & (0x3 << 20)) >> 6)
                   1354:                  | (sparc_insn & 0x3fff)));
                   1355:          }
                   1356: 
                   1357:          /* otherwise, this is a BPcc: */
                   1358:          else {
                   1359: 
                   1360:            /* if this is an unimplemented BPcc instruction: */
                   1361:            if (__tme_predict_false((sparc_insn & (1 << 20)) != 0)) {
                   1362: 
                   1363:              /* do a full assist and stop recoding: */
                   1364:              sparc_recode |= TME_SPARC_RECODE_INSN_ASSIST_FULL | TME_SPARC_RECODE_INSN_LAST;
                   1365:              sparc_assist = _tme_sparc_recode_insn_assist_unimpl;
                   1366:              break;
                   1367:            }
                   1368: 
                   1369:            /* if this BPcc is testing the xcc flags: */
                   1370:            if (sparc_insn & (1 << 21)) {
                   1371:              conds_thunk = ic->tme_sparc_recode_conds_thunk_xcc;
                   1372:            }
                   1373: 
                   1374:            /* get the raw branch displacement: */
                   1375:            branch_displacement_raw
                   1376:              = ((tme_int32_t)
                   1377:                 TME_FIELD_MASK_EXTRACTS(sparc_insn, 0x0007ffff));
                   1378:          }
                   1379:        }
                   1380: 
                   1381:        /* otherwise, this is a Bicc: */
                   1382:        else {
                   1383: 
                   1384:          /* get the raw branch displacement: */
                   1385:          branch_displacement_raw
                   1386:            = ((tme_int32_t)
                   1387:               TME_FIELD_MASK_EXTRACTS(sparc_insn, 0x003fffff));
                   1388:        }
                   1389: 
                   1390:        /* fully sign-extend the branch displacement: */
                   1391:        branch_displacement
                   1392:          = ((tme_sparc_ireg_t)
                   1393:             (tme_recode_guest_t)
                   1394:             branch_displacement_raw);
                   1395: 
                   1396:        /* advance PC to the branch delay slot: */
                   1397:        recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                   1398:        recode_insn->tme_recode_insn_size = recode_size_address;
                   1399:        recode_insn->tme_recode_insn_operand_src[0] = reg_guest_pc;
                   1400:        recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_OPERAND_IMM;
                   1401:        recode_insn->tme_recode_insn_imm_uguest = pc_advance + sizeof(sparc_insn);
                   1402:        recode_insn->tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC);
                   1403:        recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1404:        recode_insn++;
                   1405:        pc_advance = 0 - (tme_uint32_t) sizeof(sparc_insn);
                   1406:        reg_guest_pc = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC);
                   1407: 
                   1408:        /* if this isn't a branch always: */
                   1409:        if (cond != (TME_SPARC_COND_NOT | TME_SPARC_COND_N)) {
                   1410: 
                   1411:          /* assuming that the branch won't be taken, set PC_next to
                   1412:             the instruction following the branch delay slot: */
                   1413:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                   1414:          recode_insn->tme_recode_insn_size = recode_size_address;
                   1415:          recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC);
                   1416:          recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_OPERAND_IMM;
                   1417:          recode_insn->tme_recode_insn_imm_uguest = sizeof(sparc_insn);
                   1418:          recode_insn->tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC_NEXT);
                   1419:          recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1420:          recode_insn++;
                   1421:        }
                   1422: 
                   1423:        /* if this is a conditional branch: */
                   1424:        if (TME_SPARC_COND_IS_CONDITIONAL(cond)) {
                   1425: 
                   1426:          /* define the recode jump flag with the condition: */
                   1427:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_DEFC;
                   1428:          recode_insn->tme_recode_insn_operand_src[0] = (cond % TME_SPARC_COND_NOT);
                   1429:          recode_insn->tme_recode_insn_operand_src[1] = 0 - (int) (cond / TME_SPARC_COND_NOT);
                   1430:          recode_insn->tme_recode_insn_operand_dst = TME_RECODE_FLAG_JUMP;
                   1431:          recode_insn->tme_recode_insn_conds_thunk = conds_thunk;
                   1432:          recode_insn++;
                   1433: 
                   1434:          /* emit the if to test the recode jump flag: */
                   1435:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_IF;
                   1436:          recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_FLAG_JUMP;
                   1437:          recode_insn++;
                   1438:        }
                   1439: 
                   1440:        /* if this isn't a branch never: */
                   1441:        if (cond != TME_SPARC_COND_N) {
                   1442: 
                   1443:          /* if this branch target is within this same page, this chain
                   1444:             can be near, otherwise it must be far: */
                   1445:          if ((((pc_next - sizeof(sparc_insn))
                   1446:                + (branch_displacement * sizeof(sparc_insn)))
                   1447:               ^ (pc_next - sizeof(sparc_insn)))
                   1448:              < page_size) {
                   1449:            sparc_recode |= TME_SPARC_RECODE_CHAIN_INFO(TME_RECODE_CHAIN_INFO_NEAR);
                   1450:          }
                   1451:          else {
                   1452:            sparc_recode |= TME_SPARC_RECODE_CHAIN_INFO(TME_RECODE_CHAIN_INFO_FAR);
                   1453:          }
                   1454: 
                   1455:          /* if this is a branch always: */
                   1456:          if (cond == (TME_SPARC_COND_NOT | TME_SPARC_COND_N)) {
                   1457: 
                   1458:            /* the branch delay slot will be the last instruction, and
                   1459:               after that we need an unconditional jump: */
                   1460:            sparc_recode
                   1461:              |= (TME_SPARC_RECODE_INSN_LAST
                   1462:                  | TME_SPARC_RECODE_CHAIN_INFO(TME_RECODE_CHAIN_INFO_JUMP
                   1463:                                                + TME_RECODE_CHAIN_INFO_UNCONDITIONAL));
                   1464: 
                   1465:            /* set PC_next to the branch target, using the address of
                   1466:               the branch delay slot already in PC (which is one
                   1467:               instruction away from the PC of the branch, to which
                   1468:               the branch displacement is relative): */
                   1469:            recode_operand = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC);
                   1470:            branch_displacement -= 1;
                   1471:          }
                   1472: 
                   1473:          /* otherwise, this is a conditional branch: */
                   1474:          else {
                   1475: 
                   1476:            /* the branch delay slot will be the last instruction, and
                   1477:               after that we need an conditional jump: */
                   1478:            sparc_recode
                   1479:              |= (TME_SPARC_RECODE_INSN_LAST
                   1480:                  | TME_SPARC_RECODE_CHAIN_INFO(TME_RECODE_CHAIN_INFO_JUMP
                   1481:                                                + TME_RECODE_CHAIN_INFO_CONDITIONAL));
                   1482: 
                   1483:            /* change PC_next to the branch target, from the
                   1484:               instruction following the branch delay slot already in
                   1485:               PC_next (which is two instructions away from the PC of
                   1486:               the branch, to which the branch displacement is
                   1487:               relative): */
                   1488:            recode_operand = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC_NEXT);
                   1489:            branch_displacement -= 2;
                   1490:          }
                   1491: 
                   1492:          /* set PC_next to the branch target: */
                   1493:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                   1494:          recode_insn->tme_recode_insn_size = recode_size_address;
                   1495:          recode_insn->tme_recode_insn_operand_src[0] = recode_operand;
                   1496:          recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_OPERAND_IMM;
                   1497:          recode_insn->tme_recode_insn_imm_uguest = branch_displacement * sizeof(sparc_insn);
                   1498:          recode_insn->tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC_NEXT);
                   1499:          recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1500:          recode_insn++;
                   1501:        }
                   1502: 
                   1503:        /* if this is a conditional branch: */
                   1504:        if (TME_SPARC_COND_IS_CONDITIONAL(cond)) {
                   1505: 
                   1506:          /* if this conditional branch never annuls the branch delay
                   1507:             slot: */
                   1508:          if ((sparc_insn & TME_BIT(29)) == 0) {
                   1509: 
                   1510:            /* emit the endif: */
                   1511:            recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ENDIF;
                   1512:            recode_insn++;
                   1513:          }
                   1514: 
                   1515:          /* otherwise, this conditional branch annuls the branch
                   1516:             delay slot if the branch is not taken: */
                   1517:          else {
                   1518: 
                   1519:            /* we need to emit the endif after we emit the
                   1520:               instructions for the branch delay slot: */
                   1521:            sparc_recode |= TME_SPARC_RECODE_INSN_NEED_ENDIF;
                   1522:          }
                   1523:        }
                   1524: 
                   1525:        /* otherwise, this is an unconditional branch: */
                   1526:        else {
                   1527: 
                   1528:          /* if the branch delay slot is annulled: */
                   1529:          if (sparc_insn & TME_BIT(29)) {
                   1530: 
                   1531:            /* don't bother to recode the instruction in the branch
                   1532:               delay slot: */
                   1533:            sparc_recode |= TME_SPARC_RECODE_INSN_NONE;
                   1534:          }
                   1535:        }
                   1536: 
                   1537:        /* if this is a branch to . instruction: */
                   1538:        if (__tme_predict_false(branch_displacement_raw == 0)) {
                   1539: 
                   1540:          /* if this isn't a branch never, and this isn't an
                   1541:             unconditional branch that annuls, and the branch with its
                   1542:             delay instruction are a supported timing loop: */
                   1543:          if (cond != TME_SPARC_COND_N
                   1544:              && (sparc_recode & TME_SPARC_RECODE_INSN_NONE) == 0
                   1545:              && tme_sparc_timing_loop_ok(sparc_insn,
                   1546:                                          sparc_insn_next)) {
                   1547: 
                   1548:            /* store the branch delay instruction in the instruction register: */
                   1549:            recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                   1550:            recode_insn->tme_recode_insn_size = TME_RECODE_SIZE_32;
                   1551:            recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_OPERAND_ZERO;
                   1552:            recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_OPERAND_IMM;
                   1553:            recode_insn->tme_recode_insn_imm_u32 = sparc_insn_next;
                   1554:            recode_insn->tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(TME_SPARC_IREG_INSN);
                   1555:            recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1556:            recode_insn++;
                   1557: 
                   1558:            /* call the timing loop assist function: */
                   1559:            recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_GUEST;
                   1560:            recode_insn->tme_recode_insn_size = TME_RECODE_SIZE_32;
                   1561:            recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_OPERAND_IMM;
                   1562:            recode_insn->tme_recode_insn_imm_u32 = sparc_insn;
                   1563:            recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_OPERAND_ZERO;
                   1564:            recode_insn->tme_recode_insn_operand_dst = TME_RECODE_OPERAND_NULL;
                   1565:            recode_insn->tme_recode_insn_guest_func = tme_sparc_timing_loop_assist;
                   1566:            recode_insn++;
                   1567: 
                   1568:            /* the branch delay instruction must be the last instruction: */
                   1569:            assert (sparc_recode & TME_SPARC_RECODE_INSN_LAST);
                   1570: 
                   1571:            /* advance to the branch delay instruction: */
                   1572:            assert (((tme_uint32_t) (pc_advance + sizeof(sparc_insn))) == 0
                   1573:                    && reg_guest_pc == TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC));
                   1574:            pc_advance = 0;
                   1575: 
                   1576:            /* suppress the branch delay instruction: */
                   1577:            sparc_recode |= TME_SPARC_RECODE_INSN_NONE;
                   1578: 
                   1579:            /* do *not* loop now to fetch and recode the instruction
                   1580:               in the branch delay slot: */
                   1581:            break;
                   1582:          }
                   1583:        }
                   1584: 
                   1585:        /* loop now to fetch and recode the instruction in the branch
                   1586:           delay slot: */
                   1587:        continue;
                   1588: 
                   1589:       case 4: /* SETHI: */
                   1590: 
                   1591:        /* unless this instruction is suppressed, load the constant
                   1592:           into the register: */
                   1593:        if (!(sparc_recode & TME_SPARC_RECODE_INSN_NONE)) {
                   1594:          recode_operand = TME_RECODE_REG_GUEST(TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RD));
                   1595:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                   1596:          recode_insn->tme_recode_insn_size = TME_SPARC_RECODE_SIZE(ic);
                   1597:          recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_OPERAND_ZERO;
                   1598:          recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_OPERAND_IMM;
                   1599:          recode_insn->tme_recode_insn_imm_u32 = (tme_uint32_t) (sparc_insn << 10);
                   1600:          recode_insn->tme_recode_insn_operand_dst = recode_operand;
                   1601:          recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1602:          recode_insn++;
                   1603:        }
                   1604: 
                   1605:        /* suppress any instructions below: */
                   1606:        sparc_recode |= TME_SPARC_RECODE_INSN_NONE;
                   1607:        break;
                   1608: 
                   1609: #if TME_SPARC_VERSION(ic) >= 9
                   1610:       case 5: /* FBPfcc */
                   1611: #endif /* TME_SPARC_VERSION(ic) >= 9 */
                   1612:       case 6: /* FBfcc: */
                   1613: 
                   1614:        /* if we haven't recoded any instructions yet, recode is
                   1615:           impossible here: */
                   1616:        if (recode_insn == &ic->tme_sparc_recode_insns[0]) {
                   1617:          return (0);
                   1618:        }
                   1619: 
                   1620:        /* stop recoding immediately, leaving the updated PC
                   1621:           pointing to the instruction before the jmpl: */
                   1622:        assert (pc_advance >= sizeof(sparc_insn));
                   1623:        pc_advance -= sizeof(sparc_insn);
                   1624:        sparc_insn_count--;
                   1625:        sparc_recode
                   1626:          |= (TME_SPARC_RECODE_INSN_UPDATE_PCS
                   1627:              | TME_SPARC_RECODE_INSN_NONE
                   1628:              | TME_SPARC_RECODE_INSN_LAST);
                   1629:        break;
                   1630:       }
                   1631:     }
                   1632: 
                   1633:     /* otherwise, this is a format one instruction: */
                   1634:     else {
                   1635: 
                   1636:       /* if the next instruction can't be recoded when it's in a
                   1637:         branch delay slot (usually because it's also a control
                   1638:         transfer instruction): */
                   1639:       if (__tme_predict_false(_tme_sparc_recode_insn_branch_delay_bad(sparc_insn_next))) {
                   1640: 
                   1641:        /* if we haven't recoded any instructions yet, recode is
                   1642:           impossible here: */
                   1643:        if (recode_insn == &ic->tme_sparc_recode_insns[0]) {
                   1644:          return (0);
                   1645:        }
                   1646:          
                   1647:        /* stop recoding immediately, leaving the updated PC
                   1648:           pointing to the instruction before the jmpl: */
                   1649:        assert (pc_advance >= sizeof(sparc_insn));
                   1650:        pc_advance -= sizeof(sparc_insn);
                   1651:        sparc_insn_count--;
                   1652:        sparc_recode
                   1653:          |= (TME_SPARC_RECODE_INSN_UPDATE_PCS
                   1654:              | TME_SPARC_RECODE_INSN_NONE
                   1655:              | TME_SPARC_RECODE_INSN_LAST);
                   1656:       }
                   1657: 
                   1658:       /* otherwise, the next instruction can be recoded when it's in a
                   1659:         branch delay slot: */
                   1660:       else {
                   1661: 
                   1662:        /* write the PC of the CALL into r[15]: */
                   1663:        recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                   1664:        recode_insn->tme_recode_insn_size = recode_size_address;
                   1665:        recode_insn->tme_recode_insn_operand_src[0] = reg_guest_pc;
                   1666:        recode_insn->tme_recode_insn_operand_src[1]
                   1667:          = (pc_advance == 0
                   1668:             ? TME_RECODE_OPERAND_ZERO
                   1669:             : TME_RECODE_OPERAND_IMM);
                   1670:        recode_insn->tme_recode_insn_imm_uguest = pc_advance;
                   1671:        recode_insn->tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(15);
                   1672:        recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1673:        recode_insn++;
                   1674: 
                   1675:        /* if the PC is not guest size: */
                   1676:        if (recode_size_address < TME_SPARC_RECODE_SIZE(ic)) {
                   1677: 
                   1678:          /* zero-extend r[15] to guest size: */
                   1679:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_EXTZ;
                   1680:          recode_insn->tme_recode_insn_size = TME_SPARC_RECODE_SIZE(ic);
                   1681:          recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_REG_GUEST(15);
                   1682:          recode_insn->tme_recode_insn_operand_src[1] = recode_size_address;
                   1683:          recode_insn->tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(15);
                   1684:          recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1685:          recode_insn++;
                   1686:        }
                   1687: 
                   1688:        /* advance PC to the branch delay slot: */
                   1689:        recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                   1690:        recode_insn->tme_recode_insn_size = recode_size_address;
                   1691:        recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_REG_GUEST(15);
                   1692:        recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_OPERAND_IMM;
                   1693:        recode_insn->tme_recode_insn_imm_uguest = sizeof(sparc_insn);
                   1694:        recode_insn->tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC);
                   1695:        recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1696:        recode_insn++;
                   1697:        pc_advance = 0 - (tme_uint32_t) sizeof(sparc_insn);
                   1698:        reg_guest_pc = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC);
                   1699: 
                   1700:        /* write the target of the call into PC_next.  NB that we have
                   1701:           to account for the fact that PC already points to the
                   1702:           branch delay slot, but the displacement is relative to the
                   1703:           PC of the call instruction: */
                   1704:        recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                   1705:        recode_insn->tme_recode_insn_size = recode_size_address;
                   1706:        recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC);
                   1707:        recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_OPERAND_IMM;
                   1708:        recode_insn->tme_recode_insn_imm_uguest
                   1709:          = ((tme_sparc_ireg_t)
                   1710:             (((tme_recode_guest_t)
                   1711:               (tme_int32_t)
                   1712:               (sparc_insn << 2))
                   1713:              - sizeof(sparc_insn)));
                   1714:        recode_insn->tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC_NEXT);
                   1715:        recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1716:        recode_insn++;
                   1717: 
                   1718:        /* the branch delay slot will be the last instruction, and
                   1719:           after that we need an unconditional call: */
                   1720:        sparc_recode
                   1721:          |= (TME_SPARC_RECODE_INSN_LAST
                   1722:              | TME_SPARC_RECODE_CHAIN_INFO(TME_RECODE_CHAIN_INFO_CALL
                   1723:                                            + TME_RECODE_CHAIN_INFO_UNCONDITIONAL));
                   1724: 
                   1725:        /* if the call target is within this same page,
                   1726:           the call can be near, otherwise the call must be far: */
                   1727:        if ((((pc_next - sizeof(sparc_insn))
                   1728:              + ((tme_sparc_ireg_t)
                   1729:                 (tme_recode_guest_t)
                   1730:                 (tme_int32_t)
                   1731:                 (sparc_insn << 2)))
                   1732:             ^ (pc_next - sizeof(sparc_insn)))
                   1733:            < page_size) {
                   1734:          sparc_recode |= TME_SPARC_RECODE_CHAIN_INFO(TME_RECODE_CHAIN_INFO_NEAR);
                   1735:        }
                   1736:        else {
                   1737:          sparc_recode |= TME_SPARC_RECODE_CHAIN_INFO(TME_RECODE_CHAIN_INFO_FAR);
                   1738:        }
                   1739: 
                   1740:        continue;
                   1741:       }
                   1742:     }
                   1743: 
                   1744:     /* if we might not have enough space for recode insns for the next
                   1745:        instruction: */
                   1746:     if (__tme_predict_false(recode_insn >= recode_insns_last_start)) {
                   1747: 
                   1748:       /* this will be the last instruction, and we need to update the
                   1749:         PCs: */
                   1750:       sparc_recode |= TME_SPARC_RECODE_INSN_UPDATE_PCS | TME_SPARC_RECODE_INSN_LAST;
                   1751:     }
                   1752: 
                   1753:     /* if this is the last instruction, we must be updating PC and
                   1754:        PC_next: */
                   1755:     assert ((sparc_recode & TME_SPARC_RECODE_INSN_LAST) == 0
                   1756:            || (sparc_recode & TME_SPARC_RECODE_INSN_UPDATE_PCS)
                   1757:            || (pc_advance == 0
                   1758:                && reg_guest_pc == TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC)));
                   1759: 
                   1760:     /* if we need PC and PC_next to be valid for this instruction: */
                   1761:     if (sparc_recode & TME_SPARC_RECODE_INSN_UPDATE_PCS) {
                   1762: 
                   1763:       /* if we need to update PC and PC_next: */
                   1764:       if (pc_advance > 0
                   1765:          || reg_guest_pc != TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC)) {
                   1766: 
                   1767:        /* advance PC: */
                   1768:        recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                   1769:        recode_insn->tme_recode_insn_size = recode_size_address;
                   1770:        recode_insn->tme_recode_insn_operand_src[0] = reg_guest_pc;
                   1771:        recode_insn->tme_recode_insn_operand_src[1]
                   1772:          = (pc_advance == 0
                   1773:             ? TME_RECODE_OPERAND_ZERO
                   1774:             : TME_RECODE_OPERAND_IMM);
                   1775:        recode_insn->tme_recode_insn_imm_uguest = pc_advance;
                   1776:        recode_insn->tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC);
                   1777:        recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1778:        recode_insn++;
                   1779:        pc_advance = 0;
                   1780:        reg_guest_pc = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC);
                   1781: 
                   1782:        /* set PC_next: */
                   1783:        recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                   1784:        recode_insn->tme_recode_insn_size = recode_size_address;
                   1785:        recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC);
                   1786:        recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_OPERAND_IMM;
                   1787:        recode_insn->tme_recode_insn_imm_uguest = sizeof(sparc_insn);
                   1788:        recode_insn->tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(TME_SPARC_IREG_PC_NEXT);
                   1789:        recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1790:        recode_insn++;
                   1791:       }
                   1792:     }
                   1793: 
                   1794:     /* if this instruction hasn't been suppressed: */
                   1795:     if (__tme_predict_true((sparc_recode & TME_SPARC_RECODE_INSN_NONE) == 0)) {
                   1796: 
                   1797:       /* if this instruction always or sometimes needs an assist: */
                   1798:       if (sparc_recode & TME_SPARC_RECODE_INSN_UPDATE_INSN) {
                   1799: 
                   1800:        /* store the instruction in the instruction register: */
                   1801:        recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                   1802:        recode_insn->tme_recode_insn_size = TME_RECODE_SIZE_32;
                   1803:        recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_OPERAND_ZERO;
                   1804:        recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_OPERAND_IMM;
                   1805:        recode_insn->tme_recode_insn_imm_u32 = sparc_insn;
                   1806:        recode_insn->tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(TME_SPARC_IREG_INSN);
                   1807:        recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1808:        recode_insn++;
                   1809: 
                   1810:        /* if this instruction doesn't take rs2: */
                   1811:        if (sparc_recode & TME_SPARC_RECODE_INSN_NO_RS2) {
                   1812: 
                   1813:          /* clear the i bit and rs2, to force the second source
                   1814:             operand to be %g0, or zero: */
                   1815:          sparc_insn &= ~(TME_BIT(13) | TME_SPARC_FORMAT3_MASK_RS2);
                   1816:        }
                   1817: 
                   1818:        /* if this instruction doesn't take rs1: */
                   1819:        if (sparc_recode & TME_SPARC_RECODE_INSN_NO_RS1) {
                   1820: 
                   1821:          /* clear rs1, to force the first source operand to be %g0,
                   1822:             or zero: */
                   1823:          sparc_insn &= ~TME_SPARC_FORMAT3_MASK_RS1;
                   1824:        }
                   1825:       }
                   1826: 
                   1827:       /* otherwise, this instruction doesn't need an assist: */
                   1828:       else {
                   1829: 
                   1830:        /* if this is an addx, addxcc, subx, or subxcc instruction: */
                   1831:        if (sparc_recode & TME_SPARC_RECODE_INSN_DEFC) {
                   1832: 
                   1833:          /* define the recode carry flag with the icc C flag: */
                   1834:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_DEFC;
                   1835:          recode_insn->tme_recode_insn_operand_src[0] = TME_SPARC_COND_CS;
                   1836:          recode_insn->tme_recode_insn_operand_src[1] = 0;
                   1837:          recode_insn->tme_recode_insn_operand_dst = TME_RECODE_FLAG_CARRY;
                   1838:          recode_insn->tme_recode_insn_conds_thunk = ic->tme_sparc_recode_conds_thunk_icc;
                   1839:          recode_insn++;
                   1840:        }
                   1841: 
                   1842:        /* because we set the flags thunk on the recode instruction
                   1843:           now, this instruction can't be a shift (because a shift may
                   1844:           make more than one recode instruction): */
                   1845:        assert ((sparc_recode
                   1846:                 & (TME_SPARC_RECODE_INSN_CC
                   1847:                    | TME_SPARC_RECODE_INSN_SHIFT))
                   1848:                != (TME_SPARC_RECODE_INSN_CC
                   1849:                    | TME_SPARC_RECODE_INSN_SHIFT));
                   1850: 
                   1851:        /* assume that this instruction doesn't update flags: */
                   1852:        flags_thunk = NULL;
                   1853: 
                   1854:        /* if this instruction updates flags: */
                   1855:        if (sparc_recode & TME_SPARC_RECODE_INSN_CC) {
                   1856: 
                   1857:          /* set the right flags thunk for this instruction.  NB that
                   1858:             the least significant two bits of op3 are zero only for
                   1859:             the additive instructions, and bit two of op3 is set only
                   1860:             for a subtraction instruction: */
                   1861:          flags_thunk
                   1862:            = ((sparc_insn & (0x3 << 19)) == 0
                   1863:               ? ((sparc_insn & (0x4 << 19))
                   1864:                  ? ic->tme_sparc_recode_flags_thunk_sub
                   1865:                  : ic->tme_sparc_recode_flags_thunk_add)
                   1866:               : ic->tme_sparc_recode_flags_thunk_logical);
                   1867:        }
                   1868: 
                   1869:        /* set any flags thunk on this instruction: */
                   1870:        recode_insn->tme_recode_insn_flags_thunk = flags_thunk;
                   1871:       }
                   1872: 
                   1873:       /* assume that if this is a shift, it's a 32-bit shift: */
                   1874:       shift_count_mask = 32 - 1;
                   1875: 
                   1876:       /* if this is a v9 shift: */
                   1877:       if (TME_SPARC_VERSION(ic) >= 9
                   1878:          && (sparc_recode & TME_SPARC_RECODE_INSN_SHIFT)) {
                   1879: 
                   1880:        /* if this is a 64-bit shift: */
                   1881:        if (sparc_insn & TME_BIT(12)) {
                   1882: 
                   1883:          /* this is a 64-bit shift: */
                   1884:          shift_count_mask = 64 - 1;
                   1885:        }
                   1886: 
                   1887:        /* otherwise, this is a 32-bit shift: */
                   1888:        else {
                   1889: 
                   1890:          /* if this is a right shift: */
                   1891:          if (sparc_insn & TME_BIT(20)) {
                   1892: 
                   1893:            /* remember that this is a right shift: */
                   1894:            sparc_recode |= TME_SPARC_RECODE_INSN_SHIFT_RIGHT;
                   1895: 
                   1896:            /* decode rs1: */
                   1897:            recode_operand = TME_RECODE_REG_GUEST(TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RS1));
                   1898: 
                   1899:            /* extend rs1 into the second temporary register.  if this
                   1900:               is a shift right arithmetic, sign extend, else zero
                   1901:               extend: */
                   1902:            recode_insn->tme_recode_insn_opcode
                   1903:              = ((sparc_insn & TME_BIT(19))
                   1904:                 ? TME_RECODE_OPCODE_EXTS
                   1905:                 : TME_RECODE_OPCODE_EXTZ);
                   1906:            recode_insn->tme_recode_insn_size = TME_SPARC_RECODE_SIZE(ic);
                   1907:            recode_insn->tme_recode_insn_operand_src[0] = recode_operand;
                   1908:            recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_SIZE_32;
                   1909:            recode_insn->tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(TME_SPARC_IREG_TMP(1));
                   1910:            recode_insn++;
                   1911: 
                   1912:            /* NB that the flags thunk for the final instruction was
                   1913:               already set above (to NULL, because sparc shift
                   1914:               instructions do not affect any condition codes), so we
                   1915:               have to do the same since we advanced to the next
                   1916:               instruction: */
                   1917:            recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1918:          }
                   1919:        }
                   1920:       }
                   1921: 
                   1922:       /* if the i bit is one: */
                   1923:       if (sparc_insn & TME_BIT(13)) {
                   1924: 
                   1925:        /* decode simm13: */
                   1926:        imm = TME_FIELD_MASK_EXTRACTS(sparc_insn, (tme_sparc_ireg_t) 0x1fff);
                   1927: 
                   1928:        /* if this is a shift instruction: */
                   1929:        if (__tme_predict_false(sparc_recode & TME_SPARC_RECODE_INSN_SHIFT)) {
                   1930: 
                   1931:          /* mask the immediate with the shift count mask: */
                   1932:          imm &= shift_count_mask;
                   1933:        }
                   1934: 
                   1935:        /* if the immediate is zero, the second source operand is zero,
                   1936:           otherwise the second source operand is the immediate: */
                   1937:        recode_insn->tme_recode_insn_imm_uguest = imm;
                   1938:        recode_operand
                   1939:          = (imm == 0
                   1940:             ? TME_RECODE_OPERAND_ZERO
                   1941:             : TME_RECODE_OPERAND_IMM);
                   1942:       }
                   1943: 
                   1944:       /* otherwise, the i bit is zero: */
                   1945:       else {
                   1946: 
                   1947:        /* decode rs2: */
                   1948:        recode_operand = TME_RECODE_REG_GUEST(TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RS2));
                   1949:        if (recode_operand == TME_RECODE_REG_GUEST(TME_SPARC_IREG_G0)) {
                   1950:          recode_operand = TME_RECODE_OPERAND_ZERO;
                   1951:        }
                   1952: 
                   1953:        /* if this is a shift instruction: */
                   1954:        if (__tme_predict_false(sparc_recode & TME_SPARC_RECODE_INSN_SHIFT)) {
                   1955: 
                   1956:          /* make a temporary copy of the register with shift count,
                   1957:             masked with the shift count mask, and use the temporary
                   1958:             register as the second source operand: */
                   1959:          recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_AND;
                   1960:          recode_insn->tme_recode_insn_size = TME_SPARC_RECODE_SIZE(ic);
                   1961:          recode_insn->tme_recode_insn_operand_src[0] = recode_operand;
                   1962:          recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_OPERAND_IMM;
                   1963:          recode_insn->tme_recode_insn_imm_uguest = shift_count_mask;
                   1964:          recode_operand = TME_RECODE_REG_GUEST(TME_SPARC_IREG_TMP(0));
                   1965:          recode_insn->tme_recode_insn_operand_dst = recode_operand;
                   1966:          recode_insn++;
                   1967: 
                   1968:          /* NB that the flags thunk for the final instruction was
                   1969:             already set above (to NULL, because sparc shift
                   1970:             instructions do not affect any condition codes), so we
                   1971:             have to do the same since we advanced to the next
                   1972:             instruction: */
                   1973:          recode_insn->tme_recode_insn_flags_thunk = NULL;
                   1974:        }
                   1975:       }
                   1976: 
                   1977:       /* set the second source operand: */
                   1978:       recode_insn->tme_recode_insn_operand_src[1] = recode_operand;
                   1979: 
                   1980:       /* decode rs1: */
                   1981:       recode_operand = TME_RECODE_REG_GUEST(TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RS1));
                   1982:       if (recode_operand == TME_RECODE_REG_GUEST(TME_SPARC_IREG_G0)) {
                   1983:        recode_operand = TME_RECODE_OPERAND_ZERO;
                   1984:       }
                   1985: 
                   1986:       /* if this is a v9 32-bit right shift: */
                   1987:       if (__tme_predict_false(TME_SPARC_VERSION(ic) >= 9
                   1988:                              && (sparc_recode & TME_SPARC_RECODE_INSN_SHIFT_RIGHT))) {
                   1989: 
                   1990:        /* the first source operand is actually in the second
                   1991:           temporary register: */
                   1992:        recode_operand = TME_RECODE_REG_GUEST(TME_SPARC_IREG_TMP(1));
                   1993:       }
                   1994: 
                   1995:       /* set the first source operand: */
                   1996:       recode_insn->tme_recode_insn_operand_src[0] = recode_operand;
                   1997:        
                   1998:       /* decode rd: */
                   1999:       recode_operand = TME_RECODE_REG_GUEST(TME_FIELD_MASK_EXTRACTU(sparc_insn, TME_SPARC_FORMAT3_MASK_RD));
                   2000:       if (recode_operand == TME_RECODE_REG_GUEST(TME_SPARC_IREG_G0)) {
                   2001:        recode_operand = TME_RECODE_OPERAND_NULL;
                   2002:       }
                   2003: 
                   2004:       /* if this is a simple ld or st instruction: */
                   2005:       if (TME_SPARC_RECODE_INSN_OPCODE(sparc_recode) == TME_RECODE_OPCODE_RW) {
                   2006: 
                   2007:        /* assume that the first source operand is %g0, which makes
                   2008:           the second source operand the guest address: */
                   2009:        recode_operand_address = recode_insn->tme_recode_insn_operand_src[1];
                   2010: 
                   2011:        /* if the second source operand is %g0: */
                   2012:        if (recode_operand_address == TME_RECODE_OPERAND_ZERO) {
                   2013: 
                   2014:          /* nothing to do; the guest address is the first source
                   2015:             operand, which is already in
                   2016:             recode_insn->tme_recode_insn_operand_src[0]: */
                   2017:        }
                   2018: 
                   2019:        /* otherwise, the second source operand is not %g0: */
                   2020:        else {
                   2021: 
                   2022:          /* if the first source operand is not %g0: */
                   2023:          if (recode_insn->tme_recode_insn_operand_src[0] != TME_RECODE_OPERAND_ZERO) {
                   2024: 
                   2025:            /* add the two parts of the guest address into the
                   2026:               temporary register: */
                   2027:            recode_operand_address = TME_RECODE_REG_GUEST(TME_SPARC_IREG_TMP(0));
                   2028:            recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ADD;
                   2029:            recode_insn->tme_recode_insn_size = recode_size_address;
                   2030:            recode_insn->tme_recode_insn_operand_dst = recode_operand_address;
                   2031:            recode_insn->tme_recode_insn_flags_thunk = NULL;
                   2032:            recode_insn++;
                   2033:          }
                   2034: 
                   2035:          /* set the guest address as the first source operand: */
                   2036:          recode_insn->tme_recode_insn_operand_src[0] = recode_operand_address;
                   2037:        }
                   2038: 
                   2039:        /* if this is a st instruction: */
                   2040:        /* NB: all of the simple ld and st instructions that we handle
                   2041:           here, have bit 21 set for a st, and clear for a ld: */
                   2042:        if (sparc_insn & (4 << 19)) {
                   2043: 
                   2044:          /* the second source operand is the register being
                   2045:             stored: */
                   2046:          recode_insn->tme_recode_insn_operand_src[1]
                   2047:            = ((TME_RECODE_OPERAND_NULL != TME_RECODE_OPERAND_ZERO
                   2048:                && recode_operand == TME_RECODE_OPERAND_NULL)
                   2049:               ? TME_RECODE_OPERAND_ZERO
                   2050:               : recode_operand);
                   2051:          
                   2052:          /* the destination operand must be the recode %null: */
                   2053:          recode_operand = TME_RECODE_OPERAND_NULL;
                   2054:        }
                   2055: 
                   2056:        /* otherwise, this is an ld instruction: */
                   2057:        else {
                   2058: 
                   2059:          /* the second source operand must be zero: */
                   2060:          recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_OPERAND_ZERO;
                   2061:        }
                   2062: 
                   2063:        /* set the read/write thunk: */
                   2064:        assert (TME_SPARC_VERSION(ic) >= 9 || (sparc_insn & (16 << 19)) == 0);
                   2065:        recode_insn->tme_recode_insn_rw_thunk
                   2066:          = ic->tme_sparc_recode_rw_thunks
                   2067:              [TME_SPARC_RECODE_RW_THUNK_INDEX(ic,
                   2068:                                               (TME_SPARC_RECODE_SIZE(ic)
                   2069:                                                - (TME_SPARC_VERSION(ic) >= 9
                   2070:                                                   && (ic->tme_sparc64_ireg_pstate & TME_SPARC64_PSTATE_AM))),
                   2071:                                               ((TME_SPARC_VERSION(ic) >= 9
                   2072:                                                 && (ic->tme_sparc64_ireg_pstate & TME_SPARC64_PSTATE_CLE))
                   2073:                                                ? TME_ENDIAN_LITTLE
                   2074:                                                : TME_ENDIAN_BIG),
                   2075:                                               (TME_SPARC_VERSION(ic) >= 9
                   2076:                                                && (sparc_insn & (16 << 19))),
                   2077:                                               TME_FIELD_MASK_EXTRACTU(sparc_insn, (0xf << 19)))];
                   2078:       }
                   2079: 
                   2080:       /* otherwise, if this instruction needs an assist: */
                   2081:       else if (TME_SPARC_RECODE_INSN_OPCODE(sparc_recode) == TME_RECODE_OPCODE_GUEST) {
                   2082: 
                   2083:        /* set the assist function: */
                   2084:        recode_insn->tme_recode_insn_guest_func = sparc_assist;
                   2085: 
                   2086:        /* if this instruction needs a full assist: */
                   2087:        if ((sparc_recode
                   2088:             & (TME_SPARC_RECODE_INSN_NO_RS2
                   2089:                | TME_SPARC_RECODE_INSN_NO_RS1))
                   2090:            == (TME_SPARC_RECODE_INSN_NO_RS2
                   2091:                | TME_SPARC_RECODE_INSN_NO_RS1)) {
                   2092: 
                   2093:          /* the source operands can be the recode %undef, since the
                   2094:             full assist functions don't use the source operands: */
                   2095:          assert (sparc_assist == _tme_sparc_recode_insn_assist_full
                   2096:                  || sparc_assist == _tme_sparc_recode_insn_assist_unimpl
                   2097:                  );
                   2098:          recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_OPERAND_UNDEF;
                   2099:          recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_OPERAND_UNDEF;
                   2100: 
                   2101:          /* assume that this instruction is not also marked
                   2102:             TME_SPARC_RECODE_INSN_NO_RD, and force the destination
                   2103:             operand to be the recode %null, to indicate that the
                   2104:             assist can change any of the recode-visible ic state: */
                   2105:          recode_operand = TME_RECODE_OPERAND_NULL;
                   2106:        }
                   2107: 
                   2108:        /* if this instruction's rd field doesn't encode a destination
                   2109:           general register: */
                   2110:        if (sparc_recode & TME_SPARC_RECODE_INSN_NO_RD) {
                   2111: 
                   2112:          /* force the destination operand to be the fixed %g0: */
                   2113:          recode_operand = TME_RECODE_REG_GUEST(TME_SPARC_IREG_G0);
                   2114:        }
                   2115:       }
                   2116: 
                   2117:       /* set the destination operand: */
                   2118:       recode_insn->tme_recode_insn_operand_dst = recode_operand;
                   2119: 
                   2120:       /* set the size: */
                   2121:       recode_insn->tme_recode_insn_size = TME_SPARC_RECODE_SIZE(ic);
                   2122: 
                   2123:       /* set the opcode: */
                   2124:       recode_insn->tme_recode_insn_opcode = TME_SPARC_RECODE_INSN_OPCODE(sparc_recode);
                   2125: 
                   2126:       /* we have finished this instruction: */
                   2127:       recode_insn++;
                   2128:     }
                   2129: 
                   2130:     /* if we need to, emit an endif: */
                   2131:     if (sparc_recode & TME_SPARC_RECODE_INSN_NEED_ENDIF) {
                   2132:       recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_ENDIF;
                   2133:       recode_insn++;
                   2134: #ifndef NDEBUG
                   2135:       sparc_recode &= ~TME_SPARC_RECODE_INSN_NEED_ENDIF;
                   2136: #endif
                   2137:     }
                   2138: 
                   2139:     /* if this was the last instruction: */
                   2140:     if (sparc_recode & TME_SPARC_RECODE_INSN_LAST) {
                   2141:       break;
                   2142:     }
                   2143: 
                   2144:     /* clear sparc_recode: */
                   2145:     sparc_recode = 0;
                   2146:   }
                   2147: 
                   2148:   /* make sure we didn't forget an endif: */
                   2149:   assert ((sparc_recode & TME_SPARC_RECODE_INSN_NEED_ENDIF) == 0);
                   2150: 
                   2151: #ifdef _TME_SPARC_STATS
                   2152:   /* update the recode instructions total: */
                   2153:   recode_insn->tme_recode_insn_opcode = TME_RECODE_OPCODE_GUEST;
                   2154:   recode_insn->tme_recode_insn_size = TME_SPARC_RECODE_SIZE(ic);
                   2155:   recode_insn->tme_recode_insn_operand_src[0] = TME_RECODE_OPERAND_ZERO;
                   2156:   recode_insn->tme_recode_insn_operand_src[1] = TME_RECODE_OPERAND_IMM;
                   2157:   recode_insn->tme_recode_insn_imm_u32 = sparc_insn_count;
                   2158:   recode_insn->tme_recode_insn_operand_dst = TME_RECODE_REG_GUEST(TME_SPARC_IREG_G0);
                   2159:   recode_insn->tme_recode_insn_guest_func = _tme_sparc_recode_insns_total;
                   2160:   recode_insn++;
                   2161: #endif /* _TME_SPARC_STATS */
                   2162: 
                   2163:   /* set the end of the instructions: */
                   2164:   ic->tme_sparc_recode_insns_group.tme_recode_insns_group_insns_end = recode_insn;
                   2165: 
                   2166:   /* get the chain information: */
                   2167:   chain_info
                   2168:     = ((sparc_recode
                   2169:        / TME_SPARC_RECODE_CHAIN_INFO(1))
                   2170:        & (TME_RECODE_CHAIN_INFO_UNCONDITIONAL
                   2171:          | TME_RECODE_CHAIN_INFO_CONDITIONAL
                   2172:          | TME_RECODE_CHAIN_INFO_NEAR
                   2173:          | TME_RECODE_CHAIN_INFO_FAR
                   2174:          | TME_RECODE_CHAIN_INFO_JUMP
                   2175:          | TME_RECODE_CHAIN_INFO_RETURN
                   2176:          | TME_RECODE_CHAIN_INFO_CALL
                   2177:          ));
                   2178: 
                   2179:   /* if there is no chain information: */
                   2180:   if (chain_info == 0) {
                   2181: 
                   2182:     /* if the next PC is on the same page, make a chain jump near
                   2183:        unconditional, otherwise make a chain jump far
                   2184:        unconditional: */
                   2185:     if (pc_next <= itlb_addr_last) {
                   2186:       chain_info += TME_RECODE_CHAIN_INFO_NEAR;
                   2187:     }
                   2188:     else {
                   2189:       chain_info += TME_RECODE_CHAIN_INFO_FAR;
                   2190:     }
                   2191:     chain_info
                   2192:       += (TME_RECODE_CHAIN_INFO_JUMP
                   2193:          + TME_RECODE_CHAIN_INFO_UNCONDITIONAL);
                   2194:   }
                   2195: 
                   2196:   /* otherwise, there is chain information.  if the chain is
                   2197:      conditional: */
                   2198:   else if ((chain_info
                   2199:            & (TME_RECODE_CHAIN_INFO_UNCONDITIONAL
                   2200:               | TME_RECODE_CHAIN_INFO_CONDITIONAL))
                   2201:           != TME_RECODE_CHAIN_INFO_UNCONDITIONAL) {
                   2202: 
                   2203:     /* if the alternate PC is on the same page, the alternate target
                   2204:        can be near, otherwise it must be far: */
                   2205:     if (pc_next <= itlb_addr_last) {
                   2206:       chain_info += TME_RECODE_CHAIN_INFO_ALTERNATE_NEAR;
                   2207:     }
                   2208:     else {
                   2209:       chain_info += TME_RECODE_CHAIN_INFO_ALTERNATE_FAR;
                   2210:     }
                   2211:   }
                   2212: 
                   2213:   /* set the chain information: */
                   2214:   ic->tme_sparc_recode_insns_group.tme_recode_insns_group_chain_info = chain_info;
                   2215: 
                   2216:   /* make sure we didn't overflow the instructions buffer: */
                   2217:   assert (recode_insn <= &ic->tme_sparc_recode_insns[TME_ARRAY_ELS(ic->tme_sparc_recode_insns)]);
                   2218: 
                   2219:   /* recode these instructions: */
                   2220:   return (tme_recode_insns_thunk(ic->tme_sparc_recode_ic,
                   2221:                                 &ic->tme_sparc_recode_insns_group));
                   2222: }

unix.superglobalmegacorp.com

This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.