--- tme/ic/sparc/sparc-insns-auto.sh 2018/04/24 16:42:00 1.1 +++ tme/ic/sparc/sparc-insns-auto.sh 2018/04/24 16:43:57 1.1.1.2 @@ -1,6 +1,6 @@ #! /bin/sh -# $Id: sparc-insns-auto.sh,v 1.1 2018/04/24 16:42:00 root Exp $ +# $Id: sparc-insns-auto.sh,v 1.1.1.2 2018/04/24 16:43:57 root Exp $ # ic/sparc/sparc-insns-auto.sh - automatically generates C code # for many SPARC emulation instructions: @@ -48,18 +48,24 @@ done PROG=`basename $0` cat <tme_sparc_asis[asi_data].tme_sparc_asi_mask_flags;" + echo "" + echo " /* make the ASI mask: */" + echo " if (asi_mask_flags & TME_SPARC32_ASI_MASK_FLAG_SPECIAL) {" + echo " asi_mask_data" + echo " = TME_SPARC_ASI_MASK_SPECIAL(asi_data, TRUE);" + echo " }" + echo " else {" + echo " asi_mask_data = TME_SPARC32_ASI_MASK(asi_data, asi_data);" echo " }" else echo "" echo " /* if the i bit is one, use the address space in the ASI register: */" echo " if (TME_SPARC_INSN & TME_BIT(13)) {" - echo " asi_data = ic->tme_sparc_ireg_asi;" + echo " asi_data = ic->tme_sparc64_ireg_asi;" echo " }" echo "" - echo " /* start the ASI mask: */" - echo " asi_mask_data = TME_SPARC_ASI_MASK(asi_data);" + echo " /* get the flags for this ASI: */" + echo " asi_mask_flags = ic->tme_sparc_asis[asi_data].tme_sparc_asi_mask_flags;" echo "" - echo " /* if this is an unpriviledged access: */" + echo " /* if this is a nonprivileged access: */" echo " if (!TME_SPARC_PRIV(ic)) {" echo "" - echo " /* this is a priviledge violation if bit 7 of the ASI is zero: */" - echo " if (asi_data < 0x80) {" - echo " TME_SPARC_INSN_TRAP(TME_SPARC_TRAP_privileged_instruction);" + echo " /* if this is a restricted ASI: */" + echo " if (__tme_predict_false((asi_data & TME_SPARC64_ASI_FLAG_UNRESTRICTED) == 0)) {" + echo "" + echo " /* force a slow load or store, which will generate the" + echo " privileged_action trap: */" + echo " asi_mask_flags |= TME_SPARC_ASI_MASK_FLAG_UNDEF;" echo " }" + echo "" + echo " /* force a nonprivileged access with the ASI: */" + echo " asi_mask_flags |= TME_SPARC64_ASI_MASK_FLAG_INSN_AS_IF_USER;" + echo " }" + echo "" + echo " /* make the ASI mask: */" + echo " if (asi_mask_flags & TME_SPARC64_ASI_MASK_FLAG_SPECIAL) {" + echo " asi_mask_data" + echo " = (asi_mask_flags" + echo " + TME_SPARC_ASI_MASK_SPECIAL(asi_data," + echo " ((asi_mask_flags & TME_SPARC64_ASI_MASK_FLAG_INSN_AS_IF_USER) == 0)));" + echo " }" + echo " else {" + echo " asi_mask_data = TME_SPARC64_ASI_MASK(asi_data, asi_mask_flags);" echo " }" fi echo "" + echo " /* if this ASI has a special handler: */" + echo " if (__tme_predict_false(ic->tme_sparc_asis[TME_SPARC_ASI_MASK_WHICH(asi_mask_data)].tme_sparc_asi_handler != 0)) {" + echo "" + echo " /* force a slow load or store, which will call the special handler: */" + echo " asi_mask_data |= TME_SPARC_ASI_MASK_FLAG_UNDEF;" + echo " }" + echo "" echo " return (asi_mask_data);" echo "}" fi + # the FPU load and store common function: + # + if $header; then :; else + echo "" + echo "static struct tme_float *" + echo "_tme_sparc${arch}_fpu_mem_fpreg(struct tme_sparc *ic," + echo " tme_uint32_t misaligned," + echo " struct tme_float *float_buffer)" + echo "{" + echo " unsigned int float_format;" + echo " unsigned int fpreg_format;" + echo " tme_uint32_t fp_store;" + echo " unsigned int fpu_mode;" + echo " unsigned int fpreg_number;" + echo "" + echo " /* NB: this checks for various traps by their priority order: */" + echo "" + echo " TME_SPARC_INSN_FPU_ENABLED;" + echo "" + echo " /* get the floating-point format: */" + echo " float_format = float_buffer->tme_float_format;" + echo "" + echo " /* convert the floating-point format into the ieee754" + echo " floating-point register file format: */" + echo "#if (TME_FLOAT_FORMAT_NULL | TME_IEEE754_FPREG_FORMAT_NULL) != 0" + echo "#error \"TME_FLOAT_FORMAT_ or TME_IEEE754_FPREG_FORMAT_ values changed\"" + echo "#endif" + echo "#if TME_FLOAT_FORMAT_IEEE754_SINGLE < TME_IEEE754_FPREG_FORMAT_SINGLE" + echo "#error \"TME_FLOAT_FORMAT_ or TME_IEEE754_FPREG_FORMAT_ values changed\"" + echo "#endif" + echo "#if (TME_FLOAT_FORMAT_IEEE754_SINGLE / TME_IEEE754_FPREG_FORMAT_SINGLE) != (TME_FLOAT_FORMAT_IEEE754_DOUBLE / TME_IEEE754_FPREG_FORMAT_DOUBLE)" + echo "#error \"TME_FLOAT_FORMAT_ or TME_IEEE754_FPREG_FORMAT_ values changed\"" + echo "#endif" + echo " assert (float_format == TME_FLOAT_FORMAT_NULL" + echo " || float_format == TME_FLOAT_FORMAT_IEEE754_SINGLE" + echo " || float_format == TME_FLOAT_FORMAT_IEEE754_DOUBLE);" + echo " fpreg_format = float_format / (TME_FLOAT_FORMAT_IEEE754_SINGLE / TME_IEEE754_FPREG_FORMAT_SINGLE);" + echo "" + echo " /* if the memory address is misaligned, return the" + echo " float buffer now. the eventual load or store will" + echo " cause the mem_address_not_aligned trap: */" + echo "" + echo " /* if the memory address is misaligned: */" + echo "#if TME_IEEE754_FPREG_FORMAT_NULL != 0 || TME_IEEE754_FPREG_FORMAT_SINGLE != 1 || TME_IEEE754_FPREG_FORMAT_DOUBLE != 2 || TME_IEEE754_FPREG_FORMAT_QUAD != 4" + echo "#error \"TME_IEEE754_FPREG_FORMAT_ values changed\"" + echo "#endif" + echo " assert (fpreg_format == TME_IEEE754_FPREG_FORMAT_NULL" + echo " || fpreg_format == TME_IEEE754_FPREG_FORMAT_SINGLE" + echo " || fpreg_format == TME_IEEE754_FPREG_FORMAT_DOUBLE" + echo " || fpreg_format == TME_IEEE754_FPREG_FORMAT_QUAD);" + echo " misaligned &= ((sizeof(tme_uint32_t) * fpreg_format) - 1);" + echo " if (__tme_predict_false(misaligned)) {" + echo "" + if test ${arch} = 32; then + echo " return (float_buffer);" + else + echo " /* if the memory address is not even 32-bit aligned, or" + echo " if this SPARC doesn't support loads and stores of this" + echo " size at 32-bit alignment: */" + echo " if (misaligned != sizeof(tme_uint32_t)" + echo "#if TME_IEEE754_FPREG_FORMAT_SINGLE != 1 || (TME_SPARC_MEMORY_FLAG_HAS_LDDF_STDF_32 * TME_IEEE754_FPREG_FORMAT_DOUBLE) != TME_SPARC_MEMORY_FLAG_HAS_LDQF_STQF_32" + echo "#error \"TME_IEEE754_FPREG_FORMAT_ or TME_SPARC_MEMORY_FLAG_ values changed\"" + echo "#endif" + echo " || (TME_SPARC_MEMORY_FLAGS(ic)" + echo " & (TME_SPARC_MEMORY_FLAG_HAS_LDDF_STDF_32 * fpreg_format)) == 0) {" + echo "" + echo " return (float_buffer);" + echo " }" + fi + echo " }" + echo "" + echo " /* see if this is a floating-point load or store: */" + echo " /* NB: all of the floating-point instructions that use" + echo " this preamble have bit two of op3 clear for a load," + echo " and set for a store: */" + echo " fp_store = (TME_SPARC_INSN & TME_BIT(19 + 2));" + echo "" + echo " /* if the FPU isn't in execute mode: */" + echo " fpu_mode = ic->tme_sparc_fpu_mode;" + echo " if (__tme_predict_false(fpu_mode != TME_SPARC_FPU_MODE_EXECUTE)) {" + echo "" + echo " /* if this is a floating-point load, or if this is a" + echo " floating-point store and a floating-point exception" + echo " is pending: */" + echo " if (!fp_store" + echo " || fpu_mode == TME_SPARC_FPU_MODE_EXCEPTION_PENDING) {" + echo "" + echo " /* do an FPU exception check: */" + echo " tme_sparc_fpu_exception_check(ic);" + echo " }" + echo " }" + echo "" + echo " /* if this is not a load or store of a floating-point register: */" + echo " if (fpreg_format == TME_IEEE754_FPREG_FORMAT_NULL) {" + echo " return (float_buffer);" + echo " }" + echo "" + echo " /* decode rd: */" + echo " fpreg_number" + echo " = tme_sparc_fpu_fpreg_decode(ic," + echo " TME_FIELD_MASK_EXTRACTU(TME_SPARC_INSN," + echo " TME_SPARC_FORMAT3_MASK_RD)," + echo " fpreg_format);" + echo "" + echo " /* make sure this floating-point register has the right precision: */" + echo " tme_sparc_fpu_fpreg_format(ic, fpreg_number, fpreg_format | TME_IEEE754_FPREG_FORMAT_BUILTIN);" + echo "" + echo " /* if this is a floating-point load: */" + echo " if (!fp_store) {" + echo "" + echo " /* mark rd as dirty: */" + echo " TME_SPARC_FPU_DIRTY(ic, fpreg_number);" + echo " }" + echo "" + echo " /* return the floating-point register: */" + echo " return (&ic->tme_sparc_fpu_fpregs[fpreg_number]);" + echo "}" + echo "#define _tme_sparc${arch}_fpu_mem(ic) \\" + echo " do { _tme_sparc${arch}_fpu_mem_fpreg(ic, 0, &_tme_sparc_float_null); } while (/* CONSTCOND */ 0)" + fi + # permute over instruction: # + case ${arch} in + 32) insns_arch= ;; + 64) insns_arch="ldx stx ldqf stqf ldxa stxa ldfa lddfa stfa stdfa ldqfa stqfa casa casxa mulx sdivx udivx" ;; + esac for insn in \ add \ addcc \ @@ -187,7 +347,9 @@ for arch in 32; do jmpl \ ldf lddf ldfsr \ stf stdf stfsr \ + fpop1 fpop2 \ mulscc \ + ${insns_arch} \ ; do # if we're making the header, just emit declarations: @@ -237,7 +399,7 @@ for arch in 32; do umul | umulcc) arith=mul ; size_src=32 ;; smul | smulcc) arith=mul ; size_src=32 ; sign= ;; udiv | udivcc) arith=udiv ; size_src=32 ; size_dst=32 ;; - sdiv | sdivcc) arith=sdiv ; size_src=32 ; size_dst=32 ; sign= ;; + sdiv | sdivcc) arith=sdiv ; size_src=32 ; sign= ;; mulscc) arith=add ; size_src=32 ; size_dst=32 ;; *) echo "$0 internal error: unknown ALU function ${insn}" 1>&2 ; exit 1 ;; esac @@ -265,6 +427,7 @@ for arch in 32; do if ${cc}; then echo " tme_uint32_t cc;" fi + cc_plus= echo "" echo " /* get the operands: */" @@ -280,10 +443,39 @@ for arch in 32; do echo " dst = ((tme_${sign}int64_t) val64);" ;; udiv | udivcc | sdiv | sdivcc) - echo " val64 = ((((tme_uint64_t) ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_Y${reg32_shift})) << 32) | ((tme_uint32_t) src1));" + echo " val64 = ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_Y${reg32_shift});" + echo " val64 = (val64 << 32) + (tme_uint32_t) src1;" + echo " if (__tme_predict_false(src2 == 0)) {" + echo " tme_sparc${arch}_trap(ic, TME_SPARC${arch}_TRAP_division_by_zero);" + echo " }" echo " val64 /= src2;" - echo " /* XXX FIXME - overflow handling is missing here: */" - echo " dst = val64;" + echo " dst = (tme_${sign}int32_t) val64;" + echo "" + echo " /* if the division overflowed: */" + echo " if (dst != val64) {" + echo "" + echo " /* return the largest appropriate value: */" + if test "x${sign}" = xu; then + echo " dst = 0xffffffff;" + else + echo " dst = (tme_int32_t) ((val64 < 0) + (tme_uint32_t) 0x7fffffff);" + fi + if ${cc}; then + echo "" + echo " /* set V: */" + echo " cc = TME_SPARC${ccr}_ICC_V;" + fi + echo " }" + if ${cc}; then + echo "" + echo " /* otherwise, the division didn't overflow: */" + echo " else {" + echo "" + echo " /* clear V: */" + echo " cc = !TME_SPARC${ccr}_ICC_V;" + echo " }" + cc_plus='+' + fi ;; mulscc) echo "" @@ -339,14 +531,6 @@ for arch in 32; do # if ${cc}; then - echo "" - echo " /* set Z if the destination is zero: */" - echo -n " cc = ((dst == 0) * (TME_SPARC${ccr}_ICC_Z" - if test ${arch} = 64; then - echo -n " + TME_SPARC${ccr}_XCC_Z" - fi - echo "));" - # set the 32-bit, and possibly the 64-bit, condition codes: # arch_cc=16 @@ -358,7 +542,12 @@ for arch in 32; do 64) xcc=XCC ;; esac - if test `expr ${arch_cc} \<= ${size_dst}` = 1 || test "x${sign}" = xs; then + echo "" + echo " /* set Z if the destination is zero: */" + echo " cc ${cc_plus}= ((((tme_int${arch_cc}_t) dst) == 0) * TME_SPARC${ccr}_${xcc}_Z);" + cc_plus='+' + + if test `expr ${arch_cc} \<= ${size_dst}` = 1; then echo "" echo " /* set N if the destination is negative: */" echo " cc += ((((tme_int${arch_cc}_t) dst) < 0) * TME_SPARC${ccr}_${xcc}_N);" @@ -395,9 +584,8 @@ for arch in 32; do ;; logical | mul) ;; - udiv) - ;; - sdiv) + udiv | sdiv) + # the udivcc and sdivcc V are handled in the operation code ;; *) echo "$0 internal error: unknown arithmetic type ${arith}" 1>&2 ; exit 1 ;; esac @@ -418,7 +606,7 @@ for arch in 32; do echo "" echo " /* trap on a tagged overflow: */" echo " if (cc & TME_SPARC${ccr}_ICC_V) {" - echo " tme_sparc${arch}_trap(ic, TME_SPARC_TRAP_tag_overflow);" + echo " tme_sparc${arch}_trap(ic, TME_SPARC${arch}_TRAP_tag_overflow);" echo " }" echo " /* store the destination: */" @@ -472,8 +660,8 @@ for arch in 32; do echo " count %= 32;" if test ${insn} != sll; then echo "" - echo " /* clip the value to 32 bits: */" - echo " dst = (tme_${sign}int32_t) dst;" + echo " /* clip the value to 32 bits: */" + echo " dst = (tme_${sign}int32_t) dst;" fi echo " }" fi @@ -486,7 +674,7 @@ for arch in 32; do if test "${insn}" = sra; then echo "#ifdef SHIFTSIGNED_INT${arch}_T" fi - echo "#if (SHIFTMAX_INT${arch}_T < (${arch} - 1))" + echo "#if defined(SHIFTMAX_INT${arch}_T) && (SHIFTMAX_INT${arch}_T < (${arch} - 1))" echo "#error \"cannot do full shifts of a tme_int${arch}_t\"" echo "#endif /* (SHIFTMAX_INT${arch}_T < (${arch} - 1)) */" if test ${insn} = sll; then @@ -511,6 +699,45 @@ for arch in 32; do ;; esac + # the sdivx, udivx, and mulx instructions: + # + case ${insn} in + sdivx) sign= ;; + udivx | mulx) sign=u ;; + *) sign=x ;; + esac + if test "x${sign}" != xx; then + echo "" + echo "/* the sparc${arch} ${insn} function: */" + echo "TME_SPARC_FORMAT3(tme_sparc${arch}_${insn}, tme_uint${arch}_t)" + echo "{" + echo " tme_${sign}int64_t src1;" + echo " tme_${sign}int64_t src2;" + echo " tme_${sign}int64_t dst;" + echo "" + echo " /* get the operands: */" + echo " src1 = TME_SPARC_FORMAT3_RS1;" + echo " src2 = TME_SPARC_FORMAT3_RS2;" + echo "" + echo " /* do the ${insn}: */" + if test ${insn} = mulx; then + echo " dst = src1 * src2;" + else + echo " if (__tme_predict_false(src2 == 0)) {" + echo " tme_sparc${arch}_trap(ic, TME_SPARC${arch}_TRAP_division_by_zero);" + echo " }" + if test ${insn} = sdivx; then + echo " dst = (src2 == -1 && src1 == (((tme_int64_t) 1) << 63) ? src1 : src1 / src2);" + else + echo " dst = src1 / src2;" + fi + fi + echo "" + echo " TME_SPARC_FORMAT3_RD = dst;" + echo " TME_SPARC_INSN_OK;" + echo "}" + fi + # a load or store instruction: # size= @@ -519,6 +746,8 @@ for arch in 32; do ldh | sth | ldha | stha) size=16 ;; ld | st | swap | lda | sta | swapa) size=32 ;; ldd | std | ldda | stda) size=32 ;; + ldx | stx | ldxa | stxa | casxa) size=64 ;; + casa) size=32 ;; esac if test "x${size}" != x; then @@ -534,7 +763,7 @@ for arch in 32; do atomic=false double=false case "${insn}" in - ldstub | ldstuba | swap | swapa) atomic=true ;; + ldstub | ldstuba | swap | swapa | casa | casxa) atomic=true ;; ldd* | std*) double=true ; size=64 ;; esac @@ -568,11 +797,38 @@ for arch in 32; do asi_mask_data="ic->tme_sparc_asi_mask_data" fi echo " tme_uint${arch}_t address;" + if test ${arch} = 64 && ${alternate}; then + echo " tme_bus_context_t context;" + context=context + else + context="ic->tme_sparc_memory_context_default" + fi + echo " tme_uint32_t asi_mask_flags_slow;" echo " struct tme_sparc_tlb *dtlb;" echo -n " " if test ${slow} = load; then echo -n "const "; fi echo "tme_shared tme_uint8_t *memory;" - echo " tme_uint${arch}_t value;" + echo " tme_bus_context_t dtlb_context;" + echo " tme_uint32_t endian_little;" + case "${insn}" in + ldd* | std* | swap*) echo " tme_uint32_t value32;" ;; + ldstub*) ;; + cas*a) + echo " unsigned int reg_rs2;" + echo " tme_uint${size}_t value_compare${size};" + echo " tme_uint${size}_t value_swap${size};" + echo " tme_uint${size}_t value_read${size};" + ;; + ld*) + echo " tme_uint${size}_t value${size};" + size_extend=${arch} + if test `expr ${size} \< ${arch}` = 1; then + if test `expr ${size} \< 32` = 1; then size_extend=32; fi + echo " tme_uint${size_extend}_t value${size_extend};" + fi + ;; + st*) echo " tme_uint${size}_t value${size};" ;; + esac if ${alternate}; then echo "" @@ -582,7 +838,13 @@ for arch in 32; do echo "" echo " /* get the address: */" - echo " address = TME_SPARC_FORMAT3_RS1 + TME_SPARC_FORMAT3_RS2;" + case "${insn}" in + cas*a) echo " address = TME_SPARC_FORMAT3_RS1;" ;; + *) echo " address = TME_SPARC_FORMAT3_RS1 + TME_SPARC_FORMAT3_RS2;" ;; + esac + if test ${arch} = 64; then + echo " address &= ic->tme_sparc_address_mask;" + fi echo "" echo "#ifdef _TME_SPARC_STATS" @@ -590,38 +852,110 @@ for arch in 32; do echo " ic->tme_sparc_stats.tme_sparc_stats_memory_total++;" echo "#endif /* _TME_SPARC_STATS */" + echo "" + echo " /* verify and maybe replay this transfer: */" + if ${double}; then verify_size=32; else verify_size=${size}; fi + case "${insn}" in + ld*) verify_flags=TME_SPARC_RECODE_VERIFY_MEM_LOAD ;; + swap* | cas*a) verify_flags="TME_SPARC_RECODE_VERIFY_MEM_LOAD | TME_SPARC_RECODE_VERIFY_MEM_STORE" ;; + st*) verify_flags=TME_SPARC_RECODE_VERIFY_MEM_STORE ;; + *) verify_flags= ;; + esac + echo " tme_sparc_recode_verify_mem(ic, &TME_SPARC_FORMAT3_RD," + echo " ${asi_mask_data}, address," + echo " (TME_RECODE_SIZE_${verify_size}" + echo " | ${verify_flags}));" + if ${double}; then + echo " tme_sparc_recode_verify_mem(ic, &TME_SPARC_FORMAT3_RD_ODD(tme_ic_ireg_uint${arch})," + echo " ${asi_mask_data}, address + sizeof(tme_uint32_t)," + echo " (TME_RECODE_SIZE_${verify_size}" + echo " | ${verify_flags}));" + fi + echo " if (tme_sparc_recode_verify_replay_last_pc(ic) != 0) {" + echo " TME_SPARC_INSN_OK;" + echo " }" + # if this is some kind of a store, except for an ldstub: # case "${insn}" in std*) echo "" echo " /* log the values stored: */" - # XXX FIXME - we can't verify a std yet: - # echo " tme_sparc_log(ic, 1000, TME_OK, " echo " (TME_SPARC_LOG_HANDLE(ic)," - echo " _(\"${insn}\t0x%02x:0x%0"`expr ${arch} / 4`"x:\t0x%08x 0x%08x\")," - echo " TME_SPARC_ASI_MASK_WHICH(${asi_mask_data})," + echo " _(\"${insn}\t0x%02x:0x%0"`expr ${arch} / 4`"\" TME_PRIx${arch} \":\t0x%08\" TME_PRIx32 \" 0x%08\" TME_PRIx32)," + echo " TME_SPARC_ASI_MASK_WHICH(${asi_mask_data} & ~TME_SPARC_ASI_MASK_FLAG_UNDEF)," echo " address," echo " (tme_uint32_t) TME_SPARC_FORMAT3_RD," - echo " (tme_uint32_t) TME_SPARC_FORMAT3_RD_ODD));" + echo " (tme_uint32_t) TME_SPARC_FORMAT3_RD_ODD(tme_ic_ireg_uint${arch})));" ;; - st* | swap*) + st* | swap* | cas*a) echo "" echo " /* log the value stored: */" - echo " tme_sparc_verify_mem${size}(ic, ${asi_mask_data}, address, (tme_uint${size}_t) TME_SPARC_FORMAT3_RD, TME_BUS_CYCLE_WRITE);" echo " tme_sparc_log(ic, 1000, TME_OK, " echo " (TME_SPARC_LOG_HANDLE(ic)," - echo " _(\"${insn}\t0x%02x:0x%0"`expr ${arch} / 4`"x:\t0x%0"`expr ${size} / 4`"x\")," - echo " TME_SPARC_ASI_MASK_WHICH(${asi_mask_data})," + echo " _(\"${insn}\t0x%02x:0x%0"`expr ${arch} / 4`"\" TME_PRIx${arch} \":\t0x%0"`expr ${size} / 4`"\" TME_PRIx${size})," + echo " TME_SPARC_ASI_MASK_WHICH(${asi_mask_data} & ~TME_SPARC_ASI_MASK_FLAG_UNDEF)," echo " address," echo " (tme_uint${size}_t) TME_SPARC_FORMAT3_RD));" ;; esac + if test "${context}" = context; then + echo "" + echo " /* get the context: */" + if test ${arch} = 64; then + echo " context = ic->tme_sparc_memory_context_primary;" + echo " if (__tme_predict_false(${asi_mask_data}" + echo " & (TME_SPARC64_ASI_FLAG_SECONDARY" + echo " + TME_SPARC64_ASI_MASK_FLAG_INSN_NUCLEUS))) {" + echo " if (${asi_mask_data} & TME_SPARC64_ASI_FLAG_SECONDARY) {" + echo " context = ic->tme_sparc_memory_context_secondary;" + echo " }" + echo " else if (TME_SPARC_MEMORY_FLAGS(ic) & TME_SPARC_MEMORY_FLAG_HAS_NUCLEUS) {" + echo " context = 0;" + echo " }" + echo " }" + fi + fi + + echo "" + echo " /* assume that no DTLB ASI mask flags will require a slow ${slow}: */" + echo " asi_mask_flags_slow = 0;" + if test ${arch} = 64; then + + echo "" + if test ${slow} != load || ${atomic}; then + echo " /* a ${insn} traps on no-fault addresses: */" + else + echo " /* a ${insn} without a no-fault ASI traps on no-fault addresses: */" + fi + echo " asi_mask_flags_slow |= TME_SPARC64_ASI_FLAG_NO_FAULT;" + if ${atomic}; then + echo "" + echo " /* a ${insn} traps on uncacheable addresses with side-effects: */" + echo " asi_mask_flags_slow |= TME_SPARC64_ASI_MASK_FLAG_TLB_UNCACHEABLE;" + fi + + if ${alternate}; then + echo "" + echo " /* if this ${insn} is using a no-fault ASI: */" + echo " if (__tme_predict_false(${asi_mask_data} & TME_SPARC64_ASI_FLAG_NO_FAULT)) {" + echo "" + if test ${slow} != load || ${atomic}; then + echo " /* a ${insn} with a no-fault ASI traps: */" + echo " asi_mask_flags_slow = 0 - (tme_uint32_t) 1;" + else + echo " /* a ${insn} with a no-fault ASI traps on addresses with side-effects: */" + echo " asi_mask_flags_slow = TME_SPARC64_ASI_MASK_FLAG_TLB_SIDE_EFFECTS;" + fi + echo " }" + fi + fi + echo "" echo " /* get and busy the DTLB entry: */" - echo " dtlb = TME_SPARC_DTLB_ENTRY(ic, address);" + echo " dtlb = &ic->tme_sparc_tlbs[TME_SPARC_DTLB_ENTRY(ic, TME_SPARC_TLB_HASH(ic, ${context}, address))];" echo " tme_sparc_tlb_busy(dtlb);" echo "" @@ -629,19 +963,32 @@ for arch in 32; do echo " memory = dtlb->tme_sparc_tlb_emulator_off_${cycle};" echo "" + echo " /* if this DTLB matches any context, it matches this context: */" + echo " dtlb_context = dtlb->tme_sparc_tlb_context;" + echo " if (dtlb_context > ic->tme_sparc_memory_context_max) {" + echo " dtlb_context = ${context};" + echo " }" + + echo "" echo " /* we must call the slow ${slow} function if: */" echo " if (__tme_predict_false(" echo "" echo " /* the DTLB entry is invalid: */" echo " tme_bus_tlb_is_invalid(&dtlb->tme_sparc_tlb_bus_tlb)" echo "" + echo " /* the DTLB entry does not match the context: */" + echo " || dtlb_context != ${context}" + echo "" echo " /* the DTLB entry does not cover the needed addresses: */" - echo " || (dtlb->tme_sparc_tlb_addr_first > address)" - echo " || (dtlb->tme_sparc_tlb_addr_last < (address + ((${size} / 8) - 1)))" + echo " || (address < (tme_bus_addr${arch}_t) dtlb->tme_sparc_tlb_addr_first)" + echo " || ((address + ((${size} / 8) - 1)) > (tme_bus_addr${arch}_t) dtlb->tme_sparc_tlb_addr_last)" echo "" echo " /* the DTLB entry does not cover the needed address space: */" echo " || (!TME_SPARC_TLB_ASI_MASK_OK(dtlb, ${asi_mask_data}))" echo "" + echo " /* the DTLB entry can't be used for a fast ${insn}: */" + echo " || (dtlb->tme_sparc_tlb_asi_mask & asi_mask_flags_slow) != 0" + echo "" echo " /* the DTLB entry does not allow fast transfers: */" if $atomic; then echo " || (memory != dtlb->tme_sparc_tlb_emulator_off_read)" @@ -662,33 +1009,90 @@ for arch in 32; do echo "" echo " /* call the slow ${slow} function: */" - echo " memory = tme_sparc${arch}_${slow}(ic," - echo " address," - if test ${slow} = store; then - echo " &TME_SPARC_FORMAT3_RD," - fi - echo -n " (" + echo " memory = tme_sparc${arch}_ls(ic," + echo " address," + echo " &TME_SPARC_FORMAT3_RD," + echo -n " (TME_SPARC_LSINFO_OP_" if ${atomic}; then - echo "TME_SPARC_SLOW_FLAG_ATOMIC" - echo -n " | " + echo "ATOMIC" + elif test ${slow} = store; then + echo "ST" + else + echo "LD" fi + echo -n " | " + case ${insn} in + ldd* | std*) + echo "TME_SPARC_LSINFO_LDD_STD" + echo -n " | " + ;; + esac if ${alternate}; then - echo "TME_SPARC_SLOW_FLAG_A" - echo -n " | " + echo "TME_SPARC_LSINFO_ASI(TME_SPARC_ASI_MASK_WHICH(${asi_mask_data} & ~TME_SPARC_ASI_MASK_FLAG_UNDEF))" + echo " | TME_SPARC_LSINFO_A" + echo -n " | " fi echo "(${size} / 8)));" - if test ${slow} = store; then + if test ${slow} = store || ${alternate}; then echo "" - echo " /* if the slow ${slow} function did the transfer, return now: */" + echo " /* if the slow ${slow} function did the transfer: */" echo " if (__tme_predict_false(memory == TME_EMULATOR_OFF_UNDEF)) {" + echo "" + echo " /* unbusy the TLB entry; */" echo " tme_sparc_tlb_unbusy(dtlb);" + + # if this is some kind of a load, log the value loaded: + # + case ${insn} in + ldd*) + echo "" + echo " /* log the value loaded: */" + echo " tme_sparc_recode_verify_mem_load(ic, &TME_SPARC_FORMAT3_RD);" + echo " tme_sparc_recode_verify_mem_load(ic, &TME_SPARC_FORMAT3_RD_ODD(tme_ic_ireg_uint${arch}));" + echo " tme_sparc_log(ic, 1000, TME_OK," + echo " (TME_SPARC_LOG_HANDLE(ic)," + echo " _(\"${insn}\t0x%02x:0x%0"`expr ${arch} / 4`"\" TME_PRIx${arch} \":\t0x%0"`expr ${arch} / 4`"\" TME_PRIx${arch} \" 0x%0"`expr ${arch} / 4`"\" TME_PRIx${arch} \"\")," + echo " TME_SPARC_ASI_MASK_WHICH(${asi_mask_data} & ~TME_SPARC_ASI_MASK_FLAG_UNDEF)," + echo " address," + echo " TME_SPARC_FORMAT3_RD," + echo " TME_SPARC_FORMAT3_RD_ODD(tme_ic_ireg_uint${arch})));" + ;; + ld* | ldstub* | swap* | cas*a) + echo "" + echo " /* log the value loaded: */" + echo " tme_sparc_recode_verify_mem_load(ic, &TME_SPARC_FORMAT3_RD);" + echo " tme_sparc_log(ic, 1000, TME_OK," + echo " (TME_SPARC_LOG_HANDLE(ic)," + echo " _(\"${insn}\t0x%02x:0x%0"`expr ${arch} / 4`"\" TME_PRIx${arch} \":\t0x%0"`expr ${size} / 4`"\" TME_PRIx${arch})," + echo " TME_SPARC_ASI_MASK_WHICH(${asi_mask_data} & ~TME_SPARC_ASI_MASK_FLAG_UNDEF)," + echo " address," + echo " TME_SPARC_FORMAT3_RD));" + ;; + esac + echo "" echo " TME_SPARC_INSN_OK;" echo " }" fi echo " }" echo "" + echo " /* get the byte order of this transfer: */" + if test ${arch} = 32; then + echo " endian_little = FALSE;" + elif test ${arch} = 64; then + echo " endian_little = ${asi_mask_data} & TME_SPARC64_ASI_FLAG_LITTLE;" + echo " if (__tme_predict_false(dtlb->tme_sparc_tlb_asi_mask & TME_SPARC64_ASI_FLAG_LITTLE)) {" + echo " if (TME_SPARC_MEMORY_FLAGS(ic) & TME_SPARC_MEMORY_FLAG_HAS_INVERT_ENDIAN) {" + echo " endian_little ^= TME_SPARC64_ASI_FLAG_LITTLE;" + echo " }" + echo " else {" + echo " assert (FALSE);" + echo " }" + echo " }" + fi + + echo "" echo " /* do the fast transfer: */" echo " memory += address;" @@ -696,44 +1100,63 @@ for arch in 32; do # case "${insn}" in ldd*) - if test ${arch} = 32; then - echo " value = tme_memory_bus_read32(((const tme_shared tme_uint32_t *) memory) + 0, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint32_t) * 2, sizeof(tme_uint${arch}_t));" - echo " TME_SPARC_FORMAT3_RD = tme_betoh_u32(value);" - echo " value = tme_memory_bus_read32(((const tme_shared tme_uint32_t *) memory) + 1, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint32_t), sizeof(tme_uint${arch}_t));" - echo " TME_SPARC_FORMAT3_RD_ODD = tme_betoh_u32(value);" - else - echo "$PROG fatal: need ${insn} support for sparc${arch}" 1>&2 - exit 1 - fi + echo " value32 = tme_memory_bus_read32(((const tme_shared tme_uint32_t *) memory) + 0, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint32_t) * 2, sizeof(tme_uint${arch}_t));" + echo " value32 = (endian_little ? tme_letoh_u32(value32) : tme_betoh_u32(value32));" + echo " TME_SPARC_FORMAT3_RD = value32;" + echo " value32 = tme_memory_bus_read32(((const tme_shared tme_uint32_t *) memory) + 1, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint32_t) * 1, sizeof(tme_uint${arch}_t));" + echo " value32 = (endian_little ? tme_letoh_u32(value32) : tme_betoh_u32(value32));" + echo " TME_SPARC_FORMAT3_RD_ODD(tme_ic_ireg_uint${arch}) = value32;" ;; std*) - if test ${arch} = 32; then - echo " value = tme_htobe_u32(TME_SPARC_FORMAT3_RD);" - echo " tme_memory_bus_write32(((tme_shared tme_uint32_t *) memory) + 0, value, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint32_t) * 2, sizeof(tme_uint${arch}_t));" - echo " value = tme_htobe_u32(TME_SPARC_FORMAT3_RD_ODD);" - echo " tme_memory_bus_write32(((tme_shared tme_uint32_t *) memory) + 1, value, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint32_t), sizeof(tme_uint${arch}_t));" - else - echo "$PROG fatal: need ${insn} support for sparc${arch}" 1>&2 - exit 1 - fi + echo " value32 = TME_SPARC_FORMAT3_RD;" + echo " value32 = (endian_little ? tme_htole_u32(value32) : tme_htobe_u32(value32));" + echo " tme_memory_bus_write32(((tme_shared tme_uint32_t *) memory) + 0, value32, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint32_t) * 2, sizeof(tme_uint${arch}_t));" + echo " value32 = TME_SPARC_FORMAT3_RD_ODD(tme_ic_ireg_uint${arch});" + echo " value32 = (endian_little ? tme_htole_u32(value32) : tme_htobe_u32(value32));" + echo " tme_memory_bus_write32(((tme_shared tme_uint32_t *) memory) + 1, value32, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint32_t) * 1, sizeof(tme_uint${arch}_t));" ;; ldstub*) - echo " value = tme_memory_atomic_xchg8(memory, 0xff, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint8_t));" + echo " TME_SPARC_FORMAT3_RD = tme_memory_atomic_xchg8(memory, 0xff, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint8_t));" ;; swap*) - echo " value = tme_htobe_u32(TME_SPARC_FORMAT3_RD);" - echo " value = tme_memory_atomic_xchg32((tme_shared tme_uint${size}_t *) memory, value, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint8_t));" + echo " value32 = TME_SPARC_FORMAT3_RD;" + echo " value32 = (endian_little ? tme_htole_u32(value32) : tme_htobe_u32(value32));" + echo " value32 = tme_memory_atomic_xchg32((tme_shared tme_uint${size}_t *) memory, value32, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint8_t));" + echo " value32 = (endian_little ? tme_letoh_u32(value32) : tme_betoh_u32(value32));" + echo " TME_SPARC_FORMAT3_RD = value32;" + ;; + cas*a) + echo " reg_rs2 = TME_FIELD_MASK_EXTRACTU(TME_SPARC_INSN, TME_SPARC_FORMAT3_MASK_RS2);" + echo " TME_SPARC_REG_INDEX(ic, reg_rs2);" + echo " value_compare${size} = ic->tme_sparc_ireg_uint${arch}(reg_rs2);" + echo " value_compare${size} = (endian_little ? tme_htole_u${size}(value_compare${size}) : tme_htobe_u${size}(value_compare${size}));" + echo " value_swap${size} = TME_SPARC_FORMAT3_RD;" + echo " value_swap${size} = (endian_little ? tme_htole_u${size}(value_swap${size}) : tme_htobe_u${size}(value_swap${size}));" + echo " value_read${size} = tme_memory_atomic_cx${size}((tme_shared tme_uint${size}_t *) memory, value_compare${size}, value_swap${size}, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint${size}_t));" + echo " value_read${size} = (endian_little ? tme_letoh_u${size}(value_read${size}) : tme_betoh_u${size}(value_read${size}));" + echo " TME_SPARC_FORMAT3_RD = value_read${size};" ;; ld*) - echo " value = tme_memory_bus_read${size}((const tme_shared tme_uint${size}_t *) memory, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint${size}_t), sizeof(tme_uint${arch}_t));" + echo " value${size} = tme_memory_bus_read${size}((const tme_shared tme_uint${size}_t *) memory, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint${size}_t), sizeof(tme_uint${arch}_t));" if test ${size} != 8; then - echo " value = tme_betoh_u${size}(value);" + echo " value${size} = (endian_little ? tme_letoh_u${size}(value${size}) : tme_betoh_u${size}(value${size}));" fi + if test `expr ${size} \< ${arch}` = 1; then + echo "" + echo " /* possibly sign-extend the loaded value: */" + echo " value${size_extend} = value${size};" + echo " if (TME_SPARC_INSN & TME_BIT(22)) {" + echo " value${size_extend} = (tme_uint${size_extend}_t) (tme_int${size_extend}_t) (tme_int${size}_t) value${size_extend};" + echo " }" + fi + echo " TME_SPARC_FORMAT3_RD = (tme_uint${arch}_t) (tme_int${arch}_t) (tme_int${size_extend}_t) value${size_extend};" ;; st*) - if test ${size} = 8; then swap_st= ; else swap_st="tme_htobe_u${size}"; fi - echo " value = ${swap_st}(TME_SPARC_FORMAT3_RD);" - echo " tme_memory_bus_write${size}((tme_shared tme_uint${size}_t *) memory, value, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint${size}_t), sizeof(tme_uint${arch}_t));" + echo " value${size} = TME_SPARC_FORMAT3_RD;" + if test ${size} != 8; then + echo " value${size} = (endian_little ? tme_htole_u${size}(value${size}) : tme_htobe_u${size}(value${size}));" + fi + echo " tme_memory_bus_write${size}((tme_shared tme_uint${size}_t *) memory, value${size}, dtlb->tme_sparc_tlb_bus_rwlock, sizeof(tme_uint${size}_t), sizeof(tme_uint${arch}_t));" ;; *) echo "$PROG internal error: unknown memory insn ${insn}" 1>&2 ; exit 1 ;; esac @@ -741,22 +1164,6 @@ for arch in 32; do echo "" echo " /* unbusy the DTLB entry: */" echo " tme_sparc_tlb_unbusy(dtlb);" - - case "${insn}" in - ldd*) ;; - ld* | swap*) - if test `expr ${size} \< ${arch}` = 1; then - echo "" - echo " /* possibly sign-extend the loaded value: */" - echo " if (TME_SPARC_INSN & TME_BIT(22)) {" - echo " value = (tme_uint${arch}_t) (tme_int${arch}_t) (tme_int${size}_t) value;" - echo " }" - fi - echo "" - echo " /* set the loaded value: */" - echo " TME_SPARC_FORMAT3_RD = value;" - ;; - esac # if this is some kind of a load, log the value loaded: # @@ -764,24 +1171,24 @@ for arch in 32; do ldd*) echo "" echo " /* log the value loaded: */" - # XXX FIXME - we can't verify a ldd yet: - # + echo " tme_sparc_recode_verify_mem_load(ic, &TME_SPARC_FORMAT3_RD);" + echo " tme_sparc_recode_verify_mem_load(ic, &TME_SPARC_FORMAT3_RD_ODD(tme_ic_ireg_uint${arch}));" echo " tme_sparc_log(ic, 1000, TME_OK," echo " (TME_SPARC_LOG_HANDLE(ic)," - echo " _(\"${insn}\t0x%02x:0x%0"`expr ${arch} / 4`"x:\t0x%08x 0x%08x\")," - echo " TME_SPARC_ASI_MASK_WHICH(${asi_mask_data})," + echo " _(\"${insn}\t0x%02x:0x%0"`expr ${arch} / 4`"\" TME_PRIx${arch} \":\t0x%0"`expr ${arch} / 4`"\" TME_PRIx${arch} \" 0x%0"`expr ${arch} / 4`"\" TME_PRIx${arch} \"\")," + echo " TME_SPARC_ASI_MASK_WHICH(${asi_mask_data} & ~TME_SPARC_ASI_MASK_FLAG_UNDEF)," echo " address," - echo " (tme_uint32_t) TME_SPARC_FORMAT3_RD," - echo " (tme_uint32_t) TME_SPARC_FORMAT3_RD_ODD));" + echo " TME_SPARC_FORMAT3_RD," + echo " TME_SPARC_FORMAT3_RD_ODD(tme_ic_ireg_uint${arch})));" ;; - ld* | ldstub* | swap*) + ld* | ldstub* | swap* | cas*a) echo "" echo " /* log the value loaded: */" - echo " tme_sparc_verify_mem${size}(ic, ${asi_mask_data}, address, (tme_uint${size}_t) TME_SPARC_FORMAT3_RD, TME_BUS_CYCLE_READ);" + echo " tme_sparc_recode_verify_mem_load(ic, &TME_SPARC_FORMAT3_RD);" echo " tme_sparc_log(ic, 1000, TME_OK," echo " (TME_SPARC_LOG_HANDLE(ic)," - echo " _(\"${insn}\t0x%02x:0x%0"`expr ${arch} / 4`"x:\t0x%0"`expr ${arch} / 4`"x\")," - echo " TME_SPARC_ASI_MASK_WHICH(${asi_mask_data})," + echo " _(\"${insn}\t0x%02x:0x%0"`expr ${arch} / 4`"\" TME_PRIx${arch} \":\t0x%0"`expr ${size} / 4`"\" TME_PRIx${arch})," + echo " TME_SPARC_ASI_MASK_WHICH(${asi_mask_data} & ~TME_SPARC_ASI_MASK_FLAG_UNDEF)," echo " address," echo " TME_SPARC_FORMAT3_RD));" ;; @@ -801,6 +1208,7 @@ for arch in 32; do echo "TME_SPARC_FORMAT3(tme_sparc${arch}_${insn}, tme_uint${arch}_t)" echo "{" echo " tme_uint${arch}_t pc_next_next;" + echo " tme_uint32_t ls_faults;" echo "" echo " /* \"The JMPL instruction causes a register-indirect delayed control" echo " transfer to the address given by r[rs1] + r[rs2] if the i field is" @@ -810,9 +1218,29 @@ for arch in 32; do echo " bits of the jump address is nonzero, a mem_address_not_aligned" echo " trap occurs.\" */" echo "" + echo " /* get the target address: */" echo " pc_next_next = TME_SPARC_FORMAT3_RS1 + TME_SPARC_FORMAT3_RS2;" + if test ${arch} = 64; then + echo " pc_next_next &= ic->tme_sparc_address_mask;" + fi + echo "" + echo " /* set the delayed control transfer: */" + echo " ic->tme_sparc_ireg_uint${arch}(TME_SPARC_IREG_PC_NEXT_NEXT) = pc_next_next;" + echo "" + echo " /* check the target address: */" + echo " ls_faults = TME_SPARC_LS_FAULT_NONE;" + if test ${arch} = 64; then + echo " if (__tme_predict_false((pc_next_next" + echo " + ic->tme_sparc${arch}_ireg_va_hole_start)" + echo " > ((ic->tme_sparc${arch}_ireg_va_hole_start * 2) - 1))) {" + echo " ls_faults += TME_SPARC64_LS_FAULT_VA_RANGE_NNPC;" + echo " }" + fi echo " if (__tme_predict_false((pc_next_next % sizeof(tme_uint32_t)) != 0)) {" - echo " TME_SPARC_INSN_TRAP(TME_SPARC_TRAP_mem_address_not_aligned);" + echo " ls_faults += TME_SPARC_LS_FAULT_ADDRESS_NOT_ALIGNED;" + echo " }" + echo " if (__tme_predict_false(ls_faults != TME_SPARC_LS_FAULT_NONE)) {" + echo " tme_sparc_nnpc_trap(ic, ls_faults);" echo " }" echo "" echo " /* write the PC of the jmpl into r[rd]: */" @@ -822,7 +1250,7 @@ for arch in 32; do echo " if (TME_FIELD_MASK_EXTRACTU(TME_SPARC_INSN, TME_SPARC_FORMAT3_MASK_RD) == 15) {" echo " tme_sparc_log(ic, 250, TME_OK," echo " (TME_SPARC_LOG_HANDLE(ic)," - echo " _(\"call 0x%08x\")," + echo " _(\"call 0x%0"`expr ${arch} / 4`"\" TME_PRIx${arch})," echo " pc_next_next));" echo " }" echo "" @@ -832,41 +1260,54 @@ for arch in 32; do echo " == ((tme_uint32_t) (0x2 << 30) | (0 << 25) | (0x38 << 19) | (31 << 14) | (0x1 << 13) | 8)) {" echo " tme_sparc_log(ic, 250, TME_OK," echo " (TME_SPARC_LOG_HANDLE(ic)," - echo " _(\"retl 0x%08x\")," + echo " _(\"retl 0x%0"`expr ${arch} / 4`"\" TME_PRIx${arch})," echo " pc_next_next));" echo " }" echo "" - echo " /* set the delayed control transfer: */" - echo " ic->tme_sparc_ireg_uint${arch}(TME_SPARC_IREG_PC_NEXT_NEXT) = pc_next_next;" - echo "" echo " TME_SPARC_INSN_OK;" echo "}" fi + # this may be an alternate-space version of a floating-point + # load or store instruction: + # + case ${insn} in *a) alternate=a ;; *) alternate= ;; esac + # the ldf instruction: # - if test ${insn} = ldf; then + if test ${insn} = "ldf${alternate}"; then echo "" echo "/* this does a sparc${arch} ${insn}: */" echo "TME_SPARC_FORMAT3(tme_sparc${arch}_${insn}, tme_uint${arch}_t)" echo "{" - echo " unsigned int fpreg_number;" - echo "" - echo " TME_SPARC_INSN_FPU;" + echo " tme_uint32_t misaligned;" + echo " struct tme_float float_buffer;" + echo " struct tme_float *fpreg;" + echo "" + echo " /* get the least significant 32 bits of the address: */" + echo " misaligned = TME_SPARC_FORMAT3_RS1;" + echo " misaligned += (tme_uint32_t) TME_SPARC_FORMAT3_RS2;" + if test "${arch}${alternate}" = 64a; then + echo "" + echo " /* see if the address is misaligned for the ASI: */" + echo " misaligned = (*ic->_tme_sparc_ls_asi_misaligned)(ic, misaligned);" + fi echo "" echo " /* decode rd: */" - echo " fpreg_number = TME_FIELD_MASK_EXTRACTU(TME_SPARC_INSN, TME_SPARC_FORMAT3_MASK_RD);" - echo "" - echo " /* make sure this floating-point register is single-precision: */" - echo " tme_sparc_fpu_fpreg_format(ic, fpreg_number, TME_IEEE754_FPREG_FORMAT_SINGLE | TME_IEEE754_FPREG_FORMAT_BUILTIN);" + echo " float_buffer.tme_float_format = TME_FLOAT_FORMAT_IEEE754_SINGLE;" + echo " fpreg" + echo " = _tme_sparc${arch}_fpu_mem_fpreg(ic," + echo " misaligned," + echo " &float_buffer);" echo "" echo " /* do the load: */" - echo " tme_sparc${arch}_ld(ic, _rs1, _rs2, &ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_FPX${reg32_shift}));" + echo " tme_sparc${arch}_ld${alternate}(ic, _rs1, _rs2, &ic->tme_sparc_ireg_uint${arch}(TME_SPARC_IREG_FPX));" echo "" echo " /* set the floating-point register value: */" - echo " ic->tme_sparc_fpu_fpregs[fpreg_number].tme_float_format = TME_FLOAT_FORMAT_IEEE754_SINGLE;" - echo " ic->tme_sparc_fpu_fpregs[fpreg_number].tme_float_value_ieee754_single" + echo " assert (fpreg != &float_buffer);" + echo " fpreg->tme_float_format = TME_FLOAT_FORMAT_IEEE754_SINGLE;" + echo " fpreg->tme_float_value_ieee754_single" echo " = ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_FPX${reg32_shift});" echo "" echo " TME_SPARC_INSN_OK;" @@ -875,69 +1316,126 @@ for arch in 32; do # the stf instruction: # - if test ${insn} = stf; then + if test ${insn} = "stf${alternate}"; then echo "" echo "/* this does a sparc${arch} ${insn}: */" echo "TME_SPARC_FORMAT3(tme_sparc${arch}_${insn}, tme_uint${arch}_t)" echo "{" - echo " unsigned int fpreg_number;" + echo " tme_uint32_t misaligned;" + echo " struct tme_float float_buffer;" + echo " const struct tme_float *fpreg;" echo " const tme_uint32_t *value_single;" - echo " tme_uint32_t value_single_buffer;" echo "" - echo " TME_SPARC_INSN_FPU_STORE(sizeof(tme_uint32_t));" + echo " /* get the least significant 32 bits of the address: */" + echo " misaligned = TME_SPARC_FORMAT3_RS1;" + echo " misaligned += (tme_uint32_t) TME_SPARC_FORMAT3_RS2;" + if test "${arch}${alternate}" = 64a; then + echo "" + echo " /* see if the address is misaligned for the ASI: */" + echo " misaligned = (*ic->_tme_sparc_ls_asi_misaligned)(ic, misaligned);" + fi echo "" echo " /* decode rd: */" - echo " fpreg_number = TME_FIELD_MASK_EXTRACTU(TME_SPARC_INSN, TME_SPARC_FORMAT3_MASK_RD);" - echo "" - echo " /* make sure this floating-point register is single-precision: */" - echo " tme_sparc_fpu_fpreg_format(ic, fpreg_number, TME_IEEE754_FPREG_FORMAT_SINGLE | TME_IEEE754_FPREG_FORMAT_BUILTIN);" + echo " float_buffer.tme_float_format = TME_FLOAT_FORMAT_IEEE754_SINGLE;" + echo " fpreg" + echo " = _tme_sparc${arch}_fpu_mem_fpreg(ic," + echo " misaligned," + echo " &float_buffer);" echo "" echo " /* get this single floating-point register in IEEE754 single-precision format: */" - echo " value_single = tme_ieee754_single_value_get(&ic->tme_sparc_fpu_fpregs[fpreg_number], &value_single_buffer);" + echo " value_single = tme_ieee754_single_value_get(fpreg, &float_buffer.tme_float_value_ieee754_single);" echo "" echo " /* set the floating-point register value: */" echo " ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_FPX${reg32_shift}) = *value_single;" echo "" echo " /* do the store: */" - echo " tme_sparc${arch}_st(ic, _rs1, _rs2, &ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_FPX${reg32_shift}));" + echo " tme_sparc${arch}_st${alternate}(ic, _rs1, _rs2, &ic->tme_sparc_ireg_uint${arch}(TME_SPARC_IREG_FPX));" echo "" + echo " assert (fpreg != &float_buffer);" echo " TME_SPARC_INSN_OK;" echo "}" fi # the lddf instruction: # - if test ${insn} = lddf; then + if test ${insn} = "lddf${alternate}"; then echo "" echo "/* this does a sparc${arch} ${insn}: */" echo "TME_SPARC_FORMAT3(tme_sparc${arch}_${insn}, tme_uint${arch}_t)" echo "{" - echo " unsigned int fpreg_number;" - echo "" - echo " TME_SPARC_INSN_FPU;" - echo "" - echo " /* decode rd: */" - echo " fpreg_number = TME_FIELD_MASK_EXTRACTU(TME_SPARC_INSN, TME_SPARC_FORMAT3_MASK_RD);" - echo "" - echo " /* make sure rd is aligned: */" - echo " if (!tme_sparc_fpu_fpreg_aligned(ic, fpreg_number, TME_IEEE754_FPREG_FORMAT_DOUBLE)) {" - echo " fpreg_number -= 1;" - echo " }" + echo " tme_uint${arch}_t address;" + echo " tme_uint32_t misaligned;" + echo " struct tme_float float_buffer;" + echo " struct tme_float *fpreg;" + if test ${arch} != 32; then + echo " tme_uint${arch}_t offset;" + fi echo "" - echo " /* make sure that the double floating-point register is double-precision: */" - echo " tme_sparc_fpu_fpreg_format(ic, fpreg_number, TME_IEEE754_FPREG_FORMAT_DOUBLE | TME_IEEE754_FPREG_FORMAT_BUILTIN);" + echo " /* get the address: */" + echo " address = TME_SPARC_FORMAT3_RS1 + TME_SPARC_FORMAT3_RS2;" echo "" - echo " /* do the load: */" - echo " tme_sparc${arch}_ldd(ic, _rs1, _rs2, &ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_FPX${reg32_shift}));" + echo " /* get the least significant 32 bits of the address: */" + echo " misaligned = address;" + if test "${arch}${alternate}" = 64a; then + echo "" + echo " /* see if the address is misaligned for the ASI: */" + echo " misaligned = (*ic->_tme_sparc_ls_asi_misaligned)(ic, misaligned);" + fi echo "" - echo " /* set the double floating-point register value: */" - echo " ic->tme_sparc_fpu_fpregs[fpreg_number].tme_float_format = TME_FLOAT_FORMAT_IEEE754_DOUBLE;" - echo " ic->tme_sparc_fpu_fpregs[fpreg_number].tme_float_value_ieee754_double.tme_value64_uint32_hi" - echo " = ic->tme_sparc_ireg_uint32((TME_SPARC_IREG_FPX${reg32_shift}) + 0);" - echo " ic->tme_sparc_fpu_fpregs[fpreg_number].tme_float_value_ieee754_double.tme_value64_uint32_lo" - echo " = ic->tme_sparc_ireg_uint32((TME_SPARC_IREG_FPX${reg32_shift}) + 1);" + echo " /* decode rd: */" + echo " float_buffer.tme_float_format = TME_FLOAT_FORMAT_IEEE754_DOUBLE;" + echo " fpreg" + echo " = _tme_sparc${arch}_fpu_mem_fpreg(ic," + echo " misaligned," + echo " &float_buffer);" + echo "" + if test ${arch} = 32; then + echo " /* do the load: */" + echo " tme_sparc${arch}_ldd(ic, _rs1, _rs2, &ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_FPX${reg32_shift}));" + echo "" + echo " /* set the double floating-point register value: */" + echo " assert (fpreg != &float_buffer);" + echo " fpreg->tme_float_format = TME_FLOAT_FORMAT_IEEE754_DOUBLE;" + echo " fpreg->tme_float_value_ieee754_double.tme_value64_uint32_hi" + echo " = ic->tme_sparc_ireg_uint32((TME_SPARC_IREG_FPX${reg32_shift}) + 0);" + echo " fpreg->tme_float_value_ieee754_double.tme_value64_uint32_lo" + echo " = ic->tme_sparc_ireg_uint32((TME_SPARC_IREG_FPX${reg32_shift}) + 1);" + else + echo " /* if bit two of the address is set, and this SPARC supports" + echo " 32-bit-aligned ${insn} instructions: */" + echo " if ((misaligned & sizeof(tme_uint32_t))" + echo " && fpreg != &float_buffer) {" + echo "" + echo " /* do two 32-bit loads: */" + echo " offset = sizeof(tme_uint32_t) * 0;" + echo " tme_sparc${arch}_ld${alternate}(ic, &address, &offset, &ic->tme_sparc_ireg_uint${arch}(TME_SPARC_IREG_FPX + 0));" + echo " offset = sizeof(tme_uint32_t) * 1;" + echo " tme_sparc${arch}_ld${alternate}(ic, &address, &offset, &ic->tme_sparc_ireg_uint${arch}(TME_SPARC_IREG_FPX + 1));" + echo "" + echo " /* set the double floating-point register value: */" + echo " fpreg->tme_float_format = TME_FLOAT_FORMAT_IEEE754_DOUBLE;" + echo " fpreg->tme_float_value_ieee754_double.tme_value64_uint32_hi" + echo " = ic->tme_sparc_ireg_uint64(TME_SPARC_IREG_FPX + 0);" + echo " fpreg->tme_float_value_ieee754_double.tme_value64_uint32_lo" + echo " = ic->tme_sparc_ireg_uint64(TME_SPARC_IREG_FPX + 1);" + echo " }" + echo "" + echo " /* otherwise, bit two of the address is not set, or this SPARC" + echo " doesn't support 32-bit-aligned ${insn} instructions: */" + echo " else {" + echo "" + echo " /* do an ldx${alternate}-style load: */" + echo " tme_sparc${arch}_ldx${alternate}(ic, _rs1, _rs2, &ic->tme_sparc_ireg_uint${arch}(TME_SPARC_IREG_FPX));" + echo "" + echo " /* set the double floating-point register value: */" + echo " assert (fpreg != &float_buffer);" + echo " fpreg->tme_float_format = TME_FLOAT_FORMAT_IEEE754_DOUBLE;" + echo " fpreg->tme_float_value_ieee754_double.tme_value64_uint" + echo " = ic->tme_sparc_ireg_uint64(TME_SPARC_IREG_FPX);" + echo " }" + fi echo "" echo " TME_SPARC_INSN_OK;" echo "}" @@ -945,41 +1443,84 @@ for arch in 32; do # the stdf instruction: # - if test ${insn} = stdf; then + if test ${insn} = "stdf${alternate}"; then echo "" echo "/* this does a sparc${arch} ${insn}: */" echo "TME_SPARC_FORMAT3(tme_sparc${arch}_${insn}, tme_uint${arch}_t)" echo "{" - echo " unsigned int fpreg_number;" + echo " tme_uint${arch}_t address;" + echo " tme_uint32_t misaligned;" + echo " struct tme_float float_buffer;" + echo " struct tme_float *fpreg;" echo " const union tme_value64 *value_double;" - echo " union tme_value64 value_double_buffer;" - echo "" - echo " TME_SPARC_INSN_FPU_STORE(sizeof(tme_uint32_t) * 2);" + if test ${arch} != 32; then + echo " tme_uint${arch}_t offset;" + fi echo "" - echo " /* decode rd: */" - echo " fpreg_number = TME_FIELD_MASK_EXTRACTU(TME_SPARC_INSN, TME_SPARC_FORMAT3_MASK_RD);" + echo " /* get the address: */" + echo " address = TME_SPARC_FORMAT3_RS1 + TME_SPARC_FORMAT3_RS2;" echo "" - echo " /* make sure rd is aligned: */" - echo " if (!tme_sparc_fpu_fpreg_aligned(ic, fpreg_number, TME_IEEE754_FPREG_FORMAT_DOUBLE)) {" - echo " fpreg_number -= 1;" - echo " }" + echo " /* get the least significant 32 bits of the address: */" + echo " misaligned = address;" + if test "${arch}${alternate}" = 64a; then + echo "" + echo " /* see if the address is misaligned for the ASI: */" + echo " misaligned = (*ic->_tme_sparc_ls_asi_misaligned)(ic, misaligned);" + fi echo "" - echo " /* make sure the double floating-point register is double-precision: */" - echo " tme_sparc_fpu_fpreg_format(ic, fpreg_number, TME_IEEE754_FPREG_FORMAT_DOUBLE | TME_IEEE754_FPREG_FORMAT_BUILTIN);" + echo " /* decode rd: */" + echo " float_buffer.tme_float_format = TME_FLOAT_FORMAT_IEEE754_DOUBLE;" + echo " fpreg" + echo " = _tme_sparc${arch}_fpu_mem_fpreg(ic," + echo " misaligned," + echo " &float_buffer);" echo "" echo " /* get this double floating-point register in IEEE754 double-precision format: */" - echo " value_double = tme_ieee754_double_value_get(&ic->tme_sparc_fpu_fpregs[fpreg_number], &value_double_buffer);" - echo "" - echo " /* set the floating-point register value: */" - echo " ic->tme_sparc_ireg_uint32((TME_SPARC_IREG_FPX${reg32_shift}) + 0)" - echo " = value_double->tme_value64_uint32_hi;" - echo " ic->tme_sparc_ireg_uint32((TME_SPARC_IREG_FPX${reg32_shift}) + 1)" - echo " = value_double->tme_value64_uint32_lo;" + echo " value_double = tme_ieee754_double_value_get(fpreg, &float_buffer.tme_float_value_ieee754_double);" echo "" - echo " /* do the store: */" - echo " tme_sparc${arch}_std(ic, _rs1, _rs2, &ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_FPX${reg32_shift}));" + if test ${arch} = 32; then + echo " /* set the floating-point register value: */" + echo " ic->tme_sparc_ireg_uint32((TME_SPARC_IREG_FPX${reg32_shift}) + 0)" + echo " = value_double->tme_value64_uint32_hi;" + echo " ic->tme_sparc_ireg_uint32((TME_SPARC_IREG_FPX${reg32_shift}) + 1)" + echo " = value_double->tme_value64_uint32_lo;" + echo "" + echo " /* do the store: */" + echo " tme_sparc${arch}_std(ic, _rs1, _rs2, &ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_FPX${reg32_shift}));" + else + echo " /* if bit two of the address is set, and this SPARC supports" + echo " 32-bit-aligned ${insn} instructions: */" + echo " if ((misaligned & sizeof(tme_uint32_t))" + echo " && fpreg != &float_buffer) {" + echo "" + echo " /* set the floating-point register value: */" + echo " ic->tme_sparc_ireg_uint64(TME_SPARC_IREG_FPX + 0)" + echo " = value_double->tme_value64_uint32_hi;" + echo " ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_FPX + 1)" + echo " = value_double->tme_value64_uint32_lo;" + echo "" + echo " /* do two 32-bit stores: */" + echo " offset = sizeof(tme_uint32_t) * 0;" + echo " tme_sparc${arch}_st${alternate}(ic, &address, &offset, &ic->tme_sparc_ireg_uint${arch}(TME_SPARC_IREG_FPX + 0));" + echo " offset = sizeof(tme_uint32_t) * 1;" + echo " tme_sparc${arch}_st${alternate}(ic, &address, &offset, &ic->tme_sparc_ireg_uint${arch}(TME_SPARC_IREG_FPX + 1));" + echo " }" + echo "" + echo " /* otherwise, bit two of the address is not set, or this SPARC" + echo " doesn't support 32-bit-aligned ${insn} instructions: */" + echo " else {" + echo "" + echo " /* set the floating-point register value: */" + echo " ic->tme_sparc_ireg_uint64(TME_SPARC_IREG_FPX)" + echo " = value_double->tme_value64_uint;" + echo "" + echo " /* do an stx${alternate}-style store: */" + echo " tme_sparc${arch}_stx${alternate}(ic, _rs1, _rs2, &ic->tme_sparc_ireg_uint${arch}(TME_SPARC_IREG_FPX));" + echo " }" + fi echo "" + echo " assert (fpreg != &float_buffer);" echo " TME_SPARC_INSN_OK;" echo "}" fi @@ -993,11 +1534,38 @@ for arch in 32; do echo "TME_SPARC_FORMAT3(tme_sparc${arch}_${insn}, tme_uint${arch}_t)" echo "{" echo " tme_uint32_t fsr;" + if test ${arch} != 32; then + echo " tme_uint32_t reg_rd;" + echo "" + echo " /* see if this is an ldfsr or an ldxfsr: */" + echo " reg_rd = TME_FIELD_MASK_EXTRACTU(TME_SPARC_INSN, TME_SPARC_FORMAT3_MASK_RD);" + echo " if (__tme_predict_false(reg_rd > 1)) {" + echo " TME_SPARC_INSN_ILL(ic);" + echo " }" + fi echo "" - echo " TME_SPARC_INSN_FPU;" + echo " _tme_sparc${arch}_fpu_mem(ic);" echo "" - echo " /* do the load: */" - echo " tme_sparc${arch}_ld(ic, _rs1, _rs2, &ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_FPX${reg32_shift}));" + if test ${arch} = 32; then + echo " /* do the load: */" + else + echo " /* if this is an ldxfsr: */" + echo " if (reg_rd == 1) {" + echo "" + echo " /* do the load: */" + echo " tme_sparc${arch}_ldx(ic, _rs1, _rs2, &ic->tme_sparc_ireg_uint${arch}(TME_SPARC_IREG_FPX));" + echo "" + echo " /* update the extended FSR: */" + echo " ic->tme_sparc_fpu_xfsr" + echo " = (ic->tme_sparc_ireg_uint32((TME_SPARC_IREG_FPX${reg32_shift}) + 1)" + echo " & 0x3f /* fcc3 .. fcc1 */);" + echo " }" + echo "" + echo " /* otherwise, this is an ldfsr. do the load: */" + echo " else" + echo -n " " + fi + echo " tme_sparc${arch}_ld(ic, _rs1, _rs2, &ic->tme_sparc_ireg_uint${arch}(TME_SPARC_IREG_FPX));" echo "" echo " /* update the FSR: */" echo " fsr = ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_FPX${reg32_shift});" @@ -1018,14 +1586,37 @@ for arch in 32; do echo "/* this does a sparc${arch} ${insn}: */" echo "TME_SPARC_FORMAT3(tme_sparc${arch}_${insn}, tme_uint${arch}_t)" echo "{" + if test ${arch} = 64; then + echo " tme_uint32_t reg_rd;" + echo "" + echo " /* see if this is an stfsr or an stxfsr: */" + echo " reg_rd = TME_FIELD_MASK_EXTRACTU(TME_SPARC_INSN, TME_SPARC_FORMAT3_MASK_RD);" + echo " if (__tme_predict_false(reg_rd > 1)) {" + echo " TME_SPARC_INSN_ILL(ic);" + echo " }" + fi echo "" - echo " TME_SPARC_INSN_FPU_STORE(sizeof(ic->tme_sparc_fpu_fsr));" + echo " _tme_sparc${arch}_fpu_mem(ic);" echo "" echo " /* set the FSR value to store: */" echo " ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_FPX${reg32_shift}) = ic->tme_sparc_fpu_fsr;" echo "" - echo " /* do the store: */" - echo " tme_sparc${arch}_st(ic, _rs1, _rs2, &ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_FPX${reg32_shift}));" + if test ${arch} = 32; then + echo " /* do the store: */" + else + echo " /* if this is an stxfsr: */" + echo " if (reg_rd == 1) {" + echo "" + echo " /* set in the extended FSR to store and do the store: */" + echo " ic->tme_sparc_ireg_uint32((TME_SPARC_IREG_FPX${reg32_shift}) + 1) = ic->tme_sparc_fpu_xfsr;" + echo " tme_sparc${arch}_stx(ic, _rs1, _rs2, &ic->tme_sparc_ireg_uint${arch}(TME_SPARC_IREG_FPX));" + echo " }" + echo "" + echo " /* otherwise, this is a stfsr. do the store: */" + echo " else" + echo -n " " + fi + echo " tme_sparc${arch}_st(ic, _rs1, _rs2, &ic->tme_sparc_ireg_uint${arch}(TME_SPARC_IREG_FPX));" echo "" echo " TME_SPARC_INSN_OK;" echo "}" @@ -1047,6 +1638,150 @@ for arch in 32; do done + # the slow atomic function: + # + if $header; then + echo "void tme_sparc${arch}_atomic _TME_P((struct tme_sparc *, struct tme_sparc_ls *));" + else + echo "" + echo "/* this does a slow atomic operation: */" + echo "void" + echo "tme_sparc${arch}_atomic(struct tme_sparc *ic, struct tme_sparc_ls *ls)" + echo "{" + echo " tme_uint32_t endian_little;" + echo " tme_uint32_t insn;" + if test ${arch} = 64; then + echo " tme_uint${arch}_t value${arch};" + echo " tme_uint${arch}_t value_swap${arch};" + echo " unsigned int reg_rs2;" + fi + echo " tme_uint32_t value32;" + echo " tme_uint32_t value_swap32;" + echo " tme_uint32_t size;" + echo "" + echo " /* if this is the beginning of the operation: */" + echo " if (ls->tme_sparc_ls_state == 0) {" + echo "" + echo " /* start the load part of the operation: */" + echo " ls->tme_sparc_ls_state = ls->tme_sparc_ls_size;" + echo " assert (ls->tme_sparc_ls_state != 0" + echo " && (ls->tme_sparc_ls_state & TME_BIT(7)) == 0);" + echo "" + echo " /* the load must start at the beginning of the buffer: */" + echo " assert (ls->tme_sparc_ls_buffer_offset == 0);" + echo " }" + echo "" + echo " /* if this is the load part of the operation: */" + echo " if ((ls->tme_sparc_ls_state & TME_BIT(7)) == 0) {" + echo "" + echo " /* do one slow load cycle: */" + echo " tme_sparc${arch}_load(ic, ls);" + echo "" + echo " /* if the slow load cycle did not load all of the data: */" + echo " if (__tme_predict_false(ls->tme_sparc_ls_size != 0)) {" + echo " return;" + echo " }" + echo "" + echo " /* get the byte order of this transfer: */" + if test ${arch} = 32; then + echo " endian_little = FALSE;" + else + echo " endian_little = ls->tme_sparc_ls_lsinfo & TME_SPARC_LSINFO_ENDIAN_LITTLE;" + fi + echo "" + echo " /* dispatch on the op3 of the instruction: */" + echo " insn = TME_SPARC_INSN;" + echo " switch ((insn >> 19) & 0x3f) {" + if test ${arch} = 64; then + for size in 32 64; do + echo "" + if test ${size} = 32; then + echo " case 0x3c: /* casa */" + else + echo " case 0x3e: /* casxa */" + fi + echo "" + echo " /* finish the load part of the compare and swap: */" + echo " assert (ls->tme_sparc_ls_state == sizeof(tme_uint${size}_t));" + echo " value${size} = ic->tme_sparc_memory_buffer.tme_sparc_memory_buffer${size}s[0];" + echo " value_swap${size} = *ls->tme_sparc_ls_rd64;" + echo " if (endian_little) {" + echo " value${size} = tme_letoh_u${size}(value${size});" + echo " value_swap${size} = tme_htole_u${size}(value_swap${size});" + echo " }" + echo " else {" + echo " value${size} = tme_betoh_u${size}(value${size});" + echo " value_swap${size} = tme_htobe_u${size}(value_swap${size});" + echo " }" + echo " *ls->tme_sparc_ls_rd64 = value${size};" + echo "" + echo " /* if the comparison fails: */" + echo " reg_rs2 = TME_FIELD_MASK_EXTRACTU(insn, TME_SPARC_FORMAT3_MASK_RS2);" + echo " TME_SPARC_REG_INDEX(ic, reg_rs2);" + echo " if (value${size} != (tme_uint${size}_t) ic->tme_sparc_ireg_uint${arch}(reg_rs2)) {" + echo " return;" + echo " }" + echo "" + echo " /* start the store part of the compare and swap: */" + echo " ic->tme_sparc_memory_buffer.tme_sparc_memory_buffer${size}s[0] = value_swap${size};" + echo " break;" + done + fi + echo "" + echo " case 0x0d: /* ldstub */" + echo " case 0x1d: /* ldstuba */" + echo "" + echo " /* finish the load part of the ldstub: */" + echo " assert (ls->tme_sparc_ls_state == sizeof(tme_uint8_t));" + echo " *ls->tme_sparc_ls_rd${arch} = ic->tme_sparc_memory_buffer.tme_sparc_memory_buffer8s[0];" + echo "" + echo " /* start the store part of the ldstub: */" + echo " ic->tme_sparc_memory_buffer.tme_sparc_memory_buffer8s[0] = 0xff;" + echo " break;" + echo "" + echo " /* otherwise, this must be swap: */" + echo " default:" + echo " assert (((insn >> 19) & 0x2f) == 0x0f /* swap, swapa */);" + echo "" + echo " /* finish the load part of the swap: */" + echo " assert (ls->tme_sparc_ls_state == sizeof(tme_uint32_t));" + echo " value32 = ic->tme_sparc_memory_buffer.tme_sparc_memory_buffer32s[0];" + echo " value_swap32 = *ls->tme_sparc_ls_rd${arch};" + echo " if (endian_little) {" + echo " value32 = tme_letoh_u32(value32);" + echo " value_swap32 = tme_htole_u32(value32);" + echo " }" + echo " else {" + echo " value32 = tme_betoh_u32(value32);" + echo " value_swap32 = tme_htobe_u32(value32);" + echo " }" + echo " *ls->tme_sparc_ls_rd${arch} = value32;" + echo "" + echo " /* start the store part of the swap: */" + echo " ic->tme_sparc_memory_buffer.tme_sparc_memory_buffer32s[0] = value_swap32;" + echo " break;" + echo " }" + echo "" + echo " /* start the store part of the operation: */" + echo " size = ls->tme_sparc_ls_state;" + echo " ls->tme_sparc_ls_address${arch} -= size;" + echo " ls->tme_sparc_ls_size = size;" + echo " ls->tme_sparc_ls_buffer_offset = 0;" + echo " ls->tme_sparc_ls_state = size | TME_BIT(7);" + echo " }" + echo "" + echo " /* this is the store part of the operation: */" + echo "" + echo " /* do one slow store cycle: */" + echo " tme_sparc${arch}_store(ic, ls);" + echo "" + echo " /* if the slow store cycle did not store all of the data: */" + echo " if (__tme_predict_false(ls->tme_sparc_ls_size != 0)) {" + echo " return;" + echo " }" + echo "}" + fi + # the slow load and store functions: # for slow in load store; do @@ -1054,290 +1789,765 @@ for arch in 32; do if test ${slow} = load; then cycle="read" capcycle="READ" - if $header; then - echo "const tme_shared tme_uint8_t *tme_sparc${arch}_${slow} _TME_P((struct tme_sparc *, tme_uint${arch}_t, unsigned int));" - continue - fi else cycle="write" capcycle="WRITE" - if $header; then - echo "tme_shared tme_uint8_t *tme_sparc${arch}_${slow} _TME_P((struct tme_sparc *, tme_uint${arch}_t, const tme_uint${arch}_t *, unsigned int));" - continue - fi + fi + if $header; then + echo "void tme_sparc${arch}_${slow} _TME_P((struct tme_sparc *, struct tme_sparc_ls *));" + continue fi echo "" - echo "/* this does a slow ${slow}: */" - if test ${slow} = load; then echo -n "const "; fi - echo "tme_shared tme_uint8_t *" + echo "/* this does one slow ${slow} cycle: */" + echo "void" echo "tme_sparc${arch}_${slow}(struct tme_sparc *ic, " - echo " tme_uint${arch}_t address, " - if test ${slow} = store; then - echo " const tme_uint${arch}_t * const _rd," - fi - echo " const unsigned int flags)" + echo " struct tme_sparc_ls *ls)" echo "{" # our locals: # - echo " unsigned int size;" - echo " tme_uint32_t asi_mask_data;" - echo " struct tme_sparc_tlb *dtlb;" + echo " struct tme_sparc_tlb *tlb;" + echo " tme_uint${arch}_t address;" + echo " unsigned int cycle_size;" echo " tme_bus_addr_t physical_address;" echo " int shift;" - echo " struct tme_bus_cycle cycle;" - echo " unsigned int transferred, resid, cycle_size;" - echo " unsigned int trap;" echo " int err;" + + echo "" + echo " /* get the TLB entry: */" + echo " tlb = ls->tme_sparc_ls_tlb;" + echo "" + echo " /* the TLB entry must be busy and valid: */" + echo " assert (tme_bus_tlb_is_valid(&tlb->tme_sparc_tlb_bus_tlb));" + echo "" + echo " /* start the bus cycle structure: */" + echo " ls->tme_sparc_ls_bus_cycle.tme_bus_cycle_type = TME_BUS_CYCLE_${capcycle};" + echo "" + echo " /* get the buffer: */" + echo " ls->tme_sparc_ls_bus_cycle.tme_bus_cycle_buffer = &ic->tme_sparc_memory_buffer.tme_sparc_memory_buffer8s[ls->tme_sparc_ls_buffer_offset];" + echo " ls->tme_sparc_ls_bus_cycle.tme_bus_cycle_buffer_increment = 1;" + echo "" + echo " /* get the current address: */" + echo " address = ls->tme_sparc_ls_address${arch};" + echo " ls->tme_sparc_ls_bus_cycle.tme_bus_cycle_address = address;" + echo "" + echo " /* start the cycle size: */" + echo " cycle_size = ls->tme_sparc_ls_size;" + echo " assert (cycle_size > 0);" + echo " cycle_size--;" + echo " cycle_size = TME_MIN(cycle_size, (((tme_bus_addr${arch}_t) tlb->tme_sparc_tlb_addr_last) - address)) + 1;" + echo "" + echo " /* if this TLB entry allows fast ${cycle}s: */" + echo " if (__tme_predict_true(tlb->tme_sparc_tlb_emulator_off_${cycle} != TME_EMULATOR_OFF_UNDEF)) {" + echo "" + echo " /* do a ${cycle}: */" + echo " ls->tme_sparc_ls_bus_cycle.tme_bus_cycle_size = cycle_size;" + echo " tme_memory_bus_${cycle}_buffer((tlb->tme_sparc_tlb_emulator_off_${cycle} + (tme_uint${arch}_t) ls->tme_sparc_ls_bus_cycle.tme_bus_cycle_address)," + echo " ls->tme_sparc_ls_bus_cycle.tme_bus_cycle_buffer," + echo " ls->tme_sparc_ls_bus_cycle.tme_bus_cycle_size," + echo " tlb->tme_sparc_tlb_bus_rwlock," + echo " sizeof(tme_uint8_t)," + echo " sizeof(tme_uint${arch}_t));" + echo " }" + echo "" + echo " /* otherwise, this TLB entry does not allow fast ${cycle}s: */" + echo " else {" + echo "" + echo " /* finish the cycle size: */" + echo " cycle_size = TME_MIN(cycle_size, 1 + ((~ (unsigned int) address) % sizeof(tme_uint${arch}_t)));" + echo " ls->tme_sparc_ls_bus_cycle.tme_bus_cycle_size = cycle_size;" + echo "" + echo " /* form the physical address for the bus cycle handler: */" + echo " physical_address = ls->tme_sparc_ls_bus_cycle.tme_bus_cycle_address;" + echo " physical_address += tlb->tme_sparc_tlb_addr_offset;" + echo " shift = tlb->tme_sparc_tlb_addr_shift;" + echo " if (shift < 0) {" + echo " physical_address <<= (0 - shift);" + echo " }" + echo " else if (shift > 0) {" + echo " physical_address >>= shift;" + echo " }" + echo " ls->tme_sparc_ls_bus_cycle.tme_bus_cycle_address = physical_address;" + echo "" + echo " /* finish the bus cycle structure: */" + echo " (*ic->_tme_sparc_ls_bus_cycle)(ic, ls);" + + echo " tme_sparc_log(ic, 10000, TME_OK," + echo " (TME_SPARC_LOG_HANDLE(ic)," + echo " _(\"cycle-${slow}%u\t0x%0"`expr ${arch} / 4`"\" TME_PRIx${arch})," + echo " (unsigned int) (ls->tme_sparc_ls_bus_cycle.tme_bus_cycle_size * 8)," + echo " (tme_bus_addr${arch}_t) ls->tme_sparc_ls_bus_cycle.tme_bus_cycle_address));" + + echo "" + echo " /* callout the bus cycle: */" + echo " tme_sparc_tlb_unbusy(tlb);" + echo " tme_sparc_callout_unlock(ic);" + echo " err = (*tlb->tme_sparc_tlb_bus_tlb.tme_bus_tlb_cycle)" + echo " (tlb->tme_sparc_tlb_bus_tlb.tme_bus_tlb_cycle_private," + echo " &ls->tme_sparc_ls_bus_cycle);" + echo " tme_sparc_callout_relock(ic);" + echo " tme_sparc_tlb_busy(tlb);" + echo "" + echo " /* the TLB entry can't have been invalidated before the ${slow}: */" + echo " assert (err != EBADF);" + echo "" + echo " /* if the bus cycle didn't complete normally: */" + echo " if (err != TME_OK) {" + echo "" + echo " /* if a real bus fault may have happened, instead of" + echo " some synchronous event: */" + echo " if (err != TME_BUS_CYCLE_SYNCHRONOUS_EVENT) {" + echo "" + echo " /* call the bus fault handlers: */" + echo " err = tme_bus_tlb_fault(&tlb->tme_sparc_tlb_bus_tlb, &ls->tme_sparc_ls_bus_cycle, err);" + echo " }" + echo "" + echo " /* if some synchronous event has happened: */" + echo " if (err == TME_BUS_CYCLE_SYNCHRONOUS_EVENT) {" + echo "" + echo " /* after the currently executing instruction finishes, check" + echo " for external resets, halts, or interrupts: */" + echo " ic->_tme_sparc_instruction_burst_remaining = 0;" + echo " ic->_tme_sparc_instruction_burst_other = TRUE;" + echo " }" + echo "" + echo " /* otherwise, if no real bus fault happened: */" + echo " else if (err == TME_OK) {" + echo "" + echo " /* nothing to do */" + echo " }" + echo "" + echo " /* otherwise, a real bus fault happened: */" + echo " else {" + echo " (*ic->_tme_sparc_ls_bus_fault)(ic, ls, err);" + echo " return;" + echo " }" + echo " }" + echo " }" + + echo "" + echo " /* some data must have been transferred: */" + echo " assert (ls->tme_sparc_ls_bus_cycle.tme_bus_cycle_size > 0);" + if test ${slow} = store; then - echo " tme_uint${arch}_t stored_registers[2];" - echo " const tme_uint8_t *buffer;" - buffer=buffer - else - buffer="&ic->tme_sparc_memory_buffer[0]" + echo "" + echo " /* if this was an atomic operation: */" + echo " if (__tme_predict_false(ls->tme_sparc_ls_lsinfo & TME_SPARC_LSINFO_OP_ATOMIC)) {" + echo "" + echo " /* we do not support atomic operations in TLB entries that" + echo " do not support both fast reads and fast writes. assuming" + echo " that all atomic operations are to regular memory, we" + echo " should always get fast read and fast write TLBs. when" + echo " we do not, it should only be because the memory has been" + echo " made read-only in the MMU. the write above was supposed" + echo " to cause a fault (with the instruction rerun later with" + echo " a fast read and fast write TLB entry), but instead it" + echo " succeeded and transferred some data. we have modified" + echo " memory and cannot recover: */" + echo " abort();" + echo " }" fi echo "" - echo " /* get the transfer size. we can do at most a 64-bit transfer: */" - echo " size = (flags % (sizeof(tme_uint64_t) * 2));" + echo " /* update: */" + echo " cycle_size = ls->tme_sparc_ls_bus_cycle.tme_bus_cycle_size;" + echo " ls->tme_sparc_ls_address${arch} += cycle_size;" + echo " ls->tme_sparc_ls_buffer_offset += cycle_size;" + echo " ls->tme_sparc_ls_size -= cycle_size;" + echo "}" + done - echo "" - echo " /* get the DTLB entry. our caller has already busied it: */" - echo " dtlb = TME_SPARC_DTLB_ENTRY(ic, address);" + # the load/store function: + # + if $header; then + echo "tme_shared tme_uint8_t *tme_sparc${arch}_ls _TME_P((struct tme_sparc *, tme_uint${arch}_t, tme_uint${arch}_t *, tme_uint32_t));" + else echo "" + echo "/* this does a slow load or store: */" + echo "tme_shared tme_uint8_t *" + echo "tme_sparc${arch}_ls(struct tme_sparc *ic," + echo " tme_uint${arch}_t const address_first," + echo " tme_uint${arch}_t *_rd," + echo " tme_uint32_t lsinfo)" + echo "{" + echo " struct tme_sparc_ls ls;" + echo " tme_uint32_t size;" + echo " tme_uint32_t asi;" + echo " tme_uint32_t asi_mask_flags;" + echo " tme_uint32_t asi_mask;" + echo " tme_bus_context_t context;" + echo " tme_uint32_t tlb_hash;" + echo " unsigned long tlb_i;" + echo " unsigned long handler_i;" + echo " struct tme_sparc_tlb *tlb;" + echo " unsigned int cycle_type;" + echo " tme_uint${arch}_t address;" + echo " void (*address_map) _TME_P((struct tme_sparc *, struct tme_sparc_ls *));" + echo " tme_bus_addr_t address_bus;" + echo " int rc;" + echo " const tme_shared tme_uint8_t *emulator_off;" + echo " unsigned int buffer_offset;" + echo " tme_uint${arch}_t value;" + echo " tme_uint32_t value32;" + echo "" + echo " /* we must not be replaying instructions: */" + echo " assert (tme_sparc_recode_verify_replay_last_pc(ic) == 0);" + echo "" + echo " /* initialize the pointer to the rd register: */" + echo " ls.tme_sparc_ls_rd${arch} = _rd;" + echo "" + echo "#ifndef NDEBUG" + echo "" + echo " /* initialize the cycle function: */" + echo " ls.tme_sparc_ls_cycle = NULL;" + echo "" + echo " /* initialize the TLB entry pointer: */" + echo " ls.tme_sparc_ls_tlb = NULL;" + echo "" + echo "#endif /* NDEBUG */" + echo "" + echo " /* initialize the faults: */" + echo " ls.tme_sparc_ls_faults = TME_SPARC_LS_FAULT_NONE;" + echo "" + echo " /* initialize the address: */" + echo " ls.tme_sparc_ls_address${arch} = address_first;" + echo "" + echo " /* initialize the size: */" + echo " size = TME_SPARC_LSINFO_WHICH_SIZE(lsinfo);" + echo " ls.tme_sparc_ls_size = size;" + echo "" + echo " /* initialize the info: */" + echo " ls.tme_sparc_ls_lsinfo = lsinfo;" + echo "" echo " /* if the address is not aligned: */" - echo " if (__tme_predict_false((address & (size - 1)) != 0)) {" - echo " tme_sparc_tlb_unbusy(dtlb);" - echo " tme_sparc${arch}_trap(ic, TME_SPARC_TRAP_mem_address_not_aligned);" + echo " if (__tme_predict_false(((size - 1) & (tme_uint32_t) address_first) != 0)) {" + echo " ls.tme_sparc_ls_faults |= TME_SPARC_LS_FAULT_ADDRESS_NOT_ALIGNED;" echo " }" - echo "" - echo " /* if this is a ldd or a std (and not an lddf or an stdf)" - echo " and the destination register address is odd: */" - echo " /* NB: we detect lddf and stdf by testing bit 24 in the" - echo " instruction word: */" - echo " if (__tme_predict_false(size == (sizeof(tme_uint32_t) * 2)" - echo " && (TME_SPARC_INSN & TME_BIT(25)) != 0" - echo " && (TME_SPARC_INSN & TME_BIT(24)) == 0)) {" - echo " tme_sparc_tlb_unbusy(dtlb);" - echo " tme_sparc${arch}_trap(ic, TME_SPARC_TRAP_illegal_instruction);" - echo " }" - + echo " /* otherwise, the address is aligned: */" + echo " else {" echo "" - echo " /* get the ASI and its mask: */" - echo " asi_mask_data = ic->tme_sparc_asi_mask_data;" - echo " if (__tme_predict_false((flags & TME_SPARC_SLOW_FLAG_INSN) != 0)) {" - echo " asi_mask_data = ic->tme_sparc_asi_mask_insn;" + echo " /* the transfer must not cross a 32-bit boundary: */" + echo " assert ((size - 1) <= (tme_uint32_t) ~address_first);" echo " }" - echo " if (__tme_predict_false((flags & TME_SPARC_SLOW_FLAG_A) != 0)) {" - echo " asi_mask_data = _tme_sparc${arch}_alternate_asi_mask(ic);" + echo "" + echo " /* initialize the address map: */" + echo " ls.tme_sparc_ls_address_map = ic->_tme_sparc_ls_address_map;" + echo "" + echo " /* if this is a ldd, ldda, std, or stda, or an instruction" + echo " that loads or stores in the same way: */" + echo " if (lsinfo & TME_SPARC_LSINFO_LDD_STD) {" + echo "" + echo " /* if the rd register is odd: */" + echo " /* NB: we don't check the rd field in the instruction," + echo " because the register number there might be encoded" + echo " in some way, or the architecture might ignore bit" + echo " zero in the rd field (for example, the sparc32 lddf)." + echo " instead, we test the rd register pointer: */" + echo " if (__tme_predict_false((ls.tme_sparc_ls_rd${arch}" + echo " - ic->tme_sparc_ic.tme_ic_iregs.tme_ic_iregs_uint${arch}s)" + echo " % 2)) {" + echo " ls.tme_sparc_ls_faults |= TME_SPARC_LS_FAULT_LDD_STD_RD_ODD;" + echo " }" echo " }" - - if test ${slow} = store; then + + echo "" + echo " /* if the ASI has been specified: */" + echo " if (lsinfo & TME_SPARC_LSINFO_A) {" + echo "" + echo " /* get the ASI: */" + echo " asi = TME_SPARC_LSINFO_WHICH_ASI(lsinfo);" + echo "" + echo " /* get the flags for this ASI: */" + echo " asi_mask_flags = ic->tme_sparc_asis[asi].tme_sparc_asi_mask_flags;" + echo "" + if test ${arch} = 32; then + echo " /* make the ASI mask: */" + echo " if (asi_mask_flags & TME_SPARC32_ASI_MASK_FLAG_SPECIAL) {" + echo " asi_mask" + echo " = TME_SPARC_ASI_MASK_SPECIAL(asi, TRUE);" + echo " }" + echo " else {" + echo " asi_mask = TME_SPARC32_ASI_MASK(asi, asi);" + echo " }" + elif test ${arch} = 64; then + echo " /* if this is a nonprivileged access: */" + echo " if (!TME_SPARC_PRIV(ic)) {" + echo "" + echo " /* if this is a restricted ASI: */" + echo " if (__tme_predict_false((asi & TME_SPARC64_ASI_FLAG_UNRESTRICTED) == 0)) {" + echo " ls.tme_sparc_ls_faults |= TME_SPARC64_LS_FAULT_PRIVILEGED_ASI;" + echo " }" echo "" - echo " /* store big-endian versions of the registers to store in" - echo " a buffer. we always assume that this is a std and copy" - echo " r[rd] and r[rd + 1] into the buffer: */" - echo " stored_registers[0] = tme_htobe_u${arch}(TME_SPARC_FORMAT3_RD);" - echo " stored_registers[1] = tme_htobe_u${arch}(TME_SPARC_FORMAT3_RD_ODD);" - echo "" - echo " /* make the initial pointer into the buffer: */" - echo " buffer = (const tme_uint8_t *) stored_registers;" - echo " if (size < sizeof(tme_uint32_t)) {" - echo " buffer += sizeof(tme_uint32_t) - size;" - echo " }" + echo " /* force a nonprivileged access with the ASI: */" + echo " asi_mask_flags |= TME_SPARC64_ASI_MASK_FLAG_INSN_AS_IF_USER;" + echo " }" + echo "" + echo " /* make the ASI mask: */" + echo " if (asi_mask_flags & TME_SPARC64_ASI_MASK_FLAG_SPECIAL) {" + echo " asi_mask" + echo " = (asi_mask_flags" + echo " + TME_SPARC_ASI_MASK_SPECIAL(asi," + echo " (asi_mask_flags & TME_SPARC64_ASI_MASK_FLAG_INSN_AS_IF_USER) == 0));" + echo " }" + echo " else {" + echo " asi_mask = TME_SPARC64_ASI_MASK(asi, asi_mask_flags);" + echo " }" + fi + echo " ls.tme_sparc_ls_asi_mask = asi_mask;" + if test ${arch} = 64; then + echo "" + echo " /* if this is a no-fault ASI with a non-load instruction: */" + echo " if (asi_mask & TME_SPARC64_ASI_FLAG_NO_FAULT) {" + echo " if (__tme_predict_false(lsinfo & (TME_SPARC_LSINFO_OP_ST | TME_SPARC_LSINFO_OP_ATOMIC))) {" + echo " ls.tme_sparc_ls_faults |= TME_SPARC64_LS_FAULT_NO_FAULT_NON_LOAD;" + echo " }" + echo " }" fi echo "" - echo " /* loop until the ${slow} is completed: */" - echo " trap = TME_SPARC_TRAP_none;" - echo " transferred = 0;" - echo " do {" - echo "" - echo " /* while the DTLB entry is invalid, or does not cover this address" - echo " and address space, or if it does not allow for slow or fast ${cycle}s: */" - echo " for (; (tme_bus_tlb_is_invalid(&dtlb->tme_sparc_tlb_bus_tlb)" - echo " || (dtlb->tme_sparc_tlb_addr_first > address)" - echo " || (dtlb->tme_sparc_tlb_addr_last < address)" - echo " || (!TME_SPARC_TLB_ASI_MASK_OK(dtlb, asi_mask_data))" - echo " || (dtlb->tme_sparc_tlb_emulator_off_${cycle} == TME_EMULATOR_OFF_UNDEF" - echo " && (dtlb->tme_sparc_tlb_cycles_ok & TME_BUS_CYCLE_${capcycle}) == 0)); ) {" - echo "" - echo " /* unbusy this DTLB entry for filling: */" - echo " tme_bus_tlb_unbusy_fill(&dtlb->tme_sparc_tlb_bus_tlb);" - echo "" - echo " /* we never fill TLB entries on the stack because we never" - echo " callout multiple fills at the same time, so the global TLB" - echo " entry pointer always points back to the TLB entry. this" - echo " also means that we do not have to call tme_bus_tlb_back()" - echo " after the fill: */" - echo " dtlb->tme_sparc_tlb_bus_tlb.tme_bus_tlb_global = &dtlb->tme_sparc_tlb_bus_tlb;" + echo " /* get the context for the alternate address space: */" + if test ${arch} = 32; then + echo " context = ic->tme_sparc_memory_context_default;" + elif test ${arch} = 64; then + echo " context = ic->tme_sparc_memory_context_primary;" + echo " if (asi_mask & TME_SPARC64_ASI_FLAG_SECONDARY) {" + echo " context = ic->tme_sparc_memory_context_secondary;" + echo " }" + echo " if (__tme_predict_false(asi_mask & TME_SPARC64_ASI_MASK_FLAG_INSN_NUCLEUS)) {" + echo " if (TME_SPARC_MEMORY_FLAGS(ic) & TME_SPARC_MEMORY_FLAG_HAS_NUCLEUS) {" + echo " context = 0;" + echo " }" + echo " }" + fi + echo " ls.tme_sparc_ls_context = context;" echo "" - echo " /* reload the DTLB entry: */" - echo " tme_sparc_callout_unlock(ic);" - echo "#ifdef _TME_SPARC_STATS" - echo " ic->tme_sparc_stats.tme_sparc_stats_dtlb_fill++;" - echo "#endif /* _TME_SPARC_STATS */" - echo " err = (*ic->_tme_sparc_bus_connection->tme_sparc_bus_tlb_fill)" - echo " (ic->_tme_sparc_bus_connection," - echo " dtlb," - echo " asi_mask_data," - echo " address," - echo " TME_BUS_CYCLE_${capcycle});" - echo " assert (err == TME_OK);" - echo " tme_sparc_callout_relock(ic);" + echo " /* get the default TLB entry index: */" + echo " tlb_hash = TME_SPARC_TLB_HASH(ic, context, address_first);" + echo " if (lsinfo & TME_SPARC_LSINFO_OP_FETCH) {" + echo " tlb_i = TME_SPARC_ITLB_ENTRY(ic, tlb_hash);" + echo " }" + echo " else {" + echo " tlb_i = TME_SPARC_DTLB_ENTRY(ic, tlb_hash);" + echo " }" + echo " ls.tme_sparc_ls_tlb_i = tlb_i;" echo "" - echo " /* rebusy the DTLB entry: */" - echo " tme_sparc_tlb_busy(dtlb);" + echo " /* call any special handler for this ASI: */" + echo " handler_i = ic->tme_sparc_asis[TME_SPARC_ASI_MASK_WHICH(asi_mask)].tme_sparc_asi_handler;" + echo " if (__tme_predict_false(handler_i != 0)) {" + echo " (*ic->_tme_sparc_ls_asi_handlers[handler_i])(ic, &ls);" echo " }" + echo "" + echo " /* get the final TLB entry index: */" + echo " tlb_i = ls.tme_sparc_ls_tlb_i;" + echo " }" echo "" - echo " /* if this DTLB entry allows fast ${cycle}s: */" - echo " if (__tme_predict_true(dtlb->tme_sparc_tlb_emulator_off_${cycle} != TME_EMULATOR_OFF_UNDEF)) {" + echo " /* otherwise, the ASI has not been specified: */" + echo " else {" echo "" - echo " /* if we have not transferred anything yet, and this" - echo " DTLB entry covers all of the addresses and allows" - echo " fast transfers: */" - echo " if (__tme_predict_true(transferred == 0" - echo " && dtlb->tme_sparc_tlb_addr_last >= address + (size - 1)" - if test ${slow} = store; then - echo "" - echo " && ((flags & TME_SPARC_SLOW_FLAG_ATOMIC) == 0" - echo -n " || dtlb->tme_sparc_tlb_emulator_off_read == dtlb->tme_sparc_tlb_emulator_off_${cycle})" + echo " /* get the ASI mask: */" + echo " asi_mask = ic->tme_sparc_asi_mask_data;" + echo "" + echo " /* add in any ASI mask flags from the instruction: */" + if test ${arch} = 64; then + echo " /* NB: initially, TME_SPARC64_ASI_FLAG_NO_FAULT is the" + echo " only flag allowed, and only the flush instruction" + echo " can use it: */" fi - echo ")) {" - + echo " assert (TME_SPARC_LSINFO_WHICH_ASI_FLAGS(lsinfo) == 0" + if test ${arch} = 64; then + echo " || (TME_SPARC_LSINFO_WHICH_ASI_FLAGS(lsinfo) == TME_SPARC64_ASI_FLAG_NO_FAULT" + echo " && ((ic->_tme_sparc_insn >> 19) & 0x3f) == 0x3b)" + fi + echo " );" + echo " asi_mask |= TME_SPARC_LSINFO_WHICH_ASI_FLAGS(lsinfo);" echo "" - echo " /* return and let our caller do the transfer: */" - echo " return (dtlb->tme_sparc_tlb_emulator_off_${cycle});" - echo " }" + echo " /* set the ASI mask: */" + echo " ls.tme_sparc_ls_asi_mask = asi_mask;" + echo "" + echo " /* get the context: */" + echo " context = ic->tme_sparc_memory_context_default;" + echo " ls.tme_sparc_ls_context = context;" + echo "" + echo " /* this must not be a fetch: */" + echo " assert ((lsinfo & TME_SPARC_LSINFO_OP_FETCH) == 0);" + echo "" + echo " /* get the TLB entry index: */" + echo " tlb_hash = TME_SPARC_TLB_HASH(ic, context, address_first);" + echo " tlb_i = TME_SPARC_DTLB_ENTRY(ic, tlb_hash);" + echo " ls.tme_sparc_ls_tlb_i = tlb_i;" + echo " }" + + echo "" + echo " /* get the TLB entry pointer: */" + echo " tlb = &ic->tme_sparc_tlbs[tlb_i];" + echo " ls.tme_sparc_ls_tlb = tlb;" + + echo "" + echo " /* get the cycle type: */" + echo " /* NB: we deliberately set this once, now, since the lsinfo" + echo " may change once we start transferring: */" + echo " cycle_type" + echo " = ((lsinfo" + echo " & (TME_SPARC_LSINFO_OP_ST" + echo " | TME_SPARC_LSINFO_OP_ATOMIC))" + echo " ? TME_BUS_CYCLE_WRITE" + echo " : TME_BUS_CYCLE_READ);" + + echo "" + echo " /* loop until the transfer is complete: */" + echo " for (;;) {" + + echo "" + echo " /* if we have faulted: */" + echo " if (__tme_predict_false(ls.tme_sparc_ls_faults != TME_SPARC_LS_FAULT_NONE)) {" + echo "" + echo " /* unbusy this TLB, since the trap function may not return: */" + echo " tme_bus_tlb_unbusy(&tlb->tme_sparc_tlb_bus_tlb);" + echo "" + echo " /* call the trap function, which will not return if it traps: */" + echo " (*ic->_tme_sparc_ls_trap)(ic, &ls);" + echo "" + echo " /* rebusy this TLB: */" + echo " tme_bus_tlb_busy(&tlb->tme_sparc_tlb_bus_tlb);" echo "" - echo " /* calculate the cycle size: */" - echo " cycle_size = (dtlb->tme_sparc_tlb_addr_last - address) + 1;" - echo " cycle_size = TME_MIN(cycle_size, sizeof(tme_uint${arch}_t) - (address % sizeof(tme_uint${arch}_t)));" - echo " cycle_size = TME_MIN(cycle_size, size - transferred);" - echo " cycle.tme_bus_cycle_size = cycle_size;" - echo "" - echo " /* do a ${cycle}: */" - echo " tme_memory_bus_${cycle}_buffer((dtlb->tme_sparc_tlb_emulator_off_${cycle} + address)," - echo " (${buffer} + transferred)," - echo " cycle_size," - echo " dtlb->tme_sparc_tlb_bus_rwlock," - echo " sizeof(tme_uint8_t)," - echo " sizeof(tme_uint32_t));" + echo " /* since the trap function returned, it must have cleared the fault: */" + echo " assert (ls.tme_sparc_ls_faults == TME_SPARC_LS_FAULT_NONE);" echo " }" + echo "" - echo " /* otherwise, this DTLB entry does not allow fast ${cycle}s: */" - echo " else {" + echo " /* if the transfer is complete, stop now: */" + echo " if (__tme_predict_false(ls.tme_sparc_ls_size == 0)) {" + echo " break;" + echo " }" + + echo "" + echo " /* get the current address: */" + echo " address = ls.tme_sparc_ls_address${arch};" + + echo "" + echo " /* if this TLB entry does not apply or is invalid: */" + echo " if ((tlb->tme_sparc_tlb_context != ls.tme_sparc_ls_context" + echo " && tlb->tme_sparc_tlb_context <= ic->tme_sparc_memory_context_max)" + echo " || address < (tme_bus_addr${arch}_t) tlb->tme_sparc_tlb_addr_first" + echo " || address > (tme_bus_addr${arch}_t) tlb->tme_sparc_tlb_addr_last" + echo " || !TME_SPARC_TLB_ASI_MASK_OK(tlb, ls.tme_sparc_ls_asi_mask)" + echo " || ((tlb->tme_sparc_tlb_cycles_ok & cycle_type) == 0" + echo " && (cycle_type == TME_BUS_CYCLE_READ" + echo " ? tlb->tme_sparc_tlb_emulator_off_read" + echo " : tlb->tme_sparc_tlb_emulator_off_write) == TME_EMULATOR_OFF_UNDEF)" + echo " || tme_bus_tlb_is_invalid(&tlb->tme_sparc_tlb_bus_tlb)) {" + echo "" + echo " /* unbusy this TLB entry for filling: */" + echo " tme_bus_tlb_unbusy_fill(&tlb->tme_sparc_tlb_bus_tlb);" + echo "" + echo " /* if we haven't mapped this address yet: */" + echo " address_map = ls.tme_sparc_ls_address_map;" + echo " if (address_map != NULL) {" + echo " ls.tme_sparc_ls_address_map = NULL;" + echo "" + echo " /* count this mapping: */" + echo " if (ls.tme_sparc_ls_lsinfo & TME_SPARC_LSINFO_OP_FETCH) {" + echo " TME_SPARC_STAT(ic, tme_sparc_stats_itlb_map);" + echo " }" + echo " else {" + echo " TME_SPARC_STAT(ic, tme_sparc_stats_dtlb_map);" + echo " }" + echo "" + echo " /* initialize the ASI mask and context on this TLB entry: */" + echo " /* NB that the ASI mask will likely be updated by either the" + echo " address mapping or the TLB fill: */" + echo " tlb->tme_sparc_tlb_asi_mask" + echo " = (ls.tme_sparc_ls_asi_mask" + echo " & ~TME_SPARC_ASI_MASK_FLAGS_AVAIL);" + echo " tlb->tme_sparc_tlb_context = ls.tme_sparc_ls_context;" + echo "" + echo " /* NB: if the address mapping traps, we won't get a chance" + echo " to finish updating this TLB entry, which is currently in" + echo " an inconsistent state - but not necessarily an unusable" + echo " state. poison it to be unusable, including any recode" + echo " TLB entry: */" + echo " tlb->tme_sparc_tlb_addr_first = 1;" + echo " tlb->tme_sparc_tlb_addr_last = 0;" + echo "#if TME_SPARC_HAVE_RECODE(ic)" + echo " if (ls.tme_sparc_ls_lsinfo & TME_SPARC_LSINFO_OP_FETCH) {" + echo " tme_sparc${arch}_recode_chain_tlb_update(ic, &ls);" + echo " }" + echo " else {" + echo " tme_sparc${arch}_recode_ls_tlb_update(ic, &ls);" + echo " }" + echo "#endif /* TME_SPARC_HAVE_RECODE(ic) */" + echo "" + echo "#ifndef NDEBUG" + echo "" + echo " /* initialize the mapping TLB entry: */" + echo " ls.tme_sparc_ls_tlb_map.tme_bus_tlb_addr_first = 0 - (tme_bus_addr_t) 1;" + echo " ls.tme_sparc_ls_tlb_map.tme_bus_tlb_addr_last = 0 - (tme_bus_addr_t) 2;" + echo " ls.tme_sparc_ls_tlb_map.tme_bus_tlb_cycles_ok = 0;" + echo " ls.tme_sparc_ls_tlb_map.tme_bus_tlb_addr_offset = 0 - (tme_bus_addr_t) 1;" + echo "" + echo "#endif /* !NDEBUG */" echo "" - echo " /* make the bus cycle structure: */" - echo " resid = size - transferred;" - echo " cycle.tme_bus_cycle_type = TME_BUS_CYCLE_${capcycle};" - echo " cycle.tme_bus_cycle_buffer = (tme_uint8_t *) (${buffer} + transferred);" - echo " cycle.tme_bus_cycle_buffer_increment = 1;" - echo " cycle_size = TME_MIN(resid, sizeof(tme_uint${arch}_t) - (address % sizeof(tme_uint${arch}_t)));" - echo " cycle.tme_bus_cycle_size = cycle_size;" - echo " cycle.tme_bus_cycle_port = TME_BUS_CYCLE_PORT(0, TME_BUS${arch}_LOG2);" - echo " cycle.tme_bus_cycle_lane_routing = " - echo " &tme_sparc${arch}_router[TME_SPARC_BUS_ROUTER_INDEX(TME_BUS${arch}_LOG2, cycle_size, address)];" - echo "" - echo " /* form the physical address for the bus cycle handler: */" - echo " physical_address = dtlb->tme_sparc_tlb_addr_offset + address;" - echo " shift = dtlb->tme_sparc_tlb_addr_shift;" - echo " if (shift < 0) {" - echo " physical_address <<= (0 - shift);" + echo " /* map the address: */" + echo " (*address_map)(ic, &ls);" + echo "" + echo " /* the address mapping must do any trapping itself: */" + echo " assert (ls.tme_sparc_ls_faults == TME_SPARC_LS_FAULT_NONE);" + echo "" + echo " /* if the address mapping completed the transfer: */" + echo " if (__tme_predict_false(ls.tme_sparc_ls_size == 0)) {" + echo "" + echo " /* rebusy the TLB entry: */" + echo " tme_sparc_tlb_busy(tlb);" + echo "" + echo " /* stop now: */" + echo " break;" + echo " }" + echo "" + echo " /* the mapping must have actually made a mapping: */" + echo " assert (ls.tme_sparc_ls_tlb_map.tme_bus_tlb_addr_first != 0 - (tme_bus_addr_t) 1);" + echo " assert (ls.tme_sparc_ls_tlb_map.tme_bus_tlb_addr_last != 0 - (tme_bus_addr_t) 2);" + echo " assert (ls.tme_sparc_ls_tlb_map.tme_bus_tlb_cycles_ok != 0);" + echo " assert (ls.tme_sparc_ls_tlb_map.tme_bus_tlb_addr_offset != 0 - (tme_bus_addr_t) 1);" echo " }" - echo " else if (shift > 0) {" - echo " physical_address >>= shift;" + echo "" + echo " /* count this fill: */" + echo " if (ls.tme_sparc_ls_lsinfo & TME_SPARC_LSINFO_OP_FETCH) {" + echo " TME_SPARC_STAT(ic, tme_sparc_stats_itlb_fill);" echo " }" - echo " cycle.tme_bus_cycle_address = physical_address;" - + echo " else {" + echo " TME_SPARC_STAT(ic, tme_sparc_stats_dtlb_fill);" + echo " }" + echo "" + echo " /* get the bus address: */" + echo " address_bus = ls.tme_sparc_ls_address${arch} + ls.tme_sparc_ls_tlb_map.tme_bus_tlb_addr_offset;" echo "" - echo " /* callout the bus cycle: */" - echo " tme_sparc_tlb_unbusy(dtlb);" + echo " /* fill the TLB entry: */" echo " tme_sparc_callout_unlock(ic);" - echo " err = (*dtlb->tme_sparc_tlb_bus_tlb.tme_bus_tlb_cycle)" - echo " (dtlb->tme_sparc_tlb_bus_tlb.tme_bus_tlb_cycle_private, &cycle);" + echo " rc = (*ic->_tme_sparc_bus_connection->tme_sparc_bus_tlb_fill)" + echo " (ic->_tme_sparc_bus_connection," + echo " tlb," + echo " ls.tme_sparc_ls_asi_mask," + echo " address_bus," + echo " cycle_type);" + echo " assert (rc == TME_OK);" echo " tme_sparc_callout_relock(ic);" - echo " tme_sparc_tlb_busy(dtlb);" echo "" - echo " /* if the TLB entry was invalidated before the ${slow}: */" - echo " if (err == EBADF" - echo " && tme_bus_tlb_is_invalid(&dtlb->tme_sparc_tlb_bus_tlb)) {" - echo " cycle.tme_bus_cycle_size = 0;" + echo " /* map the TLB entry: */" + echo " tme_bus_tlb_map(&tlb->tme_sparc_tlb_bus_tlb, address_bus," + echo " &ls.tme_sparc_ls_tlb_map, ls.tme_sparc_ls_address${arch});" + echo "" + echo " /* update any recode TLB entry: */" + echo "#if TME_SPARC_HAVE_RECODE(ic)" + echo " if (ls.tme_sparc_ls_lsinfo & TME_SPARC_LSINFO_OP_FETCH) {" + echo " tme_sparc${arch}_recode_chain_tlb_update(ic, &ls);" + echo " }" + echo " else {" + echo " tme_sparc${arch}_recode_ls_tlb_update(ic, &ls);" + echo " }" + echo "#endif /* TME_SPARC_HAVE_RECODE(ic) */" + echo "" + echo " /* rebusy the TLB entry: */" + echo " tme_sparc_tlb_busy(tlb);" + echo "" + echo " /* if this TLB entry is already invalid: */" + echo " if (tme_bus_tlb_is_invalid(&tlb->tme_sparc_tlb_bus_tlb)) {" + echo " continue;" echo " }" + echo " }" + echo "" - echo " /* otherwise, any other error might be a bus error: */" - echo " else if (err != TME_OK) {" + echo " /* this TLB entry must apply: */" + echo " assert ((tlb->tme_sparc_tlb_context == ls.tme_sparc_ls_context" + echo " || tlb->tme_sparc_tlb_context > ic->tme_sparc_memory_context_max)" + echo " && ls.tme_sparc_ls_address${arch} >= (tme_bus_addr${arch}_t) tlb->tme_sparc_tlb_addr_first" + echo " && ls.tme_sparc_ls_address${arch} <= (tme_bus_addr${arch}_t) tlb->tme_sparc_tlb_addr_last" + echo " && ((tlb->tme_sparc_tlb_cycles_ok & cycle_type)" + echo " || (cycle_type == TME_BUS_CYCLE_READ" + echo " ? tlb->tme_sparc_tlb_emulator_off_read" + echo " : tlb->tme_sparc_tlb_emulator_off_write) != TME_EMULATOR_OFF_UNDEF)" + echo " && TME_SPARC_TLB_ASI_MASK_OK(tlb, ls.tme_sparc_ls_asi_mask));" + + echo "" + echo " /* get the current lsinfo: */" + echo " lsinfo = ls.tme_sparc_ls_lsinfo;" + + echo "" + echo " /* if we have to check the TLB: */" + echo " if (__tme_predict_true((lsinfo & TME_SPARC_LSINFO_NO_CHECK_TLB) == 0)) {" + echo "" + echo " /* get the ASI mask for this TLB entry: */" + echo " asi_mask = tlb->tme_sparc_tlb_asi_mask;" + if test ${arch} = 64; then + echo "" + echo " /* if this TLB entry is for no-fault accesses only: */" + echo " if (__tme_predict_false(asi_mask & TME_SPARC64_ASI_FLAG_NO_FAULT)) {" + echo "" + echo " /* if this access is not using a no-fault ASI: */" + echo " if (__tme_predict_false((ls.tme_sparc_ls_asi_mask & TME_SPARC64_ASI_FLAG_NO_FAULT) == 0)) {" + echo " ls.tme_sparc_ls_faults |= TME_SPARC64_LS_FAULT_NO_FAULT_FAULT;" + echo " continue;" + echo " }" + echo " }" + echo "" + echo " /* if this TLB entry is for addresses with side effects: */" + echo " if (asi_mask & TME_SPARC64_ASI_MASK_FLAG_TLB_SIDE_EFFECTS) {" + echo "" + echo " /* if this access is using a no-fault ASI: */" + echo " /* NB: a flush may be implemented as a load with a no-fault ASI: */" + echo " if (__tme_predict_false(ls.tme_sparc_ls_asi_mask & TME_SPARC64_ASI_FLAG_NO_FAULT)) {" + echo " ls.tme_sparc_ls_faults |= TME_SPARC64_LS_FAULT_SIDE_EFFECTS;" + echo " continue;" + echo " }" + echo " }" + echo "" + echo " /* if this TLB entry is for uncacheable addresses: */" + echo " if (asi_mask & TME_SPARC64_ASI_MASK_FLAG_TLB_UNCACHEABLE) {" + echo "" + echo " /* if this is an atomic access: */" + echo " if (__tme_predict_false(lsinfo & TME_SPARC_LSINFO_OP_ATOMIC)) {" + echo " ls.tme_sparc_ls_faults |= TME_SPARC64_LS_FAULT_UNCACHEABLE;" + echo " continue;" + echo " }" + echo " }" + echo "" + echo " /* see if this is a little-endian instruction: */" + echo " lsinfo" + echo " = ((lsinfo" + echo " & ~TME_SPARC_LSINFO_ENDIAN_LITTLE)" + echo " + ((ls.tme_sparc_ls_asi_mask" + echo " & TME_SPARC64_ASI_FLAG_LITTLE)" + echo "#if TME_SPARC_LSINFO_ENDIAN_LITTLE < TME_SPARC64_ASI_FLAG_LITTLE" + echo "#error \"TME_SPARC_LSINFO_ENDIAN_ values changed\"" + echo "#endif" + echo " * (TME_SPARC_LSINFO_ENDIAN_LITTLE" + echo " / TME_SPARC64_ASI_FLAG_LITTLE)));" + echo "" + echo " /* if this TLB entry has its little-endian bit set: */" + echo " if (__tme_predict_false(asi_mask & TME_SPARC64_ASI_FLAG_LITTLE)) {" + echo " assert (TME_SPARC_MEMORY_FLAGS(ic) & TME_SPARC_MEMORY_FLAG_HAS_INVERT_ENDIAN);" + echo " if (TRUE) {" + echo " lsinfo ^= TME_SPARC_LSINFO_ENDIAN_LITTLE;" + echo " }" + echo " }" + fi + echo " }" + echo "" - echo " /* if a real bus error may have happened, instead of" - echo " some synchronous event: */" - echo " if (err != TME_BUS_CYCLE_SYNCHRONOUS_EVENT) {" + echo " /* if we might not have to call a slow cycle function: */" + echo " if (__tme_predict_true((lsinfo & TME_SPARC_LSINFO_SLOW_CYCLES) == 0)) {" echo "" - echo " /* call the bus fault handlers: */" - echo " err = tme_bus_tlb_fault(&dtlb->tme_sparc_tlb_bus_tlb, &cycle, err);" + echo " /* if this TLB entry allows fast transfer of all of the addresses: */" + echo " if (__tme_predict_true(((tme_bus_addr${arch}_t) tlb->tme_sparc_tlb_addr_last) >= (address_first + (ls.tme_sparc_ls_size - 1)))) {" + echo " emulator_off = tlb->tme_sparc_tlb_emulator_off_read;" + echo " if (lsinfo & TME_SPARC_LSINFO_OP_ST) {" + echo " emulator_off = tlb->tme_sparc_tlb_emulator_off_write;" + echo " }" + echo " if (__tme_predict_true(emulator_off != TME_EMULATOR_OFF_UNDEF" + echo " && (((lsinfo & TME_SPARC_LSINFO_OP_ATOMIC) == 0)" + echo " || emulator_off == tlb->tme_sparc_tlb_emulator_off_write))) {" + echo "" + echo " /* return and let our caller do the transfer: */" + echo " /* NB: we break const here: */" + echo " return ((tme_shared tme_uint8_t *) emulator_off);" echo " }" + echo " }" + echo "" + echo " /* we have to call a slow cycle function: */" + echo " lsinfo |= TME_SPARC_LSINFO_SLOW_CYCLES;" + echo " assert (ls.tme_sparc_ls_cycle == NULL);" + if test ${arch} = 32; then + endian_little="FALSE" + else + endian_little="(lsinfo & TME_SPARC_LSINFO_ENDIAN_LITTLE)" + fi + echo "" + echo " /* assume that this operation will transfer the start of the buffer: */" + echo " buffer_offset = 0;" echo "" - echo " /* if some synchronous event has happened: */" - echo " if (err == TME_BUS_CYCLE_SYNCHRONOUS_EVENT) {" + echo " /* assume that this is a load or a fetch: */" + echo " ls.tme_sparc_ls_cycle = tme_sparc${arch}_load;" echo "" - echo " /* after the currently executing instruction finishes, check" - echo " for external resets, halts, or interrupts: */" - echo " ic->_tme_sparc_instruction_burst_remaining = 0;" + echo " /* if this is a store: */" + echo " if (lsinfo & TME_SPARC_LSINFO_OP_ST) {" + echo "" + echo " /* put the (first) register to store in the memory buffer: */" + echo " value = TME_SPARC_FORMAT3_RD;" + echo " value = (${endian_little} ? tme_htole_u${arch}(value) : tme_htobe_u${arch}(value));" + echo " ic->tme_sparc_memory_buffer.tme_sparc_memory_buffer${arch}s[0] = value;" + echo "" + echo " /* find the offset in the memory buffer corresponding to the" + echo " first address: */" + echo " buffer_offset = sizeof(tme_uint${arch}_t) - ls.tme_sparc_ls_size;" + echo " if (${endian_little}) {" + echo " buffer_offset = 0;" echo " }" + echo "" - echo " /* otherwise, if a real bus fault happened: */" - echo " else if (err != TME_OK) {" - echo " trap = (*ic->_tme_sparc_bus_fault)(ic, &cycle, flags, err);" - echo " if (trap != TME_SPARC_TRAP_none) {" - echo " break;" + echo " /* if this is a std or stda: */" + echo " if (lsinfo & TME_SPARC_LSINFO_LDD_STD) {" + echo "" + echo " /* put the odd 32-bit register to store in the memory buffer" + echo " after the even 32-bit register. exactly where this is depends" + echo " on the architecture and on the byte order of the store: */" + echo " value32 = TME_SPARC_FORMAT3_RD_ODD(tme_ic_ireg_uint${arch});" + echo " if (${endian_little}) {" + echo " value32 = tme_htole_u32(value32);" + echo " ic->tme_sparc_memory_buffer.tme_sparc_memory_buffer32s[1] = value32;" + echo " buffer_offset = 0;" + echo " }" + echo " else {" + echo " value32 = tme_htobe_u32(value32);" + echo " ic->tme_sparc_memory_buffer.tme_sparc_memory_buffer32s[(${arch} / 32)] = value32;" + echo " buffer_offset = sizeof(tme_uint${arch}_t) - sizeof(tme_uint32_t);" echo " }" echo " }" + echo "" + echo " /* set the cycle function: */" + echo " ls.tme_sparc_ls_cycle = tme_sparc${arch}_store;" echo " }" - - if test ${slow} = store; then - echo "" - echo " /* if this was an atomic operation, and data was transferred: */" - echo " if (__tme_predict_false((flags & TME_SPARC_SLOW_FLAG_ATOMIC) != 0" - echo " && cycle.tme_bus_cycle_size > 0)) {" - echo "" - echo " /* we do not support atomic operations in TLB entries that" - echo " do not support both fast reads and fast writes. assuming" - echo " that all atomic operations are to regular memory, we" - echo " should always get fast read and fast write TLBs. when" - echo " we do not, it should only be because the memory has been" - echo " made read-only in the MMU. the write above was supposed" - echo " to cause a fault (with the instruction rerun later with" - echo " a fast read and fast write TLB entry), but instead it" - echo " succeeded and transferred some data. we have modified" - echo " memory and cannot recover: */" - echo " abort();" - echo " }" - fi + echo "" + echo " /* otherwise, if this is an atomic: */" + echo " else if (lsinfo & TME_SPARC_LSINFO_OP_ATOMIC) {" + echo "" + echo " /* set the cycle function: */" + echo " ls.tme_sparc_ls_cycle = tme_sparc${arch}_atomic;" + echo " }" + echo "" + echo " /* set the buffer offset for the (first) slow cycle: */" + echo " ls.tme_sparc_ls_buffer_offset = buffer_offset;" + echo "" + echo " /* clear the state for this operation: */" + echo " ls.tme_sparc_ls_state = 0;" echo " }" echo "" - echo " /* update: */" - echo " address += cycle.tme_bus_cycle_size;" - echo " transferred += cycle.tme_bus_cycle_size;" - echo " } while (transferred < size);" - - echo "" - echo " /* if we faulted, start trap processing: */" - echo " if (__tme_predict_false(trap != TME_SPARC_TRAP_none)) {" - echo " tme_sparc_tlb_unbusy(dtlb);" - echo " tme_sparc${arch}_trap(ic, trap);" - echo " }" + echo " /* assume that we won't have to check the TLB again: */" + echo " ls.tme_sparc_ls_lsinfo = lsinfo | TME_SPARC_LSINFO_NO_CHECK_TLB;" + echo " /* call the slow cycle function: */" + echo " (*ls.tme_sparc_ls_cycle)(ic, &ls);" + echo " }" echo "" - if test ${slow} = load; then - echo " return (ic->tme_sparc_memory_buffer - (address - size));" - else - echo " return ((tme_uint8_t *) TME_EMULATOR_OFF_UNDEF);" - fi + echo " /* if this was a load that has already completed, a store," + echo " or an atomic, make sure our caller doesn't try to complete" + echo " a fast transfer: */" + echo " if (ls.tme_sparc_ls_lsinfo" + echo " & (TME_SPARC_LSINFO_LD_COMPLETED" + echo " | TME_SPARC_LSINFO_OP_ST" + echo " | TME_SPARC_LSINFO_OP_ATOMIC)) {" + echo " return (TME_EMULATOR_OFF_UNDEF);" + echo " }" + echo "" + echo " /* otherwise, this was a load that did slow cycles into the" + echo " memory buffer and hasn't updated rd yet. return a pointer" + echo " to the memory buffer so our caller can complete the load: */" + echo " return (ic->tme_sparc_memory_buffer.tme_sparc_memory_buffer8s" + echo " - address_first);" echo "}" - done + fi # unfix the architecture version: #