Source to osfmk/ppc/hw_vm.s
/*
* Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
*
* @APPLE_LICENSE_HEADER_START@
*
* The contents of this file constitute Original Code as defined in and
* are subject to the Apple Public Source License Version 1.1 (the
* "License"). You may not use this file except in compliance with the
* License. Please obtain a copy of the License at
* http://www.apple.com/publicsource and read it before using this file.
*
* This Original Code and all software distributed under the License are
* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
* License for the specific language governing rights and limitations
* under the License.
*
* @APPLE_LICENSE_HEADER_END@
*/
#include <assym.s>
#include <debug.h>
#include <cpus.h>
#include <db_machine_commands.h>
#include <mach_rt.h>
#include <mach_debug.h>
#include <ppc/asm.h>
#include <ppc/proc_reg.h>
#include <ppc/exception.h>
#include <ppc/Performance.h>
#include <ppc/exception.h>
#include <ppc/pmap_internals.h>
#include <mach/ppc/vm_param.h>
#define PERFTIMES 0
.text
/*
*
* Random notes and musings...
*
* Access to mappings via the PTEG hash must be done with the list locked.
* Access via the physical entries is controlled by the physent lock.
* Access to mappings is controlled by the PTEG lock once they are queued.
* If they are not on the list, they don't really exist, so
* only one processor at a time can find them, so no access control is needed.
*
* The second half of the PTE is kept in the physical entry. It is done this
* way, because there may be multiple mappings that refer to the same physical
* page (i.e., address aliases or synonymns). We must do it this way, because
* maintenance of the reference and change bits becomes nightmarish if each mapping
* has its own. One side effect of this, and not necessarily a bad one, is that
* all mappings for a single page can have a single WIMG, protection state, and RC bits.
* The only "bad" thing, is the reference bit. With a single copy, we can not get
* a completely accurate working set calculation, i.e., we can't tell which mapping was
* used to reference the page, all we can tell is that the physical page was
* referenced.
*
* The master copys of the reference and change bits are kept in the phys_entry.
* Other than the reference and change bits, changes to the phys_entry are not
* allowed if it has any mappings. The master reference and change bits must be
* changed via atomic update.
*
* Invalidating a PTE merges the RC bits into the phys_entry.
*
* Before checking the reference and/or bits, ALL mappings to the physical page are
* invalidated.
*
* PTEs are never explicitly validated, they are always faulted in. They are also
* not visible outside of the hw_vm modules. Complete seperation of church and state.
*
* Removal of a mapping is invalidates its PTE.
*
* So, how do we deal with mappings to I/O space? We don't have a physent for it.
* Within the mapping is a copy of the second half of the PTE. This is used
* ONLY when there is no physical entry. It is swapped into the PTE whenever
* it is built. There is no need to swap it back out, because RC is not
* maintained for these mappings.
*
* So, I'm starting to get concerned about the number of lwarx/stcwx loops in
* this. Satisfying a mapped address with no stealing requires one lock. If we
* steal an entry, there's two locks and an atomic update. Invalidation of an entry
* takes one lock and, if there is a PTE, another lock and an atomic update. Other
* operations are multiples (per mapping) of the above. Maybe we should look for
* an alternative. So far, I haven't found one, but I haven't looked hard.
*/
/* hw_add_map(struct mapping *mp, space_t space, vm_offset_t va) - Adds a mapping
*
* Adds a mapping to the PTEG hash list.
*
* Interrupts must be disabled before calling.
*
* Using the space and the virtual address, we hash into the hash table
* and get a lock on the PTEG hash chain. Then we chain the
* mapping to the front of the list.
*
*/
.align 5
.globl EXT(hw_add_map)
LEXT(hw_add_map)
#if PERFTIMES && DEBUG
mr r7,r3
mflr r11
li r3,20
bl EXT(dbgLog2) ; Start of hw_add_map
mr r3,r7
mtlr r11
#endif
mfmsr r0 /* Get the MSR */
eqv r6,r6,r6 /* Fill the bottom with foxes */
rlwinm r11,r4,6,6,25 /* Position the space for the VSID */
mfspr r10,sdr1 /* Get hash table base and size */
rlwimi r11,r5,30,2,5 /* Insert the segment no. to make a VSID */
rlwimi r6,r10,16,0,15 /* Make table size -1 out of mask */
rlwinm r7,r5,26,10,25 /* Isolate the page index */
or r8,r10,r6 /* Point to the last byte in table */
rlwinm r9,r5,4,0,3 ; Move nybble 1 up to 0
xor r7,r7,r11 /* Get primary hash */
andi. r12,r0,0x7FCF /* Disable translation and interruptions */
rlwinm r11,r11,1,1,24 /* Position VSID for pte ID */
addi r8,r8,1 /* Point to the PTEG Control Area */
xor r9,r9,r5 ; Splooch vaddr nybble 0 and 1 together
and r7,r7,r6 /* Wrap the hash */
rlwimi r11,r5,10,26,31 /* Move API into pte ID */
rlwinm r9,r9,6,27,29 ; Get splooched bits in place
add r8,r8,r7 /* Point to our PCA entry */
rlwinm r10,r4,2,27,29 ; Get low 3 bits of the VSID for look-aside hash
mtmsr r12 /* Get the stuff disabled */
la r4,PCAhash(r8) /* Point to the mapping hash area */
xor r9,r9,r10 ; Finish splooching nybble 0, 1, and the low bits of the VSID
isync /* Get rid of anything prefetched before we ref storage */
/*
* We've now got the address of our PCA, the hash chain anchor, our API subhash,
* and word 0 of the PTE (the virtual part).
*
* Now, we just lock the PCA.
*/
li r12,1 /* Get the locked value */
dcbt 0,r4 /* We'll need the hash area in a sec, so get it */
add r4,r4,r9 /* Point to the right mapping hash slot */
lwarx r10,0,r8 ; ?
ptegLckx: lwarx r10,0,r8 /* Get the PTEG lock */
mr. r10,r10 /* Is it locked? */
bne- ptegLckwx /* Yeah... */
stwcx. r12,0,r8 /* Take take it */
bne- ptegLckx /* Someone else was trying, try again... */
b ptegSXgx /* All done... */
.align 4
ptegLckwx: mr. r10,r10 /* Check if it's already held */
beq+ ptegLckx /* It's clear... */
lwz r10,0(r8) /* Get lock word again... */
b ptegLckwx /* Wait... */
.align 4
ptegSXgx: isync /* Make sure we haven't used anything yet */
lwz r7,0(r4) /* Pick up the anchor of hash list */
stw r3,0(r4) /* Save the new head */
stw r7,mmhashnext(r3) /* Chain in the old head */
stw r4,mmPTEhash(r3) /* Point to the head of the hash list */
sync /* Make sure the chain is updated */
stw r10,0(r8) /* Unlock the hash list */
mtmsr r0 /* Restore translation and interruptions */
isync /* Toss anything doe with DAT off */
#if PERFTIMES && DEBUG
mflr r11
mr r4,r3
li r3,21
bl EXT(dbgLog2) ; end of hw_add_map
mr r3,r4
mtlr r11
#endif
blr /* Leave... */
/* mp=hw_lock_phys_vir(space, va) - Finds and locks a physical entry by vaddr.
*
* Returns the mapping with the associated physent locked if found, or a
* zero and no lock if not. It we timed out trying to get a the lock on
* the physical entry, we retun a 1. A physical entry can never be on an
* odd boundary, so we can distinguish between a mapping and a timeout code.
*
* Interrupts must be disabled before calling.
*
* Using the space and the virtual address, we hash into the hash table
* and get a lock on the PTEG hash chain. Then we search the chain for the
* mapping for our virtual address. From there, we extract the pointer to
* the physical entry.
*
* Next comes a bit of monkey business. we need to get a lock on the physical
* entry. But, according to our rules, we can't get it after we've gotten the
* PTEG hash lock, we could deadlock if we do. So, we need to release the
* hash lock. The problem is, though, that as soon as we release it, some
* other yahoo may remove our mapping between the time that we release the
* hash lock and obtain the phys entry lock. So, we can't count on the
* mapping once we release the lock. Instead, after we lock the phys entry,
* we search the mapping list (phys_link) for our translation. If we don't find it,
* we unlock the phys entry, bail out, and return a 0 for the mapping address. If we
* did find it, we keep the lock and return the address of the mapping block.
*
* What happens when a mapping is found, but there is no physical entry?
* This is what happens when there is I/O area mapped. It one of these mappings
* is found, the mapping is returned, as is usual for this call, but we don't
* try to lock anything. There could possibly be some problems here if another
* processor releases the mapping while we still alre using it. Hope this
* ain't gonna happen.
*
* Taaa-dahhh! Easy as pie, huh?
*
* So, we have a few hacks hacks for running translate off in here.
* First, when we call the lock routine, we have carnel knowlege of the registers is uses.
* That way, we don't need a stack frame, which we can't have 'cause the stack is in
* virtual storage. But wait, as if that's not enough... We need one more register. So,
* we cram the LR into the CTR and return from there.
*
*/
.align 5
.globl EXT(hw_lock_phys_vir)
LEXT(hw_lock_phys_vir)
#if PERFTIMES && DEBUG
mflr r11
mr r5,r3
li r3,22
bl EXT(dbgLog2) ; Start of hw_add_map
mr r3,r5
mtlr r11
#endif
mfmsr r12 /* Get the MSR */
eqv r6,r6,r6 /* Fill the bottom with foxes */
rlwinm r11,r3,6,6,25 /* Position the space for the VSID */
mfspr r5,sdr1 /* Get hash table base and size */
rlwimi r11,r4,30,2,5 /* Insert the segment no. to make a VSID */
rlwimi r6,r5,16,0,15 /* Make table size -1 out of mask */
andi. r0,r12,0x7FCF /* Disable translation and interruptions */
rlwinm r9,r4,4,0,3 ; Move nybble 1 up to 0
rlwinm r7,r4,26,10,25 /* Isolate the page index */
or r8,r5,r6 /* Point to the last byte in table */
xor r7,r7,r11 /* Get primary hash */
rlwinm r11,r11,1,1,24 /* Position VSID for pte ID */
addi r8,r8,1 /* Point to the PTEG Control Area */
xor r9,r9,r4 ; Splooch vaddr nybble 0 and 1 together
and r7,r7,r6 /* Wrap the hash */
rlwimi r11,r4,10,26,31 /* Move API into pte ID */
rlwinm r9,r9,6,27,29 ; Get splooched bits in place
add r8,r8,r7 /* Point to our PCA entry */
rlwinm r10,r3,2,27,29 ; Get low 3 bits of the VSID for look-aside hash
mtmsr r0 /* Get the trans and intr off */
la r3,PCAhash(r8) /* Point to the mapping hash area */
xor r9,r9,r10 ; Finish splooching nybble 0, 1, and the low bits of the VSID
isync /* Make sure translation is off before we ref storage */
/*
* We've now got the address of our PCA, the hash chain anchor, our API subhash,
* and word 0 of the PTE (the virtual part).
*
* Now, we just lock the PCA and find our mapping, if it exists.
*/
dcbt 0,r3 /* We'll need the hash area in a sec, so get it */
add r3,r3,r9 /* Point to the right mapping hash slot */
lwarx r10,0,r8 ; ?
ptegLcka: lwarx r10,0,r8 /* Get the PTEG lock */
li r5,1 /* Get the locked value */
mr. r10,r10 /* Is it locked? */
bne- ptegLckwa /* Yeah... */
stwcx. r5,0,r8 /* Take take it */
bne- ptegLcka /* Someone else was trying, try again... */
b ptegSXga /* All done... */
.align 4
ptegLckwa: mr. r10,r10 /* Check if it's already held */
beq+ ptegLcka /* It's clear... */
lwz r10,0(r8) /* Get lock word again... */
b ptegLckwa /* Wait... */
.align 4
ptegSXga: isync /* Make sure we haven't used anything yet */
mflr r0 /* Get the LR */
lwz r9,0(r3) /* Pick up the first mapping block */
mtctr r0 /* Stuff it into the CTR */
findmapa:
mr. r3,r9 /* Did we hit the end? */
bne+ chkmapa /* Nope... */
stw r3,0(r8) /* Unlock the PTEG lock
Note: we never saved anything while we
had the lock, so we don't need a sync
before we unlock it */
vbail: mtmsr r12 /* Restore translation and interruptions */
isync /* Make sure translation is cool */
#if PERFTIMES && DEBUG
mflr r11
mr r4,r3
li r3,23
bl EXT(dbgLog2) ; Start of hw_add_map
mr r3,r4
mtlr r11
#endif
bctr /* Return in abject failure... */
.align 4
chkmapa: lwz r10,mmPTEv(r3) /* Pick up our virtual ID */
lwz r9,mmhashnext(r3) /* Pick up next mapping block */
cmplw r10,r11 /* Have we found ourself? */
bne- findmapa /* Nope, still wandering... */
lwz r9,mmphysent(r3) /* Get our physical entry pointer */
li r5,0 /* Clear this out */
mr. r9,r9 /* Is there, like, a physical entry? */
stw r5,0(r8) /* Unlock the PTEG lock
Note: we never saved anything while we
had the lock, so we don't need a sync
before we unlock it */
beq- vbail /* If there is no physical entry, it's time
to leave... */
/* Here we want to call hw_lock_bit. We don't want to use the stack, 'cause it's
* in virtual storage, and we're in real. So, we've carefully looked at the code
* in hw_lock_bit (and unlock) and cleverly don't use any of the registers that it uses.
* Be very, very aware of how you change this code. By the way, it uses:
* R0, R6, R7, R8, and R9. R3, R4, and R5 contain parameters
* Unfortunatly, we need to stash R9 still. So... Since we know we will not be interrupted
* ('cause we turned off interruptions and translation is off) we will use SPRG3...
*/
lwz r10,mmPTEhash(r3) /* Save the head of the hash-alike chain. We need it to find ourselves later */
lis r5,HIGH_ADDR(EXT(LockTimeOut)) /* Get address of timeout value */
la r3,pephyslink(r9) /* Point to the lock word */
ori r5,r5,LOW_ADDR(EXT(LockTimeOut)) /* Get second half of address */
li r4,PHYS_LOCK /* Get the lock bit value */
lwz r5,0(r5) /* Pick up the timeout value */
mtsprg 3,r9 /* Save R9 in SPRG3 */
bl EXT(hw_lock_bit) /* Go do the lock */
mfsprg r9,3 /* Restore pointer to the phys_entry */
mr. r3,r3 /* Did we timeout? */
lwz r4,pephyslink(r9) /* Pick up first mapping block */
beq- penterr /* Bad deal, we timed out... */
rlwinm r4,r4,0,0,26 ; Clear out the flags from first link
findmapb: mr. r3,r4 /* Did we hit the end? */
bne+ chkmapb /* Nope... */
la r3,pephyslink(r9) /* Point to where the lock is */
li r4,PHYS_LOCK /* Get the lock bit value */
bl EXT(hw_unlock_bit) /* Go unlock the physentry */
li r3,0 /* Say we failed */
b vbail /* Return in abject failure... */
penterr: li r3,1 /* Set timeout */
b vbail /* Return in abject failure... */
.align 5
chkmapb: lwz r6,mmPTEv(r3) /* Pick up our virtual ID */
lwz r4,mmnext(r3) /* Pick up next mapping block */
cmplw r6,r11 /* Have we found ourself? */
lwz r5,mmPTEhash(r3) /* Get the start of our hash chain */
bne- findmapb /* Nope, still wandering... */
cmplw r5,r10 /* On the same hash chain? */
bne- findmapb /* Nope, keep looking... */
b vbail /* Return in glorious triumph... */
/*
* hw_rem_map(mapping) - remove a mapping from the system.
*
* Upon entry, R3 contains a pointer to a mapping block and the associated
* physical entry is locked if there is one.
*
* If the mapping entry indicates that there is a PTE entry, we invalidate
* if and merge the reference and change information into the phys_entry.
*
* Next, we remove the mapping from the phys_ent and the PTEG hash list.
*
* Unlock any locks that are left, and exit.
*
* Note that this must be done with both interruptions off and VM off
*
* Note that this code depends upon the VSID being of the format 00SXXXXX
* where S is the segment number.
*
*
*/
.align 5
.globl EXT(hw_rem_map)
LEXT(hw_rem_map)
#if PERFTIMES && DEBUG
mflr r11
mr r4,r3
li r3,24
bl EXT(dbgLog2) ; Start of hw_add_map
mr r3,r4
mtlr r11
#endif
mfmsr r0 /* Save the MSR */
rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
rlwinm r12,r12,0,28,25 /* Clear IR and DR */
mtmsr r12 /* Clear interruptions and turn off translation */
isync /* Make sure that translation is off */
lwz r6,mmPTEhash(r3) /* Get pointer to hash list anchor */
lwz r5,mmPTEv(r3) /* Get the VSID */
dcbt 0,r6 /* We'll need that chain in a bit */
rlwinm r7,r6,0,0,25 /* Round hash list down to PCA boundary */
li r12,1 /* Get the locked value */
subi r6,r6,mmhashnext /* Make the anchor look like an entry */
lwarx r10,0,r7 ; ?
ptegLck1: lwarx r10,0,r7 /* Get the PTEG lock */
mr. r10,r10 /* Is it locked? */
bne- ptegLckw1 /* Yeah... */
stwcx. r12,0,r7 /* Try to take it */
bne- ptegLck1 /* Someone else was trying, try again... */
b ptegSXg1 /* All done... */
.align 4
ptegLckw1: mr. r10,r10 /* Check if it's already held */
beq+ ptegLck1 /* It's clear... */
lwz r10,0(r7) /* Get lock word again... */
b ptegLckw1 /* Wait... */
.align 4
ptegSXg1: isync /* Make sure we haven't used anything yet */
lwz r12,mmhashnext(r3) /* Prime with our forward pointer */
lwz r4,mmPTEent(r3) /* Get the pointer to the PTE now that the lock's set */
srchmaps: mr. r10,r6 /* Save the previous entry */
bne+ mapok /* No error... */
lis r0,HIGH_ADDR(Choke) /* We have a kernel choke!!! */
ori r0,r0,LOW_ADDR(Choke)
sc /* Firmware Heimlich manuever */
.align 4
mapok: lwz r6,mmhashnext(r6) /* Look at the next one */
cmplwi cr5,r4,0 /* Is there a PTE? */
cmplw r6,r3 /* Have we found ourselves? */
bne+ srchmaps /* Nope, get your head together... */
stw r12,mmhashnext(r10) /* Remove us from the queue */
rlwinm r9,r5,1,0,3 /* Move in the segment */
rlwinm r8,r4,6,4,19 /* Line PTEG disp up to a page */
rlwinm r11,r5,5,4,19 /* Line up the VSID */
lwz r10,mmphysent(r3) /* Point to the physical entry */
beq+ cr5,nopte /* There's no PTE to invalidate... */
xor r8,r8,r11 /* Back hash to virt index */
lis r12,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
rlwimi r9,r5,22,4,9 /* Move in the API */
ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
mfspr r11,pvr /* Find out what kind of machine we are */
rlwimi r9,r8,0,10,19 /* Create the virtual address */
rlwinm r11,r11,16,16,31 /* Isolate CPU type */
stw r5,0(r4) /* Make the PTE invalid */
cmplwi cr1,r11,3 /* Is this a 603? */
sync /* Make sure the invalid is stored */
lwarx r5,0,r12 ; ?
tlbhang1: lwarx r5,0,r12 /* Get the TLBIE lock */
rlwinm r11,r4,29,29,31 /* Get the bit position of entry */
mr. r5,r5 /* Is it locked? */
lis r6,0x8000 /* Start up a bit mask */
li r5,1 /* Get our lock word */
bne- tlbhang1 /* It's locked, go wait... */
stwcx. r5,0,r12 /* Try to get it */
bne- tlbhang1 /* We was beat... */
srw r6,r6,r11 /* Make a "free slot" mask */
lwz r5,PCAallo(r7) /* Get the allocation control bits */
rlwinm r11,r6,24,8,15 /* Make the autogen bit to turn off */
or r5,r5,r6 /* turn on the free bit */
rlwimi r11,r11,24,16,23 /* Get lock bit mask to turn it off */
andc r5,r5,r11 /* Turn off the lock and autogen bits in allocation flags */
li r11,0 /* Lock clear value */
tlbie r9 /* Invalidate it everywhere */
beq- cr1,its603a /* It's a 603, skip the tlbsync... */
eieio /* Make sure that the tlbie happens first */
tlbsync /* wait for everyone to catch up */
its603a: sync /* Make sure of it all */
stw r11,0(r12) /* Clear the tlbie lock */
eieio /* Make sure those RC bit are loaded */
stw r5,PCAallo(r7) /* Show that the slot is free */
stw r11,mmPTEent(r3) /* Clear the pointer to the PTE */
nopte: mr. r10,r10 /* See if there is a physical entry */
la r9,pephyslink(r10) /* Point to the physical mapping chain */
beq- nophys /* No physical entry, we're done... */
beq- cr5,nadamrg /* Not PTE to merge... */
lwz r6,4(r4) /* Get the latest reference and change bits */
la r12,pepte1(r10) /* Point right at the master copy */
rlwinm r6,r6,0,23,24 /* Extract just the RC bits */
lwarx r8,0,r12 ; ?
mrgrc: lwarx r8,0,r12 /* Get the master copy */
or r8,r8,r6 /* Merge in latest RC */
stwcx. r8,0,r12 /* Save it back */
bne- mrgrc /* If it changed, try again... */
nadamrg: li r11,0 /* Clear this out */
lwz r12,mmnext(r3) /* Prime with our next */
stw r11,0(r7) /* Unlock the hash chain now so we don't
lock out another processor during the
our next little search */
srchpmap: mr. r10,r9 /* Save the previous entry */
bne+ mapok1 /* No error... */
lis r0,HIGH_ADDR(Choke) /* We have a kernel choke!!! */
ori r0,r0,LOW_ADDR(Choke)
sc /* Firmware Heimlich maneuver */
.align 4
mapok1: lwz r9,mmnext(r9) /* Look at the next one */
rlwinm r9,r9,0,0,26 ; Clear out the flags from first link
cmplw r9,r3 /* Have we found ourselves? */
bne+ srchpmap /* Nope, get your head together... */
stw r12,mmnext(r10) /* Remove us from the queue */
mtmsr r0 /* Interrupts and translation back on */
isync
#if PERFTIMES && DEBUG
mflr r11
li r3,25
bl EXT(dbgLog2) ; Start of hw_add_map
mtlr r11
#endif
blr /* Return... */
.align 4
nophys: li r4,0 /* Make sure this is 0 */
sync /* Make sure that chain is updated */
stw r4,0(r7) /* Unlock the hash chain */
mtmsr r0 /* Interrupts and translation back on */
isync
#if PERFTIMES && DEBUG
mflr r11
li r3,25
bl EXT(dbgLog2) ; Start of hw_add_map
mtlr r11
#endif
blr /* Return... */
/*
* hw_prot(physent, prot) - Change the protection of a physical page
*
* Upon entry, R3 contains a pointer to a physical entry which is locked.
* R4 contains the PPC protection bits.
*
* The first thing we do is to slam the new protection into the phys entry.
* Then we scan the mappings and process each one.
*
* Acquire the lock on the PTEG hash list for the mapping being processed.
*
* If the current mapping has a PTE entry, we invalidate
* it and merge the reference and change information into the phys_entry.
*
* Next, slam the protection bits into the entry and unlock the hash list.
*
* Note that this must be done with both interruptions off and VM off
*
*
*/
.align 5
.globl EXT(hw_prot)
LEXT(hw_prot)
#if PERFTIMES && DEBUG
mflr r11
mr r7,r3
// lwz r5,4(r3)
li r5,0x1111
li r3,26
bl EXT(dbgLog2) ; Start of hw_add_map
mr r3,r7
mtlr r11
#endif
mfmsr r0 /* Save the MSR */
rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
li r5,pepte1 /* Get displacement to the second word of master pte */
rlwinm r12,r12,0,28,25 /* Clear IR and DR */
mtmsr r12 /* Clear interruptions and turn off translation */
/* NOTE: we can get away with just turning the
instruction translation off here because we
are running with virtual = real. If not,
there would be an "implied branch", which wouldn't
be too good. */
isync /* Make sure that translation is off */
lwz r10,pephyslink(r3) /* Get the first mapping block */
rlwinm r10,r10,0,0,26 ; Clear out the flags from first link
/*
* Note that we need to to do the interlocked update here because another processor
* can be updating the reference and change bits even though the physical entry
* is locked. All modifications to the PTE portion of the physical entry must be
* done via interlocked update.
*/
lwarx r8,r5,r3 ; ?
protcng: lwarx r8,r5,r3 /* Get the master copy */
rlwimi r8,r4,0,30,31 /* Move in the protection bits */
stwcx. r8,r5,r3 /* Save it back */
bne- protcng /* If it changed, try again... */
protnext: mr. r10,r10 /* Are there any more mappings? */
beq- protdone /* Naw... */
lwz r7,mmPTEhash(r10) /* Get pointer to hash list anchor */
lwz r5,mmPTEv(r10) /* Get the virtual address */
rlwinm r7,r7,0,0,25 /* Round hash list down to PCA boundary */
li r12,1 /* Get the locked value */
lwarx r11,0,r7 ; ?
protLck1: lwarx r11,0,r7 /* Get the PTEG lock */
mr. r11,r11 /* Is it locked? */
bne- protLckw1 /* Yeah... */
stwcx. r12,0,r7 /* Try to take it */
bne- protLck1 /* Someone else was trying, try again... */
b protSXg1 /* All done... */
.align 4
protLckw1: mr. r11,r11 /* Check if it's already held */
beq+ protLck1 /* It's clear... */
lwz r11,0(r7) /* Get lock word again... */
b protLckw1 /* Wait... */
.align 4
protSXg1: isync /* Make sure we haven't used anything yet */
lwz r6,mmPTEent(r10) /* Get the pointer to the PTE now that the lock's set */
rlwinm r9,r5,1,0,3 /* Move in the segment */
mr. r6,r6 /* See if there is a PTE here */
rlwinm r8,r5,31,2,25 /* Line it up and check if empty */
beq+ protul /* There's no PTE to invalidate... */
xor r8,r8,r6 /* Back hash to virt index */
rlwimi r9,r5,22,4,9 /* Move in the API */
lis r12,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
rlwinm r5,r5,0,1,31 /* Clear the valid bit */
ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
mfspr r11,pvr /* Find out what kind of machine we are */
rlwimi r9,r8,6,10,19 /* Create the virtual address */
rlwinm r11,r11,16,16,31 /* Isolate CPU type */
stw r5,0(r6) /* Make the PTE invalid */
cmplwi cr1,r11,3 /* Is this a 603? */
sync /* Make sure the invalid is stored */
lwarx r11,0,r12 ; ?
tlbhangp: lwarx r11,0,r12 /* Get the TLBIE lock */
rlwinm r8,r6,29,29,31 /* Get the bit position of entry */
mr. r11,r11 /* Is it locked? */
lis r5,0x8000 /* Start up a bit mask */
li r11,1 /* Get our lock word */
bne- tlbhangp /* It's locked, go wait... */
stwcx. r11,0,r12 /* Try to get it */
bne- tlbhangp /* We was beat... */
li r11,0 /* Lock clear value */
tlbie r9 /* Invalidate it everywhere */
beq- cr1,its603p /* It's a 603, skip the tlbsync... */
eieio /* Make sure that the tlbie happens first */
tlbsync /* wait for everyone to catch up */
its603p: stw r11,0(r12) /* Clear the lock */
srw r5,r5,r8 /* Make a "free slot" mask */
sync /* Make sure of it all */
lwz r6,4(r6) /* Get the latest reference and change bits */
stw r11,mmPTEent(r10) /* Clear the pointer to the PTE */
rlwinm r6,r6,0,23,24 /* Extract the RC bits */
lwz r9,PCAallo(r7) /* Get the allocation control bits */
rlwinm r8,r5,24,8,15 /* Make the autogen bit to turn off */
or r9,r9,r5 /* Set the slot free */
rlwimi r8,r8,24,16,23 /* Get lock bit mask to turn it off */
andc r9,r9,r8 /* Clear the auto and lock bits */
li r5,pepte1 /* Get displacement to the second word of master pte */
stw r9,PCAallo(r7) /* Store the allocation controls */
lwarx r11,r5,r3 ; ?
protmod: lwarx r11,r5,r3 /* Get the master copy */
or r11,r11,r6 /* Merge in latest RC */
stwcx. r11,r5,r3 /* Save it back */
bne- protmod /* If it changed, try again... */
sync /* Make sure that chain is updated */
protul: li r4,0 /* Get a 0 */
lwz r10,mmnext(r10) /* Get the next */
stw r4,0(r7) /* Unlock the hash chain */
b protnext /* Go get the next one */
.align 4
protdone: mtmsr r0 /* Interrupts and translation back on */
isync
#if PERFTIMES && DEBUG
mflr r11
li r3,27
bl EXT(dbgLog2) ; Start of hw_add_map
mtlr r11
#endif
blr /* Return... */
/*
* hw_pte_comm(physent) - Do something to the PTE pointing to a physical page
*
* Upon entry, R3 contains a pointer to a physical entry which is locked.
* Note that this must be done with both interruptions off and VM off
*
* First, we set up CRs 5 and 7 to indicate which of the 7 calls this is.
*
* Now we scan the mappings to invalidate any with and active PTE.
*
* Acquire the lock on the PTEG hash list for the mapping being processed.
*
* If the current mapping has a PTE entry, we invalidate
* it and merge the reference and change information into the phys_entry.
*
* Next, unlock the hash list and go on to the next mapping.
*
*
*
*/
.align 5
.globl EXT(hw_inv_all)
LEXT(hw_inv_all)
li r9,0x800 /* Indicate invalidate all */
b hw_pte_comm /* Join in the fun... */
.align 5
.globl EXT(hw_tst_mod)
LEXT(hw_tst_mod)
lwz r8,pepte1(r3) ; Get the saved PTE image
li r9,0x400 /* Indicate test modify */
rlwinm. r8,r8,25,31,31 ; Make change bit into return code
beq+ hw_pte_comm ; Assume we do not know if it is set...
mr r3,r8 ; Set the return code
blr ; Return quickly...
.align 5
.globl EXT(hw_tst_ref)
LEXT(hw_tst_ref)
lwz r8,pepte1(r3) ; Get the saved PTE image
li r9,0x200 /* Indicate test reference bit */
rlwinm. r8,r8,24,31,31 ; Make reference bit into return code
beq+ hw_pte_comm ; Assume we do not know if it is set...
mr r3,r8 ; Set the return code
blr ; Return quickly...
/*
* Note that the following are all in one CR for ease of use later
*/
.align 4
.globl EXT(hw_set_mod)
LEXT(hw_set_mod)
li r9,0x008 /* Indicate set modify bit */
b hw_pte_comm /* Join in the fun... */
.align 4
.globl EXT(hw_clr_mod)
LEXT(hw_clr_mod)
li r9,0x004 /* Indicate clear modify bit */
b hw_pte_comm /* Join in the fun... */
.align 4
.globl EXT(hw_set_ref)
LEXT(hw_set_ref)
li r9,0x002 /* Indicate set reference */
b hw_pte_comm /* Join in the fun... */
.align 5
.globl EXT(hw_clr_ref)
LEXT(hw_clr_ref)
li r9,0x001 /* Indicate clear reference bit */
b hw_pte_comm /* Join in the fun... */
/*
* This is the common stuff.
*/
.align 5
hw_pte_comm: /* Common routine for pte tests and manips */
#if PERFTIMES && DEBUG
mflr r11
mr r7,r3
lwz r4,4(r3)
mr r5,r9
li r3,28
bl EXT(dbgLog2) ; Start of hw_add_map
mr r3,r7
mtlr r11
#endif
lwz r10,pephyslink(r3) /* Get the first mapping block */
mfmsr r0 /* Save the MSR */
rlwinm. r10,r10,0,0,26 ; Clear out the flags from first link and see if we are mapped
rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
mtcrf 0x05,r9 /* Set the call type flags into cr5 and 7 */
rlwinm r12,r12,0,28,25 /* Clear IR and DR */
beq- comnmap ; No mapping
dcbt br0,r10 ; Touch the first mapping in before the isync
comnmap: mtmsr r12 /* Clear interruptions and turn off translation */
/* NOTE: we can get away with just turning the
instruction translation off here because we
are running with virtual equals real. If not,
there would be an "implied branch", which wouldn't
be too good. */
isync /* Make sure that translation is off */
beq- commdone ; Nothing us mapped to this page...
b commnext ; Jump to first pass (jump here so we can align loop)
.align 5
commnext: lwz r11,mmnext(r10) ; Get the pointer to the next mapping (if any)
lwz r7,mmPTEhash(r10) /* Get pointer to hash list anchor */
lwz r5,mmPTEv(r10) /* Get the virtual address */
mr. r11,r11 ; More mappings to go?
rlwinm r7,r7,0,0,25 /* Round hash list down to PCA boundary */
beq- commnxtch ; No more mappings...
dcbt br0,r11 ; Touch the next mapping
commnxtch: li r12,1 /* Get the locked value */
lwarx r11,0,r7 ; ?
commLck1: lwarx r11,0,r7 /* Get the PTEG lock */
mr. r11,r11 /* Is it locked? */
bne- commLckw1 /* Yeah... */
stwcx. r12,0,r7 /* Try to take it */
bne- commLck1 /* Someone else was trying, try again... */
b commSXg1 /* All done... */
.align 4
commLckw1: mr. r11,r11 /* Check if it's already held */
beq+ commLck1 /* It's clear... */
lwz r11,0(r7) /* Get lock word again... */
b commLckw1 /* Wait... */
.align 4
commSXg1: isync /* Make sure we haven't used anything yet */
lwz r6,mmPTEent(r10) /* Get the pointer to the PTE now that the lock's set */
rlwinm r9,r5,1,0,3 /* Move in the segment */
mr. r6,r6 /* See if there is a PTE entry here */
rlwinm r8,r5,31,2,25 /* Line it up and check if empty */
beq+ commul /* There's no PTE to invalidate... */
xor r8,r8,r6 /* Back hash to virt index */
rlwimi r9,r5,22,4,9 /* Move in the API */
lis r12,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
rlwinm r5,r5,0,1,31 /* Clear the valid bit */
ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
rlwimi r9,r8,6,10,19 /* Create the virtual address */
stw r5,0(r6) /* Make the PTE invalid */
mfspr r4,pvr /* Find out what kind of machine we are */
sync /* Make sure the invalid is stored */
lwarx r11,0,r12 ; ?
tlbhangco: lwarx r11,0,r12 /* Get the TLBIE lock */
rlwinm r8,r6,29,29,31 /* Get the bit position of entry */
mr. r11,r11 /* Is it locked? */
lis r5,0x8000 /* Start up a bit mask */
li r11,1 /* Get our lock word */
bne- tlbhangco /* It's locked, go wait... */
stwcx. r11,0,r12 /* Try to get it */
bne- tlbhangco /* We was beat... */
rlwinm r4,r4,16,16,31 /* Isolate CPU type */
li r11,0 /* Lock clear value */
cmplwi r4,3 /* Is this a 603? */
tlbie r9 /* Invalidate it everywhere */
beq- its603co /* It's a 603, skip the tlbsync... */
eieio /* Make sure that the tlbie happens first */
tlbsync /* wait for everyone to catch up */
its603co: stw r11,0(r12) /* Clear the lock */
srw r5,r5,r8 /* Make a "free slot" mask */
sync /* Make sure of it all */
lwz r6,4(r6) /* Get the latest reference and change bits */
lwz r9,PCAallo(r7) /* Get the allocation control bits */
stw r11,mmPTEent(r10) /* Clear the pointer to the PTE */
rlwinm r8,r5,24,8,15 /* Make the autogen bit to turn off */
or r9,r9,r5 /* Set the slot free */
rlwimi r8,r8,24,16,23 /* Get lock bit mask to turn it off */
rlwinm r4,r6,0,23,24 /* Extract the RC bits */
andc r9,r9,r8 /* Clear the auto and lock bits */
li r5,pepte1 /* Get displacement to the second word of master pte */
stw r9,PCAallo(r7) /* Store the allocation controls */
lwarx r11,r5,r3 ; ?
commmod: lwarx r11,r5,r3 /* Get the master copy */
or r11,r11,r4 /* Merge in latest RC */
stwcx. r11,r5,r3 /* Save it back */
bne- commmod /* If it changed, try again... */
sync /* Make sure that chain is updated */
commul: lwz r10,mmnext(r10) /* Get the next */
li r4,0 /* Make sure this is 0 */
mr. r10,r10 ; Is there another mapping?
stw r4,0(r7) /* Unlock the hash chain */
bne+ commnext ; Go get the next if there is one...
/*
* Now that all PTEs have been invalidated and the master RC bits are updated,
* we go ahead and figure out what the original call was and do that. Note that
* another processor could be messing around and may have entered one of the
* PTEs we just removed into the hash table. Too bad... You takes yer chances.
* If there's a problem with that, it's because some higher level was trying to
* do something with a mapping that it shouldn't. So, the problem's really
* there, nyaaa, nyaaa, nyaaa... nyaaa, nyaaa... nyaaa! So there!
*/
commdone: li r5,pepte1 /* Get displacement to the second word of master pte */
blt cr5,commfini /* We're finished, it was invalidate all... */
bgt cr5,commtst /* It was a test modified... */
beq cr5,commtst /* It was a test reference... */
/*
* Note that we need to to do the interlocked update here because another processor
* can be updating the reference and change bits even though the physical entry
* is locked. All modifications to the PTE portion of the physical entry must be
* done via interlocked update.
*/
lwarx r8,r5,r3 ; ?
commcng: lwarx r8,r5,r3 /* Get the master copy */
bng cr7,commclrr /* Jump if not clear change bit... */
rlwinm r8,r8,0,25,23 /* Clear the change bit */
b commsave /* Go save the new version... */
.align 4
commclrr: bns cr7,commsetm /* Jump away if not clear reference... */
rlwinm r8,r8,0,24,22 /* Clear the change bit */
b commsave /* Go save the new version... */
.align 4
commsetm: bnl cr7,commsetr /* Jump away if not set modified... */
ori r8,r8,0x0080 /* Set the modified bit */
b commsave /* Go save the new version... */
.align 4
commsetr: ori r8,r8,0x0100 /* Set the referenced bit */
commsave: stwcx. r8,r5,r3 /* Save it back */
bne- commcng /* If it changed, try again... */
mtmsr r0 /* Interrupts and translation back on */
isync
#if PERFTIMES && DEBUG
mflr r11
mr r4,r3
li r3,29
bl EXT(dbgLog2) ; Start of hw_add_map
mr r3,r4
mtlr r11
#endif
blr /* Return... */
.align 4
commtst: lwz r8,pepte1(r3) /* Get the PTE */
bne- cr5,commtcb ; This is for the change bit...
mtmsr r0 ; Interrupts and translation back on
rlwinm r3,r8,24,31,31 ; Copy reference bit to bit 31
isync ; Toss prefetching
#if PERFTIMES && DEBUG
mflr r11
mr r4,r3
li r3,29
bl EXT(dbgLog2) ; Start of hw_add_map
mr r3,r4
mtlr r11
#endif
blr ; Return...
.align 4
commtcb: rlwinm r3,r8,25,31,31 ; Copy change bit to bit 31
commfini: mtmsr r0 ; Interrupts and translation back on
isync ; Toss prefetching
#if PERFTIMES && DEBUG
mflr r11
mr r4,r3
li r3,29
bl EXT(dbgLog2) ; Start of hw_add_map
mr r3,r4
mtlr r11
#endif
blr ; Return...
/*
* hw_phys_attr(struct phys_entry *pp, vm_prot_t prot, unsigned int wimg) - Sets the default physical page attributes
*
* Note that this must be done with both interruptions off and VM off
* Move the passed in attributes into the pte image in the phys entry
*
*
*/
.align 5
.globl EXT(hw_phys_attr)
LEXT(hw_phys_attr)
#if PERFTIMES && DEBUG
mflr r11
mr r8,r3
mr r7,r5
mr r5,r4
// lwz r4,4(r3)
li r4,0x1111
li r3,30
bl EXT(dbgLog2) ; Start of hw_add_map
mr r3,r8
mr r4,r5
mr r5,r7
mtlr r11
#endif
mfmsr r0 /* Save the MSR */
andi. r5,r5,0x0078 /* Clean up the WIMG */
rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
rlwimi r5,r4,0,30,31 /* Move the protection into the wimg register */
la r6,pepte1(r3) /* Point to the default pte */
rlwinm r12,r12,0,28,25 /* Clear IR and DR */
mtmsr r12 /* Clear interruptions and turn off translation */
/* NOTE: we can get away with just turning the
instruction translation off here because we
are running with virtual equals real. If not,
there would be an "implied branch", which wouldn't
be too good. */
isync /* Make sure that translation is off */
lwarx r10,0,r6 ; ?
atmattr: lwarx r10,0,r6 /* Get the pte */
rlwimi r10,r5,0,25,31 /* Move in the new attributes */
stwcx. r10,0,r6 /* Try it on for size */
bne- atmattr /* Someone else was trying, try again... */
mtmsr r0 /* Interrupts and translation back on */
isync
#if PERFTIMES && DEBUG
mflr r11
mr r4,r10
li r3,31
bl EXT(dbgLog2) ; Start of hw_add_map
mtlr r11
#endif
blr /* All done... */
/*
* handlePF - handle a page fault interruption
*
* If the fault can be handled, this routine will RFI directly,
* otherwise it will return with all registers as in entry.
*
* Upon entry, state and all registers have been saved in savearea.
* This is pointed to by R13.
* IR and DR are off, interrupts are masked,
* Floating point be disabled.
* R3 is the interrupt code.
*
* If we bail, we must restore cr5, and all registers except 6 and
* 3.
*
*/
.align 5
.globl EXT(handlePF)
LEXT(handlePF)
/*
* This first part does a quick check to see if we can handle the fault.
* We can't handle any kind of protection exceptions here, so we pass
* them up to the next level.
*
* The mapping lists are kept in MRS (most recently stolen)
* order on queues anchored within from the
* PTEG to which the virtual address hashes. This is further segregated by
* the low-order 3 bits of the VSID XORed with the segment number and XORed
* with bits 4-7 of the vaddr in an attempt to keep the searches
* short.
*
* MRS is handled by moving the entry to the head of its list when stolen in the
* assumption that it will be revalidated soon. Entries are created on the head
* of the list because they will be used again almost immediately.
*
* We need R13 set to the savearea, R3 set to the interrupt code, and R2
* set to the per_proc.
*
* NOTE: In order for a page-fault redrive to work, the translation miss
* bit must be set in the DSISR (or SRR1 for IFETCH). That must occur
* before we come here.
*/
cmplwi r3,T_INSTRUCTION_ACCESS /* See if this is for the instruction */
lwz r8,savesrr1(r13) ; Get the MSR to determine mode
beq- gotIfetch ; We have an IFETCH here...
lwz r7,savedsisr(r13) /* Get the DSISR */
lwz r6,savedar(r13) /* Get the fault address */
b ckIfProt ; Go check if this is a protection fault...
gotIfetch: mr r7,r8 ; IFETCH info is in SRR1
lwz r6,savesrr0(r13) /* Get the instruction address */
ckIfProt: rlwinm. r7,r7,0,1,1 ; Is this a protection exception?
beqlr- ; Yes... (probably not though)
/*
* We will need to restore registers if we bail after this point.
* Note that at this point several SRs have been changed to the kernel versions.
* Therefore, for these we must build these values.
*/
#if PERFTIMES && DEBUG
mflr r11
mr r5,r6
mr r4,r3
li r3,32
bl EXT(dbgLog2) ; Start of hw_add_map
mr r3,r4
mtlr r11
mfsprg r2,0
#endif
lwz r3,PP_USERSPACE(r2) ; Get the user space
rlwinm. r8,r8,0,MSR_PR_BIT,MSR_PR_BIT ; Supervisor state access?
eqv r1,r1,r1 /* Fill the bottom with foxes */
rlwimi r3,r6,24,8,11 ; Insert the segment number into the VSID to make the user VSID
bne+ notsuper ; If user access, we have the VSID already...
mfsrin r3,r6 ; Get the faulting segment reg for kernel accesses
notsuper: mfspr r5,sdr1 /* Get hash table base and size */
rlwinm r9,r6,2,2,5 ; Move nybble 1 up to 0 (keep aligned with VSID)
rlwinm r3,r3,6,2,25 /* Position the space for the VSID */
rlwimi r1,r5,16,0,15 /* Make table size -1 out of mask */
rlwinm r7,r6,26,10,25 /* Isolate the page index */
xor r9,r9,r3 ; Splooch vaddr nybble 0 (from VSID) and 1 together
or r8,r5,r1 /* Point to the last byte in table */
xor r7,r7,r3 /* Get primary hash */
rlwinm r3,r3,1,1,24 /* Position VSID for pte ID */
addi r8,r8,1 /* Point to the PTEG Control Area */
rlwinm r9,r9,8,27,29 ; Get splooched bits in place
and r7,r7,r1 /* Wrap the hash */
rlwimi r3,r6,10,26,31 /* Move API into pte ID */
add r8,r8,r7 /* Point to our PCA entry */
rlwinm r12,r3,27,27,29 ; Get low 3 bits of the VSID for look-aside hash
la r11,PCAhash(r8) /* Point to the mapping hash area */
xor r9,r9,r12 ; Finish splooching nybble 0, 1, and the low bits of the VSID
/*
* We have about as much as we need to start searching the autogen
* and mappings. From here on, any kind of failure will bail, and
* contention will either bail or restart from here.
*
*
*/
li r12,1 /* Get the locked value */
dcbt 0,r11 /* We'll need the hash area in a sec, so get it */
add r11,r11,r9 /* Point to the right mapping hash slot */
lwarx r10,0,r8 ; ?
ptegLck: lwarx r10,0,r8 /* Get the PTEG lock */
mr. r10,r10 /* Is it locked? */
bne- ptegLckw /* Yeah... */
stwcx. r12,0,r8 /* Take take it */
bne- ptegLck /* Someone else was trying, try again... */
b ptegSXg /* All done... */
.align 4
ptegLckw: mr. r10,r10 /* Check if it's already held */
beq+ ptegLck /* It's clear... */
lwz r10,0(r8) /* Get lock word again... */
b ptegLckw /* Wait... */
.align 5
nop ; Force ISYNC to last instruction in IFETCH
nop
nop
ptegSXg: isync /* Make sure we haven't used anything yet */
lwz r9,0(r11) /* Pick up first mapping block */
mr r5,r11 /* Get the address of the anchor */
mr r7,r9 /* Save the first in line */
b findmap ; Take space and force loop to cache line
findmap: mr. r12,r9 /* Are there more? */
beq- tryAuto /* Nope, nothing in mapping list for us... */
lwz r10,mmPTEv(r12) /* Get unique PTE identification */
lwz r9,mmhashnext(r12) /* Get the chain, just in case */
cmplw r10,r3 /* Did we hit our PTE? */
lwz r6,mmPTEent(r12) /* Get the pointer to the hash table entry */
mr r5,r12 /* Save the current as previous */
bne- findmap ; Nothing here, try the next...
; Cache line bounary here
cmplwi cr1,r6,0 /* Is there actually a PTE entry in the hash? */
lwz r2,mmphysent(r12) /* Get the physical entry */
bne- cr1,MustBeOK /* There's an entry in the hash table, so, this must
have been taken care of already... */
cmplwi cr2,r2,0 /* Is there a physical entry? */
li r0,0x0100 /* Force on the reference bit whenever we make a PTE valid */
addi r2,r2,pepte1 /* Point to the second half of PTE in physent */
lis r4,0x8000 /* Tell, PTE inserter that this wasn't an auto */
bne+ cr2,gotphys /* Skip down if we have a physical entry */
la r2,mmPTEr(r12) /* Point to the PTE in the mapping */
li r0,0x0180 /* When there is no physical entry, force on
both R and C bits to keep hardware from
updating the PTE to set them. We don't
keep track of RC for I/O areas, so this is ok */
gotphys: lwz r2,0(r2) /* Get the second part of the PTE */
b insert /* Go insert into the PTEG... */
MustBeOK: li r10,0 /* Get lock clear value */
li r3,T_IN_VAIN /* Say that we handled it */
stw r10,PCAlock(r8) /* Clear the PTEG lock */
sync
#if PERFTIMES && DEBUG
mflr r11
mr r4,r3
li r3,33
bl EXT(dbgLog2) ; Start of hw_add_map
mr r3,r4
mtlr r11
#endif
blr /* Blow back and handle exception */
/*
* We couldn't find it in the mapping list. As a last try, we will
* see if we can autogen it.
*
* An autogen area is defined as a contiguous range of virtual addresses that
* map to a fixed contiguous area of physical storage. This implimentation
* does not support an area that is not in the kernal address space. We may
* want to consider doing that later. The number and size of autogen areas
* are not defined, but it would be silly to define them for one or two pages
* or hundreds of individual areas.
*
* These areas are accessed at this level read-only without locks. Higher
* levels can add an autogen area to the front of the list by moving the anchor
* to the forward pointer and then doing a compare-and-swap to set the new
* anchor. Removals are tougher. They shouldn't happen frequently, if at all.
* The only problem is at the moment of the breaking of the chain. It is possible
* that another processor is using the element being removed. So, unless we know
* for a fact that the element is unused, we can't delete it. So, removal works
* like this: first, the we lock the chain at the higher level, to insure that
* no more than one processor is deleteing elements. Next, we move the forward
* pointer from the removee to the forward of its previous. (we find the previous
* under the lock.) Note that the removee's forward still points to the next.
* The lock on the chain can be released now.
*
* At this point, a second processor could be traversing the list. If it hasn't
* reached the removee yet, it can't see it because to it, it has been removed.
* If it happens to be at the removed one, it will process it and go to the next,
* It is up to the remover to insure that the there is enough time for the other
* processor(s) to finish. What it does is to send a processor synchronize
* SIGP to all other CPU and wait until they have been received. Since all
* code that traverses the list is uninterruptable, the SYNC command can't
* occur until all CPUs are done with the removee.
*
* The autogen list is ordered ascending. By relaxing the
* "must-link-at-head" restriction, and requiring a lock for addition,
* we can order the updates of the links such that a
* non-interlocked search can proceed during the update. We
* get a zero-cost early bail-out from the autogen search, and the
* additional overhead from maintanence is nearly nil.
*
* The one problem we have here is autogenned execute protected areas, we can
* end up with a persistant ISI. So, the solution is just say NO. No to execute
* protected autogen areas.
*/
.align 4
tryAuto: lis r10,HIGH_ADDR(EXT(AutoGenList)) /* Get the top part of autogen list anchor */
rlwinm. r11,r3,0,5,24 /* Check if we are in the kernel */
ori r10,r10,LOW_ADDR(EXT(AutoGenList)) /* Get the bottom part */
beq- checkAuto /* In kernel, see if we can autogen... */
/*
* When we come here, we know that we can't handle this. Restore whatever
* state that we trashed and go back to continue handling the interrupt.
*/
realFault: li r10,0 /* Get lock clear value */
lwz r3,saveexception(r13) /* Figure out the exception code again */
stw r10,PCAlock(r8) /* Clear the PTEG lock */
#if PERFTIMES && DEBUG
mflr r11
mr r4,r3
li r3,33
bl EXT(dbgLog2) ; Start of hw_add_map
mr r3,r4
mtlr r11
#endif
blr /* Blow back and handle exception */
.align 4
checkAuto: lwz r10,0(r10) /* Get the first autogen area */
findAuto: mr. r11,r10 /* Are there more? */
beq- realFault /* Nope, no autogen, it's a fault... */
lwz r4,AGNstart(r11) /* Get start of range */
lwz r9,AGNsize(r11) /* Get the size */
sub. r5,r6,r4 /* Get distance into region */
lwz r10,AGNnext(r11) /* Assure we'll need the next one */
blt- realFault /* Bail out when we're past any possible hit */
cmplw r5,r9 /* See if we fit in the range. This works even
on an unordered list. If the start address
is after the address under test, the subtract
will give us a negative number, but with a
logical test, it will be a bigger number */
lwz r2,AGNpteX(r11) /* Get translation--should be in cache */
bgt+ findAuto /* Not in this slot, try again... */
lis r4,0x8080 /* Indicate that this was autogened */
li r0,0x0180 /* Autogenned areas always set RC bits.
This keeps the hardware from having
to do two storage writes */
/*
* Here where we insert the PTE into the hash. The PTE image is in R3, R2.
* The PTEG allocation controls are a bit map of the state of the PTEG. The
* PCAlock bits are a temporary lock for the specified PTE. PCAfree indicates that
* the PTE slot is empty. PCAauto means that it comes from an autogen area. These
* guys do not keep track of reference and change and are actually "wired".
* They're easy to maintain and are second in priority for a steal. PCAsteal
* is a sliding position mask used to "randomize" PTE slot stealing. All 4 of these
* fields fit in a single word and are loaded and stored under control of the
* PTEG control area lock (PCAlock).
*
* Physically, the fields are arranged:
* 0: PCAfree
* 1: PCAauto
* 2: PCAlock
* 3: PCAsteal
*/
insert: lwz r10,PCAallo(r8) /* Get the PTEG controls */
eqv r6,r6,r6 /* Get all ones */
mr r11,r10 /* Make a copy */
rlwimi r6,r10,8,16,23 /* Insert sliding steal position */
rlwimi r11,r11,24,24,31 /* Duplicate the locked field */
addi r6,r6,-256 /* Form mask */
rlwimi r11,r11,16,0,15 /* This gives us a quadrupled lock mask */
rlwinm r5,r10,31,24,0 /* Slide over the mask for next time */
mr r9,r10 /* Make a copy to test */
not r11,r11 /* Invert the quadrupled lock */
or r2,r2,r0 /* Force on R, and maybe C bit */
and r9,r9,r11 /* Remove the locked guys */
rlwimi r5,r5,8,24,24 /* Wrap bottom bit to top in mask */
rlwimi r9,r11,0,16,31 /* Put two copies of the unlocked entries at the end */
rlwimi r10,r5,0,24,31 /* Move steal map back in */
and r9,r9,r6 /* Set the starting point for stealing */
/* So, now we have in R9:
byte 0 = ~locked & free
byte 1 = ~locked & autogen
byte 2 = ~locked & (PCAsteal - 1)
byte 3 = ~locked
Each bit position represents (modulo 8) a PTE. If it is 1, it is available for
allocation at its priority level, left to right.
Additionally, the PCA steal field in R10 has been rotated right one bit.
*/
cntlzw r6,r9 /* Allocate a slot */
mr r14,r12 /* Save our mapping for later */
cmplwi r6,32 /* Was there anything available? */
rlwinm r7,r6,29,30,31 /* Get the priority slot we got this from */
rlwinm r6,r6,0,29,31 ; Isolate bit position
srw r11,r4,r6 /* Position the PTEG control bits */
rlwinm r6,r6,0,29,31 /* Calculate the PTE slot mask */
andc r10,r10,r11 /* Turn off the free and auto bits */
beq- realFault /* Arghh, no slots! Take the long way 'round... */
/* Remember, we've already set up the mask pattern
depending upon how we got here:
if got here from simple mapping, R4=0x80000000,
if we got here from autogenit is 0x80800000. */
rlwinm r6,r6,3,26,28 /* Start calculating actual PTE address */
rlwinm. r11,r11,0,8,15 /* Isolate just the auto bit (remember about it too) */
add r6,r8,r6 /* Get position into PTEG control area */
cmplwi cr1,r7,1 /* Set the condition based upon the old PTE type */
sub r6,r6,r1 /* Switch it to the hash table */
or r10,r10,r11 /* Turn auto on if it is (PTEG control all set up now) */
subi r6,r6,1 /* Point right */
stw r10,PCAallo(r8) /* Allocate our slot */
dcbt br0,r6 ; Touch in the PTE
bne wasauto /* This was autogenned... */
stw r6,mmPTEent(r14) /* Link the mapping to the PTE slot */
/*
* So, now we're here and what exactly do we have? We've got:
* 1) a full PTE entry, both top and bottom words in R3 and R2
* 2) an allocated slot in the PTEG.
* 3) R8 still points to the PTEG Control Area (PCA)
* 4) R6 points to the PTE entry.
* 5) R1 contains length of the hash table-1. We use this to back-translate
* a PTE to a virtual address so we can invalidate TLBs.
* 6) R11 has a copy of the PCA controls we set.
* 7) R7 indicates what the PTE slot was before we got to it. 0 shows
* that it was empty, 1 that it was an autogen, and 2 or 3, that it was
* a non-autogen entry. CR1 is set to LT for empty, EQ for auto, and GT
* otherwise.
* 8) So far as our selected PTE, it should be valid if it was stolen
* and invalid if not. We could put some kind of assert here to
* check, but I think that I'd rather leave it in as a mysterious,
* non-reproducable bug.
* 9) The new PTE's mapping has been moved to the front of its PTEG hash list
* so that it's kept in some semblance of a MRU list.
* 10) R14 points to the mapping we're adding.
*
* So, what do we have to do yet?
* 1) If we stole a slot, we need to invalidate the PTE completely.
* 2) If we stole one AND it was not an autogen,
* copy the entire old PTE (including R and C bits) to its mapping.
* 3) Set the new PTE in the PTEG and make sure it is valid.
* 4) Unlock the PTEG control area.
* 5) Go back to the interrupt handler, changing the interrupt
* code to "in vain" which will restore the registers and bail out.
*
*/
wasauto: oris r3,r3,0x8000 /* Turn on the valid bit */
blt+ cr1,slamit /* It was empty, go slam it on in... */
lwz r10,0(r6) /* Grab the top part of the PTE */
rlwinm r12,r6,6,4,19 /* Match up the hash to a page boundary */
rlwinm r5,r10,5,4,19 /* Extract the VSID to a page boundary */
rlwinm r10,r10,0,1,31 /* Make it invalid */
xor r12,r5,r12 /* Calculate vaddr */
stw r10,0(r6) /* Invalidate the PTE */
rlwinm r5,r10,7,27,29 ; Move nybble 0 up to subhash position
rlwimi r12,r10,1,0,3 /* Move in the segment portion */
lis r9,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
xor r5,r5,r10 ; Splooch nybble 0 and 1
rlwimi r12,r10,22,4,9 /* Move in the API */
ori r9,r9,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
rlwinm r4,r10,27,27,29 ; Get low 3 bits of the VSID for look-aside hash
sync /* Make sure the invalid is stored */
xor r4,r4,r5 ; Finish splooching nybble 0, 1, and the low bits of the VSID
lwarx r5,0,r9 ; ?
tlbhang: lwarx r5,0,r9 /* Get the TLBIE lock */
rlwinm r4,r4,0,27,29 ; Clean up splooched hash value
mr. r5,r5 /* Is it locked? */
add r4,r4,r8 /* Point to the offset into the PCA area */
li r5,1 /* Get our lock word */
bne- tlbhang /* It's locked, go wait... */
la r4,PCAhash(r4) /* Point to the start of the hash chain for the PTE we're replacing */
stwcx. r5,0,r9 /* Try to get it */
bne- tlbhang /* We was beat... */
mfspr r7,pvr /* Find out what kind of machine we are */
li r5,0 /* Lock clear value */
rlwinm r7,r7,16,16,31 /* Isolate CPU type */
tlbie r12 /* Invalidate it everywhere */
cmplwi r7,3 /* Is this a 603? */
stw r5,0(r9) /* Clear the lock */
beq- its603 /* It's a 603, skip the tlbsync... */
eieio /* Make sure that the tlbie happens first */
tlbsync /* wait for everyone to catch up */
its603: sync /* Make sure of it all */
beq- cr1,slamit /* The old was an autogen, time to slam it in... */
lwz r9,4(r6) /* Get the real portion of old PTE */
lwz r7,0(r4) /* Get the first element. We can't get to here
if we aren't working with a mapping... */
mr r0,r7 ; Save pointer to first element
findold: mr r1,r11 ; Save the previous guy
mr. r11,r7 /* Copy and test the chain */
beq- bebad /* Assume it's not zero... */
lwz r5,mmPTEv(r11) /* See if this is the old active one */
cmplw cr2,r11,r14 /* Check if this is actually the new one */
cmplw r5,r10 /* Is this us? (Note: valid bit kept off in mappings) */
lwz r7,mmhashnext(r11) /* Get the next one in line */
beq- cr2,findold /* Don't count the new one... */
cmplw cr2,r11,r0 ; Check if we are first on the list
bne+ findold /* Not it (and assume the worst)... */
lwz r12,mmphysent(r11) /* Get the pointer to the physical entry */
beq- cr2,nomove ; We are first, no need to requeue...
stw r11,0(r4) ; Chain us to the head
stw r0,mmhashnext(r11) ; Chain the old head to us
stw r7,mmhashnext(r1) ; Unlink us
nomove: li r5,0 /* Clear this on out */
mr. r12,r12 /* Is there a physical entry? */
stw r5,mmPTEent(r11) ; Clear the PTE entry pointer
li r5,pepte1 /* Point to the PTE last half */
bne+ mrgmrc /* Yeah, merge RC into it... */
stw r9,mmPTEr(r11) ; Squirrel away the whole thing (RC bits are in here)
b slamit /* We've got the old saved... */
bebad: lis r0,HIGH_ADDR(Choke) /* We have a kernel choke!!! */
ori r0,r0,LOW_ADDR(Choke)
sc /* Firmware Heimlich maneuver */
.align 5
mrgmrc: rlwinm r11,r9,0,23,24 /* Keep only the RC bits */
lwarx r9,r7,r12 ; ?
mrgmrcx: lwarx r9,r5,r12 /* Get the master copy */
or r9,r9,r11 /* Merge in latest RC */
stwcx. r9,r5,r12 /* Save it back */
bne- mrgmrcx /* If it changed, try again... */
/*
* Here's where we finish up. We save the real part of the PTE, eieio it, to make sure it's
* out there before the top half (with the valid bit set).
*/
slamit: stw r2,4(r6) /* Stash the real part */
li r4,0 /* Get a lock clear value */
eieio /* Erect a barricade */
stw r3,0(r6) /* Stash the virtual part and set valid on */
stw r4,PCAlock(r8) /* Clear the PCA lock */
li r3,T_IN_VAIN /* Say that we handled it */
sync /* Go no further until the stores complete */
#if PERFTIMES && DEBUG
mflr r11
mr r4,r3
li r3,33
bl EXT(dbgLog2) ; Start of hw_add_map
mr r3,r4
mtlr r11
#endif
blr /* Back to the fold... */
/*
* This walks the hash table or DBATs to locate the physical address of a virtual one.
* The space is provided. If it is the kernel space, the DBATs are searched first. Failing
* that, the hash table is accessed. Zero is returned for failure, so it must be special cased.
* This is usually used for debugging, so we try not to rely
* on anything that we don't have to.
*/
ENTRY(LRA, TAG_NO_FRAME_USED)
mfmsr r10 /* Save the current MSR */
xoris r0,r3,HIGH_ADDR(PPC_SID_KERNEL) /* Clear the top half if equal */
andi. r9,r10,0x7FCF /* Turn off interrupts and translation */
eqv r12,r12,r12 /* Fill the bottom with foxes */
mtmsr r9 /* Cheat and turn off translation without an RFI.
This only works when instructions are in a V=R area. */
cmplwi r0,LOW_ADDR(PPC_SID_KERNEL) /* See if this is kernel space */
rlwinm r11,r3,6,6,25 /* Position the space for the VSID */
isync /* Purge pipe */
bne- notkernsp /* This is not for the kernel... */
mfspr r5,dbat0u /* Get the virtual address and length */
eqv r8,r8,r8 /* Get all foxes */
rlwinm. r0,r5,0,30,30 /* Check if valid for supervisor state */
rlwinm r7,r5,0,0,14 /* Clean up the base virtual address */
beq- ckbat1 /* not valid, skip this one... */
sub r7,r4,r7 /* Subtract out the base */
rlwimi r8,r5,15,0,14 /* Get area length - 1 */
mfspr r6,dbat0l /* Get the real part */
cmplw r7,r8 /* Check if it is in the range */
bng+ fndbat /* Yup, she's a good un... */
ckbat1: mfspr r5,dbat1u /* Get the virtual address and length */
eqv r8,r8,r8 /* Get all foxes */
rlwinm. r0,r5,0,30,30 /* Check if valid for supervisor state */
rlwinm r7,r5,0,0,14 /* Clean up the base virtual address */
beq- ckbat2 /* not valid, skip this one... */
sub r7,r4,r7 /* Subtract out the base */
rlwimi r8,r5,15,0,14 /* Get area length - 1 */
mfspr r6,dbat1l /* Get the real part */
cmplw r7,r8 /* Check if it is in the range */
bng+ fndbat /* Yup, she's a good un... */
ckbat2: mfspr r5,dbat2u /* Get the virtual address and length */
eqv r8,r8,r8 /* Get all foxes */
rlwinm. r0,r5,0,30,30 /* Check if valid for supervisor state */
rlwinm r7,r5,0,0,14 /* Clean up the base virtual address */
beq- ckbat3 /* not valid, skip this one... */
sub r7,r4,r7 /* Subtract out the base */
rlwimi r8,r5,15,0,14 /* Get area length - 1 */
mfspr r6,dbat2l /* Get the real part */
cmplw r7,r8 /* Check if it is in the range */
bng- fndbat /* Yup, she's a good un... */
ckbat3: mfspr r5,dbat3u /* Get the virtual address and length */
eqv r8,r8,r8 /* Get all foxes */
rlwinm. r0,r5,0,30,30 /* Check if valid for supervisor state */
rlwinm r7,r5,0,0,14 /* Clean up the base virtual address */
beq- notkernsp /* not valid, skip this one... */
sub r7,r4,r7 /* Subtract out the base */
rlwimi r8,r5,15,0,14 /* Get area length - 1 */
mfspr r6,dbat3l /* Get the real part */
cmplw r7,r8 /* Check if it is in the range */
bgt+ notkernsp /* No good... */
fndbat: rlwinm r6,r6,0,0,14 /* Clean up the real address */
mtmsr r10 /* Restore state */
add r3,r7,r6 /* Relocate the offset to real */
isync /* Purge pipe */
blr /* Bye, bye... */
notkernsp: mfspr r5,sdr1 /* Get hash table base and size */
rlwimi r11,r4,30,2,5 /* Insert the segment no. to make a VSID */
rlwimi r12,r5,16,0,15 /* Make table size -1 out of mask */
rlwinm r7,r4,26,10,25 /* Isolate the page index */
andc r5,r5,r12 /* Clean up the hash table */
xor r7,r7,r11 /* Get primary hash */
rlwinm r11,r11,1,1,24 /* Position VSID for pte ID */
and r7,r7,r12 /* Wrap the hash */
rlwimi r11,r4,10,26,31 /* Move API into pte ID */
add r5,r7,r5 /* Point to the PTEG */
oris r11,r11,0x8000 /* Slam on valid bit so's we don't match an invalid one */
li r9,8 /* Get the number of PTEs to check */
lwz r6,0(r5) /* Preload the virtual half */
fndpte: subi r9,r9,1 /* Count the pte */
lwz r3,4(r5) /* Get the real half */
cmplw cr1,r6,r11 /* Is this what we want? */
lwz r6,8(r5) /* Start to get the next virtual half */
mr. r9,r9 /* Any more to try? */
addi r5,r5,8 /* Bump to next slot */
beq cr1,gotxlate /* We found what we were looking for... */
bne+ fndpte /* Go try the next PTE... */
mtmsr r10 /* Restore state */
li r3,0 /* Show failure */
isync /* Purge pipe */
blr /* Leave... */
gotxlate: mtmsr r10 /* Restore state */
rlwimi r3,r4,0,20,31 /* Cram in the page displacement */
isync /* Purge pipe */
blr /* Return... */
/*
* hw_set_user_space(space) - Indicate whether memory space needs to be switched.
* We really need to turn off interrupts here, because we need to be non-preemptable
*/
.align 5
.globl EXT(hw_set_user_space)
LEXT(hw_set_user_space)
mfmsr r10 /* Get the current MSR */
rlwinm r9,r10,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Turn off 'rupts */
mtmsr r9 /* Disable 'em */
mfsprg r6,0 /* Get the per_proc_info address */
stw r3,PP_USERSPACE(r6) /* Show our new address space */
mtmsr r10 /* Restore interruptions */
blr /* Return... */
/* struct mapping *hw_cpv(struct mapping *mp) - Converts a physcial mapping CB address to virtual
*
*/
.align 5
.globl EXT(hw_cpv)
LEXT(hw_cpv)
mfmsr r10 ; Get the current MSR
rlwinm r4,r3,0,0,19 ; Round back to the mapping block allocation control block
andi. r9,r10,0x7FEF ; Turn off interrupts and data translation
mtmsr r9 ; Disable DR and EE
isync
lwz r4,mbvrswap(r4) ; Get the conversion value
mtmsr r10 ; Interrupts and DR back on
isync
xor r3,r3,r4 ; Convert to physical
rlwinm r3,r3,0,0,26 ; Clean out any flags
blr
/* struct mapping *hw_cvp(struct mapping *mp) - Converts a virtual mapping CB address to physcial
*
* Translation must be on for this
*
*/
.align 5
.globl EXT(hw_cvp)
LEXT(hw_cvp)
rlwinm r4,r3,0,0,19 ; Round back to the mapping block allocation control block
rlwinm r3,r3,0,0,26 ; Clean out any flags
lwz r4,mbvrswap(r4) ; Get the conversion value
xor r3,r3,r4 ; Convert to virtual
blr
/* int mapalc(struct mappingblok *mb) - Finds, allocates, and checks a free mapping entry in a block
*
* Lock must already be held on mapping block list
* returns 0 if all slots filled.
* returns n if a slot is found and it is not the last
* returns -n if a slot os found and it is the last
* when n and -n are returned, the corresponding bit is cleared
*
*/
.align 5
.globl EXT(mapalc)
LEXT(mapalc)
lwz r4,mbfree(r3) ; Get the first mask
lis r0,0x8000 ; Get the mask to clear the first free bit
lwz r5,mbfree+4(r3) ; Get the second mask
mr r12,r3 ; Save the return
cntlzw r8,r4 ; Get first free field
lwz r6,mbfree+8(r3) ; Get the third mask
srw. r9,r0,r8 ; Get bit corresponding to first free one
lwz r7,mbfree+12(r3) ; Get the fourth mask
cntlzw r10,r5 ; Get first free field in second word
andc r4,r4,r9 ; Turn it off
bne malcfnd0 ; Found one...
srw. r9,r0,r10 ; Get bit corresponding to first free one in second word
cntlzw r11,r6 ; Get first free field in third word
andc r5,r5,r9 ; Turn it off
bne malcfnd1 ; Found one...
srw. r9,r0,r11 ; Get bit corresponding to first free one in third word
cntlzw r10,r7 ; Get first free field in fourth word
andc r6,r6,r9 ; Turn it off
bne malcfnd2 ; Found one...
srw. r9,r0,r10 ; Get bit corresponding to first free one in second word
li r3,0 ; Assume abject failure
andc r7,r7,r9 ; Turn it off
beqlr ; There are none any left...
addi r3,r10,96 ; Set the correct bit number
stw r7,mbfree+12(r12) ; Actually allocate the slot
mapafin: or r4,r4,r5 ; Merge the first two allocation maps
or r6,r6,r7 ; Then the last two
or. r4,r4,r6 ; Merge both halves
bnelr+ ; Return if some left for next time...
neg r3,r3 ; Indicate we just allocated the last one
blr ; Leave...
malcfnd0: stw r4,mbfree(r12) ; Actually allocate the slot
mr r3,r8 ; Set the correct bit number
b mapafin ; Exit now...
malcfnd1: stw r5,mbfree+4(r12) ; Actually allocate the slot
addi r3,r10,32 ; Set the correct bit number
b mapafin ; Exit now...
malcfnd2: stw r6,mbfree+8(r12) ; Actually allocate the slot
addi r3,r11,64 ; Set the correct bit number
b mapafin ; Exit now...
/*
* Log out all memory usage
*/
.align 5
.globl EXT(logmem)
LEXT(logmem)
mfmsr r2 ; Get the MSR
lis r10,hi16(EXT(DebugWork)) ; High part of area
lis r12,hi16(EXT(mem_actual)) ; High part of actual
andi. r0,r10,0x7FCF ; Interrupts and translation off
ori r10,r10,lo16(EXT(DebugWork)) ; Get the entry
mtmsr r0 ; Turn stuff off
ori r12,r12,lo16(EXT(mem_actual)) ; Get the actual
li r0,1 ; Get a one
isync
stw r0,4(r10) ; Force logging off
lwz r0,0(r12) ; Get the end of memory
lis r12,hi16(EXT(mem_size)) ; High part of defined memory
ori r12,r12,lo16(EXT(mem_size)) ; Low part of defined memory
lwz r12,0(r12) ; Make it end of defined
cmplw r0,r12 ; Is there room for the data?
ble- logmemexit ; No, do not even try...
stw r12,0(r12) ; Set defined memory size
stw r0,4(r12) ; Set the actual amount of memory
lis r3,hi16(EXT(hash_table_base)) ; Hash table address
lis r4,hi16(EXT(hash_table_size)) ; Hash table size
lis r5,hi16(EXT(pmap_mem_regions)) ; Memory regions
lis r6,hi16(EXT(mapCtl)) ; Mappings
ori r3,r3,lo16(EXT(hash_table_base))
ori r4,r4,lo16(EXT(hash_table_size))
ori r5,r5,lo16(EXT(pmap_mem_regions))
ori r6,r6,lo16(EXT(mapCtl))
lwz r3,0(r3)
lwz r4,0(r4)
lwz r5,4(r5) ; Get the pointer to the phys_ent table
lwz r6,0(r6) ; Get the pointer to the current mapping block
stw r3,8(r12) ; Save the hash table address
stw r4,12(r12) ; Save the hash table size
stw r5,16(r12) ; Save the physent pointer
stw r6,20(r12) ; Save the mappings
addi r11,r12,0x1000 ; Point to area to move hash table and PCA
add r4,r4,r4 ; Double size for both
copyhash: lwz r7,0(r3) ; Copy both of them
lwz r8,4(r3)
lwz r9,8(r3)
lwz r10,12(r3)
subic. r4,r4,0x10
addi r3,r3,0x10
stw r7,0(r11)
stw r8,4(r11)
stw r9,8(r11)
stw r10,12(r11)
addi r11,r11,0x10
bgt+ copyhash
rlwinm r4,r12,20,12,31 ; Get number of phys_ents
copyphys: lwz r7,0(r5) ; Copy physents
lwz r8,4(r5)
subic. r4,r4,1
addi r5,r5,8
stw r7,0(r11)
stw r8,4(r11)
addi r11,r11,8
bgt+ copyphys
addi r11,r11,4095 ; Round up to next page
rlwinm r11,r11,0,0,19
lwz r4,4(r6) ; Get the size of the mapping area
copymaps: lwz r7,0(r6) ; Copy the mappings
lwz r8,4(r6)
lwz r9,8(r6)
lwz r10,12(r6)
subic. r4,r4,0x10
addi r6,r6,0x10
stw r7,0(r11)
stw r8,4(r11)
stw r9,8(r11)
stw r10,12(r11)
addi r11,r11,0x10
bgt+ copymaps
sub r11,r11,r12 ; Get the total length we saved
stw r11,24(r12) ; Save the size
logmemexit: mtmsr r2 ; Back to normal
li r3,0
isync
blr