![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to hw_vm.s CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple XNU] / XNU / osfmk / ppc|
Return to hw_vm.s CVS log | Up to [Apple XNU] / XNU / osfmk / ppc |
1.1 root 1: /*
2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3: *
4: * @APPLE_LICENSE_HEADER_START@
5: *
6: * The contents of this file constitute Original Code as defined in and
7: * are subject to the Apple Public Source License Version 1.1 (the
8: * "License"). You may not use this file except in compliance with the
9: * License. Please obtain a copy of the License at
10: * http://www.apple.com/publicsource and read it before using this file.
11: *
12: * This Original Code and all software distributed under the License are
13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17: * License for the specific language governing rights and limitations
18: * under the License.
19: *
20: * @APPLE_LICENSE_HEADER_END@
21: */
22: #include <assym.s>
23: #include <debug.h>
24: #include <cpus.h>
25: #include <db_machine_commands.h>
26: #include <mach_rt.h>
27:
28: #include <mach_debug.h>
29: #include <ppc/asm.h>
30: #include <ppc/proc_reg.h>
31: #include <ppc/exception.h>
32: #include <ppc/Performance.h>
33: #include <ppc/exception.h>
34: #include <ppc/pmap_internals.h>
35: #include <mach/ppc/vm_param.h>
36:
37: #define PERFTIMES 0
38:
39: .text
40:
41: /*
42: *
43: * Random notes and musings...
44: *
45: * Access to mappings via the PTEG hash must be done with the list locked.
46: * Access via the physical entries is controlled by the physent lock.
47: * Access to mappings is controlled by the PTEG lock once they are queued.
48: * If they are not on the list, they don't really exist, so
49: * only one processor at a time can find them, so no access control is needed.
50: *
51: * The second half of the PTE is kept in the physical entry. It is done this
52: * way, because there may be multiple mappings that refer to the same physical
53: * page (i.e., address aliases or synonymns). We must do it this way, because
54: * maintenance of the reference and change bits becomes nightmarish if each mapping
55: * has its own. One side effect of this, and not necessarily a bad one, is that
56: * all mappings for a single page can have a single WIMG, protection state, and RC bits.
57: * The only "bad" thing, is the reference bit. With a single copy, we can not get
58: * a completely accurate working set calculation, i.e., we can't tell which mapping was
59: * used to reference the page, all we can tell is that the physical page was
60: * referenced.
61: *
62: * The master copys of the reference and change bits are kept in the phys_entry.
63: * Other than the reference and change bits, changes to the phys_entry are not
64: * allowed if it has any mappings. The master reference and change bits must be
65: * changed via atomic update.
66: *
67: * Invalidating a PTE merges the RC bits into the phys_entry.
68: *
69: * Before checking the reference and/or bits, ALL mappings to the physical page are
70: * invalidated.
71: *
72: * PTEs are never explicitly validated, they are always faulted in. They are also
73: * not visible outside of the hw_vm modules. Complete seperation of church and state.
74: *
75: * Removal of a mapping is invalidates its PTE.
76: *
77: * So, how do we deal with mappings to I/O space? We don't have a physent for it.
78: * Within the mapping is a copy of the second half of the PTE. This is used
79: * ONLY when there is no physical entry. It is swapped into the PTE whenever
80: * it is built. There is no need to swap it back out, because RC is not
81: * maintained for these mappings.
82: *
83: * So, I'm starting to get concerned about the number of lwarx/stcwx loops in
84: * this. Satisfying a mapped address with no stealing requires one lock. If we
85: * steal an entry, there's two locks and an atomic update. Invalidation of an entry
86: * takes one lock and, if there is a PTE, another lock and an atomic update. Other
87: * operations are multiples (per mapping) of the above. Maybe we should look for
88: * an alternative. So far, I haven't found one, but I haven't looked hard.
89: */
90:
91:
92: /* hw_add_map(struct mapping *mp, space_t space, vm_offset_t va) - Adds a mapping
93: *
94: * Adds a mapping to the PTEG hash list.
95: *
96: * Interrupts must be disabled before calling.
97: *
98: * Using the space and the virtual address, we hash into the hash table
99: * and get a lock on the PTEG hash chain. Then we chain the
100: * mapping to the front of the list.
101: *
102: */
103:
104: .align 5
105: .globl EXT(hw_add_map)
106:
107: LEXT(hw_add_map)
108:
109: #if PERFTIMES && DEBUG
110: mr r7,r3
111: mflr r11
112: li r3,20
113: bl EXT(dbgLog2) ; Start of hw_add_map
114: mr r3,r7
115: mtlr r11
116: #endif
117:
118: mfmsr r0 /* Get the MSR */
119: eqv r6,r6,r6 /* Fill the bottom with foxes */
120: rlwinm r11,r4,6,6,25 /* Position the space for the VSID */
121: mfspr r10,sdr1 /* Get hash table base and size */
122: rlwimi r11,r5,30,2,5 /* Insert the segment no. to make a VSID */
123: rlwimi r6,r10,16,0,15 /* Make table size -1 out of mask */
124: rlwinm r7,r5,26,10,25 /* Isolate the page index */
125: or r8,r10,r6 /* Point to the last byte in table */
126: rlwinm r9,r5,4,0,3 ; Move nybble 1 up to 0
127: xor r7,r7,r11 /* Get primary hash */
128: andi. r12,r0,0x7FCF /* Disable translation and interruptions */
129: rlwinm r11,r11,1,1,24 /* Position VSID for pte ID */
130: addi r8,r8,1 /* Point to the PTEG Control Area */
131: xor r9,r9,r5 ; Splooch vaddr nybble 0 and 1 together
132: and r7,r7,r6 /* Wrap the hash */
133: rlwimi r11,r5,10,26,31 /* Move API into pte ID */
134: rlwinm r9,r9,6,27,29 ; Get splooched bits in place
135: add r8,r8,r7 /* Point to our PCA entry */
136: rlwinm r10,r4,2,27,29 ; Get low 3 bits of the VSID for look-aside hash
137: mtmsr r12 /* Get the stuff disabled */
138: la r4,PCAhash(r8) /* Point to the mapping hash area */
139: xor r9,r9,r10 ; Finish splooching nybble 0, 1, and the low bits of the VSID
140: isync /* Get rid of anything prefetched before we ref storage */
141: /*
142: * We've now got the address of our PCA, the hash chain anchor, our API subhash,
143: * and word 0 of the PTE (the virtual part).
144: *
145: * Now, we just lock the PCA.
146: */
147:
148: li r12,1 /* Get the locked value */
149: dcbt 0,r4 /* We'll need the hash area in a sec, so get it */
150: add r4,r4,r9 /* Point to the right mapping hash slot */
151:
152: lwarx r10,0,r8 ; ?
153:
154: ptegLckx: lwarx r10,0,r8 /* Get the PTEG lock */
155: mr. r10,r10 /* Is it locked? */
156: bne- ptegLckwx /* Yeah... */
157: stwcx. r12,0,r8 /* Take take it */
158: bne- ptegLckx /* Someone else was trying, try again... */
159: b ptegSXgx /* All done... */
160:
161: .align 4
162:
163: ptegLckwx: mr. r10,r10 /* Check if it's already held */
164: beq+ ptegLckx /* It's clear... */
165: lwz r10,0(r8) /* Get lock word again... */
166: b ptegLckwx /* Wait... */
167:
168: .align 4
169:
170: ptegSXgx: isync /* Make sure we haven't used anything yet */
171:
172: lwz r7,0(r4) /* Pick up the anchor of hash list */
173: stw r3,0(r4) /* Save the new head */
174: stw r7,mmhashnext(r3) /* Chain in the old head */
175:
176: stw r4,mmPTEhash(r3) /* Point to the head of the hash list */
177:
178: sync /* Make sure the chain is updated */
179: stw r10,0(r8) /* Unlock the hash list */
180: mtmsr r0 /* Restore translation and interruptions */
181: isync /* Toss anything doe with DAT off */
182: #if PERFTIMES && DEBUG
183: mflr r11
184: mr r4,r3
185: li r3,21
186: bl EXT(dbgLog2) ; end of hw_add_map
187: mr r3,r4
188: mtlr r11
189: #endif
190: blr /* Leave... */
191:
192:
193: /* mp=hw_lock_phys_vir(space, va) - Finds and locks a physical entry by vaddr.
194: *
195: * Returns the mapping with the associated physent locked if found, or a
196: * zero and no lock if not. It we timed out trying to get a the lock on
197: * the physical entry, we retun a 1. A physical entry can never be on an
198: * odd boundary, so we can distinguish between a mapping and a timeout code.
199: *
200: * Interrupts must be disabled before calling.
201: *
202: * Using the space and the virtual address, we hash into the hash table
203: * and get a lock on the PTEG hash chain. Then we search the chain for the
204: * mapping for our virtual address. From there, we extract the pointer to
205: * the physical entry.
206: *
207: * Next comes a bit of monkey business. we need to get a lock on the physical
208: * entry. But, according to our rules, we can't get it after we've gotten the
209: * PTEG hash lock, we could deadlock if we do. So, we need to release the
210: * hash lock. The problem is, though, that as soon as we release it, some
211: * other yahoo may remove our mapping between the time that we release the
212: * hash lock and obtain the phys entry lock. So, we can't count on the
213: * mapping once we release the lock. Instead, after we lock the phys entry,
214: * we search the mapping list (phys_link) for our translation. If we don't find it,
215: * we unlock the phys entry, bail out, and return a 0 for the mapping address. If we
216: * did find it, we keep the lock and return the address of the mapping block.
217: *
218: * What happens when a mapping is found, but there is no physical entry?
219: * This is what happens when there is I/O area mapped. It one of these mappings
220: * is found, the mapping is returned, as is usual for this call, but we don't
221: * try to lock anything. There could possibly be some problems here if another
222: * processor releases the mapping while we still alre using it. Hope this
223: * ain't gonna happen.
224: *
225: * Taaa-dahhh! Easy as pie, huh?
226: *
227: * So, we have a few hacks hacks for running translate off in here.
228: * First, when we call the lock routine, we have carnel knowlege of the registers is uses.
229: * That way, we don't need a stack frame, which we can't have 'cause the stack is in
230: * virtual storage. But wait, as if that's not enough... We need one more register. So,
231: * we cram the LR into the CTR and return from there.
232: *
233: */
234:
235: .align 5
236: .globl EXT(hw_lock_phys_vir)
237:
238: LEXT(hw_lock_phys_vir)
239:
240: #if PERFTIMES && DEBUG
241: mflr r11
242: mr r5,r3
243: li r3,22
244: bl EXT(dbgLog2) ; Start of hw_add_map
245: mr r3,r5
246: mtlr r11
247: #endif
248: mfmsr r12 /* Get the MSR */
249: eqv r6,r6,r6 /* Fill the bottom with foxes */
250: rlwinm r11,r3,6,6,25 /* Position the space for the VSID */
251: mfspr r5,sdr1 /* Get hash table base and size */
252: rlwimi r11,r4,30,2,5 /* Insert the segment no. to make a VSID */
253: rlwimi r6,r5,16,0,15 /* Make table size -1 out of mask */
254: andi. r0,r12,0x7FCF /* Disable translation and interruptions */
255: rlwinm r9,r4,4,0,3 ; Move nybble 1 up to 0
256: rlwinm r7,r4,26,10,25 /* Isolate the page index */
257: or r8,r5,r6 /* Point to the last byte in table */
258: xor r7,r7,r11 /* Get primary hash */
259: rlwinm r11,r11,1,1,24 /* Position VSID for pte ID */
260: addi r8,r8,1 /* Point to the PTEG Control Area */
261: xor r9,r9,r4 ; Splooch vaddr nybble 0 and 1 together
262: and r7,r7,r6 /* Wrap the hash */
263: rlwimi r11,r4,10,26,31 /* Move API into pte ID */
264: rlwinm r9,r9,6,27,29 ; Get splooched bits in place
265: add r8,r8,r7 /* Point to our PCA entry */
266: rlwinm r10,r3,2,27,29 ; Get low 3 bits of the VSID for look-aside hash
267: mtmsr r0 /* Get the trans and intr off */
268: la r3,PCAhash(r8) /* Point to the mapping hash area */
269: xor r9,r9,r10 ; Finish splooching nybble 0, 1, and the low bits of the VSID
270: isync /* Make sure translation is off before we ref storage */
271:
272: /*
273: * We've now got the address of our PCA, the hash chain anchor, our API subhash,
274: * and word 0 of the PTE (the virtual part).
275: *
276: * Now, we just lock the PCA and find our mapping, if it exists.
277: */
278:
279: dcbt 0,r3 /* We'll need the hash area in a sec, so get it */
280: add r3,r3,r9 /* Point to the right mapping hash slot */
281:
282: lwarx r10,0,r8 ; ?
283:
284: ptegLcka: lwarx r10,0,r8 /* Get the PTEG lock */
285: li r5,1 /* Get the locked value */
286: mr. r10,r10 /* Is it locked? */
287: bne- ptegLckwa /* Yeah... */
288: stwcx. r5,0,r8 /* Take take it */
289: bne- ptegLcka /* Someone else was trying, try again... */
290: b ptegSXga /* All done... */
291:
292: .align 4
293:
294: ptegLckwa: mr. r10,r10 /* Check if it's already held */
295: beq+ ptegLcka /* It's clear... */
296: lwz r10,0(r8) /* Get lock word again... */
297: b ptegLckwa /* Wait... */
298:
299: .align 4
300:
301: ptegSXga: isync /* Make sure we haven't used anything yet */
302:
303: mflr r0 /* Get the LR */
304: lwz r9,0(r3) /* Pick up the first mapping block */
305: mtctr r0 /* Stuff it into the CTR */
306:
307: findmapa:
308:
309: mr. r3,r9 /* Did we hit the end? */
310: bne+ chkmapa /* Nope... */
311:
312: stw r3,0(r8) /* Unlock the PTEG lock
313: Note: we never saved anything while we
314: had the lock, so we don't need a sync
315: before we unlock it */
316:
317: vbail: mtmsr r12 /* Restore translation and interruptions */
318: isync /* Make sure translation is cool */
319: #if PERFTIMES && DEBUG
320: mflr r11
321: mr r4,r3
322: li r3,23
323: bl EXT(dbgLog2) ; Start of hw_add_map
324: mr r3,r4
325: mtlr r11
326: #endif
327: bctr /* Return in abject failure... */
328:
329: .align 4
330:
331: chkmapa: lwz r10,mmPTEv(r3) /* Pick up our virtual ID */
332: lwz r9,mmhashnext(r3) /* Pick up next mapping block */
333: cmplw r10,r11 /* Have we found ourself? */
334: bne- findmapa /* Nope, still wandering... */
335:
336: lwz r9,mmphysent(r3) /* Get our physical entry pointer */
337: li r5,0 /* Clear this out */
338: mr. r9,r9 /* Is there, like, a physical entry? */
339: stw r5,0(r8) /* Unlock the PTEG lock
340: Note: we never saved anything while we
341: had the lock, so we don't need a sync
342: before we unlock it */
343:
344: beq- vbail /* If there is no physical entry, it's time
345: to leave... */
346:
347: /* Here we want to call hw_lock_bit. We don't want to use the stack, 'cause it's
348: * in virtual storage, and we're in real. So, we've carefully looked at the code
349: * in hw_lock_bit (and unlock) and cleverly don't use any of the registers that it uses.
350: * Be very, very aware of how you change this code. By the way, it uses:
351: * R0, R6, R7, R8, and R9. R3, R4, and R5 contain parameters
352: * Unfortunatly, we need to stash R9 still. So... Since we know we will not be interrupted
353: * ('cause we turned off interruptions and translation is off) we will use SPRG3...
354: */
355:
356: lwz r10,mmPTEhash(r3) /* Save the head of the hash-alike chain. We need it to find ourselves later */
357: lis r5,HIGH_ADDR(EXT(LockTimeOut)) /* Get address of timeout value */
358: la r3,pephyslink(r9) /* Point to the lock word */
359: ori r5,r5,LOW_ADDR(EXT(LockTimeOut)) /* Get second half of address */
360: li r4,PHYS_LOCK /* Get the lock bit value */
361: lwz r5,0(r5) /* Pick up the timeout value */
362: mtsprg 3,r9 /* Save R9 in SPRG3 */
363:
364: bl EXT(hw_lock_bit) /* Go do the lock */
365:
366: mfsprg r9,3 /* Restore pointer to the phys_entry */
367: mr. r3,r3 /* Did we timeout? */
368: lwz r4,pephyslink(r9) /* Pick up first mapping block */
369: beq- penterr /* Bad deal, we timed out... */
370:
371: rlwinm r4,r4,0,0,26 ; Clear out the flags from first link
372:
373: findmapb: mr. r3,r4 /* Did we hit the end? */
374: bne+ chkmapb /* Nope... */
375:
376: la r3,pephyslink(r9) /* Point to where the lock is */
377: li r4,PHYS_LOCK /* Get the lock bit value */
378: bl EXT(hw_unlock_bit) /* Go unlock the physentry */
379:
380: li r3,0 /* Say we failed */
381: b vbail /* Return in abject failure... */
382:
383: penterr: li r3,1 /* Set timeout */
384: b vbail /* Return in abject failure... */
385:
386: .align 5
387:
388: chkmapb: lwz r6,mmPTEv(r3) /* Pick up our virtual ID */
389: lwz r4,mmnext(r3) /* Pick up next mapping block */
390: cmplw r6,r11 /* Have we found ourself? */
391: lwz r5,mmPTEhash(r3) /* Get the start of our hash chain */
392: bne- findmapb /* Nope, still wandering... */
393: cmplw r5,r10 /* On the same hash chain? */
394: bne- findmapb /* Nope, keep looking... */
395:
396: b vbail /* Return in glorious triumph... */
397:
398:
399: /*
400: * hw_rem_map(mapping) - remove a mapping from the system.
401: *
402: * Upon entry, R3 contains a pointer to a mapping block and the associated
403: * physical entry is locked if there is one.
404: *
405: * If the mapping entry indicates that there is a PTE entry, we invalidate
406: * if and merge the reference and change information into the phys_entry.
407: *
408: * Next, we remove the mapping from the phys_ent and the PTEG hash list.
409: *
410: * Unlock any locks that are left, and exit.
411: *
412: * Note that this must be done with both interruptions off and VM off
413: *
414: * Note that this code depends upon the VSID being of the format 00SXXXXX
415: * where S is the segment number.
416: *
417: *
418: */
419:
420: .align 5
421: .globl EXT(hw_rem_map)
422:
423: LEXT(hw_rem_map)
424: #if PERFTIMES && DEBUG
425: mflr r11
426: mr r4,r3
427: li r3,24
428: bl EXT(dbgLog2) ; Start of hw_add_map
429: mr r3,r4
430: mtlr r11
431: #endif
432: mfmsr r0 /* Save the MSR */
433: rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
434: rlwinm r12,r12,0,28,25 /* Clear IR and DR */
435: mtmsr r12 /* Clear interruptions and turn off translation */
436: isync /* Make sure that translation is off */
437:
438: lwz r6,mmPTEhash(r3) /* Get pointer to hash list anchor */
439: lwz r5,mmPTEv(r3) /* Get the VSID */
440: dcbt 0,r6 /* We'll need that chain in a bit */
441:
442: rlwinm r7,r6,0,0,25 /* Round hash list down to PCA boundary */
443: li r12,1 /* Get the locked value */
444: subi r6,r6,mmhashnext /* Make the anchor look like an entry */
445:
446: lwarx r10,0,r7 ; ?
447:
448: ptegLck1: lwarx r10,0,r7 /* Get the PTEG lock */
449: mr. r10,r10 /* Is it locked? */
450: bne- ptegLckw1 /* Yeah... */
451: stwcx. r12,0,r7 /* Try to take it */
452: bne- ptegLck1 /* Someone else was trying, try again... */
453: b ptegSXg1 /* All done... */
454:
455: .align 4
456:
457: ptegLckw1: mr. r10,r10 /* Check if it's already held */
458: beq+ ptegLck1 /* It's clear... */
459: lwz r10,0(r7) /* Get lock word again... */
460: b ptegLckw1 /* Wait... */
461:
462: .align 4
463:
464: ptegSXg1: isync /* Make sure we haven't used anything yet */
465:
466: lwz r12,mmhashnext(r3) /* Prime with our forward pointer */
467: lwz r4,mmPTEent(r3) /* Get the pointer to the PTE now that the lock's set */
468:
469: srchmaps: mr. r10,r6 /* Save the previous entry */
470: bne+ mapok /* No error... */
471:
472: lis r0,HIGH_ADDR(Choke) /* We have a kernel choke!!! */
473: ori r0,r0,LOW_ADDR(Choke)
474: sc /* Firmware Heimlich manuever */
475:
476: .align 4
477:
478: mapok: lwz r6,mmhashnext(r6) /* Look at the next one */
479: cmplwi cr5,r4,0 /* Is there a PTE? */
480: cmplw r6,r3 /* Have we found ourselves? */
481: bne+ srchmaps /* Nope, get your head together... */
482:
483: stw r12,mmhashnext(r10) /* Remove us from the queue */
484: rlwinm r9,r5,1,0,3 /* Move in the segment */
485:
486: rlwinm r8,r4,6,4,19 /* Line PTEG disp up to a page */
487: rlwinm r11,r5,5,4,19 /* Line up the VSID */
488: lwz r10,mmphysent(r3) /* Point to the physical entry */
489:
490: beq+ cr5,nopte /* There's no PTE to invalidate... */
491:
492: xor r8,r8,r11 /* Back hash to virt index */
493: lis r12,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
494: rlwimi r9,r5,22,4,9 /* Move in the API */
495: ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
496: mfspr r11,pvr /* Find out what kind of machine we are */
497: rlwimi r9,r8,0,10,19 /* Create the virtual address */
498: rlwinm r11,r11,16,16,31 /* Isolate CPU type */
499:
500: stw r5,0(r4) /* Make the PTE invalid */
501:
502: cmplwi cr1,r11,3 /* Is this a 603? */
503: sync /* Make sure the invalid is stored */
504:
505: lwarx r5,0,r12 ; ?
506:
507: tlbhang1: lwarx r5,0,r12 /* Get the TLBIE lock */
508: rlwinm r11,r4,29,29,31 /* Get the bit position of entry */
509: mr. r5,r5 /* Is it locked? */
510: lis r6,0x8000 /* Start up a bit mask */
511: li r5,1 /* Get our lock word */
512: bne- tlbhang1 /* It's locked, go wait... */
513: stwcx. r5,0,r12 /* Try to get it */
514: bne- tlbhang1 /* We was beat... */
515:
516: srw r6,r6,r11 /* Make a "free slot" mask */
517: lwz r5,PCAallo(r7) /* Get the allocation control bits */
518: rlwinm r11,r6,24,8,15 /* Make the autogen bit to turn off */
519: or r5,r5,r6 /* turn on the free bit */
520: rlwimi r11,r11,24,16,23 /* Get lock bit mask to turn it off */
521:
522: andc r5,r5,r11 /* Turn off the lock and autogen bits in allocation flags */
523: li r11,0 /* Lock clear value */
524:
525: tlbie r9 /* Invalidate it everywhere */
526:
527:
528: beq- cr1,its603a /* It's a 603, skip the tlbsync... */
529:
530: eieio /* Make sure that the tlbie happens first */
531: tlbsync /* wait for everyone to catch up */
532:
533: its603a: sync /* Make sure of it all */
534: stw r11,0(r12) /* Clear the tlbie lock */
535: eieio /* Make sure those RC bit are loaded */
536: stw r5,PCAallo(r7) /* Show that the slot is free */
537: stw r11,mmPTEent(r3) /* Clear the pointer to the PTE */
538:
539: nopte: mr. r10,r10 /* See if there is a physical entry */
540: la r9,pephyslink(r10) /* Point to the physical mapping chain */
541: beq- nophys /* No physical entry, we're done... */
542: beq- cr5,nadamrg /* Not PTE to merge... */
543:
544: lwz r6,4(r4) /* Get the latest reference and change bits */
545: la r12,pepte1(r10) /* Point right at the master copy */
546: rlwinm r6,r6,0,23,24 /* Extract just the RC bits */
547:
548: lwarx r8,0,r12 ; ?
549:
550: mrgrc: lwarx r8,0,r12 /* Get the master copy */
551: or r8,r8,r6 /* Merge in latest RC */
552: stwcx. r8,0,r12 /* Save it back */
553: bne- mrgrc /* If it changed, try again... */
554:
555: nadamrg: li r11,0 /* Clear this out */
556: lwz r12,mmnext(r3) /* Prime with our next */
557: stw r11,0(r7) /* Unlock the hash chain now so we don't
558: lock out another processor during the
559: our next little search */
560:
561:
562: srchpmap: mr. r10,r9 /* Save the previous entry */
563: bne+ mapok1 /* No error... */
564:
565: lis r0,HIGH_ADDR(Choke) /* We have a kernel choke!!! */
566: ori r0,r0,LOW_ADDR(Choke)
567: sc /* Firmware Heimlich maneuver */
568:
569: .align 4
570:
571: mapok1: lwz r9,mmnext(r9) /* Look at the next one */
572: rlwinm r9,r9,0,0,26 ; Clear out the flags from first link
573: cmplw r9,r3 /* Have we found ourselves? */
574: bne+ srchpmap /* Nope, get your head together... */
575:
576: stw r12,mmnext(r10) /* Remove us from the queue */
577:
578: mtmsr r0 /* Interrupts and translation back on */
579: isync
580: #if PERFTIMES && DEBUG
581: mflr r11
582: li r3,25
583: bl EXT(dbgLog2) ; Start of hw_add_map
584: mtlr r11
585: #endif
586: blr /* Return... */
587:
588: .align 4
589:
590: nophys: li r4,0 /* Make sure this is 0 */
591: sync /* Make sure that chain is updated */
592: stw r4,0(r7) /* Unlock the hash chain */
593: mtmsr r0 /* Interrupts and translation back on */
594: isync
595: #if PERFTIMES && DEBUG
596: mflr r11
597: li r3,25
598: bl EXT(dbgLog2) ; Start of hw_add_map
599: mtlr r11
600: #endif
601: blr /* Return... */
602:
603:
604: /*
605: * hw_prot(physent, prot) - Change the protection of a physical page
606: *
607: * Upon entry, R3 contains a pointer to a physical entry which is locked.
608: * R4 contains the PPC protection bits.
609: *
610: * The first thing we do is to slam the new protection into the phys entry.
611: * Then we scan the mappings and process each one.
612: *
613: * Acquire the lock on the PTEG hash list for the mapping being processed.
614: *
615: * If the current mapping has a PTE entry, we invalidate
616: * it and merge the reference and change information into the phys_entry.
617: *
618: * Next, slam the protection bits into the entry and unlock the hash list.
619: *
620: * Note that this must be done with both interruptions off and VM off
621: *
622: *
623: */
624:
625: .align 5
626: .globl EXT(hw_prot)
627:
628: LEXT(hw_prot)
629: #if PERFTIMES && DEBUG
630: mflr r11
631: mr r7,r3
632: // lwz r5,4(r3)
633: li r5,0x1111
634: li r3,26
635: bl EXT(dbgLog2) ; Start of hw_add_map
636: mr r3,r7
637: mtlr r11
638: #endif
639: mfmsr r0 /* Save the MSR */
640: rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
641: li r5,pepte1 /* Get displacement to the second word of master pte */
642: rlwinm r12,r12,0,28,25 /* Clear IR and DR */
643:
644: mtmsr r12 /* Clear interruptions and turn off translation */
645: /* NOTE: we can get away with just turning the
646: instruction translation off here because we
647: are running with virtual = real. If not,
648: there would be an "implied branch", which wouldn't
649: be too good. */
650:
651: isync /* Make sure that translation is off */
652:
653: lwz r10,pephyslink(r3) /* Get the first mapping block */
654: rlwinm r10,r10,0,0,26 ; Clear out the flags from first link
655:
656: /*
657: * Note that we need to to do the interlocked update here because another processor
658: * can be updating the reference and change bits even though the physical entry
659: * is locked. All modifications to the PTE portion of the physical entry must be
660: * done via interlocked update.
661: */
662:
663: lwarx r8,r5,r3 ; ?
664:
665: protcng: lwarx r8,r5,r3 /* Get the master copy */
666: rlwimi r8,r4,0,30,31 /* Move in the protection bits */
667: stwcx. r8,r5,r3 /* Save it back */
668: bne- protcng /* If it changed, try again... */
669:
670:
671:
672: protnext: mr. r10,r10 /* Are there any more mappings? */
673: beq- protdone /* Naw... */
674:
675: lwz r7,mmPTEhash(r10) /* Get pointer to hash list anchor */
676: lwz r5,mmPTEv(r10) /* Get the virtual address */
677: rlwinm r7,r7,0,0,25 /* Round hash list down to PCA boundary */
678:
679: li r12,1 /* Get the locked value */
680:
681: lwarx r11,0,r7 ; ?
682:
683: protLck1: lwarx r11,0,r7 /* Get the PTEG lock */
684: mr. r11,r11 /* Is it locked? */
685: bne- protLckw1 /* Yeah... */
686: stwcx. r12,0,r7 /* Try to take it */
687: bne- protLck1 /* Someone else was trying, try again... */
688: b protSXg1 /* All done... */
689:
690: .align 4
691:
692: protLckw1: mr. r11,r11 /* Check if it's already held */
693: beq+ protLck1 /* It's clear... */
694: lwz r11,0(r7) /* Get lock word again... */
695: b protLckw1 /* Wait... */
696:
697: .align 4
698:
699: protSXg1: isync /* Make sure we haven't used anything yet */
700:
701: lwz r6,mmPTEent(r10) /* Get the pointer to the PTE now that the lock's set */
702:
703: rlwinm r9,r5,1,0,3 /* Move in the segment */
704: mr. r6,r6 /* See if there is a PTE here */
705: rlwinm r8,r5,31,2,25 /* Line it up and check if empty */
706:
707: beq+ protul /* There's no PTE to invalidate... */
708:
709: xor r8,r8,r6 /* Back hash to virt index */
710: rlwimi r9,r5,22,4,9 /* Move in the API */
711: lis r12,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
712: rlwinm r5,r5,0,1,31 /* Clear the valid bit */
713: ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
714: mfspr r11,pvr /* Find out what kind of machine we are */
715: rlwimi r9,r8,6,10,19 /* Create the virtual address */
716: rlwinm r11,r11,16,16,31 /* Isolate CPU type */
717:
718: stw r5,0(r6) /* Make the PTE invalid */
719: cmplwi cr1,r11,3 /* Is this a 603? */
720: sync /* Make sure the invalid is stored */
721:
722: lwarx r11,0,r12 ; ?
723:
724: tlbhangp: lwarx r11,0,r12 /* Get the TLBIE lock */
725: rlwinm r8,r6,29,29,31 /* Get the bit position of entry */
726: mr. r11,r11 /* Is it locked? */
727: lis r5,0x8000 /* Start up a bit mask */
728: li r11,1 /* Get our lock word */
729: bne- tlbhangp /* It's locked, go wait... */
730: stwcx. r11,0,r12 /* Try to get it */
731: bne- tlbhangp /* We was beat... */
732:
733: li r11,0 /* Lock clear value */
734:
735: tlbie r9 /* Invalidate it everywhere */
736:
737: beq- cr1,its603p /* It's a 603, skip the tlbsync... */
738:
739: eieio /* Make sure that the tlbie happens first */
740: tlbsync /* wait for everyone to catch up */
741:
742: its603p: stw r11,0(r12) /* Clear the lock */
743: srw r5,r5,r8 /* Make a "free slot" mask */
744: sync /* Make sure of it all */
745:
746: lwz r6,4(r6) /* Get the latest reference and change bits */
747: stw r11,mmPTEent(r10) /* Clear the pointer to the PTE */
748: rlwinm r6,r6,0,23,24 /* Extract the RC bits */
749: lwz r9,PCAallo(r7) /* Get the allocation control bits */
750: rlwinm r8,r5,24,8,15 /* Make the autogen bit to turn off */
751: or r9,r9,r5 /* Set the slot free */
752: rlwimi r8,r8,24,16,23 /* Get lock bit mask to turn it off */
753: andc r9,r9,r8 /* Clear the auto and lock bits */
754: li r5,pepte1 /* Get displacement to the second word of master pte */
755: stw r9,PCAallo(r7) /* Store the allocation controls */
756:
757: lwarx r11,r5,r3 ; ?
758:
759: protmod: lwarx r11,r5,r3 /* Get the master copy */
760: or r11,r11,r6 /* Merge in latest RC */
761: stwcx. r11,r5,r3 /* Save it back */
762: bne- protmod /* If it changed, try again... */
763:
764: sync /* Make sure that chain is updated */
765:
766: protul: li r4,0 /* Get a 0 */
767: lwz r10,mmnext(r10) /* Get the next */
768: stw r4,0(r7) /* Unlock the hash chain */
769: b protnext /* Go get the next one */
770:
771: .align 4
772:
773: protdone: mtmsr r0 /* Interrupts and translation back on */
774: isync
775: #if PERFTIMES && DEBUG
776: mflr r11
777: li r3,27
778: bl EXT(dbgLog2) ; Start of hw_add_map
779: mtlr r11
780: #endif
781: blr /* Return... */
782:
783:
784: /*
785: * hw_pte_comm(physent) - Do something to the PTE pointing to a physical page
786: *
787: * Upon entry, R3 contains a pointer to a physical entry which is locked.
788: * Note that this must be done with both interruptions off and VM off
789: *
790: * First, we set up CRs 5 and 7 to indicate which of the 7 calls this is.
791: *
792: * Now we scan the mappings to invalidate any with and active PTE.
793: *
794: * Acquire the lock on the PTEG hash list for the mapping being processed.
795: *
796: * If the current mapping has a PTE entry, we invalidate
797: * it and merge the reference and change information into the phys_entry.
798: *
799: * Next, unlock the hash list and go on to the next mapping.
800: *
801: *
802: *
803: */
804:
805: .align 5
806: .globl EXT(hw_inv_all)
807:
808: LEXT(hw_inv_all)
809:
810: li r9,0x800 /* Indicate invalidate all */
811: b hw_pte_comm /* Join in the fun... */
812:
813:
814: .align 5
815: .globl EXT(hw_tst_mod)
816:
817: LEXT(hw_tst_mod)
818:
819: lwz r8,pepte1(r3) ; Get the saved PTE image
820: li r9,0x400 /* Indicate test modify */
821: rlwinm. r8,r8,25,31,31 ; Make change bit into return code
822: beq+ hw_pte_comm ; Assume we do not know if it is set...
823: mr r3,r8 ; Set the return code
824: blr ; Return quickly...
825:
826: .align 5
827: .globl EXT(hw_tst_ref)
828:
829: LEXT(hw_tst_ref)
830: lwz r8,pepte1(r3) ; Get the saved PTE image
831: li r9,0x200 /* Indicate test reference bit */
832: rlwinm. r8,r8,24,31,31 ; Make reference bit into return code
833: beq+ hw_pte_comm ; Assume we do not know if it is set...
834: mr r3,r8 ; Set the return code
835: blr ; Return quickly...
836:
837: /*
838: * Note that the following are all in one CR for ease of use later
839: */
840: .align 4
841: .globl EXT(hw_set_mod)
842:
843: LEXT(hw_set_mod)
844:
845: li r9,0x008 /* Indicate set modify bit */
846: b hw_pte_comm /* Join in the fun... */
847:
848:
849: .align 4
850: .globl EXT(hw_clr_mod)
851:
852: LEXT(hw_clr_mod)
853:
854: li r9,0x004 /* Indicate clear modify bit */
855: b hw_pte_comm /* Join in the fun... */
856:
857:
858: .align 4
859: .globl EXT(hw_set_ref)
860:
861: LEXT(hw_set_ref)
862:
863: li r9,0x002 /* Indicate set reference */
864: b hw_pte_comm /* Join in the fun... */
865:
866: .align 5
867: .globl EXT(hw_clr_ref)
868:
869: LEXT(hw_clr_ref)
870:
871: li r9,0x001 /* Indicate clear reference bit */
872: b hw_pte_comm /* Join in the fun... */
873:
874:
875: /*
876: * This is the common stuff.
877: */
878:
879: .align 5
880:
881: hw_pte_comm: /* Common routine for pte tests and manips */
882:
883: #if PERFTIMES && DEBUG
884: mflr r11
885: mr r7,r3
886: lwz r4,4(r3)
887: mr r5,r9
888: li r3,28
889: bl EXT(dbgLog2) ; Start of hw_add_map
890: mr r3,r7
891: mtlr r11
892: #endif
893: lwz r10,pephyslink(r3) /* Get the first mapping block */
894: mfmsr r0 /* Save the MSR */
895: rlwinm. r10,r10,0,0,26 ; Clear out the flags from first link and see if we are mapped
896: rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
897: mtcrf 0x05,r9 /* Set the call type flags into cr5 and 7 */
898: rlwinm r12,r12,0,28,25 /* Clear IR and DR */
899: beq- comnmap ; No mapping
900: dcbt br0,r10 ; Touch the first mapping in before the isync
901:
902: comnmap: mtmsr r12 /* Clear interruptions and turn off translation */
903: /* NOTE: we can get away with just turning the
904: instruction translation off here because we
905: are running with virtual equals real. If not,
906: there would be an "implied branch", which wouldn't
907: be too good. */
908:
909: isync /* Make sure that translation is off */
910: beq- commdone ; Nothing us mapped to this page...
911: b commnext ; Jump to first pass (jump here so we can align loop)
912:
913: .align 5
914:
915: commnext: lwz r11,mmnext(r10) ; Get the pointer to the next mapping (if any)
916: lwz r7,mmPTEhash(r10) /* Get pointer to hash list anchor */
917: lwz r5,mmPTEv(r10) /* Get the virtual address */
918: mr. r11,r11 ; More mappings to go?
919: rlwinm r7,r7,0,0,25 /* Round hash list down to PCA boundary */
920: beq- commnxtch ; No more mappings...
921: dcbt br0,r11 ; Touch the next mapping
922:
923: commnxtch: li r12,1 /* Get the locked value */
924:
925: lwarx r11,0,r7 ; ?
926:
927: commLck1: lwarx r11,0,r7 /* Get the PTEG lock */
928: mr. r11,r11 /* Is it locked? */
929: bne- commLckw1 /* Yeah... */
930: stwcx. r12,0,r7 /* Try to take it */
931: bne- commLck1 /* Someone else was trying, try again... */
932: b commSXg1 /* All done... */
933:
934: .align 4
935:
936: commLckw1: mr. r11,r11 /* Check if it's already held */
937: beq+ commLck1 /* It's clear... */
938: lwz r11,0(r7) /* Get lock word again... */
939: b commLckw1 /* Wait... */
940:
941: .align 4
942:
943: commSXg1: isync /* Make sure we haven't used anything yet */
944:
945: lwz r6,mmPTEent(r10) /* Get the pointer to the PTE now that the lock's set */
946:
947: rlwinm r9,r5,1,0,3 /* Move in the segment */
948: mr. r6,r6 /* See if there is a PTE entry here */
949: rlwinm r8,r5,31,2,25 /* Line it up and check if empty */
950:
951: beq+ commul /* There's no PTE to invalidate... */
952:
953: xor r8,r8,r6 /* Back hash to virt index */
954: rlwimi r9,r5,22,4,9 /* Move in the API */
955: lis r12,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
956: rlwinm r5,r5,0,1,31 /* Clear the valid bit */
957: ori r12,r12,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
958: rlwimi r9,r8,6,10,19 /* Create the virtual address */
959:
960: stw r5,0(r6) /* Make the PTE invalid */
961: mfspr r4,pvr /* Find out what kind of machine we are */
962: sync /* Make sure the invalid is stored */
963:
964: lwarx r11,0,r12 ; ?
965:
966: tlbhangco: lwarx r11,0,r12 /* Get the TLBIE lock */
967: rlwinm r8,r6,29,29,31 /* Get the bit position of entry */
968: mr. r11,r11 /* Is it locked? */
969: lis r5,0x8000 /* Start up a bit mask */
970: li r11,1 /* Get our lock word */
971: bne- tlbhangco /* It's locked, go wait... */
972: stwcx. r11,0,r12 /* Try to get it */
973: bne- tlbhangco /* We was beat... */
974:
975: rlwinm r4,r4,16,16,31 /* Isolate CPU type */
976: li r11,0 /* Lock clear value */
977: cmplwi r4,3 /* Is this a 603? */
978:
979: tlbie r9 /* Invalidate it everywhere */
980:
981: beq- its603co /* It's a 603, skip the tlbsync... */
982:
983: eieio /* Make sure that the tlbie happens first */
984: tlbsync /* wait for everyone to catch up */
985:
986: its603co: stw r11,0(r12) /* Clear the lock */
987: srw r5,r5,r8 /* Make a "free slot" mask */
988: sync /* Make sure of it all */
989:
990: lwz r6,4(r6) /* Get the latest reference and change bits */
991: lwz r9,PCAallo(r7) /* Get the allocation control bits */
992: stw r11,mmPTEent(r10) /* Clear the pointer to the PTE */
993: rlwinm r8,r5,24,8,15 /* Make the autogen bit to turn off */
994: or r9,r9,r5 /* Set the slot free */
995: rlwimi r8,r8,24,16,23 /* Get lock bit mask to turn it off */
996: rlwinm r4,r6,0,23,24 /* Extract the RC bits */
997: andc r9,r9,r8 /* Clear the auto and lock bits */
998: li r5,pepte1 /* Get displacement to the second word of master pte */
999: stw r9,PCAallo(r7) /* Store the allocation controls */
1000:
1001: lwarx r11,r5,r3 ; ?
1002:
1003: commmod: lwarx r11,r5,r3 /* Get the master copy */
1004: or r11,r11,r4 /* Merge in latest RC */
1005: stwcx. r11,r5,r3 /* Save it back */
1006: bne- commmod /* If it changed, try again... */
1007:
1008: sync /* Make sure that chain is updated */
1009:
1010: commul: lwz r10,mmnext(r10) /* Get the next */
1011: li r4,0 /* Make sure this is 0 */
1012: mr. r10,r10 ; Is there another mapping?
1013: stw r4,0(r7) /* Unlock the hash chain */
1014: bne+ commnext ; Go get the next if there is one...
1015:
1016: /*
1017: * Now that all PTEs have been invalidated and the master RC bits are updated,
1018: * we go ahead and figure out what the original call was and do that. Note that
1019: * another processor could be messing around and may have entered one of the
1020: * PTEs we just removed into the hash table. Too bad... You takes yer chances.
1021: * If there's a problem with that, it's because some higher level was trying to
1022: * do something with a mapping that it shouldn't. So, the problem's really
1023: * there, nyaaa, nyaaa, nyaaa... nyaaa, nyaaa... nyaaa! So there!
1024: */
1025:
1026: commdone: li r5,pepte1 /* Get displacement to the second word of master pte */
1027: blt cr5,commfini /* We're finished, it was invalidate all... */
1028: bgt cr5,commtst /* It was a test modified... */
1029: beq cr5,commtst /* It was a test reference... */
1030:
1031: /*
1032: * Note that we need to to do the interlocked update here because another processor
1033: * can be updating the reference and change bits even though the physical entry
1034: * is locked. All modifications to the PTE portion of the physical entry must be
1035: * done via interlocked update.
1036: */
1037:
1038: lwarx r8,r5,r3 ; ?
1039:
1040: commcng: lwarx r8,r5,r3 /* Get the master copy */
1041:
1042: bng cr7,commclrr /* Jump if not clear change bit... */
1043: rlwinm r8,r8,0,25,23 /* Clear the change bit */
1044: b commsave /* Go save the new version... */
1045:
1046: .align 4
1047:
1048: commclrr: bns cr7,commsetm /* Jump away if not clear reference... */
1049: rlwinm r8,r8,0,24,22 /* Clear the change bit */
1050: b commsave /* Go save the new version... */
1051:
1052: .align 4
1053:
1054: commsetm: bnl cr7,commsetr /* Jump away if not set modified... */
1055: ori r8,r8,0x0080 /* Set the modified bit */
1056: b commsave /* Go save the new version... */
1057:
1058: .align 4
1059:
1060: commsetr: ori r8,r8,0x0100 /* Set the referenced bit */
1061:
1062: commsave: stwcx. r8,r5,r3 /* Save it back */
1063: bne- commcng /* If it changed, try again... */
1064:
1065: mtmsr r0 /* Interrupts and translation back on */
1066: isync
1067: #if PERFTIMES && DEBUG
1068: mflr r11
1069: mr r4,r3
1070: li r3,29
1071: bl EXT(dbgLog2) ; Start of hw_add_map
1072: mr r3,r4
1073: mtlr r11
1074: #endif
1075: blr /* Return... */
1076:
1077: .align 4
1078:
1079: commtst: lwz r8,pepte1(r3) /* Get the PTE */
1080: bne- cr5,commtcb ; This is for the change bit...
1081: mtmsr r0 ; Interrupts and translation back on
1082: rlwinm r3,r8,24,31,31 ; Copy reference bit to bit 31
1083: isync ; Toss prefetching
1084: #if PERFTIMES && DEBUG
1085: mflr r11
1086: mr r4,r3
1087: li r3,29
1088: bl EXT(dbgLog2) ; Start of hw_add_map
1089: mr r3,r4
1090: mtlr r11
1091: #endif
1092: blr ; Return...
1093:
1094: .align 4
1095:
1096: commtcb: rlwinm r3,r8,25,31,31 ; Copy change bit to bit 31
1097:
1098: commfini: mtmsr r0 ; Interrupts and translation back on
1099: isync ; Toss prefetching
1100:
1101: #if PERFTIMES && DEBUG
1102: mflr r11
1103: mr r4,r3
1104: li r3,29
1105: bl EXT(dbgLog2) ; Start of hw_add_map
1106: mr r3,r4
1107: mtlr r11
1108: #endif
1109: blr ; Return...
1110:
1111: /*
1112: * hw_phys_attr(struct phys_entry *pp, vm_prot_t prot, unsigned int wimg) - Sets the default physical page attributes
1113: *
1114: * Note that this must be done with both interruptions off and VM off
1115: * Move the passed in attributes into the pte image in the phys entry
1116: *
1117: *
1118: */
1119:
1120: .align 5
1121: .globl EXT(hw_phys_attr)
1122:
1123: LEXT(hw_phys_attr)
1124:
1125: #if PERFTIMES && DEBUG
1126: mflr r11
1127: mr r8,r3
1128: mr r7,r5
1129: mr r5,r4
1130: // lwz r4,4(r3)
1131: li r4,0x1111
1132: li r3,30
1133: bl EXT(dbgLog2) ; Start of hw_add_map
1134: mr r3,r8
1135: mr r4,r5
1136: mr r5,r7
1137: mtlr r11
1138: #endif
1139: mfmsr r0 /* Save the MSR */
1140: andi. r5,r5,0x0078 /* Clean up the WIMG */
1141: rlwinm r12,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
1142: rlwimi r5,r4,0,30,31 /* Move the protection into the wimg register */
1143: la r6,pepte1(r3) /* Point to the default pte */
1144: rlwinm r12,r12,0,28,25 /* Clear IR and DR */
1145:
1146: mtmsr r12 /* Clear interruptions and turn off translation */
1147: /* NOTE: we can get away with just turning the
1148: instruction translation off here because we
1149: are running with virtual equals real. If not,
1150: there would be an "implied branch", which wouldn't
1151: be too good. */
1152:
1153: isync /* Make sure that translation is off */
1154:
1155: lwarx r10,0,r6 ; ?
1156:
1157: atmattr: lwarx r10,0,r6 /* Get the pte */
1158: rlwimi r10,r5,0,25,31 /* Move in the new attributes */
1159: stwcx. r10,0,r6 /* Try it on for size */
1160: bne- atmattr /* Someone else was trying, try again... */
1161:
1162: mtmsr r0 /* Interrupts and translation back on */
1163: isync
1164: #if PERFTIMES && DEBUG
1165: mflr r11
1166: mr r4,r10
1167: li r3,31
1168: bl EXT(dbgLog2) ; Start of hw_add_map
1169: mtlr r11
1170: #endif
1171: blr /* All done... */
1172:
1173:
1174:
1175: /*
1176: * handlePF - handle a page fault interruption
1177: *
1178: * If the fault can be handled, this routine will RFI directly,
1179: * otherwise it will return with all registers as in entry.
1180: *
1181: * Upon entry, state and all registers have been saved in savearea.
1182: * This is pointed to by R13.
1183: * IR and DR are off, interrupts are masked,
1184: * Floating point be disabled.
1185: * R3 is the interrupt code.
1186: *
1187: * If we bail, we must restore cr5, and all registers except 6 and
1188: * 3.
1189: *
1190: */
1191:
1192: .align 5
1193: .globl EXT(handlePF)
1194:
1195: LEXT(handlePF)
1196:
1197: /*
1198: * This first part does a quick check to see if we can handle the fault.
1199: * We can't handle any kind of protection exceptions here, so we pass
1200: * them up to the next level.
1201: *
1202: * The mapping lists are kept in MRS (most recently stolen)
1203: * order on queues anchored within from the
1204: * PTEG to which the virtual address hashes. This is further segregated by
1205: * the low-order 3 bits of the VSID XORed with the segment number and XORed
1206: * with bits 4-7 of the vaddr in an attempt to keep the searches
1207: * short.
1208: *
1209: * MRS is handled by moving the entry to the head of its list when stolen in the
1210: * assumption that it will be revalidated soon. Entries are created on the head
1211: * of the list because they will be used again almost immediately.
1212: *
1213: * We need R13 set to the savearea, R3 set to the interrupt code, and R2
1214: * set to the per_proc.
1215: *
1216: * NOTE: In order for a page-fault redrive to work, the translation miss
1217: * bit must be set in the DSISR (or SRR1 for IFETCH). That must occur
1218: * before we come here.
1219: */
1220:
1221: cmplwi r3,T_INSTRUCTION_ACCESS /* See if this is for the instruction */
1222: lwz r8,savesrr1(r13) ; Get the MSR to determine mode
1223: beq- gotIfetch ; We have an IFETCH here...
1224:
1225: lwz r7,savedsisr(r13) /* Get the DSISR */
1226: lwz r6,savedar(r13) /* Get the fault address */
1227: b ckIfProt ; Go check if this is a protection fault...
1228:
1229: gotIfetch: mr r7,r8 ; IFETCH info is in SRR1
1230: lwz r6,savesrr0(r13) /* Get the instruction address */
1231:
1232: ckIfProt: rlwinm. r7,r7,0,1,1 ; Is this a protection exception?
1233: beqlr- ; Yes... (probably not though)
1234:
1235: /*
1236: * We will need to restore registers if we bail after this point.
1237: * Note that at this point several SRs have been changed to the kernel versions.
1238: * Therefore, for these we must build these values.
1239: */
1240:
1241: #if PERFTIMES && DEBUG
1242: mflr r11
1243: mr r5,r6
1244: mr r4,r3
1245: li r3,32
1246: bl EXT(dbgLog2) ; Start of hw_add_map
1247: mr r3,r4
1248: mtlr r11
1249: mfsprg r2,0
1250: #endif
1251: lwz r3,PP_USERSPACE(r2) ; Get the user space
1252: rlwinm. r8,r8,0,MSR_PR_BIT,MSR_PR_BIT ; Supervisor state access?
1253: eqv r1,r1,r1 /* Fill the bottom with foxes */
1254: rlwimi r3,r6,24,8,11 ; Insert the segment number into the VSID to make the user VSID
1255: bne+ notsuper ; If user access, we have the VSID already...
1256:
1257: mfsrin r3,r6 ; Get the faulting segment reg for kernel accesses
1258:
1259: notsuper: mfspr r5,sdr1 /* Get hash table base and size */
1260: rlwinm r9,r6,2,2,5 ; Move nybble 1 up to 0 (keep aligned with VSID)
1261: rlwinm r3,r3,6,2,25 /* Position the space for the VSID */
1262: rlwimi r1,r5,16,0,15 /* Make table size -1 out of mask */
1263: rlwinm r7,r6,26,10,25 /* Isolate the page index */
1264: xor r9,r9,r3 ; Splooch vaddr nybble 0 (from VSID) and 1 together
1265: or r8,r5,r1 /* Point to the last byte in table */
1266: xor r7,r7,r3 /* Get primary hash */
1267: rlwinm r3,r3,1,1,24 /* Position VSID for pte ID */
1268: addi r8,r8,1 /* Point to the PTEG Control Area */
1269: rlwinm r9,r9,8,27,29 ; Get splooched bits in place
1270: and r7,r7,r1 /* Wrap the hash */
1271: rlwimi r3,r6,10,26,31 /* Move API into pte ID */
1272: add r8,r8,r7 /* Point to our PCA entry */
1273: rlwinm r12,r3,27,27,29 ; Get low 3 bits of the VSID for look-aside hash
1274: la r11,PCAhash(r8) /* Point to the mapping hash area */
1275: xor r9,r9,r12 ; Finish splooching nybble 0, 1, and the low bits of the VSID
1276:
1277:
1278: /*
1279: * We have about as much as we need to start searching the autogen
1280: * and mappings. From here on, any kind of failure will bail, and
1281: * contention will either bail or restart from here.
1282: *
1283: *
1284: */
1285:
1286: li r12,1 /* Get the locked value */
1287: dcbt 0,r11 /* We'll need the hash area in a sec, so get it */
1288: add r11,r11,r9 /* Point to the right mapping hash slot */
1289:
1290: lwarx r10,0,r8 ; ?
1291:
1292: ptegLck: lwarx r10,0,r8 /* Get the PTEG lock */
1293: mr. r10,r10 /* Is it locked? */
1294: bne- ptegLckw /* Yeah... */
1295: stwcx. r12,0,r8 /* Take take it */
1296: bne- ptegLck /* Someone else was trying, try again... */
1297: b ptegSXg /* All done... */
1298:
1299: .align 4
1300:
1301: ptegLckw: mr. r10,r10 /* Check if it's already held */
1302: beq+ ptegLck /* It's clear... */
1303: lwz r10,0(r8) /* Get lock word again... */
1304: b ptegLckw /* Wait... */
1305:
1306: .align 5
1307:
1308: nop ; Force ISYNC to last instruction in IFETCH
1309: nop
1310: nop
1311:
1312: ptegSXg: isync /* Make sure we haven't used anything yet */
1313:
1314: lwz r9,0(r11) /* Pick up first mapping block */
1315: mr r5,r11 /* Get the address of the anchor */
1316: mr r7,r9 /* Save the first in line */
1317: b findmap ; Take space and force loop to cache line
1318:
1319: findmap: mr. r12,r9 /* Are there more? */
1320: beq- tryAuto /* Nope, nothing in mapping list for us... */
1321:
1322: lwz r10,mmPTEv(r12) /* Get unique PTE identification */
1323: lwz r9,mmhashnext(r12) /* Get the chain, just in case */
1324: cmplw r10,r3 /* Did we hit our PTE? */
1325: lwz r6,mmPTEent(r12) /* Get the pointer to the hash table entry */
1326: mr r5,r12 /* Save the current as previous */
1327: bne- findmap ; Nothing here, try the next...
1328:
1329: ; Cache line bounary here
1330:
1331: cmplwi cr1,r6,0 /* Is there actually a PTE entry in the hash? */
1332: lwz r2,mmphysent(r12) /* Get the physical entry */
1333: bne- cr1,MustBeOK /* There's an entry in the hash table, so, this must
1334: have been taken care of already... */
1335: cmplwi cr2,r2,0 /* Is there a physical entry? */
1336: li r0,0x0100 /* Force on the reference bit whenever we make a PTE valid */
1337: addi r2,r2,pepte1 /* Point to the second half of PTE in physent */
1338: lis r4,0x8000 /* Tell, PTE inserter that this wasn't an auto */
1339: bne+ cr2,gotphys /* Skip down if we have a physical entry */
1340: la r2,mmPTEr(r12) /* Point to the PTE in the mapping */
1341: li r0,0x0180 /* When there is no physical entry, force on
1342: both R and C bits to keep hardware from
1343: updating the PTE to set them. We don't
1344: keep track of RC for I/O areas, so this is ok */
1345:
1346: gotphys: lwz r2,0(r2) /* Get the second part of the PTE */
1347: b insert /* Go insert into the PTEG... */
1348:
1349: MustBeOK: li r10,0 /* Get lock clear value */
1350: li r3,T_IN_VAIN /* Say that we handled it */
1351: stw r10,PCAlock(r8) /* Clear the PTEG lock */
1352: sync
1353: #if PERFTIMES && DEBUG
1354: mflr r11
1355: mr r4,r3
1356: li r3,33
1357: bl EXT(dbgLog2) ; Start of hw_add_map
1358: mr r3,r4
1359: mtlr r11
1360: #endif
1361: blr /* Blow back and handle exception */
1362:
1363:
1364:
1365: /*
1366: * We couldn't find it in the mapping list. As a last try, we will
1367: * see if we can autogen it.
1368: *
1369: * An autogen area is defined as a contiguous range of virtual addresses that
1370: * map to a fixed contiguous area of physical storage. This implimentation
1371: * does not support an area that is not in the kernal address space. We may
1372: * want to consider doing that later. The number and size of autogen areas
1373: * are not defined, but it would be silly to define them for one or two pages
1374: * or hundreds of individual areas.
1375: *
1376: * These areas are accessed at this level read-only without locks. Higher
1377: * levels can add an autogen area to the front of the list by moving the anchor
1378: * to the forward pointer and then doing a compare-and-swap to set the new
1379: * anchor. Removals are tougher. They shouldn't happen frequently, if at all.
1380: * The only problem is at the moment of the breaking of the chain. It is possible
1381: * that another processor is using the element being removed. So, unless we know
1382: * for a fact that the element is unused, we can't delete it. So, removal works
1383: * like this: first, the we lock the chain at the higher level, to insure that
1384: * no more than one processor is deleteing elements. Next, we move the forward
1385: * pointer from the removee to the forward of its previous. (we find the previous
1386: * under the lock.) Note that the removee's forward still points to the next.
1387: * The lock on the chain can be released now.
1388: *
1389: * At this point, a second processor could be traversing the list. If it hasn't
1390: * reached the removee yet, it can't see it because to it, it has been removed.
1391: * If it happens to be at the removed one, it will process it and go to the next,
1392: * It is up to the remover to insure that the there is enough time for the other
1393: * processor(s) to finish. What it does is to send a processor synchronize
1394: * SIGP to all other CPU and wait until they have been received. Since all
1395: * code that traverses the list is uninterruptable, the SYNC command can't
1396: * occur until all CPUs are done with the removee.
1397: *
1398: * The autogen list is ordered ascending. By relaxing the
1399: * "must-link-at-head" restriction, and requiring a lock for addition,
1400: * we can order the updates of the links such that a
1401: * non-interlocked search can proceed during the update. We
1402: * get a zero-cost early bail-out from the autogen search, and the
1403: * additional overhead from maintanence is nearly nil.
1404: *
1405: * The one problem we have here is autogenned execute protected areas, we can
1406: * end up with a persistant ISI. So, the solution is just say NO. No to execute
1407: * protected autogen areas.
1408: */
1409:
1410: .align 4
1411:
1412: tryAuto: lis r10,HIGH_ADDR(EXT(AutoGenList)) /* Get the top part of autogen list anchor */
1413: rlwinm. r11,r3,0,5,24 /* Check if we are in the kernel */
1414: ori r10,r10,LOW_ADDR(EXT(AutoGenList)) /* Get the bottom part */
1415: beq- checkAuto /* In kernel, see if we can autogen... */
1416:
1417: /*
1418: * When we come here, we know that we can't handle this. Restore whatever
1419: * state that we trashed and go back to continue handling the interrupt.
1420: */
1421:
1422: realFault: li r10,0 /* Get lock clear value */
1423: lwz r3,saveexception(r13) /* Figure out the exception code again */
1424: stw r10,PCAlock(r8) /* Clear the PTEG lock */
1425: #if PERFTIMES && DEBUG
1426: mflr r11
1427: mr r4,r3
1428: li r3,33
1429: bl EXT(dbgLog2) ; Start of hw_add_map
1430: mr r3,r4
1431: mtlr r11
1432: #endif
1433: blr /* Blow back and handle exception */
1434:
1435: .align 4
1436:
1437: checkAuto: lwz r10,0(r10) /* Get the first autogen area */
1438:
1439: findAuto: mr. r11,r10 /* Are there more? */
1440: beq- realFault /* Nope, no autogen, it's a fault... */
1441:
1442: lwz r4,AGNstart(r11) /* Get start of range */
1443: lwz r9,AGNsize(r11) /* Get the size */
1444: sub. r5,r6,r4 /* Get distance into region */
1445: lwz r10,AGNnext(r11) /* Assure we'll need the next one */
1446: blt- realFault /* Bail out when we're past any possible hit */
1447: cmplw r5,r9 /* See if we fit in the range. This works even
1448: on an unordered list. If the start address
1449: is after the address under test, the subtract
1450: will give us a negative number, but with a
1451: logical test, it will be a bigger number */
1452: lwz r2,AGNpteX(r11) /* Get translation--should be in cache */
1453: bgt+ findAuto /* Not in this slot, try again... */
1454:
1455: lis r4,0x8080 /* Indicate that this was autogened */
1456: li r0,0x0180 /* Autogenned areas always set RC bits.
1457: This keeps the hardware from having
1458: to do two storage writes */
1459:
1460: /*
1461: * Here where we insert the PTE into the hash. The PTE image is in R3, R2.
1462: * The PTEG allocation controls are a bit map of the state of the PTEG. The
1463: * PCAlock bits are a temporary lock for the specified PTE. PCAfree indicates that
1464: * the PTE slot is empty. PCAauto means that it comes from an autogen area. These
1465: * guys do not keep track of reference and change and are actually "wired".
1466: * They're easy to maintain and are second in priority for a steal. PCAsteal
1467: * is a sliding position mask used to "randomize" PTE slot stealing. All 4 of these
1468: * fields fit in a single word and are loaded and stored under control of the
1469: * PTEG control area lock (PCAlock).
1470: *
1471: * Physically, the fields are arranged:
1472: * 0: PCAfree
1473: * 1: PCAauto
1474: * 2: PCAlock
1475: * 3: PCAsteal
1476: */
1477:
1478: insert: lwz r10,PCAallo(r8) /* Get the PTEG controls */
1479: eqv r6,r6,r6 /* Get all ones */
1480: mr r11,r10 /* Make a copy */
1481: rlwimi r6,r10,8,16,23 /* Insert sliding steal position */
1482: rlwimi r11,r11,24,24,31 /* Duplicate the locked field */
1483: addi r6,r6,-256 /* Form mask */
1484: rlwimi r11,r11,16,0,15 /* This gives us a quadrupled lock mask */
1485: rlwinm r5,r10,31,24,0 /* Slide over the mask for next time */
1486: mr r9,r10 /* Make a copy to test */
1487: not r11,r11 /* Invert the quadrupled lock */
1488: or r2,r2,r0 /* Force on R, and maybe C bit */
1489: and r9,r9,r11 /* Remove the locked guys */
1490: rlwimi r5,r5,8,24,24 /* Wrap bottom bit to top in mask */
1491: rlwimi r9,r11,0,16,31 /* Put two copies of the unlocked entries at the end */
1492: rlwimi r10,r5,0,24,31 /* Move steal map back in */
1493: and r9,r9,r6 /* Set the starting point for stealing */
1494:
1495: /* So, now we have in R9:
1496: byte 0 = ~locked & free
1497: byte 1 = ~locked & autogen
1498: byte 2 = ~locked & (PCAsteal - 1)
1499: byte 3 = ~locked
1500:
1501: Each bit position represents (modulo 8) a PTE. If it is 1, it is available for
1502: allocation at its priority level, left to right.
1503:
1504: Additionally, the PCA steal field in R10 has been rotated right one bit.
1505: */
1506:
1507:
1508: cntlzw r6,r9 /* Allocate a slot */
1509: mr r14,r12 /* Save our mapping for later */
1510: cmplwi r6,32 /* Was there anything available? */
1511: rlwinm r7,r6,29,30,31 /* Get the priority slot we got this from */
1512: rlwinm r6,r6,0,29,31 ; Isolate bit position
1513: srw r11,r4,r6 /* Position the PTEG control bits */
1514: rlwinm r6,r6,0,29,31 /* Calculate the PTE slot mask */
1515: andc r10,r10,r11 /* Turn off the free and auto bits */
1516: beq- realFault /* Arghh, no slots! Take the long way 'round... */
1517:
1518: /* Remember, we've already set up the mask pattern
1519: depending upon how we got here:
1520: if got here from simple mapping, R4=0x80000000,
1521: if we got here from autogenit is 0x80800000. */
1522:
1523: rlwinm r6,r6,3,26,28 /* Start calculating actual PTE address */
1524: rlwinm. r11,r11,0,8,15 /* Isolate just the auto bit (remember about it too) */
1525: add r6,r8,r6 /* Get position into PTEG control area */
1526: cmplwi cr1,r7,1 /* Set the condition based upon the old PTE type */
1527: sub r6,r6,r1 /* Switch it to the hash table */
1528: or r10,r10,r11 /* Turn auto on if it is (PTEG control all set up now) */
1529: subi r6,r6,1 /* Point right */
1530: stw r10,PCAallo(r8) /* Allocate our slot */
1531: dcbt br0,r6 ; Touch in the PTE
1532: bne wasauto /* This was autogenned... */
1533:
1534: stw r6,mmPTEent(r14) /* Link the mapping to the PTE slot */
1535:
1536: /*
1537: * So, now we're here and what exactly do we have? We've got:
1538: * 1) a full PTE entry, both top and bottom words in R3 and R2
1539: * 2) an allocated slot in the PTEG.
1540: * 3) R8 still points to the PTEG Control Area (PCA)
1541: * 4) R6 points to the PTE entry.
1542: * 5) R1 contains length of the hash table-1. We use this to back-translate
1543: * a PTE to a virtual address so we can invalidate TLBs.
1544: * 6) R11 has a copy of the PCA controls we set.
1545: * 7) R7 indicates what the PTE slot was before we got to it. 0 shows
1546: * that it was empty, 1 that it was an autogen, and 2 or 3, that it was
1547: * a non-autogen entry. CR1 is set to LT for empty, EQ for auto, and GT
1548: * otherwise.
1549: * 8) So far as our selected PTE, it should be valid if it was stolen
1550: * and invalid if not. We could put some kind of assert here to
1551: * check, but I think that I'd rather leave it in as a mysterious,
1552: * non-reproducable bug.
1553: * 9) The new PTE's mapping has been moved to the front of its PTEG hash list
1554: * so that it's kept in some semblance of a MRU list.
1555: * 10) R14 points to the mapping we're adding.
1556: *
1557: * So, what do we have to do yet?
1558: * 1) If we stole a slot, we need to invalidate the PTE completely.
1559: * 2) If we stole one AND it was not an autogen,
1560: * copy the entire old PTE (including R and C bits) to its mapping.
1561: * 3) Set the new PTE in the PTEG and make sure it is valid.
1562: * 4) Unlock the PTEG control area.
1563: * 5) Go back to the interrupt handler, changing the interrupt
1564: * code to "in vain" which will restore the registers and bail out.
1565: *
1566: */
1567:
1568: wasauto: oris r3,r3,0x8000 /* Turn on the valid bit */
1569: blt+ cr1,slamit /* It was empty, go slam it on in... */
1570:
1571: lwz r10,0(r6) /* Grab the top part of the PTE */
1572: rlwinm r12,r6,6,4,19 /* Match up the hash to a page boundary */
1573: rlwinm r5,r10,5,4,19 /* Extract the VSID to a page boundary */
1574: rlwinm r10,r10,0,1,31 /* Make it invalid */
1575: xor r12,r5,r12 /* Calculate vaddr */
1576: stw r10,0(r6) /* Invalidate the PTE */
1577: rlwinm r5,r10,7,27,29 ; Move nybble 0 up to subhash position
1578: rlwimi r12,r10,1,0,3 /* Move in the segment portion */
1579: lis r9,HIGH_ADDR(EXT(tlb_system_lock)) /* Get the TLBIE lock */
1580: xor r5,r5,r10 ; Splooch nybble 0 and 1
1581: rlwimi r12,r10,22,4,9 /* Move in the API */
1582: ori r9,r9,LOW_ADDR(EXT(tlb_system_lock)) /* Grab up the bottom part */
1583: rlwinm r4,r10,27,27,29 ; Get low 3 bits of the VSID for look-aside hash
1584:
1585: sync /* Make sure the invalid is stored */
1586:
1587: xor r4,r4,r5 ; Finish splooching nybble 0, 1, and the low bits of the VSID
1588:
1589: lwarx r5,0,r9 ; ?
1590:
1591: tlbhang: lwarx r5,0,r9 /* Get the TLBIE lock */
1592:
1593: rlwinm r4,r4,0,27,29 ; Clean up splooched hash value
1594:
1595: mr. r5,r5 /* Is it locked? */
1596: add r4,r4,r8 /* Point to the offset into the PCA area */
1597: li r5,1 /* Get our lock word */
1598: bne- tlbhang /* It's locked, go wait... */
1599:
1600: la r4,PCAhash(r4) /* Point to the start of the hash chain for the PTE we're replacing */
1601:
1602: stwcx. r5,0,r9 /* Try to get it */
1603: bne- tlbhang /* We was beat... */
1604:
1605: mfspr r7,pvr /* Find out what kind of machine we are */
1606: li r5,0 /* Lock clear value */
1607: rlwinm r7,r7,16,16,31 /* Isolate CPU type */
1608:
1609: tlbie r12 /* Invalidate it everywhere */
1610:
1611: cmplwi r7,3 /* Is this a 603? */
1612: stw r5,0(r9) /* Clear the lock */
1613:
1614: beq- its603 /* It's a 603, skip the tlbsync... */
1615:
1616: eieio /* Make sure that the tlbie happens first */
1617: tlbsync /* wait for everyone to catch up */
1618:
1619: its603: sync /* Make sure of it all */
1620:
1621: beq- cr1,slamit /* The old was an autogen, time to slam it in... */
1622:
1623: lwz r9,4(r6) /* Get the real portion of old PTE */
1624: lwz r7,0(r4) /* Get the first element. We can't get to here
1625: if we aren't working with a mapping... */
1626: mr r0,r7 ; Save pointer to first element
1627:
1628: findold: mr r1,r11 ; Save the previous guy
1629: mr. r11,r7 /* Copy and test the chain */
1630: beq- bebad /* Assume it's not zero... */
1631:
1632: lwz r5,mmPTEv(r11) /* See if this is the old active one */
1633: cmplw cr2,r11,r14 /* Check if this is actually the new one */
1634: cmplw r5,r10 /* Is this us? (Note: valid bit kept off in mappings) */
1635: lwz r7,mmhashnext(r11) /* Get the next one in line */
1636: beq- cr2,findold /* Don't count the new one... */
1637: cmplw cr2,r11,r0 ; Check if we are first on the list
1638: bne+ findold /* Not it (and assume the worst)... */
1639:
1640: lwz r12,mmphysent(r11) /* Get the pointer to the physical entry */
1641: beq- cr2,nomove ; We are first, no need to requeue...
1642:
1643: stw r11,0(r4) ; Chain us to the head
1644: stw r0,mmhashnext(r11) ; Chain the old head to us
1645: stw r7,mmhashnext(r1) ; Unlink us
1646:
1647: nomove: li r5,0 /* Clear this on out */
1648:
1649: mr. r12,r12 /* Is there a physical entry? */
1650: stw r5,mmPTEent(r11) ; Clear the PTE entry pointer
1651: li r5,pepte1 /* Point to the PTE last half */
1652:
1653: bne+ mrgmrc /* Yeah, merge RC into it... */
1654:
1655: stw r9,mmPTEr(r11) ; Squirrel away the whole thing (RC bits are in here)
1656: b slamit /* We've got the old saved... */
1657:
1658:
1659: bebad: lis r0,HIGH_ADDR(Choke) /* We have a kernel choke!!! */
1660: ori r0,r0,LOW_ADDR(Choke)
1661: sc /* Firmware Heimlich maneuver */
1662:
1663: .align 5
1664:
1665: mrgmrc: rlwinm r11,r9,0,23,24 /* Keep only the RC bits */
1666:
1667: lwarx r9,r7,r12 ; ?
1668:
1669: mrgmrcx: lwarx r9,r5,r12 /* Get the master copy */
1670: or r9,r9,r11 /* Merge in latest RC */
1671: stwcx. r9,r5,r12 /* Save it back */
1672: bne- mrgmrcx /* If it changed, try again... */
1673:
1674: /*
1675: * Here's where we finish up. We save the real part of the PTE, eieio it, to make sure it's
1676: * out there before the top half (with the valid bit set).
1677: */
1678:
1679: slamit: stw r2,4(r6) /* Stash the real part */
1680: li r4,0 /* Get a lock clear value */
1681: eieio /* Erect a barricade */
1682: stw r3,0(r6) /* Stash the virtual part and set valid on */
1683:
1684: stw r4,PCAlock(r8) /* Clear the PCA lock */
1685:
1686: li r3,T_IN_VAIN /* Say that we handled it */
1687: sync /* Go no further until the stores complete */
1688: #if PERFTIMES && DEBUG
1689: mflr r11
1690: mr r4,r3
1691: li r3,33
1692: bl EXT(dbgLog2) ; Start of hw_add_map
1693: mr r3,r4
1694: mtlr r11
1695: #endif
1696: blr /* Back to the fold... */
1697:
1698:
1699: /*
1700: * This walks the hash table or DBATs to locate the physical address of a virtual one.
1701: * The space is provided. If it is the kernel space, the DBATs are searched first. Failing
1702: * that, the hash table is accessed. Zero is returned for failure, so it must be special cased.
1703: * This is usually used for debugging, so we try not to rely
1704: * on anything that we don't have to.
1705: */
1706:
1707: ENTRY(LRA, TAG_NO_FRAME_USED)
1708:
1709: mfmsr r10 /* Save the current MSR */
1710: xoris r0,r3,HIGH_ADDR(PPC_SID_KERNEL) /* Clear the top half if equal */
1711: andi. r9,r10,0x7FCF /* Turn off interrupts and translation */
1712: eqv r12,r12,r12 /* Fill the bottom with foxes */
1713: mtmsr r9 /* Cheat and turn off translation without an RFI.
1714: This only works when instructions are in a V=R area. */
1715: cmplwi r0,LOW_ADDR(PPC_SID_KERNEL) /* See if this is kernel space */
1716: rlwinm r11,r3,6,6,25 /* Position the space for the VSID */
1717: isync /* Purge pipe */
1718: bne- notkernsp /* This is not for the kernel... */
1719:
1720: mfspr r5,dbat0u /* Get the virtual address and length */
1721: eqv r8,r8,r8 /* Get all foxes */
1722: rlwinm. r0,r5,0,30,30 /* Check if valid for supervisor state */
1723: rlwinm r7,r5,0,0,14 /* Clean up the base virtual address */
1724: beq- ckbat1 /* not valid, skip this one... */
1725: sub r7,r4,r7 /* Subtract out the base */
1726: rlwimi r8,r5,15,0,14 /* Get area length - 1 */
1727: mfspr r6,dbat0l /* Get the real part */
1728: cmplw r7,r8 /* Check if it is in the range */
1729: bng+ fndbat /* Yup, she's a good un... */
1730:
1731: ckbat1: mfspr r5,dbat1u /* Get the virtual address and length */
1732: eqv r8,r8,r8 /* Get all foxes */
1733: rlwinm. r0,r5,0,30,30 /* Check if valid for supervisor state */
1734: rlwinm r7,r5,0,0,14 /* Clean up the base virtual address */
1735: beq- ckbat2 /* not valid, skip this one... */
1736: sub r7,r4,r7 /* Subtract out the base */
1737: rlwimi r8,r5,15,0,14 /* Get area length - 1 */
1738: mfspr r6,dbat1l /* Get the real part */
1739: cmplw r7,r8 /* Check if it is in the range */
1740: bng+ fndbat /* Yup, she's a good un... */
1741:
1742: ckbat2: mfspr r5,dbat2u /* Get the virtual address and length */
1743: eqv r8,r8,r8 /* Get all foxes */
1744: rlwinm. r0,r5,0,30,30 /* Check if valid for supervisor state */
1745: rlwinm r7,r5,0,0,14 /* Clean up the base virtual address */
1746: beq- ckbat3 /* not valid, skip this one... */
1747: sub r7,r4,r7 /* Subtract out the base */
1748: rlwimi r8,r5,15,0,14 /* Get area length - 1 */
1749: mfspr r6,dbat2l /* Get the real part */
1750: cmplw r7,r8 /* Check if it is in the range */
1751: bng- fndbat /* Yup, she's a good un... */
1752:
1753: ckbat3: mfspr r5,dbat3u /* Get the virtual address and length */
1754: eqv r8,r8,r8 /* Get all foxes */
1755: rlwinm. r0,r5,0,30,30 /* Check if valid for supervisor state */
1756: rlwinm r7,r5,0,0,14 /* Clean up the base virtual address */
1757: beq- notkernsp /* not valid, skip this one... */
1758: sub r7,r4,r7 /* Subtract out the base */
1759: rlwimi r8,r5,15,0,14 /* Get area length - 1 */
1760: mfspr r6,dbat3l /* Get the real part */
1761: cmplw r7,r8 /* Check if it is in the range */
1762: bgt+ notkernsp /* No good... */
1763:
1764: fndbat: rlwinm r6,r6,0,0,14 /* Clean up the real address */
1765: mtmsr r10 /* Restore state */
1766: add r3,r7,r6 /* Relocate the offset to real */
1767: isync /* Purge pipe */
1768: blr /* Bye, bye... */
1769:
1770: notkernsp: mfspr r5,sdr1 /* Get hash table base and size */
1771: rlwimi r11,r4,30,2,5 /* Insert the segment no. to make a VSID */
1772: rlwimi r12,r5,16,0,15 /* Make table size -1 out of mask */
1773: rlwinm r7,r4,26,10,25 /* Isolate the page index */
1774: andc r5,r5,r12 /* Clean up the hash table */
1775: xor r7,r7,r11 /* Get primary hash */
1776: rlwinm r11,r11,1,1,24 /* Position VSID for pte ID */
1777: and r7,r7,r12 /* Wrap the hash */
1778: rlwimi r11,r4,10,26,31 /* Move API into pte ID */
1779: add r5,r7,r5 /* Point to the PTEG */
1780: oris r11,r11,0x8000 /* Slam on valid bit so's we don't match an invalid one */
1781:
1782: li r9,8 /* Get the number of PTEs to check */
1783: lwz r6,0(r5) /* Preload the virtual half */
1784:
1785: fndpte: subi r9,r9,1 /* Count the pte */
1786: lwz r3,4(r5) /* Get the real half */
1787: cmplw cr1,r6,r11 /* Is this what we want? */
1788: lwz r6,8(r5) /* Start to get the next virtual half */
1789: mr. r9,r9 /* Any more to try? */
1790: addi r5,r5,8 /* Bump to next slot */
1791: beq cr1,gotxlate /* We found what we were looking for... */
1792: bne+ fndpte /* Go try the next PTE... */
1793:
1794: mtmsr r10 /* Restore state */
1795: li r3,0 /* Show failure */
1796: isync /* Purge pipe */
1797: blr /* Leave... */
1798:
1799: gotxlate: mtmsr r10 /* Restore state */
1800: rlwimi r3,r4,0,20,31 /* Cram in the page displacement */
1801: isync /* Purge pipe */
1802: blr /* Return... */
1803:
1804:
1805:
1806: /*
1807: * hw_set_user_space(space) - Indicate whether memory space needs to be switched.
1808: * We really need to turn off interrupts here, because we need to be non-preemptable
1809: */
1810:
1811:
1812: .align 5
1813: .globl EXT(hw_set_user_space)
1814:
1815: LEXT(hw_set_user_space)
1816:
1817: mfmsr r10 /* Get the current MSR */
1818: rlwinm r9,r10,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Turn off 'rupts */
1819: mtmsr r9 /* Disable 'em */
1820: mfsprg r6,0 /* Get the per_proc_info address */
1821: stw r3,PP_USERSPACE(r6) /* Show our new address space */
1822: mtmsr r10 /* Restore interruptions */
1823: blr /* Return... */
1824:
1825:
1826: /* struct mapping *hw_cpv(struct mapping *mp) - Converts a physcial mapping CB address to virtual
1827: *
1828: */
1829:
1830: .align 5
1831: .globl EXT(hw_cpv)
1832:
1833: LEXT(hw_cpv)
1834:
1835: mfmsr r10 ; Get the current MSR
1836: rlwinm r4,r3,0,0,19 ; Round back to the mapping block allocation control block
1837: andi. r9,r10,0x7FEF ; Turn off interrupts and data translation
1838: mtmsr r9 ; Disable DR and EE
1839: isync
1840:
1841: lwz r4,mbvrswap(r4) ; Get the conversion value
1842: mtmsr r10 ; Interrupts and DR back on
1843: isync
1844: xor r3,r3,r4 ; Convert to physical
1845: rlwinm r3,r3,0,0,26 ; Clean out any flags
1846: blr
1847:
1848:
1849: /* struct mapping *hw_cvp(struct mapping *mp) - Converts a virtual mapping CB address to physcial
1850: *
1851: * Translation must be on for this
1852: *
1853: */
1854:
1855: .align 5
1856: .globl EXT(hw_cvp)
1857:
1858: LEXT(hw_cvp)
1859:
1860: rlwinm r4,r3,0,0,19 ; Round back to the mapping block allocation control block
1861: rlwinm r3,r3,0,0,26 ; Clean out any flags
1862: lwz r4,mbvrswap(r4) ; Get the conversion value
1863: xor r3,r3,r4 ; Convert to virtual
1864: blr
1865:
1866:
1867: /* int mapalc(struct mappingblok *mb) - Finds, allocates, and checks a free mapping entry in a block
1868: *
1869: * Lock must already be held on mapping block list
1870: * returns 0 if all slots filled.
1871: * returns n if a slot is found and it is not the last
1872: * returns -n if a slot os found and it is the last
1873: * when n and -n are returned, the corresponding bit is cleared
1874: *
1875: */
1876:
1877: .align 5
1878: .globl EXT(mapalc)
1879:
1880: LEXT(mapalc)
1881:
1882: lwz r4,mbfree(r3) ; Get the first mask
1883: lis r0,0x8000 ; Get the mask to clear the first free bit
1884: lwz r5,mbfree+4(r3) ; Get the second mask
1885: mr r12,r3 ; Save the return
1886: cntlzw r8,r4 ; Get first free field
1887: lwz r6,mbfree+8(r3) ; Get the third mask
1888: srw. r9,r0,r8 ; Get bit corresponding to first free one
1889: lwz r7,mbfree+12(r3) ; Get the fourth mask
1890: cntlzw r10,r5 ; Get first free field in second word
1891: andc r4,r4,r9 ; Turn it off
1892: bne malcfnd0 ; Found one...
1893:
1894: srw. r9,r0,r10 ; Get bit corresponding to first free one in second word
1895: cntlzw r11,r6 ; Get first free field in third word
1896: andc r5,r5,r9 ; Turn it off
1897: bne malcfnd1 ; Found one...
1898:
1899: srw. r9,r0,r11 ; Get bit corresponding to first free one in third word
1900: cntlzw r10,r7 ; Get first free field in fourth word
1901: andc r6,r6,r9 ; Turn it off
1902: bne malcfnd2 ; Found one...
1903:
1904: srw. r9,r0,r10 ; Get bit corresponding to first free one in second word
1905: li r3,0 ; Assume abject failure
1906: andc r7,r7,r9 ; Turn it off
1907: beqlr ; There are none any left...
1908:
1909: addi r3,r10,96 ; Set the correct bit number
1910: stw r7,mbfree+12(r12) ; Actually allocate the slot
1911:
1912: mapafin: or r4,r4,r5 ; Merge the first two allocation maps
1913: or r6,r6,r7 ; Then the last two
1914: or. r4,r4,r6 ; Merge both halves
1915: bnelr+ ; Return if some left for next time...
1916:
1917: neg r3,r3 ; Indicate we just allocated the last one
1918: blr ; Leave...
1919:
1920: malcfnd0: stw r4,mbfree(r12) ; Actually allocate the slot
1921: mr r3,r8 ; Set the correct bit number
1922: b mapafin ; Exit now...
1923:
1924: malcfnd1: stw r5,mbfree+4(r12) ; Actually allocate the slot
1925: addi r3,r10,32 ; Set the correct bit number
1926: b mapafin ; Exit now...
1927:
1928: malcfnd2: stw r6,mbfree+8(r12) ; Actually allocate the slot
1929: addi r3,r11,64 ; Set the correct bit number
1930: b mapafin ; Exit now...
1931:
1932:
1933:
1934: /*
1935: * Log out all memory usage
1936: */
1937:
1938: .align 5
1939: .globl EXT(logmem)
1940:
1941: LEXT(logmem)
1942:
1943: mfmsr r2 ; Get the MSR
1944: lis r10,hi16(EXT(DebugWork)) ; High part of area
1945: lis r12,hi16(EXT(mem_actual)) ; High part of actual
1946: andi. r0,r10,0x7FCF ; Interrupts and translation off
1947: ori r10,r10,lo16(EXT(DebugWork)) ; Get the entry
1948: mtmsr r0 ; Turn stuff off
1949: ori r12,r12,lo16(EXT(mem_actual)) ; Get the actual
1950: li r0,1 ; Get a one
1951:
1952: isync
1953:
1954: stw r0,4(r10) ; Force logging off
1955: lwz r0,0(r12) ; Get the end of memory
1956:
1957: lis r12,hi16(EXT(mem_size)) ; High part of defined memory
1958: ori r12,r12,lo16(EXT(mem_size)) ; Low part of defined memory
1959: lwz r12,0(r12) ; Make it end of defined
1960:
1961: cmplw r0,r12 ; Is there room for the data?
1962: ble- logmemexit ; No, do not even try...
1963:
1964: stw r12,0(r12) ; Set defined memory size
1965: stw r0,4(r12) ; Set the actual amount of memory
1966:
1967: lis r3,hi16(EXT(hash_table_base)) ; Hash table address
1968: lis r4,hi16(EXT(hash_table_size)) ; Hash table size
1969: lis r5,hi16(EXT(pmap_mem_regions)) ; Memory regions
1970: lis r6,hi16(EXT(mapCtl)) ; Mappings
1971: ori r3,r3,lo16(EXT(hash_table_base))
1972: ori r4,r4,lo16(EXT(hash_table_size))
1973: ori r5,r5,lo16(EXT(pmap_mem_regions))
1974: ori r6,r6,lo16(EXT(mapCtl))
1975: lwz r3,0(r3)
1976: lwz r4,0(r4)
1977: lwz r5,4(r5) ; Get the pointer to the phys_ent table
1978: lwz r6,0(r6) ; Get the pointer to the current mapping block
1979: stw r3,8(r12) ; Save the hash table address
1980: stw r4,12(r12) ; Save the hash table size
1981: stw r5,16(r12) ; Save the physent pointer
1982: stw r6,20(r12) ; Save the mappings
1983:
1984: addi r11,r12,0x1000 ; Point to area to move hash table and PCA
1985:
1986: add r4,r4,r4 ; Double size for both
1987:
1988: copyhash: lwz r7,0(r3) ; Copy both of them
1989: lwz r8,4(r3)
1990: lwz r9,8(r3)
1991: lwz r10,12(r3)
1992: subic. r4,r4,0x10
1993: addi r3,r3,0x10
1994: stw r7,0(r11)
1995: stw r8,4(r11)
1996: stw r9,8(r11)
1997: stw r10,12(r11)
1998: addi r11,r11,0x10
1999: bgt+ copyhash
2000:
2001: rlwinm r4,r12,20,12,31 ; Get number of phys_ents
2002:
2003: copyphys: lwz r7,0(r5) ; Copy physents
2004: lwz r8,4(r5)
2005: subic. r4,r4,1
2006: addi r5,r5,8
2007: stw r7,0(r11)
2008: stw r8,4(r11)
2009: addi r11,r11,8
2010: bgt+ copyphys
2011:
2012: addi r11,r11,4095 ; Round up to next page
2013: rlwinm r11,r11,0,0,19
2014:
2015: lwz r4,4(r6) ; Get the size of the mapping area
2016:
2017: copymaps: lwz r7,0(r6) ; Copy the mappings
2018: lwz r8,4(r6)
2019: lwz r9,8(r6)
2020: lwz r10,12(r6)
2021: subic. r4,r4,0x10
2022: addi r6,r6,0x10
2023: stw r7,0(r11)
2024: stw r8,4(r11)
2025: stw r9,8(r11)
2026: stw r10,12(r11)
2027: addi r11,r11,0x10
2028: bgt+ copymaps
2029:
2030: sub r11,r11,r12 ; Get the total length we saved
2031: stw r11,24(r12) ; Save the size
2032:
2033: logmemexit: mtmsr r2 ; Back to normal
2034: li r3,0
2035: isync
2036: blr
2037:
2038:
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.