![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to hw_lock.s CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple XNU] / XNU / osfmk / ppc|
Return to hw_lock.s CVS log | Up to [Apple XNU] / XNU / osfmk / ppc |
1.1 root 1: /*
2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3: *
4: * @APPLE_LICENSE_HEADER_START@
5: *
6: * The contents of this file constitute Original Code as defined in and
7: * are subject to the Apple Public Source License Version 1.1 (the
8: * "License"). You may not use this file except in compliance with the
9: * License. Please obtain a copy of the License at
10: * http://www.apple.com/publicsource and read it before using this file.
11: *
12: * This Original Code and all software distributed under the License are
13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17: * License for the specific language governing rights and limitations
18: * under the License.
19: *
20: * @APPLE_LICENSE_HEADER_END@
21: */
22:
23: #include <cpus.h>
24: #include <mach_assert.h>
25: #include <mach_ldebug.h>
26: #include <mach_rt.h>
27:
28: #include <kern/etap_options.h>
29:
30: #include <ppc/asm.h>
31: #include <ppc/proc_reg.h>
32: #include <assym.s>
33:
34: #define STRING ascii
35:
36: #define SWT_HI 0+FM_SIZE
37: #define SWT_LO 4+FM_SIZE
38: #define MISSED 8+FM_SIZE
39:
40: #include <ppc/POWERMAC/mp/mp.h>
41:
42: #define PROLOG(space) \
43: stwu r1,-(FM_ALIGN(space)+FM_SIZE)(r1) __ASMNL__ \
44: mflr r0 __ASMNL__ \
45: stw r3,FM_ARG0(r1) __ASMNL__ \
46: stw r0,(FM_ALIGN(space)+FM_SIZE+FM_LR_SAVE)(r1) __ASMNL__
47:
48: #define EPILOG \
49: lwz r1,0(r1) __ASMNL__ \
50: lwz r0,FM_LR_SAVE(r1) __ASMNL__ \
51: mtlr r0 __ASMNL__
52:
53: #if MACH_LDEBUG
54: /*
55: * Routines for general lock debugging.
56: */
57:
58: /*
59: * Checks for expected lock types and calls "panic" on
60: * mismatch. Detects calls to Mutex functions with
61: * type simplelock and vice versa.
62: */
63: #define CHECK_MUTEX_TYPE() \
64: lwz r10,MUTEX_TYPE(r3) __ASMNL__ \
65: cmpwi r10,MUTEX_TAG __ASMNL__ \
66: beq+ 1f __ASMNL__ \
67: lis r3,HIGH_ADDR(not_a_mutex) __ASMNL__ \
68: ori r3,r3,LOW_ADDR(not_a_mutex) __ASMNL__ \
69: bl EXT(panic) __ASMNL__ \
70: lwz r3,FM_ARG0(r1) __ASMNL__ \
71: 1:
72:
73: .data
74: not_a_mutex:
75: STRINGD "not a mutex!\n\000"
76: .text
77:
78: #define CHECK_SIMPLE_LOCK_TYPE() \
79: lwz r10,SLOCK_TYPE(r3) __ASMNL__ \
80: cmpwi r10,USLOCK_TAG __ASMNL__ \
81: beq+ 1f __ASMNL__ \
82: lis r3,HIGH_ADDR(not_a_slock) __ASMNL__ \
83: ori r3,r3,LOW_ADDR(not_a_slock) __ASMNL__ \
84: bl EXT(panic) __ASMNL__ \
85: lwz r3,FM_ARG0(r1) __ASMNL__ \
86: 1:
87:
88: .data
89: not_a_slock:
90: STRINGD "not a simple lock!\n\000"
91: .text
92:
93: #define CHECK_NO_SIMPLELOCKS() \
94: mfsprg r10,0 __ASMNL__ \
95: lwz r10,PP_CPU_DATA(r10) __ASMNL__ \
96: lwz r10,CPU_SIMPLE_LOCK_COUNT(r10) __ASMNL__ \
97: cmpwi r10,0 __ASMNL__ \
98: beq+ 1f __ASMNL__ \
99: lis r3,HIGH_ADDR(simple_locks_held) __ASMNL__ \
100: ori r3,r3,LOW_ADDR(simple_locks_held) __ASMNL__ \
101: bl EXT(panic) __ASMNL__ \
102: lwz r3,FM_ARG0(r1) __ASMNL__ \
103: 1:
104:
105: .data
106: simple_locks_held:
107: STRINGD "simple locks held!\n\000"
108: .text
109:
110: /*
111: * Verifies return to the correct thread in "unlock" situations.
112: */
113: #define CHECK_THREAD(thread_offset) \
114: mfsprg r10,0 __ASMNL__ \
115: lwz r10,PP_CPU_DATA(r10) __ASMNL__ \
116: lwz r10,CPU_ACTIVE_THREAD(r10) __ASMNL__ \
117: cmpwi r10,0 __ASMNL__ \
118: beq- 1f __ASMNL__ \
119: lwz r9,thread_offset(r3) __ASMNL__ \
120: cmpw r9,r10 __ASMNL__ \
121: beq+ 1f __ASMNL__ \
122: lis r3,HIGH_ADDR(wrong_thread) __ASMNL__ \
123: ori r3,r3,LOW_ADDR(wrong_thread) __ASMNL__ \
124: bl EXT(panic) __ASMNL__ \
125: lwz r3,FM_ARG0(r1) __ASMNL__ \
126: 1:
127:
128: .data
129: wrong_thread:
130: STRINGD "wrong thread!\n\000"
131: .text
132:
133: #define CHECK_MYLOCK(thread_offset) \
134: mfsprg r10,0 __ASMNL__ \
135: lwz r10,PP_CPU_DATA(r10) __ASMNL__ \
136: lwz r10,CPU_ACTIVE_THREAD(r10) __ASMNL__ \
137: cmpwi r10,0 __ASMNL__ \
138: beq- 1f __ASMNL__ \
139: lwz r9, thread_offset(r3) __ASMNL__ \
140: cmpw r9,r10 __ASMNL__ \
141: bne+ 1f __ASMNL__ \
142: lis r3, HIGH_ADDR(mylock_attempt) __ASMNL__ \
143: ori r3,r3,LOW_ADDR(mylock_attempt) __ASMNL__ \
144: bl EXT(panic) __ASMNL__ \
145: lwz r3,FM_ARG0(r1) __ASMNL__ \
146: 1:
147:
148: .data
149: mylock_attempt:
150: STRINGD "mylock attempt!\n\000"
151: .text
152:
153: #else /* MACH_LDEBUG */
154:
155: #define CHECK_MUTEX_TYPE()
156: #define CHECK_SIMPLE_LOCK_TYPE()
157: #define CHECK_THREAD(thread_offset)
158: #define CHECK_NO_SIMPLELOCKS()
159: #define CHECK_MYLOCK(thread_offset)
160:
161: #endif /* MACH_LDEBUG */
162:
163: /*
164: * void hw_lock_init(hw_lock_t)
165: *
166: * Initialize a hardware lock. These locks should be cache aligned and a multiple
167: * of cache size.
168: */
169:
170: ENTRY(hw_lock_init, TAG_NO_FRAME_USED)
171:
172: li r0, 0 /* set lock to free == 0 */
173: stw r0, 0(r3) /* Initialize the lock */
174: blr
175:
176: /*
177: * void hw_lock_unlock(hw_lock_t)
178: *
179: * Unconditionally release lock.
180: * MACH_RT: release preemption level.
181: */
182:
183:
184: .align 5
185: .globl EXT(hw_lock_unlock)
186:
187: LEXT(hw_lock_unlock)
188:
189: #if 0
190: lis r0,HIGH_ADDR(CutTrace) /* (TEST/DEBUG) */
191: lis r5,0xFFFF /* (TEST/DEBUG) */
192: oris r0,r0,LOW_ADDR(CutTrace) /* (TEST/DEBUG) */
193: sc /* (TEST/DEBUG) */
194: #endif
195: sync /* Flush writes done under lock */
196: li r0, 0 /* set lock to free */
197: stw r0, 0(r3)
198:
199: #if MACH_RT
200: b epStart /* Go enable preemption... */
201: #else
202: blr
203: #endif
204:
205:
206: /*
207: * Special case for internal use. Uses same lock code, but sets up so
208: * that there will be no disabling of preemption after locking. Generally
209: * used for mutex locks when obtaining the interlock although there is
210: * nothing stopping other uses.
211: */
212:
213: lockLock: lis r4,HIGH_ADDR(EXT(LockTimeOut)) /* Get the high part */
214: ori r4,r4,LOW_ADDR(EXT(LockTimeOut)) /* And the low part */
215: cmplwi cr1,r1,0 /* Set flag to disable disable preemption */
216: lwz r4,0(r4) /* Get the timerout value */
217: b lockComm /* Join on up... */
218:
219: /*
220: * void hw_lock_lock(hw_lock_t)
221: *
222: * Acquire lock, spinning until it becomes available.
223: * MACH_RT: also return with preemption disabled.
224: * Apparently not used except by mach_perf.
225: * We will just set a default timeout and jump into the NORMAL timeout lock.
226: */
227:
228: .align 5
229: .globl EXT(hw_lock_lock)
230:
231: LEXT(hw_lock_lock)
232:
233: lockDisa: lis r4,HIGH_ADDR(EXT(LockTimeOut)) /* Get the high part */
234: ori r4,r4,LOW_ADDR(EXT(LockTimeOut)) /* And the low part */
235: cmplw cr1,r1,r1 /* Set flag to enable disable preemption */
236: lwz r4,0(r4) /* Get the timerout value */
237: b lockComm /* Join on up... */
238:
239: /*
240: * unsigned int hw_lock_to(hw_lock_t, unsigned int timeout)
241: *
242: * Try to acquire spin-lock. Return success (1) or failure (0).
243: * Attempt will fail after timeout ticks of the timebase.
244: * We try fairly hard to get this lock. We disable for interruptions, but
245: * reenable after a "short" timeout (128 ticks, we may want to change this).
246: * After checking to see if the large timeout value (passed in) has expired and a
247: * sufficient number of cycles have gone by (to insure pending 'rupts are taken),
248: * we return either in abject failure, or disable and go back to the lock sniff routine.
249: * If the sniffer finds the lock free, it jumps right up and tries to grab it.
250: *
251: * One programming note: NEVER DO NOTHING IN HERE NO HOW THAT WILL FORCE US TO CALL
252: * THIS WITH TRANSLATION OR INTERRUPTIONS EITHER ON OR OFF, GOSH DARN IT!
253: *
254: */
255: .align 5
256: .globl EXT(hw_lock_to)
257:
258: LEXT(hw_lock_to)
259:
260: #if 0
261: lis r0,HIGH_ADDR(CutTrace) /* (TEST/DEBUG) */
262: lis r5,0xEEEE /* (TEST/DEBUG) */
263: oris r0,r0,LOW_ADDR(CutTrace) /* (TEST/DEBUG) */
264: sc /* (TEST/DEBUG) */
265: #endif
266: cmplw cr1,r1,r1 /* Set flag to enable disable preemption */
267:
268: lockComm: mfmsr r9 /* Get the MSR value */
269: mr r5,r3 /* Get the address of the lock */
270: rlwinm r7,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Get MSR that is uninterruptible */
271:
272: mtmsr r7 /* Turn off interruptions */
273: mftb r8 /* Get the low part of the time base */
274:
275: lwarx r6,0,r5 ; ?
276:
277: lcktry: lwarx r6,0,r5 /* Grab the lock value */
278: li r3,1 /* Use part of the delay time */
279: mr. r6,r6 /* Is it locked? */
280: bne- lcksniff /* Yeah, wait for it to clear... */
281: stwcx. r3,0,r5 /* Try to seize that there durn lock */
282: #if MACH_RT
283: bne- lcktry /* Couldn't get it... */
284: beq+ cr1,daPreComm /* We got it, go disable preemption if we're supposed to... */
285: mtmsr r9 ; Restore interrupt state
286: blr /* Go on home... */
287: #else /* MACH_RT */
288: beq+ lckgot /* We got it, yahoo... */
289: b lcktry /* Just start up again if the store failed... */
290: #endif /* MACH_RT */
291:
292: .align 5
293:
294: lcksniff: lwz r3,0(r5) /* Get that lock in here */
295: mr. r3,r3 /* Is it free yet? */
296: beq+ lcktry /* Yeah, try for it again... */
297:
298: mftb r10 /* Time stamp us now */
299: sub r10,r10,r8 /* Get the elapsed time */
300: cmplwi r10,128 /* Have we been spinning for 128 tb ticks? */
301: blt+ lcksniff /* Not yet... */
302:
303: mtmsr r9 /* Say, any interrupts pending? */
304:
305: /* The following instructions force the pipeline to be interlocked to that only one
306: instruction is issued per cycle. The insures that we stay enabled for a long enough
307: time; if it's too short, pending interruptions will not have a chance to be taken */
308:
309: subi r4,r4,128 /* Back off elapsed time from timeout value */
310: or r4,r4,r4 /* Do nothing here but force a single cycle delay */
311: mr. r4,r4 /* See if we used the whole timeout */
312: li r3,0 /* Assume a timeout return code */
313: or r4,r4,r4 /* Do nothing here but force a single cycle delay */
314:
315: ble- lckfail /* We failed */
316: mtmsr r7 /* Disable for interruptions */
317: mftb r8 /* Get the low part of the time base */
318: b lcksniff /* Now that we've opened an enable window, keep trying... */
319:
320: #if !MACH_RT
321: lckgot: mtmsr r9 /* Enable for interruptions */
322: isync /* Make sure we don't use a speculativily loaded value */
323: blr
324: #endif /* !MACH_RT */
325:
326: lckfail: /* We couldn't get the lock */
327: li r3,0 /* Set failure return code */
328: blr /* Return, head hanging low... */
329:
330:
331: /*
332: * unsigned int hw_lock_bit(hw_lock_t, unsigned int bit, unsigned int timeout)
333: *
334: * Try to acquire spin-lock. The second parameter is the bit mask to test and set.
335: * multiple bits may be set. Return success (1) or failure (0).
336: * Attempt will fail after timeout ticks of the timebase.
337: * We try fairly hard to get this lock. We disable for interruptions, but
338: * reenable after a "short" timeout (128 ticks, we may want to shorten this).
339: * After checking to see if the large timeout value (passed in) has expired and a
340: * sufficient number of cycles have gone by (to insure pending 'rupts are taken),
341: * we return either in abject failure, or disable and go back to the lock sniff routine.
342: * If the sniffer finds the lock free, it jumps right up and tries to grab it.
343: *
344: * NOTE WELL!!!! THE ROUTINE hw_lock_phys_vir KNOWS WHAT REGISTERS THIS GUY
345: * USES. THIS SAVES A TRANSLATION OFF TO ON TRANSITION AND BACK AND A SAVE AND
346: * RESTORE FROM THE STACK.
347: *
348: */
349:
350: .align 5
351:
352: nop ; Force loop alignment to cache line
353: nop
354: nop
355: nop
356:
357: .globl EXT(hw_lock_bit)
358:
359: LEXT(hw_lock_bit)
360:
361: mfmsr r9 /* Get the MSR value */
362: rlwinm r7,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Get MSR that is uninterruptible */
363:
364: mtmsr r7 /* Turn off interruptions */
365:
366: mftb r8 /* Get the low part of the time base */
367:
368: lwarx r0,0,r3 ; ?
369:
370: bittry: lwarx r6,0,r3 /* Grab the lock value */
371: and. r0,r6,r4 /* See if any of the lock bits are on */
372: or r6,r6,r4 /* Turn on the lock bits */
373: bne- bitsniff /* Yeah, wait for it to clear... */
374: stwcx. r6,0,r3 /* Try to seize that there durn lock */
375: beq+ bitgot /* We got it, yahoo... */
376: b bittry /* Just start up again if the store failed... */
377:
378: .align 5
379:
380: bitsniff: lwz r6,0(r3) /* Get that lock in here */
381: and. r0,r6,r4 /* See if any of the lock bits are on */
382: beq+ bittry /* Yeah, try for it again... */
383:
384: mftb r6 /* Time stamp us now */
385: sub r6,r6,r8 /* Get the elapsed time */
386: cmplwi r6,128 /* Have we been spinning for 128 tb ticks? */
387: blt+ bitsniff /* Not yet... */
388:
389: mtmsr r9 /* Say, any interrupts pending? */
390:
391: /* The following instructions force the pipeline to be interlocked to that only one
392: instruction is issued per cycle. The insures that we stay enabled for a long enough
393: time. If it's too short, pending interruptions will not have a chance to be taken
394: */
395:
396: subi r5,r5,128 /* Back off elapsed time from timeout value */
397: or r5,r5,r5 /* Do nothing here but force a single cycle delay */
398: mr. r5,r5 /* See if we used the whole timeout */
399: or r5,r5,r5 /* Do nothing here but force a single cycle delay */
400:
401: ble- bitfail /* We failed */
402: mtmsr r7 /* Disable for interruptions */
403: mftb r8 /* Get the low part of the time base */
404: b bitsniff /* Now that we've opened an enable window, keep trying... */
405:
406: .align 5
407:
408: bitgot: mtmsr r9 /* Enable for interruptions */
409: li r3,1 /* Set good return code */
410: isync /* Make sure we don't use a speculativily loaded value */
411: blr
412:
413: bitfail: li r3,0 /* Set failure return code */
414: blr /* Return, head hanging low... */
415:
416:
417: /*
418: * unsigned int hw_unlock_bit(hw_lock_t, unsigned int bit)
419: *
420: * Release bit based spin-lock. The second parameter is the bit mask to clear.
421: * Multiple bits may be cleared.
422: *
423: * NOTE WELL!!!! THE ROUTINE hw_lock_phys_vir KNOWS WHAT REGISTERS THIS GUY
424: * USES. THIS SAVES A TRANSLATION OFF TO ON TRANSITION AND BACK AND A SAVE AND
425: * RESTORE FROM THE STACK.
426: */
427:
428: .align 5
429: .globl EXT(hw_unlock_bit)
430:
431: LEXT(hw_unlock_bit)
432:
433: sync
434: lwarx r0,0,r3 ; ?
435:
436: ubittry: lwarx r0,0,r3 /* Grab the lock value */
437: andc r0,r0,r4 /* Clear the lock bits */
438: stwcx. r0,0,r3 /* Try to clear that there durn lock */
439: bne- ubittry /* Try again, couldn't save it... */
440:
441: blr /* Leave... */
442:
443: /*
444: * unsigned int hw_lock_mbits(hw_lock_t, unsigned int bits, unsigned int value,
445: * unsigned int newb, unsigned int timeout)
446: *
447: * Try to acquire spin-lock. The second parameter is the bit mask to check.
448: * The third is the value of those bits and the 4th is what to set them to.
449: * Return success (1) or failure (0).
450: * Attempt will fail after timeout ticks of the timebase.
451: * We try fairly hard to get this lock. We disable for interruptions, but
452: * reenable after a "short" timeout (128 ticks, we may want to shorten this).
453: * After checking to see if the large timeout value (passed in) has expired and a
454: * sufficient number of cycles have gone by (to insure pending 'rupts are taken),
455: * we return either in abject failure, or disable and go back to the lock sniff routine.
456: * If the sniffer finds the lock free, it jumps right up and tries to grab it.
457: *
458: */
459:
460: .align 5
461:
462: nop ; Force loop alignment to cache line
463: nop
464: nop
465: nop
466:
467: .globl EXT(hw_lock_mbits)
468:
469: LEXT(hw_lock_mbits)
470:
471: mfmsr r9 ; Get the MSR value
472: rlwinm r8,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Get MSR that is uninterruptible
473:
474: mtmsr r8 ; Turn off interruptions
475:
476: mftb r10 ; Get the low part of the time base
477:
478: lwarx r0,0,r3 ; ?
479:
480: mbittry: lwarx r12,0,r3 ; Grab the lock value
481: and r0,r12,r4 ; Clear extra bits
482: or r12,r12,r6 ; Turn on the lock bits
483: cmplw r0,r5 ; Are these the right bits?
484: bne- mbitsniff ; Nope, wait for it to clear...
485: stwcx. r12,0,r3 ; Try to seize that there durn lock
486: beq+ mbitgot ; We got it, yahoo...
487: b mbittry ; Just start up again if the store failed...
488:
489: .align 5
490:
491: mbitsniff: lwz r12,0(r3) ; Get that lock in here
492: and r0,r12,r4 ; Clear extra bits
493: or r12,r12,r6 ; Turn on the lock bits
494: cmplw r0,r5 ; Are these the right bits?
495: beq+ mbittry ; Yeah, try for it again...
496:
497: mftb r11 ; Time stamp us now
498: sub r11,r11,r10 ; Get the elapsed time
499: cmplwi r11,128 ; Have we been spinning for 128 tb ticks?
500: blt+ mbitsniff ; Not yet...
501:
502: mtmsr r9 ; Say, any interrupts pending?
503:
504: ; The following instructions force the pipeline to be interlocked to that only one
505: ; instruction is issued per cycle. The insures that we stay enabled for a long enough
506: ; time. If it is too short, pending interruptions will not have a chance to be taken
507:
508: subi r7,r7,128 ; Back off elapsed time from timeout value
509: or r7,r7,r7 ; Do nothing here but force a single cycle delay
510: mr. r7,r7 ; See if we used the whole timeout
511: or r7,r7,r7 ; Do nothing here but force a single cycle delay
512:
513: ble- mbitfail ; We failed
514: mtmsr r8 ; Disable for interruptions
515: mftb r10 ; Get the low part of the time base
516: b mbitsniff ; Now that we have opened an enable window, keep trying...
517:
518: .align 5
519:
520: mbitgot: mtmsr r9 ; Enable for interruptions
521: li r3,1 ; Set good return code
522: isync ; Make sure we do not use a speculativily loaded value
523: blr
524:
525: mbitfail: li r3,0 ; Set failure return code
526: blr ; Return, head hanging low...
527:
528:
529: /*
530: * unsigned int hw_cpu_sync(unsigned int *, unsigned int timeout)
531: *
532: * Spin until word hits 0 or timeout.
533: * Return success (1) or failure (0).
534: * Attempt will fail after timeout ticks of the timebase.
535: *
536: * The theory is that a processor will bump a counter as it signals
537: * other processors. Then it will spin untl the counter hits 0 (or
538: * times out). The other processors, as it receives the signal will
539: * decrement the counter.
540: *
541: * The other processors use interlocked update to decrement, this one
542: * does not need to interlock.
543: *
544: */
545:
546: .align 5
547:
548: .globl EXT(hw_cpu_sync)
549:
550: LEXT(hw_cpu_sync)
551:
552: mftb r10 ; Get the low part of the time base
553: mr r9,r3 ; Save the sync word address
554: li r3,1 ; Assume we work
555:
556: csynctry: lwz r11,0(r9) ; Grab the sync value
557: mr. r11,r11 ; Counter hit 0?
558: beqlr- ; Yeah, we are sunk...
559: mftb r12 ; Time stamp us now
560:
561: sub r12,r12,r10 ; Get the elapsed time
562: cmplw r4,r12 ; Have we gone too long?
563: bge+ csynctry ; Not yet...
564:
565: li r3,0 ; Set failure...
566: blr ; Return, head hanging low...
567:
568:
569: /*
570: * unsigned int hw_lock_try(hw_lock_t)
571: *
572: * try to acquire spin-lock. Return success (1) or failure (0)
573: * MACH_RT: returns with preemption disabled on success.
574: *
575: */
576: .align 5
577: .globl EXT(hw_lock_try)
578:
579: LEXT(hw_lock_try)
580:
581: #if 0
582: lis r0,HIGH_ADDR(CutTrace) /* (TEST/DEBUG) */
583: lis r5,0x9999 /* (TEST/DEBUG) */
584: oris r0,r0,LOW_ADDR(CutTrace) /* (TEST/DEBUG) */
585: sc /* (TEST/DEBUG) */
586: #endif
587: mfmsr r9 /* Save the MSR value */
588: li r4, 1 /* value to be stored... 1==taken */
589: rlwinm r7,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruption bit */
590:
591: #if MACH_LDEBUG
592: lis r5, 0x10 /* roughly 1E6 */
593: mtctr r5
594: #endif /* MACH_LDEBUG */
595:
596: mtmsr r7 /* Disable interruptions and thus, preemption */
597:
598: lwarx r5,0,r3 ; ?
599:
600: .L_lock_try_loop:
601:
602: #if MACH_LDEBUG
603: bdnz+ 0f /* Count attempts */
604: mtmsr r9 /* Restore enablement */
605: BREAKPOINT_TRAP /* Get to debugger */
606: mtmsr r7 /* Disable interruptions and thus, preemption */
607: 0:
608: #endif /* MACH_LDEBUG */
609:
610: lwarx r5,0,r3 /* Ld from addr of arg and reserve */
611:
612: cmpwi r5, 0 /* TEST... */
613: bne- .L_lock_try_failed /* branch if taken. Predict free */
614:
615: stwcx. r4, 0,r3 /* And SET (if still reserved) */
616: mfsprg r6,0 /* Get the per_proc block */
617: bne- .L_lock_try_loop /* If set failed, loop back */
618:
619: lwz r6,PP_CPU_DATA(r6) /* Get the pointer to the CPU data from per proc */
620: isync
621:
622: #if MACH_RT
623: lwz r5,CPU_PREEMPTION_LEVEL(r6) /* Get the preemption level */
624: addi r5,r5,1 /* Bring up the disable count */
625: stw r5,CPU_PREEMPTION_LEVEL(r6) /* Save it back */
626:
627: #endif /* MACH_RT */
628:
629: mtmsr r9 /* Allow interruptions now */
630: li r3,1 /* Set that the lock was free */
631: blr
632:
633: .L_lock_try_failed:
634: mtmsr r9 /* Allow interruptions now */
635: li r3,0 /* FAILURE - lock was taken */
636: blr
637:
638: /*
639: * unsigned int hw_lock_held(hw_lock_t)
640: *
641: * Return 1 if lock is held
642: * MACH_RT: doesn't change preemption state.
643: * N.B. Racy, of course.
644: *
645: */
646: .align 5
647: .globl EXT(hw_lock_held)
648:
649: LEXT(hw_lock_held)
650:
651: #if 0
652: lis r0,HIGH_ADDR(CutTrace) /* (TEST/DEBUG) */
653: lis r5,0x8888 /* (TEST/DEBUG) */
654: oris r0,r0,LOW_ADDR(CutTrace) /* (TEST/DEBUG) */
655: sc /* (TEST/DEBUG) */
656: #endif
657: isync /* Make sure we don't use a speculativily fetched lock */
658: lwz r3, 0(r3) /* Return value of lock */
659: blr
660:
661: /*
662: * unsigned int hw_compare_and_store(unsigned int old, unsigned int new, unsigned int *area)
663: *
664: * Compare old to area if equal, store new, and return true
665: * else return false and no store
666: * This is an atomic operation
667: *
668: */
669: .align 5
670: .globl EXT(hw_compare_and_store)
671:
672: LEXT(hw_compare_and_store)
673:
674: mr r6,r3 /* Save the old value */
675:
676: lwarx r9,0,r5 ; ?
677:
678: cstry: lwarx r9,0,r5 /* Grab the area value */
679: li r3,1 /* Assume it works */
680: cmplw cr0,r9,r6 /* Does it match the old value? */
681: bne- csfail /* No, it must have changed... */
682: stwcx. r4,0,r5 /* Try to save the new value */
683: bne- cstry /* Didn't get it, try again... */
684: isync /* Just hold up prefetch */
685: blr /* Return... */
686:
687: csfail: li r3,0 /* Set failure */
688: blr /* Better luck next time... */
689:
690:
691: /*
692: * unsigned int hw_atomic_add(unsigned int *area, int *val)
693: *
694: * Atomically add the second parameter to the first.
695: * Returns the result.
696: *
697: */
698: .align 5
699: .globl EXT(hw_atomic_add)
700:
701: LEXT(hw_atomic_add)
702:
703: mr r6,r3 /* Save the area */
704:
705: lwarx r3,0,r6 ; ?
706:
707: addtry: lwarx r3,0,r6 /* Grab the area value */
708: add r3,r3,r4 /* Add the value */
709: stwcx. r3,0,r6 /* Try to save the new value */
710: bne- addtry /* Didn't get it, try again... */
711: blr /* Return... */
712:
713:
714: /*
715: * unsigned int hw_atomic_sub(unsigned int *area, int *val)
716: *
717: * Atomically subtract the second parameter from the first.
718: * Returns the result.
719: *
720: */
721: .align 5
722: .globl EXT(hw_atomic_sub)
723:
724: LEXT(hw_atomic_sub)
725:
726: mr r6,r3 /* Save the area */
727:
728: lwarx r3,0,r6 ; ?
729:
730: subtry: lwarx r3,0,r6 /* Grab the area value */
731: sub r3,r3,r4 /* Subtract the value */
732: stwcx. r3,0,r6 /* Try to save the new value */
733: bne- subtry /* Didn't get it, try again... */
734: blr /* Return... */
735:
736:
737: /*
738: * void hw_queue_atomic(unsigned int * anchor, unsigned int * elem, unsigned int disp)
739: *
740: * Atomically inserts the element at the head of the list
741: * anchor is the pointer to the first element
742: * element is the pointer to the element to insert
743: * disp is the displacement into the element to the chain pointer
744: *
745: */
746: .align 5
747: .globl EXT(hw_queue_atomic)
748:
749: LEXT(hw_queue_atomic)
750:
751: mr r7,r4 /* Make end point the same as start */
752: mr r8,r5 /* Copy the displacement also */
753: b hw_queue_comm /* Join common code... */
754:
755: /*
756: * void hw_queue_atomic_list(unsigned int * anchor, unsigned int * first, unsigned int * last, unsigned int disp)
757: *
758: * Atomically inserts the list of elements at the head of the list
759: * anchor is the pointer to the first element
760: * first is the pointer to the first element to insert
761: * last is the pointer to the last element to insert
762: * disp is the displacement into the element to the chain pointer
763: *
764: */
765: .align 5
766: .globl EXT(hw_queue_atomic_list)
767:
768: LEXT(hw_queue_atomic_list)
769:
770: mr r7,r5 /* Make end point the same as start */
771: mr r8,r6 /* Copy the displacement also */
772:
773: hw_queue_comm:
774: lwarx r9,0,r3 ; ?
775:
776: hw_queue_comm2:
777: lwarx r9,0,r3 /* Pick up the anchor */
778: stwx r9,r8,r7 /* Chain that to the end of the new stuff */
779: stwcx. r4,0,r3 /* Try to chain into the front */
780: bne- hw_queue_comm2 /* Didn't make it, try again... */
781:
782: blr /* Return... */
783:
784: /*
785: * unsigned int *hw_dequeue_atomic(unsigned int *anchor, unsigned int disp)
786: *
787: * Atomically removes the first element in a list and returns it.
788: * anchor is the pointer to the first element
789: * disp is the displacement into the element to the chain pointer
790: * Returns element if found, 0 if empty.
791: *
792: */
793: .align 5
794: .globl EXT(hw_dequeue_atomic)
795:
796: LEXT(hw_dequeue_atomic)
797:
798: mr r5,r3 /* Save the anchor */
799:
800: hw_dequeue_comm:
801: lwarx r9,0,r3 ; ?
802:
803: hw_dequeue_comm2:
804: lwarx r3,0,r5 /* Pick up the anchor */
805: mr. r3,r3 /* Is the list empty? */
806: beqlr- /* Leave it list empty... */
807: lwzx r9,r4,r3 /* Get the next in line */
808: stwcx. r9,0,r5 /* Try to chain into the front */
809: beqlr+ ; Got the thing, go away with it...
810: b hw_dequeue_comm2 ; Did not make it, try again...
811:
812: /*
813: * void mutex_init(mutex_t* l, etap_event_t etap)
814: */
815:
816: ENTRY(mutex_init,TAG_NO_FRAME_USED)
817:
818: PROLOG(0)
819: li r10, 0
820: stw r10, MUTEX_ILK(r3) /* clear interlock */
821: stw r10, MUTEX_LOCKED(r3) /* clear locked flag */
822: sth r10, MUTEX_WAITERS(r3) /* init waiter count */
823:
824: #if MACH_LDEBUG
825: stw r10, MUTEX_PC(r3) /* init caller pc */
826: stw r10, MUTEX_THREAD(r3) /* and owning thread */
827: li r10, MUTEX_TAG
828: stw r10, MUTEX_TYPE(r3) /* set lock type */
829: #endif /* MACH_LDEBUG */
830:
831: #if ETAP_LOCK_TRACE
832: bl EXT(etap_mutex_init) /* init ETAP data */
833: #endif /* ETAP_LOCK_TRACE */
834:
835: EPILOG
836: blr
837:
838: /*
839: * void _mutex_lock(mutex_t*)
840: */
841:
842: .align 5
843: .globl EXT(_mutex_lock)
844:
845: LEXT(_mutex_lock)
846:
847: PROLOG(12)
848:
849: #if ETAP_LOCK_TRACE
850: li r0, 0
851: stw r0,SWT_HI(r1) /* set wait time to 0 (HI) */
852: stw r0,SWT_LO(r1) /* set wait time to 0 (LO) */
853: stw r0,MISSED(r1) /* clear local miss marker */
854: #endif /* ETAP_LOCK_TRACE */
855:
856: CHECK_MUTEX_TYPE()
857: CHECK_NO_SIMPLELOCKS()
858:
859: .L_ml_retry:
860: #if 0
861: mfsprg r4,0 /* (TEST/DEBUG) */
862: lwz r4,PP_CPU_DATA(r4) /* (TEST/DEBUG) */
863: lis r0,HIGH_ADDR(CutTrace) /* (TEST/DEBUG) */
864: lwz r4,CPU_ACTIVE_THREAD(r4) /* (TEST/DEBUG) */
865: lis r5,0xAAAA /* (TEST/DEBUG) */
866: oris r0,r0,LOW_ADDR(CutTrace) /* (TEST/DEBUG) */
867: sc /* (TEST/DEBUG) */
868: #endif
869:
870: bl lockDisa /* Go get a lock on the mutex's interlock lock */
871: mr. r4,r3 /* Did we get it? */
872: lwz r3,FM_ARG0(r1) /* Restore the lock address */
873: bne+ mlGotInt /* We got it just fine... */
874:
875: lis r3,HIGH_ADDR(mutex_failed1) ; Get the failed mutex message
876: ori r3,r3,LOW_ADDR(mutex_failed1) ; Get the failed mutex message
877: bl EXT(panic) ; Call panic
878: BREAKPOINT_TRAP ; We die here anyway, can not get the lock
879:
880: .data
881: mutex_failed1:
882: STRINGD "We can't get a mutex interlock lock on mutex_lock\n\000"
883: .text
884:
885: mlGotInt:
886:
887: /* Note that there is no reason to do a load and reserve here. We already
888: hold the interlock lock and no one can touch this field unless they
889: have that, so, we're free to play */
890:
891: lwz r4,MUTEX_LOCKED(r3) /* Get the mutex's lock field */
892:
893: li r10,1 /* Set the lock value */
894:
895: mr. r4,r4 /* So, can we have it? */
896: bne- mlInUse /* Nope, sombody's playing already... */
897:
898: stw r10,MUTEX_LOCKED(r3) /* Take it unto ourselves */
899:
900: #if MACH_LDEBUG
901: mfsprg r9,0 /* Get the per_proc block */
902: lwz r10,0(r1) /* Get previous save frame */
903: lwz r9,PP_CPU_DATA(r9) /* Point to the cpu data area */
904: lwz r10,FM_LR_SAVE(r10) /* Get our caller's address */
905: lwz r8, CPU_ACTIVE_THREAD(r9) /* Get the active thread */
906: stw r10,MUTEX_PC(r3) /* Save our caller */
907: mr. r8,r8 /* Is there any thread? */
908: stw r8,MUTEX_THREAD(r3) /* Set the mutex's holding thread */
909: beq- .L_ml_no_active_thread /* No owning thread... */
910: lwz r9,THREAD_MUTEX_COUNT(r8) /* Get the mutex count */
911: addi r9,r9,1 /* Bump it up */
912: stw r9,THREAD_MUTEX_COUNT(r8) /* Stash it back */
913: .L_ml_no_active_thread:
914: #endif /* MACH_LDEBUG */
915:
916: li r10,0 /* Get the unlock value */
917: sync /* Push it all out */
918: stw r10,MUTEX_ILK(r3) /* free the interlock */
919:
920: #if ETAP_LOCK_TRACE
921: mflr r4
922: lwz r5,SWT_HI(r1)
923: lwz r6,SWT_LO(r1)
924: bl EXT(etap_mutex_hold) /* collect hold timestamp */
925: #endif /* ETAP_LOCK_TRACE */
926:
927: EPILOG /* Restore all saved registers */
928:
929: #if MACH_RT
930: b epStart /* Go enable preemption... */
931: #else
932: blr /* Return... */
933: #endif
934:
935: /*
936: * We come to here when we have a resource conflict. In other words,
937: * the mutex is held.
938: */
939:
940: mlInUse:
941:
942: #if ETAP_LOCK_TRACE
943: lwz r7,MISSED(r1)
944: cmpwi r7,0 /* did we already take a wait timestamp ? */
945: bne .L_ml_block /* yup. carry-on */
946: bl EXT(etap_mutex_miss) /* get wait timestamp */
947: stw r3,SWT_HI(r1) /* store timestamp */
948: stw r4,SWT_LO(r1)
949: li r7, 1 /* mark wait timestamp as taken */
950: stw r7,MISSED(r1)
951: lwz r3,FM_ARG0(r1) /* restore r3 (saved in prolog) */
952: .L_ml_block:
953: #endif /* ETAP_LOCK_TRACE */
954:
955: CHECK_MYLOCK(MUTEX_THREAD) /* Assert we don't own the lock already */
956:
957:
958: /* Note that we come in here with the interlock set. The wait routine
959: * will unlock it before waiting.
960: */
961: bl EXT(mutex_lock_wait) /* Wait for our turn at the lock */
962:
963: lwz r3,FM_ARG0(r1) /* restore r3 (saved in prolog) */
964: b .L_ml_retry /* and try again... */
965:
966:
967: /*
968: * void _mutex_try(mutex_t*)
969: *
970: */
971:
972: .align 5
973: .globl EXT(_mutex_try)
974:
975: LEXT(_mutex_try)
976:
977: PROLOG(8) /* reserve space for SWT_HI and SWT_LO */
978:
979: #if ETAP_LOCK_TRACE
980: li r5, 0
981: stw r5, STW_HI(r1) /* set wait time to 0 (HI) */
982: stw r5, SWT_LO(r1) /* set wait time to 0 (LO) */
983: #endif /* ETAP_LOCK_TRACE */
984:
985: #if 0
986: lis r0,HIGH_ADDR(CutTrace) /* (TEST/DEBUG) */
987: lis r5,0xBBBB /* (TEST/DEBUG) */
988: oris r0,r0,LOW_ADDR(CutTrace) /* (TEST/DEBUG) */
989: sc /* (TEST/DEBUG) */
990: #endif
991: CHECK_MUTEX_TYPE()
992: CHECK_NO_SIMPLELOCKS()
993:
994: lwz r6,MUTEX_LOCKED(r3) /* Quick check */
995: mr. r6,r6 /* to see if someone has this lock already */
996: bne- mtFail /* Someone's got it already... */
997:
998: bl lockLock /* Go get a lock on the mutex's interlock lock */
999: mr. r4,r3 /* Did we get it? */
1000: lwz r3,FM_ARG0(r1) /* Restore the lock address */
1001: bne+ mtGotInt /* We got it just fine... */
1002:
1003: lis r3,HIGH_ADDR(mutex_failed2) ; Get the failed mutex message
1004: ori r3,r3,LOW_ADDR(mutex_failed2) ; Get the failed mutex message
1005: bl EXT(panic) ; Call panic
1006: BREAKPOINT_TRAP ; We die here anyway, can not get the lock
1007:
1008: .data
1009: mutex_failed2:
1010: STRINGD "We can't get a mutex interlock lock on mutex_try\n\000"
1011: .text
1012:
1013: mtGotInt:
1014:
1015: /* Note that there is no reason to do a load and reserve here. We already
1016: hold the interlock and no one can touch at this field unless they
1017: have that, so, we're free to play */
1018:
1019: lwz r4,MUTEX_LOCKED(r3) /* Get the mutex's lock field */
1020:
1021: li r10,1 /* Set the lock value */
1022:
1023: mr. r4,r4 /* So, can we have it? */
1024: bne- mtInUse /* Nope, sombody's playing already... */
1025:
1026: stw r10,MUTEX_LOCKED(r3) /* Take it unto ourselves */
1027:
1028: #if MACH_LDEBUG
1029: mfsprg r9,0 /* Get the per_proc block */
1030: lwz r10,0(r1) /* Get previous save frame */
1031: lwz r9,PP_CPU_DATA(r9) /* Point to the cpu data area */
1032: lwz r10,FM_LR_SAVE(r10) /* Get our caller's address */
1033: lwz r8, CPU_ACTIVE_THREAD(r9) /* Get the active thread */
1034: stw r10,MUTEX_PC(r3) /* Save our caller */
1035: mr. r8,r8 /* Is there any thread? */
1036: stw r8,MUTEX_THREAD(r3) /* Set the mutex's holding thread */
1037: beq- .L_mt_no_active_thread /* No owning thread... */
1038: lwz r9, THREAD_MUTEX_COUNT(r8) /* Get the mutex count */
1039: addi r9, r9, 1 /* Bump it up */
1040: stw r9, THREAD_MUTEX_COUNT(r8) /* Stash it back */
1041: .L_mt_no_active_thread:
1042: #endif /* MACH_LDEBUG */
1043:
1044: li r10,0 /* Get the unlock value */
1045: sync /* Push it all out */
1046: stw r10,MUTEX_ILK(r3) /* free the interlock */
1047:
1048: #if ETAP_LOCK_TRACE
1049: lwz r4,0(r1) /* Back chain the stack */
1050: lwz r5,SWT_HI(r1)
1051: lwz r4,FM_LR_SAVE(r4) /* Get our caller's address */
1052: lwz r6,SWT_LO(r1)
1053: bl EXT(etap_mutex_hold) /* collect hold timestamp */
1054: #endif /* ETAP_LOCK_TRACE */
1055:
1056: EPILOG /* Restore all saved registers */
1057: blr /* Return... */
1058:
1059: /*
1060: * We come to here when we have a resource conflict. In other words,
1061: * the mutex is held.
1062: */
1063:
1064: mtInUse: li r10,0 /* Get the unlock value */
1065: sync /* Push it all out */
1066: stw r10,MUTEX_ILK(r3) /* free the interlock */
1067:
1068: mtFail: li r3,0 /* Set failure code */
1069: EPILOG /* Restore all saved registers */
1070: blr /* Return... */
1071:
1072:
1073: /*
1074: * void mutex_unlock(mutex_t* l)
1075: */
1076:
1077: .align 5
1078: .globl EXT(mutex_unlock)
1079:
1080: LEXT(mutex_unlock)
1081:
1082: PROLOG(0)
1083:
1084: #if ETAP_LOCK_TRACE
1085: bl EXT(etap_mutex_unlock) /* collect ETAP data */
1086: lwz r3,FM_ARG0(r1) /* restore r3 (saved in prolog) */
1087: #endif /* ETAP_LOCK_TRACE */
1088:
1089: CHECK_MUTEX_TYPE()
1090: CHECK_THREAD(MUTEX_THREAD)
1091:
1092: #if 0
1093: mfsprg r4,0 /* (TEST/DEBUG) */
1094: lwz r4,PP_CPU_DATA(r4) /* (TEST/DEBUG) */
1095: lis r0,HIGH_ADDR(CutTrace) /* (TEST/DEBUG) */
1096: lwz r4,CPU_ACTIVE_THREAD(r4) /* (TEST/DEBUG) */
1097: lis r5,0xCCCC /* (TEST/DEBUG) */
1098: oris r0,r0,LOW_ADDR(CutTrace) /* (TEST/DEBUG) */
1099: sc /* (TEST/DEBUG) */
1100: #endif
1101: bl lockDisa /* Go get a lock on the mutex's interlock lock */
1102: mr. r4,r3 /* Did we get it? */
1103: lwz r3,FM_ARG0(r1) /* Restore the lock address */
1104: bne+ muGotInt /* We got it just fine... */
1105:
1106: lis r3,HIGH_ADDR(mutex_failed3) ; Get the failed mutex message
1107: ori r3,r3,LOW_ADDR(mutex_failed3) ; Get the failed mutex message
1108: bl EXT(panic) ; Call panic
1109: BREAKPOINT_TRAP ; We die here anyway, can not get the lock
1110:
1111: .data
1112: mutex_failed3:
1113: STRINGD "We can't get a mutex interlock lock on mutex_unlock\n\000"
1114: .text
1115:
1116:
1117: muGotInt:
1118: lhz r10,MUTEX_WAITERS(r3) /* are there any waiters ? */
1119: cmpwi r10,0
1120: beq+ muUnlock /* Nope, we're done... */
1121:
1122: bl EXT(mutex_unlock_wakeup) /* yes, wake a thread */
1123: lwz r3,FM_ARG0(r1) /* restore r3 (saved in prolog) */
1124: li r10,0 /* Get unlock value */
1125:
1126: muUnlock:
1127: #if MACH_LDEBUG
1128: mfsprg r9,0
1129: lwz r9,PP_CPU_DATA(r9)
1130: lwz r9,CPU_ACTIVE_THREAD(r9)
1131: stw r10,MUTEX_THREAD(r3) /* disown thread */
1132: cmpwi r9,0
1133: beq- .L_mu_no_active_thread
1134: lwz r8,THREAD_MUTEX_COUNT(r9)
1135: subi r8,r8,1
1136: stw r8,THREAD_MUTEX_COUNT(r9)
1137: .L_mu_no_active_thread:
1138: #endif /* MACH_LDEBUG */
1139:
1140: stw r10,MUTEX_LOCKED(r3) /* release the mutex */
1141: sync /* Make sure it's all there before we release */
1142: stw r10,MUTEX_ILK(r3) /* unlock the interlock */
1143:
1144: EPILOG /* Deal with the stack now, enable_preemption doesn't always want one */
1145: #if MACH_RT
1146: b epStart /* Go enable preemption... */
1147: #else
1148: blr /* Return... */
1149: #endif
1150:
1151: /*
1152: * void interlock_unlock(hw_lock_t lock)
1153: */
1154:
1155: .align 5
1156: .globl EXT(interlock_unlock)
1157:
1158: LEXT(interlock_unlock)
1159:
1160: #if 0
1161: lis r0,HIGH_ADDR(CutTrace) /* (TEST/DEBUG) */
1162: lis r5,0xDDDD /* (TEST/DEBUG) */
1163: oris r0,r0,LOW_ADDR(CutTrace) /* (TEST/DEBUG) */
1164: sc /* (TEST/DEBUG) */
1165: #endif
1166: li r10,0
1167: sync
1168: stw r10,0(r3)
1169:
1170: #if MACH_RT
1171: b epStart /* Go enable preemption... */
1172: #else
1173: blr /* Return... */
1174: #endif
1175:
1176: #if MACH_RT
1177: /*
1178: * Here is where we enable preemption. We need to be protected
1179: * against ourselves, we can't chance getting interrupted and modifying
1180: * our processor wide preemption count after we'sve loaded it up. So,
1181: * we need to disable all 'rupts. Actually, we could use a compare
1182: * and swap to do this, but, since there are no MP considerations
1183: * (we are dealing with a CPU local field) it is much, much faster
1184: * to disable.
1185: *
1186: * Note that if we are not genned MP, the calls here will be no-opped via
1187: * a #define and since the _mp forms are the same, likewise a #define
1188: * will be used to route to the other forms
1189: */
1190:
1191: /* This version does not check if we get preempted or not */
1192:
1193:
1194: .align 4
1195: .globl EXT(_enable_preemption_no_check)
1196:
1197: LEXT(_enable_preemption_no_check)
1198: cmplw cr1,r1,r1 /* Force zero cr so we know not to check if preempted */
1199: b epCommn /* Join up with the other enable code... */
1200:
1201:
1202: /* This version checks if we get preempted or not */
1203:
1204: .align 5
1205: .globl EXT(_enable_preemption)
1206:
1207: LEXT(_enable_preemption)
1208:
1209: epStart: cmplwi cr1,r1,0 /* Force non-zero cr so we know to check if preempted */
1210:
1211: /*
1212: * Common enable preemption code
1213: */
1214:
1215: epCommn: mfmsr r9 /* Save the old MSR */
1216: rlwinm r8,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
1217: mtmsr r8 /* Interrupts off */
1218:
1219: mfsprg r3,0 /* Get the per_proc block */
1220: lwz r6,PP_CPU_DATA(r3) /* Get the pointer to the CPU data from per proc */
1221: li r8,-1 /* Get a decrimenter */
1222: lwz r5,CPU_PREEMPTION_LEVEL(r6) /* Get the preemption level */
1223: add. r5,r5,r8 /* Bring down the disable count */
1224: #if 0
1225: mfsprg r4,1 ; (TEST/DEBUG) Note the next 3 keep from interrpting too early
1226: mr. r4,r4 ; (TEST/DEBUG)
1227: beq- epskptrc0 ; (TEST/DEBUG)
1228: lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1229: lis r4,0xBBBB ; (TEST/DEBUG)
1230: oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1231: sc ; (TEST/DEBUG)
1232: epskptrc0: mr. r5,r5 ; (TEST/DEBUG)
1233: #endif
1234: #if MACH_LDEBUG
1235: blt- epTooFar /* Yeah, we did... */
1236: #endif /* MACH_LDEBUG */
1237: stw r5,CPU_PREEMPTION_LEVEL(r6) /* Save it back */
1238:
1239: beq+ epCheckPreempt /* Go check if we need to be preempted... */
1240:
1241: epNoCheck: mtmsr r9 /* Restore the interrupt level */
1242: blr /* Leave... */
1243:
1244: #if MACH_LDEBUG
1245: epTooFar:
1246: lis r6,HIGH_ADDR(EXT(panic)) /* First half of panic call */
1247: lis r3,HIGH_ADDR(epTooFarStr) /* First half of panic string */
1248: ori r6,r6,LOW_ADDR(EXT(panic)) /* Second half of panic call */
1249: ori r3,r3,LOW_ADDR(epTooFarStr) /* Second half of panic string */
1250: mtlr r6 /* Get the address of the panic routine */
1251: mtmsr r9 /* Restore interruptions */
1252: blrl /* Panic... */
1253:
1254: .data
1255: epTooFarStr:
1256: STRINGD "_enable_preemption: preemption_level <= 0!\000"
1257: .text
1258: #endif /* MACH_LDEBUG */
1259:
1260: .align 5
1261:
1262: epCheckPreempt:
1263: lwz r7,PP_NEED_AST(r3) /* Get the AST request address */
1264: lwz r6,PP_CPU_DATA(r3) /* Get the pointer to the CPU data from per proc */
1265: li r5,AST_URGENT /* Get the requests we do honor */
1266: lwz r7,0(r7) /* Get the actual, real live, extra special AST word */
1267: lis r0,HIGH_ADDR(DoPreemptCall) /* Just in case, get the top of firmware call */
1268: and. r7,r7,r5 ; Should we preempt?
1269: ori r0,r0,LOW_ADDR(DoPreemptCall) /* Merge in bottom part */
1270: beq+ epCPno ; No preemption here...
1271:
1272: andi. r3,r9,lo16(MASK(MSR_EE)) ; We cannot preempt if interruptions are off
1273:
1274: epCPno: mtmsr r9 /* Allow interrupts if we can */
1275: beqlr+ ; We probably will not preempt...
1276: sc /* Do the preemption */
1277: blr /* Now, go away now... */
1278:
1279: /*
1280: * Here is where we disable preemption. Since preemption is on a
1281: * per processor basis (a thread runs on one CPU at a time) we don't
1282: * need any cross-processor synchronization. We do, however, need to
1283: * be interrupt safe, so we don't preempt while in the process of
1284: * disabling it. We could use SPLs, but since we always want complete
1285: * disablement, and this is platform specific code, we'll just kick the
1286: * MSR. We'll save a couple of orders of magnitude over using SPLs.
1287: */
1288:
1289: .align 5
1290:
1291: nop ; Use these 5 nops to force daPreComm
1292: nop ; to a line boundary.
1293: nop
1294: nop
1295: nop
1296:
1297: .globl EXT(_disable_preemption)
1298:
1299: LEXT(_disable_preemption)
1300:
1301: daPreAll: mfmsr r9 /* Save the old MSR */
1302: rlwinm r8,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
1303: mtmsr r8 /* Interrupts off */
1304:
1305: daPreComm: mfsprg r6,0 /* Get the per_proc block */
1306: lwz r6,PP_CPU_DATA(r6) /* Get the pointer to the CPU data from per proc */
1307: lwz r5,CPU_PREEMPTION_LEVEL(r6) /* Get the preemption level */
1308: addi r5,r5,1 /* Bring up the disable count */
1309: stw r5,CPU_PREEMPTION_LEVEL(r6) /* Save it back */
1310: #if 0
1311: mfsprg r4,1 ; (TEST/DEBUG) Note the next 3 keep from interrpting too early
1312: mr. r4,r4 ; (TEST/DEBUG)
1313: beq- epskptrc1 ; (TEST/DEBUG)
1314: lis r0,hi16(CutTrace) ; (TEST/DEBUG)
1315: lis r4,0xAAAA ; (TEST/DEBUG)
1316: oris r0,r0,lo16(CutTrace) ; (TEST/DEBUG)
1317: sc ; (TEST/DEBUG)
1318: epskptrc1: ; (TEST/DEBUG)
1319: #endif
1320:
1321: #if 0
1322: lwz r6,CPU_ACTIVE_THREAD(r6) ; Get the pointer to the currently active thread
1323: mr. r6,r6 ; During boot?
1324: beq- nopredeb ; Yes, do not do backtrace...
1325: lwz r6,THREAD_TOP_ACT(r6) ; Point to the active activation
1326: lwz r6,ACT_MACT_PCB(r6) ; Get the last savearea used
1327: mr. r0,r6 ; Any saved context?
1328: beq- nosaveds ; No...
1329: lwz r0,saver1(r6) ; Get end of savearea chain
1330:
1331: nosaveds: li r11,0 ; Clear callers callers callers return
1332: li r10,0 ; Clear callers callers callers callers return
1333: li r8,0 ; Clear callers callers callers callers callers return
1334: lwz r2,0(r1) ; Get callers callers stack frame
1335: lwz r12,8(r2) ; Get our callers return
1336: lwz r4,0(r2) ; Back chain
1337:
1338: xor r2,r4,r2 ; Form difference
1339: cmplwi r2,8192 ; Within a couple of pages?
1340: mr r2,r4 ; Move register
1341: bge- nosaveher2 ; No, no back chain then...
1342: lwz r11,8(r2) ; Get our callers return
1343: lwz r4,0(r2) ; Back chain
1344:
1345: xor r2,r4,r2 ; Form difference
1346: cmplwi r2,8192 ; Within a couple of pages?
1347: mr r2,r4 ; Move register
1348: bge- nosaveher2 ; No, no back chain then...
1349: lwz r10,8(r2) ; Get our callers return
1350: lwz r4,0(r2) ; Back chain
1351:
1352: xor r2,r4,r2 ; Form difference
1353: cmplwi r2,8192 ; Within a couple of pages?
1354: mr r2,r4 ; Move register
1355: bge- nosaveher2 ; No, no back chain then...
1356: lwz r8,8(r2) ; Get our callers return
1357:
1358: nosaveher2:
1359: addi r5,r5,-1 ; Get index to slot
1360: mfspr r6,pir ; Get our processor
1361: mflr r4 ; Get our return
1362: rlwinm r6,r6,8,0,23 ; Index to processor slot
1363: lis r2,hi16(EXT(DBGpreempt)) ; Stack high order
1364: rlwinm r5,r5,4,0,27 ; Index to stack slot
1365: ori r2,r2,lo16(EXT(DBGpreempt)) ; Stack low order
1366: add r2,r2,r5 ; Point to slot
1367: add r2,r2,r6 ; Move to processor
1368: stw r4,0(r2) ; Save our return
1369: stw r11,4(r2) ; Save callers caller
1370: stw r10,8(r2) ; Save callers callers caller
1371: stw r8,12(r2) ; Save callers callers callers caller
1372: nopredeb:
1373: #endif
1374: mtmsr r9 /* Allow interruptions now */
1375:
1376: blr /* Return... */
1377:
1378: /*
1379: * Return the active thread for both inside and outside osfmk consumption
1380: */
1381:
1382: .align 5
1383: .globl EXT(current_thread)
1384:
1385: LEXT(current_thread)
1386:
1387: mfmsr r9 /* Save the old MSR */
1388: rlwinm r8,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
1389: mtmsr r8 /* Interrupts off */
1390: mfsprg r6,0 /* Get the per_proc */
1391: lwz r6,PP_CPU_DATA(r6) /* Get the pointer to the CPU data from per proc */
1392: lwz r3,CPU_ACTIVE_THREAD(r6) /* Get the active thread */
1393: mtmsr r9 /* Restore interruptions to entry */
1394: blr /* Return... */
1395:
1396:
1397: /*
1398: * Return the current preemption level
1399: */
1400:
1401: .align 5
1402: .globl EXT(get_preemption_level)
1403:
1404: LEXT(get_preemption_level)
1405:
1406: mfmsr r9 /* Save the old MSR */
1407: rlwinm r8,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
1408: mtmsr r8 /* Interrupts off */
1409: mfsprg r6,0 /* Get the per_proc */
1410: lwz r6,PP_CPU_DATA(r6) /* Get the pointer to the CPU data from per proc */
1411: lwz r3,CPU_PREEMPTION_LEVEL(r6) /* Get the preemption level */
1412: mtmsr r9 /* Restore interruptions to entry */
1413: blr /* Return... */
1414:
1415:
1416: /*
1417: * Return the simple lock count
1418: */
1419:
1420: .align 5
1421: .globl EXT(get_simple_lock_count)
1422:
1423: LEXT(get_simple_lock_count)
1424:
1425: mfmsr r9 /* Save the old MSR */
1426: rlwinm r8,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 /* Clear interruptions */
1427: mtmsr r8 /* Interrupts off */
1428: mfsprg r6,0 /* Get the per_proc */
1429: lwz r6,PP_CPU_DATA(r6) /* Get the pointer to the CPU data from per proc */
1430: lwz r3,CPU_SIMPLE_LOCK_COUNT(r6) /* Get the simple lock count */
1431: mtmsr r9 /* Restore interruptions to entry */
1432: blr /* Return... */
1433:
1434: #endif /* MACH_RT */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.