![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to locore.s CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple XNU] / XNU / osfmk / i386|
Return to locore.s CVS log | Up to [Apple XNU] / XNU / osfmk / i386 |
1.1 root 1: /*
2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3: *
4: * @APPLE_LICENSE_HEADER_START@
5: *
6: * The contents of this file constitute Original Code as defined in and
7: * are subject to the Apple Public Source License Version 1.1 (the
8: * "License"). You may not use this file except in compliance with the
9: * License. Please obtain a copy of the License at
10: * http://www.apple.com/publicsource and read it before using this file.
11: *
12: * This Original Code and all software distributed under the License are
13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17: * License for the specific language governing rights and limitations
18: * under the License.
19: *
20: * @APPLE_LICENSE_HEADER_END@
21: */
22: /*
23: * @OSF_COPYRIGHT@
24: */
25: /*
26: * Mach Operating System
27: * Copyright (c) 1991,1990 Carnegie Mellon University
28: * All Rights Reserved.
29: *
30: * Permission to use, copy, modify and distribute this software and its
31: * documentation is hereby granted, provided that both the copyright
32: * notice and this permission notice appear in all copies of the
33: * software, derivative works or modified versions, and any portions
34: * thereof, and that both notices appear in supporting documentation.
35: *
36: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
37: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
38: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
39: *
40: * Carnegie Mellon requests users of this software to return to
41: *
42: * Software Distribution Coordinator or [email protected]
43: * School of Computer Science
44: * Carnegie Mellon University
45: * Pittsburgh PA 15213-3890
46: *
47: * any improvements or extensions that they make and grant Carnegie Mellon
48: * the rights to redistribute these changes.
49: */
50:
51: #include <cpus.h>
52: #include <etap.h>
53: #include <etap_event_monitor.h>
54: #include <mach_rt.h>
55: #include <platforms.h>
56: #include <mach_kdb.h>
57: #include <mach_kgdb.h>
58: #include <mach_kdp.h>
59: #include <stat_time.h>
60: #include <mach_assert.h>
61:
62: #include <sys/errno.h>
63: #include <i386/asm.h>
64: #include <i386/cpuid.h>
65: #include <i386/eflags.h>
66: #include <i386/proc_reg.h>
67: #include <i386/trap.h>
68: #include <assym.s>
69: #include <mach/exception_types.h>
70:
71: #include <i386/AT386/mp/mp.h>
72:
73: #define PREEMPT_DEBUG_LOG 0
74:
75: #if __MACHO__
76: /* Under Mach-O, etext is a variable which contains
77: * the last text address
78: */
79: #define ETEXT_ADDR (EXT(etext))
80: #else
81: /* Under ELF and other non-Mach-O formats, the address of
82: * etext represents the last text address
83: */
84: #define ETEXT_ADDR $EXT(etext)
85: #endif
86:
87: #if NCPUS > 1
88:
89: #define CX(addr,reg) addr(,reg,4)
90:
91: #else
92: #define CPU_NUMBER(reg)
93: #define CX(addr,reg) addr
94:
95: #endif /* NCPUS > 1 */
96:
97: .text
98: locore_start:
99:
100: /*
101: * Fault recovery.
102: */
103:
104: #ifdef __MACHO__
105: #define RECOVERY_SECTION .section __VECTORS, __recover
106: #define RETRY_SECTION .section __VECTORS, __retries
107: #else
108: #define RECOVERY_SECTION .text
109: #define RECOVERY_SECTION .text
110: #endif
111:
112: #define RECOVER_TABLE_START \
113: .align 2 ; \
114: .globl EXT(recover_table) ;\
115: LEXT(recover_table) ;\
116: .text
117:
118: #define RECOVER(addr) \
119: .align 2; \
120: .long 9f ;\
121: .long addr ;\
122: .text ;\
123: 9:
124:
125: #define RECOVER_TABLE_END \
126: .align 2 ;\
127: .globl EXT(recover_table_end) ;\
128: LEXT(recover_table_end) ;\
129: .text
130:
131: /*
132: * Retry table for certain successful faults.
133: */
134: #define RETRY_TABLE_START \
135: .align 3; \
136: .globl EXT(retry_table) ;\
137: LEXT(retry_table) ;\
138: .text
139:
140: #define RETRY(addr) \
141: .align 3 ;\
142: .long 9f ;\
143: .long addr ;\
144: .text ;\
145: 9:
146:
147: #define RETRY_TABLE_END \
148: .align 3; \
149: .globl EXT(retry_table_end) ;\
150: LEXT(retry_table_end) ;\
151: .text
152:
153: /*
154: * Allocate recovery and retry tables.
155: */
156: RECOVERY_SECTION
157: RECOVER_TABLE_START
158: RETRY_SECTION
159: RETRY_TABLE_START
160:
161: /*
162: * Timing routines.
163: */
164: #if STAT_TIME
165:
166: #define TIME_TRAP_UENTRY
167: #define TIME_TRAP_UEXIT
168: #define TIME_INT_ENTRY
169: #define TIME_INT_EXIT
170:
171: #else /* microsecond timing */
172:
173: /*
174: * Microsecond timing.
175: * Assumes a free-running microsecond counter.
176: * no TIMER_MAX check needed.
177: */
178:
179: /*
180: * There is only one current time-stamp per CPU, since only
181: * the time-stamp in the current timer is used.
182: * To save time, we allocate the current time-stamps here.
183: */
184: .comm EXT(current_tstamp), 4*NCPUS
185:
186: /*
187: * Update time on user trap entry.
188: * 11 instructions (including cli on entry)
189: * Assumes CPU number in %edx.
190: * Uses %ebx, %ecx.
191: */
192: #define TIME_TRAP_UENTRY \
193: cli /* block interrupts */ ;\
194: movl VA_ETC,%ebx /* get timer value */ ;\
195: movl CX(EXT(current_tstamp),%edx),%ecx /* get old time stamp */;\
196: movl %ebx,CX(EXT(current_tstamp),%edx) /* set new time stamp */;\
197: subl %ecx,%ebx /* elapsed = new-old */ ;\
198: movl CX(EXT(current_timer),%edx),%ecx /* get current timer */;\
199: addl %ebx,LOW_BITS(%ecx) /* add to low bits */ ;\
200: jns 0f /* if overflow, */ ;\
201: call timer_normalize /* normalize timer */ ;\
202: 0: addl $(TH_SYS_TIMER-TH_USER_TIMER),%ecx ;\
203: /* switch to sys timer */;\
204: movl %ecx,CX(EXT(current_timer),%edx) /* make it current */ ;\
205: sti /* allow interrupts */
206:
207: /*
208: * update time on user trap exit.
209: * 10 instructions.
210: * Assumes CPU number in %edx.
211: * Uses %ebx, %ecx.
212: */
213: #define TIME_TRAP_UEXIT \
214: cli /* block interrupts */ ;\
215: movl VA_ETC,%ebx /* get timer */ ;\
216: movl CX(EXT(current_tstamp),%edx),%ecx /* get old time stamp */;\
217: movl %ebx,CX(EXT(current_tstamp),%edx) /* set new time stamp */;\
218: subl %ecx,%ebx /* elapsed = new-old */ ;\
219: movl CX(EXT(current_timer),%edx),%ecx /* get current timer */;\
220: addl %ebx,LOW_BITS(%ecx) /* add to low bits */ ;\
221: jns 0f /* if overflow, */ ;\
222: call timer_normalize /* normalize timer */ ;\
223: 0: addl $(TH_USER_TIMER-TH_SYS_TIMER),%ecx ;\
224: /* switch to user timer */;\
225: movl %ecx,CX(EXT(current_timer),%edx) /* make it current */
226:
227: /*
228: * update time on interrupt entry.
229: * 9 instructions.
230: * Assumes CPU number in %edx.
231: * Leaves old timer in %ebx.
232: * Uses %ecx.
233: */
234: #define TIME_INT_ENTRY \
235: movl VA_ETC,%ecx /* get timer */ ;\
236: movl CX(EXT(current_tstamp),%edx),%ebx /* get old time stamp */;\
237: movl %ecx,CX(EXT(current_tstamp),%edx) /* set new time stamp */;\
238: subl %ebx,%ecx /* elapsed = new-old */ ;\
239: movl CX(EXT(current_timer),%edx),%ebx /* get current timer */;\
240: addl %ecx,LOW_BITS(%ebx) /* add to low bits */ ;\
241: leal CX(0,%edx),%ecx /* timer is 16 bytes */ ;\
242: lea CX(EXT(kernel_timer),%edx),%ecx /* get interrupt timer*/;\
243: movl %ecx,CX(EXT(current_timer),%edx) /* set timer */
244:
245: /*
246: * update time on interrupt exit.
247: * 11 instructions
248: * Assumes CPU number in %edx, old timer in %ebx.
249: * Uses %eax, %ecx.
250: */
251: #define TIME_INT_EXIT \
252: movl VA_ETC,%eax /* get timer */ ;\
253: movl CX(EXT(current_tstamp),%edx),%ecx /* get old time stamp */;\
254: movl %eax,CX(EXT(current_tstamp),%edx) /* set new time stamp */;\
255: subl %ecx,%eax /* elapsed = new-old */ ;\
256: movl CX(EXT(current_timer),%edx),%ecx /* get current timer */;\
257: addl %eax,LOW_BITS(%ecx) /* add to low bits */ ;\
258: jns 0f /* if overflow, */ ;\
259: call timer_normalize /* normalize timer */ ;\
260: 0: testb $0x80,LOW_BITS+3(%ebx) /* old timer overflow? */;\
261: jz 0f /* if overflow, */ ;\
262: movl %ebx,%ecx /* get old timer */ ;\
263: call timer_normalize /* normalize timer */ ;\
264: 0: movl %ebx,CX(EXT(current_timer),%edx) /* set timer */
265:
266:
267: /*
268: * Normalize timer in ecx.
269: * Preserves edx; clobbers eax.
270: */
271: .align ALIGN
272: timer_high_unit:
273: .long TIMER_HIGH_UNIT /* div has no immediate opnd */
274:
275: timer_normalize:
276: pushl %edx /* save registersz */
277: pushl %eax
278: xorl %edx,%edx /* clear divisor high */
279: movl LOW_BITS(%ecx),%eax /* get divisor low */
280: divl timer_high_unit,%eax /* quotient in eax */
281: /* remainder in edx */
282: addl %eax,HIGH_BITS_CHECK(%ecx) /* add high_inc to check */
283: movl %edx,LOW_BITS(%ecx) /* remainder to low_bits */
284: addl %eax,HIGH_BITS(%ecx) /* add high_inc to high bits */
285: popl %eax /* restore register */
286: popl %edx
287: ret
288:
289: /*
290: * Switch to a new timer.
291: */
292: Entry(timer_switch)
293: CPU_NUMBER(%edx) /* get this CPU */
294: movl VA_ETC,%ecx /* get timer */
295: movl CX(EXT(current_tstamp),%edx),%eax /* get old time stamp */
296: movl %ecx,CX(EXT(current_tstamp),%edx) /* set new time stamp */
297: subl %ecx,%eax /* elapsed = new - old */
298: movl CX(EXT(current_timer),%edx),%ecx /* get current timer */
299: addl %eax,LOW_BITS(%ecx) /* add to low bits */
300: jns 0f /* if overflow, */
301: call timer_normalize /* normalize timer */
302: 0:
303: movl S_ARG0,%ecx /* get new timer */
304: movl %ecx,CX(EXT(current_timer),%edx) /* set timer */
305: ret
306:
307: /*
308: * Initialize the first timer for a CPU.
309: */
310: Entry(start_timer)
311: CPU_NUMBER(%edx) /* get this CPU */
312: movl VA_ETC,%ecx /* get timer */
313: movl %ecx,CX(EXT(current_tstamp),%edx) /* set initial time stamp */
314: movl S_ARG0,%ecx /* get timer */
315: movl %ecx,CX(EXT(current_timer),%edx) /* set initial timer */
316: ret
317:
318: #endif /* accurate timing */
319:
320: /*
321: * Encapsulate the transfer of exception stack frames between a PCB
322: * and a thread stack. Since the whole point of these is to emulate
323: * a call or exception that changes privilege level, both macros
324: * assume that there is no user esp or ss stored in the source
325: * frame (because there was no change of privilege to generate them).
326: */
327:
328: /*
329: * Transfer a stack frame from a thread's user stack to its PCB.
330: * We assume the thread and stack addresses have been loaded into
331: * registers (our arguments).
332: *
333: * The macro overwrites edi, esi, ecx and whatever registers hold the
334: * thread and stack addresses (which can't be one of the above three).
335: * The thread address is overwritten with the address of its saved state
336: * (where the frame winds up).
337: *
338: * Must be called on kernel stack.
339: */
340: #define FRAME_STACK_TO_PCB(thread, stkp) ;\
341: movl ACT_PCB(thread),thread /* get act`s PCB */ ;\
342: leal PCB_ISS(thread),%edi /* point to PCB`s saved state */;\
343: movl %edi,thread /* save for later */ ;\
344: movl stkp,%esi /* point to start of frame */ ;\
345: movl $R_UESP,%ecx ;\
346: sarl $2,%ecx /* word count for transfer */ ;\
347: cld /* we`re incrementing */ ;\
348: rep ;\
349: movsl /* transfer the frame */ ;\
350: addl $R_UESP,stkp /* derive true "user" esp */ ;\
351: movl stkp,R_UESP(thread) /* store in PCB */ ;\
352: movl $0,%ecx ;\
353: mov %ss,%cx /* get current ss */ ;\
354: movl %ecx,R_SS(thread) /* store in PCB */
355:
356: /*
357: * Transfer a stack frame from a thread's PCB to the stack pointed
358: * to by the PCB. We assume the thread address has been loaded into
359: * a register (our argument).
360: *
361: * The macro overwrites edi, esi, ecx and whatever register holds the
362: * thread address (which can't be one of the above three). The
363: * thread address is overwritten with the address of its saved state
364: * (where the frame winds up).
365: *
366: * Must be called on kernel stack.
367: */
368: #define FRAME_PCB_TO_STACK(thread) ;\
369: movl ACT_PCB(thread),%esi /* get act`s PCB */ ;\
370: leal PCB_ISS(%esi),%esi /* point to PCB`s saved state */;\
371: movl R_UESP(%esi),%edi /* point to end of dest frame */;\
372: movl ACT_MAP(thread),%ecx /* get act's map */ ;\
373: movl MAP_PMAP(%ecx),%ecx /* get map's pmap */ ;\
374: cmpl EXT(kernel_pmap), %ecx /* If kernel loaded task */ ;\
375: jz 1f /* use kernel data segment */ ;\
376: movl $USER_DS,%cx /* else use user data segment */;\
377: mov %cx,%es ;\
378: 1: ;\
379: movl $R_UESP,%ecx ;\
380: subl %ecx,%edi /* derive start of frame */ ;\
381: movl %edi,thread /* save for later */ ;\
382: sarl $2,%ecx /* word count for transfer */ ;\
383: cld /* we`re incrementing */ ;\
384: rep ;\
385: movsl /* transfer the frame */ ;\
386: mov %ss,%cx /* restore kernel segments */ ;\
387: mov %cx,%es
388:
389: #undef PDEBUG
390:
391: #ifdef PDEBUG
392:
393: /*
394: * Traditional, not ANSI.
395: */
396: #define CAH(label) \
397: .data ;\
398: .globl label/**/count ;\
399: label/**/count: ;\
400: .long 0 ;\
401: .globl label/**/limit ;\
402: label/**/limit: ;\
403: .long 0 ;\
404: .text ;\
405: addl $1,%ss:label/**/count ;\
406: cmpl $0,label/**/limit ;\
407: jz label/**/exit ;\
408: pushl %eax ;\
409: label/**/loop: ;\
410: movl %ss:label/**/count,%eax ;\
411: cmpl %eax,%ss:label/**/limit ;\
412: je label/**/loop ;\
413: popl %eax ;\
414: label/**/exit:
415:
416: #else /* PDEBUG */
417:
418: #define CAH(label)
419:
420: #endif /* PDEBUG */
421:
422: #if MACH_KDB
423: /*
424: * Last-ditch debug code to handle faults that might result
425: * from entering kernel (from collocated server) on an invalid
426: * stack. On collocated entry, there's no hardware-initiated
427: * stack switch, so a valid stack must be in place when an
428: * exception occurs, or we may double-fault.
429: *
430: * In case of a double-fault, our only recourse is to switch
431: * hardware "tasks", so that we avoid using the current stack.
432: *
433: * The idea here is just to get the processor into the debugger,
434: * post-haste. No attempt is made to fix up whatever error got
435: * us here, so presumably continuing from the debugger will
436: * simply land us here again -- at best.
437: */
438: #if 0
439: /*
440: * Note that the per-fault entry points are not currently
441: * functional. The only way to make them work would be to
442: * set up separate TSS's for each fault type, which doesn't
443: * currently seem worthwhile. (The offset part of a task
444: * gate is always ignored.) So all faults that task switch
445: * currently resume at db_task_start.
446: */
447: /*
448: * Double fault (Murphy's point) - error code (0) on stack
449: */
450: Entry(db_task_dbl_fault)
451: popl %eax
452: movl $(T_DOUBLE_FAULT),%ebx
453: jmp db_task_start
454: /*
455: * Segment not present - error code on stack
456: */
457: Entry(db_task_seg_np)
458: popl %eax
459: movl $(T_SEGMENT_NOT_PRESENT),%ebx
460: jmp db_task_start
461: /*
462: * Stack fault - error code on (current) stack
463: */
464: Entry(db_task_stk_fault)
465: popl %eax
466: movl $(T_STACK_FAULT),%ebx
467: jmp db_task_start
468: /*
469: * General protection fault - error code on stack
470: */
471: Entry(db_task_gen_prot)
472: popl %eax
473: movl $(T_GENERAL_PROTECTION),%ebx
474: jmp db_task_start
475: #endif /* 0 */
476: /*
477: * The entry point where execution resumes after last-ditch debugger task
478: * switch.
479: */
480: Entry(db_task_start)
481: movl %esp,%edx
482: subl $ISS_SIZE,%edx
483: movl %edx,%esp /* allocate i386_saved_state on stack */
484: movl %eax,R_ERR(%esp)
485: movl %ebx,R_TRAPNO(%esp)
486: pushl %edx
487: #if NCPUS > 1
488: CPU_NUMBER(%edx)
489: movl CX(EXT(mp_dbtss),%edx),%edx
490: movl TSS_LINK(%edx),%eax
491: #else
492: movl EXT(dbtss)+TSS_LINK,%eax
493: #endif
494: pushl %eax /* pass along selector of previous TSS */
495: call EXT(db_tss_to_frame)
496: popl %eax /* get rid of TSS selector */
497: call EXT(db_trap_from_asm)
498: addl $0x4,%esp
499: /*
500: * And now...?
501: */
502: iret /* ha, ha, ha... */
503: #endif /* MACH_KDB */
504:
505: /*
506: * Trap/interrupt entry points.
507: *
508: * All traps must create the following save area on the PCB "stack":
509: *
510: * gs
511: * fs
512: * es
513: * ds
514: * edi
515: * esi
516: * ebp
517: * cr2 if page fault - otherwise unused
518: * ebx
519: * edx
520: * ecx
521: * eax
522: * trap number
523: * error code
524: * eip
525: * cs
526: * eflags
527: * user esp - if from user
528: * user ss - if from user
529: * es - if from V86 thread
530: * ds - if from V86 thread
531: * fs - if from V86 thread
532: * gs - if from V86 thread
533: *
534: */
535:
536: /*
537: * General protection or segment-not-present fault.
538: * Check for a GP/NP fault in the kernel_return
539: * sequence; if there, report it as a GP/NP fault on the user's instruction.
540: *
541: * esp-> 0: trap code (NP or GP)
542: * 4: segment number in error
543: * 8 eip
544: * 12 cs
545: * 16 eflags
546: * 20 old registers (trap is from kernel)
547: */
548: Entry(t_gen_prot)
549: pushl $(T_GENERAL_PROTECTION) /* indicate fault type */
550: jmp trap_check_kernel_exit /* check for kernel exit sequence */
551:
552: Entry(t_segnp)
553: pushl $(T_SEGMENT_NOT_PRESENT)
554: /* indicate fault type */
555:
556: trap_check_kernel_exit:
557: testl $(EFL_VM),16(%esp) /* is trap from V86 mode? */
558: jnz EXT(alltraps) /* isn`t kernel trap if so */
559: testl $3,12(%esp) /* is trap from kernel mode? */
560: jne EXT(alltraps) /* if so: */
561: /* check for the kernel exit sequence */
562: cmpl $EXT(kret_iret),8(%esp) /* on IRET? */
563: je fault_iret
564: cmpl $EXT(kret_popl_ds),8(%esp) /* popping DS? */
565: je fault_popl_ds
566: cmpl $EXT(kret_popl_es),8(%esp) /* popping ES? */
567: je fault_popl_es
568: cmpl $EXT(kret_popl_fs),8(%esp) /* popping FS? */
569: je fault_popl_fs
570: cmpl $EXT(kret_popl_gs),8(%esp) /* popping GS? */
571: je fault_popl_gs
572: take_fault: /* if none of the above: */
573: jmp EXT(alltraps) /* treat as normal trap. */
574:
575: /*
576: * GP/NP fault on IRET: CS or SS is in error.
577: * All registers contain the user's values.
578: *
579: * on SP is
580: * 0 trap number
581: * 4 errcode
582: * 8 eip
583: * 12 cs --> trapno
584: * 16 efl --> errcode
585: * 20 user eip
586: * 24 user cs
587: * 28 user eflags
588: * 32 user esp
589: * 36 user ss
590: */
591: fault_iret:
592: movl %eax,8(%esp) /* save eax (we don`t need saved eip) */
593: popl %eax /* get trap number */
594: movl %eax,12-4(%esp) /* put in user trap number */
595: popl %eax /* get error code */
596: movl %eax,16-8(%esp) /* put in user errcode */
597: popl %eax /* restore eax */
598: CAH(fltir)
599: jmp EXT(alltraps) /* take fault */
600:
601: /*
602: * Fault restoring a segment register. The user's registers are still
603: * saved on the stack. The offending segment register has not been
604: * popped.
605: */
606: fault_popl_ds:
607: popl %eax /* get trap number */
608: popl %edx /* get error code */
609: addl $12,%esp /* pop stack to user regs */
610: jmp push_es /* (DS on top of stack) */
611: fault_popl_es:
612: popl %eax /* get trap number */
613: popl %edx /* get error code */
614: addl $12,%esp /* pop stack to user regs */
615: jmp push_fs /* (ES on top of stack) */
616: fault_popl_fs:
617: popl %eax /* get trap number */
618: popl %edx /* get error code */
619: addl $12,%esp /* pop stack to user regs */
620: jmp push_gs /* (FS on top of stack) */
621: fault_popl_gs:
622: popl %eax /* get trap number */
623: popl %edx /* get error code */
624: addl $12,%esp /* pop stack to user regs */
625: jmp push_segregs /* (GS on top of stack) */
626:
627: push_es:
628: pushl %es /* restore es, */
629: push_fs:
630: pushl %fs /* restore fs, */
631: push_gs:
632: pushl %gs /* restore gs. */
633: push_segregs:
634: movl %eax,R_TRAPNO(%esp) /* set trap number */
635: movl %edx,R_ERR(%esp) /* set error code */
636: CAH(fltpp)
637: jmp trap_set_segs /* take trap */
638:
639: /*
640: * Debug trap. Check for single-stepping across system call into
641: * kernel. If this is the case, taking the debug trap has turned
642: * off single-stepping - save the flags register with the trace
643: * bit set.
644: */
645: Entry(t_debug)
646: testl $(EFL_VM),8(%esp) /* is trap from V86 mode? */
647: jnz 0f /* isn`t kernel trap if so */
648: testl $3,4(%esp) /* is trap from kernel mode? */
649: jnz 0f /* if so: */
650: cmpl $syscall_entry,(%esp) /* system call entry? */
651: jne 0f /* if so: */
652: /* flags are sitting where syscall */
653: /* wants them */
654: addl $8,%esp /* remove eip/cs */
655: jmp syscall_entry_2 /* continue system call entry */
656:
657: 0: pushl $0 /* otherwise: */
658: pushl $(T_DEBUG) /* handle as normal */
659: jmp EXT(alltraps) /* debug fault */
660:
661: /*
662: * Page fault traps save cr2.
663: */
664: Entry(t_page_fault)
665: pushl $(T_PAGE_FAULT) /* mark a page fault trap */
666: pusha /* save the general registers */
667: movl %cr2,%eax /* get the faulting address */
668: movl %eax,12(%esp) /* save in esp save slot */
669: jmp trap_push_segs /* continue fault */
670:
671: /*
672: * All 'exceptions' enter here with:
673: * esp-> trap number
674: * error code
675: * old eip
676: * old cs
677: * old eflags
678: * old esp if trapped from user
679: * old ss if trapped from user
680: *
681: * NB: below use of CPU_NUMBER assumes that macro will use correct
682: * segment register for any kernel data accesses.
683: */
684: Entry(alltraps)
685: pusha /* save the general registers */
686: trap_push_segs:
687: pushl %ds /* save the segment registers */
688: pushl %es
689: pushl %fs
690: pushl %gs
691:
692: trap_set_segs:
693: movl %ss,%ax
694: movl %ax,%ds
695: movl %ax,%es /* switch to kernel data seg */
696: cld /* clear direction flag */
697: testl $(EFL_VM),R_EFLAGS(%esp) /* in V86 mode? */
698: jnz trap_from_user /* user mode trap if so */
699: testb $3,R_CS(%esp) /* user mode trap? */
700: jnz trap_from_user
701: CPU_NUMBER(%edx)
702: cmpl $0,CX(EXT(active_kloaded),%edx)
703: je trap_from_kernel /* if clear, truly in kernel */
704: #ifdef FIXME
705: cmpl ETEXT_ADDR,R_EIP(%esp) /* pc within kernel? */
706: jb trap_from_kernel
707: #endif
708: trap_from_kloaded:
709: /*
710: * We didn't enter here "through" PCB (i.e., using ring 0 stack),
711: * so transfer the stack frame into the PCB explicitly, then
712: * start running on resulting "PCB stack". We have to set
713: * up a simulated "uesp" manually, since there's none in the
714: * frame.
715: */
716: mov $CPU_DATA,%dx
717: mov %dx,%gs
718: CAH(atstart)
719: CPU_NUMBER(%edx)
720: movl CX(EXT(active_kloaded),%edx),%ebx
721: movl CX(EXT(kernel_stack),%edx),%eax
722: xchgl %esp,%eax
723: FRAME_STACK_TO_PCB(%ebx,%eax)
724: CAH(atend)
725: jmp EXT(take_trap)
726:
727: trap_from_user:
728: mov $CPU_DATA,%ax
729: mov %ax,%gs
730:
731: CPU_NUMBER(%edx)
732: TIME_TRAP_UENTRY
733:
734: movl CX(EXT(kernel_stack),%edx),%ebx
735: xchgl %ebx,%esp /* switch to kernel stack */
736: /* user regs pointer already set */
737: LEXT(take_trap)
738: pushl %ebx /* record register save area */
739: pushl %ebx /* pass register save area to trap */
740: call EXT(user_trap) /* call user trap routine */
741: movl 4(%esp),%esp /* switch back to PCB stack */
742:
743: /*
744: * Return from trap or system call, checking for ASTs.
745: * On PCB stack.
746: */
747:
748: LEXT(return_from_trap)
749: CPU_NUMBER(%edx)
750: cmpl $0,CX(EXT(need_ast),%edx)
751: je EXT(return_to_user) /* if we need an AST: */
752:
753: movl CX(EXT(kernel_stack),%edx),%esp
754: /* switch to kernel stack */
755: pushl $0 /* push preemption flag */
756: call EXT(i386_astintr) /* take the AST */
757: addl $4,%esp /* pop preemption flag */
758: popl %esp /* switch back to PCB stack (w/exc link) */
759: jmp EXT(return_from_trap) /* and check again (rare) */
760: /* ASTs after this point will */
761: /* have to wait */
762:
763: /*
764: * Arrange the checks needed for kernel-loaded (or kernel-loading)
765: * threads so that branch is taken in kernel-loaded case.
766: */
767: LEXT(return_to_user)
768: TIME_TRAP_UEXIT
769: CPU_NUMBER(%eax)
770: cmpl $0,CX(EXT(active_kloaded),%eax)
771: jnz EXT(return_xfer_stack)
772: movl $CPD_ACTIVE_THREAD,%ebx
773: movl %gs:(%ebx),%ebx /* get active thread */
774: movl TH_TOP_ACT(%ebx),%ebx /* get thread->top_act */
775: cmpl $0,ACT_KLOADING(%ebx) /* check if kernel-loading */
776: jnz EXT(return_kernel_loading)
777:
778: #if MACH_RT
779: #if MACH_ASSERT
780: movl $CPD_PREEMPTION_LEVEL,%ebx
781: cmpl $0,%gs:(%ebx)
782: je EXT(return_from_kernel)
783: int $3
784: #endif /* MACH_ASSERT */
785: #endif /* MACH_RT */
786:
787: /*
788: * Return from kernel mode to interrupted thread.
789: */
790:
791: LEXT(return_from_kernel)
792: LEXT(kret_popl_gs)
793: popl %gs /* restore segment registers */
794: LEXT(kret_popl_fs)
795: popl %fs
796: LEXT(kret_popl_es)
797: popl %es
798: LEXT(kret_popl_ds)
799: popl %ds
800: popa /* restore general registers */
801: addl $8,%esp /* discard trap number and error code */
802:
803: LEXT(kret_iret)
804: iret /* return from interrupt */
805:
806:
807: LEXT(return_xfer_stack)
808: /*
809: * If we're on PCB stack in a kernel-loaded task, we have
810: * to transfer saved state back to thread stack and swap
811: * stack pointers here, because the hardware's not going
812: * to do so for us.
813: */
814: CAH(rxsstart)
815: CPU_NUMBER(%eax)
816: movl CX(EXT(kernel_stack),%eax),%esp
817: movl CX(EXT(active_kloaded),%eax),%eax
818: FRAME_PCB_TO_STACK(%eax)
819: movl %eax,%esp
820: CAH(rxsend)
821: jmp EXT(return_from_kernel)
822:
823: /*
824: * Hate to put this here, but setting up a separate swap_func for
825: * kernel-loaded threads no longer works, since thread executes
826: * "for a while" (i.e., until it reaches glue code) when first
827: * created, even if it's nominally suspended. Hence we can't
828: * transfer the PCB when the thread first resumes, because we
829: * haven't initialized it yet.
830: */
831: /*
832: * Have to force transfer to new stack "manually". Use a string
833: * move to transfer all of our saved state to the stack pointed
834: * to by iss.uesp, then install a pointer to it as our current
835: * stack pointer.
836: */
837: LEXT(return_kernel_loading)
838: CPU_NUMBER(%eax)
839: movl CX(EXT(kernel_stack),%eax),%esp
840: movl $CPD_ACTIVE_THREAD,%ebx
841: movl %gs:(%ebx),%ebx /* get active thread */
842: movl TH_TOP_ACT(%ebx),%ebx /* get thread->top_act */
843: movl %ebx,%edx /* save for later */
844: movl $0,ACT_KLOADING(%edx) /* clear kernel-loading bit */
845: FRAME_PCB_TO_STACK(%ebx)
846: movl %ebx,%esp /* start running on new stack */
847: movl $1,ACT_KLOADED(%edx) /* set kernel-loaded bit */
848: movl %edx,CX(EXT(active_kloaded),%eax) /* set cached indicator */
849: jmp EXT(return_from_kernel)
850:
851: /*
852: * Trap from kernel mode. No need to switch stacks or load segment registers.
853: */
854: trap_from_kernel:
855: #if MACH_KDB || MACH_KGDB
856: mov $CPU_DATA,%ax
857: mov %ax,%gs
858: movl %esp,%ebx /* save current stack */
859:
860: cmpl EXT(int_stack_high),%esp /* on an interrupt stack? */
861: jb 6f /* OK if so */
862:
863: #if MACH_KGDB
864: cmpl $0,EXT(kgdb_active) /* Unexpected trap in kgdb */
865: je 0f /* no */
866:
867: pushl %esp /* Already on kgdb stack */
868: cli
869: call EXT(kgdb_trap)
870: addl $4,%esp
871: jmp EXT(return_from_kernel)
872: 0: /* should kgdb handle this exception? */
873: cmpl $(T_NO_FPU),R_TRAPNO(%esp) /* FPU disabled? */
874: je 2f /* yes */
875: cmpl $(T_PAGE_FAULT),R_TRAPNO(%esp) /* page fault? */
876: je 2f /* yes */
877: 1:
878: cli /* disable interrupts */
879: CPU_NUMBER(%edx) /* get CPU number */
880: movl CX(EXT(kgdb_stacks),%edx),%ebx
881: xchgl %ebx,%esp /* switch to kgdb stack */
882: pushl %ebx /* pass old sp as an arg */
883: call EXT(kgdb_from_kernel)
884: popl %esp /* switch back to kernel stack */
885: jmp EXT(return_from_kernel)
886: 2:
887: #endif /* MACH_KGDB */
888:
889: #if MACH_KDB
890: cmpl $0,EXT(db_active) /* could trap be from ddb? */
891: je 3f /* no */
892: #if NCPUS > 1
893: CPU_NUMBER(%edx) /* see if this CPU is in ddb */
894: cmpl $0,CX(EXT(kdb_active),%edx)
895: je 3f /* no */
896: #endif /* NCPUS > 1 */
897: pushl %esp
898: call EXT(db_trap_from_asm)
899: addl $0x4,%esp
900: jmp EXT(return_from_kernel)
901:
902: 3:
903: /*
904: * Dilemma: don't want to switch to kernel_stack if trap
905: * "belongs" to ddb; don't want to switch to db_stack if
906: * trap "belongs" to kernel. So have to duplicate here the
907: * set of trap types that kernel_trap() handles. Note that
908: * "unexpected" page faults will not be handled by kernel_trap().
909: * In this panic-worthy case, we fall into the debugger with
910: * kernel_stack containing the call chain that led to the
911: * bogus fault.
912: */
913: movl R_TRAPNO(%esp),%edx
914: cmpl $(T_PAGE_FAULT),%edx
915: je 4f
916: cmpl $(T_NO_FPU),%edx
917: je 4f
918: cmpl $(T_FPU_FAULT),%edx
919: je 4f
920: cmpl $(T_FLOATING_POINT_ERROR),%edx
921: je 4f
922: cmpl $(T_PREEMPT),%edx
923: jne 7f
924: 4:
925: #endif /* MACH_KDB */
926:
927: CPU_NUMBER(%edx) /* get CPU number */
928: cmpl CX(EXT(kernel_stack),%edx),%esp
929: /* if not already on kernel stack, */
930: ja 5f /* check some more */
931: cmpl CX(EXT(active_stacks),%edx),%esp
932: ja 6f /* on kernel stack: no switch */
933: 5:
934: movl CX(EXT(kernel_stack),%edx),%esp
935: 6:
936: pushl %ebx /* save old stack */
937: pushl %ebx /* pass as parameter */
938: call EXT(kernel_trap) /* to kernel trap routine */
939: addl $4,%esp /* pop parameter */
940: testl %eax,%eax
941: jne 8f
942: /*
943: * If kernel_trap returns false, trap wasn't handled.
944: */
945: 7:
946: #if MACH_KDB
947: CPU_NUMBER(%edx)
948: movl CX(EXT(db_stacks),%edx),%esp
949: pushl %ebx /* pass old stack as parameter */
950: call EXT(db_trap_from_asm)
951: #endif /* MACH_KDB */
952: #if MACH_KGDB
953: cli /* disable interrupts */
954: CPU_NUMBER(%edx) /* get CPU number */
955: movl CX(EXT(kgdb_stacks),%edx),%esp
956: pushl %ebx /* pass old stack as parameter */
957: call EXT(kgdb_from_kernel)
958: #endif /* MACH_KGDB */
959: addl $4,%esp /* pop parameter */
960: testl %eax,%eax
961: jne 8f
962: /*
963: * Likewise, if kdb_trap/kgdb_from_kernel returns false, trap
964: * wasn't handled.
965: */
966: pushl %ebx /* pass old stack as parameter */
967: call EXT(panic_trap)
968: addl $4,%esp /* pop parameter */
969: 8:
970: movl %ebx,%esp /* get old stack (from callee-saves reg) */
971: #else /* MACH_KDB || MACH_KGDB */
972: pushl %esp /* pass parameter */
973: call EXT(kernel_trap) /* to kernel trap routine */
974: addl $4,%esp /* pop parameter */
975: #endif /* MACH_KDB || MACH_KGDB */
976:
977: #if MACH_RT
978: CPU_NUMBER(%edx)
979:
980: movl CX(EXT(need_ast),%edx),%eax /* get pending asts */
981: testl $AST_URGENT,%eax /* any urgent preemption? */
982: je EXT(return_from_kernel) /* no, nothing to do */
983: cmpl $0,EXT(preemptable) /* kernel-mode, preemption enabled? */
984: je EXT(return_from_kernel) /* no, skip it */
985: cmpl $T_PREEMPT,48(%esp) /* preempt request? */
986: jne EXT(return_from_kernel) /* no, nothing to do */
987: movl CX(EXT(kernel_stack),%edx),%eax
988: movl %esp,%ecx
989: xorl %eax,%ecx
990: andl $(-KERNEL_STACK_SIZE),%ecx
991: testl %ecx,%ecx /* are we on the kernel stack? */
992: jne EXT(return_from_kernel) /* no, skip it */
993:
994: #if PREEMPT_DEBUG_LOG
995: pushl 28(%esp) /* stack pointer */
996: pushl 24+4(%esp) /* frame pointer */
997: pushl 56+8(%esp) /* stack pointer */
998: pushl $0f
999: call EXT(log_thread_action)
1000: addl $16, %esp
1001: .data
1002: 0: String "trap preempt eip"
1003: .text
1004: #endif /* PREEMPT_DEBUG_LOG */
1005:
1006: pushl $1 /* push preemption flag */
1007: call EXT(i386_astintr) /* take the AST */
1008: addl $4,%esp /* pop preemption flag */
1009: #endif /* MACH_RT */
1010:
1011: jmp EXT(return_from_kernel)
1012:
1013: /*
1014: * Called as a function, makes the current thread
1015: * return from the kernel as if from an exception.
1016: */
1017:
1018: .globl EXT(thread_exception_return)
1019: .globl EXT(thread_bootstrap_return)
1020: LEXT(thread_exception_return)
1021: LEXT(thread_bootstrap_return)
1022: movl %esp,%ecx /* get kernel stack */
1023: or $(KERNEL_STACK_SIZE-1),%ecx
1024: movl -3-IKS_SIZE(%ecx),%esp /* switch back to PCB stack */
1025: jmp EXT(return_from_trap)
1026:
1027: Entry(call_continuation)
1028: movl S_ARG0,%eax /* get continuation */
1029: movl %esp,%ecx /* get kernel stack */
1030: or $(KERNEL_STACK_SIZE-1),%ecx
1031: addl $(-3-IKS_SIZE),%ecx
1032: movl %ecx,%esp /* pop the stack */
1033: xorl %ebp,%ebp /* zero frame pointer */
1034: jmp *%eax /* goto continuation */
1035:
1036: #if 0
1037: #define LOG_INTERRUPT(info,msg) \
1038: pushal ; \
1039: pushl msg ; \
1040: pushl info ; \
1041: call EXT(log_thread_action) ; \
1042: add $8,%esp ; \
1043: popal
1044: #define CHECK_INTERRUPT_TIME(n) \
1045: pushal ; \
1046: pushl $n ; \
1047: call EXT(check_thread_time) ; \
1048: add $4,%esp ; \
1049: popal
1050: #else
1051: #define LOG_INTERRUPT(info,msg)
1052: #define CHECK_INTERRUPT_TIME(n)
1053: #endif
1054:
1055: imsg_start:
1056: String "interrupt start"
1057: imsg_end:
1058: String "interrupt end"
1059:
1060: /*
1061: * All interrupts enter here.
1062: * old %eax on stack; interrupt number in %eax.
1063: */
1064: Entry(all_intrs)
1065: pushl %ecx /* save registers */
1066: pushl %edx
1067: cld /* clear direction flag */
1068:
1069: cmpl %ss:EXT(int_stack_high),%esp /* on an interrupt stack? */
1070: jb int_from_intstack /* if not: */
1071:
1072: pushl %ds /* save segment registers */
1073: pushl %es
1074: mov %ss,%dx /* switch to kernel segments */
1075: mov %dx,%ds
1076: mov %dx,%es
1077: mov $CPU_DATA,%dx
1078: mov %dx,%gs
1079:
1080: CPU_NUMBER(%edx)
1081:
1082: movl CX(EXT(int_stack_top),%edx),%ecx
1083: xchgl %ecx,%esp /* switch to interrupt stack */
1084:
1085: #if STAT_TIME
1086: pushl %ecx /* save pointer to old stack */
1087: #else
1088: pushl %ebx /* save %ebx - out of the way */
1089: /* so stack looks the same */
1090: pushl %ecx /* save pointer to old stack */
1091: TIME_INT_ENTRY /* do timing */
1092: #endif
1093:
1094: #if MACH_RT
1095: movl $CPD_PREEMPTION_LEVEL,%edx
1096: incl %gs:(%edx)
1097: #endif /* MACH_RT */
1098:
1099: movl $CPD_INTERRUPT_LEVEL,%edx
1100: incl %gs:(%edx)
1101:
1102: pushl %eax /* Push trap number */
1103: call EXT(PE_incoming_interrupt) /* call generic interrupt routine */
1104: addl $4,%esp /* Pop trap number */
1105:
1106: .globl EXT(return_to_iret)
1107: LEXT(return_to_iret) /* (label for kdb_kintr and hardclock) */
1108:
1109: movl $CPD_INTERRUPT_LEVEL,%edx
1110: decl %gs:(%edx)
1111:
1112: #if MACH_RT
1113: movl $CPD_PREEMPTION_LEVEL,%edx
1114: decl %gs:(%edx)
1115: #endif /* MACH_RT */
1116:
1117: #if STAT_TIME
1118: #else
1119: TIME_INT_EXIT /* do timing */
1120: movl 4(%esp),%ebx /* restore the extra reg we saved */
1121: #endif
1122:
1123: popl %esp /* switch back to old stack */
1124:
1125: CPU_NUMBER(%edx)
1126: movl CX(EXT(need_ast),%edx),%eax
1127: testl %eax,%eax /* any pending asts? */
1128: je 1f /* no, nothing to do */
1129: testl $(EFL_VM),I_EFL(%esp) /* if in V86 */
1130: jnz ast_from_interrupt /* take it */
1131: testb $3,I_CS(%esp) /* user mode, */
1132: jnz ast_from_interrupt /* take it */
1133: #ifdef FIXME
1134: cmpl ETEXT_ADDR,I_EIP(%esp) /* if within kernel-loaded task, */
1135: jnb ast_from_interrupt /* take it */
1136: #endif
1137:
1138: #if MACH_RT
1139: cmpl $0,EXT(preemptable) /* kernel-mode, preemption enabled? */
1140: je 1f /* no, skip it */
1141: movl $CPD_PREEMPTION_LEVEL,%ecx
1142: cmpl $0,%gs:(%ecx) /* preemption masked? */
1143: jne 1f /* yes, skip it */
1144: testl $AST_URGENT,%eax /* any urgent requests? */
1145: je 1f /* no, skip it */
1146: cmpl $LEXT(locore_end),I_EIP(%esp) /* are we in locore code? */
1147: jb 1f /* yes, skip it */
1148: movl CX(EXT(kernel_stack),%edx),%eax
1149: movl %esp,%ecx
1150: xorl %eax,%ecx
1151: andl $(-KERNEL_STACK_SIZE),%ecx
1152: testl %ecx,%ecx /* are we on the kernel stack? */
1153: jne 1f /* no, skip it */
1154:
1155: /*
1156: * Take an AST from kernel space. We don't need (and don't want)
1157: * to do as much as the case where the interrupt came from user
1158: * space.
1159: */
1160: #if PREEMPT_DEBUG_LOG
1161: pushl $0
1162: pushl $0
1163: pushl I_EIP+8(%esp)
1164: pushl $0f
1165: call EXT(log_thread_action)
1166: addl $16, %esp
1167: .data
1168: 0: String "intr preempt eip"
1169: .text
1170: #endif /* PREEMPT_DEBUG_LOG */
1171:
1172: sti
1173: pushl $1 /* push preemption flag */
1174: call EXT(i386_astintr) /* take the AST */
1175: addl $4,%esp /* pop preemption flag */
1176: #endif /* MACH_RT */
1177:
1178: 1:
1179: pop %es /* restore segment regs */
1180: pop %ds
1181: pop %edx
1182: pop %ecx
1183: pop %eax
1184: iret /* return to caller */
1185:
1186: int_from_intstack:
1187: #if MACH_RT
1188: movl $CPD_PREEMPTION_LEVEL,%edx
1189: incl %gs:(%edx)
1190: #endif /* MACH_RT */
1191:
1192: movl $CPD_INTERRUPT_LEVEL,%edx
1193: incl %gs:(%edx)
1194:
1195: pushl %eax /* Push trap number */
1196:
1197: call EXT(PE_incoming_interrupt)
1198:
1199: LEXT(return_to_iret_i) /* ( label for kdb_kintr) */
1200:
1201: addl $4,%esp /* pop trap number */
1202:
1203: movl $CPD_INTERRUPT_LEVEL,%edx
1204: decl %gs:(%edx)
1205:
1206: #if MACH_RT
1207: movl $CPD_PREEMPTION_LEVEL,%edx
1208: decl %gs:(%edx)
1209: #endif /* MACH_RT */
1210:
1211: pop %edx /* must have been on kernel segs */
1212: pop %ecx
1213: pop %eax /* no ASTs */
1214: iret
1215:
1216: /*
1217: * Take an AST from an interrupt.
1218: * On PCB stack.
1219: * sp-> es -> edx
1220: * ds -> ecx
1221: * edx -> eax
1222: * ecx -> trapno
1223: * eax -> code
1224: * eip
1225: * cs
1226: * efl
1227: * esp
1228: * ss
1229: */
1230: ast_from_interrupt:
1231: pop %es /* restore all registers ... */
1232: pop %ds
1233: popl %edx
1234: popl %ecx
1235: popl %eax
1236: sti /* Reenable interrupts */
1237: pushl $0 /* zero code */
1238: pushl $0 /* zero trap number */
1239: pusha /* save general registers */
1240: push %ds /* save segment registers */
1241: push %es
1242: push %fs
1243: push %gs
1244: mov %ss,%dx /* switch to kernel segments */
1245: mov %dx,%ds
1246: mov %dx,%es
1247: mov $CPU_DATA,%dx
1248: mov %dx,%gs
1249:
1250: /*
1251: * See if we interrupted a kernel-loaded thread executing
1252: * in its own task.
1253: */
1254: CPU_NUMBER(%edx)
1255: testl $(EFL_VM),R_EFLAGS(%esp) /* in V86 mode? */
1256: jnz 0f /* user mode trap if so */
1257: testb $3,R_CS(%esp)
1258: jnz 0f /* user mode, back to normal */
1259: #ifdef FIXME
1260: cmpl ETEXT_ADDR,R_EIP(%esp)
1261: jb 0f /* not kernel-loaded, back to normal */
1262: #endif
1263:
1264: /*
1265: * Transfer the current stack frame by hand into the PCB.
1266: */
1267: CAH(afistart)
1268: movl CX(EXT(active_kloaded),%edx),%eax
1269: movl CX(EXT(kernel_stack),%edx),%ebx
1270: xchgl %ebx,%esp
1271: FRAME_STACK_TO_PCB(%eax,%ebx)
1272: CAH(afiend)
1273: TIME_TRAP_UENTRY
1274: jmp 3f
1275: 0:
1276: TIME_TRAP_UENTRY
1277:
1278: movl CX(EXT(kernel_stack),%edx),%eax
1279: /* switch to kernel stack */
1280: xchgl %eax,%esp
1281: 3:
1282: pushl %eax
1283: pushl $0 /* push preemption flag */
1284: call EXT(i386_astintr) /* take the AST */
1285: addl $4,%esp /* pop preemption flag */
1286: popl %esp /* back to PCB stack */
1287: jmp EXT(return_from_trap) /* return */
1288:
1289: #if MACH_KDB || MACH_KGDB
1290: /*
1291: * kdb_kintr: enter kdb from keyboard interrupt.
1292: * Chase down the stack frames until we find one whose return
1293: * address is the interrupt handler. At that point, we have:
1294: *
1295: * frame-> saved %ebp
1296: * return address in interrupt handler
1297: * ivect
1298: * saved SPL
1299: * return address == return_to_iret_i
1300: * saved %edx
1301: * saved %ecx
1302: * saved %eax
1303: * saved %eip
1304: * saved %cs
1305: * saved %efl
1306: *
1307: * OR:
1308: * frame-> saved %ebp
1309: * return address in interrupt handler
1310: * ivect
1311: * saved SPL
1312: * return address == return_to_iret
1313: * pointer to save area on old stack
1314: * [ saved %ebx, if accurate timing ]
1315: *
1316: * old stack: saved %es
1317: * saved %ds
1318: * saved %edx
1319: * saved %ecx
1320: * saved %eax
1321: * saved %eip
1322: * saved %cs
1323: * saved %efl
1324: *
1325: * Call kdb, passing it that register save area.
1326: */
1327:
1328: #if MACH_KGDB
1329: Entry(kgdb_kintr)
1330: #endif /* MACH_KGDB */
1331: #if MACH_KDB
1332: Entry(kdb_kintr)
1333: #endif /* MACH_KDB */
1334: movl %ebp,%eax /* save caller`s frame pointer */
1335: movl $EXT(return_to_iret),%ecx /* interrupt return address 1 */
1336: movl $EXT(return_to_iret_i),%edx /* interrupt return address 2 */
1337:
1338: 0: cmpl 16(%eax),%ecx /* does this frame return to */
1339: /* interrupt handler (1)? */
1340: je 1f
1341: cmpl $kdb_from_iret,16(%eax)
1342: je 1f
1343: cmpl 16(%eax),%edx /* interrupt handler (2)? */
1344: je 2f /* if not: */
1345: cmpl $kdb_from_iret_i,16(%eax)
1346: je 2f
1347: movl (%eax),%eax /* try next frame */
1348: jmp 0b
1349:
1350: 1: movl $kdb_from_iret,16(%eax) /* returns to kernel/user stack */
1351: ret
1352:
1353: 2: movl $kdb_from_iret_i,16(%eax)
1354: /* returns to interrupt stack */
1355: ret
1356:
1357: /*
1358: * On return from keyboard interrupt, we will execute
1359: * kdb_from_iret_i
1360: * if returning to an interrupt on the interrupt stack
1361: * kdb_from_iret
1362: * if returning to an interrupt on the user or kernel stack
1363: */
1364: kdb_from_iret:
1365: /* save regs in known locations */
1366: #if STAT_TIME
1367: pushl %ebx /* caller`s %ebx is in reg */
1368: #else
1369: movl 4(%esp),%eax /* get caller`s %ebx */
1370: pushl %eax /* push on stack */
1371: #endif
1372: pushl %ebp
1373: pushl %esi
1374: pushl %edi
1375: push %fs
1376: push %gs
1377: #if MACH_KGDB
1378: cli
1379: pushl %esp /* pass regs */
1380: call EXT(kgdb_kentry) /* to kgdb */
1381: addl $4,%esp /* pop parameters */
1382: #endif /* MACH_KGDB */
1383: #if MACH_KDB
1384: pushl %esp /* pass regs */
1385: call EXT(kdb_kentry) /* to kdb */
1386: addl $4,%esp /* pop parameters */
1387: #endif /* MACH_KDB */
1388: pop %gs /* restore registers */
1389: pop %fs
1390: popl %edi
1391: popl %esi
1392: popl %ebp
1393: #if STAT_TIME
1394: popl %ebx
1395: #else
1396: popl %eax
1397: movl %eax,4(%esp)
1398: #endif
1399: jmp EXT(return_to_iret) /* normal interrupt return */
1400:
1401: kdb_from_iret_i: /* on interrupt stack */
1402: pop %edx /* restore saved registers */
1403: pop %ecx
1404: pop %eax
1405: pushl $0 /* zero error code */
1406: pushl $0 /* zero trap number */
1407: pusha /* save general registers */
1408: push %ds /* save segment registers */
1409: push %es
1410: push %fs
1411: push %gs
1412: #if MACH_KGDB
1413: cli /* disable interrupts */
1414: CPU_NUMBER(%edx) /* get CPU number */
1415: movl CX(EXT(kgdb_stacks),%edx),%ebx
1416: xchgl %ebx,%esp /* switch to kgdb stack */
1417: pushl %ebx /* pass old sp as an arg */
1418: call EXT(kgdb_from_kernel)
1419: popl %esp /* switch back to interrupt stack */
1420: #endif /* MACH_KGDB */
1421: #if MACH_KDB
1422: pushl %esp /* pass regs, */
1423: pushl $0 /* code, */
1424: pushl $-1 /* type to kdb */
1425: call EXT(kdb_trap)
1426: addl $12,%esp
1427: #endif /* MACH_KDB */
1428: pop %gs /* restore segment registers */
1429: pop %fs
1430: pop %es
1431: pop %ds
1432: popa /* restore general registers */
1433: addl $8,%esp
1434: iret
1435:
1436: #endif /* MACH_KDB || MACH_KGDB */
1437:
1438:
1439: /*
1440: * Mach RPC enters through a call gate, like a system call.
1441: */
1442:
1443: Entry(mach_rpc)
1444: pushf /* save flags as soon as possible */
1445: pushl %eax /* save system call number */
1446: pushl $0 /* clear trap number slot */
1447:
1448: pusha /* save the general registers */
1449: pushl %ds /* and the segment registers */
1450: pushl %es
1451: pushl %fs
1452: pushl %gs
1453:
1454: mov %ss,%dx /* switch to kernel data segment */
1455: mov %dx,%ds
1456: mov %dx,%es
1457: mov $CPU_DATA,%dx
1458: mov %dx,%gs
1459:
1460: /*
1461: * Shuffle eflags,eip,cs into proper places
1462: */
1463:
1464: movl R_EIP(%esp),%ebx /* eflags are in EIP slot */
1465: movl R_CS(%esp),%ecx /* eip is in CS slot */
1466: movl R_EFLAGS(%esp),%edx /* cs is in EFLAGS slot */
1467: movl %ecx,R_EIP(%esp) /* fix eip */
1468: movl %edx,R_CS(%esp) /* fix cs */
1469: movl %ebx,R_EFLAGS(%esp) /* fix eflags */
1470:
1471: CPU_NUMBER(%edx)
1472: TIME_TRAP_UENTRY
1473:
1474: negl %eax /* get system call number */
1475: shll $4,%eax /* manual indexing */
1476:
1477: /*
1478: * Check here for mach_rpc from kernel-loaded task --
1479: * - Note that kernel-loaded task returns via real return.
1480: * We didn't enter here "through" PCB (i.e., using ring 0 stack),
1481: * so transfer the stack frame into the PCB explicitly, then
1482: * start running on resulting "PCB stack". We have to set
1483: * up a simulated "uesp" manually, since there's none in the
1484: * frame.
1485: */
1486: cmpl $0,CX(EXT(active_kloaded),%edx)
1487: jz 2f
1488: CAH(mrstart)
1489: movl CX(EXT(active_kloaded),%edx),%ebx
1490: movl CX(EXT(kernel_stack),%edx),%edx
1491: xchgl %edx,%esp
1492:
1493: FRAME_STACK_TO_PCB(%ebx,%edx)
1494: CAH(mrend)
1495:
1496: CPU_NUMBER(%edx)
1497: jmp 3f
1498:
1499: 2:
1500: CPU_NUMBER(%edx)
1501: movl CX(EXT(kernel_stack),%edx),%ebx
1502: /* get current kernel stack */
1503: xchgl %ebx,%esp /* switch stacks - %ebx points to */
1504: /* user registers. */
1505:
1506: 3:
1507:
1508: /*
1509: * Register use on entry:
1510: * eax contains syscall number
1511: * ebx contains user regs pointer
1512: */
1513: #undef RPC_TRAP_REGISTERS
1514: #ifdef RPC_TRAP_REGISTERS
1515: pushl R_ESI(%ebx)
1516: pushl R_EDI(%ebx)
1517: pushl R_ECX(%ebx)
1518: pushl R_EDX(%ebx)
1519: #else
1520: movl EXT(mach_trap_table)(%eax),%ecx
1521: /* get number of arguments */
1522: jecxz 2f /* skip argument copy if none */
1523: movl R_UESP(%ebx),%esi /* get user stack pointer */
1524: lea 4(%esi,%ecx,4),%esi /* skip user return address, */
1525: /* and point past last argument */
1526: /* edx holds cpu number from above */
1527: movl CX(EXT(active_kloaded),%edx),%edx
1528: /* point to current thread */
1529: orl %edx,%edx /* if ! kernel-loaded, check addr */
1530: jz 4f /* else */
1531: mov %ds,%dx /* kernel data segment access */
1532: jmp 5f
1533: 4:
1534: cmpl $(VM_MAX_ADDRESS),%esi /* in user space? */
1535: ja mach_call_addr /* address error if not */
1536: movl $USER_DS,%edx /* user data segment access */
1537: 5:
1538: mov %dx,%fs
1539: movl %esp,%edx /* save kernel ESP for error recovery */
1540: 1:
1541: subl $4,%esi
1542: RECOVERY_SECTION
1543: RECOVER(mach_call_addr_push)
1544: pushl %fs:(%esi) /* push argument on stack */
1545: loop 1b /* loop for all arguments */
1546: #endif
1547:
1548: /*
1549: * Register use on entry:
1550: * eax contains syscall number
1551: * ebx contains user regs pointer
1552: */
1553: 2:
1554: CAH(call_call)
1555: call *EXT(mach_trap_table)+4(%eax)
1556: /* call procedure */
1557: movl %esp,%ecx /* get kernel stack */
1558: or $(KERNEL_STACK_SIZE-1),%ecx
1559: movl -3-IKS_SIZE(%ecx),%esp /* switch back to PCB stack */
1560: movl %eax,R_EAX(%esp) /* save return value */
1561: jmp EXT(return_from_trap) /* return to user */
1562:
1563:
1564: /*
1565: * Special system call entry for "int 0x80", which has the "eflags"
1566: * register saved at the right place already.
1567: * Fall back to the common syscall path after saving the registers.
1568: *
1569: * esp -> old eip
1570: * old cs
1571: * old eflags
1572: * old esp if trapped from user
1573: * old ss if trapped from user
1574: *
1575: * XXX: for the moment, we don't check for int 0x80 from kernel mode.
1576: */
1577: Entry(syscall_int80)
1578: pushl %eax /* save system call number */
1579: pushl $0 /* clear trap number slot */
1580:
1581: pusha /* save the general registers */
1582: pushl %ds /* and the segment registers */
1583: pushl %es
1584: pushl %fs
1585: pushl %gs
1586:
1587: mov %ss,%dx /* switch to kernel data segment */
1588: mov %dx,%ds
1589: mov %dx,%es
1590: mov $CPU_DATA,%dx
1591: mov %dx,%gs
1592:
1593: jmp syscall_entry_3
1594:
1595: /*
1596: * System call enters through a call gate. Flags are not saved -
1597: * we must shuffle stack to look like trap save area.
1598: *
1599: * esp-> old eip
1600: * old cs
1601: * old esp
1602: * old ss
1603: *
1604: * eax contains system call number.
1605: *
1606: * NB: below use of CPU_NUMBER assumes that macro will use correct
1607: * correct segment register for any kernel data accesses.
1608: */
1609: Entry(syscall)
1610: syscall_entry:
1611: pushf /* save flags as soon as possible */
1612: syscall_entry_2:
1613: pushl %eax /* save system call number */
1614: pushl $0 /* clear trap number slot */
1615:
1616: pusha /* save the general registers */
1617: pushl %ds /* and the segment registers */
1618: pushl %es
1619: pushl %fs
1620: pushl %gs
1621:
1622: mov %ss,%dx /* switch to kernel data segment */
1623: mov %dx,%ds
1624: mov %dx,%es
1625: mov $CPU_DATA,%dx
1626: mov %dx,%gs
1627:
1628: /*
1629: * Shuffle eflags,eip,cs into proper places
1630: */
1631:
1632: movl R_EIP(%esp),%ebx /* eflags are in EIP slot */
1633: movl R_CS(%esp),%ecx /* eip is in CS slot */
1634: movl R_EFLAGS(%esp),%edx /* cs is in EFLAGS slot */
1635: movl %ecx,R_EIP(%esp) /* fix eip */
1636: movl %edx,R_CS(%esp) /* fix cs */
1637: movl %ebx,R_EFLAGS(%esp) /* fix eflags */
1638:
1639: syscall_entry_3:
1640: CPU_NUMBER(%edx)
1641: /*
1642: * Check here for syscall from kernel-loaded task --
1643: * We didn't enter here "through" PCB (i.e., using ring 0 stack),
1644: * so transfer the stack frame into the PCB explicitly, then
1645: * start running on resulting "PCB stack". We have to set
1646: * up a simulated "uesp" manually, since there's none in the
1647: * frame.
1648: */
1649: cmpl $0,CX(EXT(active_kloaded),%edx)
1650: jz 0f
1651: CAH(scstart)
1652: movl CX(EXT(active_kloaded),%edx),%ebx
1653: movl CX(EXT(kernel_stack),%edx),%edx
1654: xchgl %edx,%esp
1655: FRAME_STACK_TO_PCB(%ebx,%edx)
1656: CAH(scend)
1657: TIME_TRAP_UENTRY
1658: CPU_NUMBER(%edx)
1659: jmp 1f
1660:
1661: 0:
1662: TIME_TRAP_UENTRY
1663:
1664: CPU_NUMBER(%edx)
1665: movl CX(EXT(kernel_stack),%edx),%ebx
1666: /* get current kernel stack */
1667: xchgl %ebx,%esp /* switch stacks - %ebx points to */
1668: /* user registers. */
1669: /* user regs pointer already set */
1670:
1671: /*
1672: * Check for MACH or emulated system call
1673: * Register use (from here till we begin processing call):
1674: * eax contains system call number
1675: * ebx points to user regs
1676: */
1677: 1:
1678: movl $CPD_ACTIVE_THREAD,%edx
1679: movl %gs:(%edx),%edx /* get active thread */
1680: /* point to current thread */
1681: movl TH_TOP_ACT(%edx),%edx /* get thread->top_act */
1682: movl ACT_TASK(%edx),%edx /* point to task */
1683: movl TASK_EMUL(%edx),%edx /* get emulation vector */
1684: orl %edx,%edx /* if none, */
1685: je syscall_native /* do native system call */
1686: movl %eax,%ecx /* copy system call number */
1687: subl DISP_MIN(%edx),%ecx /* get displacement into syscall */
1688: /* vector table */
1689: jl syscall_native /* too low - native system call */
1690: cmpl DISP_COUNT(%edx),%ecx /* check range */
1691: jnl syscall_native /* too high - native system call */
1692: movl DISP_VECTOR(%edx,%ecx,4),%edx
1693: /* get the emulation vector */
1694: orl %edx,%edx /* emulated system call if not zero */
1695: jnz syscall_emul
1696:
1697: /*
1698: * Native system call.
1699: * Register use on entry:
1700: * eax contains syscall number
1701: * ebx points to user regs
1702: */
1703: syscall_native:
1704: negl %eax /* get system call number */
1705: jl mach_call_range /* out of range if it was positive */
1706:
1707: cmpl EXT(mach_trap_count),%eax /* check system call table bounds */
1708: jg mach_call_range /* error if out of range */
1709: shll $4,%eax /* manual indexing */
1710:
1711: movl EXT(mach_trap_table)+4(%eax),%edx
1712: /* get procedure */
1713: cmpl $EXT(kern_invalid),%edx /* if not "kern_invalid" */
1714: jne mach_syscall_native /* go on with Mach syscall */
1715:
1716: movl $CPD_ACTIVE_THREAD,%edx
1717: movl %gs:(%edx),%edx /* get active thread */
1718: /* point to current thread */
1719: movl TH_TOP_ACT(%edx),%edx /* get thread->top_act */
1720: movl ACT_TASK(%edx),%edx /* point to task */
1721: movl TASK_EMUL(%edx),%edx /* get emulation vector */
1722: orl %edx,%edx /* if it exists, */
1723: jne mach_syscall_native /* do native system call */
1724: shrl $4,%eax /* restore syscall number */
1725: jmp mach_call_range /* try it as a "server" syscall */
1726:
1727: mach_syscall_native:
1728: movl $CPD_ACTIVE_THREAD,%edx
1729: movl %gs:(%edx),%edx /* get active thread */
1730:
1731: movl TH_TOP_ACT(%edx),%edx /* get thread->top_act */
1732: movl ACT_MACH_EXC_PORT(%edx),%edx
1733: movl $EXT(realhost),%ecx
1734: movl HOST_NAME(%ecx),%ecx
1735: cmpl %edx,%ecx /* act->mach_exc_port = host_name ? */
1736: je do_native_call /* -> send to kernel, do not collect $200 */
1737: cmpl $0,%edx /* thread->mach_exc_port = null ? */
1738: je try_task /* try task */
1739: jmp mach_syscall_exception
1740: /* NOT REACHED */
1741:
1742: try_task:
1743: movl $CPD_ACTIVE_THREAD,%edx
1744: movl %gs:(%edx),%edx /* get active thread */
1745:
1746: movl TH_TOP_ACT(%edx),%edx /* get thread->top_act */
1747: movl ACT_TASK(%edx),%edx /* point to task */
1748: movl TASK_MACH_EXC_PORT(%edx),%edx
1749: movl $EXT(realhost),%ecx
1750: movl HOST_NAME(%ecx),%ecx
1751: cmpl %edx,%ecx /* thread->mach_exc_port = host_name ? */
1752: je do_native_call /* -> send to kernel */
1753: cmpl $0,%edx /* thread->mach_exc_port = null ? */
1754: je EXT(syscall_failed) /* try task */
1755: jmp mach_syscall_exception
1756: /* NOT REACHED */
1757:
1758: /*
1759: * Register use on entry:
1760: * eax contains syscall number
1761: * ebx contains user regs pointer
1762: */
1763: do_native_call:
1764: movl EXT(mach_trap_table)(%eax),%ecx
1765: /* get number of arguments */
1766: jecxz mach_call_call /* skip argument copy if none */
1767: movl R_UESP(%ebx),%esi /* get user stack pointer */
1768: lea 4(%esi,%ecx,4),%esi /* skip user return address, */
1769: /* and point past last argument */
1770: CPU_NUMBER(%edx)
1771: movl CX(EXT(active_kloaded),%edx),%edx
1772: /* point to current thread */
1773: orl %edx,%edx /* if kernel-loaded, skip addr check */
1774: jz 0f /* else */
1775: mov %ds,%dx /* kernel data segment access */
1776: jmp 1f
1777: 0:
1778: cmpl $(VM_MAX_ADDRESS),%esi /* in user space? */
1779: ja mach_call_addr /* address error if not */
1780: movl $USER_DS,%edx /* user data segment access */
1781: 1:
1782: mov %dx,%fs
1783: movl %esp,%edx /* save kernel ESP for error recovery */
1784: 2:
1785: subl $4,%esi
1786: RECOVERY_SECTION
1787: RECOVER(mach_call_addr_push)
1788: pushl %fs:(%esi) /* push argument on stack */
1789: loop 2b /* loop for all arguments */
1790:
1791: /*
1792: * Register use on entry:
1793: * eax contains syscall number
1794: * ebx contains user regs pointer
1795: */
1796: mach_call_call:
1797:
1798: CAH(call_call)
1799:
1800: #if ETAP_EVENT_MONITOR
1801: cmpl $0x200, %eax /* is this mach_msg? */
1802: jz make_syscall /* if yes, don't record event */
1803:
1804: pushal /* Otherwise: save registers */
1805: pushl %eax /* push syscall number on stack*/
1806: call EXT(etap_machcall_probe1) /* call event begin probe */
1807: add $4,%esp /* restore stack */
1808: popal /* restore registers */
1809:
1810: call *EXT(mach_trap_table)+4(%eax) /* call procedure */
1811: pushal
1812: call EXT(etap_machcall_probe2) /* call event end probe */
1813: popal
1814: jmp skip_syscall /* syscall already made */
1815: #endif /* ETAP_EVENT_MONITOR */
1816:
1817: make_syscall:
1818: call *EXT(mach_trap_table)+4(%eax) /* call procedure */
1819: skip_syscall:
1820:
1821: movl %esp,%ecx /* get kernel stack */
1822: or $(KERNEL_STACK_SIZE-1),%ecx
1823: movl -3-IKS_SIZE(%ecx),%esp /* switch back to PCB stack */
1824: movl %eax,R_EAX(%esp) /* save return value */
1825: jmp EXT(return_from_trap) /* return to user */
1826:
1827: /*
1828: * Address out of range. Change to page fault.
1829: * %esi holds failing address.
1830: * Register use on entry:
1831: * ebx contains user regs pointer
1832: */
1833: mach_call_addr_push:
1834: movl %edx,%esp /* clean parameters from stack */
1835: mach_call_addr:
1836: movl %esi,R_CR2(%ebx) /* set fault address */
1837: movl $(T_PAGE_FAULT),R_TRAPNO(%ebx)
1838: /* set page-fault trap */
1839: movl $(T_PF_USER),R_ERR(%ebx)
1840: /* set error code - read user space */
1841: CAH(call_addr)
1842: jmp EXT(take_trap) /* treat as a trap */
1843:
1844: /*
1845: * try sending mach system call exception to server
1846: * Register use on entry:
1847: * eax contains syscall number
1848: */
1849: mach_syscall_exception:
1850: push %eax /* code (syscall no.) */
1851: movl %esp,%edx
1852: push $1 /* code_cnt = 1 */
1853: push %edx /* exception_type_t (see i/f docky) */
1854: push $EXC_MACH_SYSCALL /* exception */
1855:
1856: CAH(exception)
1857: call EXT(exception)
1858: /* no return */
1859:
1860: /*
1861: * System call out of range. Treat as invalid-instruction trap.
1862: * (? general protection?)
1863: * Register use on entry:
1864: * eax contains syscall number
1865: */
1866: mach_call_range:
1867: movl $CPD_ACTIVE_THREAD,%edx
1868: movl %gs:(%edx),%edx /* get active thread */
1869:
1870: movl TH_TOP_ACT(%edx),%edx /* get thread->top_act */
1871: movl ACT_TASK(%edx),%edx /* point to task */
1872: movl TASK_EMUL(%edx),%edx /* get emulation vector */
1873: orl %edx,%edx /* if emulator, */
1874: jne EXT(syscall_failed) /* handle as illegal instruction */
1875: /* else generate syscall exception: */
1876: push %eax
1877: movl %esp,%edx
1878: push $1 /* code_cnt = 1 */
1879: push %edx /* exception_type_t (see i/f docky) */
1880: push $EXC_SYSCALL
1881: CAH(call_range)
1882: call EXT(exception)
1883: /* no return */
1884:
1885: /*
1886: * Restart a mach system call after raising an exception.
1887: */
1888: .globl EXT(restart_mach_syscall)
1889: LEXT(restart_mach_syscall)
1890: movl %esp,%ecx /* get kernel stack */
1891: or $(KERNEL_STACK_SIZE-1),%ecx
1892: movl -3-IKS_SIZE(%ecx),%ebx /* get user PCB */
1893: movl R_EAX(%ebx),%eax /* syscall number, */
1894: negl %eax /* negate it and */
1895: shll $4,%eax /* change to syscall table index */
1896: CAH(restart)
1897: jmp do_native_call /* try native system call */
1898:
1899: .globl EXT(syscall_failed)
1900: LEXT(syscall_failed)
1901: movl %esp,%ecx /* get kernel stack */
1902: or $(KERNEL_STACK_SIZE-1),%ecx
1903: movl -3-IKS_SIZE(%ecx),%esp /* switch back to PCB stack */
1904: CPU_NUMBER(%edx)
1905: movl CX(EXT(kernel_stack),%edx),%ebx
1906: /* get current kernel stack */
1907: xchgl %ebx,%esp /* switch stacks - %ebx points to */
1908: /* user registers. */
1909: /* user regs pointer already set */
1910:
1911: movl $(T_INVALID_OPCODE),R_TRAPNO(%ebx)
1912: /* set invalid-operation trap */
1913: movl $0,R_ERR(%ebx) /* clear error code */
1914: CAH(failed)
1915: jmp EXT(take_trap) /* treat as a trap */
1916:
1917: /*
1918: * User space emulation of system calls.
1919: * edx - user address to handle syscall
1920: *
1921: * User stack will become:
1922: * uesp-> eflags
1923: * eip
1924: * Register use on entry:
1925: * ebx contains user regs pointer
1926: * edx contains emulator vector address
1927: */
1928: syscall_emul:
1929: movl R_UESP(%ebx),%edi /* get user stack pointer */
1930: CPU_NUMBER(%eax)
1931: movl CX(EXT(active_kloaded),%eax),%eax
1932: orl %eax,%eax /* if thread not kernel-loaded, */
1933: jz 0f /* do address checks */
1934: subl $8,%edi
1935: mov %ds,%ax /* kernel data segment access */
1936: jmp 1f /* otherwise, skip them */
1937: 0:
1938: cmpl $(VM_MAX_ADDRESS),%edi /* in user space? */
1939: ja syscall_addr /* address error if not */
1940: subl $8,%edi /* push space for new arguments */
1941: cmpl $(VM_MIN_ADDRESS),%edi /* still in user space? */
1942: jb syscall_addr /* error if not */
1943: movl $USER_DS,%ax /* user data segment access */
1944: 1:
1945: mov %ax,%fs
1946: movl R_EFLAGS(%ebx),%eax /* move flags */
1947: RECOVERY_SECTION
1948: RECOVER(syscall_addr)
1949: movl %eax,%fs:0(%edi) /* to user stack */
1950: movl R_EIP(%ebx),%eax /* move eip */
1951: RECOVERY_SECTION
1952: RECOVER(syscall_addr)
1953: movl %eax,%fs:4(%edi) /* to user stack */
1954: movl %edi,R_UESP(%ebx) /* set new user stack pointer */
1955: movl %edx,R_EIP(%ebx) /* change return address to trap */
1956: movl %ebx,%esp /* back to PCB stack */
1957: CAH(emul)
1958: jmp EXT(return_from_trap) /* return to user */
1959:
1960:
1961: /*
1962: * Address error - address is in %edi.
1963: * Register use on entry:
1964: * ebx contains user regs pointer
1965: */
1966: syscall_addr:
1967: movl %edi,R_CR2(%ebx) /* set fault address */
1968: movl $(T_PAGE_FAULT),R_TRAPNO(%ebx)
1969: /* set page-fault trap */
1970: movl $(T_PF_USER),R_ERR(%ebx)
1971: /* set error code - read user space */
1972: CAH(addr)
1973: jmp EXT(take_trap) /* treat as a trap */
1974:
1975: /* */
1976: /*
1977: * Utility routines.
1978: */
1979:
1980:
1981: /*
1982: * Copy from user address space.
1983: * arg0: user address
1984: * arg1: kernel address
1985: * arg2: byte count
1986: */
1987: Entry(copyinmsg)
1988: ENTRY(copyin)
1989: pushl %esi
1990: pushl %edi /* save registers */
1991:
1992: movl 8+S_ARG0,%esi /* get user start address */
1993: movl 8+S_ARG1,%edi /* get kernel destination address */
1994: movl 8+S_ARG2,%edx /* get count */
1995:
1996: lea 0(%esi,%edx),%eax /* get user end address + 1 */
1997:
1998: movl $CPD_ACTIVE_THREAD,%ecx
1999: movl %gs:(%ecx),%ecx /* get active thread */
2000: movl TH_TOP_ACT(%ecx),%ecx /* get thread->top_act */
2001: movl ACT_MAP(%ecx),%ecx /* get act->map */
2002: movl MAP_PMAP(%ecx),%ecx /* get map->pmap */
2003: cmpl EXT(kernel_pmap), %ecx
2004: jz 1f
2005: movl $USER_DS,%cx /* user data segment access */
2006: mov %cx,%ds
2007: 1:
2008: cmpl %esi,%eax
2009: jb copyin_fail /* fail if wrap-around */
2010: cld /* count up */
2011: movl %edx,%ecx /* move by longwords first */
2012: shrl $2,%ecx
2013: RECOVERY_SECTION
2014: RECOVER(copyin_fail)
2015: rep
2016: movsl /* move longwords */
2017: movl %edx,%ecx /* now move remaining bytes */
2018: andl $3,%ecx
2019: RECOVERY_SECTION
2020: RECOVER(copyin_fail)
2021: rep
2022: movsb
2023: xorl %eax,%eax /* return 0 for success */
2024: copy_ret:
2025: mov %ss,%di /* restore kernel data segment */
2026: mov %di,%ds
2027:
2028: popl %edi /* restore registers */
2029: popl %esi
2030: ret /* and return */
2031:
2032: copyin_fail:
2033: movl $EFAULT,%eax /* return error for failure */
2034: jmp copy_ret /* pop frame and return */
2035:
2036: /*
2037: * Copy string from user address space.
2038: * arg0: user address
2039: * arg1: kernel address
2040: * arg2: max byte count
2041: * arg3: actual byte count (OUT)
2042: */
2043: Entry(copyinstr)
2044: pushl %esi
2045: pushl %edi /* save registers */
2046:
2047: movl 8+S_ARG0,%esi /* get user start address */
2048: movl 8+S_ARG1,%edi /* get kernel destination address */
2049: movl 8+S_ARG2,%edx /* get count */
2050:
2051: lea 0(%esi,%edx),%eax /* get user end address + 1 */
2052:
2053: movl $CPD_ACTIVE_THREAD,%ecx
2054: movl %gs:(%ecx),%ecx /* get active thread */
2055: movl TH_TOP_ACT(%ecx),%ecx /* get thread->top_act */
2056: movl ACT_MAP(%ecx),%ecx /* get act->map */
2057: movl MAP_PMAP(%ecx),%ecx /* get map->pmap */
2058: cmpl EXT(kernel_pmap), %ecx
2059: jne 0f
2060: mov %ds,%cx /* kernel data segment access */
2061: jmp 1f
2062: 0:
2063: movl $USER_DS,%cx /* user data segment access */
2064: 1:
2065: mov %cx,%fs
2066: xorl %eax,%eax
2067: cmpl $0,%edx
2068: je 4f
2069: 2:
2070: RECOVERY_SECTION
2071: RECOVER(copystr_fail) /* copy bytes... */
2072: movb %fs:(%esi),%eax
2073: incl %esi
2074: testl %edi,%edi /* if kernel address is ... */
2075: jz 3f /* not NULL */
2076: movb %eax,(%edi) /* copy the byte */
2077: incl %edi
2078: 3:
2079: decl %edx
2080: je 5f /* Zero count.. error out */
2081: cmpl $0,%eax
2082: jne 2b /* .. a NUL found? */
2083: jmp 4f
2084: 5:
2085: movl $ENAMETOOLONG,%eax /* String is too long.. */
2086: 4:
2087: xorl %eax,%eax /* return zero for success */
2088: movl 8+S_ARG3,%edi /* get OUT len ptr */
2089: cmpl $0,%edi
2090: jz copystr_ret /* if null, just return */
2091: subl 8+S_ARG0,%esi
2092: movl %esi,(%edi) /* else set OUT arg to xfer len */
2093: copystr_ret:
2094: popl %edi /* restore registers */
2095: popl %esi
2096: ret /* and return */
2097:
2098: copystr_fail:
2099: movl $EFAULT,%eax /* return error for failure */
2100: jmp copy_ret /* pop frame and return */
2101:
2102: /*
2103: * Copy to user address space.
2104: * arg0: kernel address
2105: * arg1: user address
2106: * arg2: byte count
2107: */
2108: Entry(copyoutmsg)
2109: ENTRY(copyout)
2110: pushl %esi
2111: pushl %edi /* save registers */
2112: pushl %ebx
2113:
2114: movl 12+S_ARG0,%esi /* get kernel start address */
2115: movl 12+S_ARG1,%edi /* get user start address */
2116: movl 12+S_ARG2,%edx /* get count */
2117:
2118: leal 0(%edi,%edx),%eax /* get user end address + 1 */
2119:
2120: movl $CPD_ACTIVE_THREAD,%ecx
2121: movl %gs:(%ecx),%ecx /* get active thread */
2122: movl TH_TOP_ACT(%ecx),%ecx /* get thread->top_act */
2123: movl ACT_MAP(%ecx),%ecx /* get act->map */
2124: movl MAP_PMAP(%ecx),%ecx /* get map->pmap */
2125: cmpl EXT(kernel_pmap), %ecx
2126: jne 0f
2127: mov %ds,%cx /* else kernel data segment access */
2128: jmp 1f
2129: 0:
2130: movl $USER_DS,%cx
2131: 1:
2132: mov %cx,%es
2133:
2134: /*
2135: * Check whether user address space is writable
2136: * before writing to it - hardware is broken.
2137: *
2138: * Skip check if "user" address is really in
2139: * kernel space (i.e., if it's in a kernel-loaded
2140: * task).
2141: *
2142: * Register usage:
2143: * esi/edi source/dest pointers for rep/mov
2144: * ecx counter for rep/mov
2145: * edx counts down from 3rd arg
2146: * eax count of bytes for each (partial) page copy
2147: * ebx shadows edi, used to adjust edx
2148: */
2149: movl %edi,%ebx /* copy edi for syncing up */
2150: copyout_retry:
2151: /* if restarting after a partial copy, put edx back in sync, */
2152: addl %ebx,%edx /* edx -= (edi - ebx); */
2153: subl %edi,%edx /
2154: movl %edi,%ebx /* ebx = edi; */
2155:
2156: mov %es,%cx
2157: cmpl $USER_DS,%cx /* If kernel data segment */
2158: jnz 0f /* skip check */
2159:
2160: cmpb $(CPUID_FAMILY_386), EXT(cpuid_family)
2161: ja 0f
2162:
2163: movl %cr3,%ecx /* point to page directory */
2164: #if NCPUS > 1
2165: andl $(~0x7), %ecx /* remove cpu number */
2166: #endif /* NCPUS > 1 && AT386 */
2167: movl %edi,%eax /* get page directory bits */
2168: shrl $(PDESHIFT),%eax /* from user address */
2169: movl KERNELBASE(%ecx,%eax,4),%ecx
2170: /* get page directory pointer */
2171: testl $(PTE_V),%ecx /* present? */
2172: jz 0f /* if not, fault is OK */
2173: andl $(PTE_PFN),%ecx /* isolate page frame address */
2174: movl %edi,%eax /* get page table bits */
2175: shrl $(PTESHIFT),%eax
2176: andl $(PTEMASK),%eax /* from user address */
2177: leal KERNELBASE(%ecx,%eax,4),%ecx
2178: /* point to page table entry */
2179: movl (%ecx),%eax /* get it */
2180: testl $(PTE_V),%eax /* present? */
2181: jz 0f /* if not, fault is OK */
2182: testl $(PTE_W),%eax /* writable? */
2183: jnz 0f /* OK if so */
2184: /*
2185: * Not writable - must fake a fault. Turn off access to the page.
2186: */
2187: andl $(PTE_INVALID),(%ecx) /* turn off valid bit */
2188: movl %cr3,%eax /* invalidate TLB */
2189: movl %eax,%cr3
2190: 0:
2191: /*
2192: * Copy only what fits on the current destination page.
2193: * Check for write-fault again on the next page.
2194: */
2195: leal NBPG(%edi),%eax /* point to */
2196: andl $(-NBPG),%eax /* start of next page */
2197: subl %edi,%eax /* get number of bytes to that point */
2198: cmpl %edx,%eax /* bigger than count? */
2199: jle 1f /* if so, */
2200: movl %edx,%eax /* use count */
2201: 1:
2202: cld /* count up */
2203: movl %eax,%ecx /* move by longwords first */
2204: shrl $2,%ecx
2205: RECOVERY_SECTION
2206: RECOVER(copyout_fail)
2207: RETRY_SECTION
2208: RETRY(copyout_retry)
2209: rep
2210: movsl
2211: movl %eax,%ecx /* now move remaining bytes */
2212: andl $3,%ecx
2213: RECOVERY_SECTION
2214: RECOVER(copyout_fail)
2215: RETRY_SECTION
2216: RETRY(copyout_retry)
2217: rep
2218: movsb /* move */
2219: movl %edi,%ebx /* copy edi for syncing up */
2220: subl %eax,%edx /* and decrement count */
2221: jg copyout_retry /* restart on next page if not done */
2222: xorl %eax,%eax /* return 0 for success */
2223: copyout_ret:
2224: mov %ss,%di /* restore kernel segment */
2225: mov %di,%es
2226:
2227: popl %ebx
2228: popl %edi /* restore registers */
2229: popl %esi
2230: ret /* and return */
2231:
2232: copyout_fail:
2233: movl $EFAULT,%eax /* return error for failure */
2234: jmp copyout_ret /* pop frame and return */
2235:
2236: /*
2237: * FPU routines.
2238: */
2239:
2240: /*
2241: * Initialize FPU.
2242: */
2243: ENTRY(_fninit)
2244: fninit
2245: ret
2246:
2247: /*
2248: * Read control word
2249: */
2250: ENTRY(_fstcw)
2251: pushl %eax /* get stack space */
2252: fstcw (%esp)
2253: popl %eax
2254: ret
2255:
2256: /*
2257: * Set control word
2258: */
2259: ENTRY(_fldcw)
2260: fldcw 4(%esp)
2261: ret
2262:
2263: /*
2264: * Read status word
2265: */
2266: ENTRY(_fnstsw)
2267: xor %eax,%eax /* clear high 16 bits of eax */
2268: fnstsw %ax /* read FP status */
2269: ret
2270:
2271: /*
2272: * Clear FPU exceptions
2273: */
2274: ENTRY(_fnclex)
2275: fnclex
2276: ret
2277:
2278: /*
2279: * Clear task-switched flag.
2280: */
2281: ENTRY(_clts)
2282: clts
2283: ret
2284:
2285: /*
2286: * Save complete FPU state. Save error for later.
2287: */
2288: ENTRY(_fpsave)
2289: movl 4(%esp),%eax /* get save area pointer */
2290: fnsave (%eax) /* save complete state, including */
2291: /* errors */
2292: ret
2293:
2294: /*
2295: * Restore FPU state.
2296: */
2297: ENTRY(_fprestore)
2298: movl 4(%esp),%eax /* get save area pointer */
2299: frstor (%eax) /* restore complete state */
2300: ret
2301:
2302: /*
2303: * Set cr3
2304: */
2305: ENTRY(set_cr3)
2306: #if NCPUS > 1
2307: CPU_NUMBER(%eax)
2308: orl 4(%esp), %eax
2309: #else /* NCPUS > 1 && AT386 */
2310: movl 4(%esp),%eax /* get new cr3 value */
2311: #endif /* NCPUS > 1 && AT386 */
2312: /*
2313: * Don't set PDBR to a new value (hence invalidating the
2314: * "paging cache") if the new value matches the current one.
2315: */
2316: movl %cr3,%edx /* get current cr3 value */
2317: cmpl %eax,%edx
2318: je 0f /* if two are equal, don't set */
2319: movl %eax,%cr3 /* load it (and flush cache) */
2320: 0:
2321: ret
2322:
2323: /*
2324: * Read cr3
2325: */
2326: ENTRY(get_cr3)
2327: movl %cr3,%eax
2328: #if NCPUS > 1
2329: andl $(~0x7), %eax /* remove cpu number */
2330: #endif /* NCPUS > 1 && AT386 */
2331: ret
2332:
2333: /*
2334: * Flush TLB
2335: */
2336: ENTRY(flush_tlb)
2337: movl %cr3,%eax /* flush tlb by reloading CR3 */
2338: movl %eax,%cr3 /* with itself */
2339: ret
2340:
2341: /*
2342: * Read cr2
2343: */
2344: ENTRY(get_cr2)
2345: movl %cr2,%eax
2346: ret
2347:
2348: /*
2349: * Read cr4
2350: */
2351: ENTRY(get_cr4)
2352: .byte 0x0f,0x20,0xe0 /* movl %cr4, %eax */
2353: ret
2354:
2355: /*
2356: * Write cr4
2357: */
2358: ENTRY(set_cr4)
2359: movl 4(%esp), %eax
2360: .byte 0x0f,0x22,0xe0 /* movl %eax, %cr4 */
2361: ret
2362:
2363: /*
2364: * Read ldtr
2365: */
2366: Entry(get_ldt)
2367: xorl %eax,%eax
2368: sldt %ax
2369: ret
2370:
2371: /*
2372: * Set ldtr
2373: */
2374: Entry(set_ldt)
2375: lldt 4(%esp)
2376: ret
2377:
2378: /*
2379: * Read task register.
2380: */
2381: ENTRY(get_tr)
2382: xorl %eax,%eax
2383: str %ax
2384: ret
2385:
2386: /*
2387: * Set task register. Also clears busy bit of task descriptor.
2388: */
2389: ENTRY(set_tr)
2390: movl S_ARG0,%eax /* get task segment number */
2391: subl $8,%esp /* push space for SGDT */
2392: sgdt 2(%esp) /* store GDT limit and base (linear) */
2393: movl 4(%esp),%edx /* address GDT */
2394: movb $(K_TSS),5(%edx,%eax) /* fix access byte in task descriptor */
2395: ltr %ax /* load task register */
2396: addl $8,%esp /* clear stack */
2397: ret /* and return */
2398:
2399: /*
2400: * Set task-switched flag.
2401: */
2402: ENTRY(_setts)
2403: movl %cr0,%eax /* get cr0 */
2404: orl $(CR0_TS),%eax /* or in TS bit */
2405: movl %eax,%cr0 /* set cr0 */
2406: ret
2407:
2408: /*
2409: * io register must not be used on slaves (no AT bus)
2410: */
2411: #define ILL_ON_SLAVE
2412:
2413:
2414: #if MACH_ASSERT
2415:
2416: #define ARG0 B_ARG0
2417: #define ARG1 B_ARG1
2418: #define ARG2 B_ARG2
2419: #define PUSH_FRAME FRAME
2420: #define POP_FRAME EMARF
2421:
2422: #else /* MACH_ASSERT */
2423:
2424: #define ARG0 S_ARG0
2425: #define ARG1 S_ARG1
2426: #define ARG2 S_ARG2
2427: #define PUSH_FRAME
2428: #define POP_FRAME
2429:
2430: #endif /* MACH_ASSERT */
2431:
2432:
2433: #if MACH_KDB || MACH_ASSERT
2434:
2435: /*
2436: * Following routines are also defined as macros in i386/pio.h
2437: * Compile then when MACH_KDB is configured so that they
2438: * can be invoked from the debugger.
2439: */
2440:
2441: /*
2442: * void outb(unsigned char *io_port,
2443: * unsigned char byte)
2444: *
2445: * Output a byte to an IO port.
2446: */
2447: ENTRY(outb)
2448: PUSH_FRAME
2449: ILL_ON_SLAVE
2450: movl ARG0,%edx /* IO port address */
2451: movl ARG1,%eax /* data to output */
2452: outb %al,%dx /* send it out */
2453: POP_FRAME
2454: ret
2455:
2456: /*
2457: * unsigned char inb(unsigned char *io_port)
2458: *
2459: * Input a byte from an IO port.
2460: */
2461: ENTRY(inb)
2462: PUSH_FRAME
2463: ILL_ON_SLAVE
2464: movl ARG0,%edx /* IO port address */
2465: xor %eax,%eax /* clear high bits of register */
2466: inb %dx,%al /* get the byte */
2467: POP_FRAME
2468: ret
2469:
2470: /*
2471: * void outw(unsigned short *io_port,
2472: * unsigned short word)
2473: *
2474: * Output a word to an IO port.
2475: */
2476: ENTRY(outw)
2477: PUSH_FRAME
2478: ILL_ON_SLAVE
2479: movl ARG0,%edx /* IO port address */
2480: movl ARG1,%eax /* data to output */
2481: outw %ax,%dx /* send it out */
2482: POP_FRAME
2483: ret
2484:
2485: /*
2486: * unsigned short inw(unsigned short *io_port)
2487: *
2488: * Input a word from an IO port.
2489: */
2490: ENTRY(inw)
2491: PUSH_FRAME
2492: ILL_ON_SLAVE
2493: movl ARG0,%edx /* IO port address */
2494: xor %eax,%eax /* clear high bits of register */
2495: inw %dx,%ax /* get the word */
2496: POP_FRAME
2497: ret
2498:
2499: /*
2500: * void outl(unsigned int *io_port,
2501: * unsigned int byte)
2502: *
2503: * Output an int to an IO port.
2504: */
2505: ENTRY(outl)
2506: PUSH_FRAME
2507: ILL_ON_SLAVE
2508: movl ARG0,%edx /* IO port address*/
2509: movl ARG1,%eax /* data to output */
2510: outl %eax,%dx /* send it out */
2511: POP_FRAME
2512: ret
2513:
2514: /*
2515: * unsigned int inl(unsigned int *io_port)
2516: *
2517: * Input an int from an IO port.
2518: */
2519: ENTRY(inl)
2520: PUSH_FRAME
2521: ILL_ON_SLAVE
2522: movl ARG0,%edx /* IO port address */
2523: inl %dx,%eax /* get the int */
2524: POP_FRAME
2525: ret
2526:
2527: #endif /* MACH_KDB || MACH_ASSERT*/
2528:
2529: /*
2530: * void loutb(unsigned byte *io_port,
2531: * unsigned byte *data,
2532: * unsigned int count)
2533: *
2534: * Output an array of bytes to an IO port.
2535: */
2536: ENTRY(loutb)
2537: ENTRY(outsb)
2538: PUSH_FRAME
2539: ILL_ON_SLAVE
2540: movl %esi,%eax /* save register */
2541: movl ARG0,%edx /* get io port number */
2542: movl ARG1,%esi /* get data address */
2543: movl ARG2,%ecx /* get count */
2544: cld /* count up */
2545: rep
2546: outsb /* output */
2547: movl %eax,%esi /* restore register */
2548: POP_FRAME
2549: ret
2550:
2551:
2552: /*
2553: * void loutw(unsigned short *io_port,
2554: * unsigned short *data,
2555: * unsigned int count)
2556: *
2557: * Output an array of shorts to an IO port.
2558: */
2559: ENTRY(loutw)
2560: ENTRY(outsw)
2561: PUSH_FRAME
2562: ILL_ON_SLAVE
2563: movl %esi,%eax /* save register */
2564: movl ARG0,%edx /* get io port number */
2565: movl ARG1,%esi /* get data address */
2566: movl ARG2,%ecx /* get count */
2567: cld /* count up */
2568: rep
2569: outsw /* output */
2570: movl %eax,%esi /* restore register */
2571: POP_FRAME
2572: ret
2573:
2574: /*
2575: * void loutw(unsigned short io_port,
2576: * unsigned int *data,
2577: * unsigned int count)
2578: *
2579: * Output an array of longs to an IO port.
2580: */
2581: ENTRY(loutl)
2582: ENTRY(outsl)
2583: PUSH_FRAME
2584: ILL_ON_SLAVE
2585: movl %esi,%eax /* save register */
2586: movl ARG0,%edx /* get io port number */
2587: movl ARG1,%esi /* get data address */
2588: movl ARG2,%ecx /* get count */
2589: cld /* count up */
2590: rep
2591: outsl /* output */
2592: movl %eax,%esi /* restore register */
2593: POP_FRAME
2594: ret
2595:
2596:
2597: /*
2598: * void linb(unsigned char *io_port,
2599: * unsigned char *data,
2600: * unsigned int count)
2601: *
2602: * Input an array of bytes from an IO port.
2603: */
2604: ENTRY(linb)
2605: ENTRY(insb)
2606: PUSH_FRAME
2607: ILL_ON_SLAVE
2608: movl %edi,%eax /* save register */
2609: movl ARG0,%edx /* get io port number */
2610: movl ARG1,%edi /* get data address */
2611: movl ARG2,%ecx /* get count */
2612: cld /* count up */
2613: rep
2614: insb /* input */
2615: movl %eax,%edi /* restore register */
2616: POP_FRAME
2617: ret
2618:
2619:
2620: /*
2621: * void linw(unsigned short *io_port,
2622: * unsigned short *data,
2623: * unsigned int count)
2624: *
2625: * Input an array of shorts from an IO port.
2626: */
2627: ENTRY(linw)
2628: ENTRY(insw)
2629: PUSH_FRAME
2630: ILL_ON_SLAVE
2631: movl %edi,%eax /* save register */
2632: movl ARG0,%edx /* get io port number */
2633: movl ARG1,%edi /* get data address */
2634: movl ARG2,%ecx /* get count */
2635: cld /* count up */
2636: rep
2637: insw /* input */
2638: movl %eax,%edi /* restore register */
2639: POP_FRAME
2640: ret
2641:
2642:
2643: /*
2644: * void linl(unsigned short io_port,
2645: * unsigned int *data,
2646: * unsigned int count)
2647: *
2648: * Input an array of longs from an IO port.
2649: */
2650: ENTRY(linl)
2651: ENTRY(insl)
2652: PUSH_FRAME
2653: ILL_ON_SLAVE
2654: movl %edi,%eax /* save register */
2655: movl ARG0,%edx /* get io port number */
2656: movl ARG1,%edi /* get data address */
2657: movl ARG2,%ecx /* get count */
2658: cld /* count up */
2659: rep
2660: insl /* input */
2661: movl %eax,%edi /* restore register */
2662: POP_FRAME
2663: ret
2664:
2665:
2666: /*
2667: * int inst_fetch(int eip, int cs);
2668: *
2669: * Fetch instruction byte. Return -1 if invalid address.
2670: */
2671: .globl EXT(inst_fetch)
2672: LEXT(inst_fetch)
2673: movl S_ARG1, %eax /* get segment */
2674: movw %ax,%fs /* into FS */
2675: movl S_ARG0, %eax /* get offset */
2676: RETRY_SECTION
2677: RETRY(EXT(inst_fetch)) /* re-load FS on retry */
2678: RECOVERY_SECTION
2679: RECOVER(EXT(inst_fetch_fault))
2680: movzbl %fs:(%eax),%eax /* load instruction byte */
2681: ret
2682:
2683: LEXT(inst_fetch_fault)
2684: movl $-1,%eax /* return -1 if error */
2685: ret
2686:
2687:
2688: #if MACH_KDP
2689: /*
2690: * kdp_copy_kmem(char *src, char *dst, int count)
2691: *
2692: * Similar to copyin except that both addresses are kernel addresses.
2693: */
2694:
2695: ENTRY(kdp_copy_kmem)
2696: pushl %esi
2697: pushl %edi /* save registers */
2698:
2699: movl 8+S_ARG0,%esi /* get kernel start address */
2700: movl 8+S_ARG1,%edi /* get kernel destination address */
2701:
2702: movl 8+S_ARG2,%edx /* get count */
2703:
2704: lea 0(%esi,%edx),%eax /* get kernel end address + 1 */
2705:
2706: cmpl %esi,%eax
2707: jb kdp_vm_read_fail /* fail if wrap-around */
2708: cld /* count up */
2709: movl %edx,%ecx /* move by longwords first */
2710: shrl $2,%ecx
2711: RECOVERY_SECTION
2712: RECOVER(kdp_vm_read_fail)
2713: rep
2714: movsl /* move longwords */
2715: movl %edx,%ecx /* now move remaining bytes */
2716: andl $3,%ecx
2717: RECOVERY_SECTION
2718: RECOVER(kdp_vm_read_fail)
2719: rep
2720: movsb
2721: kdp_vm_read_done:
2722: movl 8+S_ARG2,%edx /* get count */
2723: subl %ecx,%edx /* Return number of bytes transfered */
2724: movl %edx,%eax
2725:
2726: popl %edi /* restore registers */
2727: popl %esi
2728: ret /* and return */
2729:
2730: kdp_vm_read_fail:
2731: xorl %eax,%eax /* didn't copy a thing. */
2732:
2733: popl %edi
2734: popl %esi
2735: ret
2736: #endif
2737:
2738:
2739: /*
2740: * Done with recovery and retry tables.
2741: */
2742: RECOVERY_SECTION
2743: RECOVER_TABLE_END
2744: RETRY_SECTION
2745: RETRY_TABLE_END
2746:
2747:
2748:
2749: ENTRY(dr6)
2750: movl %db6, %eax
2751: ret
2752:
2753: /* dr<i>(address, type, len, persistence)
2754: */
2755: ENTRY(dr0)
2756: movl S_ARG0, %eax
2757: movl %eax,EXT(dr_addr)
2758: movl %eax, %db0
2759: movl $0, %ecx
2760: jmp 0f
2761: ENTRY(dr1)
2762: movl S_ARG0, %eax
2763: movl %eax,EXT(dr_addr)+1*4
2764: movl %eax, %db1
2765: movl $2, %ecx
2766: jmp 0f
2767: ENTRY(dr2)
2768: movl S_ARG0, %eax
2769: movl %eax,EXT(dr_addr)+2*4
2770: movl %eax, %db2
2771: movl $4, %ecx
2772: jmp 0f
2773:
2774: ENTRY(dr3)
2775: movl S_ARG0, %eax
2776: movl %eax,EXT(dr_addr)+3*4
2777: movl %eax, %db3
2778: movl $6, %ecx
2779:
2780: 0:
2781: pushl %ebp
2782: movl %esp, %ebp
2783:
2784: movl %db7, %edx
2785: movl %edx,EXT(dr_addr)+4*4
2786: andl dr_msk(,%ecx,2),%edx /* clear out new entry */
2787: movl %edx,EXT(dr_addr)+5*4
2788: movzbl B_ARG3, %eax
2789: andb $3, %al
2790: shll %cl, %eax
2791: orl %eax, %edx
2792:
2793: movzbl B_ARG1, %eax
2794: andb $3, %al
2795: addb $0x10, %ecx
2796: shll %cl, %eax
2797: orl %eax, %edx
2798:
2799: movzbl B_ARG2, %eax
2800: andb $3, %al
2801: addb $0x2, %ecx
2802: shll %cl, %eax
2803: orl %eax, %edx
2804:
2805: movl %edx, %db7
2806: movl %edx,EXT(dr_addr)+7*4
2807: movl %edx, %eax
2808: leave
2809: ret
2810:
2811: .data
2812:
2813: DATA(preemptable) /* Not on an MP (makes cpu_number() usage unsafe) */
2814: #if MACH_RT && (NCPUS == 1)
2815: .long 0 /* FIXME -- Currently disabled */
2816: #else
2817: .long 0 /* FIX ME -- Currently disabled */
2818: #endif /* MACH_RT && (NCPUS == 1) */
2819:
2820: dr_msk:
2821: .long ~0x000f0003
2822: .long ~0x00f0000c
2823: .long ~0x0f000030
2824: .long ~0xf00000c0
2825: ENTRY(dr_addr)
2826: .long 0,0,0,0
2827: .long 0,0,0,0
2828: .text
2829:
2830: /*
2831: * Determine cpu model and set global cpuid_xxx variables
2832: *
2833: * Relies on 386 eflags bit 18 (AC) always being zero & 486 preserving it.
2834: * Relies on 486 eflags bit 21 (ID) always being zero & 586 preserving it.
2835: * Relies on CPUID instruction for next x86 generations
2836: * (assumes cpuid-family-homogenous MPs; else convert to per-cpu array)
2837: */
2838:
2839: ENTRY(set_cpu_model)
2840: FRAME
2841: pushl %ebx /* save ebx */
2842: andl $~0x3,%esp /* Align stack to avoid AC fault */
2843: pushfl /* push EFLAGS */
2844: popl %eax /* pop into eax */
2845: movl %eax,%ecx /* Save original EFLAGS */
2846: xorl $(EFL_AC+EFL_ID),%eax /* toggle ID,AC bits */
2847: pushl %eax /* push new value */
2848: popfl /* through the EFLAGS register */
2849: pushfl /* and back */
2850: popl %eax /* into eax */
2851: movb $(CPUID_FAMILY_386),EXT(cpuid_family)
2852: pushl %ecx /* push original EFLAGS */
2853: popfl /* restore EFLAGS */
2854: xorl %ecx,%eax /* see what changed */
2855: testl $EFL_AC,%eax /* test AC bit */
2856: jz 0f /* if AC toggled (486 or higher) */
2857:
2858: movb $(CPUID_FAMILY_486),EXT(cpuid_family)
2859: testl $EFL_ID,%eax /* test ID bit */
2860: jz 0f /* if ID toggled use cpuid instruction */
2861:
2862: xorl %eax,%eax /* get vendor identification string */
2863: .word 0xA20F /* cpuid instruction */
2864: movl %eax,EXT(cpuid_value) /* Store high value */
2865: movl %ebx,EXT(cpuid_vid) /* Store byte 0-3 of Vendor ID */
2866: movl %edx,EXT(cpuid_vid)+4 /* Store byte 4-7 of Vendor ID */
2867: movl %ecx,EXT(cpuid_vid)+8 /* Store byte 8-B of Vendor ID */
2868: movl $1,%eax /* get processor signature */
2869: .word 0xA20F /* cpuid instruction */
2870: movl %edx,EXT(cpuid_feature) /* Store feature flags */
2871: movl %eax,%ecx /* Save original signature */
2872: andb $0xF,%al /* Get Stepping ID */
2873: movb %al,EXT(cpuid_stepping) /* Save Stepping ID */
2874: movl %ecx,%eax /* Get original signature */
2875: shrl $4,%eax /* Shift Stepping ID */
2876: movl %eax,%ecx /* Save original signature */
2877: andb $0xF,%al /* Get Model */
2878: movb %al,EXT(cpuid_model) /* Save Model */
2879: movl %ecx,%eax /* Get original signature */
2880: shrl $4,%eax /* Shift Stepping ID */
2881: movl %eax,%ecx /* Save original signature */
2882: andb $0xF,%al /* Get Family */
2883: movb %al,EXT(cpuid_family) /* Save Family */
2884: movl %ecx,%eax /* Get original signature */
2885: shrl $4,%eax /* Shift Stepping ID */
2886: andb $0x3,%al /* Get Type */
2887: movb %al,EXT(cpuid_type) /* Save Type */
2888:
2889: movl EXT(cpuid_value),%eax /* Get high value */
2890: cmpl $2,%eax /* Test if processor configuration */
2891: jle 0f /* is present */
2892: movl $2,%eax /* get processor configuration */
2893: .word 0xA20F /* cpuid instruction */
2894: movl %eax,EXT(cpuid_cache) /* Store byte 0-3 of configuration */
2895: movl %ebx,EXT(cpuid_cache)+4 /* Store byte 4-7 of configuration */
2896: movl %ecx,EXT(cpuid_cache)+8 /* Store byte 8-B of configuration */
2897: movl %edx,EXT(cpuid_cache)+12 /* Store byte C-F of configuration */
2898: 0:
2899: popl %ebx /* restore ebx */
2900: EMARF
2901: ret /* return */
2902:
2903: ENTRY(get_cr0)
2904: movl %cr0, %eax
2905: ret
2906:
2907: ENTRY(set_cr0)
2908: movl 4(%esp), %eax
2909: movl %eax, %cr0
2910: ret
2911:
2912: #ifndef SYMMETRY
2913:
2914: /*
2915: * ffs(mask)
2916: */
2917: ENTRY(ffs)
2918: bsfl S_ARG0, %eax
2919: jz 0f
2920: incl %eax
2921: ret
2922: 0: xorl %eax, %eax
2923: ret
2924:
2925: /*
2926: * cpu_shutdown()
2927: * Force reboot
2928: */
2929:
2930: null_idtr:
2931: .word 0
2932: .long 0
2933:
2934: Entry(cpu_shutdown)
2935: lidt null_idtr /* disable the interrupt handler */
2936: xor %ecx,%ecx /* generate a divide by zero */
2937: div %ecx,%eax /* reboot now */
2938: ret /* this will "never" be executed */
2939:
2940: #endif /* SYMMETRY */
2941:
2942:
2943: /*
2944: * setbit(int bitno, int *s) - set bit in bit string
2945: */
2946: ENTRY(setbit)
2947: movl S_ARG0, %ecx /* bit number */
2948: movl S_ARG1, %eax /* address */
2949: btsl %ecx, (%eax) /* set bit */
2950: ret
2951:
2952: /*
2953: * clrbit(int bitno, int *s) - clear bit in bit string
2954: */
2955: ENTRY(clrbit)
2956: movl S_ARG0, %ecx /* bit number */
2957: movl S_ARG1, %eax /* address */
2958: btrl %ecx, (%eax) /* clear bit */
2959: ret
2960:
2961: /*
2962: * ffsbit(int *s) - find first set bit in bit string
2963: */
2964: ENTRY(ffsbit)
2965: movl S_ARG0, %ecx /* address */
2966: movl $0, %edx /* base offset */
2967: 0:
2968: bsfl (%ecx), %eax /* check argument bits */
2969: jnz 1f /* found bit, return */
2970: addl $4, %ecx /* increment address */
2971: addl $32, %edx /* increment offset */
2972: jmp 0b /* try again */
2973: 1:
2974: addl %edx, %eax /* return offset */
2975: ret
2976:
2977: /*
2978: * testbit(int nr, volatile void *array)
2979: *
2980: * Test to see if the bit is set within the bit string
2981: */
2982:
2983: ENTRY(testbit)
2984: movl S_ARG0,%eax /* Get the bit to test */
2985: movl S_ARG1,%ecx /* get the array string */
2986: btl %eax,(%ecx)
2987: sbbl %eax,%eax
2988: ret
2989:
2990: ENTRY(get_pc)
2991: movl 4(%ebp),%eax
2992: ret
2993:
2994: #if ETAP
2995:
2996: ENTRY(etap_get_pc)
2997: movl 4(%ebp), %eax /* fetch pc of caller */
2998: ret
2999:
3000: ENTRY(tvals_to_etap)
3001: movl S_ARG0, %eax
3002: movl $1000000000, %ecx
3003: mull %ecx
3004: addl S_ARG1, %eax
3005: adc $0, %edx
3006: ret
3007:
3008: /* etap_time_t
3009: * etap_time_sub(etap_time_t stop, etap_time_t start)
3010: *
3011: * 64bit subtract, returns stop - start
3012: */
3013: ENTRY(etap_time_sub)
3014: movl S_ARG0, %eax /* stop.low */
3015: movl S_ARG1, %edx /* stop.hi */
3016: subl S_ARG2, %eax /* stop.lo - start.lo */
3017: sbbl S_ARG3, %edx /* stop.hi - start.hi */
3018: ret
3019:
3020: #endif /* ETAP */
3021:
3022: #if NCPUS > 1
3023:
3024: ENTRY(minsecurity)
3025: pushl %ebp
3026: movl %esp,%ebp
3027: /*
3028: * jail: set the EIP to "jail" to block a kernel thread.
3029: * Useful to debug synchronization problems on MPs.
3030: */
3031: ENTRY(jail)
3032: jmp EXT(jail)
3033:
3034: #endif /* NCPUS > 1 */
3035:
3036: /*
3037: * delay(microseconds)
3038: */
3039:
3040: ENTRY(delay)
3041: movl 4(%esp),%eax
3042: testl %eax, %eax
3043: jle 3f
3044: movl EXT(delaycount), %ecx
3045: 1:
3046: movl %ecx, %edx
3047: 2:
3048: decl %edx
3049: jne 2b
3050: decl %eax
3051: jne 1b
3052: 3:
3053: ret
3054:
3055: /*
3056: * unsigned int
3057: * div_scale(unsigned int dividend,
3058: * unsigned int divisor,
3059: * unsigned int *scale)
3060: *
3061: * This function returns (dividend << *scale) //divisor where *scale
3062: * is the largest possible value before overflow. This is used in
3063: * computation where precision must be achieved in order to avoid
3064: * floating point usage.
3065: *
3066: * Algorithm:
3067: * *scale = 0;
3068: * while (((dividend >> *scale) >= divisor))
3069: * (*scale)++;
3070: * *scale = 32 - *scale;
3071: * return ((dividend << *scale) / divisor);
3072: */
3073: ENTRY(div_scale)
3074: PUSH_FRAME
3075: xorl %ecx, %ecx /* *scale = 0 */
3076: xorl %eax, %eax
3077: movl ARG0, %edx /* get dividend */
3078: 0:
3079: cmpl ARG1, %edx /* if (divisor > dividend) */
3080: jle 1f /* goto 1f */
3081: addl $1, %ecx /* (*scale)++ */
3082: shrdl $1, %edx, %eax /* dividend >> 1 */
3083: shrl $1, %edx /* dividend >> 1 */
3084: jmp 0b /* goto 0b */
3085: 1:
3086: divl ARG1 /* (dividend << (32 - *scale)) / divisor */
3087: movl ARG2, %edx /* get scale */
3088: movl $32, (%edx) /* *scale = 32 */
3089: subl %ecx, (%edx) /* *scale -= %ecx */
3090: POP_FRAME
3091: ret
3092:
3093: /*
3094: * unsigned int
3095: * mul_scale(unsigned int multiplicand,
3096: * unsigned int multiplier,
3097: * unsigned int *scale)
3098: *
3099: * This function returns ((multiplicand * multiplier) >> *scale) where
3100: * scale is the largest possible value before overflow. This is used in
3101: * computation where precision must be achieved in order to avoid
3102: * floating point usage.
3103: *
3104: * Algorithm:
3105: * *scale = 0;
3106: * while (overflow((multiplicand * multiplier) >> *scale))
3107: * (*scale)++;
3108: * return ((multiplicand * multiplier) >> *scale);
3109: */
3110: ENTRY(mul_scale)
3111: PUSH_FRAME
3112: xorl %ecx, %ecx /* *scale = 0 */
3113: movl ARG0, %eax /* get multiplicand */
3114: mull ARG1 /* multiplicand * multiplier */
3115: 0:
3116: cmpl $0, %edx /* if (!overflow()) */
3117: je 1f /* goto 1 */
3118: addl $1, %ecx /* (*scale)++ */
3119: shrdl $1, %edx, %eax /* (multiplicand * multiplier) >> 1 */
3120: shrl $1, %edx /* (multiplicand * multiplier) >> 1 */
3121: jmp 0b
3122: 1:
3123: movl ARG2, %edx /* get scale */
3124: movl %ecx, (%edx) /* set *scale */
3125: POP_FRAME
3126: ret
3127:
3128: #if NCPUS > 1
3129: ENTRY(_cpu_number)
3130: CPU_NUMBER(%eax)
3131: ret
3132: #endif /* NCPUS > 1 */
3133:
3134: #ifdef MACH_BSD
3135: /*
3136: * BSD System call entry point..
3137: */
3138:
3139: Entry(trap_unix_syscall)
3140: pushf /* save flags as soon as possible */
3141: pushl %eax /* save system call number */
3142: pushl $0 /* clear trap number slot */
3143:
3144: pusha /* save the general registers */
3145: pushl %ds /* and the segment registers */
3146: pushl %es
3147: pushl %fs
3148: pushl %gs
3149:
3150: mov %ss,%dx /* switch to kernel data segment */
3151: mov %dx,%ds
3152: mov %dx,%es
3153: mov $CPU_DATA,%dx
3154: mov %dx,%gs
3155:
3156: /*
3157: * Shuffle eflags,eip,cs into proper places
3158: */
3159:
3160: movl R_EIP(%esp),%ebx /* eflags are in EIP slot */
3161: movl R_CS(%esp),%ecx /* eip is in CS slot */
3162: movl R_EFLAGS(%esp),%edx /* cs is in EFLAGS slot */
3163: movl %ecx,R_EIP(%esp) /* fix eip */
3164: movl %edx,R_CS(%esp) /* fix cs */
3165: movl %ebx,R_EFLAGS(%esp) /* fix eflags */
3166:
3167: CPU_NUMBER(%edx)
3168: TIME_TRAP_UENTRY
3169:
3170: negl %eax /* get system call number */
3171: shll $4,%eax /* manual indexing */
3172:
3173: CPU_NUMBER(%edx)
3174: movl CX(EXT(kernel_stack),%edx),%ebx
3175: /* get current kernel stack */
3176: xchgl %ebx,%esp /* switch stacks - %ebx points to */
3177: /* user registers. */
3178:
3179: /*
3180: * Register use on entry:
3181: * eax contains syscall number
3182: * ebx contains user regs pointer
3183: */
3184: CAH(call_call)
3185: pushl %ebx /* Push the regs set onto stack */
3186: call EXT(unix_syscall)
3187: popl %ebx
3188: movl %esp,%ecx /* get kernel stack */
3189: or $(KERNEL_STACK_SIZE-1),%ecx
3190: movl -3-IKS_SIZE(%ecx),%esp /* switch back to PCB stack */
3191: movl %eax,R_EAX(%esp) /* save return value */
3192: jmp EXT(return_from_trap) /* return to user */
3193:
3194: /*
3195: * Entry point for machdep system calls..
3196: */
3197:
3198: Entry(trap_machdep_syscall)
3199: pushf /* save flags as soon as possible */
3200: pushl %eax /* save system call number */
3201: pushl $0 /* clear trap number slot */
3202:
3203: pusha /* save the general registers */
3204: pushl %ds /* and the segment registers */
3205: pushl %es
3206: pushl %fs
3207: pushl %gs
3208:
3209: mov %ss,%dx /* switch to kernel data segment */
3210: mov %dx,%ds
3211: mov %dx,%es
3212: mov $CPU_DATA,%dx
3213: mov %dx,%gs
3214:
3215: /*
3216: * Shuffle eflags,eip,cs into proper places
3217: */
3218:
3219: movl R_EIP(%esp),%ebx /* eflags are in EIP slot */
3220: movl R_CS(%esp),%ecx /* eip is in CS slot */
3221: movl R_EFLAGS(%esp),%edx /* cs is in EFLAGS slot */
3222: movl %ecx,R_EIP(%esp) /* fix eip */
3223: movl %edx,R_CS(%esp) /* fix cs */
3224: movl %ebx,R_EFLAGS(%esp) /* fix eflags */
3225:
3226: CPU_NUMBER(%edx)
3227: TIME_TRAP_UENTRY
3228:
3229: negl %eax /* get system call number */
3230: shll $4,%eax /* manual indexing */
3231:
3232: CPU_NUMBER(%edx)
3233: movl CX(EXT(kernel_stack),%edx),%ebx
3234: /* get current kernel stack */
3235: xchgl %ebx,%esp /* switch stacks - %ebx points to */
3236: /* user registers. */
3237:
3238: /*
3239: * Register use on entry:
3240: * eax contains syscall number
3241: * ebx contains user regs pointer
3242: */
3243: CAH(call_call)
3244: pushl %ebx
3245: call EXT(machdep_syscall)
3246: popl %ebx
3247: movl %esp,%ecx /* get kernel stack */
3248: or $(KERNEL_STACK_SIZE-1),%ecx
3249: movl -3-IKS_SIZE(%ecx),%esp /* switch back to PCB stack */
3250: movl %eax,R_EAX(%esp) /* save return value */
3251: jmp EXT(return_from_trap) /* return to user */
3252:
3253: Entry(trap_mach25_syscall)
3254: pushf /* save flags as soon as possible */
3255: pushl %eax /* save system call number */
3256: pushl $0 /* clear trap number slot */
3257:
3258: pusha /* save the general registers */
3259: pushl %ds /* and the segment registers */
3260: pushl %es
3261: pushl %fs
3262: pushl %gs
3263:
3264: mov %ss,%dx /* switch to kernel data segment */
3265: mov %dx,%ds
3266: mov %dx,%es
3267: mov $CPU_DATA,%dx
3268: mov %dx,%gs
3269:
3270: /*
3271: * Shuffle eflags,eip,cs into proper places
3272: */
3273:
3274: movl R_EIP(%esp),%ebx /* eflags are in EIP slot */
3275: movl R_CS(%esp),%ecx /* eip is in CS slot */
3276: movl R_EFLAGS(%esp),%edx /* cs is in EFLAGS slot */
3277: movl %ecx,R_EIP(%esp) /* fix eip */
3278: movl %edx,R_CS(%esp) /* fix cs */
3279: movl %ebx,R_EFLAGS(%esp) /* fix eflags */
3280:
3281: CPU_NUMBER(%edx)
3282: TIME_TRAP_UENTRY
3283:
3284: negl %eax /* get system call number */
3285: shll $4,%eax /* manual indexing */
3286:
3287: CPU_NUMBER(%edx)
3288: movl CX(EXT(kernel_stack),%edx),%ebx
3289: /* get current kernel stack */
3290: xchgl %ebx,%esp /* switch stacks - %ebx points to */
3291: /* user registers. */
3292:
3293: /*
3294: * Register use on entry:
3295: * eax contains syscall number
3296: * ebx contains user regs pointer
3297: */
3298: CAH(call_call)
3299: pushl %ebx
3300: call EXT(mach25_syscall)
3301: popl %ebx
3302: movl %esp,%ecx /* get kernel stack */
3303: or $(KERNEL_STACK_SIZE-1),%ecx
3304: movl -3-IKS_SIZE(%ecx),%esp /* switch back to PCB stack */
3305: movl %eax,R_EAX(%esp) /* save return value */
3306: jmp EXT(return_from_trap) /* return to user */
3307:
3308: #endif
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.