![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_fault.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_fault.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1994,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm_fault.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: *
33: * Page fault handling module.
34: */
35:
1.1.1.3 ! root 36: #include <kern/printf.h>
1.1 root 37: #include <vm/vm_fault.h>
38: #include <mach/kern_return.h>
39: #include <mach/message.h> /* for error codes */
40: #include <kern/counters.h>
1.1.1.3 ! root 41: #include <kern/debug.h>
1.1 root 42: #include <kern/thread.h>
43: #include <kern/sched_prim.h>
44: #include <vm/vm_map.h>
45: #include <vm/vm_object.h>
46: #include <vm/vm_page.h>
47: #include <vm/pmap.h>
48: #include <mach/vm_statistics.h>
49: #include <vm/vm_pageout.h>
50: #include <mach/vm_param.h>
51: #include <mach/memory_object.h>
1.1.1.3 ! root 52: #include <vm/memory_object_user.user.h>
1.1 root 53: /* For memory_object_data_{request,unlock} */
54: #include <kern/macro_help.h>
1.1.1.3 ! root 55: #include <kern/slab.h>
1.1 root 56:
57: #if MACH_PCSAMPLE
58: #include <kern/pc_sample.h>
59: #endif
60:
61:
62:
63: /*
64: * State needed by vm_fault_continue.
65: * This is a little hefty to drop directly
66: * into the thread structure.
67: */
68: typedef struct vm_fault_state {
69: struct vm_map *vmf_map;
70: vm_offset_t vmf_vaddr;
71: vm_prot_t vmf_fault_type;
72: boolean_t vmf_change_wiring;
73: void (*vmf_continuation)();
74: vm_map_version_t vmf_version;
75: boolean_t vmf_wired;
76: struct vm_object *vmf_object;
77: vm_offset_t vmf_offset;
78: vm_prot_t vmf_prot;
79:
80: boolean_t vmfp_backoff;
81: struct vm_object *vmfp_object;
82: vm_offset_t vmfp_offset;
83: struct vm_page *vmfp_first_m;
84: vm_prot_t vmfp_access;
85: } vm_fault_state_t;
86:
1.1.1.3 ! root 87: struct kmem_cache vm_fault_state_cache;
1.1 root 88:
89: int vm_object_absent_max = 50;
90:
91: int vm_fault_debug = 0;
92:
93: boolean_t vm_fault_dirty_handling = FALSE;
94: boolean_t vm_fault_interruptible = TRUE;
95:
96: boolean_t software_reference_bits = TRUE;
97:
98: #if MACH_KDB
99: extern struct db_watchpoint *db_watchpoint_list;
1.1.1.2 root 100: #endif /* MACH_KDB */
1.1 root 101:
102: /*
103: * Routine: vm_fault_init
104: * Purpose:
105: * Initialize our private data structures.
106: */
1.1.1.3 ! root 107: void vm_fault_init(void)
1.1 root 108: {
1.1.1.3 ! root 109: kmem_cache_init(&vm_fault_state_cache, "vm_fault_state",
! 110: sizeof(vm_fault_state_t), 0, NULL, NULL, NULL, 0);
1.1 root 111: }
112:
113: /*
114: * Routine: vm_fault_cleanup
115: * Purpose:
116: * Clean up the result of vm_fault_page.
117: * Results:
118: * The paging reference for "object" is released.
119: * "object" is unlocked.
120: * If "top_page" is not null, "top_page" is
121: * freed and the paging reference for the object
122: * containing it is released.
123: *
124: * In/out conditions:
125: * "object" must be locked.
126: */
127: void
128: vm_fault_cleanup(object, top_page)
129: register vm_object_t object;
130: register vm_page_t top_page;
131: {
132: vm_object_paging_end(object);
133: vm_object_unlock(object);
134:
135: if (top_page != VM_PAGE_NULL) {
136: object = top_page->object;
137: vm_object_lock(object);
138: VM_PAGE_FREE(top_page);
139: vm_object_paging_end(object);
140: vm_object_unlock(object);
141: }
142: }
143:
144:
145: #if MACH_PCSAMPLE
146: /*
147: * Do PC sampling on current thread, assuming
148: * that it is the thread taking this page fault.
149: *
150: * Must check for THREAD_NULL, since faults
151: * can occur before threads are running.
152: */
153:
154: #define vm_stat_sample(flavor) \
155: MACRO_BEGIN \
156: thread_t _thread_ = current_thread(); \
157: \
158: if (_thread_ != THREAD_NULL) \
159: take_pc_sample_macro(_thread_, (flavor)); \
160: MACRO_END
161:
162: #else
163: #define vm_stat_sample(x)
164: #endif /* MACH_PCSAMPLE */
165:
166:
167:
168: /*
169: * Routine: vm_fault_page
170: * Purpose:
171: * Find the resident page for the virtual memory
172: * specified by the given virtual memory object
173: * and offset.
174: * Additional arguments:
175: * The required permissions for the page is given
176: * in "fault_type". Desired permissions are included
177: * in "protection".
178: *
179: * If the desired page is known to be resident (for
180: * example, because it was previously wired down), asserting
181: * the "unwiring" parameter will speed the search.
182: *
183: * If the operation can be interrupted (by thread_abort
184: * or thread_terminate), then the "interruptible"
185: * parameter should be asserted.
186: *
187: * Results:
188: * The page containing the proper data is returned
189: * in "result_page".
190: *
191: * In/out conditions:
192: * The source object must be locked and referenced,
193: * and must donate one paging reference. The reference
194: * is not affected. The paging reference and lock are
195: * consumed.
196: *
197: * If the call succeeds, the object in which "result_page"
198: * resides is left locked and holding a paging reference.
199: * If this is not the original object, a busy page in the
200: * original object is returned in "top_page", to prevent other
201: * callers from pursuing this same data, along with a paging
202: * reference for the original object. The "top_page" should
203: * be destroyed when this guarantee is no longer required.
204: * The "result_page" is also left busy. It is not removed
205: * from the pageout queues.
206: */
207: vm_fault_return_t vm_fault_page(first_object, first_offset,
208: fault_type, must_be_resident, interruptible,
209: protection,
210: result_page, top_page,
211: resume, continuation)
212: /* Arguments: */
213: vm_object_t first_object; /* Object to begin search */
214: vm_offset_t first_offset; /* Offset into object */
215: vm_prot_t fault_type; /* What access is requested */
216: boolean_t must_be_resident;/* Must page be resident? */
217: boolean_t interruptible; /* May fault be interrupted? */
218: /* Modifies in place: */
219: vm_prot_t *protection; /* Protection for mapping */
220: /* Returns: */
221: vm_page_t *result_page; /* Page found, if successful */
222: vm_page_t *top_page; /* Page in top object, if
223: * not result_page.
224: */
225: /* More arguments: */
226: boolean_t resume; /* We are restarting. */
227: void (*continuation)(); /* Continuation for blocking. */
228: {
229: register
230: vm_page_t m;
231: register
232: vm_object_t object;
233: register
234: vm_offset_t offset;
235: vm_page_t first_m;
236: vm_object_t next_object;
237: vm_object_t copy_object;
238: boolean_t look_for_page;
239: vm_prot_t access_required;
240:
241: if (resume) {
242: register vm_fault_state_t *state =
243: (vm_fault_state_t *) current_thread()->ith_other;
244:
245: if (state->vmfp_backoff)
246: goto after_block_and_backoff;
247:
248: object = state->vmfp_object;
249: offset = state->vmfp_offset;
250: first_m = state->vmfp_first_m;
251: access_required = state->vmfp_access;
252: goto after_thread_block;
253: }
254:
255: vm_stat_sample(SAMPLED_PC_VM_FAULTS_ANY);
256: vm_stat.faults++; /* needs lock XXX */
1.1.1.3 ! root 257: current_task()->faults++;
1.1 root 258:
259: /*
260: * Recovery actions
261: */
262: #define RELEASE_PAGE(m) \
263: MACRO_BEGIN \
264: PAGE_WAKEUP_DONE(m); \
265: vm_page_lock_queues(); \
266: if (!m->active && !m->inactive) \
267: vm_page_activate(m); \
268: vm_page_unlock_queues(); \
269: MACRO_END
270:
271: if (vm_fault_dirty_handling
272: #if MACH_KDB
273: /*
274: * If there are watchpoints set, then
275: * we don't want to give away write permission
276: * on a read fault. Make the task write fault,
277: * so that the watchpoint code notices the access.
278: */
279: || db_watchpoint_list
1.1.1.2 root 280: #endif /* MACH_KDB */
1.1 root 281: ) {
282: /*
283: * If we aren't asking for write permission,
284: * then don't give it away. We're using write
285: * faults to set the dirty bit.
286: */
287: if (!(fault_type & VM_PROT_WRITE))
288: *protection &= ~VM_PROT_WRITE;
289: }
290:
291: if (!vm_fault_interruptible)
292: interruptible = FALSE;
293:
294: /*
295: * INVARIANTS (through entire routine):
296: *
297: * 1) At all times, we must either have the object
298: * lock or a busy page in some object to prevent
299: * some other thread from trying to bring in
300: * the same page.
301: *
302: * Note that we cannot hold any locks during the
303: * pager access or when waiting for memory, so
304: * we use a busy page then.
305: *
306: * Note also that we aren't as concerned about more than
307: * one thread attempting to memory_object_data_unlock
308: * the same page at once, so we don't hold the page
309: * as busy then, but do record the highest unlock
310: * value so far. [Unlock requests may also be delivered
311: * out of order.]
312: *
313: * 2) To prevent another thread from racing us down the
314: * shadow chain and entering a new page in the top
315: * object before we do, we must keep a busy page in
316: * the top object while following the shadow chain.
317: *
318: * 3) We must increment paging_in_progress on any object
319: * for which we have a busy page, to prevent
320: * vm_object_collapse from removing the busy page
321: * without our noticing.
322: *
323: * 4) We leave busy pages on the pageout queues.
324: * If the pageout daemon comes across a busy page,
325: * it will remove the page from the pageout queues.
326: */
327:
328: /*
329: * Search for the page at object/offset.
330: */
331:
332: object = first_object;
333: offset = first_offset;
334: first_m = VM_PAGE_NULL;
335: access_required = fault_type;
336:
337: /*
338: * See whether this page is resident
339: */
340:
341: while (TRUE) {
342: m = vm_page_lookup(object, offset);
343: if (m != VM_PAGE_NULL) {
344: /*
345: * If the page is being brought in,
346: * wait for it and then retry.
347: *
348: * A possible optimization: if the page
349: * is known to be resident, we can ignore
350: * pages that are absent (regardless of
351: * whether they're busy).
352: */
353:
354: if (m->busy) {
355: kern_return_t wait_result;
356:
357: PAGE_ASSERT_WAIT(m, interruptible);
358: vm_object_unlock(object);
359: if (continuation != (void (*)()) 0) {
360: register vm_fault_state_t *state =
361: (vm_fault_state_t *) current_thread()->ith_other;
362:
363: /*
364: * Save variables in case
365: * thread_block discards
366: * our kernel stack.
367: */
368:
369: state->vmfp_backoff = FALSE;
370: state->vmfp_object = object;
371: state->vmfp_offset = offset;
372: state->vmfp_first_m = first_m;
373: state->vmfp_access =
374: access_required;
375: state->vmf_prot = *protection;
376:
377: counter(c_vm_fault_page_block_busy_user++);
378: thread_block(continuation);
379: } else
380: {
381: counter(c_vm_fault_page_block_busy_kernel++);
382: thread_block((void (*)()) 0);
383: }
384: after_thread_block:
385: wait_result = current_thread()->wait_result;
386: vm_object_lock(object);
387: if (wait_result != THREAD_AWAKENED) {
388: vm_fault_cleanup(object, first_m);
389: if (wait_result == THREAD_RESTART)
390: return(VM_FAULT_RETRY);
391: else
392: return(VM_FAULT_INTERRUPTED);
393: }
394: continue;
395: }
396:
397: /*
398: * If the page is in error, give up now.
399: */
400:
401: if (m->error) {
402: VM_PAGE_FREE(m);
403: vm_fault_cleanup(object, first_m);
404: return(VM_FAULT_MEMORY_ERROR);
405: }
406:
407: /*
408: * If the page isn't busy, but is absent,
409: * then it was deemed "unavailable".
410: */
411:
412: if (m->absent) {
413: /*
414: * Remove the non-existent page (unless it's
415: * in the top object) and move on down to the
416: * next object (if there is one).
417: */
418:
419: offset += object->shadow_offset;
420: access_required = VM_PROT_READ;
421: next_object = object->shadow;
422: if (next_object == VM_OBJECT_NULL) {
423: vm_page_t real_m;
424:
425: assert(!must_be_resident);
426:
427: /*
428: * Absent page at bottom of shadow
429: * chain; zero fill the page we left
430: * busy in the first object, and flush
431: * the absent page. But first we
432: * need to allocate a real page.
433: */
434:
1.1.1.2 root 435: real_m = vm_page_grab(!object->internal);
1.1 root 436: if (real_m == VM_PAGE_NULL) {
437: vm_fault_cleanup(object, first_m);
438: return(VM_FAULT_MEMORY_SHORTAGE);
439: }
440:
441: if (object != first_object) {
442: VM_PAGE_FREE(m);
443: vm_object_paging_end(object);
444: vm_object_unlock(object);
445: object = first_object;
446: offset = first_offset;
447: m = first_m;
448: first_m = VM_PAGE_NULL;
449: vm_object_lock(object);
450: }
451:
452: VM_PAGE_FREE(m);
453: assert(real_m->busy);
454: vm_page_lock_queues();
455: vm_page_insert(real_m, object, offset);
456: vm_page_unlock_queues();
457: m = real_m;
458:
459: /*
460: * Drop the lock while zero filling
461: * page. Then break because this
462: * is the page we wanted. Checking
463: * the page lock is a waste of time;
464: * this page was either absent or
465: * newly allocated -- in both cases
466: * it can't be page locked by a pager.
467: */
468: vm_object_unlock(object);
469:
470: vm_page_zero_fill(m);
471:
472: vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS);
1.1.1.2 root 473:
1.1 root 474: vm_stat.zero_fill_count++;
1.1.1.3 ! root 475: current_task()->zero_fills++;
1.1 root 476: vm_object_lock(object);
477: pmap_clear_modify(m->phys_addr);
478: break;
479: } else {
480: if (must_be_resident) {
481: vm_object_paging_end(object);
482: } else if (object != first_object) {
483: vm_object_paging_end(object);
484: VM_PAGE_FREE(m);
485: } else {
486: first_m = m;
487: m->absent = FALSE;
488: vm_object_absent_release(object);
489: m->busy = TRUE;
490:
491: vm_page_lock_queues();
492: VM_PAGE_QUEUES_REMOVE(m);
493: vm_page_unlock_queues();
494: }
495: vm_object_lock(next_object);
496: vm_object_unlock(object);
497: object = next_object;
498: vm_object_paging_begin(object);
499: continue;
500: }
501: }
502:
503: /*
504: * If the desired access to this page has
505: * been locked out, request that it be unlocked.
506: */
507:
508: if (access_required & m->page_lock) {
509: if ((access_required & m->unlock_request) != access_required) {
510: vm_prot_t new_unlock_request;
511: kern_return_t rc;
1.1.1.2 root 512:
1.1 root 513: if (!object->pager_ready) {
514: vm_object_assert_wait(object,
515: VM_OBJECT_EVENT_PAGER_READY,
516: interruptible);
517: goto block_and_backoff;
518: }
519:
520: new_unlock_request = m->unlock_request =
521: (access_required | m->unlock_request);
522: vm_object_unlock(object);
523: if ((rc = memory_object_data_unlock(
524: object->pager,
525: object->pager_request,
526: offset + object->paging_offset,
527: PAGE_SIZE,
528: new_unlock_request))
529: != KERN_SUCCESS) {
530: printf("vm_fault: memory_object_data_unlock failed\n");
531: vm_object_lock(object);
532: vm_fault_cleanup(object, first_m);
533: return((rc == MACH_SEND_INTERRUPTED) ?
534: VM_FAULT_INTERRUPTED :
535: VM_FAULT_MEMORY_ERROR);
536: }
537: vm_object_lock(object);
538: continue;
539: }
540:
541: PAGE_ASSERT_WAIT(m, interruptible);
542: goto block_and_backoff;
543: }
544:
545: /*
546: * We mark the page busy and leave it on
547: * the pageout queues. If the pageout
548: * deamon comes across it, then it will
549: * remove the page.
550: */
551:
552: if (!software_reference_bits) {
553: vm_page_lock_queues();
554: if (m->inactive) {
555: vm_stat_sample(SAMPLED_PC_VM_REACTIVATION_FAULTS);
556: vm_stat.reactivations++;
1.1.1.3 ! root 557: current_task()->reactivations++;
1.1 root 558: }
559:
560: VM_PAGE_QUEUES_REMOVE(m);
561: vm_page_unlock_queues();
562: }
563:
564: assert(!m->busy);
565: m->busy = TRUE;
566: assert(!m->absent);
567: break;
568: }
569:
570: look_for_page =
571: (object->pager_created)
572: #if MACH_PAGEMAP
573: && (vm_external_state_get(object->existence_info, offset + object->paging_offset) !=
574: VM_EXTERNAL_STATE_ABSENT)
1.1.1.2 root 575: #endif /* MACH_PAGEMAP */
1.1 root 576: ;
577:
578: if ((look_for_page || (object == first_object))
579: && !must_be_resident) {
580: /*
581: * Allocate a new page for this object/offset
582: * pair.
583: */
584:
585: m = vm_page_grab_fictitious();
586: if (m == VM_PAGE_NULL) {
587: vm_fault_cleanup(object, first_m);
588: return(VM_FAULT_FICTITIOUS_SHORTAGE);
589: }
590:
591: vm_page_lock_queues();
592: vm_page_insert(m, object, offset);
593: vm_page_unlock_queues();
594: }
595:
596: if (look_for_page && !must_be_resident) {
597: kern_return_t rc;
598:
599: /*
600: * If the memory manager is not ready, we
601: * cannot make requests.
602: */
603: if (!object->pager_ready) {
604: vm_object_assert_wait(object,
605: VM_OBJECT_EVENT_PAGER_READY,
606: interruptible);
607: VM_PAGE_FREE(m);
608: goto block_and_backoff;
609: }
610:
611: if (object->internal) {
612: /*
613: * Requests to the default pager
614: * must reserve a real page in advance,
615: * because the pager's data-provided
616: * won't block for pages.
617: */
618:
1.1.1.2 root 619: if (m->fictitious && !vm_page_convert(m, FALSE)) {
1.1 root 620: VM_PAGE_FREE(m);
621: vm_fault_cleanup(object, first_m);
622: return(VM_FAULT_MEMORY_SHORTAGE);
623: }
624: } else if (object->absent_count >
625: vm_object_absent_max) {
626: /*
627: * If there are too many outstanding page
628: * requests pending on this object, we
629: * wait for them to be resolved now.
630: */
631:
632: vm_object_absent_assert_wait(object, interruptible);
633: VM_PAGE_FREE(m);
634: goto block_and_backoff;
635: }
636:
637: /*
638: * Indicate that the page is waiting for data
639: * from the memory manager.
640: */
641:
642: m->absent = TRUE;
643: object->absent_count++;
644:
645: /*
646: * We have a busy page, so we can
647: * release the object lock.
648: */
649: vm_object_unlock(object);
650:
651: /*
652: * Call the memory manager to retrieve the data.
653: */
654:
655: vm_stat.pageins++;
656: vm_stat_sample(SAMPLED_PC_VM_PAGEIN_FAULTS);
1.1.1.3 ! root 657: current_task()->pageins++;
1.1 root 658:
1.1.1.2 root 659: if ((rc = memory_object_data_request(object->pager,
1.1 root 660: object->pager_request,
1.1.1.2 root 661: m->offset + object->paging_offset,
1.1 root 662: PAGE_SIZE, access_required)) != KERN_SUCCESS) {
663: if (rc != MACH_SEND_INTERRUPTED)
1.1.1.3 ! root 664: printf("%s(0x%p, 0x%p, 0x%lx, 0x%x, 0x%x) failed, %x\n",
1.1 root 665: "memory_object_data_request",
666: object->pager,
667: object->pager_request,
1.1.1.2 root 668: m->offset + object->paging_offset,
1.1 root 669: PAGE_SIZE, access_required, rc);
670: /*
671: * Don't want to leave a busy page around,
672: * but the data request may have blocked,
673: * so check if it's still there and busy.
674: */
675: vm_object_lock(object);
676: if (m == vm_page_lookup(object,offset) &&
677: m->absent && m->busy)
678: VM_PAGE_FREE(m);
679: vm_fault_cleanup(object, first_m);
680: return((rc == MACH_SEND_INTERRUPTED) ?
681: VM_FAULT_INTERRUPTED :
682: VM_FAULT_MEMORY_ERROR);
683: }
1.1.1.2 root 684:
1.1 root 685: /*
686: * Retry with same object/offset, since new data may
687: * be in a different page (i.e., m is meaningless at
688: * this point).
689: */
690: vm_object_lock(object);
691: continue;
692: }
693:
694: /*
695: * For the XP system, the only case in which we get here is if
696: * object has no pager (or unwiring). If the pager doesn't
697: * have the page this is handled in the m->absent case above
698: * (and if you change things here you should look above).
699: */
700: if (object == first_object)
701: first_m = m;
702: else
703: {
704: assert(m == VM_PAGE_NULL);
705: }
706:
707: /*
708: * Move on to the next object. Lock the next
709: * object before unlocking the current one.
710: */
711: access_required = VM_PROT_READ;
712:
713: offset += object->shadow_offset;
714: next_object = object->shadow;
715: if (next_object == VM_OBJECT_NULL) {
716: assert(!must_be_resident);
717:
718: /*
719: * If there's no object left, fill the page
720: * in the top object with zeros. But first we
721: * need to allocate a real page.
722: */
723:
724: if (object != first_object) {
725: vm_object_paging_end(object);
726: vm_object_unlock(object);
727:
728: object = first_object;
729: offset = first_offset;
730: vm_object_lock(object);
731: }
732:
733: m = first_m;
734: assert(m->object == object);
735: first_m = VM_PAGE_NULL;
736:
1.1.1.2 root 737: if (m->fictitious && !vm_page_convert(m, !object->internal)) {
1.1 root 738: VM_PAGE_FREE(m);
739: vm_fault_cleanup(object, VM_PAGE_NULL);
740: return(VM_FAULT_MEMORY_SHORTAGE);
741: }
742:
743: vm_object_unlock(object);
744: vm_page_zero_fill(m);
745: vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS);
746: vm_stat.zero_fill_count++;
1.1.1.3 ! root 747: current_task()->zero_fills++;
1.1 root 748: vm_object_lock(object);
749: pmap_clear_modify(m->phys_addr);
750: break;
751: }
752: else {
753: vm_object_lock(next_object);
754: if ((object != first_object) || must_be_resident)
755: vm_object_paging_end(object);
756: vm_object_unlock(object);
757: object = next_object;
758: vm_object_paging_begin(object);
759: }
760: }
761:
762: /*
763: * PAGE HAS BEEN FOUND.
764: *
765: * This page (m) is:
766: * busy, so that we can play with it;
767: * not absent, so that nobody else will fill it;
768: * possibly eligible for pageout;
769: *
770: * The top-level page (first_m) is:
771: * VM_PAGE_NULL if the page was found in the
772: * top-level object;
773: * busy, not absent, and ineligible for pageout.
774: *
775: * The current object (object) is locked. A paging
776: * reference is held for the current and top-level
777: * objects.
778: */
779:
780: #if EXTRA_ASSERTIONS
781: assert(m->busy && !m->absent);
782: assert((first_m == VM_PAGE_NULL) ||
783: (first_m->busy && !first_m->absent &&
784: !first_m->active && !first_m->inactive));
1.1.1.2 root 785: #endif /* EXTRA_ASSERTIONS */
1.1 root 786:
787: /*
788: * If the page is being written, but isn't
789: * already owned by the top-level object,
790: * we have to copy it into a new page owned
791: * by the top-level object.
792: */
793:
794: if (object != first_object) {
795: /*
796: * We only really need to copy if we
797: * want to write it.
798: */
799:
800: if (fault_type & VM_PROT_WRITE) {
801: vm_page_t copy_m;
802:
803: assert(!must_be_resident);
804:
805: /*
806: * If we try to collapse first_object at this
807: * point, we may deadlock when we try to get
808: * the lock on an intermediate object (since we
809: * have the bottom object locked). We can't
810: * unlock the bottom object, because the page
811: * we found may move (by collapse) if we do.
812: *
813: * Instead, we first copy the page. Then, when
814: * we have no more use for the bottom object,
815: * we unlock it and try to collapse.
816: *
817: * Note that we copy the page even if we didn't
818: * need to... that's the breaks.
819: */
820:
821: /*
822: * Allocate a page for the copy
823: */
1.1.1.2 root 824: copy_m = vm_page_grab(!first_object->internal);
1.1 root 825: if (copy_m == VM_PAGE_NULL) {
826: RELEASE_PAGE(m);
827: vm_fault_cleanup(object, first_m);
828: return(VM_FAULT_MEMORY_SHORTAGE);
829: }
830:
831: vm_object_unlock(object);
832: vm_page_copy(m, copy_m);
833: vm_object_lock(object);
834:
835: /*
836: * If another map is truly sharing this
837: * page with us, we have to flush all
838: * uses of the original page, since we
839: * can't distinguish those which want the
840: * original from those which need the
841: * new copy.
842: *
843: * XXXO If we know that only one map has
844: * access to this page, then we could
845: * avoid the pmap_page_protect() call.
846: */
847:
848: vm_page_lock_queues();
849: vm_page_deactivate(m);
850: pmap_page_protect(m->phys_addr, VM_PROT_NONE);
851: vm_page_unlock_queues();
852:
853: /*
854: * We no longer need the old page or object.
855: */
856:
857: PAGE_WAKEUP_DONE(m);
858: vm_object_paging_end(object);
859: vm_object_unlock(object);
860:
861: vm_stat.cow_faults++;
862: vm_stat_sample(SAMPLED_PC_VM_COW_FAULTS);
1.1.1.3 ! root 863: current_task()->cow_faults++;
1.1 root 864: object = first_object;
865: offset = first_offset;
866:
867: vm_object_lock(object);
868: VM_PAGE_FREE(first_m);
869: first_m = VM_PAGE_NULL;
870: assert(copy_m->busy);
871: vm_page_lock_queues();
872: vm_page_insert(copy_m, object, offset);
873: vm_page_unlock_queues();
874: m = copy_m;
875:
876: /*
877: * Now that we've gotten the copy out of the
878: * way, let's try to collapse the top object.
879: * But we have to play ugly games with
880: * paging_in_progress to do that...
881: */
882:
883: vm_object_paging_end(object);
884: vm_object_collapse(object);
885: vm_object_paging_begin(object);
886: }
887: else {
888: *protection &= (~VM_PROT_WRITE);
889: }
890: }
891:
892: /*
893: * Now check whether the page needs to be pushed into the
894: * copy object. The use of asymmetric copy on write for
895: * shared temporary objects means that we may do two copies to
896: * satisfy the fault; one above to get the page from a
897: * shadowed object, and one here to push it into the copy.
898: */
899:
900: while ((copy_object = first_object->copy) != VM_OBJECT_NULL) {
901: vm_offset_t copy_offset;
902: vm_page_t copy_m;
903:
904: /*
905: * If the page is being written, but hasn't been
906: * copied to the copy-object, we have to copy it there.
907: */
908:
909: if ((fault_type & VM_PROT_WRITE) == 0) {
910: *protection &= ~VM_PROT_WRITE;
911: break;
912: }
913:
914: /*
915: * If the page was guaranteed to be resident,
916: * we must have already performed the copy.
917: */
918:
919: if (must_be_resident)
920: break;
921:
922: /*
923: * Try to get the lock on the copy_object.
924: */
925: if (!vm_object_lock_try(copy_object)) {
926: vm_object_unlock(object);
927:
928: simple_lock_pause(); /* wait a bit */
929:
930: vm_object_lock(object);
931: continue;
932: }
933:
934: /*
935: * Make another reference to the copy-object,
936: * to keep it from disappearing during the
937: * copy.
938: */
939: assert(copy_object->ref_count > 0);
940: copy_object->ref_count++;
941:
942: /*
943: * Does the page exist in the copy?
944: */
945: copy_offset = first_offset - copy_object->shadow_offset;
946: copy_m = vm_page_lookup(copy_object, copy_offset);
947: if (copy_m != VM_PAGE_NULL) {
948: if (copy_m->busy) {
949: /*
950: * If the page is being brought
951: * in, wait for it and then retry.
952: */
953: PAGE_ASSERT_WAIT(copy_m, interruptible);
954: RELEASE_PAGE(m);
955: copy_object->ref_count--;
956: assert(copy_object->ref_count > 0);
957: vm_object_unlock(copy_object);
958: goto block_and_backoff;
959: }
960: }
961: else {
962: /*
963: * Allocate a page for the copy
964: */
965: copy_m = vm_page_alloc(copy_object, copy_offset);
966: if (copy_m == VM_PAGE_NULL) {
967: RELEASE_PAGE(m);
968: copy_object->ref_count--;
969: assert(copy_object->ref_count > 0);
970: vm_object_unlock(copy_object);
971: vm_fault_cleanup(object, first_m);
972: return(VM_FAULT_MEMORY_SHORTAGE);
973: }
974:
975: /*
976: * Must copy page into copy-object.
977: */
978:
979: vm_page_copy(m, copy_m);
1.1.1.2 root 980:
1.1 root 981: /*
982: * If the old page was in use by any users
983: * of the copy-object, it must be removed
984: * from all pmaps. (We can't know which
985: * pmaps use it.)
986: */
987:
988: vm_page_lock_queues();
989: pmap_page_protect(m->phys_addr, VM_PROT_NONE);
990: copy_m->dirty = TRUE;
991: vm_page_unlock_queues();
992:
993: /*
994: * If there's a pager, then immediately
995: * page out this page, using the "initialize"
996: * option. Else, we use the copy.
997: */
998:
999: if (!copy_object->pager_created) {
1000: vm_page_lock_queues();
1001: vm_page_activate(copy_m);
1002: vm_page_unlock_queues();
1003: PAGE_WAKEUP_DONE(copy_m);
1004: } else {
1005: /*
1006: * The page is already ready for pageout:
1007: * not on pageout queues and busy.
1008: * Unlock everything except the
1009: * copy_object itself.
1010: */
1011:
1012: vm_object_unlock(object);
1013:
1014: /*
1015: * Write the page to the copy-object,
1016: * flushing it from the kernel.
1017: */
1018:
1019: vm_pageout_page(copy_m, TRUE, TRUE);
1020:
1021: /*
1022: * Since the pageout may have
1023: * temporarily dropped the
1024: * copy_object's lock, we
1025: * check whether we'll have
1026: * to deallocate the hard way.
1027: */
1028:
1029: if ((copy_object->shadow != object) ||
1030: (copy_object->ref_count == 1)) {
1031: vm_object_unlock(copy_object);
1032: vm_object_deallocate(copy_object);
1033: vm_object_lock(object);
1034: continue;
1035: }
1036:
1037: /*
1038: * Pick back up the old object's
1039: * lock. [It is safe to do so,
1040: * since it must be deeper in the
1041: * object tree.]
1042: */
1043:
1044: vm_object_lock(object);
1045: }
1046:
1047: /*
1048: * Because we're pushing a page upward
1049: * in the object tree, we must restart
1050: * any faults that are waiting here.
1051: * [Note that this is an expansion of
1052: * PAGE_WAKEUP that uses the THREAD_RESTART
1053: * wait result]. Can't turn off the page's
1054: * busy bit because we're not done with it.
1055: */
1.1.1.2 root 1056:
1.1 root 1057: if (m->wanted) {
1058: m->wanted = FALSE;
1059: thread_wakeup_with_result((event_t) m,
1060: THREAD_RESTART);
1061: }
1062: }
1063:
1064: /*
1065: * The reference count on copy_object must be
1066: * at least 2: one for our extra reference,
1067: * and at least one from the outside world
1068: * (we checked that when we last locked
1069: * copy_object).
1070: */
1071: copy_object->ref_count--;
1072: assert(copy_object->ref_count > 0);
1073: vm_object_unlock(copy_object);
1074:
1075: break;
1076: }
1077:
1078: *result_page = m;
1079: *top_page = first_m;
1080:
1081: /*
1082: * If the page can be written, assume that it will be.
1083: * [Earlier, we restrict the permission to allow write
1084: * access only if the fault so required, so we don't
1085: * mark read-only data as dirty.]
1086: */
1087:
1088: if (vm_fault_dirty_handling && (*protection & VM_PROT_WRITE))
1089: m->dirty = TRUE;
1090:
1091: return(VM_FAULT_SUCCESS);
1092:
1093: block_and_backoff:
1094: vm_fault_cleanup(object, first_m);
1095:
1096: if (continuation != (void (*)()) 0) {
1097: register vm_fault_state_t *state =
1098: (vm_fault_state_t *) current_thread()->ith_other;
1099:
1100: /*
1101: * Save variables in case we must restart.
1102: */
1103:
1104: state->vmfp_backoff = TRUE;
1105: state->vmf_prot = *protection;
1106:
1107: counter(c_vm_fault_page_block_backoff_user++);
1108: thread_block(continuation);
1109: } else
1110: {
1111: counter(c_vm_fault_page_block_backoff_kernel++);
1112: thread_block((void (*)()) 0);
1113: }
1114: after_block_and_backoff:
1115: if (current_thread()->wait_result == THREAD_AWAKENED)
1116: return VM_FAULT_RETRY;
1117: else
1118: return VM_FAULT_INTERRUPTED;
1119:
1120: #undef RELEASE_PAGE
1121: }
1122:
1123: /*
1124: * Routine: vm_fault
1125: * Purpose:
1126: * Handle page faults, including pseudo-faults
1127: * used to change the wiring status of pages.
1128: * Returns:
1129: * If an explicit (expression) continuation is supplied,
1130: * then we call the continuation instead of returning.
1131: * Implementation:
1132: * Explicit continuations make this a little icky,
1133: * because it hasn't been rewritten to embrace CPS.
1134: * Instead, we have resume arguments for vm_fault and
1135: * vm_fault_page, to let continue the fault computation.
1136: *
1137: * vm_fault and vm_fault_page save mucho state
1138: * in the moral equivalent of a closure. The state
1139: * structure is allocated when first entering vm_fault
1140: * and deallocated when leaving vm_fault.
1141: */
1142:
1143: void
1144: vm_fault_continue()
1145: {
1146: register vm_fault_state_t *state =
1147: (vm_fault_state_t *) current_thread()->ith_other;
1148:
1149: (void) vm_fault(state->vmf_map,
1150: state->vmf_vaddr,
1151: state->vmf_fault_type,
1152: state->vmf_change_wiring,
1153: TRUE, state->vmf_continuation);
1154: /*NOTREACHED*/
1155: }
1156:
1157: kern_return_t vm_fault(map, vaddr, fault_type, change_wiring,
1158: resume, continuation)
1159: vm_map_t map;
1160: vm_offset_t vaddr;
1161: vm_prot_t fault_type;
1162: boolean_t change_wiring;
1163: boolean_t resume;
1164: void (*continuation)();
1165: {
1166: vm_map_version_t version; /* Map version for verificiation */
1167: boolean_t wired; /* Should mapping be wired down? */
1168: vm_object_t object; /* Top-level object */
1169: vm_offset_t offset; /* Top-level offset */
1170: vm_prot_t prot; /* Protection for mapping */
1171: vm_object_t old_copy_object; /* Saved copy object */
1172: vm_page_t result_page; /* Result of vm_fault_page */
1173: vm_page_t top_page; /* Placeholder page */
1174: kern_return_t kr;
1175:
1176: register
1177: vm_page_t m; /* Fast access to result_page */
1178:
1179: if (resume) {
1180: register vm_fault_state_t *state =
1181: (vm_fault_state_t *) current_thread()->ith_other;
1182:
1183: /*
1184: * Retrieve cached variables and
1185: * continue vm_fault_page.
1186: */
1187:
1188: object = state->vmf_object;
1189: if (object == VM_OBJECT_NULL)
1190: goto RetryFault;
1191: version = state->vmf_version;
1192: wired = state->vmf_wired;
1193: offset = state->vmf_offset;
1194: prot = state->vmf_prot;
1195:
1196: kr = vm_fault_page(object, offset, fault_type,
1197: (change_wiring && !wired), !change_wiring,
1198: &prot, &result_page, &top_page,
1199: TRUE, vm_fault_continue);
1200: goto after_vm_fault_page;
1201: }
1202:
1203: if (continuation != (void (*)()) 0) {
1204: /*
1205: * We will probably need to save state.
1206: */
1207:
1208: char * state;
1209:
1210: /*
1211: * if this assignment stmt is written as
1.1.1.3 ! root 1212: * 'active_threads[cpu_number()] = kmem_cache_alloc()',
! 1213: * cpu_number may be evaluated before kmem_cache_alloc;
! 1214: * if kmem_cache_alloc blocks, cpu_number will be wrong
1.1 root 1215: */
1216:
1.1.1.3 ! root 1217: state = (char *) kmem_cache_alloc(&vm_fault_state_cache);
1.1 root 1218: current_thread()->ith_other = state;
1219:
1220: }
1221:
1222: RetryFault: ;
1223:
1224: /*
1225: * Find the backing store object and offset into
1226: * it to begin the search.
1227: */
1228:
1229: if ((kr = vm_map_lookup(&map, vaddr, fault_type, &version,
1230: &object, &offset,
1231: &prot, &wired)) != KERN_SUCCESS) {
1232: goto done;
1233: }
1234:
1235: /*
1236: * If the page is wired, we must fault for the current protection
1237: * value, to avoid further faults.
1238: */
1239:
1240: if (wired)
1241: fault_type = prot;
1242:
1243: /*
1244: * Make a reference to this object to
1245: * prevent its disposal while we are messing with
1246: * it. Once we have the reference, the map is free
1247: * to be diddled. Since objects reference their
1248: * shadows (and copies), they will stay around as well.
1249: */
1250:
1251: assert(object->ref_count > 0);
1252: object->ref_count++;
1253: vm_object_paging_begin(object);
1254:
1255: if (continuation != (void (*)()) 0) {
1256: register vm_fault_state_t *state =
1257: (vm_fault_state_t *) current_thread()->ith_other;
1258:
1259: /*
1260: * Save variables, in case vm_fault_page discards
1261: * our kernel stack and we have to restart.
1262: */
1263:
1264: state->vmf_map = map;
1265: state->vmf_vaddr = vaddr;
1266: state->vmf_fault_type = fault_type;
1267: state->vmf_change_wiring = change_wiring;
1268: state->vmf_continuation = continuation;
1269:
1270: state->vmf_version = version;
1271: state->vmf_wired = wired;
1272: state->vmf_object = object;
1273: state->vmf_offset = offset;
1274: state->vmf_prot = prot;
1275:
1276: kr = vm_fault_page(object, offset, fault_type,
1277: (change_wiring && !wired), !change_wiring,
1278: &prot, &result_page, &top_page,
1279: FALSE, vm_fault_continue);
1280: } else
1281: {
1282: kr = vm_fault_page(object, offset, fault_type,
1283: (change_wiring && !wired), !change_wiring,
1284: &prot, &result_page, &top_page,
1285: FALSE, (void (*)()) 0);
1286: }
1287: after_vm_fault_page:
1288:
1289: /*
1290: * If we didn't succeed, lose the object reference immediately.
1291: */
1292:
1293: if (kr != VM_FAULT_SUCCESS)
1294: vm_object_deallocate(object);
1295:
1296: /*
1297: * See why we failed, and take corrective action.
1298: */
1299:
1300: switch (kr) {
1301: case VM_FAULT_SUCCESS:
1302: break;
1303: case VM_FAULT_RETRY:
1304: goto RetryFault;
1305: case VM_FAULT_INTERRUPTED:
1306: kr = KERN_SUCCESS;
1307: goto done;
1308: case VM_FAULT_MEMORY_SHORTAGE:
1309: if (continuation != (void (*)()) 0) {
1310: register vm_fault_state_t *state =
1311: (vm_fault_state_t *) current_thread()->ith_other;
1312:
1313: /*
1314: * Save variables in case VM_PAGE_WAIT
1315: * discards our kernel stack.
1316: */
1317:
1318: state->vmf_map = map;
1319: state->vmf_vaddr = vaddr;
1320: state->vmf_fault_type = fault_type;
1321: state->vmf_change_wiring = change_wiring;
1322: state->vmf_continuation = continuation;
1323: state->vmf_object = VM_OBJECT_NULL;
1324:
1325: VM_PAGE_WAIT(vm_fault_continue);
1326: } else
1327: VM_PAGE_WAIT((void (*)()) 0);
1328: goto RetryFault;
1329: case VM_FAULT_FICTITIOUS_SHORTAGE:
1330: vm_page_more_fictitious();
1331: goto RetryFault;
1332: case VM_FAULT_MEMORY_ERROR:
1333: kr = KERN_MEMORY_ERROR;
1334: goto done;
1335: }
1336:
1337: m = result_page;
1338:
1339: assert((change_wiring && !wired) ?
1340: (top_page == VM_PAGE_NULL) :
1341: ((top_page == VM_PAGE_NULL) == (m->object == object)));
1342:
1343: /*
1344: * How to clean up the result of vm_fault_page. This
1345: * happens whether the mapping is entered or not.
1346: */
1347:
1348: #define UNLOCK_AND_DEALLOCATE \
1349: MACRO_BEGIN \
1350: vm_fault_cleanup(m->object, top_page); \
1351: vm_object_deallocate(object); \
1352: MACRO_END
1353:
1354: /*
1355: * What to do with the resulting page from vm_fault_page
1356: * if it doesn't get entered into the physical map:
1357: */
1358:
1359: #define RELEASE_PAGE(m) \
1360: MACRO_BEGIN \
1361: PAGE_WAKEUP_DONE(m); \
1362: vm_page_lock_queues(); \
1363: if (!m->active && !m->inactive) \
1364: vm_page_activate(m); \
1365: vm_page_unlock_queues(); \
1366: MACRO_END
1367:
1368: /*
1369: * We must verify that the maps have not changed
1370: * since our last lookup.
1371: */
1372:
1373: old_copy_object = m->object->copy;
1374:
1375: vm_object_unlock(m->object);
1376: while (!vm_map_verify(map, &version)) {
1377: vm_object_t retry_object;
1378: vm_offset_t retry_offset;
1379: vm_prot_t retry_prot;
1380:
1381: /*
1382: * To avoid trying to write_lock the map while another
1383: * thread has it read_locked (in vm_map_pageable), we
1384: * do not try for write permission. If the page is
1385: * still writable, we will get write permission. If it
1386: * is not, or has been marked needs_copy, we enter the
1387: * mapping without write permission, and will merely
1388: * take another fault.
1389: */
1390: kr = vm_map_lookup(&map, vaddr,
1391: fault_type & ~VM_PROT_WRITE, &version,
1392: &retry_object, &retry_offset, &retry_prot,
1393: &wired);
1394:
1395: if (kr != KERN_SUCCESS) {
1396: vm_object_lock(m->object);
1397: RELEASE_PAGE(m);
1398: UNLOCK_AND_DEALLOCATE;
1399: goto done;
1400: }
1401:
1402: vm_object_unlock(retry_object);
1403: vm_object_lock(m->object);
1404:
1405: if ((retry_object != object) ||
1406: (retry_offset != offset)) {
1407: RELEASE_PAGE(m);
1408: UNLOCK_AND_DEALLOCATE;
1409: goto RetryFault;
1410: }
1411:
1412: /*
1413: * Check whether the protection has changed or the object
1414: * has been copied while we left the map unlocked.
1415: */
1416: prot &= retry_prot;
1417: vm_object_unlock(m->object);
1418: }
1419: vm_object_lock(m->object);
1420:
1421: /*
1422: * If the copy object changed while the top-level object
1423: * was unlocked, then we must take away write permission.
1424: */
1425:
1426: if (m->object->copy != old_copy_object)
1427: prot &= ~VM_PROT_WRITE;
1428:
1429: /*
1430: * If we want to wire down this page, but no longer have
1431: * adequate permissions, we must start all over.
1432: */
1433:
1434: if (wired && (prot != fault_type)) {
1435: vm_map_verify_done(map, &version);
1436: RELEASE_PAGE(m);
1437: UNLOCK_AND_DEALLOCATE;
1438: goto RetryFault;
1439: }
1440:
1441: /*
1442: * It's critically important that a wired-down page be faulted
1443: * only once in each map for which it is wired.
1444: */
1445:
1446: vm_object_unlock(m->object);
1447:
1448: /*
1449: * Put this page into the physical map.
1450: * We had to do the unlock above because pmap_enter
1451: * may cause other faults. The page may be on
1452: * the pageout queues. If the pageout daemon comes
1453: * across the page, it will remove it from the queues.
1454: */
1455:
1456: PMAP_ENTER(map->pmap, vaddr, m, prot, wired);
1457:
1458: /*
1459: * If the page is not wired down and isn't already
1460: * on a pageout queue, then put it where the
1461: * pageout daemon can find it.
1462: */
1463: vm_object_lock(m->object);
1464: vm_page_lock_queues();
1465: if (change_wiring) {
1466: if (wired)
1467: vm_page_wire(m);
1468: else
1469: vm_page_unwire(m);
1470: } else if (software_reference_bits) {
1471: if (!m->active && !m->inactive)
1472: vm_page_activate(m);
1473: m->reference = TRUE;
1474: } else {
1475: vm_page_activate(m);
1476: }
1477: vm_page_unlock_queues();
1478:
1479: /*
1480: * Unlock everything, and return
1481: */
1482:
1483: vm_map_verify_done(map, &version);
1484: PAGE_WAKEUP_DONE(m);
1485: kr = KERN_SUCCESS;
1486: UNLOCK_AND_DEALLOCATE;
1487:
1488: #undef UNLOCK_AND_DEALLOCATE
1489: #undef RELEASE_PAGE
1490:
1491: done:
1492: if (continuation != (void (*)()) 0) {
1493: register vm_fault_state_t *state =
1494: (vm_fault_state_t *) current_thread()->ith_other;
1495:
1.1.1.3 ! root 1496: kmem_cache_free(&vm_fault_state_cache, (vm_offset_t) state);
1.1 root 1497: (*continuation)(kr);
1498: /*NOTREACHED*/
1499: }
1500:
1501: return(kr);
1502: }
1503:
1504: kern_return_t vm_fault_wire_fast();
1505:
1506: /*
1507: * vm_fault_wire:
1508: *
1509: * Wire down a range of virtual addresses in a map.
1510: */
1511: void vm_fault_wire(map, entry)
1512: vm_map_t map;
1513: vm_map_entry_t entry;
1514: {
1515:
1516: register vm_offset_t va;
1517: register pmap_t pmap;
1518: register vm_offset_t end_addr = entry->vme_end;
1519:
1520: pmap = vm_map_pmap(map);
1521:
1522: /*
1523: * Inform the physical mapping system that the
1524: * range of addresses may not fault, so that
1525: * page tables and such can be locked down as well.
1526: */
1527:
1528: pmap_pageable(pmap, entry->vme_start, end_addr, FALSE);
1529:
1530: /*
1531: * We simulate a fault to get the page and enter it
1532: * in the physical map.
1533: */
1534:
1535: for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
1536: if (vm_fault_wire_fast(map, va, entry) != KERN_SUCCESS)
1537: (void) vm_fault(map, va, VM_PROT_NONE, TRUE,
1538: FALSE, (void (*)()) 0);
1539: }
1540: }
1541:
1542: /*
1543: * vm_fault_unwire:
1544: *
1545: * Unwire a range of virtual addresses in a map.
1546: */
1547: void vm_fault_unwire(map, entry)
1548: vm_map_t map;
1549: vm_map_entry_t entry;
1550: {
1551: register vm_offset_t va;
1552: register pmap_t pmap;
1553: register vm_offset_t end_addr = entry->vme_end;
1554: vm_object_t object;
1555:
1556: pmap = vm_map_pmap(map);
1557:
1558: object = (entry->is_sub_map)
1559: ? VM_OBJECT_NULL : entry->object.vm_object;
1560:
1561: /*
1562: * Since the pages are wired down, we must be able to
1563: * get their mappings from the physical map system.
1564: */
1565:
1566: for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
1567: pmap_change_wiring(pmap, va, FALSE);
1568:
1569: if (object == VM_OBJECT_NULL) {
1570: vm_map_lock_set_recursive(map);
1571: (void) vm_fault(map, va, VM_PROT_NONE, TRUE,
1572: FALSE, (void (*)()) 0);
1573: vm_map_lock_clear_recursive(map);
1574: } else {
1575: vm_prot_t prot;
1576: vm_page_t result_page;
1577: vm_page_t top_page;
1578: vm_fault_return_t result;
1579:
1580: do {
1581: prot = VM_PROT_NONE;
1582:
1583: vm_object_lock(object);
1584: vm_object_paging_begin(object);
1585: result = vm_fault_page(object,
1586: entry->offset +
1587: (va - entry->vme_start),
1588: VM_PROT_NONE, TRUE,
1589: FALSE, &prot,
1590: &result_page,
1591: &top_page,
1592: FALSE, (void (*)()) 0);
1593: } while (result == VM_FAULT_RETRY);
1594:
1595: if (result != VM_FAULT_SUCCESS)
1596: panic("vm_fault_unwire: failure");
1597:
1598: vm_page_lock_queues();
1599: vm_page_unwire(result_page);
1600: vm_page_unlock_queues();
1601: PAGE_WAKEUP_DONE(result_page);
1602:
1603: vm_fault_cleanup(result_page->object, top_page);
1604: }
1605: }
1606:
1607: /*
1608: * Inform the physical mapping system that the range
1609: * of addresses may fault, so that page tables and
1610: * such may be unwired themselves.
1611: */
1612:
1613: pmap_pageable(pmap, entry->vme_start, end_addr, TRUE);
1614: }
1615:
1616: /*
1617: * vm_fault_wire_fast:
1618: *
1619: * Handle common case of a wire down page fault at the given address.
1620: * If successful, the page is inserted into the associated physical map.
1621: * The map entry is passed in to avoid the overhead of a map lookup.
1622: *
1623: * NOTE: the given address should be truncated to the
1624: * proper page address.
1625: *
1626: * KERN_SUCCESS is returned if the page fault is handled; otherwise,
1627: * a standard error specifying why the fault is fatal is returned.
1628: *
1629: * The map in question must be referenced, and remains so.
1630: * Caller has a read lock on the map.
1631: *
1632: * This is a stripped version of vm_fault() for wiring pages. Anything
1633: * other than the common case will return KERN_FAILURE, and the caller
1634: * is expected to call vm_fault().
1635: */
1636: kern_return_t vm_fault_wire_fast(map, va, entry)
1637: vm_map_t map;
1638: vm_offset_t va;
1639: vm_map_entry_t entry;
1640: {
1641: vm_object_t object;
1642: vm_offset_t offset;
1643: register vm_page_t m;
1644: vm_prot_t prot;
1645:
1646: vm_stat.faults++; /* needs lock XXX */
1.1.1.3 ! root 1647: current_task()->faults++;
1.1 root 1648: /*
1649: * Recovery actions
1650: */
1651:
1652: #undef RELEASE_PAGE
1653: #define RELEASE_PAGE(m) { \
1654: PAGE_WAKEUP_DONE(m); \
1655: vm_page_lock_queues(); \
1656: vm_page_unwire(m); \
1657: vm_page_unlock_queues(); \
1658: }
1659:
1660:
1661: #undef UNLOCK_THINGS
1662: #define UNLOCK_THINGS { \
1663: object->paging_in_progress--; \
1664: vm_object_unlock(object); \
1665: }
1666:
1667: #undef UNLOCK_AND_DEALLOCATE
1668: #define UNLOCK_AND_DEALLOCATE { \
1669: UNLOCK_THINGS; \
1670: vm_object_deallocate(object); \
1671: }
1672: /*
1673: * Give up and have caller do things the hard way.
1674: */
1675:
1676: #define GIVE_UP { \
1677: UNLOCK_AND_DEALLOCATE; \
1678: return(KERN_FAILURE); \
1679: }
1680:
1681:
1682: /*
1683: * If this entry is not directly to a vm_object, bail out.
1684: */
1685: if (entry->is_sub_map)
1686: return(KERN_FAILURE);
1687:
1688: /*
1689: * Find the backing store object and offset into it.
1690: */
1691:
1692: object = entry->object.vm_object;
1693: offset = (va - entry->vme_start) + entry->offset;
1694: prot = entry->protection;
1695:
1696: /*
1697: * Make a reference to this object to prevent its
1698: * disposal while we are messing with it.
1699: */
1700:
1701: vm_object_lock(object);
1702: assert(object->ref_count > 0);
1703: object->ref_count++;
1704: object->paging_in_progress++;
1705:
1706: /*
1707: * INVARIANTS (through entire routine):
1708: *
1709: * 1) At all times, we must either have the object
1710: * lock or a busy page in some object to prevent
1711: * some other thread from trying to bring in
1712: * the same page.
1713: *
1714: * 2) Once we have a busy page, we must remove it from
1715: * the pageout queues, so that the pageout daemon
1716: * will not grab it away.
1717: *
1718: */
1719:
1720: /*
1721: * Look for page in top-level object. If it's not there or
1722: * there's something going on, give up.
1723: */
1724: m = vm_page_lookup(object, offset);
1725: if ((m == VM_PAGE_NULL) || (m->error) ||
1726: (m->busy) || (m->absent) || (prot & m->page_lock)) {
1727: GIVE_UP;
1728: }
1729:
1730: /*
1731: * Wire the page down now. All bail outs beyond this
1.1.1.2 root 1732: * point must unwire the page.
1.1 root 1733: */
1734:
1735: vm_page_lock_queues();
1736: vm_page_wire(m);
1737: vm_page_unlock_queues();
1738:
1739: /*
1740: * Mark page busy for other threads.
1741: */
1742: assert(!m->busy);
1743: m->busy = TRUE;
1744: assert(!m->absent);
1745:
1746: /*
1747: * Give up if the page is being written and there's a copy object
1748: */
1749: if ((object->copy != VM_OBJECT_NULL) && (prot & VM_PROT_WRITE)) {
1750: RELEASE_PAGE(m);
1751: GIVE_UP;
1752: }
1753:
1754: /*
1755: * Put this page into the physical map.
1756: * We have to unlock the object because pmap_enter
1.1.1.2 root 1757: * may cause other faults.
1.1 root 1758: */
1759: vm_object_unlock(object);
1760:
1761: PMAP_ENTER(map->pmap, va, m, prot, TRUE);
1762:
1763: /*
1764: * Must relock object so that paging_in_progress can be cleared.
1765: */
1766: vm_object_lock(object);
1767:
1768: /*
1769: * Unlock everything, and return
1770: */
1771:
1772: PAGE_WAKEUP_DONE(m);
1773: UNLOCK_AND_DEALLOCATE;
1774:
1775: return(KERN_SUCCESS);
1776:
1777: }
1778:
1779: /*
1780: * Routine: vm_fault_copy_cleanup
1781: * Purpose:
1782: * Release a page used by vm_fault_copy.
1783: */
1784:
1785: void vm_fault_copy_cleanup(page, top_page)
1786: vm_page_t page;
1787: vm_page_t top_page;
1788: {
1789: vm_object_t object = page->object;
1790:
1791: vm_object_lock(object);
1792: PAGE_WAKEUP_DONE(page);
1793: vm_page_lock_queues();
1794: if (!page->active && !page->inactive)
1795: vm_page_activate(page);
1796: vm_page_unlock_queues();
1797: vm_fault_cleanup(object, top_page);
1798: }
1799:
1800: /*
1801: * Routine: vm_fault_copy
1802: *
1803: * Purpose:
1804: * Copy pages from one virtual memory object to another --
1805: * neither the source nor destination pages need be resident.
1806: *
1807: * Before actually copying a page, the version associated with
1808: * the destination address map wil be verified.
1809: *
1810: * In/out conditions:
1811: * The caller must hold a reference, but not a lock, to
1812: * each of the source and destination objects and to the
1813: * destination map.
1814: *
1815: * Results:
1816: * Returns KERN_SUCCESS if no errors were encountered in
1817: * reading or writing the data. Returns KERN_INTERRUPTED if
1818: * the operation was interrupted (only possible if the
1819: * "interruptible" argument is asserted). Other return values
1820: * indicate a permanent error in copying the data.
1821: *
1822: * The actual amount of data copied will be returned in the
1823: * "copy_size" argument. In the event that the destination map
1824: * verification failed, this amount may be less than the amount
1825: * requested.
1826: */
1827: kern_return_t vm_fault_copy(
1828: src_object,
1829: src_offset,
1830: src_size,
1831: dst_object,
1832: dst_offset,
1833: dst_map,
1834: dst_version,
1835: interruptible
1836: )
1837: vm_object_t src_object;
1838: vm_offset_t src_offset;
1839: vm_size_t *src_size; /* INOUT */
1840: vm_object_t dst_object;
1841: vm_offset_t dst_offset;
1842: vm_map_t dst_map;
1843: vm_map_version_t *dst_version;
1844: boolean_t interruptible;
1845: {
1846: vm_page_t result_page;
1847: vm_prot_t prot;
1.1.1.2 root 1848:
1.1 root 1849: vm_page_t src_page;
1850: vm_page_t src_top_page;
1851:
1852: vm_page_t dst_page;
1853: vm_page_t dst_top_page;
1854:
1855: vm_size_t amount_done;
1856: vm_object_t old_copy_object;
1857:
1858: #define RETURN(x) \
1859: MACRO_BEGIN \
1860: *src_size = amount_done; \
1861: MACRO_RETURN(x); \
1862: MACRO_END
1863:
1864: amount_done = 0;
1865: do { /* while (amount_done != *src_size) */
1866:
1867: RetrySourceFault: ;
1868:
1869: if (src_object == VM_OBJECT_NULL) {
1870: /*
1871: * No source object. We will just
1872: * zero-fill the page in dst_object.
1873: */
1874:
1875: src_page = VM_PAGE_NULL;
1876: } else {
1877: prot = VM_PROT_READ;
1878:
1879: vm_object_lock(src_object);
1880: vm_object_paging_begin(src_object);
1881:
1882: switch (vm_fault_page(src_object, src_offset,
1883: VM_PROT_READ, FALSE, interruptible,
1884: &prot, &result_page, &src_top_page,
1885: FALSE, (void (*)()) 0)) {
1886:
1887: case VM_FAULT_SUCCESS:
1888: break;
1889: case VM_FAULT_RETRY:
1890: goto RetrySourceFault;
1891: case VM_FAULT_INTERRUPTED:
1892: RETURN(MACH_SEND_INTERRUPTED);
1893: case VM_FAULT_MEMORY_SHORTAGE:
1894: VM_PAGE_WAIT((void (*)()) 0);
1895: goto RetrySourceFault;
1896: case VM_FAULT_FICTITIOUS_SHORTAGE:
1897: vm_page_more_fictitious();
1898: goto RetrySourceFault;
1899: case VM_FAULT_MEMORY_ERROR:
1900: return(KERN_MEMORY_ERROR);
1901: }
1902:
1903: src_page = result_page;
1904:
1905: assert((src_top_page == VM_PAGE_NULL) ==
1906: (src_page->object == src_object));
1907:
1908: assert ((prot & VM_PROT_READ) != VM_PROT_NONE);
1909:
1910: vm_object_unlock(src_page->object);
1911: }
1912:
1913: RetryDestinationFault: ;
1914:
1915: prot = VM_PROT_WRITE;
1916:
1917: vm_object_lock(dst_object);
1918: vm_object_paging_begin(dst_object);
1919:
1920: switch (vm_fault_page(dst_object, dst_offset, VM_PROT_WRITE,
1921: FALSE, FALSE /* interruptible */,
1922: &prot, &result_page, &dst_top_page,
1923: FALSE, (void (*)()) 0)) {
1924:
1925: case VM_FAULT_SUCCESS:
1926: break;
1927: case VM_FAULT_RETRY:
1928: goto RetryDestinationFault;
1929: case VM_FAULT_INTERRUPTED:
1930: if (src_page != VM_PAGE_NULL)
1931: vm_fault_copy_cleanup(src_page,
1932: src_top_page);
1933: RETURN(MACH_SEND_INTERRUPTED);
1934: case VM_FAULT_MEMORY_SHORTAGE:
1935: VM_PAGE_WAIT((void (*)()) 0);
1936: goto RetryDestinationFault;
1937: case VM_FAULT_FICTITIOUS_SHORTAGE:
1938: vm_page_more_fictitious();
1939: goto RetryDestinationFault;
1940: case VM_FAULT_MEMORY_ERROR:
1941: if (src_page != VM_PAGE_NULL)
1942: vm_fault_copy_cleanup(src_page,
1943: src_top_page);
1944: return(KERN_MEMORY_ERROR);
1945: }
1946: assert ((prot & VM_PROT_WRITE) != VM_PROT_NONE);
1947:
1948: dst_page = result_page;
1949:
1950: old_copy_object = dst_page->object->copy;
1951:
1952: vm_object_unlock(dst_page->object);
1953:
1954: if (!vm_map_verify(dst_map, dst_version)) {
1955:
1956: BailOut: ;
1957:
1958: if (src_page != VM_PAGE_NULL)
1959: vm_fault_copy_cleanup(src_page, src_top_page);
1960: vm_fault_copy_cleanup(dst_page, dst_top_page);
1961: break;
1962: }
1963:
1964:
1965: vm_object_lock(dst_page->object);
1966: if (dst_page->object->copy != old_copy_object) {
1967: vm_object_unlock(dst_page->object);
1968: vm_map_verify_done(dst_map, dst_version);
1969: goto BailOut;
1970: }
1971: vm_object_unlock(dst_page->object);
1972:
1973: /*
1974: * Copy the page, and note that it is dirty
1975: * immediately.
1976: */
1977:
1978: if (src_page == VM_PAGE_NULL)
1979: vm_page_zero_fill(dst_page);
1980: else
1981: vm_page_copy(src_page, dst_page);
1982: dst_page->dirty = TRUE;
1983:
1984: /*
1985: * Unlock everything, and return
1986: */
1987:
1988: vm_map_verify_done(dst_map, dst_version);
1989:
1990: if (src_page != VM_PAGE_NULL)
1991: vm_fault_copy_cleanup(src_page, src_top_page);
1992: vm_fault_copy_cleanup(dst_page, dst_top_page);
1993:
1994: amount_done += PAGE_SIZE;
1995: src_offset += PAGE_SIZE;
1996: dst_offset += PAGE_SIZE;
1997:
1998: } while (amount_done != *src_size);
1999:
2000: RETURN(KERN_SUCCESS);
2001: #undef RETURN
2002:
1.1.1.2 root 2003: /*NOTREACHED*/
1.1 root 2004: }
2005:
2006:
2007:
2008:
2009:
2010: #ifdef notdef
2011:
2012: /*
2013: * Routine: vm_fault_page_overwrite
2014: *
2015: * Description:
2016: * A form of vm_fault_page that assumes that the
2017: * resulting page will be overwritten in its entirety,
2018: * making it unnecessary to obtain the correct *contents*
2019: * of the page.
2020: *
2021: * Implementation:
2022: * XXX Untested. Also unused. Eventually, this technology
2023: * could be used in vm_fault_copy() to advantage.
2024: */
2025: vm_fault_return_t vm_fault_page_overwrite(dst_object, dst_offset, result_page)
2026: register
2027: vm_object_t dst_object;
2028: vm_offset_t dst_offset;
2029: vm_page_t *result_page; /* OUT */
2030: {
2031: register
2032: vm_page_t dst_page;
2033:
2034: #define interruptible FALSE /* XXX */
2035:
2036: while (TRUE) {
2037: /*
2038: * Look for a page at this offset
2039: */
2040:
2041: while ((dst_page = vm_page_lookup(dst_object, dst_offset))
2042: == VM_PAGE_NULL) {
2043: /*
2044: * No page, no problem... just allocate one.
2045: */
2046:
2047: dst_page = vm_page_alloc(dst_object, dst_offset);
2048: if (dst_page == VM_PAGE_NULL) {
2049: vm_object_unlock(dst_object);
2050: VM_PAGE_WAIT((void (*)()) 0);
2051: vm_object_lock(dst_object);
2052: continue;
2053: }
2054:
2055: /*
2056: * Pretend that the memory manager
2057: * write-protected the page.
2058: *
2059: * Note that we will be asking for write
2060: * permission without asking for the data
2061: * first.
2062: */
2063:
2064: dst_page->overwriting = TRUE;
2065: dst_page->page_lock = VM_PROT_WRITE;
2066: dst_page->absent = TRUE;
2067: dst_object->absent_count++;
2068:
2069: break;
2070:
2071: /*
2072: * When we bail out, we might have to throw
2073: * away the page created here.
2074: */
2075:
2076: #define DISCARD_PAGE \
2077: MACRO_BEGIN \
2078: vm_object_lock(dst_object); \
2079: dst_page = vm_page_lookup(dst_object, dst_offset); \
2080: if ((dst_page != VM_PAGE_NULL) && dst_page->overwriting) \
2081: VM_PAGE_FREE(dst_page); \
2082: vm_object_unlock(dst_object); \
2083: MACRO_END
2084: }
2085:
2086: /*
2087: * If the page is write-protected...
2088: */
2089:
2090: if (dst_page->page_lock & VM_PROT_WRITE) {
2091: /*
2092: * ... and an unlock request hasn't been sent
2093: */
2094:
2095: if ( ! (dst_page->unlock_request & VM_PROT_WRITE)) {
2096: vm_prot_t u;
2097: kern_return_t rc;
2098:
2099: /*
2100: * ... then send one now.
2101: */
2102:
2103: if (!dst_object->pager_ready) {
2104: vm_object_assert_wait(dst_object,
2105: VM_OBJECT_EVENT_PAGER_READY,
2106: interruptible);
2107: vm_object_unlock(dst_object);
2108: thread_block((void (*)()) 0);
2109: if (current_thread()->wait_result !=
2110: THREAD_AWAKENED) {
2111: DISCARD_PAGE;
2112: return(VM_FAULT_INTERRUPTED);
2113: }
2114: continue;
2115: }
2116:
2117: u = dst_page->unlock_request |= VM_PROT_WRITE;
2118: vm_object_unlock(dst_object);
2119:
2120: if ((rc = memory_object_data_unlock(
2121: dst_object->pager,
2122: dst_object->pager_request,
2123: dst_offset + dst_object->paging_offset,
2124: PAGE_SIZE,
2125: u)) != KERN_SUCCESS) {
2126: printf("vm_object_overwrite: memory_object_data_unlock failed\n");
2127: DISCARD_PAGE;
2128: return((rc == MACH_SEND_INTERRUPTED) ?
2129: VM_FAULT_INTERRUPTED :
2130: VM_FAULT_MEMORY_ERROR);
2131: }
2132: vm_object_lock(dst_object);
2133: continue;
2134: }
2135:
2136: /* ... fall through to wait below */
2137: } else {
2138: /*
2139: * If the page isn't being used for other
2140: * purposes, then we're done.
2141: */
2142: if ( ! (dst_page->busy || dst_page->absent || dst_page->error) )
2143: break;
2144: }
2145:
2146: PAGE_ASSERT_WAIT(dst_page, interruptible);
2147: vm_object_unlock(dst_object);
2148: thread_block((void (*)()) 0);
2149: if (current_thread()->wait_result != THREAD_AWAKENED) {
2150: DISCARD_PAGE;
2151: return(VM_FAULT_INTERRUPTED);
2152: }
2153: }
2154:
2155: *result_page = dst_page;
2156: return(VM_FAULT_SUCCESS);
2157:
2158: #undef interruptible
2159: #undef DISCARD_PAGE
2160: }
2161:
1.1.1.2 root 2162: #endif /* notdef */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.