![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_fault.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_fault.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1994,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm_fault.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: *
33: * Page fault handling module.
34: */
35:
1.1.1.3 root 36: #include <kern/printf.h>
1.1 root 37: #include <vm/vm_fault.h>
38: #include <mach/kern_return.h>
39: #include <mach/message.h> /* for error codes */
40: #include <kern/counters.h>
1.1.1.3 root 41: #include <kern/debug.h>
1.1 root 42: #include <kern/thread.h>
43: #include <kern/sched_prim.h>
44: #include <vm/vm_map.h>
45: #include <vm/vm_object.h>
46: #include <vm/vm_page.h>
47: #include <vm/pmap.h>
48: #include <mach/vm_statistics.h>
49: #include <vm/vm_pageout.h>
50: #include <mach/vm_param.h>
51: #include <mach/memory_object.h>
1.1.1.3 root 52: #include <vm/memory_object_user.user.h>
1.1 root 53: /* For memory_object_data_{request,unlock} */
54: #include <kern/macro_help.h>
1.1.1.3 root 55: #include <kern/slab.h>
1.1 root 56:
57: #if MACH_PCSAMPLE
58: #include <kern/pc_sample.h>
59: #endif
60:
61:
62:
63: /*
64: * State needed by vm_fault_continue.
65: * This is a little hefty to drop directly
66: * into the thread structure.
67: */
68: typedef struct vm_fault_state {
69: struct vm_map *vmf_map;
70: vm_offset_t vmf_vaddr;
71: vm_prot_t vmf_fault_type;
72: boolean_t vmf_change_wiring;
73: void (*vmf_continuation)();
74: vm_map_version_t vmf_version;
75: boolean_t vmf_wired;
76: struct vm_object *vmf_object;
77: vm_offset_t vmf_offset;
78: vm_prot_t vmf_prot;
79:
80: boolean_t vmfp_backoff;
81: struct vm_object *vmfp_object;
82: vm_offset_t vmfp_offset;
83: struct vm_page *vmfp_first_m;
84: vm_prot_t vmfp_access;
85: } vm_fault_state_t;
86:
1.1.1.3 root 87: struct kmem_cache vm_fault_state_cache;
1.1 root 88:
89: int vm_object_absent_max = 50;
90:
91: boolean_t vm_fault_dirty_handling = FALSE;
92: boolean_t vm_fault_interruptible = TRUE;
93:
94: boolean_t software_reference_bits = TRUE;
95:
96: #if MACH_KDB
97: extern struct db_watchpoint *db_watchpoint_list;
1.1.1.2 root 98: #endif /* MACH_KDB */
1.1 root 99:
100: /*
101: * Routine: vm_fault_init
102: * Purpose:
103: * Initialize our private data structures.
104: */
1.1.1.3 root 105: void vm_fault_init(void)
1.1 root 106: {
1.1.1.3 root 107: kmem_cache_init(&vm_fault_state_cache, "vm_fault_state",
108: sizeof(vm_fault_state_t), 0, NULL, NULL, NULL, 0);
1.1 root 109: }
110:
111: /*
112: * Routine: vm_fault_cleanup
113: * Purpose:
114: * Clean up the result of vm_fault_page.
115: * Results:
116: * The paging reference for "object" is released.
117: * "object" is unlocked.
118: * If "top_page" is not null, "top_page" is
119: * freed and the paging reference for the object
120: * containing it is released.
121: *
122: * In/out conditions:
123: * "object" must be locked.
124: */
125: void
1.1.1.4 ! root 126: vm_fault_cleanup(
! 127: vm_object_t object,
! 128: vm_page_t top_page)
1.1 root 129: {
130: vm_object_paging_end(object);
131: vm_object_unlock(object);
132:
133: if (top_page != VM_PAGE_NULL) {
134: object = top_page->object;
135: vm_object_lock(object);
136: VM_PAGE_FREE(top_page);
137: vm_object_paging_end(object);
138: vm_object_unlock(object);
139: }
140: }
141:
142:
143: #if MACH_PCSAMPLE
144: /*
145: * Do PC sampling on current thread, assuming
146: * that it is the thread taking this page fault.
147: *
148: * Must check for THREAD_NULL, since faults
149: * can occur before threads are running.
150: */
151:
152: #define vm_stat_sample(flavor) \
153: MACRO_BEGIN \
154: thread_t _thread_ = current_thread(); \
155: \
156: if (_thread_ != THREAD_NULL) \
157: take_pc_sample_macro(_thread_, (flavor)); \
158: MACRO_END
159:
160: #else
161: #define vm_stat_sample(x)
162: #endif /* MACH_PCSAMPLE */
163:
164:
165:
166: /*
167: * Routine: vm_fault_page
168: * Purpose:
169: * Find the resident page for the virtual memory
170: * specified by the given virtual memory object
171: * and offset.
172: * Additional arguments:
173: * The required permissions for the page is given
174: * in "fault_type". Desired permissions are included
175: * in "protection".
176: *
177: * If the desired page is known to be resident (for
178: * example, because it was previously wired down), asserting
179: * the "unwiring" parameter will speed the search.
180: *
181: * If the operation can be interrupted (by thread_abort
182: * or thread_terminate), then the "interruptible"
183: * parameter should be asserted.
184: *
185: * Results:
186: * The page containing the proper data is returned
187: * in "result_page".
188: *
189: * In/out conditions:
190: * The source object must be locked and referenced,
191: * and must donate one paging reference. The reference
192: * is not affected. The paging reference and lock are
193: * consumed.
194: *
195: * If the call succeeds, the object in which "result_page"
196: * resides is left locked and holding a paging reference.
197: * If this is not the original object, a busy page in the
198: * original object is returned in "top_page", to prevent other
199: * callers from pursuing this same data, along with a paging
200: * reference for the original object. The "top_page" should
201: * be destroyed when this guarantee is no longer required.
202: * The "result_page" is also left busy. It is not removed
203: * from the pageout queues.
204: */
1.1.1.4 ! root 205: vm_fault_return_t vm_fault_page(
1.1 root 206: /* Arguments: */
1.1.1.4 ! root 207: vm_object_t first_object, /* Object to begin search */
! 208: vm_offset_t first_offset, /* Offset into object */
! 209: vm_prot_t fault_type, /* What access is requested */
! 210: boolean_t must_be_resident,/* Must page be resident? */
! 211: boolean_t interruptible, /* May fault be interrupted? */
1.1 root 212: /* Modifies in place: */
1.1.1.4 ! root 213: vm_prot_t *protection, /* Protection for mapping */
1.1 root 214: /* Returns: */
1.1.1.4 ! root 215: vm_page_t *result_page, /* Page found, if successful */
! 216: vm_page_t *top_page, /* Page in top object, if
1.1 root 217: * not result_page.
218: */
219: /* More arguments: */
1.1.1.4 ! root 220: boolean_t resume, /* We are restarting. */
! 221: void (*continuation)()) /* Continuation for blocking. */
1.1 root 222: {
223: vm_page_t m;
224: vm_object_t object;
225: vm_offset_t offset;
226: vm_page_t first_m;
227: vm_object_t next_object;
228: vm_object_t copy_object;
229: boolean_t look_for_page;
230: vm_prot_t access_required;
231:
232: if (resume) {
1.1.1.4 ! root 233: vm_fault_state_t *state =
1.1 root 234: (vm_fault_state_t *) current_thread()->ith_other;
235:
236: if (state->vmfp_backoff)
237: goto after_block_and_backoff;
238:
239: object = state->vmfp_object;
240: offset = state->vmfp_offset;
241: first_m = state->vmfp_first_m;
242: access_required = state->vmfp_access;
243: goto after_thread_block;
244: }
245:
246: vm_stat_sample(SAMPLED_PC_VM_FAULTS_ANY);
247: vm_stat.faults++; /* needs lock XXX */
1.1.1.3 root 248: current_task()->faults++;
1.1 root 249:
250: /*
251: * Recovery actions
252: */
253: #define RELEASE_PAGE(m) \
254: MACRO_BEGIN \
255: PAGE_WAKEUP_DONE(m); \
256: vm_page_lock_queues(); \
257: if (!m->active && !m->inactive) \
258: vm_page_activate(m); \
259: vm_page_unlock_queues(); \
260: MACRO_END
261:
262: if (vm_fault_dirty_handling
263: #if MACH_KDB
264: /*
265: * If there are watchpoints set, then
266: * we don't want to give away write permission
267: * on a read fault. Make the task write fault,
268: * so that the watchpoint code notices the access.
269: */
270: || db_watchpoint_list
1.1.1.2 root 271: #endif /* MACH_KDB */
1.1 root 272: ) {
273: /*
274: * If we aren't asking for write permission,
275: * then don't give it away. We're using write
276: * faults to set the dirty bit.
277: */
278: if (!(fault_type & VM_PROT_WRITE))
279: *protection &= ~VM_PROT_WRITE;
280: }
281:
282: if (!vm_fault_interruptible)
283: interruptible = FALSE;
284:
285: /*
286: * INVARIANTS (through entire routine):
287: *
288: * 1) At all times, we must either have the object
289: * lock or a busy page in some object to prevent
290: * some other thread from trying to bring in
291: * the same page.
292: *
293: * Note that we cannot hold any locks during the
294: * pager access or when waiting for memory, so
295: * we use a busy page then.
296: *
297: * Note also that we aren't as concerned about more than
298: * one thread attempting to memory_object_data_unlock
299: * the same page at once, so we don't hold the page
300: * as busy then, but do record the highest unlock
301: * value so far. [Unlock requests may also be delivered
302: * out of order.]
303: *
304: * 2) To prevent another thread from racing us down the
305: * shadow chain and entering a new page in the top
306: * object before we do, we must keep a busy page in
307: * the top object while following the shadow chain.
308: *
309: * 3) We must increment paging_in_progress on any object
310: * for which we have a busy page, to prevent
311: * vm_object_collapse from removing the busy page
312: * without our noticing.
313: *
314: * 4) We leave busy pages on the pageout queues.
315: * If the pageout daemon comes across a busy page,
316: * it will remove the page from the pageout queues.
317: */
318:
319: /*
320: * Search for the page at object/offset.
321: */
322:
323: object = first_object;
324: offset = first_offset;
325: first_m = VM_PAGE_NULL;
326: access_required = fault_type;
327:
328: /*
329: * See whether this page is resident
330: */
331:
332: while (TRUE) {
333: m = vm_page_lookup(object, offset);
334: if (m != VM_PAGE_NULL) {
335: /*
336: * If the page is being brought in,
337: * wait for it and then retry.
338: *
339: * A possible optimization: if the page
340: * is known to be resident, we can ignore
341: * pages that are absent (regardless of
342: * whether they're busy).
343: */
344:
345: if (m->busy) {
346: kern_return_t wait_result;
347:
348: PAGE_ASSERT_WAIT(m, interruptible);
349: vm_object_unlock(object);
350: if (continuation != (void (*)()) 0) {
1.1.1.4 ! root 351: vm_fault_state_t *state =
1.1 root 352: (vm_fault_state_t *) current_thread()->ith_other;
353:
354: /*
355: * Save variables in case
356: * thread_block discards
357: * our kernel stack.
358: */
359:
360: state->vmfp_backoff = FALSE;
361: state->vmfp_object = object;
362: state->vmfp_offset = offset;
363: state->vmfp_first_m = first_m;
364: state->vmfp_access =
365: access_required;
366: state->vmf_prot = *protection;
367:
368: counter(c_vm_fault_page_block_busy_user++);
369: thread_block(continuation);
370: } else
371: {
372: counter(c_vm_fault_page_block_busy_kernel++);
373: thread_block((void (*)()) 0);
374: }
375: after_thread_block:
376: wait_result = current_thread()->wait_result;
377: vm_object_lock(object);
378: if (wait_result != THREAD_AWAKENED) {
379: vm_fault_cleanup(object, first_m);
380: if (wait_result == THREAD_RESTART)
381: return(VM_FAULT_RETRY);
382: else
383: return(VM_FAULT_INTERRUPTED);
384: }
385: continue;
386: }
387:
388: /*
389: * If the page is in error, give up now.
390: */
391:
392: if (m->error) {
393: VM_PAGE_FREE(m);
394: vm_fault_cleanup(object, first_m);
395: return(VM_FAULT_MEMORY_ERROR);
396: }
397:
398: /*
399: * If the page isn't busy, but is absent,
400: * then it was deemed "unavailable".
401: */
402:
403: if (m->absent) {
404: /*
405: * Remove the non-existent page (unless it's
406: * in the top object) and move on down to the
407: * next object (if there is one).
408: */
409:
410: offset += object->shadow_offset;
411: access_required = VM_PROT_READ;
412: next_object = object->shadow;
413: if (next_object == VM_OBJECT_NULL) {
414: vm_page_t real_m;
415:
416: assert(!must_be_resident);
417:
418: /*
419: * Absent page at bottom of shadow
420: * chain; zero fill the page we left
421: * busy in the first object, and flush
422: * the absent page. But first we
423: * need to allocate a real page.
424: */
425:
1.1.1.2 root 426: real_m = vm_page_grab(!object->internal);
1.1 root 427: if (real_m == VM_PAGE_NULL) {
428: vm_fault_cleanup(object, first_m);
429: return(VM_FAULT_MEMORY_SHORTAGE);
430: }
431:
432: if (object != first_object) {
433: VM_PAGE_FREE(m);
434: vm_object_paging_end(object);
435: vm_object_unlock(object);
436: object = first_object;
437: offset = first_offset;
438: m = first_m;
439: first_m = VM_PAGE_NULL;
440: vm_object_lock(object);
441: }
442:
443: VM_PAGE_FREE(m);
444: assert(real_m->busy);
445: vm_page_lock_queues();
446: vm_page_insert(real_m, object, offset);
447: vm_page_unlock_queues();
448: m = real_m;
449:
450: /*
451: * Drop the lock while zero filling
452: * page. Then break because this
453: * is the page we wanted. Checking
454: * the page lock is a waste of time;
455: * this page was either absent or
456: * newly allocated -- in both cases
457: * it can't be page locked by a pager.
458: */
459: vm_object_unlock(object);
460:
461: vm_page_zero_fill(m);
462:
463: vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS);
1.1.1.2 root 464:
1.1 root 465: vm_stat.zero_fill_count++;
1.1.1.3 root 466: current_task()->zero_fills++;
1.1 root 467: vm_object_lock(object);
468: pmap_clear_modify(m->phys_addr);
469: break;
470: } else {
471: if (must_be_resident) {
472: vm_object_paging_end(object);
473: } else if (object != first_object) {
474: vm_object_paging_end(object);
475: VM_PAGE_FREE(m);
476: } else {
477: first_m = m;
478: m->absent = FALSE;
479: vm_object_absent_release(object);
480: m->busy = TRUE;
481:
482: vm_page_lock_queues();
483: VM_PAGE_QUEUES_REMOVE(m);
484: vm_page_unlock_queues();
485: }
486: vm_object_lock(next_object);
487: vm_object_unlock(object);
488: object = next_object;
489: vm_object_paging_begin(object);
490: continue;
491: }
492: }
493:
494: /*
495: * If the desired access to this page has
496: * been locked out, request that it be unlocked.
497: */
498:
499: if (access_required & m->page_lock) {
500: if ((access_required & m->unlock_request) != access_required) {
501: vm_prot_t new_unlock_request;
502: kern_return_t rc;
1.1.1.2 root 503:
1.1 root 504: if (!object->pager_ready) {
505: vm_object_assert_wait(object,
506: VM_OBJECT_EVENT_PAGER_READY,
507: interruptible);
508: goto block_and_backoff;
509: }
510:
511: new_unlock_request = m->unlock_request =
512: (access_required | m->unlock_request);
513: vm_object_unlock(object);
514: if ((rc = memory_object_data_unlock(
515: object->pager,
516: object->pager_request,
517: offset + object->paging_offset,
518: PAGE_SIZE,
519: new_unlock_request))
520: != KERN_SUCCESS) {
521: printf("vm_fault: memory_object_data_unlock failed\n");
522: vm_object_lock(object);
523: vm_fault_cleanup(object, first_m);
524: return((rc == MACH_SEND_INTERRUPTED) ?
525: VM_FAULT_INTERRUPTED :
526: VM_FAULT_MEMORY_ERROR);
527: }
528: vm_object_lock(object);
529: continue;
530: }
531:
532: PAGE_ASSERT_WAIT(m, interruptible);
533: goto block_and_backoff;
534: }
535:
536: /*
537: * We mark the page busy and leave it on
538: * the pageout queues. If the pageout
539: * deamon comes across it, then it will
540: * remove the page.
541: */
542:
543: if (!software_reference_bits) {
544: vm_page_lock_queues();
545: if (m->inactive) {
546: vm_stat_sample(SAMPLED_PC_VM_REACTIVATION_FAULTS);
547: vm_stat.reactivations++;
1.1.1.3 root 548: current_task()->reactivations++;
1.1 root 549: }
550:
551: VM_PAGE_QUEUES_REMOVE(m);
552: vm_page_unlock_queues();
553: }
554:
555: assert(!m->busy);
556: m->busy = TRUE;
557: assert(!m->absent);
558: break;
559: }
560:
561: look_for_page =
562: (object->pager_created)
563: #if MACH_PAGEMAP
564: && (vm_external_state_get(object->existence_info, offset + object->paging_offset) !=
565: VM_EXTERNAL_STATE_ABSENT)
1.1.1.2 root 566: #endif /* MACH_PAGEMAP */
1.1 root 567: ;
568:
569: if ((look_for_page || (object == first_object))
570: && !must_be_resident) {
571: /*
572: * Allocate a new page for this object/offset
573: * pair.
574: */
575:
576: m = vm_page_grab_fictitious();
577: if (m == VM_PAGE_NULL) {
578: vm_fault_cleanup(object, first_m);
579: return(VM_FAULT_FICTITIOUS_SHORTAGE);
580: }
581:
582: vm_page_lock_queues();
583: vm_page_insert(m, object, offset);
584: vm_page_unlock_queues();
585: }
586:
587: if (look_for_page && !must_be_resident) {
588: kern_return_t rc;
589:
590: /*
591: * If the memory manager is not ready, we
592: * cannot make requests.
593: */
594: if (!object->pager_ready) {
595: vm_object_assert_wait(object,
596: VM_OBJECT_EVENT_PAGER_READY,
597: interruptible);
598: VM_PAGE_FREE(m);
599: goto block_and_backoff;
600: }
601:
602: if (object->internal) {
603: /*
604: * Requests to the default pager
605: * must reserve a real page in advance,
606: * because the pager's data-provided
607: * won't block for pages.
608: */
609:
1.1.1.2 root 610: if (m->fictitious && !vm_page_convert(m, FALSE)) {
1.1 root 611: VM_PAGE_FREE(m);
612: vm_fault_cleanup(object, first_m);
613: return(VM_FAULT_MEMORY_SHORTAGE);
614: }
615: } else if (object->absent_count >
616: vm_object_absent_max) {
617: /*
618: * If there are too many outstanding page
619: * requests pending on this object, we
620: * wait for them to be resolved now.
621: */
622:
623: vm_object_absent_assert_wait(object, interruptible);
624: VM_PAGE_FREE(m);
625: goto block_and_backoff;
626: }
627:
628: /*
629: * Indicate that the page is waiting for data
630: * from the memory manager.
631: */
632:
633: m->absent = TRUE;
634: object->absent_count++;
635:
636: /*
637: * We have a busy page, so we can
638: * release the object lock.
639: */
640: vm_object_unlock(object);
641:
642: /*
643: * Call the memory manager to retrieve the data.
644: */
645:
646: vm_stat.pageins++;
647: vm_stat_sample(SAMPLED_PC_VM_PAGEIN_FAULTS);
1.1.1.3 root 648: current_task()->pageins++;
1.1 root 649:
1.1.1.2 root 650: if ((rc = memory_object_data_request(object->pager,
1.1 root 651: object->pager_request,
1.1.1.2 root 652: m->offset + object->paging_offset,
1.1 root 653: PAGE_SIZE, access_required)) != KERN_SUCCESS) {
654: if (rc != MACH_SEND_INTERRUPTED)
1.1.1.3 root 655: printf("%s(0x%p, 0x%p, 0x%lx, 0x%x, 0x%x) failed, %x\n",
1.1 root 656: "memory_object_data_request",
657: object->pager,
658: object->pager_request,
1.1.1.2 root 659: m->offset + object->paging_offset,
1.1 root 660: PAGE_SIZE, access_required, rc);
661: /*
662: * Don't want to leave a busy page around,
663: * but the data request may have blocked,
664: * so check if it's still there and busy.
665: */
666: vm_object_lock(object);
667: if (m == vm_page_lookup(object,offset) &&
668: m->absent && m->busy)
669: VM_PAGE_FREE(m);
670: vm_fault_cleanup(object, first_m);
671: return((rc == MACH_SEND_INTERRUPTED) ?
672: VM_FAULT_INTERRUPTED :
673: VM_FAULT_MEMORY_ERROR);
674: }
1.1.1.2 root 675:
1.1 root 676: /*
677: * Retry with same object/offset, since new data may
678: * be in a different page (i.e., m is meaningless at
679: * this point).
680: */
681: vm_object_lock(object);
682: continue;
683: }
684:
685: /*
686: * For the XP system, the only case in which we get here is if
687: * object has no pager (or unwiring). If the pager doesn't
688: * have the page this is handled in the m->absent case above
689: * (and if you change things here you should look above).
690: */
691: if (object == first_object)
692: first_m = m;
693: else
694: {
695: assert(m == VM_PAGE_NULL);
696: }
697:
698: /*
699: * Move on to the next object. Lock the next
700: * object before unlocking the current one.
701: */
702: access_required = VM_PROT_READ;
703:
704: offset += object->shadow_offset;
705: next_object = object->shadow;
706: if (next_object == VM_OBJECT_NULL) {
707: assert(!must_be_resident);
708:
709: /*
710: * If there's no object left, fill the page
711: * in the top object with zeros. But first we
712: * need to allocate a real page.
713: */
714:
715: if (object != first_object) {
716: vm_object_paging_end(object);
717: vm_object_unlock(object);
718:
719: object = first_object;
720: offset = first_offset;
721: vm_object_lock(object);
722: }
723:
724: m = first_m;
725: assert(m->object == object);
726: first_m = VM_PAGE_NULL;
727:
1.1.1.2 root 728: if (m->fictitious && !vm_page_convert(m, !object->internal)) {
1.1 root 729: VM_PAGE_FREE(m);
730: vm_fault_cleanup(object, VM_PAGE_NULL);
731: return(VM_FAULT_MEMORY_SHORTAGE);
732: }
733:
734: vm_object_unlock(object);
735: vm_page_zero_fill(m);
736: vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS);
737: vm_stat.zero_fill_count++;
1.1.1.3 root 738: current_task()->zero_fills++;
1.1 root 739: vm_object_lock(object);
740: pmap_clear_modify(m->phys_addr);
741: break;
742: }
743: else {
744: vm_object_lock(next_object);
745: if ((object != first_object) || must_be_resident)
746: vm_object_paging_end(object);
747: vm_object_unlock(object);
748: object = next_object;
749: vm_object_paging_begin(object);
750: }
751: }
752:
753: /*
754: * PAGE HAS BEEN FOUND.
755: *
756: * This page (m) is:
757: * busy, so that we can play with it;
758: * not absent, so that nobody else will fill it;
759: * possibly eligible for pageout;
760: *
761: * The top-level page (first_m) is:
762: * VM_PAGE_NULL if the page was found in the
763: * top-level object;
764: * busy, not absent, and ineligible for pageout.
765: *
766: * The current object (object) is locked. A paging
767: * reference is held for the current and top-level
768: * objects.
769: */
770:
771: #if EXTRA_ASSERTIONS
772: assert(m->busy && !m->absent);
773: assert((first_m == VM_PAGE_NULL) ||
774: (first_m->busy && !first_m->absent &&
775: !first_m->active && !first_m->inactive));
1.1.1.2 root 776: #endif /* EXTRA_ASSERTIONS */
1.1 root 777:
778: /*
779: * If the page is being written, but isn't
780: * already owned by the top-level object,
781: * we have to copy it into a new page owned
782: * by the top-level object.
783: */
784:
785: if (object != first_object) {
786: /*
787: * We only really need to copy if we
788: * want to write it.
789: */
790:
791: if (fault_type & VM_PROT_WRITE) {
792: vm_page_t copy_m;
793:
794: assert(!must_be_resident);
795:
796: /*
797: * If we try to collapse first_object at this
798: * point, we may deadlock when we try to get
799: * the lock on an intermediate object (since we
800: * have the bottom object locked). We can't
801: * unlock the bottom object, because the page
802: * we found may move (by collapse) if we do.
803: *
804: * Instead, we first copy the page. Then, when
805: * we have no more use for the bottom object,
806: * we unlock it and try to collapse.
807: *
808: * Note that we copy the page even if we didn't
809: * need to... that's the breaks.
810: */
811:
812: /*
813: * Allocate a page for the copy
814: */
1.1.1.2 root 815: copy_m = vm_page_grab(!first_object->internal);
1.1 root 816: if (copy_m == VM_PAGE_NULL) {
817: RELEASE_PAGE(m);
818: vm_fault_cleanup(object, first_m);
819: return(VM_FAULT_MEMORY_SHORTAGE);
820: }
821:
822: vm_object_unlock(object);
823: vm_page_copy(m, copy_m);
824: vm_object_lock(object);
825:
826: /*
827: * If another map is truly sharing this
828: * page with us, we have to flush all
829: * uses of the original page, since we
830: * can't distinguish those which want the
831: * original from those which need the
832: * new copy.
833: *
834: * XXXO If we know that only one map has
835: * access to this page, then we could
836: * avoid the pmap_page_protect() call.
837: */
838:
839: vm_page_lock_queues();
840: vm_page_deactivate(m);
841: pmap_page_protect(m->phys_addr, VM_PROT_NONE);
842: vm_page_unlock_queues();
843:
844: /*
845: * We no longer need the old page or object.
846: */
847:
848: PAGE_WAKEUP_DONE(m);
849: vm_object_paging_end(object);
850: vm_object_unlock(object);
851:
852: vm_stat.cow_faults++;
853: vm_stat_sample(SAMPLED_PC_VM_COW_FAULTS);
1.1.1.3 root 854: current_task()->cow_faults++;
1.1 root 855: object = first_object;
856: offset = first_offset;
857:
858: vm_object_lock(object);
859: VM_PAGE_FREE(first_m);
860: first_m = VM_PAGE_NULL;
861: assert(copy_m->busy);
862: vm_page_lock_queues();
863: vm_page_insert(copy_m, object, offset);
864: vm_page_unlock_queues();
865: m = copy_m;
866:
867: /*
868: * Now that we've gotten the copy out of the
869: * way, let's try to collapse the top object.
870: * But we have to play ugly games with
871: * paging_in_progress to do that...
872: */
873:
874: vm_object_paging_end(object);
875: vm_object_collapse(object);
876: vm_object_paging_begin(object);
877: }
878: else {
879: *protection &= (~VM_PROT_WRITE);
880: }
881: }
882:
883: /*
884: * Now check whether the page needs to be pushed into the
885: * copy object. The use of asymmetric copy on write for
886: * shared temporary objects means that we may do two copies to
887: * satisfy the fault; one above to get the page from a
888: * shadowed object, and one here to push it into the copy.
889: */
890:
891: while ((copy_object = first_object->copy) != VM_OBJECT_NULL) {
892: vm_offset_t copy_offset;
893: vm_page_t copy_m;
894:
895: /*
896: * If the page is being written, but hasn't been
897: * copied to the copy-object, we have to copy it there.
898: */
899:
900: if ((fault_type & VM_PROT_WRITE) == 0) {
901: *protection &= ~VM_PROT_WRITE;
902: break;
903: }
904:
905: /*
906: * If the page was guaranteed to be resident,
907: * we must have already performed the copy.
908: */
909:
910: if (must_be_resident)
911: break;
912:
913: /*
914: * Try to get the lock on the copy_object.
915: */
916: if (!vm_object_lock_try(copy_object)) {
917: vm_object_unlock(object);
918:
919: simple_lock_pause(); /* wait a bit */
920:
921: vm_object_lock(object);
922: continue;
923: }
924:
925: /*
926: * Make another reference to the copy-object,
927: * to keep it from disappearing during the
928: * copy.
929: */
930: assert(copy_object->ref_count > 0);
931: copy_object->ref_count++;
932:
933: /*
934: * Does the page exist in the copy?
935: */
936: copy_offset = first_offset - copy_object->shadow_offset;
937: copy_m = vm_page_lookup(copy_object, copy_offset);
938: if (copy_m != VM_PAGE_NULL) {
939: if (copy_m->busy) {
940: /*
941: * If the page is being brought
942: * in, wait for it and then retry.
943: */
944: PAGE_ASSERT_WAIT(copy_m, interruptible);
945: RELEASE_PAGE(m);
946: copy_object->ref_count--;
947: assert(copy_object->ref_count > 0);
948: vm_object_unlock(copy_object);
949: goto block_and_backoff;
950: }
951: }
952: else {
953: /*
954: * Allocate a page for the copy
955: */
956: copy_m = vm_page_alloc(copy_object, copy_offset);
957: if (copy_m == VM_PAGE_NULL) {
958: RELEASE_PAGE(m);
959: copy_object->ref_count--;
960: assert(copy_object->ref_count > 0);
961: vm_object_unlock(copy_object);
962: vm_fault_cleanup(object, first_m);
963: return(VM_FAULT_MEMORY_SHORTAGE);
964: }
965:
966: /*
967: * Must copy page into copy-object.
968: */
969:
970: vm_page_copy(m, copy_m);
1.1.1.2 root 971:
1.1 root 972: /*
973: * If the old page was in use by any users
974: * of the copy-object, it must be removed
975: * from all pmaps. (We can't know which
976: * pmaps use it.)
977: */
978:
979: vm_page_lock_queues();
980: pmap_page_protect(m->phys_addr, VM_PROT_NONE);
981: copy_m->dirty = TRUE;
982: vm_page_unlock_queues();
983:
984: /*
985: * If there's a pager, then immediately
986: * page out this page, using the "initialize"
987: * option. Else, we use the copy.
988: */
989:
990: if (!copy_object->pager_created) {
991: vm_page_lock_queues();
992: vm_page_activate(copy_m);
993: vm_page_unlock_queues();
994: PAGE_WAKEUP_DONE(copy_m);
995: } else {
996: /*
997: * The page is already ready for pageout:
998: * not on pageout queues and busy.
999: * Unlock everything except the
1000: * copy_object itself.
1001: */
1002:
1003: vm_object_unlock(object);
1004:
1005: /*
1006: * Write the page to the copy-object,
1007: * flushing it from the kernel.
1008: */
1009:
1010: vm_pageout_page(copy_m, TRUE, TRUE);
1011:
1012: /*
1013: * Since the pageout may have
1014: * temporarily dropped the
1015: * copy_object's lock, we
1016: * check whether we'll have
1017: * to deallocate the hard way.
1018: */
1019:
1020: if ((copy_object->shadow != object) ||
1021: (copy_object->ref_count == 1)) {
1022: vm_object_unlock(copy_object);
1023: vm_object_deallocate(copy_object);
1024: vm_object_lock(object);
1025: continue;
1026: }
1027:
1028: /*
1029: * Pick back up the old object's
1030: * lock. [It is safe to do so,
1031: * since it must be deeper in the
1032: * object tree.]
1033: */
1034:
1035: vm_object_lock(object);
1036: }
1037:
1038: /*
1039: * Because we're pushing a page upward
1040: * in the object tree, we must restart
1041: * any faults that are waiting here.
1042: * [Note that this is an expansion of
1043: * PAGE_WAKEUP that uses the THREAD_RESTART
1044: * wait result]. Can't turn off the page's
1045: * busy bit because we're not done with it.
1046: */
1.1.1.2 root 1047:
1.1 root 1048: if (m->wanted) {
1049: m->wanted = FALSE;
1050: thread_wakeup_with_result((event_t) m,
1051: THREAD_RESTART);
1052: }
1053: }
1054:
1055: /*
1056: * The reference count on copy_object must be
1057: * at least 2: one for our extra reference,
1058: * and at least one from the outside world
1059: * (we checked that when we last locked
1060: * copy_object).
1061: */
1062: copy_object->ref_count--;
1063: assert(copy_object->ref_count > 0);
1064: vm_object_unlock(copy_object);
1065:
1066: break;
1067: }
1068:
1069: *result_page = m;
1070: *top_page = first_m;
1071:
1072: /*
1073: * If the page can be written, assume that it will be.
1074: * [Earlier, we restrict the permission to allow write
1075: * access only if the fault so required, so we don't
1076: * mark read-only data as dirty.]
1077: */
1078:
1079: if (vm_fault_dirty_handling && (*protection & VM_PROT_WRITE))
1080: m->dirty = TRUE;
1081:
1082: return(VM_FAULT_SUCCESS);
1083:
1084: block_and_backoff:
1085: vm_fault_cleanup(object, first_m);
1086:
1087: if (continuation != (void (*)()) 0) {
1.1.1.4 ! root 1088: vm_fault_state_t *state =
1.1 root 1089: (vm_fault_state_t *) current_thread()->ith_other;
1090:
1091: /*
1092: * Save variables in case we must restart.
1093: */
1094:
1095: state->vmfp_backoff = TRUE;
1096: state->vmf_prot = *protection;
1097:
1098: counter(c_vm_fault_page_block_backoff_user++);
1099: thread_block(continuation);
1100: } else
1101: {
1102: counter(c_vm_fault_page_block_backoff_kernel++);
1103: thread_block((void (*)()) 0);
1104: }
1105: after_block_and_backoff:
1106: if (current_thread()->wait_result == THREAD_AWAKENED)
1107: return VM_FAULT_RETRY;
1108: else
1109: return VM_FAULT_INTERRUPTED;
1110:
1111: #undef RELEASE_PAGE
1112: }
1113:
1114: /*
1115: * Routine: vm_fault
1116: * Purpose:
1117: * Handle page faults, including pseudo-faults
1118: * used to change the wiring status of pages.
1119: * Returns:
1120: * If an explicit (expression) continuation is supplied,
1121: * then we call the continuation instead of returning.
1122: * Implementation:
1123: * Explicit continuations make this a little icky,
1124: * because it hasn't been rewritten to embrace CPS.
1125: * Instead, we have resume arguments for vm_fault and
1126: * vm_fault_page, to let continue the fault computation.
1127: *
1128: * vm_fault and vm_fault_page save mucho state
1129: * in the moral equivalent of a closure. The state
1130: * structure is allocated when first entering vm_fault
1131: * and deallocated when leaving vm_fault.
1132: */
1133:
1134: void
1.1.1.4 ! root 1135: vm_fault_continue(void)
1.1 root 1136: {
1.1.1.4 ! root 1137: vm_fault_state_t *state =
1.1 root 1138: (vm_fault_state_t *) current_thread()->ith_other;
1139:
1140: (void) vm_fault(state->vmf_map,
1141: state->vmf_vaddr,
1142: state->vmf_fault_type,
1143: state->vmf_change_wiring,
1144: TRUE, state->vmf_continuation);
1145: /*NOTREACHED*/
1146: }
1147:
1.1.1.4 ! root 1148: kern_return_t vm_fault(
! 1149: vm_map_t map,
! 1150: vm_offset_t vaddr,
! 1151: vm_prot_t fault_type,
! 1152: boolean_t change_wiring,
! 1153: boolean_t resume,
! 1154: void (*continuation)())
1.1 root 1155: {
1156: vm_map_version_t version; /* Map version for verificiation */
1157: boolean_t wired; /* Should mapping be wired down? */
1158: vm_object_t object; /* Top-level object */
1159: vm_offset_t offset; /* Top-level offset */
1160: vm_prot_t prot; /* Protection for mapping */
1161: vm_object_t old_copy_object; /* Saved copy object */
1162: vm_page_t result_page; /* Result of vm_fault_page */
1163: vm_page_t top_page; /* Placeholder page */
1164: kern_return_t kr;
1165:
1166: vm_page_t m; /* Fast access to result_page */
1167:
1168: if (resume) {
1.1.1.4 ! root 1169: vm_fault_state_t *state =
1.1 root 1170: (vm_fault_state_t *) current_thread()->ith_other;
1171:
1172: /*
1173: * Retrieve cached variables and
1174: * continue vm_fault_page.
1175: */
1176:
1177: object = state->vmf_object;
1178: if (object == VM_OBJECT_NULL)
1179: goto RetryFault;
1180: version = state->vmf_version;
1181: wired = state->vmf_wired;
1182: offset = state->vmf_offset;
1183: prot = state->vmf_prot;
1184:
1185: kr = vm_fault_page(object, offset, fault_type,
1186: (change_wiring && !wired), !change_wiring,
1187: &prot, &result_page, &top_page,
1188: TRUE, vm_fault_continue);
1189: goto after_vm_fault_page;
1190: }
1191:
1192: if (continuation != (void (*)()) 0) {
1193: /*
1194: * We will probably need to save state.
1195: */
1196:
1197: char * state;
1198:
1199: /*
1200: * if this assignment stmt is written as
1.1.1.3 root 1201: * 'active_threads[cpu_number()] = kmem_cache_alloc()',
1202: * cpu_number may be evaluated before kmem_cache_alloc;
1203: * if kmem_cache_alloc blocks, cpu_number will be wrong
1.1 root 1204: */
1205:
1.1.1.3 root 1206: state = (char *) kmem_cache_alloc(&vm_fault_state_cache);
1.1 root 1207: current_thread()->ith_other = state;
1208:
1209: }
1210:
1211: RetryFault: ;
1212:
1213: /*
1214: * Find the backing store object and offset into
1215: * it to begin the search.
1216: */
1217:
1218: if ((kr = vm_map_lookup(&map, vaddr, fault_type, &version,
1219: &object, &offset,
1220: &prot, &wired)) != KERN_SUCCESS) {
1221: goto done;
1222: }
1223:
1224: /*
1225: * If the page is wired, we must fault for the current protection
1226: * value, to avoid further faults.
1227: */
1228:
1229: if (wired)
1230: fault_type = prot;
1231:
1232: /*
1233: * Make a reference to this object to
1234: * prevent its disposal while we are messing with
1235: * it. Once we have the reference, the map is free
1236: * to be diddled. Since objects reference their
1237: * shadows (and copies), they will stay around as well.
1238: */
1239:
1240: assert(object->ref_count > 0);
1241: object->ref_count++;
1242: vm_object_paging_begin(object);
1243:
1244: if (continuation != (void (*)()) 0) {
1.1.1.4 ! root 1245: vm_fault_state_t *state =
1.1 root 1246: (vm_fault_state_t *) current_thread()->ith_other;
1247:
1248: /*
1249: * Save variables, in case vm_fault_page discards
1250: * our kernel stack and we have to restart.
1251: */
1252:
1253: state->vmf_map = map;
1254: state->vmf_vaddr = vaddr;
1255: state->vmf_fault_type = fault_type;
1256: state->vmf_change_wiring = change_wiring;
1257: state->vmf_continuation = continuation;
1258:
1259: state->vmf_version = version;
1260: state->vmf_wired = wired;
1261: state->vmf_object = object;
1262: state->vmf_offset = offset;
1263: state->vmf_prot = prot;
1264:
1265: kr = vm_fault_page(object, offset, fault_type,
1266: (change_wiring && !wired), !change_wiring,
1267: &prot, &result_page, &top_page,
1268: FALSE, vm_fault_continue);
1269: } else
1270: {
1271: kr = vm_fault_page(object, offset, fault_type,
1272: (change_wiring && !wired), !change_wiring,
1273: &prot, &result_page, &top_page,
1274: FALSE, (void (*)()) 0);
1275: }
1276: after_vm_fault_page:
1277:
1278: /*
1279: * If we didn't succeed, lose the object reference immediately.
1280: */
1281:
1282: if (kr != VM_FAULT_SUCCESS)
1283: vm_object_deallocate(object);
1284:
1285: /*
1286: * See why we failed, and take corrective action.
1287: */
1288:
1289: switch (kr) {
1290: case VM_FAULT_SUCCESS:
1291: break;
1292: case VM_FAULT_RETRY:
1293: goto RetryFault;
1294: case VM_FAULT_INTERRUPTED:
1295: kr = KERN_SUCCESS;
1296: goto done;
1297: case VM_FAULT_MEMORY_SHORTAGE:
1298: if (continuation != (void (*)()) 0) {
1.1.1.4 ! root 1299: vm_fault_state_t *state =
1.1 root 1300: (vm_fault_state_t *) current_thread()->ith_other;
1301:
1302: /*
1303: * Save variables in case VM_PAGE_WAIT
1304: * discards our kernel stack.
1305: */
1306:
1307: state->vmf_map = map;
1308: state->vmf_vaddr = vaddr;
1309: state->vmf_fault_type = fault_type;
1310: state->vmf_change_wiring = change_wiring;
1311: state->vmf_continuation = continuation;
1312: state->vmf_object = VM_OBJECT_NULL;
1313:
1314: VM_PAGE_WAIT(vm_fault_continue);
1315: } else
1316: VM_PAGE_WAIT((void (*)()) 0);
1317: goto RetryFault;
1318: case VM_FAULT_FICTITIOUS_SHORTAGE:
1319: vm_page_more_fictitious();
1320: goto RetryFault;
1321: case VM_FAULT_MEMORY_ERROR:
1322: kr = KERN_MEMORY_ERROR;
1323: goto done;
1324: }
1325:
1326: m = result_page;
1327:
1328: assert((change_wiring && !wired) ?
1329: (top_page == VM_PAGE_NULL) :
1330: ((top_page == VM_PAGE_NULL) == (m->object == object)));
1331:
1332: /*
1333: * How to clean up the result of vm_fault_page. This
1334: * happens whether the mapping is entered or not.
1335: */
1336:
1337: #define UNLOCK_AND_DEALLOCATE \
1338: MACRO_BEGIN \
1339: vm_fault_cleanup(m->object, top_page); \
1340: vm_object_deallocate(object); \
1341: MACRO_END
1342:
1343: /*
1344: * What to do with the resulting page from vm_fault_page
1345: * if it doesn't get entered into the physical map:
1346: */
1347:
1348: #define RELEASE_PAGE(m) \
1349: MACRO_BEGIN \
1350: PAGE_WAKEUP_DONE(m); \
1351: vm_page_lock_queues(); \
1352: if (!m->active && !m->inactive) \
1353: vm_page_activate(m); \
1354: vm_page_unlock_queues(); \
1355: MACRO_END
1356:
1357: /*
1358: * We must verify that the maps have not changed
1359: * since our last lookup.
1360: */
1361:
1362: old_copy_object = m->object->copy;
1363:
1364: vm_object_unlock(m->object);
1365: while (!vm_map_verify(map, &version)) {
1366: vm_object_t retry_object;
1367: vm_offset_t retry_offset;
1368: vm_prot_t retry_prot;
1369:
1370: /*
1371: * To avoid trying to write_lock the map while another
1372: * thread has it read_locked (in vm_map_pageable), we
1373: * do not try for write permission. If the page is
1374: * still writable, we will get write permission. If it
1375: * is not, or has been marked needs_copy, we enter the
1376: * mapping without write permission, and will merely
1377: * take another fault.
1378: */
1379: kr = vm_map_lookup(&map, vaddr,
1380: fault_type & ~VM_PROT_WRITE, &version,
1381: &retry_object, &retry_offset, &retry_prot,
1382: &wired);
1383:
1384: if (kr != KERN_SUCCESS) {
1385: vm_object_lock(m->object);
1386: RELEASE_PAGE(m);
1387: UNLOCK_AND_DEALLOCATE;
1388: goto done;
1389: }
1390:
1391: vm_object_unlock(retry_object);
1392: vm_object_lock(m->object);
1393:
1394: if ((retry_object != object) ||
1395: (retry_offset != offset)) {
1396: RELEASE_PAGE(m);
1397: UNLOCK_AND_DEALLOCATE;
1398: goto RetryFault;
1399: }
1400:
1401: /*
1402: * Check whether the protection has changed or the object
1403: * has been copied while we left the map unlocked.
1404: */
1405: prot &= retry_prot;
1406: vm_object_unlock(m->object);
1407: }
1408: vm_object_lock(m->object);
1409:
1410: /*
1411: * If the copy object changed while the top-level object
1412: * was unlocked, then we must take away write permission.
1413: */
1414:
1415: if (m->object->copy != old_copy_object)
1416: prot &= ~VM_PROT_WRITE;
1417:
1418: /*
1419: * If we want to wire down this page, but no longer have
1420: * adequate permissions, we must start all over.
1421: */
1422:
1423: if (wired && (prot != fault_type)) {
1424: vm_map_verify_done(map, &version);
1425: RELEASE_PAGE(m);
1426: UNLOCK_AND_DEALLOCATE;
1427: goto RetryFault;
1428: }
1429:
1430: /*
1431: * It's critically important that a wired-down page be faulted
1432: * only once in each map for which it is wired.
1433: */
1434:
1435: vm_object_unlock(m->object);
1436:
1437: /*
1438: * Put this page into the physical map.
1439: * We had to do the unlock above because pmap_enter
1440: * may cause other faults. The page may be on
1441: * the pageout queues. If the pageout daemon comes
1442: * across the page, it will remove it from the queues.
1443: */
1444:
1445: PMAP_ENTER(map->pmap, vaddr, m, prot, wired);
1446:
1447: /*
1448: * If the page is not wired down and isn't already
1449: * on a pageout queue, then put it where the
1450: * pageout daemon can find it.
1451: */
1452: vm_object_lock(m->object);
1453: vm_page_lock_queues();
1454: if (change_wiring) {
1455: if (wired)
1456: vm_page_wire(m);
1457: else
1458: vm_page_unwire(m);
1459: } else if (software_reference_bits) {
1460: if (!m->active && !m->inactive)
1461: vm_page_activate(m);
1462: m->reference = TRUE;
1463: } else {
1464: vm_page_activate(m);
1465: }
1466: vm_page_unlock_queues();
1467:
1468: /*
1469: * Unlock everything, and return
1470: */
1471:
1472: vm_map_verify_done(map, &version);
1473: PAGE_WAKEUP_DONE(m);
1474: kr = KERN_SUCCESS;
1475: UNLOCK_AND_DEALLOCATE;
1476:
1477: #undef UNLOCK_AND_DEALLOCATE
1478: #undef RELEASE_PAGE
1479:
1480: done:
1481: if (continuation != (void (*)()) 0) {
1.1.1.4 ! root 1482: vm_fault_state_t *state =
1.1 root 1483: (vm_fault_state_t *) current_thread()->ith_other;
1484:
1.1.1.3 root 1485: kmem_cache_free(&vm_fault_state_cache, (vm_offset_t) state);
1.1 root 1486: (*continuation)(kr);
1487: /*NOTREACHED*/
1488: }
1489:
1490: return(kr);
1491: }
1492:
1493: /*
1494: * vm_fault_wire:
1495: *
1496: * Wire down a range of virtual addresses in a map.
1497: */
1.1.1.4 ! root 1498: void vm_fault_wire(
! 1499: vm_map_t map,
! 1500: vm_map_entry_t entry)
1.1 root 1501: {
1502:
1.1.1.4 ! root 1503: vm_offset_t va;
! 1504: pmap_t pmap;
! 1505: vm_offset_t end_addr = entry->vme_end;
1.1 root 1506:
1507: pmap = vm_map_pmap(map);
1508:
1509: /*
1510: * Inform the physical mapping system that the
1511: * range of addresses may not fault, so that
1512: * page tables and such can be locked down as well.
1513: */
1514:
1515: pmap_pageable(pmap, entry->vme_start, end_addr, FALSE);
1516:
1517: /*
1518: * We simulate a fault to get the page and enter it
1519: * in the physical map.
1520: */
1521:
1522: for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
1523: if (vm_fault_wire_fast(map, va, entry) != KERN_SUCCESS)
1524: (void) vm_fault(map, va, VM_PROT_NONE, TRUE,
1525: FALSE, (void (*)()) 0);
1526: }
1527: }
1528:
1529: /*
1530: * vm_fault_unwire:
1531: *
1532: * Unwire a range of virtual addresses in a map.
1533: */
1.1.1.4 ! root 1534: void vm_fault_unwire(
! 1535: vm_map_t map,
! 1536: vm_map_entry_t entry)
1.1 root 1537: {
1.1.1.4 ! root 1538: vm_offset_t va;
! 1539: pmap_t pmap;
! 1540: vm_offset_t end_addr = entry->vme_end;
! 1541: vm_object_t object;
1.1 root 1542:
1543: pmap = vm_map_pmap(map);
1544:
1545: object = (entry->is_sub_map)
1546: ? VM_OBJECT_NULL : entry->object.vm_object;
1547:
1548: /*
1549: * Since the pages are wired down, we must be able to
1550: * get their mappings from the physical map system.
1551: */
1552:
1553: for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
1554: pmap_change_wiring(pmap, va, FALSE);
1555:
1556: if (object == VM_OBJECT_NULL) {
1557: vm_map_lock_set_recursive(map);
1558: (void) vm_fault(map, va, VM_PROT_NONE, TRUE,
1559: FALSE, (void (*)()) 0);
1560: vm_map_lock_clear_recursive(map);
1561: } else {
1562: vm_prot_t prot;
1563: vm_page_t result_page;
1564: vm_page_t top_page;
1565: vm_fault_return_t result;
1566:
1567: do {
1568: prot = VM_PROT_NONE;
1569:
1570: vm_object_lock(object);
1571: vm_object_paging_begin(object);
1572: result = vm_fault_page(object,
1573: entry->offset +
1574: (va - entry->vme_start),
1575: VM_PROT_NONE, TRUE,
1576: FALSE, &prot,
1577: &result_page,
1578: &top_page,
1579: FALSE, (void (*)()) 0);
1580: } while (result == VM_FAULT_RETRY);
1581:
1582: if (result != VM_FAULT_SUCCESS)
1583: panic("vm_fault_unwire: failure");
1584:
1585: vm_page_lock_queues();
1586: vm_page_unwire(result_page);
1587: vm_page_unlock_queues();
1588: PAGE_WAKEUP_DONE(result_page);
1589:
1590: vm_fault_cleanup(result_page->object, top_page);
1591: }
1592: }
1593:
1594: /*
1595: * Inform the physical mapping system that the range
1596: * of addresses may fault, so that page tables and
1597: * such may be unwired themselves.
1598: */
1599:
1600: pmap_pageable(pmap, entry->vme_start, end_addr, TRUE);
1601: }
1602:
1603: /*
1604: * vm_fault_wire_fast:
1605: *
1606: * Handle common case of a wire down page fault at the given address.
1607: * If successful, the page is inserted into the associated physical map.
1608: * The map entry is passed in to avoid the overhead of a map lookup.
1609: *
1610: * NOTE: the given address should be truncated to the
1611: * proper page address.
1612: *
1613: * KERN_SUCCESS is returned if the page fault is handled; otherwise,
1614: * a standard error specifying why the fault is fatal is returned.
1615: *
1616: * The map in question must be referenced, and remains so.
1617: * Caller has a read lock on the map.
1618: *
1619: * This is a stripped version of vm_fault() for wiring pages. Anything
1620: * other than the common case will return KERN_FAILURE, and the caller
1621: * is expected to call vm_fault().
1622: */
1.1.1.4 ! root 1623: kern_return_t vm_fault_wire_fast(
! 1624: vm_map_t map,
! 1625: vm_offset_t va,
! 1626: vm_map_entry_t entry)
1.1 root 1627: {
1628: vm_object_t object;
1629: vm_offset_t offset;
1.1.1.4 ! root 1630: vm_page_t m;
1.1 root 1631: vm_prot_t prot;
1632:
1633: vm_stat.faults++; /* needs lock XXX */
1.1.1.3 root 1634: current_task()->faults++;
1.1 root 1635: /*
1636: * Recovery actions
1637: */
1638:
1639: #undef RELEASE_PAGE
1640: #define RELEASE_PAGE(m) { \
1641: PAGE_WAKEUP_DONE(m); \
1642: vm_page_lock_queues(); \
1643: vm_page_unwire(m); \
1644: vm_page_unlock_queues(); \
1645: }
1646:
1647:
1648: #undef UNLOCK_THINGS
1649: #define UNLOCK_THINGS { \
1650: object->paging_in_progress--; \
1651: vm_object_unlock(object); \
1652: }
1653:
1654: #undef UNLOCK_AND_DEALLOCATE
1655: #define UNLOCK_AND_DEALLOCATE { \
1656: UNLOCK_THINGS; \
1657: vm_object_deallocate(object); \
1658: }
1659: /*
1660: * Give up and have caller do things the hard way.
1661: */
1662:
1663: #define GIVE_UP { \
1664: UNLOCK_AND_DEALLOCATE; \
1665: return(KERN_FAILURE); \
1666: }
1667:
1668:
1669: /*
1670: * If this entry is not directly to a vm_object, bail out.
1671: */
1672: if (entry->is_sub_map)
1673: return(KERN_FAILURE);
1674:
1675: /*
1676: * Find the backing store object and offset into it.
1677: */
1678:
1679: object = entry->object.vm_object;
1680: offset = (va - entry->vme_start) + entry->offset;
1681: prot = entry->protection;
1682:
1683: /*
1684: * Make a reference to this object to prevent its
1685: * disposal while we are messing with it.
1686: */
1687:
1688: vm_object_lock(object);
1689: assert(object->ref_count > 0);
1690: object->ref_count++;
1691: object->paging_in_progress++;
1692:
1693: /*
1694: * INVARIANTS (through entire routine):
1695: *
1696: * 1) At all times, we must either have the object
1697: * lock or a busy page in some object to prevent
1698: * some other thread from trying to bring in
1699: * the same page.
1700: *
1701: * 2) Once we have a busy page, we must remove it from
1702: * the pageout queues, so that the pageout daemon
1703: * will not grab it away.
1704: *
1705: */
1706:
1707: /*
1708: * Look for page in top-level object. If it's not there or
1709: * there's something going on, give up.
1710: */
1711: m = vm_page_lookup(object, offset);
1712: if ((m == VM_PAGE_NULL) || (m->error) ||
1713: (m->busy) || (m->absent) || (prot & m->page_lock)) {
1714: GIVE_UP;
1715: }
1716:
1717: /*
1718: * Wire the page down now. All bail outs beyond this
1.1.1.2 root 1719: * point must unwire the page.
1.1 root 1720: */
1721:
1722: vm_page_lock_queues();
1723: vm_page_wire(m);
1724: vm_page_unlock_queues();
1725:
1726: /*
1727: * Mark page busy for other threads.
1728: */
1729: assert(!m->busy);
1730: m->busy = TRUE;
1731: assert(!m->absent);
1732:
1733: /*
1734: * Give up if the page is being written and there's a copy object
1735: */
1736: if ((object->copy != VM_OBJECT_NULL) && (prot & VM_PROT_WRITE)) {
1737: RELEASE_PAGE(m);
1738: GIVE_UP;
1739: }
1740:
1741: /*
1742: * Put this page into the physical map.
1743: * We have to unlock the object because pmap_enter
1.1.1.2 root 1744: * may cause other faults.
1.1 root 1745: */
1746: vm_object_unlock(object);
1747:
1748: PMAP_ENTER(map->pmap, va, m, prot, TRUE);
1749:
1750: /*
1751: * Must relock object so that paging_in_progress can be cleared.
1752: */
1753: vm_object_lock(object);
1754:
1755: /*
1756: * Unlock everything, and return
1757: */
1758:
1759: PAGE_WAKEUP_DONE(m);
1760: UNLOCK_AND_DEALLOCATE;
1761:
1762: return(KERN_SUCCESS);
1763:
1764: }
1765:
1766: /*
1767: * Routine: vm_fault_copy_cleanup
1768: * Purpose:
1769: * Release a page used by vm_fault_copy.
1770: */
1771:
1.1.1.4 ! root 1772: void vm_fault_copy_cleanup(
! 1773: vm_page_t page,
! 1774: vm_page_t top_page)
1.1 root 1775: {
1776: vm_object_t object = page->object;
1777:
1778: vm_object_lock(object);
1779: PAGE_WAKEUP_DONE(page);
1780: vm_page_lock_queues();
1781: if (!page->active && !page->inactive)
1782: vm_page_activate(page);
1783: vm_page_unlock_queues();
1784: vm_fault_cleanup(object, top_page);
1785: }
1786:
1787: /*
1788: * Routine: vm_fault_copy
1789: *
1790: * Purpose:
1791: * Copy pages from one virtual memory object to another --
1792: * neither the source nor destination pages need be resident.
1793: *
1794: * Before actually copying a page, the version associated with
1795: * the destination address map wil be verified.
1796: *
1797: * In/out conditions:
1798: * The caller must hold a reference, but not a lock, to
1799: * each of the source and destination objects and to the
1800: * destination map.
1801: *
1802: * Results:
1803: * Returns KERN_SUCCESS if no errors were encountered in
1804: * reading or writing the data. Returns KERN_INTERRUPTED if
1805: * the operation was interrupted (only possible if the
1806: * "interruptible" argument is asserted). Other return values
1807: * indicate a permanent error in copying the data.
1808: *
1809: * The actual amount of data copied will be returned in the
1810: * "copy_size" argument. In the event that the destination map
1811: * verification failed, this amount may be less than the amount
1812: * requested.
1813: */
1814: kern_return_t vm_fault_copy(
1.1.1.4 ! root 1815: vm_object_t src_object,
! 1816: vm_offset_t src_offset,
! 1817: vm_size_t *src_size, /* INOUT */
! 1818: vm_object_t dst_object,
! 1819: vm_offset_t dst_offset,
! 1820: vm_map_t dst_map,
! 1821: vm_map_version_t *dst_version,
! 1822: boolean_t interruptible)
1.1 root 1823: {
1824: vm_page_t result_page;
1825: vm_prot_t prot;
1.1.1.2 root 1826:
1.1 root 1827: vm_page_t src_page;
1828: vm_page_t src_top_page;
1829:
1830: vm_page_t dst_page;
1831: vm_page_t dst_top_page;
1832:
1833: vm_size_t amount_done;
1834: vm_object_t old_copy_object;
1835:
1836: #define RETURN(x) \
1837: MACRO_BEGIN \
1838: *src_size = amount_done; \
1839: MACRO_RETURN(x); \
1840: MACRO_END
1841:
1842: amount_done = 0;
1843: do { /* while (amount_done != *src_size) */
1844:
1845: RetrySourceFault: ;
1846:
1847: if (src_object == VM_OBJECT_NULL) {
1848: /*
1849: * No source object. We will just
1850: * zero-fill the page in dst_object.
1851: */
1852:
1853: src_page = VM_PAGE_NULL;
1854: } else {
1855: prot = VM_PROT_READ;
1856:
1857: vm_object_lock(src_object);
1858: vm_object_paging_begin(src_object);
1859:
1860: switch (vm_fault_page(src_object, src_offset,
1861: VM_PROT_READ, FALSE, interruptible,
1862: &prot, &result_page, &src_top_page,
1863: FALSE, (void (*)()) 0)) {
1864:
1865: case VM_FAULT_SUCCESS:
1866: break;
1867: case VM_FAULT_RETRY:
1868: goto RetrySourceFault;
1869: case VM_FAULT_INTERRUPTED:
1870: RETURN(MACH_SEND_INTERRUPTED);
1871: case VM_FAULT_MEMORY_SHORTAGE:
1872: VM_PAGE_WAIT((void (*)()) 0);
1873: goto RetrySourceFault;
1874: case VM_FAULT_FICTITIOUS_SHORTAGE:
1875: vm_page_more_fictitious();
1876: goto RetrySourceFault;
1877: case VM_FAULT_MEMORY_ERROR:
1878: return(KERN_MEMORY_ERROR);
1879: }
1880:
1881: src_page = result_page;
1882:
1883: assert((src_top_page == VM_PAGE_NULL) ==
1884: (src_page->object == src_object));
1885:
1886: assert ((prot & VM_PROT_READ) != VM_PROT_NONE);
1887:
1888: vm_object_unlock(src_page->object);
1889: }
1890:
1891: RetryDestinationFault: ;
1892:
1893: prot = VM_PROT_WRITE;
1894:
1895: vm_object_lock(dst_object);
1896: vm_object_paging_begin(dst_object);
1897:
1898: switch (vm_fault_page(dst_object, dst_offset, VM_PROT_WRITE,
1899: FALSE, FALSE /* interruptible */,
1900: &prot, &result_page, &dst_top_page,
1901: FALSE, (void (*)()) 0)) {
1902:
1903: case VM_FAULT_SUCCESS:
1904: break;
1905: case VM_FAULT_RETRY:
1906: goto RetryDestinationFault;
1907: case VM_FAULT_INTERRUPTED:
1908: if (src_page != VM_PAGE_NULL)
1909: vm_fault_copy_cleanup(src_page,
1910: src_top_page);
1911: RETURN(MACH_SEND_INTERRUPTED);
1912: case VM_FAULT_MEMORY_SHORTAGE:
1913: VM_PAGE_WAIT((void (*)()) 0);
1914: goto RetryDestinationFault;
1915: case VM_FAULT_FICTITIOUS_SHORTAGE:
1916: vm_page_more_fictitious();
1917: goto RetryDestinationFault;
1918: case VM_FAULT_MEMORY_ERROR:
1919: if (src_page != VM_PAGE_NULL)
1920: vm_fault_copy_cleanup(src_page,
1921: src_top_page);
1922: return(KERN_MEMORY_ERROR);
1923: }
1924: assert ((prot & VM_PROT_WRITE) != VM_PROT_NONE);
1925:
1926: dst_page = result_page;
1927:
1928: old_copy_object = dst_page->object->copy;
1929:
1930: vm_object_unlock(dst_page->object);
1931:
1932: if (!vm_map_verify(dst_map, dst_version)) {
1933:
1934: BailOut: ;
1935:
1936: if (src_page != VM_PAGE_NULL)
1937: vm_fault_copy_cleanup(src_page, src_top_page);
1938: vm_fault_copy_cleanup(dst_page, dst_top_page);
1939: break;
1940: }
1941:
1942:
1943: vm_object_lock(dst_page->object);
1944: if (dst_page->object->copy != old_copy_object) {
1945: vm_object_unlock(dst_page->object);
1946: vm_map_verify_done(dst_map, dst_version);
1947: goto BailOut;
1948: }
1949: vm_object_unlock(dst_page->object);
1950:
1951: /*
1952: * Copy the page, and note that it is dirty
1953: * immediately.
1954: */
1955:
1956: if (src_page == VM_PAGE_NULL)
1957: vm_page_zero_fill(dst_page);
1958: else
1959: vm_page_copy(src_page, dst_page);
1960: dst_page->dirty = TRUE;
1961:
1962: /*
1963: * Unlock everything, and return
1964: */
1965:
1966: vm_map_verify_done(dst_map, dst_version);
1967:
1968: if (src_page != VM_PAGE_NULL)
1969: vm_fault_copy_cleanup(src_page, src_top_page);
1970: vm_fault_copy_cleanup(dst_page, dst_top_page);
1971:
1972: amount_done += PAGE_SIZE;
1973: src_offset += PAGE_SIZE;
1974: dst_offset += PAGE_SIZE;
1975:
1976: } while (amount_done != *src_size);
1977:
1978: RETURN(KERN_SUCCESS);
1979: #undef RETURN
1980:
1.1.1.2 root 1981: /*NOTREACHED*/
1.1 root 1982: }
1983:
1984:
1985:
1986:
1987:
1988: #ifdef notdef
1989:
1990: /*
1991: * Routine: vm_fault_page_overwrite
1992: *
1993: * Description:
1994: * A form of vm_fault_page that assumes that the
1995: * resulting page will be overwritten in its entirety,
1996: * making it unnecessary to obtain the correct *contents*
1997: * of the page.
1998: *
1999: * Implementation:
2000: * XXX Untested. Also unused. Eventually, this technology
2001: * could be used in vm_fault_copy() to advantage.
2002: */
1.1.1.4 ! root 2003: vm_fault_return_t vm_fault_page_overwrite(
! 2004: vm_object_t dst_object,
! 2005: vm_offset_t dst_offset,
! 2006: vm_page_t *result_page) /* OUT */
1.1 root 2007: {
2008: vm_page_t dst_page;
2009:
2010: #define interruptible FALSE /* XXX */
2011:
2012: while (TRUE) {
2013: /*
2014: * Look for a page at this offset
2015: */
2016:
2017: while ((dst_page = vm_page_lookup(dst_object, dst_offset))
2018: == VM_PAGE_NULL) {
2019: /*
2020: * No page, no problem... just allocate one.
2021: */
2022:
2023: dst_page = vm_page_alloc(dst_object, dst_offset);
2024: if (dst_page == VM_PAGE_NULL) {
2025: vm_object_unlock(dst_object);
2026: VM_PAGE_WAIT((void (*)()) 0);
2027: vm_object_lock(dst_object);
2028: continue;
2029: }
2030:
2031: /*
2032: * Pretend that the memory manager
2033: * write-protected the page.
2034: *
2035: * Note that we will be asking for write
2036: * permission without asking for the data
2037: * first.
2038: */
2039:
2040: dst_page->overwriting = TRUE;
2041: dst_page->page_lock = VM_PROT_WRITE;
2042: dst_page->absent = TRUE;
2043: dst_object->absent_count++;
2044:
2045: break;
2046:
2047: /*
2048: * When we bail out, we might have to throw
2049: * away the page created here.
2050: */
2051:
2052: #define DISCARD_PAGE \
2053: MACRO_BEGIN \
2054: vm_object_lock(dst_object); \
2055: dst_page = vm_page_lookup(dst_object, dst_offset); \
2056: if ((dst_page != VM_PAGE_NULL) && dst_page->overwriting) \
2057: VM_PAGE_FREE(dst_page); \
2058: vm_object_unlock(dst_object); \
2059: MACRO_END
2060: }
2061:
2062: /*
2063: * If the page is write-protected...
2064: */
2065:
2066: if (dst_page->page_lock & VM_PROT_WRITE) {
2067: /*
2068: * ... and an unlock request hasn't been sent
2069: */
2070:
2071: if ( ! (dst_page->unlock_request & VM_PROT_WRITE)) {
2072: vm_prot_t u;
2073: kern_return_t rc;
2074:
2075: /*
2076: * ... then send one now.
2077: */
2078:
2079: if (!dst_object->pager_ready) {
2080: vm_object_assert_wait(dst_object,
2081: VM_OBJECT_EVENT_PAGER_READY,
2082: interruptible);
2083: vm_object_unlock(dst_object);
2084: thread_block((void (*)()) 0);
2085: if (current_thread()->wait_result !=
2086: THREAD_AWAKENED) {
2087: DISCARD_PAGE;
2088: return(VM_FAULT_INTERRUPTED);
2089: }
2090: continue;
2091: }
2092:
2093: u = dst_page->unlock_request |= VM_PROT_WRITE;
2094: vm_object_unlock(dst_object);
2095:
2096: if ((rc = memory_object_data_unlock(
2097: dst_object->pager,
2098: dst_object->pager_request,
2099: dst_offset + dst_object->paging_offset,
2100: PAGE_SIZE,
2101: u)) != KERN_SUCCESS) {
2102: printf("vm_object_overwrite: memory_object_data_unlock failed\n");
2103: DISCARD_PAGE;
2104: return((rc == MACH_SEND_INTERRUPTED) ?
2105: VM_FAULT_INTERRUPTED :
2106: VM_FAULT_MEMORY_ERROR);
2107: }
2108: vm_object_lock(dst_object);
2109: continue;
2110: }
2111:
2112: /* ... fall through to wait below */
2113: } else {
2114: /*
2115: * If the page isn't being used for other
2116: * purposes, then we're done.
2117: */
2118: if ( ! (dst_page->busy || dst_page->absent || dst_page->error) )
2119: break;
2120: }
2121:
2122: PAGE_ASSERT_WAIT(dst_page, interruptible);
2123: vm_object_unlock(dst_object);
2124: thread_block((void (*)()) 0);
2125: if (current_thread()->wait_result != THREAD_AWAKENED) {
2126: DISCARD_PAGE;
2127: return(VM_FAULT_INTERRUPTED);
2128: }
2129: }
2130:
2131: *result_page = dst_page;
2132: return(VM_FAULT_SUCCESS);
2133:
2134: #undef interruptible
2135: #undef DISCARD_PAGE
2136: }
2137:
1.1.1.2 root 2138: #endif /* notdef */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.