![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_fault.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_fault.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1994,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm_fault.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: *
33: * Page fault handling module.
34: */
35:
1.1.1.3 root 36: #include <kern/printf.h>
1.1 root 37: #include <vm/vm_fault.h>
38: #include <mach/kern_return.h>
39: #include <mach/message.h> /* for error codes */
40: #include <kern/counters.h>
1.1.1.3 root 41: #include <kern/debug.h>
1.1 root 42: #include <kern/thread.h>
43: #include <kern/sched_prim.h>
44: #include <vm/vm_map.h>
45: #include <vm/vm_object.h>
46: #include <vm/vm_page.h>
47: #include <vm/pmap.h>
48: #include <mach/vm_statistics.h>
49: #include <vm/vm_pageout.h>
50: #include <mach/vm_param.h>
51: #include <mach/memory_object.h>
1.1.1.3 root 52: #include <vm/memory_object_user.user.h>
1.1 root 53: /* For memory_object_data_{request,unlock} */
1.1.1.5 ! root 54: #include <kern/macros.h>
1.1.1.3 root 55: #include <kern/slab.h>
1.1 root 56:
57: #if MACH_PCSAMPLE
58: #include <kern/pc_sample.h>
59: #endif
60:
61:
62:
63: /*
64: * State needed by vm_fault_continue.
65: * This is a little hefty to drop directly
66: * into the thread structure.
67: */
68: typedef struct vm_fault_state {
69: struct vm_map *vmf_map;
70: vm_offset_t vmf_vaddr;
71: vm_prot_t vmf_fault_type;
72: boolean_t vmf_change_wiring;
73: void (*vmf_continuation)();
74: vm_map_version_t vmf_version;
75: boolean_t vmf_wired;
76: struct vm_object *vmf_object;
77: vm_offset_t vmf_offset;
78: vm_prot_t vmf_prot;
79:
80: boolean_t vmfp_backoff;
81: struct vm_object *vmfp_object;
82: vm_offset_t vmfp_offset;
83: struct vm_page *vmfp_first_m;
84: vm_prot_t vmfp_access;
85: } vm_fault_state_t;
86:
1.1.1.3 root 87: struct kmem_cache vm_fault_state_cache;
1.1 root 88:
89: int vm_object_absent_max = 50;
90:
91: boolean_t vm_fault_dirty_handling = FALSE;
92: boolean_t vm_fault_interruptible = TRUE;
93:
94: boolean_t software_reference_bits = TRUE;
95:
96: #if MACH_KDB
97: extern struct db_watchpoint *db_watchpoint_list;
1.1.1.2 root 98: #endif /* MACH_KDB */
1.1 root 99:
100: /*
101: * Routine: vm_fault_init
102: * Purpose:
103: * Initialize our private data structures.
104: */
1.1.1.3 root 105: void vm_fault_init(void)
1.1 root 106: {
1.1.1.3 root 107: kmem_cache_init(&vm_fault_state_cache, "vm_fault_state",
108: sizeof(vm_fault_state_t), 0, NULL, NULL, NULL, 0);
1.1 root 109: }
110:
111: /*
112: * Routine: vm_fault_cleanup
113: * Purpose:
114: * Clean up the result of vm_fault_page.
115: * Results:
116: * The paging reference for "object" is released.
117: * "object" is unlocked.
118: * If "top_page" is not null, "top_page" is
119: * freed and the paging reference for the object
120: * containing it is released.
121: *
122: * In/out conditions:
123: * "object" must be locked.
124: */
125: void
1.1.1.4 root 126: vm_fault_cleanup(
127: vm_object_t object,
128: vm_page_t top_page)
1.1 root 129: {
130: vm_object_paging_end(object);
131: vm_object_unlock(object);
132:
133: if (top_page != VM_PAGE_NULL) {
134: object = top_page->object;
135: vm_object_lock(object);
136: VM_PAGE_FREE(top_page);
137: vm_object_paging_end(object);
138: vm_object_unlock(object);
139: }
140: }
141:
142:
143: #if MACH_PCSAMPLE
144: /*
145: * Do PC sampling on current thread, assuming
146: * that it is the thread taking this page fault.
147: *
148: * Must check for THREAD_NULL, since faults
149: * can occur before threads are running.
150: */
151:
152: #define vm_stat_sample(flavor) \
153: MACRO_BEGIN \
154: thread_t _thread_ = current_thread(); \
155: \
156: if (_thread_ != THREAD_NULL) \
157: take_pc_sample_macro(_thread_, (flavor)); \
158: MACRO_END
159:
160: #else
161: #define vm_stat_sample(x)
162: #endif /* MACH_PCSAMPLE */
163:
164:
165:
166: /*
167: * Routine: vm_fault_page
168: * Purpose:
169: * Find the resident page for the virtual memory
170: * specified by the given virtual memory object
171: * and offset.
172: * Additional arguments:
173: * The required permissions for the page is given
174: * in "fault_type". Desired permissions are included
175: * in "protection".
176: *
177: * If the desired page is known to be resident (for
178: * example, because it was previously wired down), asserting
179: * the "unwiring" parameter will speed the search.
180: *
181: * If the operation can be interrupted (by thread_abort
182: * or thread_terminate), then the "interruptible"
183: * parameter should be asserted.
184: *
185: * Results:
186: * The page containing the proper data is returned
187: * in "result_page".
188: *
189: * In/out conditions:
190: * The source object must be locked and referenced,
191: * and must donate one paging reference. The reference
192: * is not affected. The paging reference and lock are
193: * consumed.
194: *
195: * If the call succeeds, the object in which "result_page"
196: * resides is left locked and holding a paging reference.
197: * If this is not the original object, a busy page in the
198: * original object is returned in "top_page", to prevent other
199: * callers from pursuing this same data, along with a paging
200: * reference for the original object. The "top_page" should
201: * be destroyed when this guarantee is no longer required.
202: * The "result_page" is also left busy. It is not removed
203: * from the pageout queues.
204: */
1.1.1.4 root 205: vm_fault_return_t vm_fault_page(
1.1 root 206: /* Arguments: */
1.1.1.4 root 207: vm_object_t first_object, /* Object to begin search */
208: vm_offset_t first_offset, /* Offset into object */
209: vm_prot_t fault_type, /* What access is requested */
210: boolean_t must_be_resident,/* Must page be resident? */
211: boolean_t interruptible, /* May fault be interrupted? */
1.1 root 212: /* Modifies in place: */
1.1.1.4 root 213: vm_prot_t *protection, /* Protection for mapping */
1.1 root 214: /* Returns: */
1.1.1.4 root 215: vm_page_t *result_page, /* Page found, if successful */
216: vm_page_t *top_page, /* Page in top object, if
1.1 root 217: * not result_page.
218: */
219: /* More arguments: */
1.1.1.4 root 220: boolean_t resume, /* We are restarting. */
221: void (*continuation)()) /* Continuation for blocking. */
1.1 root 222: {
223: vm_page_t m;
224: vm_object_t object;
225: vm_offset_t offset;
226: vm_page_t first_m;
227: vm_object_t next_object;
228: vm_object_t copy_object;
229: boolean_t look_for_page;
230: vm_prot_t access_required;
231:
232: if (resume) {
1.1.1.4 root 233: vm_fault_state_t *state =
1.1 root 234: (vm_fault_state_t *) current_thread()->ith_other;
235:
236: if (state->vmfp_backoff)
237: goto after_block_and_backoff;
238:
239: object = state->vmfp_object;
240: offset = state->vmfp_offset;
241: first_m = state->vmfp_first_m;
242: access_required = state->vmfp_access;
243: goto after_thread_block;
244: }
245:
246: vm_stat_sample(SAMPLED_PC_VM_FAULTS_ANY);
247: vm_stat.faults++; /* needs lock XXX */
1.1.1.3 root 248: current_task()->faults++;
1.1 root 249:
250: /*
251: * Recovery actions
252: */
253: #define RELEASE_PAGE(m) \
254: MACRO_BEGIN \
255: PAGE_WAKEUP_DONE(m); \
256: vm_page_lock_queues(); \
257: if (!m->active && !m->inactive) \
258: vm_page_activate(m); \
259: vm_page_unlock_queues(); \
260: MACRO_END
261:
262: if (vm_fault_dirty_handling
263: #if MACH_KDB
264: /*
265: * If there are watchpoints set, then
266: * we don't want to give away write permission
267: * on a read fault. Make the task write fault,
268: * so that the watchpoint code notices the access.
269: */
270: || db_watchpoint_list
1.1.1.2 root 271: #endif /* MACH_KDB */
1.1 root 272: ) {
273: /*
274: * If we aren't asking for write permission,
275: * then don't give it away. We're using write
276: * faults to set the dirty bit.
277: */
278: if (!(fault_type & VM_PROT_WRITE))
279: *protection &= ~VM_PROT_WRITE;
280: }
281:
282: if (!vm_fault_interruptible)
283: interruptible = FALSE;
284:
285: /*
286: * INVARIANTS (through entire routine):
287: *
288: * 1) At all times, we must either have the object
289: * lock or a busy page in some object to prevent
290: * some other thread from trying to bring in
291: * the same page.
292: *
293: * Note that we cannot hold any locks during the
294: * pager access or when waiting for memory, so
295: * we use a busy page then.
296: *
297: * Note also that we aren't as concerned about more than
298: * one thread attempting to memory_object_data_unlock
299: * the same page at once, so we don't hold the page
300: * as busy then, but do record the highest unlock
301: * value so far. [Unlock requests may also be delivered
302: * out of order.]
303: *
304: * 2) To prevent another thread from racing us down the
305: * shadow chain and entering a new page in the top
306: * object before we do, we must keep a busy page in
307: * the top object while following the shadow chain.
308: *
309: * 3) We must increment paging_in_progress on any object
310: * for which we have a busy page, to prevent
311: * vm_object_collapse from removing the busy page
312: * without our noticing.
313: *
314: * 4) We leave busy pages on the pageout queues.
315: * If the pageout daemon comes across a busy page,
316: * it will remove the page from the pageout queues.
317: */
318:
319: /*
320: * Search for the page at object/offset.
321: */
322:
323: object = first_object;
324: offset = first_offset;
325: first_m = VM_PAGE_NULL;
326: access_required = fault_type;
327:
328: /*
329: * See whether this page is resident
330: */
331:
332: while (TRUE) {
333: m = vm_page_lookup(object, offset);
334: if (m != VM_PAGE_NULL) {
335: /*
336: * If the page is being brought in,
337: * wait for it and then retry.
338: *
339: * A possible optimization: if the page
340: * is known to be resident, we can ignore
341: * pages that are absent (regardless of
342: * whether they're busy).
343: */
344:
345: if (m->busy) {
346: kern_return_t wait_result;
347:
348: PAGE_ASSERT_WAIT(m, interruptible);
349: vm_object_unlock(object);
350: if (continuation != (void (*)()) 0) {
1.1.1.4 root 351: vm_fault_state_t *state =
1.1 root 352: (vm_fault_state_t *) current_thread()->ith_other;
353:
354: /*
355: * Save variables in case
356: * thread_block discards
357: * our kernel stack.
358: */
359:
360: state->vmfp_backoff = FALSE;
361: state->vmfp_object = object;
362: state->vmfp_offset = offset;
363: state->vmfp_first_m = first_m;
364: state->vmfp_access =
365: access_required;
366: state->vmf_prot = *protection;
367:
368: counter(c_vm_fault_page_block_busy_user++);
369: thread_block(continuation);
370: } else
371: {
372: counter(c_vm_fault_page_block_busy_kernel++);
373: thread_block((void (*)()) 0);
374: }
375: after_thread_block:
376: wait_result = current_thread()->wait_result;
377: vm_object_lock(object);
378: if (wait_result != THREAD_AWAKENED) {
379: vm_fault_cleanup(object, first_m);
380: if (wait_result == THREAD_RESTART)
381: return(VM_FAULT_RETRY);
382: else
383: return(VM_FAULT_INTERRUPTED);
384: }
385: continue;
386: }
387:
388: /*
389: * If the page is in error, give up now.
390: */
391:
392: if (m->error) {
393: VM_PAGE_FREE(m);
394: vm_fault_cleanup(object, first_m);
395: return(VM_FAULT_MEMORY_ERROR);
396: }
397:
398: /*
399: * If the page isn't busy, but is absent,
400: * then it was deemed "unavailable".
401: */
402:
403: if (m->absent) {
404: /*
405: * Remove the non-existent page (unless it's
406: * in the top object) and move on down to the
407: * next object (if there is one).
408: */
409:
410: offset += object->shadow_offset;
411: access_required = VM_PROT_READ;
412: next_object = object->shadow;
413: if (next_object == VM_OBJECT_NULL) {
414: vm_page_t real_m;
415:
416: assert(!must_be_resident);
417:
418: /*
419: * Absent page at bottom of shadow
420: * chain; zero fill the page we left
421: * busy in the first object, and flush
422: * the absent page. But first we
423: * need to allocate a real page.
424: */
425:
1.1.1.2 root 426: real_m = vm_page_grab(!object->internal);
1.1 root 427: if (real_m == VM_PAGE_NULL) {
428: vm_fault_cleanup(object, first_m);
429: return(VM_FAULT_MEMORY_SHORTAGE);
430: }
431:
432: if (object != first_object) {
433: VM_PAGE_FREE(m);
434: vm_object_paging_end(object);
435: vm_object_unlock(object);
436: object = first_object;
437: offset = first_offset;
438: m = first_m;
439: first_m = VM_PAGE_NULL;
440: vm_object_lock(object);
441: }
442:
443: VM_PAGE_FREE(m);
444: assert(real_m->busy);
445: vm_page_lock_queues();
446: vm_page_insert(real_m, object, offset);
447: vm_page_unlock_queues();
448: m = real_m;
449:
450: /*
451: * Drop the lock while zero filling
452: * page. Then break because this
453: * is the page we wanted. Checking
454: * the page lock is a waste of time;
455: * this page was either absent or
456: * newly allocated -- in both cases
457: * it can't be page locked by a pager.
458: */
459: vm_object_unlock(object);
460:
461: vm_page_zero_fill(m);
462:
463: vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS);
1.1.1.2 root 464:
1.1 root 465: vm_stat.zero_fill_count++;
1.1.1.3 root 466: current_task()->zero_fills++;
1.1 root 467: vm_object_lock(object);
468: pmap_clear_modify(m->phys_addr);
469: break;
470: } else {
471: if (must_be_resident) {
472: vm_object_paging_end(object);
473: } else if (object != first_object) {
474: vm_object_paging_end(object);
475: VM_PAGE_FREE(m);
476: } else {
477: first_m = m;
478: m->absent = FALSE;
479: vm_object_absent_release(object);
480: m->busy = TRUE;
481:
482: vm_page_lock_queues();
483: VM_PAGE_QUEUES_REMOVE(m);
484: vm_page_unlock_queues();
485: }
486: vm_object_lock(next_object);
487: vm_object_unlock(object);
488: object = next_object;
489: vm_object_paging_begin(object);
490: continue;
491: }
492: }
493:
494: /*
495: * If the desired access to this page has
496: * been locked out, request that it be unlocked.
497: */
498:
499: if (access_required & m->page_lock) {
500: if ((access_required & m->unlock_request) != access_required) {
501: vm_prot_t new_unlock_request;
502: kern_return_t rc;
1.1.1.2 root 503:
1.1 root 504: if (!object->pager_ready) {
505: vm_object_assert_wait(object,
506: VM_OBJECT_EVENT_PAGER_READY,
507: interruptible);
508: goto block_and_backoff;
509: }
510:
511: new_unlock_request = m->unlock_request =
512: (access_required | m->unlock_request);
513: vm_object_unlock(object);
514: if ((rc = memory_object_data_unlock(
515: object->pager,
516: object->pager_request,
517: offset + object->paging_offset,
518: PAGE_SIZE,
519: new_unlock_request))
520: != KERN_SUCCESS) {
521: printf("vm_fault: memory_object_data_unlock failed\n");
522: vm_object_lock(object);
523: vm_fault_cleanup(object, first_m);
524: return((rc == MACH_SEND_INTERRUPTED) ?
525: VM_FAULT_INTERRUPTED :
526: VM_FAULT_MEMORY_ERROR);
527: }
528: vm_object_lock(object);
529: continue;
530: }
531:
532: PAGE_ASSERT_WAIT(m, interruptible);
533: goto block_and_backoff;
534: }
535:
536: /*
537: * We mark the page busy and leave it on
538: * the pageout queues. If the pageout
539: * deamon comes across it, then it will
540: * remove the page.
541: */
542:
543: if (!software_reference_bits) {
544: vm_page_lock_queues();
545: if (m->inactive) {
546: vm_stat_sample(SAMPLED_PC_VM_REACTIVATION_FAULTS);
547: vm_stat.reactivations++;
1.1.1.3 root 548: current_task()->reactivations++;
1.1 root 549: }
550:
551: VM_PAGE_QUEUES_REMOVE(m);
552: vm_page_unlock_queues();
553: }
554:
555: assert(!m->busy);
556: m->busy = TRUE;
557: assert(!m->absent);
558: break;
559: }
560:
561: look_for_page =
562: (object->pager_created)
563: #if MACH_PAGEMAP
564: && (vm_external_state_get(object->existence_info, offset + object->paging_offset) !=
565: VM_EXTERNAL_STATE_ABSENT)
1.1.1.2 root 566: #endif /* MACH_PAGEMAP */
1.1 root 567: ;
568:
569: if ((look_for_page || (object == first_object))
570: && !must_be_resident) {
571: /*
572: * Allocate a new page for this object/offset
573: * pair.
574: */
575:
576: m = vm_page_grab_fictitious();
577: if (m == VM_PAGE_NULL) {
578: vm_fault_cleanup(object, first_m);
579: return(VM_FAULT_FICTITIOUS_SHORTAGE);
580: }
581:
582: vm_page_lock_queues();
583: vm_page_insert(m, object, offset);
584: vm_page_unlock_queues();
585: }
586:
587: if (look_for_page && !must_be_resident) {
588: kern_return_t rc;
589:
590: /*
591: * If the memory manager is not ready, we
592: * cannot make requests.
593: */
594: if (!object->pager_ready) {
595: vm_object_assert_wait(object,
596: VM_OBJECT_EVENT_PAGER_READY,
597: interruptible);
598: VM_PAGE_FREE(m);
599: goto block_and_backoff;
600: }
601:
602: if (object->internal) {
603: /*
604: * Requests to the default pager
605: * must reserve a real page in advance,
606: * because the pager's data-provided
607: * won't block for pages.
608: */
609:
1.1.1.2 root 610: if (m->fictitious && !vm_page_convert(m, FALSE)) {
1.1 root 611: VM_PAGE_FREE(m);
612: vm_fault_cleanup(object, first_m);
613: return(VM_FAULT_MEMORY_SHORTAGE);
614: }
615: } else if (object->absent_count >
616: vm_object_absent_max) {
617: /*
618: * If there are too many outstanding page
619: * requests pending on this object, we
620: * wait for them to be resolved now.
621: */
622:
623: vm_object_absent_assert_wait(object, interruptible);
624: VM_PAGE_FREE(m);
625: goto block_and_backoff;
626: }
627:
628: /*
629: * Indicate that the page is waiting for data
630: * from the memory manager.
631: */
632:
633: m->absent = TRUE;
634: object->absent_count++;
635:
636: /*
637: * We have a busy page, so we can
638: * release the object lock.
639: */
640: vm_object_unlock(object);
641:
642: /*
643: * Call the memory manager to retrieve the data.
644: */
645:
646: vm_stat.pageins++;
647: vm_stat_sample(SAMPLED_PC_VM_PAGEIN_FAULTS);
1.1.1.3 root 648: current_task()->pageins++;
1.1 root 649:
1.1.1.2 root 650: if ((rc = memory_object_data_request(object->pager,
1.1 root 651: object->pager_request,
1.1.1.2 root 652: m->offset + object->paging_offset,
1.1 root 653: PAGE_SIZE, access_required)) != KERN_SUCCESS) {
654: if (rc != MACH_SEND_INTERRUPTED)
1.1.1.3 root 655: printf("%s(0x%p, 0x%p, 0x%lx, 0x%x, 0x%x) failed, %x\n",
1.1 root 656: "memory_object_data_request",
657: object->pager,
658: object->pager_request,
1.1.1.2 root 659: m->offset + object->paging_offset,
1.1 root 660: PAGE_SIZE, access_required, rc);
661: /*
662: * Don't want to leave a busy page around,
663: * but the data request may have blocked,
664: * so check if it's still there and busy.
665: */
666: vm_object_lock(object);
667: if (m == vm_page_lookup(object,offset) &&
668: m->absent && m->busy)
669: VM_PAGE_FREE(m);
670: vm_fault_cleanup(object, first_m);
671: return((rc == MACH_SEND_INTERRUPTED) ?
672: VM_FAULT_INTERRUPTED :
673: VM_FAULT_MEMORY_ERROR);
674: }
1.1.1.2 root 675:
1.1 root 676: /*
677: * Retry with same object/offset, since new data may
678: * be in a different page (i.e., m is meaningless at
679: * this point).
680: */
681: vm_object_lock(object);
682: continue;
683: }
684:
685: /*
686: * For the XP system, the only case in which we get here is if
687: * object has no pager (or unwiring). If the pager doesn't
688: * have the page this is handled in the m->absent case above
689: * (and if you change things here you should look above).
690: */
691: if (object == first_object)
692: first_m = m;
693: else
694: {
695: assert(m == VM_PAGE_NULL);
696: }
697:
698: /*
699: * Move on to the next object. Lock the next
700: * object before unlocking the current one.
701: */
702: access_required = VM_PROT_READ;
703:
704: offset += object->shadow_offset;
705: next_object = object->shadow;
706: if (next_object == VM_OBJECT_NULL) {
707: assert(!must_be_resident);
708:
709: /*
710: * If there's no object left, fill the page
711: * in the top object with zeros. But first we
712: * need to allocate a real page.
713: */
714:
715: if (object != first_object) {
716: vm_object_paging_end(object);
717: vm_object_unlock(object);
718:
719: object = first_object;
720: offset = first_offset;
721: vm_object_lock(object);
722: }
723:
724: m = first_m;
725: assert(m->object == object);
726: first_m = VM_PAGE_NULL;
727:
1.1.1.2 root 728: if (m->fictitious && !vm_page_convert(m, !object->internal)) {
1.1 root 729: VM_PAGE_FREE(m);
730: vm_fault_cleanup(object, VM_PAGE_NULL);
731: return(VM_FAULT_MEMORY_SHORTAGE);
732: }
733:
734: vm_object_unlock(object);
735: vm_page_zero_fill(m);
736: vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS);
737: vm_stat.zero_fill_count++;
1.1.1.3 root 738: current_task()->zero_fills++;
1.1 root 739: vm_object_lock(object);
740: pmap_clear_modify(m->phys_addr);
741: break;
742: }
743: else {
744: vm_object_lock(next_object);
745: if ((object != first_object) || must_be_resident)
746: vm_object_paging_end(object);
747: vm_object_unlock(object);
748: object = next_object;
749: vm_object_paging_begin(object);
750: }
751: }
752:
753: /*
754: * PAGE HAS BEEN FOUND.
755: *
756: * This page (m) is:
757: * busy, so that we can play with it;
758: * not absent, so that nobody else will fill it;
759: * possibly eligible for pageout;
760: *
761: * The top-level page (first_m) is:
762: * VM_PAGE_NULL if the page was found in the
763: * top-level object;
764: * busy, not absent, and ineligible for pageout.
765: *
766: * The current object (object) is locked. A paging
767: * reference is held for the current and top-level
768: * objects.
769: */
770:
771: assert(m->busy && !m->absent);
772: assert((first_m == VM_PAGE_NULL) ||
773: (first_m->busy && !first_m->absent &&
774: !first_m->active && !first_m->inactive));
775:
776: /*
777: * If the page is being written, but isn't
778: * already owned by the top-level object,
779: * we have to copy it into a new page owned
780: * by the top-level object.
781: */
782:
783: if (object != first_object) {
784: /*
785: * We only really need to copy if we
786: * want to write it.
787: */
788:
789: if (fault_type & VM_PROT_WRITE) {
790: vm_page_t copy_m;
791:
792: assert(!must_be_resident);
793:
794: /*
795: * If we try to collapse first_object at this
796: * point, we may deadlock when we try to get
797: * the lock on an intermediate object (since we
798: * have the bottom object locked). We can't
799: * unlock the bottom object, because the page
800: * we found may move (by collapse) if we do.
801: *
802: * Instead, we first copy the page. Then, when
803: * we have no more use for the bottom object,
804: * we unlock it and try to collapse.
805: *
806: * Note that we copy the page even if we didn't
807: * need to... that's the breaks.
808: */
809:
810: /*
811: * Allocate a page for the copy
812: */
1.1.1.2 root 813: copy_m = vm_page_grab(!first_object->internal);
1.1 root 814: if (copy_m == VM_PAGE_NULL) {
815: RELEASE_PAGE(m);
816: vm_fault_cleanup(object, first_m);
817: return(VM_FAULT_MEMORY_SHORTAGE);
818: }
819:
820: vm_object_unlock(object);
821: vm_page_copy(m, copy_m);
822: vm_object_lock(object);
823:
824: /*
825: * If another map is truly sharing this
826: * page with us, we have to flush all
827: * uses of the original page, since we
828: * can't distinguish those which want the
829: * original from those which need the
830: * new copy.
831: *
832: * XXXO If we know that only one map has
833: * access to this page, then we could
834: * avoid the pmap_page_protect() call.
835: */
836:
837: vm_page_lock_queues();
838: vm_page_deactivate(m);
839: pmap_page_protect(m->phys_addr, VM_PROT_NONE);
840: vm_page_unlock_queues();
841:
842: /*
843: * We no longer need the old page or object.
844: */
845:
846: PAGE_WAKEUP_DONE(m);
847: vm_object_paging_end(object);
848: vm_object_unlock(object);
849:
850: vm_stat.cow_faults++;
851: vm_stat_sample(SAMPLED_PC_VM_COW_FAULTS);
1.1.1.3 root 852: current_task()->cow_faults++;
1.1 root 853: object = first_object;
854: offset = first_offset;
855:
856: vm_object_lock(object);
857: VM_PAGE_FREE(first_m);
858: first_m = VM_PAGE_NULL;
859: assert(copy_m->busy);
860: vm_page_lock_queues();
861: vm_page_insert(copy_m, object, offset);
862: vm_page_unlock_queues();
863: m = copy_m;
864:
865: /*
866: * Now that we've gotten the copy out of the
867: * way, let's try to collapse the top object.
868: * But we have to play ugly games with
869: * paging_in_progress to do that...
870: */
871:
872: vm_object_paging_end(object);
873: vm_object_collapse(object);
874: vm_object_paging_begin(object);
875: }
876: else {
877: *protection &= (~VM_PROT_WRITE);
878: }
879: }
880:
881: /*
882: * Now check whether the page needs to be pushed into the
883: * copy object. The use of asymmetric copy on write for
884: * shared temporary objects means that we may do two copies to
885: * satisfy the fault; one above to get the page from a
886: * shadowed object, and one here to push it into the copy.
887: */
888:
889: while ((copy_object = first_object->copy) != VM_OBJECT_NULL) {
890: vm_offset_t copy_offset;
891: vm_page_t copy_m;
892:
893: /*
894: * If the page is being written, but hasn't been
895: * copied to the copy-object, we have to copy it there.
896: */
897:
898: if ((fault_type & VM_PROT_WRITE) == 0) {
899: *protection &= ~VM_PROT_WRITE;
900: break;
901: }
902:
903: /*
904: * If the page was guaranteed to be resident,
905: * we must have already performed the copy.
906: */
907:
908: if (must_be_resident)
909: break;
910:
911: /*
912: * Try to get the lock on the copy_object.
913: */
914: if (!vm_object_lock_try(copy_object)) {
915: vm_object_unlock(object);
916:
917: simple_lock_pause(); /* wait a bit */
918:
919: vm_object_lock(object);
920: continue;
921: }
922:
923: /*
924: * Make another reference to the copy-object,
925: * to keep it from disappearing during the
926: * copy.
927: */
928: assert(copy_object->ref_count > 0);
929: copy_object->ref_count++;
930:
931: /*
932: * Does the page exist in the copy?
933: */
934: copy_offset = first_offset - copy_object->shadow_offset;
935: copy_m = vm_page_lookup(copy_object, copy_offset);
936: if (copy_m != VM_PAGE_NULL) {
937: if (copy_m->busy) {
938: /*
939: * If the page is being brought
940: * in, wait for it and then retry.
941: */
942: PAGE_ASSERT_WAIT(copy_m, interruptible);
943: RELEASE_PAGE(m);
944: copy_object->ref_count--;
945: assert(copy_object->ref_count > 0);
946: vm_object_unlock(copy_object);
947: goto block_and_backoff;
948: }
949: }
950: else {
951: /*
952: * Allocate a page for the copy
953: */
954: copy_m = vm_page_alloc(copy_object, copy_offset);
955: if (copy_m == VM_PAGE_NULL) {
956: RELEASE_PAGE(m);
957: copy_object->ref_count--;
958: assert(copy_object->ref_count > 0);
959: vm_object_unlock(copy_object);
960: vm_fault_cleanup(object, first_m);
961: return(VM_FAULT_MEMORY_SHORTAGE);
962: }
963:
964: /*
965: * Must copy page into copy-object.
966: */
967:
968: vm_page_copy(m, copy_m);
1.1.1.2 root 969:
1.1 root 970: /*
971: * If the old page was in use by any users
972: * of the copy-object, it must be removed
973: * from all pmaps. (We can't know which
974: * pmaps use it.)
975: */
976:
977: vm_page_lock_queues();
978: pmap_page_protect(m->phys_addr, VM_PROT_NONE);
979: copy_m->dirty = TRUE;
980: vm_page_unlock_queues();
981:
982: /*
983: * If there's a pager, then immediately
984: * page out this page, using the "initialize"
985: * option. Else, we use the copy.
986: */
987:
988: if (!copy_object->pager_created) {
989: vm_page_lock_queues();
990: vm_page_activate(copy_m);
991: vm_page_unlock_queues();
992: PAGE_WAKEUP_DONE(copy_m);
993: } else {
994: /*
995: * The page is already ready for pageout:
996: * not on pageout queues and busy.
997: * Unlock everything except the
998: * copy_object itself.
999: */
1000:
1001: vm_object_unlock(object);
1002:
1003: /*
1004: * Write the page to the copy-object,
1005: * flushing it from the kernel.
1006: */
1007:
1008: vm_pageout_page(copy_m, TRUE, TRUE);
1009:
1010: /*
1011: * Since the pageout may have
1012: * temporarily dropped the
1013: * copy_object's lock, we
1014: * check whether we'll have
1015: * to deallocate the hard way.
1016: */
1017:
1018: if ((copy_object->shadow != object) ||
1019: (copy_object->ref_count == 1)) {
1020: vm_object_unlock(copy_object);
1021: vm_object_deallocate(copy_object);
1022: vm_object_lock(object);
1023: continue;
1024: }
1025:
1026: /*
1027: * Pick back up the old object's
1028: * lock. [It is safe to do so,
1029: * since it must be deeper in the
1030: * object tree.]
1031: */
1032:
1033: vm_object_lock(object);
1034: }
1035:
1036: /*
1037: * Because we're pushing a page upward
1038: * in the object tree, we must restart
1039: * any faults that are waiting here.
1040: * [Note that this is an expansion of
1041: * PAGE_WAKEUP that uses the THREAD_RESTART
1042: * wait result]. Can't turn off the page's
1043: * busy bit because we're not done with it.
1044: */
1.1.1.2 root 1045:
1.1 root 1046: if (m->wanted) {
1047: m->wanted = FALSE;
1048: thread_wakeup_with_result((event_t) m,
1049: THREAD_RESTART);
1050: }
1051: }
1052:
1053: /*
1054: * The reference count on copy_object must be
1055: * at least 2: one for our extra reference,
1056: * and at least one from the outside world
1057: * (we checked that when we last locked
1058: * copy_object).
1059: */
1060: copy_object->ref_count--;
1061: assert(copy_object->ref_count > 0);
1062: vm_object_unlock(copy_object);
1063:
1064: break;
1065: }
1066:
1067: *result_page = m;
1068: *top_page = first_m;
1069:
1070: /*
1071: * If the page can be written, assume that it will be.
1072: * [Earlier, we restrict the permission to allow write
1073: * access only if the fault so required, so we don't
1074: * mark read-only data as dirty.]
1075: */
1076:
1077: if (vm_fault_dirty_handling && (*protection & VM_PROT_WRITE))
1078: m->dirty = TRUE;
1079:
1080: return(VM_FAULT_SUCCESS);
1081:
1082: block_and_backoff:
1083: vm_fault_cleanup(object, first_m);
1084:
1085: if (continuation != (void (*)()) 0) {
1.1.1.4 root 1086: vm_fault_state_t *state =
1.1 root 1087: (vm_fault_state_t *) current_thread()->ith_other;
1088:
1089: /*
1090: * Save variables in case we must restart.
1091: */
1092:
1093: state->vmfp_backoff = TRUE;
1094: state->vmf_prot = *protection;
1095:
1096: counter(c_vm_fault_page_block_backoff_user++);
1097: thread_block(continuation);
1098: } else
1099: {
1100: counter(c_vm_fault_page_block_backoff_kernel++);
1101: thread_block((void (*)()) 0);
1102: }
1103: after_block_and_backoff:
1104: if (current_thread()->wait_result == THREAD_AWAKENED)
1105: return VM_FAULT_RETRY;
1106: else
1107: return VM_FAULT_INTERRUPTED;
1108:
1109: #undef RELEASE_PAGE
1110: }
1111:
1112: /*
1113: * Routine: vm_fault
1114: * Purpose:
1115: * Handle page faults, including pseudo-faults
1116: * used to change the wiring status of pages.
1117: * Returns:
1118: * If an explicit (expression) continuation is supplied,
1119: * then we call the continuation instead of returning.
1120: * Implementation:
1121: * Explicit continuations make this a little icky,
1122: * because it hasn't been rewritten to embrace CPS.
1123: * Instead, we have resume arguments for vm_fault and
1124: * vm_fault_page, to let continue the fault computation.
1125: *
1126: * vm_fault and vm_fault_page save mucho state
1127: * in the moral equivalent of a closure. The state
1128: * structure is allocated when first entering vm_fault
1129: * and deallocated when leaving vm_fault.
1130: */
1131:
1132: void
1.1.1.4 root 1133: vm_fault_continue(void)
1.1 root 1134: {
1.1.1.4 root 1135: vm_fault_state_t *state =
1.1 root 1136: (vm_fault_state_t *) current_thread()->ith_other;
1137:
1138: (void) vm_fault(state->vmf_map,
1139: state->vmf_vaddr,
1140: state->vmf_fault_type,
1141: state->vmf_change_wiring,
1142: TRUE, state->vmf_continuation);
1143: /*NOTREACHED*/
1144: }
1145:
1.1.1.4 root 1146: kern_return_t vm_fault(
1147: vm_map_t map,
1148: vm_offset_t vaddr,
1149: vm_prot_t fault_type,
1150: boolean_t change_wiring,
1151: boolean_t resume,
1152: void (*continuation)())
1.1 root 1153: {
1154: vm_map_version_t version; /* Map version for verificiation */
1155: boolean_t wired; /* Should mapping be wired down? */
1156: vm_object_t object; /* Top-level object */
1157: vm_offset_t offset; /* Top-level offset */
1158: vm_prot_t prot; /* Protection for mapping */
1159: vm_object_t old_copy_object; /* Saved copy object */
1160: vm_page_t result_page; /* Result of vm_fault_page */
1161: vm_page_t top_page; /* Placeholder page */
1162: kern_return_t kr;
1163:
1164: vm_page_t m; /* Fast access to result_page */
1165:
1166: if (resume) {
1.1.1.4 root 1167: vm_fault_state_t *state =
1.1 root 1168: (vm_fault_state_t *) current_thread()->ith_other;
1169:
1170: /*
1171: * Retrieve cached variables and
1172: * continue vm_fault_page.
1173: */
1174:
1175: object = state->vmf_object;
1176: if (object == VM_OBJECT_NULL)
1177: goto RetryFault;
1178: version = state->vmf_version;
1179: wired = state->vmf_wired;
1180: offset = state->vmf_offset;
1181: prot = state->vmf_prot;
1182:
1183: kr = vm_fault_page(object, offset, fault_type,
1184: (change_wiring && !wired), !change_wiring,
1185: &prot, &result_page, &top_page,
1186: TRUE, vm_fault_continue);
1187: goto after_vm_fault_page;
1188: }
1189:
1190: if (continuation != (void (*)()) 0) {
1191: /*
1192: * We will probably need to save state.
1193: */
1194:
1195: char * state;
1196:
1197: /*
1198: * if this assignment stmt is written as
1.1.1.3 root 1199: * 'active_threads[cpu_number()] = kmem_cache_alloc()',
1200: * cpu_number may be evaluated before kmem_cache_alloc;
1201: * if kmem_cache_alloc blocks, cpu_number will be wrong
1.1 root 1202: */
1203:
1.1.1.3 root 1204: state = (char *) kmem_cache_alloc(&vm_fault_state_cache);
1.1 root 1205: current_thread()->ith_other = state;
1206:
1207: }
1208:
1209: RetryFault: ;
1210:
1211: /*
1212: * Find the backing store object and offset into
1213: * it to begin the search.
1214: */
1215:
1216: if ((kr = vm_map_lookup(&map, vaddr, fault_type, &version,
1217: &object, &offset,
1218: &prot, &wired)) != KERN_SUCCESS) {
1219: goto done;
1220: }
1221:
1222: /*
1223: * If the page is wired, we must fault for the current protection
1224: * value, to avoid further faults.
1225: */
1226:
1227: if (wired)
1228: fault_type = prot;
1229:
1230: /*
1231: * Make a reference to this object to
1232: * prevent its disposal while we are messing with
1233: * it. Once we have the reference, the map is free
1234: * to be diddled. Since objects reference their
1235: * shadows (and copies), they will stay around as well.
1236: */
1237:
1238: assert(object->ref_count > 0);
1239: object->ref_count++;
1240: vm_object_paging_begin(object);
1241:
1242: if (continuation != (void (*)()) 0) {
1.1.1.4 root 1243: vm_fault_state_t *state =
1.1 root 1244: (vm_fault_state_t *) current_thread()->ith_other;
1245:
1246: /*
1247: * Save variables, in case vm_fault_page discards
1248: * our kernel stack and we have to restart.
1249: */
1250:
1251: state->vmf_map = map;
1252: state->vmf_vaddr = vaddr;
1253: state->vmf_fault_type = fault_type;
1254: state->vmf_change_wiring = change_wiring;
1255: state->vmf_continuation = continuation;
1256:
1257: state->vmf_version = version;
1258: state->vmf_wired = wired;
1259: state->vmf_object = object;
1260: state->vmf_offset = offset;
1261: state->vmf_prot = prot;
1262:
1263: kr = vm_fault_page(object, offset, fault_type,
1264: (change_wiring && !wired), !change_wiring,
1265: &prot, &result_page, &top_page,
1266: FALSE, vm_fault_continue);
1267: } else
1268: {
1269: kr = vm_fault_page(object, offset, fault_type,
1270: (change_wiring && !wired), !change_wiring,
1271: &prot, &result_page, &top_page,
1272: FALSE, (void (*)()) 0);
1273: }
1274: after_vm_fault_page:
1275:
1276: /*
1277: * If we didn't succeed, lose the object reference immediately.
1278: */
1279:
1280: if (kr != VM_FAULT_SUCCESS)
1281: vm_object_deallocate(object);
1282:
1283: /*
1284: * See why we failed, and take corrective action.
1285: */
1286:
1287: switch (kr) {
1288: case VM_FAULT_SUCCESS:
1289: break;
1290: case VM_FAULT_RETRY:
1291: goto RetryFault;
1292: case VM_FAULT_INTERRUPTED:
1293: kr = KERN_SUCCESS;
1294: goto done;
1295: case VM_FAULT_MEMORY_SHORTAGE:
1296: if (continuation != (void (*)()) 0) {
1.1.1.4 root 1297: vm_fault_state_t *state =
1.1 root 1298: (vm_fault_state_t *) current_thread()->ith_other;
1299:
1300: /*
1301: * Save variables in case VM_PAGE_WAIT
1302: * discards our kernel stack.
1303: */
1304:
1305: state->vmf_map = map;
1306: state->vmf_vaddr = vaddr;
1307: state->vmf_fault_type = fault_type;
1308: state->vmf_change_wiring = change_wiring;
1309: state->vmf_continuation = continuation;
1310: state->vmf_object = VM_OBJECT_NULL;
1311:
1312: VM_PAGE_WAIT(vm_fault_continue);
1313: } else
1314: VM_PAGE_WAIT((void (*)()) 0);
1315: goto RetryFault;
1316: case VM_FAULT_FICTITIOUS_SHORTAGE:
1317: vm_page_more_fictitious();
1318: goto RetryFault;
1319: case VM_FAULT_MEMORY_ERROR:
1320: kr = KERN_MEMORY_ERROR;
1321: goto done;
1322: }
1323:
1324: m = result_page;
1325:
1326: assert((change_wiring && !wired) ?
1327: (top_page == VM_PAGE_NULL) :
1328: ((top_page == VM_PAGE_NULL) == (m->object == object)));
1329:
1330: /*
1331: * How to clean up the result of vm_fault_page. This
1332: * happens whether the mapping is entered or not.
1333: */
1334:
1335: #define UNLOCK_AND_DEALLOCATE \
1336: MACRO_BEGIN \
1337: vm_fault_cleanup(m->object, top_page); \
1338: vm_object_deallocate(object); \
1339: MACRO_END
1340:
1341: /*
1342: * What to do with the resulting page from vm_fault_page
1343: * if it doesn't get entered into the physical map:
1344: */
1345:
1346: #define RELEASE_PAGE(m) \
1347: MACRO_BEGIN \
1348: PAGE_WAKEUP_DONE(m); \
1349: vm_page_lock_queues(); \
1350: if (!m->active && !m->inactive) \
1351: vm_page_activate(m); \
1352: vm_page_unlock_queues(); \
1353: MACRO_END
1354:
1355: /*
1356: * We must verify that the maps have not changed
1357: * since our last lookup.
1358: */
1359:
1360: old_copy_object = m->object->copy;
1361:
1362: vm_object_unlock(m->object);
1363: while (!vm_map_verify(map, &version)) {
1364: vm_object_t retry_object;
1365: vm_offset_t retry_offset;
1366: vm_prot_t retry_prot;
1367:
1368: /*
1369: * To avoid trying to write_lock the map while another
1370: * thread has it read_locked (in vm_map_pageable), we
1371: * do not try for write permission. If the page is
1372: * still writable, we will get write permission. If it
1373: * is not, or has been marked needs_copy, we enter the
1374: * mapping without write permission, and will merely
1375: * take another fault.
1376: */
1377: kr = vm_map_lookup(&map, vaddr,
1378: fault_type & ~VM_PROT_WRITE, &version,
1379: &retry_object, &retry_offset, &retry_prot,
1380: &wired);
1381:
1382: if (kr != KERN_SUCCESS) {
1383: vm_object_lock(m->object);
1384: RELEASE_PAGE(m);
1385: UNLOCK_AND_DEALLOCATE;
1386: goto done;
1387: }
1388:
1389: vm_object_unlock(retry_object);
1390: vm_object_lock(m->object);
1391:
1392: if ((retry_object != object) ||
1393: (retry_offset != offset)) {
1394: RELEASE_PAGE(m);
1395: UNLOCK_AND_DEALLOCATE;
1396: goto RetryFault;
1397: }
1398:
1399: /*
1400: * Check whether the protection has changed or the object
1401: * has been copied while we left the map unlocked.
1402: */
1403: prot &= retry_prot;
1404: vm_object_unlock(m->object);
1405: }
1406: vm_object_lock(m->object);
1407:
1408: /*
1409: * If the copy object changed while the top-level object
1410: * was unlocked, then we must take away write permission.
1411: */
1412:
1413: if (m->object->copy != old_copy_object)
1414: prot &= ~VM_PROT_WRITE;
1415:
1416: /*
1417: * If we want to wire down this page, but no longer have
1418: * adequate permissions, we must start all over.
1419: */
1420:
1421: if (wired && (prot != fault_type)) {
1422: vm_map_verify_done(map, &version);
1423: RELEASE_PAGE(m);
1424: UNLOCK_AND_DEALLOCATE;
1425: goto RetryFault;
1426: }
1427:
1428: /*
1429: * It's critically important that a wired-down page be faulted
1430: * only once in each map for which it is wired.
1431: */
1432:
1433: vm_object_unlock(m->object);
1434:
1435: /*
1436: * Put this page into the physical map.
1437: * We had to do the unlock above because pmap_enter
1438: * may cause other faults. The page may be on
1439: * the pageout queues. If the pageout daemon comes
1440: * across the page, it will remove it from the queues.
1441: */
1442:
1443: PMAP_ENTER(map->pmap, vaddr, m, prot, wired);
1444:
1445: /*
1446: * If the page is not wired down and isn't already
1447: * on a pageout queue, then put it where the
1448: * pageout daemon can find it.
1449: */
1450: vm_object_lock(m->object);
1451: vm_page_lock_queues();
1452: if (change_wiring) {
1453: if (wired)
1454: vm_page_wire(m);
1455: else
1456: vm_page_unwire(m);
1457: } else if (software_reference_bits) {
1458: if (!m->active && !m->inactive)
1459: vm_page_activate(m);
1460: m->reference = TRUE;
1461: } else {
1462: vm_page_activate(m);
1463: }
1464: vm_page_unlock_queues();
1465:
1466: /*
1467: * Unlock everything, and return
1468: */
1469:
1470: vm_map_verify_done(map, &version);
1471: PAGE_WAKEUP_DONE(m);
1472: kr = KERN_SUCCESS;
1473: UNLOCK_AND_DEALLOCATE;
1474:
1475: #undef UNLOCK_AND_DEALLOCATE
1476: #undef RELEASE_PAGE
1477:
1478: done:
1479: if (continuation != (void (*)()) 0) {
1.1.1.4 root 1480: vm_fault_state_t *state =
1.1 root 1481: (vm_fault_state_t *) current_thread()->ith_other;
1482:
1.1.1.3 root 1483: kmem_cache_free(&vm_fault_state_cache, (vm_offset_t) state);
1.1 root 1484: (*continuation)(kr);
1485: /*NOTREACHED*/
1486: }
1487:
1488: return(kr);
1489: }
1490:
1491: /*
1492: * vm_fault_wire:
1493: *
1494: * Wire down a range of virtual addresses in a map.
1495: */
1.1.1.4 root 1496: void vm_fault_wire(
1497: vm_map_t map,
1498: vm_map_entry_t entry)
1.1 root 1499: {
1500:
1.1.1.4 root 1501: vm_offset_t va;
1502: pmap_t pmap;
1503: vm_offset_t end_addr = entry->vme_end;
1.1 root 1504:
1505: pmap = vm_map_pmap(map);
1506:
1507: /*
1508: * Inform the physical mapping system that the
1509: * range of addresses may not fault, so that
1510: * page tables and such can be locked down as well.
1511: */
1512:
1513: pmap_pageable(pmap, entry->vme_start, end_addr, FALSE);
1514:
1515: /*
1516: * We simulate a fault to get the page and enter it
1517: * in the physical map.
1518: */
1519:
1520: for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
1521: if (vm_fault_wire_fast(map, va, entry) != KERN_SUCCESS)
1522: (void) vm_fault(map, va, VM_PROT_NONE, TRUE,
1523: FALSE, (void (*)()) 0);
1524: }
1525: }
1526:
1527: /*
1528: * vm_fault_unwire:
1529: *
1530: * Unwire a range of virtual addresses in a map.
1531: */
1.1.1.4 root 1532: void vm_fault_unwire(
1533: vm_map_t map,
1534: vm_map_entry_t entry)
1.1 root 1535: {
1.1.1.4 root 1536: vm_offset_t va;
1537: pmap_t pmap;
1538: vm_offset_t end_addr = entry->vme_end;
1539: vm_object_t object;
1.1 root 1540:
1541: pmap = vm_map_pmap(map);
1542:
1543: object = (entry->is_sub_map)
1544: ? VM_OBJECT_NULL : entry->object.vm_object;
1545:
1546: /*
1547: * Since the pages are wired down, we must be able to
1548: * get their mappings from the physical map system.
1549: */
1550:
1551: for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
1552: pmap_change_wiring(pmap, va, FALSE);
1553:
1554: if (object == VM_OBJECT_NULL) {
1555: vm_map_lock_set_recursive(map);
1556: (void) vm_fault(map, va, VM_PROT_NONE, TRUE,
1557: FALSE, (void (*)()) 0);
1558: vm_map_lock_clear_recursive(map);
1559: } else {
1560: vm_prot_t prot;
1561: vm_page_t result_page;
1562: vm_page_t top_page;
1563: vm_fault_return_t result;
1564:
1565: do {
1566: prot = VM_PROT_NONE;
1567:
1568: vm_object_lock(object);
1569: vm_object_paging_begin(object);
1570: result = vm_fault_page(object,
1571: entry->offset +
1572: (va - entry->vme_start),
1573: VM_PROT_NONE, TRUE,
1574: FALSE, &prot,
1575: &result_page,
1576: &top_page,
1577: FALSE, (void (*)()) 0);
1578: } while (result == VM_FAULT_RETRY);
1579:
1580: if (result != VM_FAULT_SUCCESS)
1581: panic("vm_fault_unwire: failure");
1582:
1583: vm_page_lock_queues();
1584: vm_page_unwire(result_page);
1585: vm_page_unlock_queues();
1586: PAGE_WAKEUP_DONE(result_page);
1587:
1588: vm_fault_cleanup(result_page->object, top_page);
1589: }
1590: }
1591:
1592: /*
1593: * Inform the physical mapping system that the range
1594: * of addresses may fault, so that page tables and
1595: * such may be unwired themselves.
1596: */
1597:
1598: pmap_pageable(pmap, entry->vme_start, end_addr, TRUE);
1599: }
1600:
1601: /*
1602: * vm_fault_wire_fast:
1603: *
1604: * Handle common case of a wire down page fault at the given address.
1605: * If successful, the page is inserted into the associated physical map.
1606: * The map entry is passed in to avoid the overhead of a map lookup.
1607: *
1608: * NOTE: the given address should be truncated to the
1609: * proper page address.
1610: *
1611: * KERN_SUCCESS is returned if the page fault is handled; otherwise,
1612: * a standard error specifying why the fault is fatal is returned.
1613: *
1614: * The map in question must be referenced, and remains so.
1615: * Caller has a read lock on the map.
1616: *
1617: * This is a stripped version of vm_fault() for wiring pages. Anything
1618: * other than the common case will return KERN_FAILURE, and the caller
1619: * is expected to call vm_fault().
1620: */
1.1.1.4 root 1621: kern_return_t vm_fault_wire_fast(
1622: vm_map_t map,
1623: vm_offset_t va,
1624: vm_map_entry_t entry)
1.1 root 1625: {
1626: vm_object_t object;
1627: vm_offset_t offset;
1.1.1.4 root 1628: vm_page_t m;
1.1 root 1629: vm_prot_t prot;
1630:
1631: vm_stat.faults++; /* needs lock XXX */
1.1.1.3 root 1632: current_task()->faults++;
1.1 root 1633: /*
1634: * Recovery actions
1635: */
1636:
1637: #undef RELEASE_PAGE
1638: #define RELEASE_PAGE(m) { \
1639: PAGE_WAKEUP_DONE(m); \
1640: vm_page_lock_queues(); \
1641: vm_page_unwire(m); \
1642: vm_page_unlock_queues(); \
1643: }
1644:
1645:
1646: #undef UNLOCK_THINGS
1647: #define UNLOCK_THINGS { \
1648: object->paging_in_progress--; \
1649: vm_object_unlock(object); \
1650: }
1651:
1652: #undef UNLOCK_AND_DEALLOCATE
1653: #define UNLOCK_AND_DEALLOCATE { \
1654: UNLOCK_THINGS; \
1655: vm_object_deallocate(object); \
1656: }
1657: /*
1658: * Give up and have caller do things the hard way.
1659: */
1660:
1661: #define GIVE_UP { \
1662: UNLOCK_AND_DEALLOCATE; \
1663: return(KERN_FAILURE); \
1664: }
1665:
1666:
1667: /*
1668: * If this entry is not directly to a vm_object, bail out.
1669: */
1670: if (entry->is_sub_map)
1671: return(KERN_FAILURE);
1672:
1673: /*
1674: * Find the backing store object and offset into it.
1675: */
1676:
1677: object = entry->object.vm_object;
1678: offset = (va - entry->vme_start) + entry->offset;
1679: prot = entry->protection;
1680:
1681: /*
1682: * Make a reference to this object to prevent its
1683: * disposal while we are messing with it.
1684: */
1685:
1686: vm_object_lock(object);
1687: assert(object->ref_count > 0);
1688: object->ref_count++;
1689: object->paging_in_progress++;
1690:
1691: /*
1692: * INVARIANTS (through entire routine):
1693: *
1694: * 1) At all times, we must either have the object
1695: * lock or a busy page in some object to prevent
1696: * some other thread from trying to bring in
1697: * the same page.
1698: *
1699: * 2) Once we have a busy page, we must remove it from
1700: * the pageout queues, so that the pageout daemon
1701: * will not grab it away.
1702: *
1703: */
1704:
1705: /*
1706: * Look for page in top-level object. If it's not there or
1707: * there's something going on, give up.
1708: */
1709: m = vm_page_lookup(object, offset);
1710: if ((m == VM_PAGE_NULL) || (m->error) ||
1711: (m->busy) || (m->absent) || (prot & m->page_lock)) {
1712: GIVE_UP;
1713: }
1714:
1715: /*
1716: * Wire the page down now. All bail outs beyond this
1.1.1.2 root 1717: * point must unwire the page.
1.1 root 1718: */
1719:
1720: vm_page_lock_queues();
1721: vm_page_wire(m);
1722: vm_page_unlock_queues();
1723:
1724: /*
1725: * Mark page busy for other threads.
1726: */
1727: assert(!m->busy);
1728: m->busy = TRUE;
1729: assert(!m->absent);
1730:
1731: /*
1732: * Give up if the page is being written and there's a copy object
1733: */
1734: if ((object->copy != VM_OBJECT_NULL) && (prot & VM_PROT_WRITE)) {
1735: RELEASE_PAGE(m);
1736: GIVE_UP;
1737: }
1738:
1739: /*
1740: * Put this page into the physical map.
1741: * We have to unlock the object because pmap_enter
1.1.1.2 root 1742: * may cause other faults.
1.1 root 1743: */
1744: vm_object_unlock(object);
1745:
1746: PMAP_ENTER(map->pmap, va, m, prot, TRUE);
1747:
1748: /*
1749: * Must relock object so that paging_in_progress can be cleared.
1750: */
1751: vm_object_lock(object);
1752:
1753: /*
1754: * Unlock everything, and return
1755: */
1756:
1757: PAGE_WAKEUP_DONE(m);
1758: UNLOCK_AND_DEALLOCATE;
1759:
1760: return(KERN_SUCCESS);
1761:
1762: }
1763:
1764: /*
1765: * Routine: vm_fault_copy_cleanup
1766: * Purpose:
1767: * Release a page used by vm_fault_copy.
1768: */
1769:
1.1.1.4 root 1770: void vm_fault_copy_cleanup(
1771: vm_page_t page,
1772: vm_page_t top_page)
1.1 root 1773: {
1774: vm_object_t object = page->object;
1775:
1776: vm_object_lock(object);
1777: PAGE_WAKEUP_DONE(page);
1778: vm_page_lock_queues();
1779: if (!page->active && !page->inactive)
1780: vm_page_activate(page);
1781: vm_page_unlock_queues();
1782: vm_fault_cleanup(object, top_page);
1783: }
1784:
1785: /*
1786: * Routine: vm_fault_copy
1787: *
1788: * Purpose:
1789: * Copy pages from one virtual memory object to another --
1790: * neither the source nor destination pages need be resident.
1791: *
1792: * Before actually copying a page, the version associated with
1793: * the destination address map wil be verified.
1794: *
1795: * In/out conditions:
1796: * The caller must hold a reference, but not a lock, to
1797: * each of the source and destination objects and to the
1798: * destination map.
1799: *
1800: * Results:
1801: * Returns KERN_SUCCESS if no errors were encountered in
1802: * reading or writing the data. Returns KERN_INTERRUPTED if
1803: * the operation was interrupted (only possible if the
1804: * "interruptible" argument is asserted). Other return values
1805: * indicate a permanent error in copying the data.
1806: *
1807: * The actual amount of data copied will be returned in the
1808: * "copy_size" argument. In the event that the destination map
1809: * verification failed, this amount may be less than the amount
1810: * requested.
1811: */
1812: kern_return_t vm_fault_copy(
1.1.1.4 root 1813: vm_object_t src_object,
1814: vm_offset_t src_offset,
1815: vm_size_t *src_size, /* INOUT */
1816: vm_object_t dst_object,
1817: vm_offset_t dst_offset,
1818: vm_map_t dst_map,
1819: vm_map_version_t *dst_version,
1820: boolean_t interruptible)
1.1 root 1821: {
1822: vm_page_t result_page;
1823: vm_prot_t prot;
1.1.1.2 root 1824:
1.1 root 1825: vm_page_t src_page;
1826: vm_page_t src_top_page;
1827:
1828: vm_page_t dst_page;
1829: vm_page_t dst_top_page;
1830:
1831: vm_size_t amount_done;
1832: vm_object_t old_copy_object;
1833:
1834: #define RETURN(x) \
1835: MACRO_BEGIN \
1836: *src_size = amount_done; \
1837: MACRO_RETURN(x); \
1838: MACRO_END
1839:
1840: amount_done = 0;
1841: do { /* while (amount_done != *src_size) */
1842:
1843: RetrySourceFault: ;
1844:
1845: if (src_object == VM_OBJECT_NULL) {
1846: /*
1847: * No source object. We will just
1848: * zero-fill the page in dst_object.
1849: */
1850:
1851: src_page = VM_PAGE_NULL;
1852: } else {
1853: prot = VM_PROT_READ;
1854:
1855: vm_object_lock(src_object);
1856: vm_object_paging_begin(src_object);
1857:
1858: switch (vm_fault_page(src_object, src_offset,
1859: VM_PROT_READ, FALSE, interruptible,
1860: &prot, &result_page, &src_top_page,
1861: FALSE, (void (*)()) 0)) {
1862:
1863: case VM_FAULT_SUCCESS:
1864: break;
1865: case VM_FAULT_RETRY:
1866: goto RetrySourceFault;
1867: case VM_FAULT_INTERRUPTED:
1868: RETURN(MACH_SEND_INTERRUPTED);
1869: case VM_FAULT_MEMORY_SHORTAGE:
1870: VM_PAGE_WAIT((void (*)()) 0);
1871: goto RetrySourceFault;
1872: case VM_FAULT_FICTITIOUS_SHORTAGE:
1873: vm_page_more_fictitious();
1874: goto RetrySourceFault;
1875: case VM_FAULT_MEMORY_ERROR:
1876: return(KERN_MEMORY_ERROR);
1877: }
1878:
1879: src_page = result_page;
1880:
1881: assert((src_top_page == VM_PAGE_NULL) ==
1882: (src_page->object == src_object));
1883:
1884: assert ((prot & VM_PROT_READ) != VM_PROT_NONE);
1885:
1886: vm_object_unlock(src_page->object);
1887: }
1888:
1889: RetryDestinationFault: ;
1890:
1891: prot = VM_PROT_WRITE;
1892:
1893: vm_object_lock(dst_object);
1894: vm_object_paging_begin(dst_object);
1895:
1896: switch (vm_fault_page(dst_object, dst_offset, VM_PROT_WRITE,
1897: FALSE, FALSE /* interruptible */,
1898: &prot, &result_page, &dst_top_page,
1899: FALSE, (void (*)()) 0)) {
1900:
1901: case VM_FAULT_SUCCESS:
1902: break;
1903: case VM_FAULT_RETRY:
1904: goto RetryDestinationFault;
1905: case VM_FAULT_INTERRUPTED:
1906: if (src_page != VM_PAGE_NULL)
1907: vm_fault_copy_cleanup(src_page,
1908: src_top_page);
1909: RETURN(MACH_SEND_INTERRUPTED);
1910: case VM_FAULT_MEMORY_SHORTAGE:
1911: VM_PAGE_WAIT((void (*)()) 0);
1912: goto RetryDestinationFault;
1913: case VM_FAULT_FICTITIOUS_SHORTAGE:
1914: vm_page_more_fictitious();
1915: goto RetryDestinationFault;
1916: case VM_FAULT_MEMORY_ERROR:
1917: if (src_page != VM_PAGE_NULL)
1918: vm_fault_copy_cleanup(src_page,
1919: src_top_page);
1920: return(KERN_MEMORY_ERROR);
1921: }
1922: assert ((prot & VM_PROT_WRITE) != VM_PROT_NONE);
1923:
1924: dst_page = result_page;
1925:
1926: old_copy_object = dst_page->object->copy;
1927:
1928: vm_object_unlock(dst_page->object);
1929:
1930: if (!vm_map_verify(dst_map, dst_version)) {
1931:
1932: BailOut: ;
1933:
1934: if (src_page != VM_PAGE_NULL)
1935: vm_fault_copy_cleanup(src_page, src_top_page);
1936: vm_fault_copy_cleanup(dst_page, dst_top_page);
1937: break;
1938: }
1939:
1940:
1941: vm_object_lock(dst_page->object);
1942: if (dst_page->object->copy != old_copy_object) {
1943: vm_object_unlock(dst_page->object);
1944: vm_map_verify_done(dst_map, dst_version);
1945: goto BailOut;
1946: }
1947: vm_object_unlock(dst_page->object);
1948:
1949: /*
1950: * Copy the page, and note that it is dirty
1951: * immediately.
1952: */
1953:
1954: if (src_page == VM_PAGE_NULL)
1955: vm_page_zero_fill(dst_page);
1956: else
1957: vm_page_copy(src_page, dst_page);
1958: dst_page->dirty = TRUE;
1959:
1960: /*
1961: * Unlock everything, and return
1962: */
1963:
1964: vm_map_verify_done(dst_map, dst_version);
1965:
1966: if (src_page != VM_PAGE_NULL)
1967: vm_fault_copy_cleanup(src_page, src_top_page);
1968: vm_fault_copy_cleanup(dst_page, dst_top_page);
1969:
1970: amount_done += PAGE_SIZE;
1971: src_offset += PAGE_SIZE;
1972: dst_offset += PAGE_SIZE;
1973:
1974: } while (amount_done != *src_size);
1975:
1976: RETURN(KERN_SUCCESS);
1977: #undef RETURN
1978:
1.1.1.2 root 1979: /*NOTREACHED*/
1.1 root 1980: }
1981:
1982:
1983:
1984:
1985:
1986: #ifdef notdef
1987:
1988: /*
1989: * Routine: vm_fault_page_overwrite
1990: *
1991: * Description:
1992: * A form of vm_fault_page that assumes that the
1993: * resulting page will be overwritten in its entirety,
1994: * making it unnecessary to obtain the correct *contents*
1995: * of the page.
1996: *
1997: * Implementation:
1998: * XXX Untested. Also unused. Eventually, this technology
1999: * could be used in vm_fault_copy() to advantage.
2000: */
1.1.1.4 root 2001: vm_fault_return_t vm_fault_page_overwrite(
2002: vm_object_t dst_object,
2003: vm_offset_t dst_offset,
2004: vm_page_t *result_page) /* OUT */
1.1 root 2005: {
2006: vm_page_t dst_page;
2007:
2008: #define interruptible FALSE /* XXX */
2009:
2010: while (TRUE) {
2011: /*
2012: * Look for a page at this offset
2013: */
2014:
2015: while ((dst_page = vm_page_lookup(dst_object, dst_offset))
2016: == VM_PAGE_NULL) {
2017: /*
2018: * No page, no problem... just allocate one.
2019: */
2020:
2021: dst_page = vm_page_alloc(dst_object, dst_offset);
2022: if (dst_page == VM_PAGE_NULL) {
2023: vm_object_unlock(dst_object);
2024: VM_PAGE_WAIT((void (*)()) 0);
2025: vm_object_lock(dst_object);
2026: continue;
2027: }
2028:
2029: /*
2030: * Pretend that the memory manager
2031: * write-protected the page.
2032: *
2033: * Note that we will be asking for write
2034: * permission without asking for the data
2035: * first.
2036: */
2037:
2038: dst_page->overwriting = TRUE;
2039: dst_page->page_lock = VM_PROT_WRITE;
2040: dst_page->absent = TRUE;
2041: dst_object->absent_count++;
2042:
2043: break;
2044:
2045: /*
2046: * When we bail out, we might have to throw
2047: * away the page created here.
2048: */
2049:
2050: #define DISCARD_PAGE \
2051: MACRO_BEGIN \
2052: vm_object_lock(dst_object); \
2053: dst_page = vm_page_lookup(dst_object, dst_offset); \
2054: if ((dst_page != VM_PAGE_NULL) && dst_page->overwriting) \
2055: VM_PAGE_FREE(dst_page); \
2056: vm_object_unlock(dst_object); \
2057: MACRO_END
2058: }
2059:
2060: /*
2061: * If the page is write-protected...
2062: */
2063:
2064: if (dst_page->page_lock & VM_PROT_WRITE) {
2065: /*
2066: * ... and an unlock request hasn't been sent
2067: */
2068:
2069: if ( ! (dst_page->unlock_request & VM_PROT_WRITE)) {
2070: vm_prot_t u;
2071: kern_return_t rc;
2072:
2073: /*
2074: * ... then send one now.
2075: */
2076:
2077: if (!dst_object->pager_ready) {
2078: vm_object_assert_wait(dst_object,
2079: VM_OBJECT_EVENT_PAGER_READY,
2080: interruptible);
2081: vm_object_unlock(dst_object);
2082: thread_block((void (*)()) 0);
2083: if (current_thread()->wait_result !=
2084: THREAD_AWAKENED) {
2085: DISCARD_PAGE;
2086: return(VM_FAULT_INTERRUPTED);
2087: }
2088: continue;
2089: }
2090:
2091: u = dst_page->unlock_request |= VM_PROT_WRITE;
2092: vm_object_unlock(dst_object);
2093:
2094: if ((rc = memory_object_data_unlock(
2095: dst_object->pager,
2096: dst_object->pager_request,
2097: dst_offset + dst_object->paging_offset,
2098: PAGE_SIZE,
2099: u)) != KERN_SUCCESS) {
2100: printf("vm_object_overwrite: memory_object_data_unlock failed\n");
2101: DISCARD_PAGE;
2102: return((rc == MACH_SEND_INTERRUPTED) ?
2103: VM_FAULT_INTERRUPTED :
2104: VM_FAULT_MEMORY_ERROR);
2105: }
2106: vm_object_lock(dst_object);
2107: continue;
2108: }
2109:
2110: /* ... fall through to wait below */
2111: } else {
2112: /*
2113: * If the page isn't being used for other
2114: * purposes, then we're done.
2115: */
2116: if ( ! (dst_page->busy || dst_page->absent || dst_page->error) )
2117: break;
2118: }
2119:
2120: PAGE_ASSERT_WAIT(dst_page, interruptible);
2121: vm_object_unlock(dst_object);
2122: thread_block((void (*)()) 0);
2123: if (current_thread()->wait_result != THREAD_AWAKENED) {
2124: DISCARD_PAGE;
2125: return(VM_FAULT_INTERRUPTED);
2126: }
2127: }
2128:
2129: *result_page = dst_page;
2130: return(VM_FAULT_SUCCESS);
2131:
2132: #undef interruptible
2133: #undef DISCARD_PAGE
2134: }
2135:
1.1.1.2 root 2136: #endif /* notdef */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.