![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_fault.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / OSKit-Mach / vm|
Return to vm_fault.c CVS log | Up to [GnuMach] / OSKit-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1994,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm_fault.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: *
33: * Page fault handling module.
34: */
35: #include <mach_pagemap.h>
36: #include <mach_kdb.h>
37: #include <mach_pcsample.h>
38:
39:
40: #include <vm/vm_fault.h>
41: #include <mach/kern_return.h>
42: #include <mach/message.h> /* for error codes */
43: #include <kern/counters.h>
44: #include <kern/thread.h>
45: #include <kern/sched_prim.h>
46: #include <vm/vm_map.h>
47: #include <vm/vm_object.h>
48: #include <vm/vm_page.h>
49: #include <vm/pmap.h>
50: #include <mach/vm_statistics.h>
51: #include <vm/vm_pageout.h>
52: #include <mach/vm_param.h>
53: #include <mach/memory_object.h>
54: #include "memory_object_user.h"
55: /* For memory_object_data_{request,unlock} */
56: #include <kern/mach_param.h>
57: #include <kern/macro_help.h>
58: #include <kern/zalloc.h>
59:
60: #if MACH_PCSAMPLE
61: #include <kern/pc_sample.h>
62: #endif
63:
64:
65:
66: /*
67: * State needed by vm_fault_continue.
68: * This is a little hefty to drop directly
69: * into the thread structure.
70: */
71: typedef struct vm_fault_state {
72: struct vm_map *vmf_map;
73: vm_offset_t vmf_vaddr;
74: vm_prot_t vmf_fault_type;
75: boolean_t vmf_change_wiring;
76: void (*vmf_continuation)();
77: vm_map_version_t vmf_version;
78: boolean_t vmf_wired;
79: struct vm_object *vmf_object;
80: vm_offset_t vmf_offset;
81: vm_prot_t vmf_prot;
82:
83: boolean_t vmfp_backoff;
84: struct vm_object *vmfp_object;
85: vm_offset_t vmfp_offset;
86: struct vm_page *vmfp_first_m;
87: vm_prot_t vmfp_access;
88: } vm_fault_state_t;
89:
90: zone_t vm_fault_state_zone = 0;
91:
92: int vm_object_absent_max = 50;
93:
94: int vm_fault_debug = 0;
95:
96: boolean_t vm_fault_dirty_handling = FALSE;
97: boolean_t vm_fault_interruptible = TRUE;
98:
99: boolean_t software_reference_bits = TRUE;
100:
101: #if MACH_KDB
102: extern struct db_watchpoint *db_watchpoint_list;
103: #endif /* MACH_KDB */
104:
105: /*
106: * Routine: vm_fault_init
107: * Purpose:
108: * Initialize our private data structures.
109: */
110: void vm_fault_init()
111: {
112: vm_fault_state_zone = zinit(sizeof(vm_fault_state_t),
113: THREAD_MAX * sizeof(vm_fault_state_t),
114: sizeof(vm_fault_state_t),
115: 0, "vm fault state");
116: }
117:
118: /*
119: * Routine: vm_fault_cleanup
120: * Purpose:
121: * Clean up the result of vm_fault_page.
122: * Results:
123: * The paging reference for "object" is released.
124: * "object" is unlocked.
125: * If "top_page" is not null, "top_page" is
126: * freed and the paging reference for the object
127: * containing it is released.
128: *
129: * In/out conditions:
130: * "object" must be locked.
131: */
132: void
133: vm_fault_cleanup(object, top_page)
134: register vm_object_t object;
135: register vm_page_t top_page;
136: {
137: vm_object_paging_end(object);
138: vm_object_unlock(object);
139:
140: if (top_page != VM_PAGE_NULL) {
141: object = top_page->object;
142: vm_object_lock(object);
143: VM_PAGE_FREE(top_page);
144: vm_object_paging_end(object);
145: vm_object_unlock(object);
146: }
147: }
148:
149:
150: #if MACH_PCSAMPLE
151: /*
152: * Do PC sampling on current thread, assuming
153: * that it is the thread taking this page fault.
154: *
155: * Must check for THREAD_NULL, since faults
156: * can occur before threads are running.
157: */
158:
159: #define vm_stat_sample(flavor) \
160: MACRO_BEGIN \
161: thread_t _thread_ = current_thread(); \
162: \
163: if (_thread_ != THREAD_NULL) \
164: take_pc_sample_macro(_thread_, (flavor)); \
165: MACRO_END
166:
167: #else
168: #define vm_stat_sample(x)
169: #endif /* MACH_PCSAMPLE */
170:
171:
172:
173: /*
174: * Routine: vm_fault_page
175: * Purpose:
176: * Find the resident page for the virtual memory
177: * specified by the given virtual memory object
178: * and offset.
179: * Additional arguments:
180: * The required permissions for the page is given
181: * in "fault_type". Desired permissions are included
182: * in "protection".
183: *
184: * If the desired page is known to be resident (for
185: * example, because it was previously wired down), asserting
186: * the "unwiring" parameter will speed the search.
187: *
188: * If the operation can be interrupted (by thread_abort
189: * or thread_terminate), then the "interruptible"
190: * parameter should be asserted.
191: *
192: * Results:
193: * The page containing the proper data is returned
194: * in "result_page".
195: *
196: * In/out conditions:
197: * The source object must be locked and referenced,
198: * and must donate one paging reference. The reference
199: * is not affected. The paging reference and lock are
200: * consumed.
201: *
202: * If the call succeeds, the object in which "result_page"
203: * resides is left locked and holding a paging reference.
204: * If this is not the original object, a busy page in the
205: * original object is returned in "top_page", to prevent other
206: * callers from pursuing this same data, along with a paging
207: * reference for the original object. The "top_page" should
208: * be destroyed when this guarantee is no longer required.
209: * The "result_page" is also left busy. It is not removed
210: * from the pageout queues.
211: */
212: vm_fault_return_t vm_fault_page(first_object, first_offset,
213: fault_type, must_be_resident, interruptible,
214: protection,
215: result_page, top_page,
216: resume, continuation)
217: /* Arguments: */
218: vm_object_t first_object; /* Object to begin search */
219: vm_offset_t first_offset; /* Offset into object */
220: vm_prot_t fault_type; /* What access is requested */
221: boolean_t must_be_resident;/* Must page be resident? */
222: boolean_t interruptible; /* May fault be interrupted? */
223: /* Modifies in place: */
224: vm_prot_t *protection; /* Protection for mapping */
225: /* Returns: */
226: vm_page_t *result_page; /* Page found, if successful */
227: vm_page_t *top_page; /* Page in top object, if
228: * not result_page.
229: */
230: /* More arguments: */
231: boolean_t resume; /* We are restarting. */
232: void (*continuation)(); /* Continuation for blocking. */
233: {
234: register
235: vm_page_t m;
236: register
237: vm_object_t object;
238: register
239: vm_offset_t offset;
240: vm_page_t first_m;
241: vm_object_t next_object;
242: vm_object_t copy_object;
243: boolean_t look_for_page;
244: vm_prot_t access_required;
245:
246: #ifdef CONTINUATIONS
247: if (resume) {
248: register vm_fault_state_t *state =
249: (vm_fault_state_t *) current_thread()->ith_other;
250:
251: if (state->vmfp_backoff)
252: goto after_block_and_backoff;
253:
254: object = state->vmfp_object;
255: offset = state->vmfp_offset;
256: first_m = state->vmfp_first_m;
257: access_required = state->vmfp_access;
258: goto after_thread_block;
259: }
260: #else /* not CONTINUATIONS */
261: assert(continuation == 0);
262: assert(!resume);
263: #endif /* not CONTINUATIONS */
264:
265: vm_stat_sample(SAMPLED_PC_VM_FAULTS_ANY);
266: vm_stat.faults++; /* needs lock XXX */
267:
268: /*
269: * Recovery actions
270: */
271: #define RELEASE_PAGE(m) \
272: MACRO_BEGIN \
273: PAGE_WAKEUP_DONE(m); \
274: vm_page_lock_queues(); \
275: if (!m->active && !m->inactive) \
276: vm_page_activate(m); \
277: vm_page_unlock_queues(); \
278: MACRO_END
279:
280: if (vm_fault_dirty_handling
281: #if MACH_KDB
282: /*
283: * If there are watchpoints set, then
284: * we don't want to give away write permission
285: * on a read fault. Make the task write fault,
286: * so that the watchpoint code notices the access.
287: */
288: || db_watchpoint_list
289: #endif /* MACH_KDB */
290: ) {
291: /*
292: * If we aren't asking for write permission,
293: * then don't give it away. We're using write
294: * faults to set the dirty bit.
295: */
296: if (!(fault_type & VM_PROT_WRITE))
297: *protection &= ~VM_PROT_WRITE;
298: }
299:
300: if (!vm_fault_interruptible)
301: interruptible = FALSE;
302:
303: /*
304: * INVARIANTS (through entire routine):
305: *
306: * 1) At all times, we must either have the object
307: * lock or a busy page in some object to prevent
308: * some other thread from trying to bring in
309: * the same page.
310: *
311: * Note that we cannot hold any locks during the
312: * pager access or when waiting for memory, so
313: * we use a busy page then.
314: *
315: * Note also that we aren't as concerned about more than
316: * one thread attempting to memory_object_data_unlock
317: * the same page at once, so we don't hold the page
318: * as busy then, but do record the highest unlock
319: * value so far. [Unlock requests may also be delivered
320: * out of order.]
321: *
322: * 2) To prevent another thread from racing us down the
323: * shadow chain and entering a new page in the top
324: * object before we do, we must keep a busy page in
325: * the top object while following the shadow chain.
326: *
327: * 3) We must increment paging_in_progress on any object
328: * for which we have a busy page, to prevent
329: * vm_object_collapse from removing the busy page
330: * without our noticing.
331: *
332: * 4) We leave busy pages on the pageout queues.
333: * If the pageout daemon comes across a busy page,
334: * it will remove the page from the pageout queues.
335: */
336:
337: /*
338: * Search for the page at object/offset.
339: */
340:
341: object = first_object;
342: offset = first_offset;
343: first_m = VM_PAGE_NULL;
344: access_required = fault_type;
345:
346: /*
347: * See whether this page is resident
348: */
349:
350: while (TRUE) {
351: m = vm_page_lookup(object, offset);
352: if (m != VM_PAGE_NULL) {
353: /*
354: * If the page is being brought in,
355: * wait for it and then retry.
356: *
357: * A possible optimization: if the page
358: * is known to be resident, we can ignore
359: * pages that are absent (regardless of
360: * whether they're busy).
361: */
362:
363: if (m->busy) {
364: kern_return_t wait_result;
365:
366: PAGE_ASSERT_WAIT(m, interruptible);
367: vm_object_unlock(object);
368: #ifdef CONTINUATIONS
369: if (continuation != (void (*)()) 0) {
370: register vm_fault_state_t *state =
371: (vm_fault_state_t *) current_thread()->ith_other;
372:
373: /*
374: * Save variables in case
375: * thread_block discards
376: * our kernel stack.
377: */
378:
379: state->vmfp_backoff = FALSE;
380: state->vmfp_object = object;
381: state->vmfp_offset = offset;
382: state->vmfp_first_m = first_m;
383: state->vmfp_access =
384: access_required;
385: state->vmf_prot = *protection;
386:
387: counter(c_vm_fault_page_block_busy_user++);
388: thread_block(continuation);
389: } else
390: #endif /* CONTINUATIONS */
391: {
392: counter(c_vm_fault_page_block_busy_kernel++);
393: thread_block((void (*)()) 0);
394: }
395: after_thread_block:
396: wait_result = current_thread()->wait_result;
397: vm_object_lock(object);
398: if (wait_result != THREAD_AWAKENED) {
399: vm_fault_cleanup(object, first_m);
400: if (wait_result == THREAD_RESTART)
401: return(VM_FAULT_RETRY);
402: else
403: return(VM_FAULT_INTERRUPTED);
404: }
405: continue;
406: }
407:
408: /*
409: * If the page is in error, give up now.
410: */
411:
412: if (m->error) {
413: VM_PAGE_FREE(m);
414: vm_fault_cleanup(object, first_m);
415: return(VM_FAULT_MEMORY_ERROR);
416: }
417:
418: /*
419: * If the page isn't busy, but is absent,
420: * then it was deemed "unavailable".
421: */
422:
423: if (m->absent) {
424: /*
425: * Remove the non-existent page (unless it's
426: * in the top object) and move on down to the
427: * next object (if there is one).
428: */
429:
430: offset += object->shadow_offset;
431: access_required = VM_PROT_READ;
432: next_object = object->shadow;
433: if (next_object == VM_OBJECT_NULL) {
434: vm_page_t real_m;
435:
436: assert(!must_be_resident);
437:
438: /*
439: * Absent page at bottom of shadow
440: * chain; zero fill the page we left
441: * busy in the first object, and flush
442: * the absent page. But first we
443: * need to allocate a real page.
444: */
445:
446: real_m = vm_page_grab(!object->internal);
447: if (real_m == VM_PAGE_NULL) {
448: vm_fault_cleanup(object, first_m);
449: return(VM_FAULT_MEMORY_SHORTAGE);
450: }
451:
452: if (object != first_object) {
453: VM_PAGE_FREE(m);
454: vm_object_paging_end(object);
455: vm_object_unlock(object);
456: object = first_object;
457: offset = first_offset;
458: m = first_m;
459: first_m = VM_PAGE_NULL;
460: vm_object_lock(object);
461: }
462:
463: VM_PAGE_FREE(m);
464: assert(real_m->busy);
465: vm_page_lock_queues();
466: vm_page_insert(real_m, object, offset);
467: vm_page_unlock_queues();
468: m = real_m;
469:
470: /*
471: * Drop the lock while zero filling
472: * page. Then break because this
473: * is the page we wanted. Checking
474: * the page lock is a waste of time;
475: * this page was either absent or
476: * newly allocated -- in both cases
477: * it can't be page locked by a pager.
478: */
479: vm_object_unlock(object);
480:
481: vm_page_zero_fill(m);
482:
483: vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS);
484:
485: vm_stat.zero_fill_count++;
486: vm_object_lock(object);
487: pmap_clear_modify(m->phys_addr);
488: break;
489: } else {
490: if (must_be_resident) {
491: vm_object_paging_end(object);
492: } else if (object != first_object) {
493: vm_object_paging_end(object);
494: VM_PAGE_FREE(m);
495: } else {
496: first_m = m;
497: m->absent = FALSE;
498: vm_object_absent_release(object);
499: m->busy = TRUE;
500:
501: vm_page_lock_queues();
502: VM_PAGE_QUEUES_REMOVE(m);
503: vm_page_unlock_queues();
504: }
505: vm_object_lock(next_object);
506: vm_object_unlock(object);
507: object = next_object;
508: vm_object_paging_begin(object);
509: continue;
510: }
511: }
512:
513: /*
514: * If the desired access to this page has
515: * been locked out, request that it be unlocked.
516: */
517:
518: if (access_required & m->page_lock) {
519: if ((access_required & m->unlock_request) != access_required) {
520: vm_prot_t new_unlock_request;
521: kern_return_t rc;
522:
523: if (!object->pager_ready) {
524: vm_object_assert_wait(object,
525: VM_OBJECT_EVENT_PAGER_READY,
526: interruptible);
527: goto block_and_backoff;
528: }
529:
530: new_unlock_request = m->unlock_request =
531: (access_required | m->unlock_request);
532: vm_object_unlock(object);
533: if ((rc = memory_object_data_unlock(
534: object->pager,
535: object->pager_request,
536: offset + object->paging_offset,
537: PAGE_SIZE,
538: new_unlock_request))
539: != KERN_SUCCESS) {
540: printf("vm_fault: memory_object_data_unlock failed\n");
541: vm_object_lock(object);
542: vm_fault_cleanup(object, first_m);
543: return((rc == MACH_SEND_INTERRUPTED) ?
544: VM_FAULT_INTERRUPTED :
545: VM_FAULT_MEMORY_ERROR);
546: }
547: vm_object_lock(object);
548: continue;
549: }
550:
551: PAGE_ASSERT_WAIT(m, interruptible);
552: goto block_and_backoff;
553: }
554:
555: /*
556: * We mark the page busy and leave it on
557: * the pageout queues. If the pageout
558: * deamon comes across it, then it will
559: * remove the page.
560: */
561:
562: if (!software_reference_bits) {
563: vm_page_lock_queues();
564: if (m->inactive) {
565: vm_stat_sample(SAMPLED_PC_VM_REACTIVATION_FAULTS);
566: vm_stat.reactivations++;
567: }
568:
569: VM_PAGE_QUEUES_REMOVE(m);
570: vm_page_unlock_queues();
571: }
572:
573: assert(!m->busy);
574: m->busy = TRUE;
575: assert(!m->absent);
576: break;
577: }
578:
579: look_for_page =
580: (object->pager_created)
581: #if MACH_PAGEMAP
582: && (vm_external_state_get(object->existence_info, offset + object->paging_offset) !=
583: VM_EXTERNAL_STATE_ABSENT)
584: #endif /* MACH_PAGEMAP */
585: ;
586:
587: if ((look_for_page || (object == first_object))
588: && !must_be_resident) {
589: /*
590: * Allocate a new page for this object/offset
591: * pair.
592: */
593:
594: m = vm_page_grab_fictitious();
595: if (m == VM_PAGE_NULL) {
596: vm_fault_cleanup(object, first_m);
597: return(VM_FAULT_FICTITIOUS_SHORTAGE);
598: }
599:
600: vm_page_lock_queues();
601: vm_page_insert(m, object, offset);
602: vm_page_unlock_queues();
603: }
604:
605: if (look_for_page && !must_be_resident) {
606: kern_return_t rc;
607:
608: /*
609: * If the memory manager is not ready, we
610: * cannot make requests.
611: */
612: if (!object->pager_ready) {
613: vm_object_assert_wait(object,
614: VM_OBJECT_EVENT_PAGER_READY,
615: interruptible);
616: VM_PAGE_FREE(m);
617: goto block_and_backoff;
618: }
619:
620: if (object->internal) {
621: /*
622: * Requests to the default pager
623: * must reserve a real page in advance,
624: * because the pager's data-provided
625: * won't block for pages.
626: */
627:
628: if (m->fictitious && !vm_page_convert(m, FALSE)) {
629: VM_PAGE_FREE(m);
630: vm_fault_cleanup(object, first_m);
631: return(VM_FAULT_MEMORY_SHORTAGE);
632: }
633: } else if (object->absent_count >
634: vm_object_absent_max) {
635: /*
636: * If there are too many outstanding page
637: * requests pending on this object, we
638: * wait for them to be resolved now.
639: */
640:
641: vm_object_absent_assert_wait(object, interruptible);
642: VM_PAGE_FREE(m);
643: goto block_and_backoff;
644: }
645:
646: /*
647: * Indicate that the page is waiting for data
648: * from the memory manager.
649: */
650:
651: m->absent = TRUE;
652: object->absent_count++;
653:
654: /*
655: * We have a busy page, so we can
656: * release the object lock.
657: */
658: vm_object_unlock(object);
659:
660: /*
661: * Call the memory manager to retrieve the data.
662: */
663:
664: vm_stat.pageins++;
665: vm_stat_sample(SAMPLED_PC_VM_PAGEIN_FAULTS);
666:
667: if ((rc = memory_object_data_request(object->pager,
668: object->pager_request,
669: m->offset + object->paging_offset,
670: PAGE_SIZE, access_required)) != KERN_SUCCESS) {
671: if (rc != MACH_SEND_INTERRUPTED)
672: printf("%s(0x%x, 0x%x, 0x%x, 0x%x, 0x%x) failed, %d\n",
673: "memory_object_data_request",
674: object->pager,
675: object->pager_request,
676: m->offset + object->paging_offset,
677: PAGE_SIZE, access_required, rc);
678: /*
679: * Don't want to leave a busy page around,
680: * but the data request may have blocked,
681: * so check if it's still there and busy.
682: */
683: vm_object_lock(object);
684: if (m == vm_page_lookup(object,offset) &&
685: m->absent && m->busy)
686: VM_PAGE_FREE(m);
687: vm_fault_cleanup(object, first_m);
688: return((rc == MACH_SEND_INTERRUPTED) ?
689: VM_FAULT_INTERRUPTED :
690: VM_FAULT_MEMORY_ERROR);
691: }
692:
693: /*
694: * Retry with same object/offset, since new data may
695: * be in a different page (i.e., m is meaningless at
696: * this point).
697: */
698: vm_object_lock(object);
699: continue;
700: }
701:
702: /*
703: * For the XP system, the only case in which we get here is if
704: * object has no pager (or unwiring). If the pager doesn't
705: * have the page this is handled in the m->absent case above
706: * (and if you change things here you should look above).
707: */
708: if (object == first_object)
709: first_m = m;
710: else
711: {
712: assert(m == VM_PAGE_NULL);
713: }
714:
715: /*
716: * Move on to the next object. Lock the next
717: * object before unlocking the current one.
718: */
719: access_required = VM_PROT_READ;
720:
721: offset += object->shadow_offset;
722: next_object = object->shadow;
723: if (next_object == VM_OBJECT_NULL) {
724: assert(!must_be_resident);
725:
726: /*
727: * If there's no object left, fill the page
728: * in the top object with zeros. But first we
729: * need to allocate a real page.
730: */
731:
732: if (object != first_object) {
733: vm_object_paging_end(object);
734: vm_object_unlock(object);
735:
736: object = first_object;
737: offset = first_offset;
738: vm_object_lock(object);
739: }
740:
741: m = first_m;
742: assert(m->object == object);
743: first_m = VM_PAGE_NULL;
744:
745: if (m->fictitious && !vm_page_convert(m, !object->internal)) {
746: VM_PAGE_FREE(m);
747: vm_fault_cleanup(object, VM_PAGE_NULL);
748: return(VM_FAULT_MEMORY_SHORTAGE);
749: }
750:
751: vm_object_unlock(object);
752: vm_page_zero_fill(m);
753: vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS);
754: vm_stat.zero_fill_count++;
755: vm_object_lock(object);
756: pmap_clear_modify(m->phys_addr);
757: break;
758: }
759: else {
760: vm_object_lock(next_object);
761: if ((object != first_object) || must_be_resident)
762: vm_object_paging_end(object);
763: vm_object_unlock(object);
764: object = next_object;
765: vm_object_paging_begin(object);
766: }
767: }
768:
769: /*
770: * PAGE HAS BEEN FOUND.
771: *
772: * This page (m) is:
773: * busy, so that we can play with it;
774: * not absent, so that nobody else will fill it;
775: * possibly eligible for pageout;
776: *
777: * The top-level page (first_m) is:
778: * VM_PAGE_NULL if the page was found in the
779: * top-level object;
780: * busy, not absent, and ineligible for pageout.
781: *
782: * The current object (object) is locked. A paging
783: * reference is held for the current and top-level
784: * objects.
785: */
786:
787: #if EXTRA_ASSERTIONS
788: assert(m->busy && !m->absent);
789: assert((first_m == VM_PAGE_NULL) ||
790: (first_m->busy && !first_m->absent &&
791: !first_m->active && !first_m->inactive));
792: #endif /* EXTRA_ASSERTIONS */
793:
794: /*
795: * If the page is being written, but isn't
796: * already owned by the top-level object,
797: * we have to copy it into a new page owned
798: * by the top-level object.
799: */
800:
801: if (object != first_object) {
802: /*
803: * We only really need to copy if we
804: * want to write it.
805: */
806:
807: if (fault_type & VM_PROT_WRITE) {
808: vm_page_t copy_m;
809:
810: assert(!must_be_resident);
811:
812: /*
813: * If we try to collapse first_object at this
814: * point, we may deadlock when we try to get
815: * the lock on an intermediate object (since we
816: * have the bottom object locked). We can't
817: * unlock the bottom object, because the page
818: * we found may move (by collapse) if we do.
819: *
820: * Instead, we first copy the page. Then, when
821: * we have no more use for the bottom object,
822: * we unlock it and try to collapse.
823: *
824: * Note that we copy the page even if we didn't
825: * need to... that's the breaks.
826: */
827:
828: /*
829: * Allocate a page for the copy
830: */
831: copy_m = vm_page_grab(!first_object->internal);
832: if (copy_m == VM_PAGE_NULL) {
833: RELEASE_PAGE(m);
834: vm_fault_cleanup(object, first_m);
835: return(VM_FAULT_MEMORY_SHORTAGE);
836: }
837:
838: vm_object_unlock(object);
839: vm_page_copy(m, copy_m);
840: vm_object_lock(object);
841:
842: /*
843: * If another map is truly sharing this
844: * page with us, we have to flush all
845: * uses of the original page, since we
846: * can't distinguish those which want the
847: * original from those which need the
848: * new copy.
849: *
850: * XXXO If we know that only one map has
851: * access to this page, then we could
852: * avoid the pmap_page_protect() call.
853: */
854:
855: vm_page_lock_queues();
856: vm_page_deactivate(m);
857: pmap_page_protect(m->phys_addr, VM_PROT_NONE);
858: vm_page_unlock_queues();
859:
860: /*
861: * We no longer need the old page or object.
862: */
863:
864: PAGE_WAKEUP_DONE(m);
865: vm_object_paging_end(object);
866: vm_object_unlock(object);
867:
868: vm_stat.cow_faults++;
869: vm_stat_sample(SAMPLED_PC_VM_COW_FAULTS);
870: object = first_object;
871: offset = first_offset;
872:
873: vm_object_lock(object);
874: VM_PAGE_FREE(first_m);
875: first_m = VM_PAGE_NULL;
876: assert(copy_m->busy);
877: vm_page_lock_queues();
878: vm_page_insert(copy_m, object, offset);
879: vm_page_unlock_queues();
880: m = copy_m;
881:
882: /*
883: * Now that we've gotten the copy out of the
884: * way, let's try to collapse the top object.
885: * But we have to play ugly games with
886: * paging_in_progress to do that...
887: */
888:
889: vm_object_paging_end(object);
890: vm_object_collapse(object);
891: vm_object_paging_begin(object);
892: }
893: else {
894: *protection &= (~VM_PROT_WRITE);
895: }
896: }
897:
898: /*
899: * Now check whether the page needs to be pushed into the
900: * copy object. The use of asymmetric copy on write for
901: * shared temporary objects means that we may do two copies to
902: * satisfy the fault; one above to get the page from a
903: * shadowed object, and one here to push it into the copy.
904: */
905:
906: while ((copy_object = first_object->copy) != VM_OBJECT_NULL) {
907: vm_offset_t copy_offset;
908: vm_page_t copy_m;
909:
910: /*
911: * If the page is being written, but hasn't been
912: * copied to the copy-object, we have to copy it there.
913: */
914:
915: if ((fault_type & VM_PROT_WRITE) == 0) {
916: *protection &= ~VM_PROT_WRITE;
917: break;
918: }
919:
920: /*
921: * If the page was guaranteed to be resident,
922: * we must have already performed the copy.
923: */
924:
925: if (must_be_resident)
926: break;
927:
928: /*
929: * Try to get the lock on the copy_object.
930: */
931: if (!vm_object_lock_try(copy_object)) {
932: vm_object_unlock(object);
933:
934: simple_lock_pause(); /* wait a bit */
935:
936: vm_object_lock(object);
937: continue;
938: }
939:
940: /*
941: * Make another reference to the copy-object,
942: * to keep it from disappearing during the
943: * copy.
944: */
945: assert(copy_object->ref_count > 0);
946: copy_object->ref_count++;
947:
948: /*
949: * Does the page exist in the copy?
950: */
951: copy_offset = first_offset - copy_object->shadow_offset;
952: copy_m = vm_page_lookup(copy_object, copy_offset);
953: if (copy_m != VM_PAGE_NULL) {
954: if (copy_m->busy) {
955: /*
956: * If the page is being brought
957: * in, wait for it and then retry.
958: */
959: PAGE_ASSERT_WAIT(copy_m, interruptible);
960: RELEASE_PAGE(m);
961: copy_object->ref_count--;
962: assert(copy_object->ref_count > 0);
963: vm_object_unlock(copy_object);
964: goto block_and_backoff;
965: }
966: }
967: else {
968: /*
969: * Allocate a page for the copy
970: */
971: copy_m = vm_page_alloc(copy_object, copy_offset);
972: if (copy_m == VM_PAGE_NULL) {
973: RELEASE_PAGE(m);
974: copy_object->ref_count--;
975: assert(copy_object->ref_count > 0);
976: vm_object_unlock(copy_object);
977: vm_fault_cleanup(object, first_m);
978: return(VM_FAULT_MEMORY_SHORTAGE);
979: }
980:
981: /*
982: * Must copy page into copy-object.
983: */
984:
985: vm_page_copy(m, copy_m);
986:
987: /*
988: * If the old page was in use by any users
989: * of the copy-object, it must be removed
990: * from all pmaps. (We can't know which
991: * pmaps use it.)
992: */
993:
994: vm_page_lock_queues();
995: pmap_page_protect(m->phys_addr, VM_PROT_NONE);
996: copy_m->dirty = TRUE;
997: vm_page_unlock_queues();
998:
999: /*
1000: * If there's a pager, then immediately
1001: * page out this page, using the "initialize"
1002: * option. Else, we use the copy.
1003: */
1004:
1005: if (!copy_object->pager_created) {
1006: vm_page_lock_queues();
1007: vm_page_activate(copy_m);
1008: vm_page_unlock_queues();
1009: PAGE_WAKEUP_DONE(copy_m);
1010: } else {
1011: /*
1012: * The page is already ready for pageout:
1013: * not on pageout queues and busy.
1014: * Unlock everything except the
1015: * copy_object itself.
1016: */
1017:
1018: vm_object_unlock(object);
1019:
1020: /*
1021: * Write the page to the copy-object,
1022: * flushing it from the kernel.
1023: */
1024:
1025: vm_pageout_page(copy_m, TRUE, TRUE);
1026:
1027: /*
1028: * Since the pageout may have
1029: * temporarily dropped the
1030: * copy_object's lock, we
1031: * check whether we'll have
1032: * to deallocate the hard way.
1033: */
1034:
1035: if ((copy_object->shadow != object) ||
1036: (copy_object->ref_count == 1)) {
1037: vm_object_unlock(copy_object);
1038: vm_object_deallocate(copy_object);
1039: vm_object_lock(object);
1040: continue;
1041: }
1042:
1043: /*
1044: * Pick back up the old object's
1045: * lock. [It is safe to do so,
1046: * since it must be deeper in the
1047: * object tree.]
1048: */
1049:
1050: vm_object_lock(object);
1051: }
1052:
1053: /*
1054: * Because we're pushing a page upward
1055: * in the object tree, we must restart
1056: * any faults that are waiting here.
1057: * [Note that this is an expansion of
1058: * PAGE_WAKEUP that uses the THREAD_RESTART
1059: * wait result]. Can't turn off the page's
1060: * busy bit because we're not done with it.
1061: */
1062:
1063: if (m->wanted) {
1064: m->wanted = FALSE;
1065: thread_wakeup_with_result((event_t) m,
1066: THREAD_RESTART);
1067: }
1068: }
1069:
1070: /*
1071: * The reference count on copy_object must be
1072: * at least 2: one for our extra reference,
1073: * and at least one from the outside world
1074: * (we checked that when we last locked
1075: * copy_object).
1076: */
1077: copy_object->ref_count--;
1078: assert(copy_object->ref_count > 0);
1079: vm_object_unlock(copy_object);
1080:
1081: break;
1082: }
1083:
1084: *result_page = m;
1085: *top_page = first_m;
1086:
1087: /*
1088: * If the page can be written, assume that it will be.
1089: * [Earlier, we restrict the permission to allow write
1090: * access only if the fault so required, so we don't
1091: * mark read-only data as dirty.]
1092: */
1093:
1094: if (vm_fault_dirty_handling && (*protection & VM_PROT_WRITE))
1095: m->dirty = TRUE;
1096:
1097: return(VM_FAULT_SUCCESS);
1098:
1099: block_and_backoff:
1100: vm_fault_cleanup(object, first_m);
1101:
1102: #ifdef CONTINUATIONS
1103: if (continuation != (void (*)()) 0) {
1104: register vm_fault_state_t *state =
1105: (vm_fault_state_t *) current_thread()->ith_other;
1106:
1107: /*
1108: * Save variables in case we must restart.
1109: */
1110:
1111: state->vmfp_backoff = TRUE;
1112: state->vmf_prot = *protection;
1113:
1114: counter(c_vm_fault_page_block_backoff_user++);
1115: thread_block(continuation);
1116: } else
1117: #endif /* CONTINUATIONS */
1118: {
1119: counter(c_vm_fault_page_block_backoff_kernel++);
1120: thread_block((void (*)()) 0);
1121: }
1122: after_block_and_backoff:
1123: if (current_thread()->wait_result == THREAD_AWAKENED)
1124: return VM_FAULT_RETRY;
1125: else
1126: return VM_FAULT_INTERRUPTED;
1127:
1128: #undef RELEASE_PAGE
1129: }
1130:
1131: /*
1132: * Routine: vm_fault
1133: * Purpose:
1134: * Handle page faults, including pseudo-faults
1135: * used to change the wiring status of pages.
1136: * Returns:
1137: * If an explicit (expression) continuation is supplied,
1138: * then we call the continuation instead of returning.
1139: * Implementation:
1140: * Explicit continuations make this a little icky,
1141: * because it hasn't been rewritten to embrace CPS.
1142: * Instead, we have resume arguments for vm_fault and
1143: * vm_fault_page, to let continue the fault computation.
1144: *
1145: * vm_fault and vm_fault_page save mucho state
1146: * in the moral equivalent of a closure. The state
1147: * structure is allocated when first entering vm_fault
1148: * and deallocated when leaving vm_fault.
1149: */
1150:
1151: #ifdef CONTINUATIONS
1152: void
1153: vm_fault_continue()
1154: {
1155: register vm_fault_state_t *state =
1156: (vm_fault_state_t *) current_thread()->ith_other;
1157:
1158: (void) vm_fault(state->vmf_map,
1159: state->vmf_vaddr,
1160: state->vmf_fault_type,
1161: state->vmf_change_wiring,
1162: TRUE, state->vmf_continuation);
1163: /*NOTREACHED*/
1164: }
1165: #endif /* CONTINUATIONS */
1166:
1167: kern_return_t vm_fault(map, vaddr, fault_type, change_wiring,
1168: resume, continuation)
1169: vm_map_t map;
1170: vm_offset_t vaddr;
1171: vm_prot_t fault_type;
1172: boolean_t change_wiring;
1173: boolean_t resume;
1174: void (*continuation)();
1175: {
1176: vm_map_version_t version; /* Map version for verificiation */
1177: boolean_t wired; /* Should mapping be wired down? */
1178: vm_object_t object; /* Top-level object */
1179: vm_offset_t offset; /* Top-level offset */
1180: vm_prot_t prot; /* Protection for mapping */
1181: vm_object_t old_copy_object; /* Saved copy object */
1182: vm_page_t result_page; /* Result of vm_fault_page */
1183: vm_page_t top_page; /* Placeholder page */
1184: kern_return_t kr;
1185:
1186: register
1187: vm_page_t m; /* Fast access to result_page */
1188:
1189: #ifdef CONTINUATIONS
1190: if (resume) {
1191: register vm_fault_state_t *state =
1192: (vm_fault_state_t *) current_thread()->ith_other;
1193:
1194: /*
1195: * Retrieve cached variables and
1196: * continue vm_fault_page.
1197: */
1198:
1199: object = state->vmf_object;
1200: if (object == VM_OBJECT_NULL)
1201: goto RetryFault;
1202: version = state->vmf_version;
1203: wired = state->vmf_wired;
1204: offset = state->vmf_offset;
1205: prot = state->vmf_prot;
1206:
1207: kr = vm_fault_page(object, offset, fault_type,
1208: (change_wiring && !wired), !change_wiring,
1209: &prot, &result_page, &top_page,
1210: TRUE, vm_fault_continue);
1211: goto after_vm_fault_page;
1212: }
1213:
1214: if (continuation != (void (*)()) 0) {
1215: /*
1216: * We will probably need to save state.
1217: */
1218:
1219: char * state;
1220:
1221: /*
1222: * if this assignment stmt is written as
1223: * 'active_threads[cpu_number()] = zalloc()',
1224: * cpu_number may be evaluated before zalloc;
1225: * if zalloc blocks, cpu_number will be wrong
1226: */
1227:
1228: state = (char *) zalloc(vm_fault_state_zone);
1229: current_thread()->ith_other = state;
1230:
1231: }
1232: #else /* not CONTINUATIONS */
1233: assert(continuation == 0);
1234: assert(!resume);
1235: #endif /* not CONTINUATIONS */
1236:
1237: RetryFault: ;
1238:
1239: /*
1240: * Find the backing store object and offset into
1241: * it to begin the search.
1242: */
1243:
1244: if ((kr = vm_map_lookup(&map, vaddr, fault_type, &version,
1245: &object, &offset,
1246: &prot, &wired)) != KERN_SUCCESS) {
1247: goto done;
1248: }
1249:
1250: /*
1251: * If the page is wired, we must fault for the current protection
1252: * value, to avoid further faults.
1253: */
1254:
1255: if (wired)
1256: fault_type = prot;
1257:
1258: /*
1259: * Make a reference to this object to
1260: * prevent its disposal while we are messing with
1261: * it. Once we have the reference, the map is free
1262: * to be diddled. Since objects reference their
1263: * shadows (and copies), they will stay around as well.
1264: */
1265:
1266: assert(object->ref_count > 0);
1267: object->ref_count++;
1268: vm_object_paging_begin(object);
1269:
1270: #ifdef CONTINUATIONS
1271: if (continuation != (void (*)()) 0) {
1272: register vm_fault_state_t *state =
1273: (vm_fault_state_t *) current_thread()->ith_other;
1274:
1275: /*
1276: * Save variables, in case vm_fault_page discards
1277: * our kernel stack and we have to restart.
1278: */
1279:
1280: state->vmf_map = map;
1281: state->vmf_vaddr = vaddr;
1282: state->vmf_fault_type = fault_type;
1283: state->vmf_change_wiring = change_wiring;
1284: state->vmf_continuation = continuation;
1285:
1286: state->vmf_version = version;
1287: state->vmf_wired = wired;
1288: state->vmf_object = object;
1289: state->vmf_offset = offset;
1290: state->vmf_prot = prot;
1291:
1292: kr = vm_fault_page(object, offset, fault_type,
1293: (change_wiring && !wired), !change_wiring,
1294: &prot, &result_page, &top_page,
1295: FALSE, vm_fault_continue);
1296: } else
1297: #endif /* CONTINUATIONS */
1298: {
1299: kr = vm_fault_page(object, offset, fault_type,
1300: (change_wiring && !wired), !change_wiring,
1301: &prot, &result_page, &top_page,
1302: FALSE, (void (*)()) 0);
1303: }
1304: after_vm_fault_page:
1305:
1306: /*
1307: * If we didn't succeed, lose the object reference immediately.
1308: */
1309:
1310: if (kr != VM_FAULT_SUCCESS)
1311: vm_object_deallocate(object);
1312:
1313: /*
1314: * See why we failed, and take corrective action.
1315: */
1316:
1317: switch (kr) {
1318: case VM_FAULT_SUCCESS:
1319: break;
1320: case VM_FAULT_RETRY:
1321: goto RetryFault;
1322: case VM_FAULT_INTERRUPTED:
1323: kr = KERN_SUCCESS;
1324: goto done;
1325: case VM_FAULT_MEMORY_SHORTAGE:
1326: #ifdef CONTINUATIONS
1327: if (continuation != (void (*)()) 0) {
1328: register vm_fault_state_t *state =
1329: (vm_fault_state_t *) current_thread()->ith_other;
1330:
1331: /*
1332: * Save variables in case VM_PAGE_WAIT
1333: * discards our kernel stack.
1334: */
1335:
1336: state->vmf_map = map;
1337: state->vmf_vaddr = vaddr;
1338: state->vmf_fault_type = fault_type;
1339: state->vmf_change_wiring = change_wiring;
1340: state->vmf_continuation = continuation;
1341: state->vmf_object = VM_OBJECT_NULL;
1342:
1343: VM_PAGE_WAIT(vm_fault_continue);
1344: } else
1345: #endif /* CONTINUATIONS */
1346: VM_PAGE_WAIT((void (*)()) 0);
1347: goto RetryFault;
1348: case VM_FAULT_FICTITIOUS_SHORTAGE:
1349: vm_page_more_fictitious();
1350: goto RetryFault;
1351: case VM_FAULT_MEMORY_ERROR:
1352: kr = KERN_MEMORY_ERROR;
1353: goto done;
1354: }
1355:
1356: m = result_page;
1357:
1358: assert((change_wiring && !wired) ?
1359: (top_page == VM_PAGE_NULL) :
1360: ((top_page == VM_PAGE_NULL) == (m->object == object)));
1361:
1362: /*
1363: * How to clean up the result of vm_fault_page. This
1364: * happens whether the mapping is entered or not.
1365: */
1366:
1367: #define UNLOCK_AND_DEALLOCATE \
1368: MACRO_BEGIN \
1369: vm_fault_cleanup(m->object, top_page); \
1370: vm_object_deallocate(object); \
1371: MACRO_END
1372:
1373: /*
1374: * What to do with the resulting page from vm_fault_page
1375: * if it doesn't get entered into the physical map:
1376: */
1377:
1378: #define RELEASE_PAGE(m) \
1379: MACRO_BEGIN \
1380: PAGE_WAKEUP_DONE(m); \
1381: vm_page_lock_queues(); \
1382: if (!m->active && !m->inactive) \
1383: vm_page_activate(m); \
1384: vm_page_unlock_queues(); \
1385: MACRO_END
1386:
1387: /*
1388: * We must verify that the maps have not changed
1389: * since our last lookup.
1390: */
1391:
1392: old_copy_object = m->object->copy;
1393:
1394: vm_object_unlock(m->object);
1395: while (!vm_map_verify(map, &version)) {
1396: vm_object_t retry_object;
1397: vm_offset_t retry_offset;
1398: vm_prot_t retry_prot;
1399:
1400: /*
1401: * To avoid trying to write_lock the map while another
1402: * thread has it read_locked (in vm_map_pageable), we
1403: * do not try for write permission. If the page is
1404: * still writable, we will get write permission. If it
1405: * is not, or has been marked needs_copy, we enter the
1406: * mapping without write permission, and will merely
1407: * take another fault.
1408: */
1409: kr = vm_map_lookup(&map, vaddr,
1410: fault_type & ~VM_PROT_WRITE, &version,
1411: &retry_object, &retry_offset, &retry_prot,
1412: &wired);
1413:
1414: if (kr != KERN_SUCCESS) {
1415: vm_object_lock(m->object);
1416: RELEASE_PAGE(m);
1417: UNLOCK_AND_DEALLOCATE;
1418: goto done;
1419: }
1420:
1421: vm_object_unlock(retry_object);
1422: vm_object_lock(m->object);
1423:
1424: if ((retry_object != object) ||
1425: (retry_offset != offset)) {
1426: RELEASE_PAGE(m);
1427: UNLOCK_AND_DEALLOCATE;
1428: goto RetryFault;
1429: }
1430:
1431: /*
1432: * Check whether the protection has changed or the object
1433: * has been copied while we left the map unlocked.
1434: */
1435: prot &= retry_prot;
1436: vm_object_unlock(m->object);
1437: }
1438: vm_object_lock(m->object);
1439:
1440: /*
1441: * If the copy object changed while the top-level object
1442: * was unlocked, then we must take away write permission.
1443: */
1444:
1445: if (m->object->copy != old_copy_object)
1446: prot &= ~VM_PROT_WRITE;
1447:
1448: /*
1449: * If we want to wire down this page, but no longer have
1450: * adequate permissions, we must start all over.
1451: */
1452:
1453: if (wired && (prot != fault_type)) {
1454: vm_map_verify_done(map, &version);
1455: RELEASE_PAGE(m);
1456: UNLOCK_AND_DEALLOCATE;
1457: goto RetryFault;
1458: }
1459:
1460: /*
1461: * It's critically important that a wired-down page be faulted
1462: * only once in each map for which it is wired.
1463: */
1464:
1465: vm_object_unlock(m->object);
1466:
1467: /*
1468: * Put this page into the physical map.
1469: * We had to do the unlock above because pmap_enter
1470: * may cause other faults. The page may be on
1471: * the pageout queues. If the pageout daemon comes
1472: * across the page, it will remove it from the queues.
1473: */
1474:
1475: PMAP_ENTER(map->pmap, vaddr, m, prot, wired);
1476:
1477: /*
1478: * If the page is not wired down and isn't already
1479: * on a pageout queue, then put it where the
1480: * pageout daemon can find it.
1481: */
1482: vm_object_lock(m->object);
1483: vm_page_lock_queues();
1484: if (change_wiring) {
1485: if (wired)
1486: vm_page_wire(m);
1487: else
1488: vm_page_unwire(m);
1489: } else if (software_reference_bits) {
1490: if (!m->active && !m->inactive)
1491: vm_page_activate(m);
1492: m->reference = TRUE;
1493: } else {
1494: vm_page_activate(m);
1495: }
1496: vm_page_unlock_queues();
1497:
1498: /*
1499: * Unlock everything, and return
1500: */
1501:
1502: vm_map_verify_done(map, &version);
1503: PAGE_WAKEUP_DONE(m);
1504: kr = KERN_SUCCESS;
1505: UNLOCK_AND_DEALLOCATE;
1506:
1507: #undef UNLOCK_AND_DEALLOCATE
1508: #undef RELEASE_PAGE
1509:
1510: done:
1511: #ifdef CONTINUATIONS
1512: if (continuation != (void (*)()) 0) {
1513: register vm_fault_state_t *state =
1514: (vm_fault_state_t *) current_thread()->ith_other;
1515:
1516: zfree(vm_fault_state_zone, (vm_offset_t) state);
1517: (*continuation)(kr);
1518: /*NOTREACHED*/
1519: }
1520: #endif /* CONTINUATIONS */
1521:
1522: return(kr);
1523: }
1524:
1525: kern_return_t vm_fault_wire_fast();
1526:
1527: /*
1528: * vm_fault_wire:
1529: *
1530: * Wire down a range of virtual addresses in a map.
1531: */
1532: void vm_fault_wire(map, entry)
1533: vm_map_t map;
1534: vm_map_entry_t entry;
1535: {
1536:
1537: register vm_offset_t va;
1538: register pmap_t pmap;
1539: register vm_offset_t end_addr = entry->vme_end;
1540:
1541: pmap = vm_map_pmap(map);
1542:
1543: /*
1544: * Inform the physical mapping system that the
1545: * range of addresses may not fault, so that
1546: * page tables and such can be locked down as well.
1547: */
1548:
1549: pmap_pageable(pmap, entry->vme_start, end_addr, FALSE);
1550:
1551: /*
1552: * We simulate a fault to get the page and enter it
1553: * in the physical map.
1554: */
1555:
1556: for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
1557: if (vm_fault_wire_fast(map, va, entry) != KERN_SUCCESS)
1558: (void) vm_fault(map, va, VM_PROT_NONE, TRUE,
1559: FALSE, (void (*)()) 0);
1560: }
1561: }
1562:
1563: /*
1564: * vm_fault_unwire:
1565: *
1566: * Unwire a range of virtual addresses in a map.
1567: */
1568: void vm_fault_unwire(map, entry)
1569: vm_map_t map;
1570: vm_map_entry_t entry;
1571: {
1572: register vm_offset_t va;
1573: register pmap_t pmap;
1574: register vm_offset_t end_addr = entry->vme_end;
1575: vm_object_t object;
1576:
1577: pmap = vm_map_pmap(map);
1578:
1579: object = (entry->is_sub_map)
1580: ? VM_OBJECT_NULL : entry->object.vm_object;
1581:
1582: /*
1583: * Since the pages are wired down, we must be able to
1584: * get their mappings from the physical map system.
1585: */
1586:
1587: for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
1588: pmap_change_wiring(pmap, va, FALSE);
1589:
1590: if (object == VM_OBJECT_NULL) {
1591: vm_map_lock_set_recursive(map);
1592: (void) vm_fault(map, va, VM_PROT_NONE, TRUE,
1593: FALSE, (void (*)()) 0);
1594: vm_map_lock_clear_recursive(map);
1595: } else {
1596: vm_prot_t prot;
1597: vm_page_t result_page;
1598: vm_page_t top_page;
1599: vm_fault_return_t result;
1600:
1601: do {
1602: prot = VM_PROT_NONE;
1603:
1604: vm_object_lock(object);
1605: vm_object_paging_begin(object);
1606: result = vm_fault_page(object,
1607: entry->offset +
1608: (va - entry->vme_start),
1609: VM_PROT_NONE, TRUE,
1610: FALSE, &prot,
1611: &result_page,
1612: &top_page,
1613: FALSE, (void (*)()) 0);
1614: } while (result == VM_FAULT_RETRY);
1615:
1616: if (result != VM_FAULT_SUCCESS)
1617: panic("vm_fault_unwire: failure");
1618:
1619: vm_page_lock_queues();
1620: vm_page_unwire(result_page);
1621: vm_page_unlock_queues();
1622: PAGE_WAKEUP_DONE(result_page);
1623:
1624: vm_fault_cleanup(result_page->object, top_page);
1625: }
1626: }
1627:
1628: /*
1629: * Inform the physical mapping system that the range
1630: * of addresses may fault, so that page tables and
1631: * such may be unwired themselves.
1632: */
1633:
1634: pmap_pageable(pmap, entry->vme_start, end_addr, TRUE);
1635: }
1636:
1637: /*
1638: * vm_fault_wire_fast:
1639: *
1640: * Handle common case of a wire down page fault at the given address.
1641: * If successful, the page is inserted into the associated physical map.
1642: * The map entry is passed in to avoid the overhead of a map lookup.
1643: *
1644: * NOTE: the given address should be truncated to the
1645: * proper page address.
1646: *
1647: * KERN_SUCCESS is returned if the page fault is handled; otherwise,
1648: * a standard error specifying why the fault is fatal is returned.
1649: *
1650: * The map in question must be referenced, and remains so.
1651: * Caller has a read lock on the map.
1652: *
1653: * This is a stripped version of vm_fault() for wiring pages. Anything
1654: * other than the common case will return KERN_FAILURE, and the caller
1655: * is expected to call vm_fault().
1656: */
1657: kern_return_t vm_fault_wire_fast(map, va, entry)
1658: vm_map_t map;
1659: vm_offset_t va;
1660: vm_map_entry_t entry;
1661: {
1662: vm_object_t object;
1663: vm_offset_t offset;
1664: register vm_page_t m;
1665: vm_prot_t prot;
1666:
1667: vm_stat.faults++; /* needs lock XXX */
1668: /*
1669: * Recovery actions
1670: */
1671:
1672: #undef RELEASE_PAGE
1673: #define RELEASE_PAGE(m) { \
1674: PAGE_WAKEUP_DONE(m); \
1675: vm_page_lock_queues(); \
1676: vm_page_unwire(m); \
1677: vm_page_unlock_queues(); \
1678: }
1679:
1680:
1681: #undef UNLOCK_THINGS
1682: #define UNLOCK_THINGS { \
1683: object->paging_in_progress--; \
1684: vm_object_unlock(object); \
1685: }
1686:
1687: #undef UNLOCK_AND_DEALLOCATE
1688: #define UNLOCK_AND_DEALLOCATE { \
1689: UNLOCK_THINGS; \
1690: vm_object_deallocate(object); \
1691: }
1692: /*
1693: * Give up and have caller do things the hard way.
1694: */
1695:
1696: #define GIVE_UP { \
1697: UNLOCK_AND_DEALLOCATE; \
1698: return(KERN_FAILURE); \
1699: }
1700:
1701:
1702: /*
1703: * If this entry is not directly to a vm_object, bail out.
1704: */
1705: if (entry->is_sub_map)
1706: return(KERN_FAILURE);
1707:
1708: /*
1709: * Find the backing store object and offset into it.
1710: */
1711:
1712: object = entry->object.vm_object;
1713: offset = (va - entry->vme_start) + entry->offset;
1714: prot = entry->protection;
1715:
1716: /*
1717: * Make a reference to this object to prevent its
1718: * disposal while we are messing with it.
1719: */
1720:
1721: vm_object_lock(object);
1722: assert(object->ref_count > 0);
1723: object->ref_count++;
1724: object->paging_in_progress++;
1725:
1726: /*
1727: * INVARIANTS (through entire routine):
1728: *
1729: * 1) At all times, we must either have the object
1730: * lock or a busy page in some object to prevent
1731: * some other thread from trying to bring in
1732: * the same page.
1733: *
1734: * 2) Once we have a busy page, we must remove it from
1735: * the pageout queues, so that the pageout daemon
1736: * will not grab it away.
1737: *
1738: */
1739:
1740: /*
1741: * Look for page in top-level object. If it's not there or
1742: * there's something going on, give up.
1743: */
1744: m = vm_page_lookup(object, offset);
1745: if ((m == VM_PAGE_NULL) || (m->error) ||
1746: (m->busy) || (m->absent) || (prot & m->page_lock)) {
1747: GIVE_UP;
1748: }
1749:
1750: /*
1751: * Wire the page down now. All bail outs beyond this
1752: * point must unwire the page.
1753: */
1754:
1755: vm_page_lock_queues();
1756: vm_page_wire(m);
1757: vm_page_unlock_queues();
1758:
1759: /*
1760: * Mark page busy for other threads.
1761: */
1762: assert(!m->busy);
1763: m->busy = TRUE;
1764: assert(!m->absent);
1765:
1766: /*
1767: * Give up if the page is being written and there's a copy object
1768: */
1769: if ((object->copy != VM_OBJECT_NULL) && (prot & VM_PROT_WRITE)) {
1770: RELEASE_PAGE(m);
1771: GIVE_UP;
1772: }
1773:
1774: /*
1775: * Put this page into the physical map.
1776: * We have to unlock the object because pmap_enter
1777: * may cause other faults.
1778: */
1779: vm_object_unlock(object);
1780:
1781: PMAP_ENTER(map->pmap, va, m, prot, TRUE);
1782:
1783: /*
1784: * Must relock object so that paging_in_progress can be cleared.
1785: */
1786: vm_object_lock(object);
1787:
1788: /*
1789: * Unlock everything, and return
1790: */
1791:
1792: PAGE_WAKEUP_DONE(m);
1793: UNLOCK_AND_DEALLOCATE;
1794:
1795: return(KERN_SUCCESS);
1796:
1797: }
1798:
1799: /*
1800: * Routine: vm_fault_copy_cleanup
1801: * Purpose:
1802: * Release a page used by vm_fault_copy.
1803: */
1804:
1805: void vm_fault_copy_cleanup(page, top_page)
1806: vm_page_t page;
1807: vm_page_t top_page;
1808: {
1809: vm_object_t object = page->object;
1810:
1811: vm_object_lock(object);
1812: PAGE_WAKEUP_DONE(page);
1813: vm_page_lock_queues();
1814: if (!page->active && !page->inactive)
1815: vm_page_activate(page);
1816: vm_page_unlock_queues();
1817: vm_fault_cleanup(object, top_page);
1818: }
1819:
1820: /*
1821: * Routine: vm_fault_copy
1822: *
1823: * Purpose:
1824: * Copy pages from one virtual memory object to another --
1825: * neither the source nor destination pages need be resident.
1826: *
1827: * Before actually copying a page, the version associated with
1828: * the destination address map wil be verified.
1829: *
1830: * In/out conditions:
1831: * The caller must hold a reference, but not a lock, to
1832: * each of the source and destination objects and to the
1833: * destination map.
1834: *
1835: * Results:
1836: * Returns KERN_SUCCESS if no errors were encountered in
1837: * reading or writing the data. Returns KERN_INTERRUPTED if
1838: * the operation was interrupted (only possible if the
1839: * "interruptible" argument is asserted). Other return values
1840: * indicate a permanent error in copying the data.
1841: *
1842: * The actual amount of data copied will be returned in the
1843: * "copy_size" argument. In the event that the destination map
1844: * verification failed, this amount may be less than the amount
1845: * requested.
1846: */
1847: kern_return_t vm_fault_copy(
1848: src_object,
1849: src_offset,
1850: src_size,
1851: dst_object,
1852: dst_offset,
1853: dst_map,
1854: dst_version,
1855: interruptible
1856: )
1857: vm_object_t src_object;
1858: vm_offset_t src_offset;
1859: vm_size_t *src_size; /* INOUT */
1860: vm_object_t dst_object;
1861: vm_offset_t dst_offset;
1862: vm_map_t dst_map;
1863: vm_map_version_t *dst_version;
1864: boolean_t interruptible;
1865: {
1866: vm_page_t result_page;
1867: vm_prot_t prot;
1868:
1869: vm_page_t src_page;
1870: vm_page_t src_top_page;
1871:
1872: vm_page_t dst_page;
1873: vm_page_t dst_top_page;
1874:
1875: vm_size_t amount_done;
1876: vm_object_t old_copy_object;
1877:
1878: #define RETURN(x) \
1879: MACRO_BEGIN \
1880: *src_size = amount_done; \
1881: MACRO_RETURN(x); \
1882: MACRO_END
1883:
1884: amount_done = 0;
1885: do { /* while (amount_done != *src_size) */
1886:
1887: RetrySourceFault: ;
1888:
1889: if (src_object == VM_OBJECT_NULL) {
1890: /*
1891: * No source object. We will just
1892: * zero-fill the page in dst_object.
1893: */
1894:
1895: src_page = VM_PAGE_NULL;
1896: } else {
1897: prot = VM_PROT_READ;
1898:
1899: vm_object_lock(src_object);
1900: vm_object_paging_begin(src_object);
1901:
1902: switch (vm_fault_page(src_object, src_offset,
1903: VM_PROT_READ, FALSE, interruptible,
1904: &prot, &result_page, &src_top_page,
1905: FALSE, (void (*)()) 0)) {
1906:
1907: case VM_FAULT_SUCCESS:
1908: break;
1909: case VM_FAULT_RETRY:
1910: goto RetrySourceFault;
1911: case VM_FAULT_INTERRUPTED:
1912: RETURN(MACH_SEND_INTERRUPTED);
1913: case VM_FAULT_MEMORY_SHORTAGE:
1914: VM_PAGE_WAIT((void (*)()) 0);
1915: goto RetrySourceFault;
1916: case VM_FAULT_FICTITIOUS_SHORTAGE:
1917: vm_page_more_fictitious();
1918: goto RetrySourceFault;
1919: case VM_FAULT_MEMORY_ERROR:
1920: return(KERN_MEMORY_ERROR);
1921: }
1922:
1923: src_page = result_page;
1924:
1925: assert((src_top_page == VM_PAGE_NULL) ==
1926: (src_page->object == src_object));
1927:
1928: assert ((prot & VM_PROT_READ) != VM_PROT_NONE);
1929:
1930: vm_object_unlock(src_page->object);
1931: }
1932:
1933: RetryDestinationFault: ;
1934:
1935: prot = VM_PROT_WRITE;
1936:
1937: vm_object_lock(dst_object);
1938: vm_object_paging_begin(dst_object);
1939:
1940: switch (vm_fault_page(dst_object, dst_offset, VM_PROT_WRITE,
1941: FALSE, FALSE /* interruptible */,
1942: &prot, &result_page, &dst_top_page,
1943: FALSE, (void (*)()) 0)) {
1944:
1945: case VM_FAULT_SUCCESS:
1946: break;
1947: case VM_FAULT_RETRY:
1948: goto RetryDestinationFault;
1949: case VM_FAULT_INTERRUPTED:
1950: if (src_page != VM_PAGE_NULL)
1951: vm_fault_copy_cleanup(src_page,
1952: src_top_page);
1953: RETURN(MACH_SEND_INTERRUPTED);
1954: case VM_FAULT_MEMORY_SHORTAGE:
1955: VM_PAGE_WAIT((void (*)()) 0);
1956: goto RetryDestinationFault;
1957: case VM_FAULT_FICTITIOUS_SHORTAGE:
1958: vm_page_more_fictitious();
1959: goto RetryDestinationFault;
1960: case VM_FAULT_MEMORY_ERROR:
1961: if (src_page != VM_PAGE_NULL)
1962: vm_fault_copy_cleanup(src_page,
1963: src_top_page);
1964: return(KERN_MEMORY_ERROR);
1965: }
1966: assert ((prot & VM_PROT_WRITE) != VM_PROT_NONE);
1967:
1968: dst_page = result_page;
1969:
1970: old_copy_object = dst_page->object->copy;
1971:
1972: vm_object_unlock(dst_page->object);
1973:
1974: if (!vm_map_verify(dst_map, dst_version)) {
1975:
1976: BailOut: ;
1977:
1978: if (src_page != VM_PAGE_NULL)
1979: vm_fault_copy_cleanup(src_page, src_top_page);
1980: vm_fault_copy_cleanup(dst_page, dst_top_page);
1981: break;
1982: }
1983:
1984:
1985: vm_object_lock(dst_page->object);
1986: if (dst_page->object->copy != old_copy_object) {
1987: vm_object_unlock(dst_page->object);
1988: vm_map_verify_done(dst_map, dst_version);
1989: goto BailOut;
1990: }
1991: vm_object_unlock(dst_page->object);
1992:
1993: /*
1994: * Copy the page, and note that it is dirty
1995: * immediately.
1996: */
1997:
1998: if (src_page == VM_PAGE_NULL)
1999: vm_page_zero_fill(dst_page);
2000: else
2001: vm_page_copy(src_page, dst_page);
2002: dst_page->dirty = TRUE;
2003:
2004: /*
2005: * Unlock everything, and return
2006: */
2007:
2008: vm_map_verify_done(dst_map, dst_version);
2009:
2010: if (src_page != VM_PAGE_NULL)
2011: vm_fault_copy_cleanup(src_page, src_top_page);
2012: vm_fault_copy_cleanup(dst_page, dst_top_page);
2013:
2014: amount_done += PAGE_SIZE;
2015: src_offset += PAGE_SIZE;
2016: dst_offset += PAGE_SIZE;
2017:
2018: } while (amount_done != *src_size);
2019:
2020: RETURN(KERN_SUCCESS);
2021: #undef RETURN
2022:
2023: /*NOTREACHED*/
2024: }
2025:
2026:
2027:
2028:
2029:
2030: #ifdef notdef
2031:
2032: /*
2033: * Routine: vm_fault_page_overwrite
2034: *
2035: * Description:
2036: * A form of vm_fault_page that assumes that the
2037: * resulting page will be overwritten in its entirety,
2038: * making it unnecessary to obtain the correct *contents*
2039: * of the page.
2040: *
2041: * Implementation:
2042: * XXX Untested. Also unused. Eventually, this technology
2043: * could be used in vm_fault_copy() to advantage.
2044: */
2045: vm_fault_return_t vm_fault_page_overwrite(dst_object, dst_offset, result_page)
2046: register
2047: vm_object_t dst_object;
2048: vm_offset_t dst_offset;
2049: vm_page_t *result_page; /* OUT */
2050: {
2051: register
2052: vm_page_t dst_page;
2053:
2054: #define interruptible FALSE /* XXX */
2055:
2056: while (TRUE) {
2057: /*
2058: * Look for a page at this offset
2059: */
2060:
2061: while ((dst_page = vm_page_lookup(dst_object, dst_offset))
2062: == VM_PAGE_NULL) {
2063: /*
2064: * No page, no problem... just allocate one.
2065: */
2066:
2067: dst_page = vm_page_alloc(dst_object, dst_offset);
2068: if (dst_page == VM_PAGE_NULL) {
2069: vm_object_unlock(dst_object);
2070: VM_PAGE_WAIT((void (*)()) 0);
2071: vm_object_lock(dst_object);
2072: continue;
2073: }
2074:
2075: /*
2076: * Pretend that the memory manager
2077: * write-protected the page.
2078: *
2079: * Note that we will be asking for write
2080: * permission without asking for the data
2081: * first.
2082: */
2083:
2084: dst_page->overwriting = TRUE;
2085: dst_page->page_lock = VM_PROT_WRITE;
2086: dst_page->absent = TRUE;
2087: dst_object->absent_count++;
2088:
2089: break;
2090:
2091: /*
2092: * When we bail out, we might have to throw
2093: * away the page created here.
2094: */
2095:
2096: #define DISCARD_PAGE \
2097: MACRO_BEGIN \
2098: vm_object_lock(dst_object); \
2099: dst_page = vm_page_lookup(dst_object, dst_offset); \
2100: if ((dst_page != VM_PAGE_NULL) && dst_page->overwriting) \
2101: VM_PAGE_FREE(dst_page); \
2102: vm_object_unlock(dst_object); \
2103: MACRO_END
2104: }
2105:
2106: /*
2107: * If the page is write-protected...
2108: */
2109:
2110: if (dst_page->page_lock & VM_PROT_WRITE) {
2111: /*
2112: * ... and an unlock request hasn't been sent
2113: */
2114:
2115: if ( ! (dst_page->unlock_request & VM_PROT_WRITE)) {
2116: vm_prot_t u;
2117: kern_return_t rc;
2118:
2119: /*
2120: * ... then send one now.
2121: */
2122:
2123: if (!dst_object->pager_ready) {
2124: vm_object_assert_wait(dst_object,
2125: VM_OBJECT_EVENT_PAGER_READY,
2126: interruptible);
2127: vm_object_unlock(dst_object);
2128: thread_block((void (*)()) 0);
2129: if (current_thread()->wait_result !=
2130: THREAD_AWAKENED) {
2131: DISCARD_PAGE;
2132: return(VM_FAULT_INTERRUPTED);
2133: }
2134: continue;
2135: }
2136:
2137: u = dst_page->unlock_request |= VM_PROT_WRITE;
2138: vm_object_unlock(dst_object);
2139:
2140: if ((rc = memory_object_data_unlock(
2141: dst_object->pager,
2142: dst_object->pager_request,
2143: dst_offset + dst_object->paging_offset,
2144: PAGE_SIZE,
2145: u)) != KERN_SUCCESS) {
2146: printf("vm_object_overwrite: memory_object_data_unlock failed\n");
2147: DISCARD_PAGE;
2148: return((rc == MACH_SEND_INTERRUPTED) ?
2149: VM_FAULT_INTERRUPTED :
2150: VM_FAULT_MEMORY_ERROR);
2151: }
2152: vm_object_lock(dst_object);
2153: continue;
2154: }
2155:
2156: /* ... fall through to wait below */
2157: } else {
2158: /*
2159: * If the page isn't being used for other
2160: * purposes, then we're done.
2161: */
2162: if ( ! (dst_page->busy || dst_page->absent || dst_page->error) )
2163: break;
2164: }
2165:
2166: PAGE_ASSERT_WAIT(dst_page, interruptible);
2167: vm_object_unlock(dst_object);
2168: thread_block((void (*)()) 0);
2169: if (current_thread()->wait_result != THREAD_AWAKENED) {
2170: DISCARD_PAGE;
2171: return(VM_FAULT_INTERRUPTED);
2172: }
2173: }
2174:
2175: *result_page = dst_page;
2176: return(VM_FAULT_SUCCESS);
2177:
2178: #undef interruptible
2179: #undef DISCARD_PAGE
2180: }
2181:
2182: #endif /* notdef */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.