![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_fault.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple Darwin 0.x] / kernel / vm|
Return to vm_fault.c CVS log | Up to [Apple Darwin 0.x] / kernel / vm |
1.1 root 1: /*
2: * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
3: *
4: * @APPLE_LICENSE_HEADER_START@
5: *
6: * Portions Copyright (c) 1999 Apple Computer, Inc. All Rights
7: * Reserved. This file contains Original Code and/or Modifications of
8: * Original Code as defined in and that are subject to the Apple Public
9: * Source License Version 1.1 (the "License"). You may not use this file
10: * except in compliance with the License. Please obtain a copy of the
11: * License at http://www.apple.com/publicsource and read it before using
12: * this file.
13: *
14: * The Original Code and all software distributed under the License are
15: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
16: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
17: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
18: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
19: * License for the specific language governing rights and limitations
20: * under the License.
21: *
22: * @APPLE_LICENSE_HEADER_END@
23: */
24:
25: /*
26: * Mach Operating System
27: * Copyright (c) 1987 Carnegie-Mellon University
28: * All rights reserved. The CMU software License Agreement specifies
29: * the terms and conditions for use and redistribution.
30: */
31: /*
32: * File: vm_fault.c
33: * Author: Avadis Tevanian, Jr., Michael Wayne Young
34: *
35: * Copyright (C) 1985, Avadis Tevanian, Jr., Michael Wayne Young
36: *
37: * Page fault handling module.
38: */
39:
40: #import <mach_xp.h>
41:
42: #define USE_VERSIONS MACH_XP
43:
44: #import <mach/kern_return.h>
45: #import <mach/message.h> /* for error codes */
46: #import <kern/thread.h>
47: #import <kern/sched_prim.h>
48: #import <vm/vm_map.h>
49: #import <vm/vm_object.h>
50: #import <vm/vm_page.h>
51: #import <vm/pmap.h>
52: #import <mach/vm_statistics.h>
53: #import <vm/vm_pageout.h>
54: #import <mach/vm_param.h>
55:
56:
57: /*
58: * vm_fault:
59: *
60: * Handle a page fault occuring at the given address,
61: * requiring the given permissions, in the map specified.
62: * If successful, the page is inserted into the
63: * associated physical map.
64: *
65: * NOTE: the given address should be truncated to the
66: * proper page address.
67: *
68: * KERN_SUCCESS is returned if the page fault is handled; otherwise,
69: * a standard error specifying why the fault is fatal is returned.
70: *
71: *
72: * The map in question must be referenced, and remains so.
73: * Caller may hold no locks.
74: */
75: #if NeXT
76: kern_return_t vm_fault(map, vaddr, fault_type, change_wiring, error)
77: vm_map_t map;
78: vm_offset_t vaddr;
79: vm_prot_t fault_type;
80: boolean_t change_wiring;
81: int *error;
82: #else NeXT
83: kern_return_t vm_fault(map, vaddr, fault_type, change_wiring)
84: vm_map_t map;
85: vm_offset_t vaddr;
86: vm_prot_t fault_type;
87: boolean_t change_wiring;
88: #endif NeXT
89: {
90: vm_object_t first_object;
91: vm_offset_t first_offset;
92: #if USE_VERSIONS
93: vm_map_version_t version;
94: #else USE_VERSIONS
95: vm_map_entry_t entry;
96: #endif USE_VERSIONS
97: register vm_object_t object;
98: register vm_offset_t offset;
99: register vm_page_t m;
100: vm_page_t first_m;
101: vm_prot_t prot;
102: kern_return_t result;
103: boolean_t wired;
104: boolean_t su;
105: #if !USE_VERSIONS
106: boolean_t lookup_still_valid;
107: #endif !USE_VERSIONS
108: boolean_t page_exists;
109: vm_page_t old_m;
110: vm_object_t next_object;
111:
112: vm_stat.faults++; /* needs lock XXX */
113: /*
114: * Recovery actions
115: */
116: #if MACH_XP
117: /* vm_page_free does a PAGE_WAKEUP anyway */
118: #define FREE_PAGE(m) { \
119: vm_page_lock_queues(); \
120: vm_page_free(m); \
121: vm_page_unlock_queues(); \
122: }
123: #else MACH_XP
124: #define FREE_PAGE(m) { \
125: PAGE_WAKEUP(m); \
126: vm_page_lock_queues(); \
127: vm_page_free(m); \
128: vm_page_unlock_queues(); \
129: }
130: #endif MACH_XP
131:
132: #define RELEASE_PAGE(m) { \
133: PAGE_WAKEUP(m); \
134: vm_page_lock_queues(); \
135: vm_page_activate(m); \
136: vm_page_unlock_queues(); \
137: }
138:
139: #if USE_VERSIONS
140: #define UNLOCK_MAP
141: #else USE_VERSIONS
142: #define UNLOCK_MAP { \
143: if (lookup_still_valid) { \
144: vm_map_lookup_done(map, entry); \
145: lookup_still_valid = FALSE; \
146: } \
147: }
148: #endif USE_VERSIONS
149:
150: #define UNLOCK_THINGS { \
151: object->paging_in_progress--; \
152: vm_object_unlock(object); \
153: if (object != first_object) { \
154: vm_object_lock(first_object); \
155: FREE_PAGE(first_m); \
156: first_object->paging_in_progress--; \
157: vm_object_unlock(first_object); \
158: } \
159: UNLOCK_MAP; \
160: }
161:
162: #define UNLOCK_AND_DEALLOCATE { \
163: UNLOCK_THINGS; \
164: vm_object_deallocate(first_object); \
165: }
166:
167: RetryFault: ;
168:
169: /*
170: * Find the backing store object and offset into
171: * it to begin the search.
172: */
173:
174: #if USE_VERSIONS
175: if ((result = vm_map_lookup(&map, vaddr, fault_type, &version,
176: #else USE_VERSIONS
177: if ((result = vm_map_lookup(&map, vaddr, fault_type, &entry,
178: #endif USE_VERSIONS
179: &first_object, &first_offset,
180: &prot, &wired, &su)) != KERN_SUCCESS) {
181: return(result);
182: }
183: #if !USE_VERSIONS
184: lookup_still_valid = TRUE;
185: #endif !USE_VERSIONS
186:
187: if (wired)
188: fault_type = prot;
189:
190: first_m = VM_PAGE_NULL;
191:
192: /*
193: * Make a reference to this object to
194: * prevent its disposal while we are messing with
195: * it. Once we have the reference, the map is free
196: * to be diddled. Since objects reference their
197: * shadows (and copies), they will stay around as well.
198: */
199:
200: #if !USE_VERSIONS
201: vm_object_lock(first_object);
202: #endif !USE_VERSIONS
203:
204: first_object->ref_count++;
205: first_object->paging_in_progress++;
206:
207: /*
208: * INVARIANTS (through entire routine):
209: *
210: * 1) At all times, we must either have the object
211: * lock or a busy page in some object to prevent
212: * some other thread from trying to bring in
213: * the same page.
214: *
215: * Note that we cannot hold any locks during the
216: * pager access or when waiting for memory, so
217: * we use a busy page then.
218: *
219: * Note also that we aren't as concerned about
220: * more than one thead attempting to pager_data_unlock
221: * the same page at once, so we don't hold the page
222: * as busy then, but do record the highest unlock
223: * value so far. [Unlock requests may also be delivered
224: * out of order.]
225: *
226: * 2) Once we have a busy page, we must remove it from
227: * the pageout queues, so that the pageout daemon
228: * will not grab it away.
229: *
230: * 3) To prevent another thread from racing us down the
231: * shadow chain and entering a new page in the top
232: * object before we do, we must keep a busy page in
233: * the top object while following the shadow chain.
234: *
235: * 4) We must increment paging_in_progress on any object
236: * for which we have a busy page, to prevent
237: * vm_object_collapse from removing the busy page
238: * without our noticing.
239: */
240:
241: /*
242: * Search for the page at object/offset.
243: */
244:
245: object = first_object;
246: offset = first_offset;
247:
248: /*
249: * See whether this page is resident
250: */
251:
252: while (TRUE) {
253: m = vm_page_lookup(object, offset);
254: if (m != VM_PAGE_NULL) {
255: /*
256: * If the page is in error, give up now.
257: */
258:
259: if (m->error) {
260: FREE_PAGE(m);
261: UNLOCK_AND_DEALLOCATE;
262: return(KERN_MEMORY_ERROR);
263: }
264:
265: /*
266: * If the page is being brought in,
267: * wait for it and then retry.
268: */
269: if (m->busy) {
270: kern_return_t wait_result;
271: #if SCRUBVM3
272: /* hint if we miss it its ok */
273: if (m->dry_vp){
274: vm_page_lock_queues();
275: (void) vm_page_completeio(m, TRUE);
276: vm_page_unlock_queues();
277: } else {
278: #endif
279: PAGE_ASSERT_WAIT(m, !change_wiring);
280: UNLOCK_MAP;
281: vm_object_unlock(object);
282: thread_block();
283: wait_result = current_thread()->wait_result;
284: vm_object_lock(object);
285: if (wait_result == THREAD_RESTART) {
286: UNLOCK_AND_DEALLOCATE;
287: goto RetryFault;
288: }
289: if (wait_result != THREAD_AWAKENED) {
290: UNLOCK_AND_DEALLOCATE;
291: return(KERN_SUCCESS);
292: }
293: continue;
294: #if SCRUBVM3
295: }
296: #endif
297: }
298:
299: /*
300: * If the page isn't busy, but is absent,
301: * then it was deemed "unavailable".
302: */
303:
304: if (m->absent) {
305: /*
306: * Remove the non-existent page (unless it's
307: * in the top object) and move on down to the
308: * next object (if there is one).
309: */
310: offset += object->shadow_offset;
311: next_object = object->shadow;
312: if (next_object == VM_OBJECT_NULL) {
313: /*
314: * Absent page at bottom of shadow
315: * chain; zero fill the page we left
316: * busy in the first object, and flush
317: * the absent page.
318: */
319: if (object != first_object) {
320: m->busy = m->absent = FALSE;
321: FREE_PAGE(m);
322: object->paging_in_progress--;
323: vm_object_unlock(object);
324: object = first_object;
325: offset = first_offset;
326: m = first_m;
327: vm_object_lock(object);
328: }
329: first_m = VM_PAGE_NULL;
330: vm_page_zero_fill(m);
331: vm_stat.zero_fill_count++;
332: m->absent = FALSE;
333: } else {
334: if (object != first_object) {
335: object->paging_in_progress--;
336: FREE_PAGE(m);
337: } else {
338: first_m = m;
339: m->absent = FALSE;
340: }
341: vm_object_lock(next_object);
342: vm_object_unlock(object);
343: object = next_object;
344: object->paging_in_progress++;
345: continue;
346: }
347: }
348:
349: /*
350: * If the desired access to this page has
351: * been locked out, request that it be unlocked.
352: */
353:
354: if (fault_type & m->page_lock) {
355: UNLOCK_AND_DEALLOCATE;
356: return (KERN_MEMORY_ERROR);
357: }
358:
359: /*
360: * Remove the page from the pageout daemon's
361: * reach while we play with it.
362: */
363:
364: vm_page_lock_queues();
365: if (m->inactive) {
366: queue_remove(&vm_page_queue_inactive, m,
367: vm_page_t, pageq);
368: m->inactive = FALSE;
369: vm_page_inactive_count--;
370: vm_stat.reactivations++;
371: }
372:
373: if (m->active) {
374: queue_remove(&vm_page_queue_active, m,
375: vm_page_t, pageq);
376: m->active = FALSE;
377: vm_page_active_count--;
378: }
379: #if NeXT
380: if (m->free) {
381: /*
382: * We only get here on reactivation of a free page,
383: * vm_page_alloc takes care of this for us in the
384: * typical case.
385: */
386: queue_remove(&vm_page_queue_free, m,
387: vm_page_t, pageq);
388: m->free = FALSE;
389: vm_page_free_count--;
390: vm_stat.reactivations++;
391: }
392: #endif NeXT
393: vm_page_unlock_queues();
394:
395: /*
396: * Mark page busy for other threads.
397: */
398: m->busy = TRUE;
399: m->absent = FALSE;
400: break;
401: }
402:
403: if (((object->pager != vm_pager_null) &&
404: (!change_wiring || wired))
405: || (object == first_object)) {
406:
407: /*
408: * Allocate a new page for this object/offset
409: * pair.
410: */
411:
412: m = vm_page_alloc(object, offset);
413:
414: if (m == VM_PAGE_NULL) {
415: UNLOCK_AND_DEALLOCATE;
416: VM_WAIT;
417: goto RetryFault;
418: }
419: }
420:
421: if ((object->pager != vm_pager_null) &&
422: (!change_wiring || wired)) {
423: #if MACH_XP
424: kern_return_t rc;
425: #else MACH_XP
426: pager_return_t rc;
427: #endif MACH_XP
428:
429: /*
430: * Now that we have a busy page, we can
431: * release the object lock.
432: */
433: vm_object_unlock(object);
434:
435: /*
436: * Call the pager to retrieve the data, if any,
437: * after releasing the lock on the map.
438: */
439: UNLOCK_MAP;
440:
441: #if MACH_XP
442: m->absent = TRUE;
443:
444: vm_stat.pageins++;
445: if ((rc = pager_data_request(object->pager,
446: object->pager_request,
447: m->offset + object->paging_offset,
448: PAGE_SIZE, fault_type)) != KERN_SUCCESS) {
449: if (rc != SEND_INTERRUPTED)
450: kprintf("%s(0x%x, 0x%x, 0x%x, 0x%x, 0x%x) failed, %d\n",
451: "pager_data_request",
452: object->pager,
453: object->pager_request,
454: m->offset + object->paging_offset,
455: PAGE_SIZE, fault_type, rc);
456: vm_object_lock(object);
457: FREE_PAGE(m);
458: UNLOCK_AND_DEALLOCATE;
459: return((rc == SEND_INTERRUPTED) ?
460: KERN_SUCCESS : KERN_MEMORY_ERROR);
461: }
462:
463: /*
464: * Retry with same object/offset, since new data may
465: * be in a different page (ie, m is meaningless at
466: * this point).
467: */
468: vm_object_lock(object);
469: continue;
470: #else MACH_XP
471: #if NeXT
472: rc = vm_pager_get(object->pager, m, error);
473: #else NeXT
474: rc = vm_pager_get(object->pager, m);
475: #endif NeXT
476: if (rc == PAGER_SUCCESS) {
477:
478: /*
479: * Found the page.
480: * Leave it busy while we play with it.
481: */
482: vm_object_lock(object);
483:
484: /*
485: * Relookup in case pager changed page.
486: * Pager is responsible for disposition
487: * of old page if moved.
488: */
489: m = vm_page_lookup(object, offset);
490:
491: vm_stat.pageins++;
492: pmap_clear_modify(VM_PAGE_TO_PHYS(m));
493: break;
494: }
495: if (rc == PAGER_ERROR) {
496: /*
497: * Pager had the page, but could not
498: * read it. Return error to stop caller.
499: */
500: vm_object_lock(object);
501: FREE_PAGE(m);
502: UNLOCK_AND_DEALLOCATE;
503: return(KERN_MEMORY_ERROR);
504: }
505:
506: /*
507: * Remove the bogus page (which does not
508: * exist at this object/offset); before
509: * doing so, we must get back our object
510: * lock to preserve our invariant.
511: *
512: * Also wake up any other thread that may want
513: * to bring in this page.
514: *
515: * If this is the top-level object, we must
516: * leave the busy page to prevent another
517: * thread from rushing past us, and inserting
518: * the page in that object at the same time
519: * that we are.
520: */
521:
522: vm_object_lock(object);
523: if (object != first_object) {
524: FREE_PAGE(m);
525: }
526: #endif MACH_XP
527: }
528:
529: /*
530: * For the XP system, the only case in which we get here is if
531: * object has no pager (or unwiring). If the pager doesn't
532: * have the page this is handled in the m->absent case above
533: * (and if you change things here you should look above).
534: */
535: if (object == first_object)
536: first_m = m;
537:
538: /*
539: * Move on to the next object. Lock the next
540: * object before unlocking the current one.
541: */
542:
543: offset += object->shadow_offset;
544: next_object = object->shadow;
545: if (next_object == VM_OBJECT_NULL) {
546: /*
547: * If there's no object left, fill the page
548: * in the top object with zeros.
549: */
550: if (object != first_object) {
551: object->paging_in_progress--;
552: vm_object_unlock(object);
553:
554: object = first_object;
555: offset = first_offset;
556: m = first_m;
557: vm_object_lock(object);
558: }
559: first_m = VM_PAGE_NULL;
560:
561: vm_page_zero_fill(m);
562: vm_stat.zero_fill_count++;
563: m->absent = FALSE;
564: break;
565: }
566: else {
567: vm_object_lock(next_object);
568: if (object != first_object)
569: object->paging_in_progress--;
570: vm_object_unlock(object);
571: object = next_object;
572: object->paging_in_progress++;
573: }
574: }
575:
576: if (m->absent || m->active || m->inactive || !m->busy)
577: panic("vm_fault: absent or active or inactive or not busy after main loop");
578:
579: /*
580: * PAGE HAS BEEN FOUND.
581: * [Loop invariant still holds -- the object lock
582: * is held.]
583: */
584:
585: old_m = m; /* save page that would be copied */
586:
587: /*
588: * If the page is being written, but isn't
589: * already owned by the top-level object,
590: * we have to copy it into a new page owned
591: * by the top-level object.
592: */
593:
594: if (object != first_object) {
595: /*
596: * We only really need to copy if we
597: * want to write it.
598: */
599:
600: if (fault_type & VM_PROT_WRITE) {
601:
602: /*
603: * If we try to collapse first_object at this
604: * point, we may deadlock when we try to get
605: * the lock on an intermediate object (since we
606: * have the bottom object locked). We can't
607: * unlock the bottom object, because the page
608: * we found may move (by collapse) if we do.
609: *
610: * Instead, we first copy the page. Then, when
611: * we have no more use for the bottom object,
612: * we unlock it and try to collapse.
613: *
614: * Note that we copy the page even if we didn't
615: * need to... that's the breaks.
616: */
617:
618: /*
619: * We already have an empty page in
620: * first_object - use it.
621: */
622:
623: vm_page_copy(m, first_m);
624: first_m->absent = FALSE;
625:
626: /*
627: * If another map is truly sharing this
628: * page with us, we have to flush all
629: * uses of the original page, since we
630: * can't distinguish those which want the
631: * original from those which need the
632: * new copy.
633: */
634:
635: vm_page_lock_queues();
636: /*
637: * We first activate the page, then deactivate
638: * it since vm_page_deactivate will only
639: * deactivate active pages.
640: */
641: vm_page_activate(m);
642: vm_page_deactivate(m);
643: if (!su)
644: pmap_remove_all(VM_PAGE_TO_PHYS(m));
645: vm_page_unlock_queues();
646:
647: /*
648: * We no longer need the old page or object.
649: */
650: PAGE_WAKEUP(m);
651: object->paging_in_progress--;
652: vm_object_unlock(object);
653:
654: /*
655: * Only use the new page below...
656: */
657:
658: vm_stat.cow_faults++;
659: m = first_m;
660: object = first_object;
661: offset = first_offset;
662:
663: /*
664: * Now that we've gotten the copy out of the
665: * way, let's try to collapse the top object.
666: */
667: vm_object_lock(object);
668: /*
669: * But we have to play ugly games with
670: * paging_in_progress to do that...
671: */
672: object->paging_in_progress--;
673: vm_object_collapse(object);
674: object->paging_in_progress++;
675: }
676: else {
677: prot &= (~VM_PROT_WRITE);
678: m->copy_on_write = TRUE;
679: }
680: }
681:
682: if (m->active || m->inactive)
683: panic("vm_fault: active or inactive before copy object handling");
684:
685: /*
686: * If the page is being written, but hasn't been
687: * copied to the copy-object, we have to copy it there.
688: */
689: RetryCopy:
690: if (first_object->copy != VM_OBJECT_NULL) {
691: vm_object_t copy_object = first_object->copy;
692: vm_offset_t copy_offset;
693: vm_page_t copy_m;
694:
695: /*
696: * We only need to copy if we want to write it.
697: */
698: if ((fault_type & VM_PROT_WRITE) == 0) {
699: prot &= ~VM_PROT_WRITE;
700: m->copy_on_write = TRUE;
701: }
702: else {
703: /*
704: * Try to get the lock on the copy_object.
705: */
706: if (!vm_object_lock_try(copy_object)) {
707: vm_object_unlock(object);
708: /* should spin a bit here... */
709: vm_object_lock(object);
710: goto RetryCopy;
711: }
712:
713: /*
714: * Make another reference to the copy-object,
715: * to keep it from disappearing during the
716: * copy.
717: */
718: copy_object->ref_count++;
719:
720: /*
721: * Does the page exist in the copy?
722: */
723: copy_offset = first_offset
724: - copy_object->shadow_offset;
725: copy_m = vm_page_lookup(copy_object, copy_offset);
726: if (page_exists = (copy_m != VM_PAGE_NULL)) {
727: if (copy_m->busy) {
728: kern_return_t wait_result;
729: #if SCRUBVM3
730: /* hint if we miss it its ok */
731: if (copy_m->dry_vp){
732: vm_page_lock_queues();
733: (void) vm_page_completeio(copy_m, TRUE);
734: vm_page_unlock_queues();
735: } else {
736: #endif
737:
738: /*
739: * If the page is being brought
740: * in, wait for it and then retry.
741: */
742: PAGE_ASSERT_WAIT(copy_m, !change_wiring);
743: RELEASE_PAGE(m);
744: copy_object->ref_count--;
745: vm_object_unlock(copy_object);
746: UNLOCK_THINGS;
747: thread_block();
748: wait_result = current_thread()->wait_result;
749: vm_object_deallocate(first_object);
750: /* may block */
751: if (wait_result != THREAD_AWAKENED)
752: return(KERN_SUCCESS);
753: goto RetryFault;
754: #if SCRUBVM3
755: }
756: #endif
757: }
758: }
759:
760: #if MACH_XP
761: else {
762: /*
763: * Allocate a page for the copy
764: */
765: copy_m = vm_page_alloc(copy_object,
766: copy_offset);
767: if (copy_m == VM_PAGE_NULL) {
768: /*
769: * Wait for a page, then retry.
770: */
771: RELEASE_PAGE(m);
772: copy_object->ref_count--;
773: vm_object_unlock(copy_object);
774: UNLOCK_AND_DEALLOCATE;
775: VM_WAIT;
776: goto RetryFault;
777: }
778:
779: /*
780: * Must copy page into copy-object.
781: */
782:
783: vm_page_copy(m, copy_m);
784: m->copy_on_write = FALSE;
785: copy_m->absent = FALSE;
786:
787: /*
788: * If the old page was in use by any users
789: * of the copy-object, it must be removed
790: * from all pmaps. (We can't know which
791: * pmaps use it.)
792: */
793:
794: vm_page_lock_queues();
795: pmap_remove_all(VM_PAGE_TO_PHYS(old_m));
796: copy_m->clean = FALSE;
797: vm_page_unlock_queues();
798:
799: /*
800: * If there's a pager, then immediately
801: * page out this page, using the "initialize"
802: * option. Else, we use the copy.
803: */
804:
805: if (copy_object->pager == vm_pager_null) {
806: vm_page_lock_queues();
807: vm_page_activate(copy_m);
808: vm_page_unlock_queues();
809: PAGE_WAKEUP(copy_m);
810: } else {
811: /*
812: * Prepare the page for pageout:
813: *
814: * Since it was just allocated,
815: * it is not on a pageout queue, but
816: * it is busy.
817: */
818:
819: copy_m->busy = FALSE;
820:
821: /*
822: * Unlock everything except the
823: * copy_object itself.
824: */
825:
826: vm_object_unlock(object);
827: UNLOCK_MAP;
828:
829: vm_pageout_page(copy_m, TRUE);
830:
831: /*
832: * Since the pageout may have
833: * temporarily dropped the
834: * copy_object's lock, we
835: * check whether we'll have
836: * to deallocate the hard way.
837: */
838:
839: if ((copy_object->shadow != object) ||
840: (copy_object->ref_count == 1)) {
841: vm_object_unlock(copy_object);
842: vm_object_deallocate(copy_object);
843: vm_object_lock(object);
844: goto RetryCopy;
845: }
846:
847: /*
848: * Pick back up the old object's
849: * lock. [It is safe to do so,
850: * since it must be deeper in the
851: * object tree.]
852: */
853:
854: vm_object_lock(object);
855: }
856: }
857: #if defined(lint) || defined(hc)
858: if (++page_exists != 0)
859: panic("lint");
860: #endif defined(lint) || defined(hc)
861: #else MACH_XP
862: /*
863: * If the page is not in memory (in the object)
864: * and the object has a pager, we have to check
865: * if the pager has the data in secondary
866: * storage.
867: */
868: if (!page_exists) {
869:
870: /*
871: * If we don't allocate a (blank) page
872: * here... another thread could try
873: * to page it in, allocate a page, and
874: * then block on the busy page in its
875: * shadow (first_object). Then we'd
876: * trip over the busy page after we
877: * found that the copy_object's pager
878: * doesn't have the page...
879: */
880: copy_m = vm_page_alloc(copy_object,
881: copy_offset);
882: if (copy_m == VM_PAGE_NULL) {
883: /*
884: * Wait for a page, then retry.
885: */
886: RELEASE_PAGE(m);
887: copy_object->ref_count--;
888: vm_object_unlock(copy_object);
889: UNLOCK_AND_DEALLOCATE;
890: VM_WAIT;
891: goto RetryFault;
892: }
893:
894: if (copy_object->pager != vm_pager_null) {
895: vm_object_unlock(object);
896: vm_object_unlock(copy_object);
897: UNLOCK_MAP;
898:
899: page_exists = vm_pager_has_page(
900: copy_object->pager,
901: (copy_offset + copy_object->paging_offset));
902:
903: vm_object_lock(copy_object);
904:
905: /*
906: * Since the map is unlocked, someone
907: * else could have copied this object
908: * and put a different copy_object
909: * between the two. Or, the last
910: * reference to the copy-object (other
911: * than the one we have) may have
912: * disappeared - if that has happened,
913: * we don't need to make the copy.
914: */
915: if (copy_object->shadow != object ||
916: copy_object->ref_count == 1) {
917: /*
918: * Gaah... start over!
919: */
920: FREE_PAGE(copy_m);
921: vm_object_unlock(copy_object);
922: vm_object_deallocate(copy_object);
923: /* may block */
924: vm_object_lock(object);
925: goto RetryCopy;
926: }
927: vm_object_lock(object);
928:
929: if (page_exists) {
930: /*
931: * We didn't need the page
932: */
933: FREE_PAGE(copy_m);
934: }
935: }
936: }
937: if (!page_exists) {
938: /*
939: * Must copy page into copy-object.
940: */
941: vm_page_copy(m, copy_m);
942: copy_m->absent = FALSE;
943:
944: /*
945: * Things to remember:
946: * 1. The copied page must be marked 'dirty'
947: * so it will be paged out to the copy
948: * object.
949: * 2. If the old page was in use by any users
950: * of the copy-object, it must be removed
951: * from all pmaps. (We can't know which
952: * pmaps use it.)
953: */
954: vm_page_lock_queues();
955: pmap_remove_all(VM_PAGE_TO_PHYS(old_m));
956: copy_m->clean = FALSE;
957: vm_page_activate(copy_m); /* XXX */
958: vm_page_unlock_queues();
959:
960: PAGE_WAKEUP(copy_m);
961: }
962: #endif MACH_XP
963: /*
964: * The reference count on copy_object must be
965: * at least 2: one for our extra reference,
966: * and at least one from the outside world
967: * (we checked that when we last locked
968: * copy_object).
969: */
970: copy_object->ref_count--;
971: vm_object_unlock(copy_object);
972: m->copy_on_write = FALSE;
973: }
974: }
975:
976: if (m->active || m->inactive)
977: panic("vm_fault: active or inactive before retrying lookup");
978:
979: /*
980: * We must verify that the maps have not changed
981: * since our last lookup.
982: */
983:
984: #if USE_VERSIONS
985: vm_object_unlock(object);
986: while (!vm_map_verify(map, &version)) {
987: vm_object_t retry_object;
988: vm_offset_t retry_offset;
989: vm_prot_t retry_prot;
990:
991: /*
992: * To avoid trying to write_lock the map while another
993: * thread has it read_locked (in vm_map_pageable), we
994: * do not try for write permission. If the page is
995: * still writable, we will get write permission. If it
996: * is not, or has been marked needs_copy, we enter the
997: * mapping without write permission, and will merely
998: * take another fault.
999: */
1000: result = vm_map_lookup(&map, vaddr,
1001: fault_type & ~VM_PROT_WRITE, &version,
1002: &retry_object, &retry_offset, &retry_prot,
1003: &wired, &su);
1004:
1005: if (result != KERN_SUCCESS) {
1006: RELEASE_PAGE(m);
1007: UNLOCK_AND_DEALLOCATE;
1008: return(result);
1009: }
1010:
1011: vm_object_unlock(retry_object);
1012: vm_object_lock(object);
1013:
1014: if ((retry_object != first_object) ||
1015: (retry_offset != first_offset)) {
1016: RELEASE_PAGE(m);
1017: UNLOCK_AND_DEALLOCATE;
1018: goto RetryFault;
1019: }
1020:
1021: /*
1022: * Check whether the protection has changed or the object
1023: * has been copied while we left the map unlocked.
1024: * Changing from read to write permission is OK - we leave
1025: * the page write-protected, and catch the write fault.
1026: * Changing from write to read permission means that we
1027: * can't mark the page write-enabled after all.
1028: */
1029: prot &= retry_prot;
1030: if (m->copy_on_write)
1031: prot &= ~VM_PROT_WRITE;
1032:
1033: vm_object_unlock(object);
1034: }
1035: vm_object_lock(object);
1036: #else USE_VERSIONS
1037:
1038: if (!lookup_still_valid) {
1039: vm_object_t retry_object;
1040: vm_offset_t retry_offset;
1041: vm_prot_t retry_prot;
1042:
1043: /*
1044: * Since map entries may be pageable, make sure we can
1045: * take a page fault on them.
1046: */
1047: vm_object_unlock(object);
1048:
1049: /*
1050: * To avoid trying to write_lock the map while another
1051: * thread has it read_locked (in vm_map_pageable), we
1052: * do not try for write permission. If the page is
1053: * still writable, we will get write permission. If it
1054: * is not, or has been marked needs_copy, we enter the
1055: * mapping without write permission, and will merely
1056: * take another fault.
1057: */
1058:
1059: result = vm_map_lookup(&map, vaddr,
1060: fault_type & ~VM_PROT_WRITE, &entry,
1061: &retry_object, &retry_offset, &retry_prot,
1062: &wired, &su);
1063: vm_object_lock(object);
1064:
1065: /*
1066: * If we don't need the page any longer, put it on the
1067: * active list (the easiest thing to do here). If no
1068: * one needs it, pageout will grab it eventually.
1069: */
1070:
1071: if (result != KERN_SUCCESS) {
1072: RELEASE_PAGE(m);
1073: UNLOCK_AND_DEALLOCATE;
1074: return(result);
1075: }
1076:
1077: lookup_still_valid = TRUE;
1078:
1079: if ((retry_object != first_object) ||
1080: (retry_offset != first_offset)) {
1081: RELEASE_PAGE(m);
1082: UNLOCK_AND_DEALLOCATE;
1083: goto RetryFault;
1084: }
1085:
1086: /*
1087: * Check whether the protection has changed or the object
1088: * has been copied while we left the map unlocked.
1089: * Changing from read to write permission is OK - we leave
1090: * the page write-protected, and catch the write fault.
1091: * Changing from write to read permission means that we
1092: * can't mark the page write-enabled after all.
1093: */
1094: prot &= retry_prot;
1095: if (m->copy_on_write)
1096: prot &= ~VM_PROT_WRITE;
1097: /*
1098: * Can't catch write fault if page is to be wired. This
1099: * should never happen because caller holds a read lock
1100: * on the map.
1101: */
1102: if (wired && (prot != fault_type)) {
1103: RELEASE_PAGE(m);
1104: UNLOCK_AND_DEALLOCATE;
1105: goto RetryFault;
1106: }
1107: }
1108: #endif USE_VERSIONS
1109:
1110: /*
1111: * (the various bits we're fiddling with here are locked by
1112: * the object's lock)
1113: */
1114:
1115: /* XXX This distorts the meaning of the copy_on_write bit */
1116:
1117: if (prot & VM_PROT_WRITE)
1118: m->copy_on_write = FALSE;
1119:
1120: /*
1121: * It's critically important that a wired-down page be faulted
1122: * only once in each map for which it is wired.
1123: */
1124:
1125: if (m->active || m->inactive)
1126: panic("vm_fault: active or inactive before pmap_enter");
1127:
1128: vm_object_unlock(object);
1129:
1130: /*
1131: * Put this page into the physical map.
1132: * We had to do the unlock above because pmap_enter
1133: * may cause other faults. We don't put the
1134: * page back on the active queue until later so
1135: * that the page-out daemon won't find us (yet).
1136: */
1137:
1138: pmap_enter(map->pmap, vaddr, VM_PAGE_TO_PHYS(m),
1139: prot & ~(m->page_lock), wired);
1140:
1141: /*
1142: * If the page is not wired down, then put it where the
1143: * pageout daemon can find it.
1144: */
1145: vm_object_lock(object);
1146: vm_page_lock_queues();
1147: if (change_wiring) {
1148: if (wired)
1149: vm_page_wire(m);
1150: else
1151: vm_page_unwire(m);
1152: }
1153: else
1154: vm_page_activate(m);
1155: vm_page_unlock_queues();
1156:
1157: /*
1158: * Unlock everything, and return
1159: */
1160:
1161: #if USE_VERSIONS
1162: vm_map_verify_done(map, &version);
1163: #endif USE_VERSIONS
1164: PAGE_WAKEUP(m);
1165: UNLOCK_AND_DEALLOCATE;
1166:
1167: return(KERN_SUCCESS);
1168:
1169: }
1170:
1171: kern_return_t vm_fault_wire_fast();
1172:
1173: /*
1174: * vm_fault_wire:
1175: *
1176: * Wire down a range of virtual addresses in a map.
1177: */
1178: void vm_fault_wire(map, entry)
1179: vm_map_t map;
1180: vm_map_entry_t entry;
1181: {
1182:
1183: register vm_offset_t va;
1184: register pmap_t pmap;
1185: register vm_offset_t end_addr = entry->vme_end;
1186:
1187: pmap = vm_map_pmap(map);
1188:
1189: /*
1190: * Inform the physical mapping system that the
1191: * range of addresses may not fault, so that
1192: * page tables and such can be locked down as well.
1193: */
1194:
1195: pmap_pageable(pmap, entry->vme_start, end_addr, FALSE);
1196:
1197: /*
1198: * We simulate a fault to get the page and enter it
1199: * in the physical map.
1200: */
1201:
1202: for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
1203: if (vm_fault_wire_fast(map, va, entry) != KERN_SUCCESS)
1204: (void) vm_fault(map, va, VM_PROT_NONE, TRUE, 0);
1205: }
1206: }
1207:
1208:
1209: /*
1210: * vm_fault_unwire:
1211: *
1212: * Unwire a range of virtual addresses in a map.
1213: */
1214: void vm_fault_unwire(map, entry)
1215: vm_map_t map;
1216: vm_map_entry_t entry;
1217: {
1218:
1219: register vm_offset_t va, pa;
1220: register pmap_t pmap;
1221: register vm_offset_t end_addr = entry->vme_end;
1222:
1223: pmap = vm_map_pmap(map);
1224:
1225: /*
1226: * Since the pages are wired down, we must be able to
1227: * get their mappings from the physical map system.
1228: */
1229:
1230: vm_page_lock_queues();
1231:
1232: for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
1233: pa = pmap_extract(pmap, va);
1234: if (pa == (vm_offset_t) 0)
1235: continue;
1236:
1237: pmap_change_wiring(pmap, va, FALSE);
1238: vm_page_unwire(PHYS_TO_VM_PAGE(pa));
1239: }
1240: vm_page_unlock_queues();
1241:
1242: /*
1243: * Inform the physical mapping system that the range
1244: * of addresses may fault, so that page tables and
1245: * such may be unwired themselves.
1246: */
1247:
1248: pmap_pageable(pmap, entry->vme_start, end_addr, TRUE);
1249:
1250: }
1251:
1252: /*
1253: * Routine:
1254: * vm_fault_copy_entry
1255: * Function:
1256: * Copy all of the pages from a wired-down map entry to another.
1257: *
1258: * In/out conditions:
1259: * The source and destination maps must be locked for write.
1260: * The source map entry must be wired down (or be a sharing map
1261: * entry corresponding to a main map entry that is wired down).
1262: */
1263:
1264: void vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry)
1265: vm_map_t dst_map;
1266: vm_map_t src_map;
1267: vm_map_entry_t dst_entry;
1268: vm_map_entry_t src_entry;
1269: {
1270:
1271: vm_object_t dst_object;
1272: vm_object_t src_object;
1273: vm_offset_t dst_offset;
1274: vm_offset_t src_offset;
1275: vm_prot_t prot;
1276: vm_offset_t vaddr;
1277: vm_page_t dst_m;
1278: vm_page_t src_m;
1279:
1280: #ifdef lint
1281: src_map++;
1282: #endif lint
1283:
1284: src_object = src_entry->object.vm_object;
1285: src_offset = src_entry->offset;
1286:
1287: /*
1288: * Create the top-level object for the destination entry.
1289: * (Doesn't actually shadow anything - we copy the pages
1290: * directly.)
1291: */
1292: dst_object = vm_object_allocate(
1293: (vm_size_t) (dst_entry->vme_end -
1294: dst_entry->vme_start));
1295:
1296: dst_entry->object.vm_object = dst_object;
1297: dst_entry->offset = 0;
1298:
1299: prot = dst_entry->max_protection;
1300:
1301: /*
1302: * Loop through all of the pages in the entry's range, copying
1303: * each one from the source object (it should be there) to the
1304: * destination object.
1305: */
1306: for (vaddr = dst_entry->vme_start, dst_offset = 0;
1307: vaddr < dst_entry->vme_end;
1308: vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) {
1309:
1310: /*
1311: * Allocate a page in the destination object
1312: */
1313: vm_object_lock(dst_object);
1314: do {
1315: dst_m = vm_page_alloc(dst_object, dst_offset);
1316: if (dst_m == VM_PAGE_NULL) {
1317: vm_object_unlock(dst_object);
1318: VM_WAIT;
1319: vm_object_lock(dst_object);
1320: }
1321: } while (dst_m == VM_PAGE_NULL);
1322:
1323: /*
1324: * Find the page in the source object, and copy it in.
1325: * (Because the source is wired down, the page will be
1326: * in memory.)
1327: */
1328: vm_object_lock(src_object);
1329: src_m = vm_page_lookup(src_object, dst_offset + src_offset);
1330: if (src_m == VM_PAGE_NULL)
1331: panic("vm_fault_copy_wired: page missing");
1332:
1333: vm_page_copy(src_m, dst_m);
1334:
1335: /*
1336: * Enter it in the pmap...
1337: */
1338: vm_object_unlock(src_object);
1339: vm_object_unlock(dst_object);
1340:
1341: pmap_enter(dst_map->pmap, vaddr, VM_PAGE_TO_PHYS(dst_m),
1342: prot, FALSE);
1343:
1344: /*
1345: * Mark it no longer busy, and put it on the active list.
1346: */
1347: vm_object_lock(dst_object);
1348: vm_page_lock_queues();
1349: vm_page_activate(dst_m);
1350: vm_page_unlock_queues();
1351: PAGE_WAKEUP(dst_m);
1352: vm_object_unlock(dst_object);
1353: }
1354:
1355: }
1356:
1357:
1358: /*
1359: * vm_fault_wire_fast:
1360: *
1361: * Handle common case of a wire down page fault at the given address.
1362: * If successful, the page is inserted into the associated physical map.
1363: * The map entry is passed in to avoid the overhead of a map lookup.
1364: *
1365: * NOTE: the given address should be truncated to the
1366: * proper page address.
1367: *
1368: * KERN_SUCCESS is returned if the page fault is handled; otherwise,
1369: * a standard error specifying why the fault is fatal is returned.
1370: *
1371: * The map in question must be referenced, and remains so.
1372: * Caller has a read lock on the map.
1373: *
1374: * This is a stripped version of vm_fault() for wiring pages. Anything
1375: * other than the common case will return KERN_FAILURE, and the caller
1376: * is expected to call vm_fault().
1377: */
1378: kern_return_t vm_fault_wire_fast(map, va, entry)
1379: vm_map_t map;
1380: vm_offset_t va;
1381: vm_map_entry_t entry;
1382: {
1383: vm_object_t object;
1384: vm_offset_t offset;
1385: register vm_page_t m;
1386: vm_prot_t prot;
1387:
1388: vm_stat.faults++; /* needs lock XXX */
1389: /*
1390: * Recovery actions
1391: */
1392:
1393: #undef RELEASE_PAGE
1394: #define RELEASE_PAGE(m) { \
1395: PAGE_WAKEUP(m); \
1396: vm_page_lock_queues(); \
1397: vm_page_unwire(m); \
1398: vm_page_unlock_queues(); \
1399: }
1400:
1401:
1402: #undef UNLOCK_THINGS
1403: #define UNLOCK_THINGS { \
1404: object->paging_in_progress--; \
1405: vm_object_unlock(object); \
1406: }
1407:
1408: #undef UNLOCK_AND_DEALLOCATE
1409: #define UNLOCK_AND_DEALLOCATE { \
1410: UNLOCK_THINGS; \
1411: vm_object_deallocate(object); \
1412: }
1413: /*
1414: * Give up and have caller do things the hard way.
1415: */
1416:
1417: #define GIVE_UP { \
1418: UNLOCK_AND_DEALLOCATE; \
1419: return(KERN_FAILURE); \
1420: }
1421:
1422:
1423: /*
1424: * If this entry is not directly to a vm_object, bail out.
1425: */
1426: if ((entry->is_a_map) || (entry->is_sub_map))
1427: return(KERN_FAILURE);
1428:
1429: /*
1430: * Find the backing store object and offset into it.
1431: */
1432:
1433: object = entry->object.vm_object;
1434: offset = va - entry->vme_start + entry->offset;
1435: prot = entry->protection;
1436:
1437: /*
1438: * Make a reference to this object to prevent its
1439: * disposal while we are messing with it.
1440: */
1441:
1442: vm_object_lock(object);
1443: object->ref_count++;
1444: object->paging_in_progress++;
1445:
1446: /*
1447: * INVARIANTS (through entire routine):
1448: *
1449: * 1) At all times, we must either have the object
1450: * lock or a busy page in some object to prevent
1451: * some other thread from trying to bring in
1452: * the same page.
1453: *
1454: * 2) Once we have a busy page, we must remove it from
1455: * the pageout queues, so that the pageout daemon
1456: * will not grab it away.
1457: *
1458: */
1459:
1460: /*
1461: * Look for page in top-level object. If it's not there or
1462: * there's something going on, give up.
1463: */
1464: m = vm_page_lookup(object, offset);
1465: #if SCRUBVM3
1466: if ((m != VM_PAGE_NULL) && (m->dry_vp)){
1467: vm_page_lock_queues();
1468: (void) vm_page_completeio(m, TRUE);
1469: vm_page_unlock_queues();
1470: }
1471: #endif
1472: if ((m == VM_PAGE_NULL) || (m->busy) || (m->absent) ||
1473: (prot & m->page_lock)) {
1474: GIVE_UP;
1475: }
1476:
1477: /*
1478: * Wire the page down now. All bail outs beyond this
1479: * point must unwire the page.
1480: */
1481:
1482: vm_page_lock_queues();
1483: vm_page_wire(m);
1484: vm_page_unlock_queues();
1485:
1486: /*
1487: * Mark page busy for other threads.
1488: */
1489: m->busy = TRUE;
1490: m->absent = FALSE;
1491:
1492: /*
1493: * Give up if the page is being written and there's a copy object
1494: */
1495: if (object->copy != VM_OBJECT_NULL) {
1496: if ((prot & VM_PROT_WRITE) == 0) {
1497: m->copy_on_write = TRUE;
1498: }
1499: else {
1500: RELEASE_PAGE(m);
1501: GIVE_UP;
1502: }
1503: }
1504:
1505: /*
1506: * (the various bits we're fiddling with here are locked by
1507: * the object's lock)
1508: */
1509:
1510: /* XXX This distorts the meaning of the copy_on_write bit */
1511:
1512: if (prot & VM_PROT_WRITE)
1513: m->copy_on_write = FALSE;
1514:
1515: /*
1516: * Put this page into the physical map.
1517: * We have to unlock the object because pmap_enter
1518: * may cause other faults.
1519: */
1520: vm_object_unlock(object);
1521:
1522: pmap_enter(map->pmap, va, VM_PAGE_TO_PHYS(m), prot , TRUE);
1523:
1524: /*
1525: * Must relock object so that paging_in_progress can be cleared.
1526: */
1527: vm_object_lock(object);
1528:
1529: /*
1530: * Unlock everything, and return
1531: */
1532:
1533: PAGE_WAKEUP(m);
1534: UNLOCK_AND_DEALLOCATE;
1535:
1536: return(KERN_SUCCESS);
1537:
1538: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.