![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_fault.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Net/2] / Net2 / vm|
Return to vm_fault.c CVS log | Up to [Net/2] / Net2 / vm |
1.1 ! root 1: /* ! 2: * Copyright (c) 1991 Regents of the University of California. ! 3: * All rights reserved. ! 4: * ! 5: * This code is derived from software contributed to Berkeley by ! 6: * The Mach Operating System project at Carnegie-Mellon University. ! 7: * ! 8: * Redistribution and use in source and binary forms, with or without ! 9: * modification, are permitted provided that the following conditions ! 10: * are met: ! 11: * 1. Redistributions of source code must retain the above copyright ! 12: * notice, this list of conditions and the following disclaimer. ! 13: * 2. Redistributions in binary form must reproduce the above copyright ! 14: * notice, this list of conditions and the following disclaimer in the ! 15: * documentation and/or other materials provided with the distribution. ! 16: * 3. All advertising materials mentioning features or use of this software ! 17: * must display the following acknowledgement: ! 18: * This product includes software developed by the University of ! 19: * California, Berkeley and its contributors. ! 20: * 4. Neither the name of the University nor the names of its contributors ! 21: * may be used to endorse or promote products derived from this software ! 22: * without specific prior written permission. ! 23: * ! 24: * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ! 25: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE ! 26: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ! 27: * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE ! 28: * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL ! 29: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS ! 30: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) ! 31: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT ! 32: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY ! 33: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ! 34: * SUCH DAMAGE. ! 35: * ! 36: * @(#)vm_fault.c 7.6 (Berkeley) 5/7/91 ! 37: * ! 38: * ! 39: * Copyright (c) 1987, 1990 Carnegie-Mellon University. ! 40: * All rights reserved. ! 41: * ! 42: * Authors: Avadis Tevanian, Jr., Michael Wayne Young ! 43: * ! 44: * Permission to use, copy, modify and distribute this software and ! 45: * its documentation is hereby granted, provided that both the copyright ! 46: * notice and this permission notice appear in all copies of the ! 47: * software, derivative works or modified versions, and any portions ! 48: * thereof, and that both notices appear in supporting documentation. ! 49: * ! 50: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" ! 51: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND ! 52: * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. ! 53: * ! 54: * Carnegie Mellon requests users of this software to return to ! 55: * ! 56: * Software Distribution Coordinator or [email protected] ! 57: * School of Computer Science ! 58: * Carnegie Mellon University ! 59: * Pittsburgh PA 15213-3890 ! 60: * ! 61: * any improvements or extensions that they make and grant Carnegie the ! 62: * rights to redistribute these changes. ! 63: */ ! 64: ! 65: /* ! 66: * Page fault handling module. ! 67: */ ! 68: ! 69: #include "param.h" ! 70: ! 71: #include "vm.h" ! 72: #include "vm_page.h" ! 73: #include "vm_pageout.h" ! 74: ! 75: /* ! 76: * vm_fault: ! 77: * ! 78: * Handle a page fault occuring at the given address, ! 79: * requiring the given permissions, in the map specified. ! 80: * If successful, the page is inserted into the ! 81: * associated physical map. ! 82: * ! 83: * NOTE: the given address should be truncated to the ! 84: * proper page address. ! 85: * ! 86: * KERN_SUCCESS is returned if the page fault is handled; otherwise, ! 87: * a standard error specifying why the fault is fatal is returned. ! 88: * ! 89: * ! 90: * The map in question must be referenced, and remains so. ! 91: * Caller may hold no locks. ! 92: */ ! 93: vm_fault(map, vaddr, fault_type, change_wiring) ! 94: vm_map_t map; ! 95: vm_offset_t vaddr; ! 96: vm_prot_t fault_type; ! 97: boolean_t change_wiring; ! 98: { ! 99: vm_object_t first_object; ! 100: vm_offset_t first_offset; ! 101: vm_map_entry_t entry; ! 102: register vm_object_t object; ! 103: register vm_offset_t offset; ! 104: register vm_page_t m; ! 105: vm_page_t first_m; ! 106: vm_prot_t prot; ! 107: int result; ! 108: boolean_t wired; ! 109: boolean_t su; ! 110: boolean_t lookup_still_valid; ! 111: boolean_t page_exists; ! 112: vm_page_t old_m; ! 113: vm_object_t next_object; ! 114: ! 115: vm_stat.faults++; /* needs lock XXX */ ! 116: /* ! 117: * Recovery actions ! 118: */ ! 119: #define FREE_PAGE(m) { \ ! 120: PAGE_WAKEUP(m); \ ! 121: vm_page_lock_queues(); \ ! 122: vm_page_free(m); \ ! 123: vm_page_unlock_queues(); \ ! 124: } ! 125: ! 126: #define RELEASE_PAGE(m) { \ ! 127: PAGE_WAKEUP(m); \ ! 128: vm_page_lock_queues(); \ ! 129: vm_page_activate(m); \ ! 130: vm_page_unlock_queues(); \ ! 131: } ! 132: ! 133: #define UNLOCK_MAP { \ ! 134: if (lookup_still_valid) { \ ! 135: vm_map_lookup_done(map, entry); \ ! 136: lookup_still_valid = FALSE; \ ! 137: } \ ! 138: } ! 139: ! 140: #define UNLOCK_THINGS { \ ! 141: object->paging_in_progress--; \ ! 142: vm_object_unlock(object); \ ! 143: if (object != first_object) { \ ! 144: vm_object_lock(first_object); \ ! 145: FREE_PAGE(first_m); \ ! 146: first_object->paging_in_progress--; \ ! 147: vm_object_unlock(first_object); \ ! 148: } \ ! 149: UNLOCK_MAP; \ ! 150: } ! 151: ! 152: #define UNLOCK_AND_DEALLOCATE { \ ! 153: UNLOCK_THINGS; \ ! 154: vm_object_deallocate(first_object); \ ! 155: } ! 156: ! 157: RetryFault: ; ! 158: ! 159: /* ! 160: * Find the backing store object and offset into ! 161: * it to begin the search. ! 162: */ ! 163: ! 164: if ((result = vm_map_lookup(&map, vaddr, fault_type, &entry, ! 165: &first_object, &first_offset, ! 166: &prot, &wired, &su)) != KERN_SUCCESS) { ! 167: return(result); ! 168: } ! 169: lookup_still_valid = TRUE; ! 170: ! 171: if (wired) ! 172: fault_type = prot; ! 173: ! 174: first_m = NULL; ! 175: ! 176: /* ! 177: * Make a reference to this object to ! 178: * prevent its disposal while we are messing with ! 179: * it. Once we have the reference, the map is free ! 180: * to be diddled. Since objects reference their ! 181: * shadows (and copies), they will stay around as well. ! 182: */ ! 183: ! 184: vm_object_lock(first_object); ! 185: ! 186: first_object->ref_count++; ! 187: first_object->paging_in_progress++; ! 188: ! 189: /* ! 190: * INVARIANTS (through entire routine): ! 191: * ! 192: * 1) At all times, we must either have the object ! 193: * lock or a busy page in some object to prevent ! 194: * some other thread from trying to bring in ! 195: * the same page. ! 196: * ! 197: * Note that we cannot hold any locks during the ! 198: * pager access or when waiting for memory, so ! 199: * we use a busy page then. ! 200: * ! 201: * Note also that we aren't as concerned about ! 202: * more than one thead attempting to pager_data_unlock ! 203: * the same page at once, so we don't hold the page ! 204: * as busy then, but do record the highest unlock ! 205: * value so far. [Unlock requests may also be delivered ! 206: * out of order.] ! 207: * ! 208: * 2) Once we have a busy page, we must remove it from ! 209: * the pageout queues, so that the pageout daemon ! 210: * will not grab it away. ! 211: * ! 212: * 3) To prevent another thread from racing us down the ! 213: * shadow chain and entering a new page in the top ! 214: * object before we do, we must keep a busy page in ! 215: * the top object while following the shadow chain. ! 216: * ! 217: * 4) We must increment paging_in_progress on any object ! 218: * for which we have a busy page, to prevent ! 219: * vm_object_collapse from removing the busy page ! 220: * without our noticing. ! 221: */ ! 222: ! 223: /* ! 224: * Search for the page at object/offset. ! 225: */ ! 226: ! 227: object = first_object; ! 228: offset = first_offset; ! 229: ! 230: /* ! 231: * See whether this page is resident ! 232: */ ! 233: ! 234: while (TRUE) { ! 235: m = vm_page_lookup(object, offset); ! 236: if (m != NULL) { ! 237: /* ! 238: * If the page is being brought in, ! 239: * wait for it and then retry. ! 240: */ ! 241: if (m->busy) { ! 242: #ifdef DOTHREADS ! 243: int wait_result; ! 244: ! 245: PAGE_ASSERT_WAIT(m, !change_wiring); ! 246: UNLOCK_THINGS; ! 247: thread_block(); ! 248: wait_result = current_thread()->wait_result; ! 249: vm_object_deallocate(first_object); ! 250: if (wait_result != THREAD_AWAKENED) ! 251: return(KERN_SUCCESS); ! 252: goto RetryFault; ! 253: #else ! 254: PAGE_ASSERT_WAIT(m, !change_wiring); ! 255: UNLOCK_THINGS; ! 256: thread_block(); ! 257: vm_object_deallocate(first_object); ! 258: goto RetryFault; ! 259: #endif ! 260: } ! 261: ! 262: if (m->absent) ! 263: panic("vm_fault: absent"); ! 264: ! 265: /* ! 266: * If the desired access to this page has ! 267: * been locked out, request that it be unlocked. ! 268: */ ! 269: ! 270: if (fault_type & m->page_lock) { ! 271: #ifdef DOTHREADS ! 272: int wait_result; ! 273: ! 274: if ((fault_type & m->unlock_request) != fault_type) ! 275: panic("vm_fault: pager_data_unlock"); ! 276: ! 277: PAGE_ASSERT_WAIT(m, !change_wiring); ! 278: UNLOCK_THINGS; ! 279: thread_block(); ! 280: wait_result = current_thread()->wait_result; ! 281: vm_object_deallocate(first_object); ! 282: if (wait_result != THREAD_AWAKENED) ! 283: return(KERN_SUCCESS); ! 284: goto RetryFault; ! 285: #else ! 286: if ((fault_type & m->unlock_request) != fault_type) ! 287: panic("vm_fault: pager_data_unlock"); ! 288: ! 289: PAGE_ASSERT_WAIT(m, !change_wiring); ! 290: UNLOCK_THINGS; ! 291: thread_block(); ! 292: vm_object_deallocate(first_object); ! 293: goto RetryFault; ! 294: #endif ! 295: } ! 296: ! 297: /* ! 298: * Remove the page from the pageout daemon's ! 299: * reach while we play with it. ! 300: */ ! 301: ! 302: vm_page_lock_queues(); ! 303: if (m->inactive) { ! 304: queue_remove(&vm_page_queue_inactive, m, ! 305: vm_page_t, pageq); ! 306: m->inactive = FALSE; ! 307: vm_page_inactive_count--; ! 308: vm_stat.reactivations++; ! 309: } ! 310: ! 311: if (m->active) { ! 312: queue_remove(&vm_page_queue_active, m, ! 313: vm_page_t, pageq); ! 314: m->active = FALSE; ! 315: vm_page_active_count--; ! 316: } ! 317: vm_page_unlock_queues(); ! 318: ! 319: /* ! 320: * Mark page busy for other threads. ! 321: */ ! 322: m->busy = TRUE; ! 323: m->absent = FALSE; ! 324: break; ! 325: } ! 326: ! 327: if (((object->pager != NULL) && ! 328: (!change_wiring || wired)) ! 329: || (object == first_object)) { ! 330: ! 331: /* ! 332: * Allocate a new page for this object/offset ! 333: * pair. ! 334: */ ! 335: ! 336: m = vm_page_alloc(object, offset); ! 337: ! 338: if (m == NULL) { ! 339: UNLOCK_AND_DEALLOCATE; ! 340: VM_WAIT; ! 341: goto RetryFault; ! 342: } ! 343: } ! 344: ! 345: if ((object->pager != NULL) && ! 346: (!change_wiring || wired)) { ! 347: int rv; ! 348: ! 349: /* ! 350: * Now that we have a busy page, we can ! 351: * release the object lock. ! 352: */ ! 353: vm_object_unlock(object); ! 354: ! 355: /* ! 356: * Call the pager to retrieve the data, if any, ! 357: * after releasing the lock on the map. ! 358: */ ! 359: UNLOCK_MAP; ! 360: ! 361: rv = vm_pager_get(object->pager, m, TRUE); ! 362: if (rv == VM_PAGER_OK) { ! 363: /* ! 364: * Found the page. ! 365: * Leave it busy while we play with it. ! 366: */ ! 367: vm_object_lock(object); ! 368: ! 369: /* ! 370: * Relookup in case pager changed page. ! 371: * Pager is responsible for disposition ! 372: * of old page if moved. ! 373: */ ! 374: m = vm_page_lookup(object, offset); ! 375: ! 376: vm_stat.pageins++; ! 377: m->fake = FALSE; ! 378: pmap_clear_modify(VM_PAGE_TO_PHYS(m)); ! 379: break; ! 380: } ! 381: ! 382: /* ! 383: * Remove the bogus page (which does not ! 384: * exist at this object/offset); before ! 385: * doing so, we must get back our object ! 386: * lock to preserve our invariant. ! 387: * ! 388: * Also wake up any other thread that may want ! 389: * to bring in this page. ! 390: * ! 391: * If this is the top-level object, we must ! 392: * leave the busy page to prevent another ! 393: * thread from rushing past us, and inserting ! 394: * the page in that object at the same time ! 395: * that we are. ! 396: */ ! 397: ! 398: vm_object_lock(object); ! 399: /* ! 400: * Data outside the range of the pager; an error ! 401: */ ! 402: if (rv == VM_PAGER_BAD) { ! 403: FREE_PAGE(m); ! 404: UNLOCK_AND_DEALLOCATE; ! 405: return(KERN_PROTECTION_FAILURE); /* XXX */ ! 406: } ! 407: if (object != first_object) { ! 408: FREE_PAGE(m); ! 409: /* ! 410: * XXX - we cannot just fall out at this ! 411: * point, m has been freed and is invalid! ! 412: */ ! 413: } ! 414: } ! 415: ! 416: /* ! 417: * We get here if the object has no pager (or unwiring) ! 418: * or the pager doesn't have the page. ! 419: */ ! 420: if (object == first_object) ! 421: first_m = m; ! 422: ! 423: /* ! 424: * Move on to the next object. Lock the next ! 425: * object before unlocking the current one. ! 426: */ ! 427: ! 428: offset += object->shadow_offset; ! 429: next_object = object->shadow; ! 430: if (next_object == NULL) { ! 431: /* ! 432: * If there's no object left, fill the page ! 433: * in the top object with zeros. ! 434: */ ! 435: if (object != first_object) { ! 436: object->paging_in_progress--; ! 437: vm_object_unlock(object); ! 438: ! 439: object = first_object; ! 440: offset = first_offset; ! 441: m = first_m; ! 442: vm_object_lock(object); ! 443: } ! 444: first_m = NULL; ! 445: ! 446: vm_page_zero_fill(m); ! 447: vm_stat.zero_fill_count++; ! 448: m->fake = FALSE; ! 449: m->absent = FALSE; ! 450: break; ! 451: } ! 452: else { ! 453: vm_object_lock(next_object); ! 454: if (object != first_object) ! 455: object->paging_in_progress--; ! 456: vm_object_unlock(object); ! 457: object = next_object; ! 458: object->paging_in_progress++; ! 459: } ! 460: } ! 461: ! 462: if (m->absent || m->active || m->inactive || !m->busy) ! 463: panic("vm_fault: absent or active or inactive or not busy after main loop"); ! 464: ! 465: /* ! 466: * PAGE HAS BEEN FOUND. ! 467: * [Loop invariant still holds -- the object lock ! 468: * is held.] ! 469: */ ! 470: ! 471: old_m = m; /* save page that would be copied */ ! 472: ! 473: /* ! 474: * If the page is being written, but isn't ! 475: * already owned by the top-level object, ! 476: * we have to copy it into a new page owned ! 477: * by the top-level object. ! 478: */ ! 479: ! 480: if (object != first_object) { ! 481: /* ! 482: * We only really need to copy if we ! 483: * want to write it. ! 484: */ ! 485: ! 486: if (fault_type & VM_PROT_WRITE) { ! 487: ! 488: /* ! 489: * If we try to collapse first_object at this ! 490: * point, we may deadlock when we try to get ! 491: * the lock on an intermediate object (since we ! 492: * have the bottom object locked). We can't ! 493: * unlock the bottom object, because the page ! 494: * we found may move (by collapse) if we do. ! 495: * ! 496: * Instead, we first copy the page. Then, when ! 497: * we have no more use for the bottom object, ! 498: * we unlock it and try to collapse. ! 499: * ! 500: * Note that we copy the page even if we didn't ! 501: * need to... that's the breaks. ! 502: */ ! 503: ! 504: /* ! 505: * We already have an empty page in ! 506: * first_object - use it. ! 507: */ ! 508: ! 509: vm_page_copy(m, first_m); ! 510: first_m->fake = FALSE; ! 511: first_m->absent = FALSE; ! 512: ! 513: /* ! 514: * If another map is truly sharing this ! 515: * page with us, we have to flush all ! 516: * uses of the original page, since we ! 517: * can't distinguish those which want the ! 518: * original from those which need the ! 519: * new copy. ! 520: * ! 521: * XXX If we know that only one map has ! 522: * access to this page, then we could ! 523: * avoid the pmap_page_protect() call. ! 524: */ ! 525: ! 526: vm_page_lock_queues(); ! 527: vm_page_deactivate(m); ! 528: pmap_page_protect(VM_PAGE_TO_PHYS(m), VM_PROT_NONE); ! 529: vm_page_unlock_queues(); ! 530: ! 531: /* ! 532: * We no longer need the old page or object. ! 533: */ ! 534: PAGE_WAKEUP(m); ! 535: object->paging_in_progress--; ! 536: vm_object_unlock(object); ! 537: ! 538: /* ! 539: * Only use the new page below... ! 540: */ ! 541: ! 542: vm_stat.cow_faults++; ! 543: m = first_m; ! 544: object = first_object; ! 545: offset = first_offset; ! 546: ! 547: /* ! 548: * Now that we've gotten the copy out of the ! 549: * way, let's try to collapse the top object. ! 550: */ ! 551: vm_object_lock(object); ! 552: /* ! 553: * But we have to play ugly games with ! 554: * paging_in_progress to do that... ! 555: */ ! 556: object->paging_in_progress--; ! 557: vm_object_collapse(object); ! 558: object->paging_in_progress++; ! 559: } ! 560: else { ! 561: prot &= (~VM_PROT_WRITE); ! 562: m->copy_on_write = TRUE; ! 563: } ! 564: } ! 565: ! 566: if (m->active || m->inactive) ! 567: panic("vm_fault: active or inactive before copy object handling"); ! 568: ! 569: /* ! 570: * If the page is being written, but hasn't been ! 571: * copied to the copy-object, we have to copy it there. ! 572: */ ! 573: RetryCopy: ! 574: if (first_object->copy != NULL) { ! 575: vm_object_t copy_object = first_object->copy; ! 576: vm_offset_t copy_offset; ! 577: vm_page_t copy_m; ! 578: ! 579: /* ! 580: * We only need to copy if we want to write it. ! 581: */ ! 582: if ((fault_type & VM_PROT_WRITE) == 0) { ! 583: prot &= ~VM_PROT_WRITE; ! 584: m->copy_on_write = TRUE; ! 585: } ! 586: else { ! 587: /* ! 588: * Try to get the lock on the copy_object. ! 589: */ ! 590: if (!vm_object_lock_try(copy_object)) { ! 591: vm_object_unlock(object); ! 592: /* should spin a bit here... */ ! 593: vm_object_lock(object); ! 594: goto RetryCopy; ! 595: } ! 596: ! 597: /* ! 598: * Make another reference to the copy-object, ! 599: * to keep it from disappearing during the ! 600: * copy. ! 601: */ ! 602: copy_object->ref_count++; ! 603: ! 604: /* ! 605: * Does the page exist in the copy? ! 606: */ ! 607: copy_offset = first_offset ! 608: - copy_object->shadow_offset; ! 609: copy_m = vm_page_lookup(copy_object, copy_offset); ! 610: if (page_exists = (copy_m != NULL)) { ! 611: if (copy_m->busy) { ! 612: #ifdef DOTHREADS ! 613: int wait_result; ! 614: ! 615: /* ! 616: * If the page is being brought ! 617: * in, wait for it and then retry. ! 618: */ ! 619: PAGE_ASSERT_WAIT(copy_m, !change_wiring); ! 620: RELEASE_PAGE(m); ! 621: copy_object->ref_count--; ! 622: vm_object_unlock(copy_object); ! 623: UNLOCK_THINGS; ! 624: thread_block(); ! 625: wait_result = current_thread()->wait_result; ! 626: vm_object_deallocate(first_object); ! 627: if (wait_result != THREAD_AWAKENED) ! 628: return(KERN_SUCCESS); ! 629: goto RetryFault; ! 630: #else ! 631: /* ! 632: * If the page is being brought ! 633: * in, wait for it and then retry. ! 634: */ ! 635: PAGE_ASSERT_WAIT(copy_m, !change_wiring); ! 636: RELEASE_PAGE(m); ! 637: copy_object->ref_count--; ! 638: vm_object_unlock(copy_object); ! 639: UNLOCK_THINGS; ! 640: thread_block(); ! 641: vm_object_deallocate(first_object); ! 642: goto RetryFault; ! 643: #endif ! 644: } ! 645: } ! 646: ! 647: /* ! 648: * If the page is not in memory (in the object) ! 649: * and the object has a pager, we have to check ! 650: * if the pager has the data in secondary ! 651: * storage. ! 652: */ ! 653: if (!page_exists) { ! 654: ! 655: /* ! 656: * If we don't allocate a (blank) page ! 657: * here... another thread could try ! 658: * to page it in, allocate a page, and ! 659: * then block on the busy page in its ! 660: * shadow (first_object). Then we'd ! 661: * trip over the busy page after we ! 662: * found that the copy_object's pager ! 663: * doesn't have the page... ! 664: */ ! 665: copy_m = vm_page_alloc(copy_object, ! 666: copy_offset); ! 667: if (copy_m == NULL) { ! 668: /* ! 669: * Wait for a page, then retry. ! 670: */ ! 671: RELEASE_PAGE(m); ! 672: copy_object->ref_count--; ! 673: vm_object_unlock(copy_object); ! 674: UNLOCK_AND_DEALLOCATE; ! 675: VM_WAIT; ! 676: goto RetryFault; ! 677: } ! 678: ! 679: if (copy_object->pager != NULL) { ! 680: vm_object_unlock(object); ! 681: vm_object_unlock(copy_object); ! 682: UNLOCK_MAP; ! 683: ! 684: page_exists = vm_pager_has_page( ! 685: copy_object->pager, ! 686: (copy_offset + copy_object->paging_offset)); ! 687: ! 688: vm_object_lock(copy_object); ! 689: ! 690: /* ! 691: * Since the map is unlocked, someone ! 692: * else could have copied this object ! 693: * and put a different copy_object ! 694: * between the two. Or, the last ! 695: * reference to the copy-object (other ! 696: * than the one we have) may have ! 697: * disappeared - if that has happened, ! 698: * we don't need to make the copy. ! 699: */ ! 700: if (copy_object->shadow != object || ! 701: copy_object->ref_count == 1) { ! 702: /* ! 703: * Gaah... start over! ! 704: */ ! 705: FREE_PAGE(copy_m); ! 706: vm_object_unlock(copy_object); ! 707: vm_object_deallocate(copy_object); ! 708: /* may block */ ! 709: vm_object_lock(object); ! 710: goto RetryCopy; ! 711: } ! 712: vm_object_lock(object); ! 713: ! 714: if (page_exists) { ! 715: /* ! 716: * We didn't need the page ! 717: */ ! 718: FREE_PAGE(copy_m); ! 719: } ! 720: } ! 721: } ! 722: if (!page_exists) { ! 723: /* ! 724: * Must copy page into copy-object. ! 725: */ ! 726: vm_page_copy(m, copy_m); ! 727: copy_m->fake = FALSE; ! 728: copy_m->absent = FALSE; ! 729: ! 730: /* ! 731: * Things to remember: ! 732: * 1. The copied page must be marked 'dirty' ! 733: * so it will be paged out to the copy ! 734: * object. ! 735: * 2. If the old page was in use by any users ! 736: * of the copy-object, it must be removed ! 737: * from all pmaps. (We can't know which ! 738: * pmaps use it.) ! 739: */ ! 740: vm_page_lock_queues(); ! 741: pmap_page_protect(VM_PAGE_TO_PHYS(old_m), ! 742: VM_PROT_NONE); ! 743: copy_m->clean = FALSE; ! 744: vm_page_activate(copy_m); /* XXX */ ! 745: vm_page_unlock_queues(); ! 746: ! 747: PAGE_WAKEUP(copy_m); ! 748: } ! 749: /* ! 750: * The reference count on copy_object must be ! 751: * at least 2: one for our extra reference, ! 752: * and at least one from the outside world ! 753: * (we checked that when we last locked ! 754: * copy_object). ! 755: */ ! 756: copy_object->ref_count--; ! 757: vm_object_unlock(copy_object); ! 758: m->copy_on_write = FALSE; ! 759: } ! 760: } ! 761: ! 762: if (m->active || m->inactive) ! 763: panic("vm_fault: active or inactive before retrying lookup"); ! 764: ! 765: /* ! 766: * We must verify that the maps have not changed ! 767: * since our last lookup. ! 768: */ ! 769: ! 770: if (!lookup_still_valid) { ! 771: vm_object_t retry_object; ! 772: vm_offset_t retry_offset; ! 773: vm_prot_t retry_prot; ! 774: ! 775: /* ! 776: * Since map entries may be pageable, make sure we can ! 777: * take a page fault on them. ! 778: */ ! 779: vm_object_unlock(object); ! 780: ! 781: /* ! 782: * To avoid trying to write_lock the map while another ! 783: * thread has it read_locked (in vm_map_pageable), we ! 784: * do not try for write permission. If the page is ! 785: * still writable, we will get write permission. If it ! 786: * is not, or has been marked needs_copy, we enter the ! 787: * mapping without write permission, and will merely ! 788: * take another fault. ! 789: */ ! 790: result = vm_map_lookup(&map, vaddr, ! 791: fault_type & ~VM_PROT_WRITE, &entry, ! 792: &retry_object, &retry_offset, &retry_prot, ! 793: &wired, &su); ! 794: ! 795: vm_object_lock(object); ! 796: ! 797: /* ! 798: * If we don't need the page any longer, put it on the ! 799: * active list (the easiest thing to do here). If no ! 800: * one needs it, pageout will grab it eventually. ! 801: */ ! 802: ! 803: if (result != KERN_SUCCESS) { ! 804: RELEASE_PAGE(m); ! 805: UNLOCK_AND_DEALLOCATE; ! 806: return(result); ! 807: } ! 808: ! 809: lookup_still_valid = TRUE; ! 810: ! 811: if ((retry_object != first_object) || ! 812: (retry_offset != first_offset)) { ! 813: RELEASE_PAGE(m); ! 814: UNLOCK_AND_DEALLOCATE; ! 815: goto RetryFault; ! 816: } ! 817: ! 818: /* ! 819: * Check whether the protection has changed or the object ! 820: * has been copied while we left the map unlocked. ! 821: * Changing from read to write permission is OK - we leave ! 822: * the page write-protected, and catch the write fault. ! 823: * Changing from write to read permission means that we ! 824: * can't mark the page write-enabled after all. ! 825: */ ! 826: prot &= retry_prot; ! 827: if (m->copy_on_write) ! 828: prot &= ~VM_PROT_WRITE; ! 829: } ! 830: ! 831: /* ! 832: * (the various bits we're fiddling with here are locked by ! 833: * the object's lock) ! 834: */ ! 835: ! 836: /* XXX This distorts the meaning of the copy_on_write bit */ ! 837: ! 838: if (prot & VM_PROT_WRITE) ! 839: m->copy_on_write = FALSE; ! 840: ! 841: /* ! 842: * It's critically important that a wired-down page be faulted ! 843: * only once in each map for which it is wired. ! 844: */ ! 845: ! 846: if (m->active || m->inactive) ! 847: panic("vm_fault: active or inactive before pmap_enter"); ! 848: ! 849: vm_object_unlock(object); ! 850: ! 851: /* ! 852: * Put this page into the physical map. ! 853: * We had to do the unlock above because pmap_enter ! 854: * may cause other faults. We don't put the ! 855: * page back on the active queue until later so ! 856: * that the page-out daemon won't find us (yet). ! 857: */ ! 858: ! 859: pmap_enter(map->pmap, vaddr, VM_PAGE_TO_PHYS(m), ! 860: prot & ~(m->page_lock), wired); ! 861: ! 862: /* ! 863: * If the page is not wired down, then put it where the ! 864: * pageout daemon can find it. ! 865: */ ! 866: vm_object_lock(object); ! 867: vm_page_lock_queues(); ! 868: if (change_wiring) { ! 869: if (wired) ! 870: vm_page_wire(m); ! 871: else ! 872: vm_page_unwire(m); ! 873: } ! 874: else ! 875: vm_page_activate(m); ! 876: vm_page_unlock_queues(); ! 877: ! 878: /* ! 879: * Unlock everything, and return ! 880: */ ! 881: ! 882: PAGE_WAKEUP(m); ! 883: UNLOCK_AND_DEALLOCATE; ! 884: ! 885: return(KERN_SUCCESS); ! 886: ! 887: } ! 888: ! 889: /* ! 890: * vm_fault_wire: ! 891: * ! 892: * Wire down a range of virtual addresses in a map. ! 893: */ ! 894: void vm_fault_wire(map, start, end) ! 895: vm_map_t map; ! 896: vm_offset_t start, end; ! 897: { ! 898: ! 899: register vm_offset_t va; ! 900: register pmap_t pmap; ! 901: ! 902: pmap = vm_map_pmap(map); ! 903: ! 904: /* ! 905: * Inform the physical mapping system that the ! 906: * range of addresses may not fault, so that ! 907: * page tables and such can be locked down as well. ! 908: */ ! 909: ! 910: pmap_pageable(pmap, start, end, FALSE); ! 911: ! 912: /* ! 913: * We simulate a fault to get the page and enter it ! 914: * in the physical map. ! 915: */ ! 916: ! 917: for (va = start; va < end; va += PAGE_SIZE) { ! 918: (void) vm_fault(map, va, VM_PROT_NONE, TRUE); ! 919: } ! 920: } ! 921: ! 922: ! 923: /* ! 924: * vm_fault_unwire: ! 925: * ! 926: * Unwire a range of virtual addresses in a map. ! 927: */ ! 928: void vm_fault_unwire(map, start, end) ! 929: vm_map_t map; ! 930: vm_offset_t start, end; ! 931: { ! 932: ! 933: register vm_offset_t va, pa; ! 934: register pmap_t pmap; ! 935: ! 936: pmap = vm_map_pmap(map); ! 937: ! 938: /* ! 939: * Since the pages are wired down, we must be able to ! 940: * get their mappings from the physical map system. ! 941: */ ! 942: ! 943: vm_page_lock_queues(); ! 944: ! 945: for (va = start; va < end; va += PAGE_SIZE) { ! 946: pa = pmap_extract(pmap, va); ! 947: if (pa == (vm_offset_t) 0) { ! 948: panic("unwire: page not in pmap"); ! 949: } ! 950: pmap_change_wiring(pmap, va, FALSE); ! 951: vm_page_unwire(PHYS_TO_VM_PAGE(pa)); ! 952: } ! 953: vm_page_unlock_queues(); ! 954: ! 955: /* ! 956: * Inform the physical mapping system that the range ! 957: * of addresses may fault, so that page tables and ! 958: * such may be unwired themselves. ! 959: */ ! 960: ! 961: pmap_pageable(pmap, start, end, TRUE); ! 962: ! 963: } ! 964: ! 965: /* ! 966: * Routine: ! 967: * vm_fault_copy_entry ! 968: * Function: ! 969: * Copy all of the pages from a wired-down map entry to another. ! 970: * ! 971: * In/out conditions: ! 972: * The source and destination maps must be locked for write. ! 973: * The source map entry must be wired down (or be a sharing map ! 974: * entry corresponding to a main map entry that is wired down). ! 975: */ ! 976: ! 977: void vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry) ! 978: vm_map_t dst_map; ! 979: vm_map_t src_map; ! 980: vm_map_entry_t dst_entry; ! 981: vm_map_entry_t src_entry; ! 982: { ! 983: ! 984: vm_object_t dst_object; ! 985: vm_object_t src_object; ! 986: vm_offset_t dst_offset; ! 987: vm_offset_t src_offset; ! 988: vm_prot_t prot; ! 989: vm_offset_t vaddr; ! 990: vm_page_t dst_m; ! 991: vm_page_t src_m; ! 992: ! 993: #ifdef lint ! 994: src_map++; ! 995: #endif lint ! 996: ! 997: src_object = src_entry->object.vm_object; ! 998: src_offset = src_entry->offset; ! 999: ! 1000: /* ! 1001: * Create the top-level object for the destination entry. ! 1002: * (Doesn't actually shadow anything - we copy the pages ! 1003: * directly.) ! 1004: */ ! 1005: dst_object = vm_object_allocate( ! 1006: (vm_size_t) (dst_entry->end - dst_entry->start)); ! 1007: ! 1008: dst_entry->object.vm_object = dst_object; ! 1009: dst_entry->offset = 0; ! 1010: ! 1011: prot = dst_entry->max_protection; ! 1012: ! 1013: /* ! 1014: * Loop through all of the pages in the entry's range, copying ! 1015: * each one from the source object (it should be there) to the ! 1016: * destination object. ! 1017: */ ! 1018: for (vaddr = dst_entry->start, dst_offset = 0; ! 1019: vaddr < dst_entry->end; ! 1020: vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) { ! 1021: ! 1022: /* ! 1023: * Allocate a page in the destination object ! 1024: */ ! 1025: vm_object_lock(dst_object); ! 1026: do { ! 1027: dst_m = vm_page_alloc(dst_object, dst_offset); ! 1028: if (dst_m == NULL) { ! 1029: vm_object_unlock(dst_object); ! 1030: VM_WAIT; ! 1031: vm_object_lock(dst_object); ! 1032: } ! 1033: } while (dst_m == NULL); ! 1034: ! 1035: /* ! 1036: * Find the page in the source object, and copy it in. ! 1037: * (Because the source is wired down, the page will be ! 1038: * in memory.) ! 1039: */ ! 1040: vm_object_lock(src_object); ! 1041: src_m = vm_page_lookup(src_object, dst_offset + src_offset); ! 1042: if (src_m == NULL) ! 1043: panic("vm_fault_copy_wired: page missing"); ! 1044: ! 1045: vm_page_copy(src_m, dst_m); ! 1046: ! 1047: /* ! 1048: * Enter it in the pmap... ! 1049: */ ! 1050: vm_object_unlock(src_object); ! 1051: vm_object_unlock(dst_object); ! 1052: ! 1053: pmap_enter(dst_map->pmap, vaddr, VM_PAGE_TO_PHYS(dst_m), ! 1054: prot, FALSE); ! 1055: ! 1056: /* ! 1057: * Mark it no longer busy, and put it on the active list. ! 1058: */ ! 1059: vm_object_lock(dst_object); ! 1060: vm_page_lock_queues(); ! 1061: vm_page_activate(dst_m); ! 1062: vm_page_unlock_queues(); ! 1063: PAGE_WAKEUP(dst_m); ! 1064: vm_object_unlock(dst_object); ! 1065: } ! 1066: ! 1067: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.