![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_fault.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / OSKit-Mach / vm|
Return to vm_fault.c CVS log | Up to [GnuMach] / OSKit-Mach / vm |
1.1 ! root 1: /* ! 2: * Mach Operating System ! 3: * Copyright (c) 1994,1990,1989,1988,1987 Carnegie Mellon University. ! 4: * Copyright (c) 1993,1994 The University of Utah and ! 5: * the Computer Systems Laboratory (CSL). ! 6: * All rights reserved. ! 7: * ! 8: * Permission to use, copy, modify and distribute this software and its ! 9: * documentation is hereby granted, provided that both the copyright ! 10: * notice and this permission notice appear in all copies of the ! 11: * software, derivative works or modified versions, and any portions ! 12: * thereof, and that both notices appear in supporting documentation. ! 13: * ! 14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF ! 15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY ! 16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF ! 17: * THIS SOFTWARE. ! 18: * ! 19: * Carnegie Mellon requests users of this software to return to ! 20: * ! 21: * Software Distribution Coordinator or [email protected] ! 22: * School of Computer Science ! 23: * Carnegie Mellon University ! 24: * Pittsburgh PA 15213-3890 ! 25: * ! 26: * any improvements or extensions that they make and grant Carnegie Mellon ! 27: * the rights to redistribute these changes. ! 28: */ ! 29: /* ! 30: * File: vm_fault.c ! 31: * Author: Avadis Tevanian, Jr., Michael Wayne Young ! 32: * ! 33: * Page fault handling module. ! 34: */ ! 35: #include <mach_pagemap.h> ! 36: #include <mach_kdb.h> ! 37: #include <mach_pcsample.h> ! 38: ! 39: ! 40: #include <vm/vm_fault.h> ! 41: #include <mach/kern_return.h> ! 42: #include <mach/message.h> /* for error codes */ ! 43: #include <kern/counters.h> ! 44: #include <kern/thread.h> ! 45: #include <kern/sched_prim.h> ! 46: #include <vm/vm_map.h> ! 47: #include <vm/vm_object.h> ! 48: #include <vm/vm_page.h> ! 49: #include <vm/pmap.h> ! 50: #include <mach/vm_statistics.h> ! 51: #include <vm/vm_pageout.h> ! 52: #include <mach/vm_param.h> ! 53: #include <mach/memory_object.h> ! 54: #include "memory_object_user.h" ! 55: /* For memory_object_data_{request,unlock} */ ! 56: #include <kern/mach_param.h> ! 57: #include <kern/macro_help.h> ! 58: #include <kern/zalloc.h> ! 59: ! 60: #if MACH_PCSAMPLE ! 61: #include <kern/pc_sample.h> ! 62: #endif ! 63: ! 64: ! 65: ! 66: /* ! 67: * State needed by vm_fault_continue. ! 68: * This is a little hefty to drop directly ! 69: * into the thread structure. ! 70: */ ! 71: typedef struct vm_fault_state { ! 72: struct vm_map *vmf_map; ! 73: vm_offset_t vmf_vaddr; ! 74: vm_prot_t vmf_fault_type; ! 75: boolean_t vmf_change_wiring; ! 76: void (*vmf_continuation)(); ! 77: vm_map_version_t vmf_version; ! 78: boolean_t vmf_wired; ! 79: struct vm_object *vmf_object; ! 80: vm_offset_t vmf_offset; ! 81: vm_prot_t vmf_prot; ! 82: ! 83: boolean_t vmfp_backoff; ! 84: struct vm_object *vmfp_object; ! 85: vm_offset_t vmfp_offset; ! 86: struct vm_page *vmfp_first_m; ! 87: vm_prot_t vmfp_access; ! 88: } vm_fault_state_t; ! 89: ! 90: zone_t vm_fault_state_zone = 0; ! 91: ! 92: int vm_object_absent_max = 50; ! 93: ! 94: int vm_fault_debug = 0; ! 95: ! 96: boolean_t vm_fault_dirty_handling = FALSE; ! 97: boolean_t vm_fault_interruptible = TRUE; ! 98: ! 99: boolean_t software_reference_bits = TRUE; ! 100: ! 101: #if MACH_KDB ! 102: extern struct db_watchpoint *db_watchpoint_list; ! 103: #endif /* MACH_KDB */ ! 104: ! 105: /* ! 106: * Routine: vm_fault_init ! 107: * Purpose: ! 108: * Initialize our private data structures. ! 109: */ ! 110: void vm_fault_init() ! 111: { ! 112: vm_fault_state_zone = zinit(sizeof(vm_fault_state_t), ! 113: THREAD_MAX * sizeof(vm_fault_state_t), ! 114: sizeof(vm_fault_state_t), ! 115: 0, "vm fault state"); ! 116: } ! 117: ! 118: /* ! 119: * Routine: vm_fault_cleanup ! 120: * Purpose: ! 121: * Clean up the result of vm_fault_page. ! 122: * Results: ! 123: * The paging reference for "object" is released. ! 124: * "object" is unlocked. ! 125: * If "top_page" is not null, "top_page" is ! 126: * freed and the paging reference for the object ! 127: * containing it is released. ! 128: * ! 129: * In/out conditions: ! 130: * "object" must be locked. ! 131: */ ! 132: void ! 133: vm_fault_cleanup(object, top_page) ! 134: register vm_object_t object; ! 135: register vm_page_t top_page; ! 136: { ! 137: vm_object_paging_end(object); ! 138: vm_object_unlock(object); ! 139: ! 140: if (top_page != VM_PAGE_NULL) { ! 141: object = top_page->object; ! 142: vm_object_lock(object); ! 143: VM_PAGE_FREE(top_page); ! 144: vm_object_paging_end(object); ! 145: vm_object_unlock(object); ! 146: } ! 147: } ! 148: ! 149: ! 150: #if MACH_PCSAMPLE ! 151: /* ! 152: * Do PC sampling on current thread, assuming ! 153: * that it is the thread taking this page fault. ! 154: * ! 155: * Must check for THREAD_NULL, since faults ! 156: * can occur before threads are running. ! 157: */ ! 158: ! 159: #define vm_stat_sample(flavor) \ ! 160: MACRO_BEGIN \ ! 161: thread_t _thread_ = current_thread(); \ ! 162: \ ! 163: if (_thread_ != THREAD_NULL) \ ! 164: take_pc_sample_macro(_thread_, (flavor)); \ ! 165: MACRO_END ! 166: ! 167: #else ! 168: #define vm_stat_sample(x) ! 169: #endif /* MACH_PCSAMPLE */ ! 170: ! 171: ! 172: ! 173: /* ! 174: * Routine: vm_fault_page ! 175: * Purpose: ! 176: * Find the resident page for the virtual memory ! 177: * specified by the given virtual memory object ! 178: * and offset. ! 179: * Additional arguments: ! 180: * The required permissions for the page is given ! 181: * in "fault_type". Desired permissions are included ! 182: * in "protection". ! 183: * ! 184: * If the desired page is known to be resident (for ! 185: * example, because it was previously wired down), asserting ! 186: * the "unwiring" parameter will speed the search. ! 187: * ! 188: * If the operation can be interrupted (by thread_abort ! 189: * or thread_terminate), then the "interruptible" ! 190: * parameter should be asserted. ! 191: * ! 192: * Results: ! 193: * The page containing the proper data is returned ! 194: * in "result_page". ! 195: * ! 196: * In/out conditions: ! 197: * The source object must be locked and referenced, ! 198: * and must donate one paging reference. The reference ! 199: * is not affected. The paging reference and lock are ! 200: * consumed. ! 201: * ! 202: * If the call succeeds, the object in which "result_page" ! 203: * resides is left locked and holding a paging reference. ! 204: * If this is not the original object, a busy page in the ! 205: * original object is returned in "top_page", to prevent other ! 206: * callers from pursuing this same data, along with a paging ! 207: * reference for the original object. The "top_page" should ! 208: * be destroyed when this guarantee is no longer required. ! 209: * The "result_page" is also left busy. It is not removed ! 210: * from the pageout queues. ! 211: */ ! 212: vm_fault_return_t vm_fault_page(first_object, first_offset, ! 213: fault_type, must_be_resident, interruptible, ! 214: protection, ! 215: result_page, top_page, ! 216: resume, continuation) ! 217: /* Arguments: */ ! 218: vm_object_t first_object; /* Object to begin search */ ! 219: vm_offset_t first_offset; /* Offset into object */ ! 220: vm_prot_t fault_type; /* What access is requested */ ! 221: boolean_t must_be_resident;/* Must page be resident? */ ! 222: boolean_t interruptible; /* May fault be interrupted? */ ! 223: /* Modifies in place: */ ! 224: vm_prot_t *protection; /* Protection for mapping */ ! 225: /* Returns: */ ! 226: vm_page_t *result_page; /* Page found, if successful */ ! 227: vm_page_t *top_page; /* Page in top object, if ! 228: * not result_page. ! 229: */ ! 230: /* More arguments: */ ! 231: boolean_t resume; /* We are restarting. */ ! 232: void (*continuation)(); /* Continuation for blocking. */ ! 233: { ! 234: register ! 235: vm_page_t m; ! 236: register ! 237: vm_object_t object; ! 238: register ! 239: vm_offset_t offset; ! 240: vm_page_t first_m; ! 241: vm_object_t next_object; ! 242: vm_object_t copy_object; ! 243: boolean_t look_for_page; ! 244: vm_prot_t access_required; ! 245: ! 246: #ifdef CONTINUATIONS ! 247: if (resume) { ! 248: register vm_fault_state_t *state = ! 249: (vm_fault_state_t *) current_thread()->ith_other; ! 250: ! 251: if (state->vmfp_backoff) ! 252: goto after_block_and_backoff; ! 253: ! 254: object = state->vmfp_object; ! 255: offset = state->vmfp_offset; ! 256: first_m = state->vmfp_first_m; ! 257: access_required = state->vmfp_access; ! 258: goto after_thread_block; ! 259: } ! 260: #else /* not CONTINUATIONS */ ! 261: assert(continuation == 0); ! 262: assert(!resume); ! 263: #endif /* not CONTINUATIONS */ ! 264: ! 265: vm_stat_sample(SAMPLED_PC_VM_FAULTS_ANY); ! 266: vm_stat.faults++; /* needs lock XXX */ ! 267: ! 268: /* ! 269: * Recovery actions ! 270: */ ! 271: #define RELEASE_PAGE(m) \ ! 272: MACRO_BEGIN \ ! 273: PAGE_WAKEUP_DONE(m); \ ! 274: vm_page_lock_queues(); \ ! 275: if (!m->active && !m->inactive) \ ! 276: vm_page_activate(m); \ ! 277: vm_page_unlock_queues(); \ ! 278: MACRO_END ! 279: ! 280: if (vm_fault_dirty_handling ! 281: #if MACH_KDB ! 282: /* ! 283: * If there are watchpoints set, then ! 284: * we don't want to give away write permission ! 285: * on a read fault. Make the task write fault, ! 286: * so that the watchpoint code notices the access. ! 287: */ ! 288: || db_watchpoint_list ! 289: #endif /* MACH_KDB */ ! 290: ) { ! 291: /* ! 292: * If we aren't asking for write permission, ! 293: * then don't give it away. We're using write ! 294: * faults to set the dirty bit. ! 295: */ ! 296: if (!(fault_type & VM_PROT_WRITE)) ! 297: *protection &= ~VM_PROT_WRITE; ! 298: } ! 299: ! 300: if (!vm_fault_interruptible) ! 301: interruptible = FALSE; ! 302: ! 303: /* ! 304: * INVARIANTS (through entire routine): ! 305: * ! 306: * 1) At all times, we must either have the object ! 307: * lock or a busy page in some object to prevent ! 308: * some other thread from trying to bring in ! 309: * the same page. ! 310: * ! 311: * Note that we cannot hold any locks during the ! 312: * pager access or when waiting for memory, so ! 313: * we use a busy page then. ! 314: * ! 315: * Note also that we aren't as concerned about more than ! 316: * one thread attempting to memory_object_data_unlock ! 317: * the same page at once, so we don't hold the page ! 318: * as busy then, but do record the highest unlock ! 319: * value so far. [Unlock requests may also be delivered ! 320: * out of order.] ! 321: * ! 322: * 2) To prevent another thread from racing us down the ! 323: * shadow chain and entering a new page in the top ! 324: * object before we do, we must keep a busy page in ! 325: * the top object while following the shadow chain. ! 326: * ! 327: * 3) We must increment paging_in_progress on any object ! 328: * for which we have a busy page, to prevent ! 329: * vm_object_collapse from removing the busy page ! 330: * without our noticing. ! 331: * ! 332: * 4) We leave busy pages on the pageout queues. ! 333: * If the pageout daemon comes across a busy page, ! 334: * it will remove the page from the pageout queues. ! 335: */ ! 336: ! 337: /* ! 338: * Search for the page at object/offset. ! 339: */ ! 340: ! 341: object = first_object; ! 342: offset = first_offset; ! 343: first_m = VM_PAGE_NULL; ! 344: access_required = fault_type; ! 345: ! 346: /* ! 347: * See whether this page is resident ! 348: */ ! 349: ! 350: while (TRUE) { ! 351: m = vm_page_lookup(object, offset); ! 352: if (m != VM_PAGE_NULL) { ! 353: /* ! 354: * If the page is being brought in, ! 355: * wait for it and then retry. ! 356: * ! 357: * A possible optimization: if the page ! 358: * is known to be resident, we can ignore ! 359: * pages that are absent (regardless of ! 360: * whether they're busy). ! 361: */ ! 362: ! 363: if (m->busy) { ! 364: kern_return_t wait_result; ! 365: ! 366: PAGE_ASSERT_WAIT(m, interruptible); ! 367: vm_object_unlock(object); ! 368: #ifdef CONTINUATIONS ! 369: if (continuation != (void (*)()) 0) { ! 370: register vm_fault_state_t *state = ! 371: (vm_fault_state_t *) current_thread()->ith_other; ! 372: ! 373: /* ! 374: * Save variables in case ! 375: * thread_block discards ! 376: * our kernel stack. ! 377: */ ! 378: ! 379: state->vmfp_backoff = FALSE; ! 380: state->vmfp_object = object; ! 381: state->vmfp_offset = offset; ! 382: state->vmfp_first_m = first_m; ! 383: state->vmfp_access = ! 384: access_required; ! 385: state->vmf_prot = *protection; ! 386: ! 387: counter(c_vm_fault_page_block_busy_user++); ! 388: thread_block(continuation); ! 389: } else ! 390: #endif /* CONTINUATIONS */ ! 391: { ! 392: counter(c_vm_fault_page_block_busy_kernel++); ! 393: thread_block((void (*)()) 0); ! 394: } ! 395: after_thread_block: ! 396: wait_result = current_thread()->wait_result; ! 397: vm_object_lock(object); ! 398: if (wait_result != THREAD_AWAKENED) { ! 399: vm_fault_cleanup(object, first_m); ! 400: if (wait_result == THREAD_RESTART) ! 401: return(VM_FAULT_RETRY); ! 402: else ! 403: return(VM_FAULT_INTERRUPTED); ! 404: } ! 405: continue; ! 406: } ! 407: ! 408: /* ! 409: * If the page is in error, give up now. ! 410: */ ! 411: ! 412: if (m->error) { ! 413: VM_PAGE_FREE(m); ! 414: vm_fault_cleanup(object, first_m); ! 415: return(VM_FAULT_MEMORY_ERROR); ! 416: } ! 417: ! 418: /* ! 419: * If the page isn't busy, but is absent, ! 420: * then it was deemed "unavailable". ! 421: */ ! 422: ! 423: if (m->absent) { ! 424: /* ! 425: * Remove the non-existent page (unless it's ! 426: * in the top object) and move on down to the ! 427: * next object (if there is one). ! 428: */ ! 429: ! 430: offset += object->shadow_offset; ! 431: access_required = VM_PROT_READ; ! 432: next_object = object->shadow; ! 433: if (next_object == VM_OBJECT_NULL) { ! 434: vm_page_t real_m; ! 435: ! 436: assert(!must_be_resident); ! 437: ! 438: /* ! 439: * Absent page at bottom of shadow ! 440: * chain; zero fill the page we left ! 441: * busy in the first object, and flush ! 442: * the absent page. But first we ! 443: * need to allocate a real page. ! 444: */ ! 445: ! 446: real_m = vm_page_grab(!object->internal); ! 447: if (real_m == VM_PAGE_NULL) { ! 448: vm_fault_cleanup(object, first_m); ! 449: return(VM_FAULT_MEMORY_SHORTAGE); ! 450: } ! 451: ! 452: if (object != first_object) { ! 453: VM_PAGE_FREE(m); ! 454: vm_object_paging_end(object); ! 455: vm_object_unlock(object); ! 456: object = first_object; ! 457: offset = first_offset; ! 458: m = first_m; ! 459: first_m = VM_PAGE_NULL; ! 460: vm_object_lock(object); ! 461: } ! 462: ! 463: VM_PAGE_FREE(m); ! 464: assert(real_m->busy); ! 465: vm_page_lock_queues(); ! 466: vm_page_insert(real_m, object, offset); ! 467: vm_page_unlock_queues(); ! 468: m = real_m; ! 469: ! 470: /* ! 471: * Drop the lock while zero filling ! 472: * page. Then break because this ! 473: * is the page we wanted. Checking ! 474: * the page lock is a waste of time; ! 475: * this page was either absent or ! 476: * newly allocated -- in both cases ! 477: * it can't be page locked by a pager. ! 478: */ ! 479: vm_object_unlock(object); ! 480: ! 481: vm_page_zero_fill(m); ! 482: ! 483: vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS); ! 484: ! 485: vm_stat.zero_fill_count++; ! 486: vm_object_lock(object); ! 487: pmap_clear_modify(m->phys_addr); ! 488: break; ! 489: } else { ! 490: if (must_be_resident) { ! 491: vm_object_paging_end(object); ! 492: } else if (object != first_object) { ! 493: vm_object_paging_end(object); ! 494: VM_PAGE_FREE(m); ! 495: } else { ! 496: first_m = m; ! 497: m->absent = FALSE; ! 498: vm_object_absent_release(object); ! 499: m->busy = TRUE; ! 500: ! 501: vm_page_lock_queues(); ! 502: VM_PAGE_QUEUES_REMOVE(m); ! 503: vm_page_unlock_queues(); ! 504: } ! 505: vm_object_lock(next_object); ! 506: vm_object_unlock(object); ! 507: object = next_object; ! 508: vm_object_paging_begin(object); ! 509: continue; ! 510: } ! 511: } ! 512: ! 513: /* ! 514: * If the desired access to this page has ! 515: * been locked out, request that it be unlocked. ! 516: */ ! 517: ! 518: if (access_required & m->page_lock) { ! 519: if ((access_required & m->unlock_request) != access_required) { ! 520: vm_prot_t new_unlock_request; ! 521: kern_return_t rc; ! 522: ! 523: if (!object->pager_ready) { ! 524: vm_object_assert_wait(object, ! 525: VM_OBJECT_EVENT_PAGER_READY, ! 526: interruptible); ! 527: goto block_and_backoff; ! 528: } ! 529: ! 530: new_unlock_request = m->unlock_request = ! 531: (access_required | m->unlock_request); ! 532: vm_object_unlock(object); ! 533: if ((rc = memory_object_data_unlock( ! 534: object->pager, ! 535: object->pager_request, ! 536: offset + object->paging_offset, ! 537: PAGE_SIZE, ! 538: new_unlock_request)) ! 539: != KERN_SUCCESS) { ! 540: printf("vm_fault: memory_object_data_unlock failed\n"); ! 541: vm_object_lock(object); ! 542: vm_fault_cleanup(object, first_m); ! 543: return((rc == MACH_SEND_INTERRUPTED) ? ! 544: VM_FAULT_INTERRUPTED : ! 545: VM_FAULT_MEMORY_ERROR); ! 546: } ! 547: vm_object_lock(object); ! 548: continue; ! 549: } ! 550: ! 551: PAGE_ASSERT_WAIT(m, interruptible); ! 552: goto block_and_backoff; ! 553: } ! 554: ! 555: /* ! 556: * We mark the page busy and leave it on ! 557: * the pageout queues. If the pageout ! 558: * deamon comes across it, then it will ! 559: * remove the page. ! 560: */ ! 561: ! 562: if (!software_reference_bits) { ! 563: vm_page_lock_queues(); ! 564: if (m->inactive) { ! 565: vm_stat_sample(SAMPLED_PC_VM_REACTIVATION_FAULTS); ! 566: vm_stat.reactivations++; ! 567: } ! 568: ! 569: VM_PAGE_QUEUES_REMOVE(m); ! 570: vm_page_unlock_queues(); ! 571: } ! 572: ! 573: assert(!m->busy); ! 574: m->busy = TRUE; ! 575: assert(!m->absent); ! 576: break; ! 577: } ! 578: ! 579: look_for_page = ! 580: (object->pager_created) ! 581: #if MACH_PAGEMAP ! 582: && (vm_external_state_get(object->existence_info, offset + object->paging_offset) != ! 583: VM_EXTERNAL_STATE_ABSENT) ! 584: #endif /* MACH_PAGEMAP */ ! 585: ; ! 586: ! 587: if ((look_for_page || (object == first_object)) ! 588: && !must_be_resident) { ! 589: /* ! 590: * Allocate a new page for this object/offset ! 591: * pair. ! 592: */ ! 593: ! 594: m = vm_page_grab_fictitious(); ! 595: if (m == VM_PAGE_NULL) { ! 596: vm_fault_cleanup(object, first_m); ! 597: return(VM_FAULT_FICTITIOUS_SHORTAGE); ! 598: } ! 599: ! 600: vm_page_lock_queues(); ! 601: vm_page_insert(m, object, offset); ! 602: vm_page_unlock_queues(); ! 603: } ! 604: ! 605: if (look_for_page && !must_be_resident) { ! 606: kern_return_t rc; ! 607: ! 608: /* ! 609: * If the memory manager is not ready, we ! 610: * cannot make requests. ! 611: */ ! 612: if (!object->pager_ready) { ! 613: vm_object_assert_wait(object, ! 614: VM_OBJECT_EVENT_PAGER_READY, ! 615: interruptible); ! 616: VM_PAGE_FREE(m); ! 617: goto block_and_backoff; ! 618: } ! 619: ! 620: if (object->internal) { ! 621: /* ! 622: * Requests to the default pager ! 623: * must reserve a real page in advance, ! 624: * because the pager's data-provided ! 625: * won't block for pages. ! 626: */ ! 627: ! 628: if (m->fictitious && !vm_page_convert(m, FALSE)) { ! 629: VM_PAGE_FREE(m); ! 630: vm_fault_cleanup(object, first_m); ! 631: return(VM_FAULT_MEMORY_SHORTAGE); ! 632: } ! 633: } else if (object->absent_count > ! 634: vm_object_absent_max) { ! 635: /* ! 636: * If there are too many outstanding page ! 637: * requests pending on this object, we ! 638: * wait for them to be resolved now. ! 639: */ ! 640: ! 641: vm_object_absent_assert_wait(object, interruptible); ! 642: VM_PAGE_FREE(m); ! 643: goto block_and_backoff; ! 644: } ! 645: ! 646: /* ! 647: * Indicate that the page is waiting for data ! 648: * from the memory manager. ! 649: */ ! 650: ! 651: m->absent = TRUE; ! 652: object->absent_count++; ! 653: ! 654: /* ! 655: * We have a busy page, so we can ! 656: * release the object lock. ! 657: */ ! 658: vm_object_unlock(object); ! 659: ! 660: /* ! 661: * Call the memory manager to retrieve the data. ! 662: */ ! 663: ! 664: vm_stat.pageins++; ! 665: vm_stat_sample(SAMPLED_PC_VM_PAGEIN_FAULTS); ! 666: ! 667: if ((rc = memory_object_data_request(object->pager, ! 668: object->pager_request, ! 669: m->offset + object->paging_offset, ! 670: PAGE_SIZE, access_required)) != KERN_SUCCESS) { ! 671: if (rc != MACH_SEND_INTERRUPTED) ! 672: printf("%s(0x%x, 0x%x, 0x%x, 0x%x, 0x%x) failed, %d\n", ! 673: "memory_object_data_request", ! 674: object->pager, ! 675: object->pager_request, ! 676: m->offset + object->paging_offset, ! 677: PAGE_SIZE, access_required, rc); ! 678: /* ! 679: * Don't want to leave a busy page around, ! 680: * but the data request may have blocked, ! 681: * so check if it's still there and busy. ! 682: */ ! 683: vm_object_lock(object); ! 684: if (m == vm_page_lookup(object,offset) && ! 685: m->absent && m->busy) ! 686: VM_PAGE_FREE(m); ! 687: vm_fault_cleanup(object, first_m); ! 688: return((rc == MACH_SEND_INTERRUPTED) ? ! 689: VM_FAULT_INTERRUPTED : ! 690: VM_FAULT_MEMORY_ERROR); ! 691: } ! 692: ! 693: /* ! 694: * Retry with same object/offset, since new data may ! 695: * be in a different page (i.e., m is meaningless at ! 696: * this point). ! 697: */ ! 698: vm_object_lock(object); ! 699: continue; ! 700: } ! 701: ! 702: /* ! 703: * For the XP system, the only case in which we get here is if ! 704: * object has no pager (or unwiring). If the pager doesn't ! 705: * have the page this is handled in the m->absent case above ! 706: * (and if you change things here you should look above). ! 707: */ ! 708: if (object == first_object) ! 709: first_m = m; ! 710: else ! 711: { ! 712: assert(m == VM_PAGE_NULL); ! 713: } ! 714: ! 715: /* ! 716: * Move on to the next object. Lock the next ! 717: * object before unlocking the current one. ! 718: */ ! 719: access_required = VM_PROT_READ; ! 720: ! 721: offset += object->shadow_offset; ! 722: next_object = object->shadow; ! 723: if (next_object == VM_OBJECT_NULL) { ! 724: assert(!must_be_resident); ! 725: ! 726: /* ! 727: * If there's no object left, fill the page ! 728: * in the top object with zeros. But first we ! 729: * need to allocate a real page. ! 730: */ ! 731: ! 732: if (object != first_object) { ! 733: vm_object_paging_end(object); ! 734: vm_object_unlock(object); ! 735: ! 736: object = first_object; ! 737: offset = first_offset; ! 738: vm_object_lock(object); ! 739: } ! 740: ! 741: m = first_m; ! 742: assert(m->object == object); ! 743: first_m = VM_PAGE_NULL; ! 744: ! 745: if (m->fictitious && !vm_page_convert(m, !object->internal)) { ! 746: VM_PAGE_FREE(m); ! 747: vm_fault_cleanup(object, VM_PAGE_NULL); ! 748: return(VM_FAULT_MEMORY_SHORTAGE); ! 749: } ! 750: ! 751: vm_object_unlock(object); ! 752: vm_page_zero_fill(m); ! 753: vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS); ! 754: vm_stat.zero_fill_count++; ! 755: vm_object_lock(object); ! 756: pmap_clear_modify(m->phys_addr); ! 757: break; ! 758: } ! 759: else { ! 760: vm_object_lock(next_object); ! 761: if ((object != first_object) || must_be_resident) ! 762: vm_object_paging_end(object); ! 763: vm_object_unlock(object); ! 764: object = next_object; ! 765: vm_object_paging_begin(object); ! 766: } ! 767: } ! 768: ! 769: /* ! 770: * PAGE HAS BEEN FOUND. ! 771: * ! 772: * This page (m) is: ! 773: * busy, so that we can play with it; ! 774: * not absent, so that nobody else will fill it; ! 775: * possibly eligible for pageout; ! 776: * ! 777: * The top-level page (first_m) is: ! 778: * VM_PAGE_NULL if the page was found in the ! 779: * top-level object; ! 780: * busy, not absent, and ineligible for pageout. ! 781: * ! 782: * The current object (object) is locked. A paging ! 783: * reference is held for the current and top-level ! 784: * objects. ! 785: */ ! 786: ! 787: #if EXTRA_ASSERTIONS ! 788: assert(m->busy && !m->absent); ! 789: assert((first_m == VM_PAGE_NULL) || ! 790: (first_m->busy && !first_m->absent && ! 791: !first_m->active && !first_m->inactive)); ! 792: #endif /* EXTRA_ASSERTIONS */ ! 793: ! 794: /* ! 795: * If the page is being written, but isn't ! 796: * already owned by the top-level object, ! 797: * we have to copy it into a new page owned ! 798: * by the top-level object. ! 799: */ ! 800: ! 801: if (object != first_object) { ! 802: /* ! 803: * We only really need to copy if we ! 804: * want to write it. ! 805: */ ! 806: ! 807: if (fault_type & VM_PROT_WRITE) { ! 808: vm_page_t copy_m; ! 809: ! 810: assert(!must_be_resident); ! 811: ! 812: /* ! 813: * If we try to collapse first_object at this ! 814: * point, we may deadlock when we try to get ! 815: * the lock on an intermediate object (since we ! 816: * have the bottom object locked). We can't ! 817: * unlock the bottom object, because the page ! 818: * we found may move (by collapse) if we do. ! 819: * ! 820: * Instead, we first copy the page. Then, when ! 821: * we have no more use for the bottom object, ! 822: * we unlock it and try to collapse. ! 823: * ! 824: * Note that we copy the page even if we didn't ! 825: * need to... that's the breaks. ! 826: */ ! 827: ! 828: /* ! 829: * Allocate a page for the copy ! 830: */ ! 831: copy_m = vm_page_grab(!first_object->internal); ! 832: if (copy_m == VM_PAGE_NULL) { ! 833: RELEASE_PAGE(m); ! 834: vm_fault_cleanup(object, first_m); ! 835: return(VM_FAULT_MEMORY_SHORTAGE); ! 836: } ! 837: ! 838: vm_object_unlock(object); ! 839: vm_page_copy(m, copy_m); ! 840: vm_object_lock(object); ! 841: ! 842: /* ! 843: * If another map is truly sharing this ! 844: * page with us, we have to flush all ! 845: * uses of the original page, since we ! 846: * can't distinguish those which want the ! 847: * original from those which need the ! 848: * new copy. ! 849: * ! 850: * XXXO If we know that only one map has ! 851: * access to this page, then we could ! 852: * avoid the pmap_page_protect() call. ! 853: */ ! 854: ! 855: vm_page_lock_queues(); ! 856: vm_page_deactivate(m); ! 857: pmap_page_protect(m->phys_addr, VM_PROT_NONE); ! 858: vm_page_unlock_queues(); ! 859: ! 860: /* ! 861: * We no longer need the old page or object. ! 862: */ ! 863: ! 864: PAGE_WAKEUP_DONE(m); ! 865: vm_object_paging_end(object); ! 866: vm_object_unlock(object); ! 867: ! 868: vm_stat.cow_faults++; ! 869: vm_stat_sample(SAMPLED_PC_VM_COW_FAULTS); ! 870: object = first_object; ! 871: offset = first_offset; ! 872: ! 873: vm_object_lock(object); ! 874: VM_PAGE_FREE(first_m); ! 875: first_m = VM_PAGE_NULL; ! 876: assert(copy_m->busy); ! 877: vm_page_lock_queues(); ! 878: vm_page_insert(copy_m, object, offset); ! 879: vm_page_unlock_queues(); ! 880: m = copy_m; ! 881: ! 882: /* ! 883: * Now that we've gotten the copy out of the ! 884: * way, let's try to collapse the top object. ! 885: * But we have to play ugly games with ! 886: * paging_in_progress to do that... ! 887: */ ! 888: ! 889: vm_object_paging_end(object); ! 890: vm_object_collapse(object); ! 891: vm_object_paging_begin(object); ! 892: } ! 893: else { ! 894: *protection &= (~VM_PROT_WRITE); ! 895: } ! 896: } ! 897: ! 898: /* ! 899: * Now check whether the page needs to be pushed into the ! 900: * copy object. The use of asymmetric copy on write for ! 901: * shared temporary objects means that we may do two copies to ! 902: * satisfy the fault; one above to get the page from a ! 903: * shadowed object, and one here to push it into the copy. ! 904: */ ! 905: ! 906: while ((copy_object = first_object->copy) != VM_OBJECT_NULL) { ! 907: vm_offset_t copy_offset; ! 908: vm_page_t copy_m; ! 909: ! 910: /* ! 911: * If the page is being written, but hasn't been ! 912: * copied to the copy-object, we have to copy it there. ! 913: */ ! 914: ! 915: if ((fault_type & VM_PROT_WRITE) == 0) { ! 916: *protection &= ~VM_PROT_WRITE; ! 917: break; ! 918: } ! 919: ! 920: /* ! 921: * If the page was guaranteed to be resident, ! 922: * we must have already performed the copy. ! 923: */ ! 924: ! 925: if (must_be_resident) ! 926: break; ! 927: ! 928: /* ! 929: * Try to get the lock on the copy_object. ! 930: */ ! 931: if (!vm_object_lock_try(copy_object)) { ! 932: vm_object_unlock(object); ! 933: ! 934: simple_lock_pause(); /* wait a bit */ ! 935: ! 936: vm_object_lock(object); ! 937: continue; ! 938: } ! 939: ! 940: /* ! 941: * Make another reference to the copy-object, ! 942: * to keep it from disappearing during the ! 943: * copy. ! 944: */ ! 945: assert(copy_object->ref_count > 0); ! 946: copy_object->ref_count++; ! 947: ! 948: /* ! 949: * Does the page exist in the copy? ! 950: */ ! 951: copy_offset = first_offset - copy_object->shadow_offset; ! 952: copy_m = vm_page_lookup(copy_object, copy_offset); ! 953: if (copy_m != VM_PAGE_NULL) { ! 954: if (copy_m->busy) { ! 955: /* ! 956: * If the page is being brought ! 957: * in, wait for it and then retry. ! 958: */ ! 959: PAGE_ASSERT_WAIT(copy_m, interruptible); ! 960: RELEASE_PAGE(m); ! 961: copy_object->ref_count--; ! 962: assert(copy_object->ref_count > 0); ! 963: vm_object_unlock(copy_object); ! 964: goto block_and_backoff; ! 965: } ! 966: } ! 967: else { ! 968: /* ! 969: * Allocate a page for the copy ! 970: */ ! 971: copy_m = vm_page_alloc(copy_object, copy_offset); ! 972: if (copy_m == VM_PAGE_NULL) { ! 973: RELEASE_PAGE(m); ! 974: copy_object->ref_count--; ! 975: assert(copy_object->ref_count > 0); ! 976: vm_object_unlock(copy_object); ! 977: vm_fault_cleanup(object, first_m); ! 978: return(VM_FAULT_MEMORY_SHORTAGE); ! 979: } ! 980: ! 981: /* ! 982: * Must copy page into copy-object. ! 983: */ ! 984: ! 985: vm_page_copy(m, copy_m); ! 986: ! 987: /* ! 988: * If the old page was in use by any users ! 989: * of the copy-object, it must be removed ! 990: * from all pmaps. (We can't know which ! 991: * pmaps use it.) ! 992: */ ! 993: ! 994: vm_page_lock_queues(); ! 995: pmap_page_protect(m->phys_addr, VM_PROT_NONE); ! 996: copy_m->dirty = TRUE; ! 997: vm_page_unlock_queues(); ! 998: ! 999: /* ! 1000: * If there's a pager, then immediately ! 1001: * page out this page, using the "initialize" ! 1002: * option. Else, we use the copy. ! 1003: */ ! 1004: ! 1005: if (!copy_object->pager_created) { ! 1006: vm_page_lock_queues(); ! 1007: vm_page_activate(copy_m); ! 1008: vm_page_unlock_queues(); ! 1009: PAGE_WAKEUP_DONE(copy_m); ! 1010: } else { ! 1011: /* ! 1012: * The page is already ready for pageout: ! 1013: * not on pageout queues and busy. ! 1014: * Unlock everything except the ! 1015: * copy_object itself. ! 1016: */ ! 1017: ! 1018: vm_object_unlock(object); ! 1019: ! 1020: /* ! 1021: * Write the page to the copy-object, ! 1022: * flushing it from the kernel. ! 1023: */ ! 1024: ! 1025: vm_pageout_page(copy_m, TRUE, TRUE); ! 1026: ! 1027: /* ! 1028: * Since the pageout may have ! 1029: * temporarily dropped the ! 1030: * copy_object's lock, we ! 1031: * check whether we'll have ! 1032: * to deallocate the hard way. ! 1033: */ ! 1034: ! 1035: if ((copy_object->shadow != object) || ! 1036: (copy_object->ref_count == 1)) { ! 1037: vm_object_unlock(copy_object); ! 1038: vm_object_deallocate(copy_object); ! 1039: vm_object_lock(object); ! 1040: continue; ! 1041: } ! 1042: ! 1043: /* ! 1044: * Pick back up the old object's ! 1045: * lock. [It is safe to do so, ! 1046: * since it must be deeper in the ! 1047: * object tree.] ! 1048: */ ! 1049: ! 1050: vm_object_lock(object); ! 1051: } ! 1052: ! 1053: /* ! 1054: * Because we're pushing a page upward ! 1055: * in the object tree, we must restart ! 1056: * any faults that are waiting here. ! 1057: * [Note that this is an expansion of ! 1058: * PAGE_WAKEUP that uses the THREAD_RESTART ! 1059: * wait result]. Can't turn off the page's ! 1060: * busy bit because we're not done with it. ! 1061: */ ! 1062: ! 1063: if (m->wanted) { ! 1064: m->wanted = FALSE; ! 1065: thread_wakeup_with_result((event_t) m, ! 1066: THREAD_RESTART); ! 1067: } ! 1068: } ! 1069: ! 1070: /* ! 1071: * The reference count on copy_object must be ! 1072: * at least 2: one for our extra reference, ! 1073: * and at least one from the outside world ! 1074: * (we checked that when we last locked ! 1075: * copy_object). ! 1076: */ ! 1077: copy_object->ref_count--; ! 1078: assert(copy_object->ref_count > 0); ! 1079: vm_object_unlock(copy_object); ! 1080: ! 1081: break; ! 1082: } ! 1083: ! 1084: *result_page = m; ! 1085: *top_page = first_m; ! 1086: ! 1087: /* ! 1088: * If the page can be written, assume that it will be. ! 1089: * [Earlier, we restrict the permission to allow write ! 1090: * access only if the fault so required, so we don't ! 1091: * mark read-only data as dirty.] ! 1092: */ ! 1093: ! 1094: if (vm_fault_dirty_handling && (*protection & VM_PROT_WRITE)) ! 1095: m->dirty = TRUE; ! 1096: ! 1097: return(VM_FAULT_SUCCESS); ! 1098: ! 1099: block_and_backoff: ! 1100: vm_fault_cleanup(object, first_m); ! 1101: ! 1102: #ifdef CONTINUATIONS ! 1103: if (continuation != (void (*)()) 0) { ! 1104: register vm_fault_state_t *state = ! 1105: (vm_fault_state_t *) current_thread()->ith_other; ! 1106: ! 1107: /* ! 1108: * Save variables in case we must restart. ! 1109: */ ! 1110: ! 1111: state->vmfp_backoff = TRUE; ! 1112: state->vmf_prot = *protection; ! 1113: ! 1114: counter(c_vm_fault_page_block_backoff_user++); ! 1115: thread_block(continuation); ! 1116: } else ! 1117: #endif /* CONTINUATIONS */ ! 1118: { ! 1119: counter(c_vm_fault_page_block_backoff_kernel++); ! 1120: thread_block((void (*)()) 0); ! 1121: } ! 1122: after_block_and_backoff: ! 1123: if (current_thread()->wait_result == THREAD_AWAKENED) ! 1124: return VM_FAULT_RETRY; ! 1125: else ! 1126: return VM_FAULT_INTERRUPTED; ! 1127: ! 1128: #undef RELEASE_PAGE ! 1129: } ! 1130: ! 1131: /* ! 1132: * Routine: vm_fault ! 1133: * Purpose: ! 1134: * Handle page faults, including pseudo-faults ! 1135: * used to change the wiring status of pages. ! 1136: * Returns: ! 1137: * If an explicit (expression) continuation is supplied, ! 1138: * then we call the continuation instead of returning. ! 1139: * Implementation: ! 1140: * Explicit continuations make this a little icky, ! 1141: * because it hasn't been rewritten to embrace CPS. ! 1142: * Instead, we have resume arguments for vm_fault and ! 1143: * vm_fault_page, to let continue the fault computation. ! 1144: * ! 1145: * vm_fault and vm_fault_page save mucho state ! 1146: * in the moral equivalent of a closure. The state ! 1147: * structure is allocated when first entering vm_fault ! 1148: * and deallocated when leaving vm_fault. ! 1149: */ ! 1150: ! 1151: #ifdef CONTINUATIONS ! 1152: void ! 1153: vm_fault_continue() ! 1154: { ! 1155: register vm_fault_state_t *state = ! 1156: (vm_fault_state_t *) current_thread()->ith_other; ! 1157: ! 1158: (void) vm_fault(state->vmf_map, ! 1159: state->vmf_vaddr, ! 1160: state->vmf_fault_type, ! 1161: state->vmf_change_wiring, ! 1162: TRUE, state->vmf_continuation); ! 1163: /*NOTREACHED*/ ! 1164: } ! 1165: #endif /* CONTINUATIONS */ ! 1166: ! 1167: kern_return_t vm_fault(map, vaddr, fault_type, change_wiring, ! 1168: resume, continuation) ! 1169: vm_map_t map; ! 1170: vm_offset_t vaddr; ! 1171: vm_prot_t fault_type; ! 1172: boolean_t change_wiring; ! 1173: boolean_t resume; ! 1174: void (*continuation)(); ! 1175: { ! 1176: vm_map_version_t version; /* Map version for verificiation */ ! 1177: boolean_t wired; /* Should mapping be wired down? */ ! 1178: vm_object_t object; /* Top-level object */ ! 1179: vm_offset_t offset; /* Top-level offset */ ! 1180: vm_prot_t prot; /* Protection for mapping */ ! 1181: vm_object_t old_copy_object; /* Saved copy object */ ! 1182: vm_page_t result_page; /* Result of vm_fault_page */ ! 1183: vm_page_t top_page; /* Placeholder page */ ! 1184: kern_return_t kr; ! 1185: ! 1186: register ! 1187: vm_page_t m; /* Fast access to result_page */ ! 1188: ! 1189: #ifdef CONTINUATIONS ! 1190: if (resume) { ! 1191: register vm_fault_state_t *state = ! 1192: (vm_fault_state_t *) current_thread()->ith_other; ! 1193: ! 1194: /* ! 1195: * Retrieve cached variables and ! 1196: * continue vm_fault_page. ! 1197: */ ! 1198: ! 1199: object = state->vmf_object; ! 1200: if (object == VM_OBJECT_NULL) ! 1201: goto RetryFault; ! 1202: version = state->vmf_version; ! 1203: wired = state->vmf_wired; ! 1204: offset = state->vmf_offset; ! 1205: prot = state->vmf_prot; ! 1206: ! 1207: kr = vm_fault_page(object, offset, fault_type, ! 1208: (change_wiring && !wired), !change_wiring, ! 1209: &prot, &result_page, &top_page, ! 1210: TRUE, vm_fault_continue); ! 1211: goto after_vm_fault_page; ! 1212: } ! 1213: ! 1214: if (continuation != (void (*)()) 0) { ! 1215: /* ! 1216: * We will probably need to save state. ! 1217: */ ! 1218: ! 1219: char * state; ! 1220: ! 1221: /* ! 1222: * if this assignment stmt is written as ! 1223: * 'active_threads[cpu_number()] = zalloc()', ! 1224: * cpu_number may be evaluated before zalloc; ! 1225: * if zalloc blocks, cpu_number will be wrong ! 1226: */ ! 1227: ! 1228: state = (char *) zalloc(vm_fault_state_zone); ! 1229: current_thread()->ith_other = state; ! 1230: ! 1231: } ! 1232: #else /* not CONTINUATIONS */ ! 1233: assert(continuation == 0); ! 1234: assert(!resume); ! 1235: #endif /* not CONTINUATIONS */ ! 1236: ! 1237: RetryFault: ; ! 1238: ! 1239: /* ! 1240: * Find the backing store object and offset into ! 1241: * it to begin the search. ! 1242: */ ! 1243: ! 1244: if ((kr = vm_map_lookup(&map, vaddr, fault_type, &version, ! 1245: &object, &offset, ! 1246: &prot, &wired)) != KERN_SUCCESS) { ! 1247: goto done; ! 1248: } ! 1249: ! 1250: /* ! 1251: * If the page is wired, we must fault for the current protection ! 1252: * value, to avoid further faults. ! 1253: */ ! 1254: ! 1255: if (wired) ! 1256: fault_type = prot; ! 1257: ! 1258: /* ! 1259: * Make a reference to this object to ! 1260: * prevent its disposal while we are messing with ! 1261: * it. Once we have the reference, the map is free ! 1262: * to be diddled. Since objects reference their ! 1263: * shadows (and copies), they will stay around as well. ! 1264: */ ! 1265: ! 1266: assert(object->ref_count > 0); ! 1267: object->ref_count++; ! 1268: vm_object_paging_begin(object); ! 1269: ! 1270: #ifdef CONTINUATIONS ! 1271: if (continuation != (void (*)()) 0) { ! 1272: register vm_fault_state_t *state = ! 1273: (vm_fault_state_t *) current_thread()->ith_other; ! 1274: ! 1275: /* ! 1276: * Save variables, in case vm_fault_page discards ! 1277: * our kernel stack and we have to restart. ! 1278: */ ! 1279: ! 1280: state->vmf_map = map; ! 1281: state->vmf_vaddr = vaddr; ! 1282: state->vmf_fault_type = fault_type; ! 1283: state->vmf_change_wiring = change_wiring; ! 1284: state->vmf_continuation = continuation; ! 1285: ! 1286: state->vmf_version = version; ! 1287: state->vmf_wired = wired; ! 1288: state->vmf_object = object; ! 1289: state->vmf_offset = offset; ! 1290: state->vmf_prot = prot; ! 1291: ! 1292: kr = vm_fault_page(object, offset, fault_type, ! 1293: (change_wiring && !wired), !change_wiring, ! 1294: &prot, &result_page, &top_page, ! 1295: FALSE, vm_fault_continue); ! 1296: } else ! 1297: #endif /* CONTINUATIONS */ ! 1298: { ! 1299: kr = vm_fault_page(object, offset, fault_type, ! 1300: (change_wiring && !wired), !change_wiring, ! 1301: &prot, &result_page, &top_page, ! 1302: FALSE, (void (*)()) 0); ! 1303: } ! 1304: after_vm_fault_page: ! 1305: ! 1306: /* ! 1307: * If we didn't succeed, lose the object reference immediately. ! 1308: */ ! 1309: ! 1310: if (kr != VM_FAULT_SUCCESS) ! 1311: vm_object_deallocate(object); ! 1312: ! 1313: /* ! 1314: * See why we failed, and take corrective action. ! 1315: */ ! 1316: ! 1317: switch (kr) { ! 1318: case VM_FAULT_SUCCESS: ! 1319: break; ! 1320: case VM_FAULT_RETRY: ! 1321: goto RetryFault; ! 1322: case VM_FAULT_INTERRUPTED: ! 1323: kr = KERN_SUCCESS; ! 1324: goto done; ! 1325: case VM_FAULT_MEMORY_SHORTAGE: ! 1326: #ifdef CONTINUATIONS ! 1327: if (continuation != (void (*)()) 0) { ! 1328: register vm_fault_state_t *state = ! 1329: (vm_fault_state_t *) current_thread()->ith_other; ! 1330: ! 1331: /* ! 1332: * Save variables in case VM_PAGE_WAIT ! 1333: * discards our kernel stack. ! 1334: */ ! 1335: ! 1336: state->vmf_map = map; ! 1337: state->vmf_vaddr = vaddr; ! 1338: state->vmf_fault_type = fault_type; ! 1339: state->vmf_change_wiring = change_wiring; ! 1340: state->vmf_continuation = continuation; ! 1341: state->vmf_object = VM_OBJECT_NULL; ! 1342: ! 1343: VM_PAGE_WAIT(vm_fault_continue); ! 1344: } else ! 1345: #endif /* CONTINUATIONS */ ! 1346: VM_PAGE_WAIT((void (*)()) 0); ! 1347: goto RetryFault; ! 1348: case VM_FAULT_FICTITIOUS_SHORTAGE: ! 1349: vm_page_more_fictitious(); ! 1350: goto RetryFault; ! 1351: case VM_FAULT_MEMORY_ERROR: ! 1352: kr = KERN_MEMORY_ERROR; ! 1353: goto done; ! 1354: } ! 1355: ! 1356: m = result_page; ! 1357: ! 1358: assert((change_wiring && !wired) ? ! 1359: (top_page == VM_PAGE_NULL) : ! 1360: ((top_page == VM_PAGE_NULL) == (m->object == object))); ! 1361: ! 1362: /* ! 1363: * How to clean up the result of vm_fault_page. This ! 1364: * happens whether the mapping is entered or not. ! 1365: */ ! 1366: ! 1367: #define UNLOCK_AND_DEALLOCATE \ ! 1368: MACRO_BEGIN \ ! 1369: vm_fault_cleanup(m->object, top_page); \ ! 1370: vm_object_deallocate(object); \ ! 1371: MACRO_END ! 1372: ! 1373: /* ! 1374: * What to do with the resulting page from vm_fault_page ! 1375: * if it doesn't get entered into the physical map: ! 1376: */ ! 1377: ! 1378: #define RELEASE_PAGE(m) \ ! 1379: MACRO_BEGIN \ ! 1380: PAGE_WAKEUP_DONE(m); \ ! 1381: vm_page_lock_queues(); \ ! 1382: if (!m->active && !m->inactive) \ ! 1383: vm_page_activate(m); \ ! 1384: vm_page_unlock_queues(); \ ! 1385: MACRO_END ! 1386: ! 1387: /* ! 1388: * We must verify that the maps have not changed ! 1389: * since our last lookup. ! 1390: */ ! 1391: ! 1392: old_copy_object = m->object->copy; ! 1393: ! 1394: vm_object_unlock(m->object); ! 1395: while (!vm_map_verify(map, &version)) { ! 1396: vm_object_t retry_object; ! 1397: vm_offset_t retry_offset; ! 1398: vm_prot_t retry_prot; ! 1399: ! 1400: /* ! 1401: * To avoid trying to write_lock the map while another ! 1402: * thread has it read_locked (in vm_map_pageable), we ! 1403: * do not try for write permission. If the page is ! 1404: * still writable, we will get write permission. If it ! 1405: * is not, or has been marked needs_copy, we enter the ! 1406: * mapping without write permission, and will merely ! 1407: * take another fault. ! 1408: */ ! 1409: kr = vm_map_lookup(&map, vaddr, ! 1410: fault_type & ~VM_PROT_WRITE, &version, ! 1411: &retry_object, &retry_offset, &retry_prot, ! 1412: &wired); ! 1413: ! 1414: if (kr != KERN_SUCCESS) { ! 1415: vm_object_lock(m->object); ! 1416: RELEASE_PAGE(m); ! 1417: UNLOCK_AND_DEALLOCATE; ! 1418: goto done; ! 1419: } ! 1420: ! 1421: vm_object_unlock(retry_object); ! 1422: vm_object_lock(m->object); ! 1423: ! 1424: if ((retry_object != object) || ! 1425: (retry_offset != offset)) { ! 1426: RELEASE_PAGE(m); ! 1427: UNLOCK_AND_DEALLOCATE; ! 1428: goto RetryFault; ! 1429: } ! 1430: ! 1431: /* ! 1432: * Check whether the protection has changed or the object ! 1433: * has been copied while we left the map unlocked. ! 1434: */ ! 1435: prot &= retry_prot; ! 1436: vm_object_unlock(m->object); ! 1437: } ! 1438: vm_object_lock(m->object); ! 1439: ! 1440: /* ! 1441: * If the copy object changed while the top-level object ! 1442: * was unlocked, then we must take away write permission. ! 1443: */ ! 1444: ! 1445: if (m->object->copy != old_copy_object) ! 1446: prot &= ~VM_PROT_WRITE; ! 1447: ! 1448: /* ! 1449: * If we want to wire down this page, but no longer have ! 1450: * adequate permissions, we must start all over. ! 1451: */ ! 1452: ! 1453: if (wired && (prot != fault_type)) { ! 1454: vm_map_verify_done(map, &version); ! 1455: RELEASE_PAGE(m); ! 1456: UNLOCK_AND_DEALLOCATE; ! 1457: goto RetryFault; ! 1458: } ! 1459: ! 1460: /* ! 1461: * It's critically important that a wired-down page be faulted ! 1462: * only once in each map for which it is wired. ! 1463: */ ! 1464: ! 1465: vm_object_unlock(m->object); ! 1466: ! 1467: /* ! 1468: * Put this page into the physical map. ! 1469: * We had to do the unlock above because pmap_enter ! 1470: * may cause other faults. The page may be on ! 1471: * the pageout queues. If the pageout daemon comes ! 1472: * across the page, it will remove it from the queues. ! 1473: */ ! 1474: ! 1475: PMAP_ENTER(map->pmap, vaddr, m, prot, wired); ! 1476: ! 1477: /* ! 1478: * If the page is not wired down and isn't already ! 1479: * on a pageout queue, then put it where the ! 1480: * pageout daemon can find it. ! 1481: */ ! 1482: vm_object_lock(m->object); ! 1483: vm_page_lock_queues(); ! 1484: if (change_wiring) { ! 1485: if (wired) ! 1486: vm_page_wire(m); ! 1487: else ! 1488: vm_page_unwire(m); ! 1489: } else if (software_reference_bits) { ! 1490: if (!m->active && !m->inactive) ! 1491: vm_page_activate(m); ! 1492: m->reference = TRUE; ! 1493: } else { ! 1494: vm_page_activate(m); ! 1495: } ! 1496: vm_page_unlock_queues(); ! 1497: ! 1498: /* ! 1499: * Unlock everything, and return ! 1500: */ ! 1501: ! 1502: vm_map_verify_done(map, &version); ! 1503: PAGE_WAKEUP_DONE(m); ! 1504: kr = KERN_SUCCESS; ! 1505: UNLOCK_AND_DEALLOCATE; ! 1506: ! 1507: #undef UNLOCK_AND_DEALLOCATE ! 1508: #undef RELEASE_PAGE ! 1509: ! 1510: done: ! 1511: #ifdef CONTINUATIONS ! 1512: if (continuation != (void (*)()) 0) { ! 1513: register vm_fault_state_t *state = ! 1514: (vm_fault_state_t *) current_thread()->ith_other; ! 1515: ! 1516: zfree(vm_fault_state_zone, (vm_offset_t) state); ! 1517: (*continuation)(kr); ! 1518: /*NOTREACHED*/ ! 1519: } ! 1520: #endif /* CONTINUATIONS */ ! 1521: ! 1522: return(kr); ! 1523: } ! 1524: ! 1525: kern_return_t vm_fault_wire_fast(); ! 1526: ! 1527: /* ! 1528: * vm_fault_wire: ! 1529: * ! 1530: * Wire down a range of virtual addresses in a map. ! 1531: */ ! 1532: void vm_fault_wire(map, entry) ! 1533: vm_map_t map; ! 1534: vm_map_entry_t entry; ! 1535: { ! 1536: ! 1537: register vm_offset_t va; ! 1538: register pmap_t pmap; ! 1539: register vm_offset_t end_addr = entry->vme_end; ! 1540: ! 1541: pmap = vm_map_pmap(map); ! 1542: ! 1543: /* ! 1544: * Inform the physical mapping system that the ! 1545: * range of addresses may not fault, so that ! 1546: * page tables and such can be locked down as well. ! 1547: */ ! 1548: ! 1549: pmap_pageable(pmap, entry->vme_start, end_addr, FALSE); ! 1550: ! 1551: /* ! 1552: * We simulate a fault to get the page and enter it ! 1553: * in the physical map. ! 1554: */ ! 1555: ! 1556: for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) { ! 1557: if (vm_fault_wire_fast(map, va, entry) != KERN_SUCCESS) ! 1558: (void) vm_fault(map, va, VM_PROT_NONE, TRUE, ! 1559: FALSE, (void (*)()) 0); ! 1560: } ! 1561: } ! 1562: ! 1563: /* ! 1564: * vm_fault_unwire: ! 1565: * ! 1566: * Unwire a range of virtual addresses in a map. ! 1567: */ ! 1568: void vm_fault_unwire(map, entry) ! 1569: vm_map_t map; ! 1570: vm_map_entry_t entry; ! 1571: { ! 1572: register vm_offset_t va; ! 1573: register pmap_t pmap; ! 1574: register vm_offset_t end_addr = entry->vme_end; ! 1575: vm_object_t object; ! 1576: ! 1577: pmap = vm_map_pmap(map); ! 1578: ! 1579: object = (entry->is_sub_map) ! 1580: ? VM_OBJECT_NULL : entry->object.vm_object; ! 1581: ! 1582: /* ! 1583: * Since the pages are wired down, we must be able to ! 1584: * get their mappings from the physical map system. ! 1585: */ ! 1586: ! 1587: for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) { ! 1588: pmap_change_wiring(pmap, va, FALSE); ! 1589: ! 1590: if (object == VM_OBJECT_NULL) { ! 1591: vm_map_lock_set_recursive(map); ! 1592: (void) vm_fault(map, va, VM_PROT_NONE, TRUE, ! 1593: FALSE, (void (*)()) 0); ! 1594: vm_map_lock_clear_recursive(map); ! 1595: } else { ! 1596: vm_prot_t prot; ! 1597: vm_page_t result_page; ! 1598: vm_page_t top_page; ! 1599: vm_fault_return_t result; ! 1600: ! 1601: do { ! 1602: prot = VM_PROT_NONE; ! 1603: ! 1604: vm_object_lock(object); ! 1605: vm_object_paging_begin(object); ! 1606: result = vm_fault_page(object, ! 1607: entry->offset + ! 1608: (va - entry->vme_start), ! 1609: VM_PROT_NONE, TRUE, ! 1610: FALSE, &prot, ! 1611: &result_page, ! 1612: &top_page, ! 1613: FALSE, (void (*)()) 0); ! 1614: } while (result == VM_FAULT_RETRY); ! 1615: ! 1616: if (result != VM_FAULT_SUCCESS) ! 1617: panic("vm_fault_unwire: failure"); ! 1618: ! 1619: vm_page_lock_queues(); ! 1620: vm_page_unwire(result_page); ! 1621: vm_page_unlock_queues(); ! 1622: PAGE_WAKEUP_DONE(result_page); ! 1623: ! 1624: vm_fault_cleanup(result_page->object, top_page); ! 1625: } ! 1626: } ! 1627: ! 1628: /* ! 1629: * Inform the physical mapping system that the range ! 1630: * of addresses may fault, so that page tables and ! 1631: * such may be unwired themselves. ! 1632: */ ! 1633: ! 1634: pmap_pageable(pmap, entry->vme_start, end_addr, TRUE); ! 1635: } ! 1636: ! 1637: /* ! 1638: * vm_fault_wire_fast: ! 1639: * ! 1640: * Handle common case of a wire down page fault at the given address. ! 1641: * If successful, the page is inserted into the associated physical map. ! 1642: * The map entry is passed in to avoid the overhead of a map lookup. ! 1643: * ! 1644: * NOTE: the given address should be truncated to the ! 1645: * proper page address. ! 1646: * ! 1647: * KERN_SUCCESS is returned if the page fault is handled; otherwise, ! 1648: * a standard error specifying why the fault is fatal is returned. ! 1649: * ! 1650: * The map in question must be referenced, and remains so. ! 1651: * Caller has a read lock on the map. ! 1652: * ! 1653: * This is a stripped version of vm_fault() for wiring pages. Anything ! 1654: * other than the common case will return KERN_FAILURE, and the caller ! 1655: * is expected to call vm_fault(). ! 1656: */ ! 1657: kern_return_t vm_fault_wire_fast(map, va, entry) ! 1658: vm_map_t map; ! 1659: vm_offset_t va; ! 1660: vm_map_entry_t entry; ! 1661: { ! 1662: vm_object_t object; ! 1663: vm_offset_t offset; ! 1664: register vm_page_t m; ! 1665: vm_prot_t prot; ! 1666: ! 1667: vm_stat.faults++; /* needs lock XXX */ ! 1668: /* ! 1669: * Recovery actions ! 1670: */ ! 1671: ! 1672: #undef RELEASE_PAGE ! 1673: #define RELEASE_PAGE(m) { \ ! 1674: PAGE_WAKEUP_DONE(m); \ ! 1675: vm_page_lock_queues(); \ ! 1676: vm_page_unwire(m); \ ! 1677: vm_page_unlock_queues(); \ ! 1678: } ! 1679: ! 1680: ! 1681: #undef UNLOCK_THINGS ! 1682: #define UNLOCK_THINGS { \ ! 1683: object->paging_in_progress--; \ ! 1684: vm_object_unlock(object); \ ! 1685: } ! 1686: ! 1687: #undef UNLOCK_AND_DEALLOCATE ! 1688: #define UNLOCK_AND_DEALLOCATE { \ ! 1689: UNLOCK_THINGS; \ ! 1690: vm_object_deallocate(object); \ ! 1691: } ! 1692: /* ! 1693: * Give up and have caller do things the hard way. ! 1694: */ ! 1695: ! 1696: #define GIVE_UP { \ ! 1697: UNLOCK_AND_DEALLOCATE; \ ! 1698: return(KERN_FAILURE); \ ! 1699: } ! 1700: ! 1701: ! 1702: /* ! 1703: * If this entry is not directly to a vm_object, bail out. ! 1704: */ ! 1705: if (entry->is_sub_map) ! 1706: return(KERN_FAILURE); ! 1707: ! 1708: /* ! 1709: * Find the backing store object and offset into it. ! 1710: */ ! 1711: ! 1712: object = entry->object.vm_object; ! 1713: offset = (va - entry->vme_start) + entry->offset; ! 1714: prot = entry->protection; ! 1715: ! 1716: /* ! 1717: * Make a reference to this object to prevent its ! 1718: * disposal while we are messing with it. ! 1719: */ ! 1720: ! 1721: vm_object_lock(object); ! 1722: assert(object->ref_count > 0); ! 1723: object->ref_count++; ! 1724: object->paging_in_progress++; ! 1725: ! 1726: /* ! 1727: * INVARIANTS (through entire routine): ! 1728: * ! 1729: * 1) At all times, we must either have the object ! 1730: * lock or a busy page in some object to prevent ! 1731: * some other thread from trying to bring in ! 1732: * the same page. ! 1733: * ! 1734: * 2) Once we have a busy page, we must remove it from ! 1735: * the pageout queues, so that the pageout daemon ! 1736: * will not grab it away. ! 1737: * ! 1738: */ ! 1739: ! 1740: /* ! 1741: * Look for page in top-level object. If it's not there or ! 1742: * there's something going on, give up. ! 1743: */ ! 1744: m = vm_page_lookup(object, offset); ! 1745: if ((m == VM_PAGE_NULL) || (m->error) || ! 1746: (m->busy) || (m->absent) || (prot & m->page_lock)) { ! 1747: GIVE_UP; ! 1748: } ! 1749: ! 1750: /* ! 1751: * Wire the page down now. All bail outs beyond this ! 1752: * point must unwire the page. ! 1753: */ ! 1754: ! 1755: vm_page_lock_queues(); ! 1756: vm_page_wire(m); ! 1757: vm_page_unlock_queues(); ! 1758: ! 1759: /* ! 1760: * Mark page busy for other threads. ! 1761: */ ! 1762: assert(!m->busy); ! 1763: m->busy = TRUE; ! 1764: assert(!m->absent); ! 1765: ! 1766: /* ! 1767: * Give up if the page is being written and there's a copy object ! 1768: */ ! 1769: if ((object->copy != VM_OBJECT_NULL) && (prot & VM_PROT_WRITE)) { ! 1770: RELEASE_PAGE(m); ! 1771: GIVE_UP; ! 1772: } ! 1773: ! 1774: /* ! 1775: * Put this page into the physical map. ! 1776: * We have to unlock the object because pmap_enter ! 1777: * may cause other faults. ! 1778: */ ! 1779: vm_object_unlock(object); ! 1780: ! 1781: PMAP_ENTER(map->pmap, va, m, prot, TRUE); ! 1782: ! 1783: /* ! 1784: * Must relock object so that paging_in_progress can be cleared. ! 1785: */ ! 1786: vm_object_lock(object); ! 1787: ! 1788: /* ! 1789: * Unlock everything, and return ! 1790: */ ! 1791: ! 1792: PAGE_WAKEUP_DONE(m); ! 1793: UNLOCK_AND_DEALLOCATE; ! 1794: ! 1795: return(KERN_SUCCESS); ! 1796: ! 1797: } ! 1798: ! 1799: /* ! 1800: * Routine: vm_fault_copy_cleanup ! 1801: * Purpose: ! 1802: * Release a page used by vm_fault_copy. ! 1803: */ ! 1804: ! 1805: void vm_fault_copy_cleanup(page, top_page) ! 1806: vm_page_t page; ! 1807: vm_page_t top_page; ! 1808: { ! 1809: vm_object_t object = page->object; ! 1810: ! 1811: vm_object_lock(object); ! 1812: PAGE_WAKEUP_DONE(page); ! 1813: vm_page_lock_queues(); ! 1814: if (!page->active && !page->inactive) ! 1815: vm_page_activate(page); ! 1816: vm_page_unlock_queues(); ! 1817: vm_fault_cleanup(object, top_page); ! 1818: } ! 1819: ! 1820: /* ! 1821: * Routine: vm_fault_copy ! 1822: * ! 1823: * Purpose: ! 1824: * Copy pages from one virtual memory object to another -- ! 1825: * neither the source nor destination pages need be resident. ! 1826: * ! 1827: * Before actually copying a page, the version associated with ! 1828: * the destination address map wil be verified. ! 1829: * ! 1830: * In/out conditions: ! 1831: * The caller must hold a reference, but not a lock, to ! 1832: * each of the source and destination objects and to the ! 1833: * destination map. ! 1834: * ! 1835: * Results: ! 1836: * Returns KERN_SUCCESS if no errors were encountered in ! 1837: * reading or writing the data. Returns KERN_INTERRUPTED if ! 1838: * the operation was interrupted (only possible if the ! 1839: * "interruptible" argument is asserted). Other return values ! 1840: * indicate a permanent error in copying the data. ! 1841: * ! 1842: * The actual amount of data copied will be returned in the ! 1843: * "copy_size" argument. In the event that the destination map ! 1844: * verification failed, this amount may be less than the amount ! 1845: * requested. ! 1846: */ ! 1847: kern_return_t vm_fault_copy( ! 1848: src_object, ! 1849: src_offset, ! 1850: src_size, ! 1851: dst_object, ! 1852: dst_offset, ! 1853: dst_map, ! 1854: dst_version, ! 1855: interruptible ! 1856: ) ! 1857: vm_object_t src_object; ! 1858: vm_offset_t src_offset; ! 1859: vm_size_t *src_size; /* INOUT */ ! 1860: vm_object_t dst_object; ! 1861: vm_offset_t dst_offset; ! 1862: vm_map_t dst_map; ! 1863: vm_map_version_t *dst_version; ! 1864: boolean_t interruptible; ! 1865: { ! 1866: vm_page_t result_page; ! 1867: vm_prot_t prot; ! 1868: ! 1869: vm_page_t src_page; ! 1870: vm_page_t src_top_page; ! 1871: ! 1872: vm_page_t dst_page; ! 1873: vm_page_t dst_top_page; ! 1874: ! 1875: vm_size_t amount_done; ! 1876: vm_object_t old_copy_object; ! 1877: ! 1878: #define RETURN(x) \ ! 1879: MACRO_BEGIN \ ! 1880: *src_size = amount_done; \ ! 1881: MACRO_RETURN(x); \ ! 1882: MACRO_END ! 1883: ! 1884: amount_done = 0; ! 1885: do { /* while (amount_done != *src_size) */ ! 1886: ! 1887: RetrySourceFault: ; ! 1888: ! 1889: if (src_object == VM_OBJECT_NULL) { ! 1890: /* ! 1891: * No source object. We will just ! 1892: * zero-fill the page in dst_object. ! 1893: */ ! 1894: ! 1895: src_page = VM_PAGE_NULL; ! 1896: } else { ! 1897: prot = VM_PROT_READ; ! 1898: ! 1899: vm_object_lock(src_object); ! 1900: vm_object_paging_begin(src_object); ! 1901: ! 1902: switch (vm_fault_page(src_object, src_offset, ! 1903: VM_PROT_READ, FALSE, interruptible, ! 1904: &prot, &result_page, &src_top_page, ! 1905: FALSE, (void (*)()) 0)) { ! 1906: ! 1907: case VM_FAULT_SUCCESS: ! 1908: break; ! 1909: case VM_FAULT_RETRY: ! 1910: goto RetrySourceFault; ! 1911: case VM_FAULT_INTERRUPTED: ! 1912: RETURN(MACH_SEND_INTERRUPTED); ! 1913: case VM_FAULT_MEMORY_SHORTAGE: ! 1914: VM_PAGE_WAIT((void (*)()) 0); ! 1915: goto RetrySourceFault; ! 1916: case VM_FAULT_FICTITIOUS_SHORTAGE: ! 1917: vm_page_more_fictitious(); ! 1918: goto RetrySourceFault; ! 1919: case VM_FAULT_MEMORY_ERROR: ! 1920: return(KERN_MEMORY_ERROR); ! 1921: } ! 1922: ! 1923: src_page = result_page; ! 1924: ! 1925: assert((src_top_page == VM_PAGE_NULL) == ! 1926: (src_page->object == src_object)); ! 1927: ! 1928: assert ((prot & VM_PROT_READ) != VM_PROT_NONE); ! 1929: ! 1930: vm_object_unlock(src_page->object); ! 1931: } ! 1932: ! 1933: RetryDestinationFault: ; ! 1934: ! 1935: prot = VM_PROT_WRITE; ! 1936: ! 1937: vm_object_lock(dst_object); ! 1938: vm_object_paging_begin(dst_object); ! 1939: ! 1940: switch (vm_fault_page(dst_object, dst_offset, VM_PROT_WRITE, ! 1941: FALSE, FALSE /* interruptible */, ! 1942: &prot, &result_page, &dst_top_page, ! 1943: FALSE, (void (*)()) 0)) { ! 1944: ! 1945: case VM_FAULT_SUCCESS: ! 1946: break; ! 1947: case VM_FAULT_RETRY: ! 1948: goto RetryDestinationFault; ! 1949: case VM_FAULT_INTERRUPTED: ! 1950: if (src_page != VM_PAGE_NULL) ! 1951: vm_fault_copy_cleanup(src_page, ! 1952: src_top_page); ! 1953: RETURN(MACH_SEND_INTERRUPTED); ! 1954: case VM_FAULT_MEMORY_SHORTAGE: ! 1955: VM_PAGE_WAIT((void (*)()) 0); ! 1956: goto RetryDestinationFault; ! 1957: case VM_FAULT_FICTITIOUS_SHORTAGE: ! 1958: vm_page_more_fictitious(); ! 1959: goto RetryDestinationFault; ! 1960: case VM_FAULT_MEMORY_ERROR: ! 1961: if (src_page != VM_PAGE_NULL) ! 1962: vm_fault_copy_cleanup(src_page, ! 1963: src_top_page); ! 1964: return(KERN_MEMORY_ERROR); ! 1965: } ! 1966: assert ((prot & VM_PROT_WRITE) != VM_PROT_NONE); ! 1967: ! 1968: dst_page = result_page; ! 1969: ! 1970: old_copy_object = dst_page->object->copy; ! 1971: ! 1972: vm_object_unlock(dst_page->object); ! 1973: ! 1974: if (!vm_map_verify(dst_map, dst_version)) { ! 1975: ! 1976: BailOut: ; ! 1977: ! 1978: if (src_page != VM_PAGE_NULL) ! 1979: vm_fault_copy_cleanup(src_page, src_top_page); ! 1980: vm_fault_copy_cleanup(dst_page, dst_top_page); ! 1981: break; ! 1982: } ! 1983: ! 1984: ! 1985: vm_object_lock(dst_page->object); ! 1986: if (dst_page->object->copy != old_copy_object) { ! 1987: vm_object_unlock(dst_page->object); ! 1988: vm_map_verify_done(dst_map, dst_version); ! 1989: goto BailOut; ! 1990: } ! 1991: vm_object_unlock(dst_page->object); ! 1992: ! 1993: /* ! 1994: * Copy the page, and note that it is dirty ! 1995: * immediately. ! 1996: */ ! 1997: ! 1998: if (src_page == VM_PAGE_NULL) ! 1999: vm_page_zero_fill(dst_page); ! 2000: else ! 2001: vm_page_copy(src_page, dst_page); ! 2002: dst_page->dirty = TRUE; ! 2003: ! 2004: /* ! 2005: * Unlock everything, and return ! 2006: */ ! 2007: ! 2008: vm_map_verify_done(dst_map, dst_version); ! 2009: ! 2010: if (src_page != VM_PAGE_NULL) ! 2011: vm_fault_copy_cleanup(src_page, src_top_page); ! 2012: vm_fault_copy_cleanup(dst_page, dst_top_page); ! 2013: ! 2014: amount_done += PAGE_SIZE; ! 2015: src_offset += PAGE_SIZE; ! 2016: dst_offset += PAGE_SIZE; ! 2017: ! 2018: } while (amount_done != *src_size); ! 2019: ! 2020: RETURN(KERN_SUCCESS); ! 2021: #undef RETURN ! 2022: ! 2023: /*NOTREACHED*/ ! 2024: } ! 2025: ! 2026: ! 2027: ! 2028: ! 2029: ! 2030: #ifdef notdef ! 2031: ! 2032: /* ! 2033: * Routine: vm_fault_page_overwrite ! 2034: * ! 2035: * Description: ! 2036: * A form of vm_fault_page that assumes that the ! 2037: * resulting page will be overwritten in its entirety, ! 2038: * making it unnecessary to obtain the correct *contents* ! 2039: * of the page. ! 2040: * ! 2041: * Implementation: ! 2042: * XXX Untested. Also unused. Eventually, this technology ! 2043: * could be used in vm_fault_copy() to advantage. ! 2044: */ ! 2045: vm_fault_return_t vm_fault_page_overwrite(dst_object, dst_offset, result_page) ! 2046: register ! 2047: vm_object_t dst_object; ! 2048: vm_offset_t dst_offset; ! 2049: vm_page_t *result_page; /* OUT */ ! 2050: { ! 2051: register ! 2052: vm_page_t dst_page; ! 2053: ! 2054: #define interruptible FALSE /* XXX */ ! 2055: ! 2056: while (TRUE) { ! 2057: /* ! 2058: * Look for a page at this offset ! 2059: */ ! 2060: ! 2061: while ((dst_page = vm_page_lookup(dst_object, dst_offset)) ! 2062: == VM_PAGE_NULL) { ! 2063: /* ! 2064: * No page, no problem... just allocate one. ! 2065: */ ! 2066: ! 2067: dst_page = vm_page_alloc(dst_object, dst_offset); ! 2068: if (dst_page == VM_PAGE_NULL) { ! 2069: vm_object_unlock(dst_object); ! 2070: VM_PAGE_WAIT((void (*)()) 0); ! 2071: vm_object_lock(dst_object); ! 2072: continue; ! 2073: } ! 2074: ! 2075: /* ! 2076: * Pretend that the memory manager ! 2077: * write-protected the page. ! 2078: * ! 2079: * Note that we will be asking for write ! 2080: * permission without asking for the data ! 2081: * first. ! 2082: */ ! 2083: ! 2084: dst_page->overwriting = TRUE; ! 2085: dst_page->page_lock = VM_PROT_WRITE; ! 2086: dst_page->absent = TRUE; ! 2087: dst_object->absent_count++; ! 2088: ! 2089: break; ! 2090: ! 2091: /* ! 2092: * When we bail out, we might have to throw ! 2093: * away the page created here. ! 2094: */ ! 2095: ! 2096: #define DISCARD_PAGE \ ! 2097: MACRO_BEGIN \ ! 2098: vm_object_lock(dst_object); \ ! 2099: dst_page = vm_page_lookup(dst_object, dst_offset); \ ! 2100: if ((dst_page != VM_PAGE_NULL) && dst_page->overwriting) \ ! 2101: VM_PAGE_FREE(dst_page); \ ! 2102: vm_object_unlock(dst_object); \ ! 2103: MACRO_END ! 2104: } ! 2105: ! 2106: /* ! 2107: * If the page is write-protected... ! 2108: */ ! 2109: ! 2110: if (dst_page->page_lock & VM_PROT_WRITE) { ! 2111: /* ! 2112: * ... and an unlock request hasn't been sent ! 2113: */ ! 2114: ! 2115: if ( ! (dst_page->unlock_request & VM_PROT_WRITE)) { ! 2116: vm_prot_t u; ! 2117: kern_return_t rc; ! 2118: ! 2119: /* ! 2120: * ... then send one now. ! 2121: */ ! 2122: ! 2123: if (!dst_object->pager_ready) { ! 2124: vm_object_assert_wait(dst_object, ! 2125: VM_OBJECT_EVENT_PAGER_READY, ! 2126: interruptible); ! 2127: vm_object_unlock(dst_object); ! 2128: thread_block((void (*)()) 0); ! 2129: if (current_thread()->wait_result != ! 2130: THREAD_AWAKENED) { ! 2131: DISCARD_PAGE; ! 2132: return(VM_FAULT_INTERRUPTED); ! 2133: } ! 2134: continue; ! 2135: } ! 2136: ! 2137: u = dst_page->unlock_request |= VM_PROT_WRITE; ! 2138: vm_object_unlock(dst_object); ! 2139: ! 2140: if ((rc = memory_object_data_unlock( ! 2141: dst_object->pager, ! 2142: dst_object->pager_request, ! 2143: dst_offset + dst_object->paging_offset, ! 2144: PAGE_SIZE, ! 2145: u)) != KERN_SUCCESS) { ! 2146: printf("vm_object_overwrite: memory_object_data_unlock failed\n"); ! 2147: DISCARD_PAGE; ! 2148: return((rc == MACH_SEND_INTERRUPTED) ? ! 2149: VM_FAULT_INTERRUPTED : ! 2150: VM_FAULT_MEMORY_ERROR); ! 2151: } ! 2152: vm_object_lock(dst_object); ! 2153: continue; ! 2154: } ! 2155: ! 2156: /* ... fall through to wait below */ ! 2157: } else { ! 2158: /* ! 2159: * If the page isn't being used for other ! 2160: * purposes, then we're done. ! 2161: */ ! 2162: if ( ! (dst_page->busy || dst_page->absent || dst_page->error) ) ! 2163: break; ! 2164: } ! 2165: ! 2166: PAGE_ASSERT_WAIT(dst_page, interruptible); ! 2167: vm_object_unlock(dst_object); ! 2168: thread_block((void (*)()) 0); ! 2169: if (current_thread()->wait_result != THREAD_AWAKENED) { ! 2170: DISCARD_PAGE; ! 2171: return(VM_FAULT_INTERRUPTED); ! 2172: } ! 2173: } ! 2174: ! 2175: *result_page = dst_page; ! 2176: return(VM_FAULT_SUCCESS); ! 2177: ! 2178: #undef interruptible ! 2179: #undef DISCARD_PAGE ! 2180: } ! 2181: ! 2182: #endif /* notdef */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.