![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_fault.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple XNU] / XNU / osfmk / vm|
Return to vm_fault.c CVS log | Up to [Apple XNU] / XNU / osfmk / vm |
1.1 ! root 1: /* ! 2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. ! 3: * ! 4: * @APPLE_LICENSE_HEADER_START@ ! 5: * ! 6: * The contents of this file constitute Original Code as defined in and ! 7: * are subject to the Apple Public Source License Version 1.1 (the ! 8: * "License"). You may not use this file except in compliance with the ! 9: * License. Please obtain a copy of the License at ! 10: * http://www.apple.com/publicsource and read it before using this file. ! 11: * ! 12: * This Original Code and all software distributed under the License are ! 13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER ! 14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, ! 15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, ! 16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the ! 17: * License for the specific language governing rights and limitations ! 18: * under the License. ! 19: * ! 20: * @APPLE_LICENSE_HEADER_END@ ! 21: */ ! 22: /* ! 23: * @OSF_COPYRIGHT@ ! 24: */ ! 25: /* ! 26: * Mach Operating System ! 27: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University ! 28: * All Rights Reserved. ! 29: * ! 30: * Permission to use, copy, modify and distribute this software and its ! 31: * documentation is hereby granted, provided that both the copyright ! 32: * notice and this permission notice appear in all copies of the ! 33: * software, derivative works or modified versions, and any portions ! 34: * thereof, and that both notices appear in supporting documentation. ! 35: * ! 36: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" ! 37: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR ! 38: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. ! 39: * ! 40: * Carnegie Mellon requests users of this software to return to ! 41: * ! 42: * Software Distribution Coordinator or [email protected] ! 43: * School of Computer Science ! 44: * Carnegie Mellon University ! 45: * Pittsburgh PA 15213-3890 ! 46: * ! 47: * any improvements or extensions that they make and grant Carnegie Mellon ! 48: * the rights to redistribute these changes. ! 49: */ ! 50: /* ! 51: */ ! 52: /* ! 53: * File: vm_fault.c ! 54: * Author: Avadis Tevanian, Jr., Michael Wayne Young ! 55: * ! 56: * Page fault handling module. ! 57: */ ! 58: #ifdef MACH_BSD ! 59: /* remove after component interface available */ ! 60: extern int vnode_pager_workaround; ! 61: #endif ! 62: ! 63: #include <mach_cluster_stats.h> ! 64: #include <mach_pagemap.h> ! 65: #include <mach_kdb.h> ! 66: ! 67: #include <vm/vm_fault.h> ! 68: #include <mach/kern_return.h> ! 69: #include <mach/message.h> /* for error codes */ ! 70: #include <kern/host_statistics.h> ! 71: #include <kern/counters.h> ! 72: #include <kern/task.h> ! 73: #include <kern/thread.h> ! 74: #include <kern/sched_prim.h> ! 75: #include <kern/host.h> ! 76: #include <kern/xpr.h> ! 77: #include <vm/vm_map.h> ! 78: #include <vm/vm_object.h> ! 79: #include <vm/vm_page.h> ! 80: #include <vm/pmap.h> ! 81: #include <vm/vm_pageout.h> ! 82: #include <mach/vm_param.h> ! 83: #include <mach/vm_behavior.h> ! 84: #include <mach/memory_object.h> ! 85: /* For memory_object_data_{request,unlock} */ ! 86: #include <kern/mach_param.h> ! 87: #include <kern/macro_help.h> ! 88: #include <kern/zalloc.h> ! 89: #include <kern/misc_protos.h> ! 90: ! 91: #include <sys/kdebug.h> ! 92: ! 93: #define VM_FAULT_CLASSIFY 0 ! 94: #define VM_FAULT_STATIC_CONFIG 1 ! 95: ! 96: #define TRACEFAULTPAGE 0 /* (TEST/DEBUG) */ ! 97: ! 98: int vm_object_absent_max = 50; ! 99: ! 100: int vm_fault_debug = 0; ! 101: boolean_t vm_page_deactivate_behind = TRUE; ! 102: ! 103: vm_machine_attribute_val_t mv_cache_sync = MATTR_VAL_CACHE_SYNC; ! 104: ! 105: #if !VM_FAULT_STATIC_CONFIG ! 106: boolean_t vm_fault_dirty_handling = FALSE; ! 107: boolean_t vm_fault_interruptible = FALSE; ! 108: boolean_t software_reference_bits = TRUE; ! 109: #endif ! 110: ! 111: #if MACH_KDB ! 112: extern struct db_watchpoint *db_watchpoint_list; ! 113: #endif /* MACH_KDB */ ! 114: ! 115: /* Forward declarations of internal routines. */ ! 116: extern kern_return_t vm_fault_wire_fast( ! 117: vm_map_t map, ! 118: vm_offset_t va, ! 119: vm_map_entry_t entry); ! 120: ! 121: extern void vm_fault_continue(void); ! 122: ! 123: extern void vm_fault_copy_cleanup( ! 124: vm_page_t page, ! 125: vm_page_t top_page); ! 126: ! 127: extern void vm_fault_copy_dst_cleanup( ! 128: vm_page_t page); ! 129: ! 130: #if VM_FAULT_CLASSIFY ! 131: extern void vm_fault_classify(vm_object_t object, ! 132: vm_offset_t offset, ! 133: vm_prot_t fault_type); ! 134: ! 135: extern void vm_fault_classify_init(void); ! 136: #endif ! 137: ! 138: /* ! 139: * Routine: vm_fault_init ! 140: * Purpose: ! 141: * Initialize our private data structures. ! 142: */ ! 143: void ! 144: vm_fault_init(void) ! 145: { ! 146: } ! 147: ! 148: /* ! 149: * Routine: vm_fault_cleanup ! 150: * Purpose: ! 151: * Clean up the result of vm_fault_page. ! 152: * Results: ! 153: * The paging reference for "object" is released. ! 154: * "object" is unlocked. ! 155: * If "top_page" is not null, "top_page" is ! 156: * freed and the paging reference for the object ! 157: * containing it is released. ! 158: * ! 159: * In/out conditions: ! 160: * "object" must be locked. ! 161: */ ! 162: void ! 163: vm_fault_cleanup( ! 164: register vm_object_t object, ! 165: register vm_page_t top_page) ! 166: { ! 167: vm_object_paging_end(object); ! 168: vm_object_unlock(object); ! 169: ! 170: if (top_page != VM_PAGE_NULL) { ! 171: object = top_page->object; ! 172: vm_object_lock(object); ! 173: VM_PAGE_FREE(top_page); ! 174: vm_object_paging_end(object); ! 175: vm_object_unlock(object); ! 176: } ! 177: } ! 178: ! 179: #if MACH_CLUSTER_STATS ! 180: #define MAXCLUSTERPAGES 16 ! 181: struct { ! 182: unsigned long pages_in_cluster; ! 183: unsigned long pages_at_higher_offsets; ! 184: unsigned long pages_at_lower_offsets; ! 185: } cluster_stats_in[MAXCLUSTERPAGES]; ! 186: #define CLUSTER_STAT(clause) clause ! 187: #define CLUSTER_STAT_HIGHER(x) \ ! 188: ((cluster_stats_in[(x)].pages_at_higher_offsets)++) ! 189: #define CLUSTER_STAT_LOWER(x) \ ! 190: ((cluster_stats_in[(x)].pages_at_lower_offsets)++) ! 191: #define CLUSTER_STAT_CLUSTER(x) \ ! 192: ((cluster_stats_in[(x)].pages_in_cluster)++) ! 193: #else /* MACH_CLUSTER_STATS */ ! 194: #define CLUSTER_STAT(clause) ! 195: #endif /* MACH_CLUSTER_STATS */ ! 196: ! 197: /* XXX - temporary */ ! 198: boolean_t vm_allow_clustered_pagein = FALSE; ! 199: int vm_pagein_cluster_used = 0; ! 200: ! 201: /* ! 202: * Prepage default sizes given VM_BEHAVIOR_DEFAULT reference behavior ! 203: */ ! 204: int vm_default_ahead = 1; /* Number of pages to prepage ahead */ ! 205: int vm_default_behind = 0; /* Number of pages to prepage behind */ ! 206: ! 207: #define ALIGNED(x) (((x) & (PAGE_SIZE - 1)) == 0) ! 208: ! 209: /* ! 210: * Routine: vm_fault_page ! 211: * Purpose: ! 212: * Find the resident page for the virtual memory ! 213: * specified by the given virtual memory object ! 214: * and offset. ! 215: * Additional arguments: ! 216: * The required permissions for the page is given ! 217: * in "fault_type". Desired permissions are included ! 218: * in "protection". The minimum and maximum valid offsets ! 219: * within the object for the relevant map entry are ! 220: * passed in "lo_offset" and "hi_offset" respectively and ! 221: * the expected page reference pattern is passed in "behavior". ! 222: * These three parameters are used to determine pagein cluster ! 223: * limits. ! 224: * ! 225: * If the desired page is known to be resident (for ! 226: * example, because it was previously wired down), asserting ! 227: * the "unwiring" parameter will speed the search. ! 228: * ! 229: * If the operation can be interrupted (by thread_abort ! 230: * or thread_terminate), then the "interruptible" ! 231: * parameter should be asserted. ! 232: * ! 233: * Results: ! 234: * The page containing the proper data is returned ! 235: * in "result_page". ! 236: * ! 237: * In/out conditions: ! 238: * The source object must be locked and referenced, ! 239: * and must donate one paging reference. The reference ! 240: * is not affected. The paging reference and lock are ! 241: * consumed. ! 242: * ! 243: * If the call succeeds, the object in which "result_page" ! 244: * resides is left locked and holding a paging reference. ! 245: * If this is not the original object, a busy page in the ! 246: * original object is returned in "top_page", to prevent other ! 247: * callers from pursuing this same data, along with a paging ! 248: * reference for the original object. The "top_page" should ! 249: * be destroyed when this guarantee is no longer required. ! 250: * The "result_page" is also left busy. It is not removed ! 251: * from the pageout queues. ! 252: */ ! 253: ! 254: vm_fault_return_t ! 255: vm_fault_page( ! 256: /* Arguments: */ ! 257: vm_object_t first_object, /* Object to begin search */ ! 258: vm_offset_t first_offset, /* Offset into object */ ! 259: vm_prot_t fault_type, /* What access is requested */ ! 260: boolean_t must_be_resident,/* Must page be resident? */ ! 261: int interruptible, /* how may fault be interrupted? */ ! 262: vm_offset_t lo_offset, /* Map entry start */ ! 263: vm_offset_t hi_offset, /* Map entry end */ ! 264: vm_behavior_t behavior, /* Page reference behavior */ ! 265: /* Modifies in place: */ ! 266: vm_prot_t *protection, /* Protection for mapping */ ! 267: /* Returns: */ ! 268: vm_page_t *result_page, /* Page found, if successful */ ! 269: vm_page_t *top_page, /* Page in top object, if ! 270: * not result_page. */ ! 271: int *type_of_fault, /* if non-null, fill in with type of fault ! 272: * COW, zero-fill, etc... returned in trace point */ ! 273: /* More arguments: */ ! 274: kern_return_t *error_code, /* code if page is in error */ ! 275: boolean_t no_zero_fill, /* don't zero fill absent pages */ ! 276: boolean_t data_supply) /* treat as data_supply if ! 277: * it is a write fault and a full ! 278: * page is provided */ ! 279: { ! 280: register ! 281: vm_page_t m; ! 282: register ! 283: vm_object_t object; ! 284: register ! 285: vm_offset_t offset; ! 286: vm_page_t first_m; ! 287: vm_object_t next_object; ! 288: vm_object_t copy_object; ! 289: boolean_t look_for_page; ! 290: vm_prot_t access_required = fault_type; ! 291: vm_prot_t wants_copy_flag; ! 292: thread_t thread = current_thread(); ! 293: vm_size_t cluster_size, length; ! 294: vm_offset_t cluster_offset; ! 295: vm_offset_t cluster_start, cluster_end, paging_offset; ! 296: vm_offset_t align_offset; ! 297: CLUSTER_STAT(int pages_at_higher_offsets;) ! 298: CLUSTER_STAT(int pages_at_lower_offsets;) ! 299: #ifdef MACH_BSD ! 300: kern_return_t vnode_pager_data_request( ! 301: ipc_port_t, ipc_port_t, vm_offset_t, vm_size_t, vm_prot_t); ! 302: #endif ! 303: ! 304: #if MACH_PAGEMAP ! 305: /* ! 306: * MACH page map - an optional optimization where a bit map is maintained ! 307: * by the VM subsystem for internal objects to indicate which pages of ! 308: * the object currently reside on backing store. This existence map ! 309: * duplicates information maintained by the vnode pager. It is ! 310: * created at the time of the first pageout against the object, i.e. ! 311: * at the same time pager for the object is created. The optimization ! 312: * is designed to eliminate pager interaction overhead, if it is ! 313: * 'known' that the page does not exist on backing store. ! 314: * ! 315: * LOOK_FOR() evaluates to TRUE if the page specified by object/offset is ! 316: * either marked as paged out in the existence map for the object or no ! 317: * existence map exists for the object. LOOK_FOR() is one of the ! 318: * criteria in the decision to invoke the pager. It is also used as one ! 319: * of the criteria to terminate the scan for adjacent pages in a clustered ! 320: * pagein operation. Note that LOOK_FOR() always evaluates to TRUE for ! 321: * permanent objects. Note also that if the pager for an internal object ! 322: * has not been created, the pager is not invoked regardless of the value ! 323: * of LOOK_FOR() and that clustered pagein scans are only done on an object ! 324: * for which a pager has been created. ! 325: * ! 326: * PAGED_OUT() evaluates to TRUE if the page specified by the object/offset ! 327: * is marked as paged out in the existence map for the object. PAGED_OUT() ! 328: * PAGED_OUT() is used to determine if a page has already been pushed ! 329: * into a copy object in order to avoid a redundant page out operation. ! 330: */ ! 331: #define LOOK_FOR(o, f) (vm_external_state_get((o)->existence_map, (f)) \ ! 332: != VM_EXTERNAL_STATE_ABSENT) ! 333: #define PAGED_OUT(o, f) (vm_external_state_get((o)->existence_map, (f)) \ ! 334: == VM_EXTERNAL_STATE_EXISTS) ! 335: #else /* MACH_PAGEMAP */ ! 336: /* ! 337: * If the MACH page map optimization is not enabled, ! 338: * LOOK_FOR() always evaluates to TRUE. The pager will always be ! 339: * invoked to resolve missing pages in an object, assuming the pager ! 340: * has been created for the object. In a clustered page operation, the ! 341: * absence of a page on backing backing store cannot be used to terminate ! 342: * a scan for adjacent pages since that information is available only in ! 343: * the pager. Hence pages that may not be paged out are potentially ! 344: * included in a clustered request. The vnode pager is coded to deal ! 345: * with any combination of absent/present pages in a clustered ! 346: * pagein request. PAGED_OUT() always evaluates to FALSE, i.e. the pager ! 347: * will always be invoked to push a dirty page into a copy object assuming ! 348: * a pager has been created. If the page has already been pushed, the ! 349: * pager will ingore the new request. ! 350: */ ! 351: #define LOOK_FOR(o, f) TRUE ! 352: #define PAGED_OUT(o, f) FALSE ! 353: #endif /* MACH_PAGEMAP */ ! 354: ! 355: /* ! 356: * Recovery actions ! 357: */ ! 358: #define PREPARE_RELEASE_PAGE(m) \ ! 359: MACRO_BEGIN \ ! 360: vm_page_lock_queues(); \ ! 361: MACRO_END ! 362: ! 363: #define DO_RELEASE_PAGE(m) \ ! 364: MACRO_BEGIN \ ! 365: PAGE_WAKEUP_DONE(m); \ ! 366: if (!m->active && !m->inactive) \ ! 367: vm_page_activate(m); \ ! 368: vm_page_unlock_queues(); \ ! 369: MACRO_END ! 370: ! 371: #define RELEASE_PAGE(m) \ ! 372: MACRO_BEGIN \ ! 373: PREPARE_RELEASE_PAGE(m); \ ! 374: DO_RELEASE_PAGE(m); \ ! 375: MACRO_END ! 376: ! 377: #if TRACEFAULTPAGE ! 378: dbgTrace(0xBEEF0002, (unsigned int) first_object, (unsigned int) first_offset); /* (TEST/DEBUG) */ ! 379: #endif ! 380: ! 381: ! 382: ! 383: #if !VM_FAULT_STATIC_CONFIG ! 384: if (vm_fault_dirty_handling ! 385: #if MACH_KDB ! 386: /* ! 387: * If there are watchpoints set, then ! 388: * we don't want to give away write permission ! 389: * on a read fault. Make the task write fault, ! 390: * so that the watchpoint code notices the access. ! 391: */ ! 392: || db_watchpoint_list ! 393: #endif /* MACH_KDB */ ! 394: ) { ! 395: /* ! 396: * If we aren't asking for write permission, ! 397: * then don't give it away. We're using write ! 398: * faults to set the dirty bit. ! 399: */ ! 400: if (!(fault_type & VM_PROT_WRITE)) ! 401: *protection &= ~VM_PROT_WRITE; ! 402: } ! 403: ! 404: if (!vm_fault_interruptible) ! 405: interruptible = THREAD_UNINT; ! 406: #else /* STATIC_CONFIG */ ! 407: #if MACH_KDB ! 408: /* ! 409: * If there are watchpoints set, then ! 410: * we don't want to give away write permission ! 411: * on a read fault. Make the task write fault, ! 412: * so that the watchpoint code notices the access. ! 413: */ ! 414: if (db_watchpoint_list) { ! 415: /* ! 416: * If we aren't asking for write permission, ! 417: * then don't give it away. We're using write ! 418: * faults to set the dirty bit. ! 419: */ ! 420: if (!(fault_type & VM_PROT_WRITE)) ! 421: *protection &= ~VM_PROT_WRITE; ! 422: } ! 423: ! 424: interruptible = THREAD_UNINT; /* vm_fault_interruptible */ ! 425: #endif /* MACH_KDB */ ! 426: #endif /* STATIC_CONFIG */ ! 427: ! 428: /* ! 429: * INVARIANTS (through entire routine): ! 430: * ! 431: * 1) At all times, we must either have the object ! 432: * lock or a busy page in some object to prevent ! 433: * some other thread from trying to bring in ! 434: * the same page. ! 435: * ! 436: * Note that we cannot hold any locks during the ! 437: * pager access or when waiting for memory, so ! 438: * we use a busy page then. ! 439: * ! 440: * Note also that we aren't as concerned about more than ! 441: * one thread attempting to memory_object_data_unlock ! 442: * the same page at once, so we don't hold the page ! 443: * as busy then, but do record the highest unlock ! 444: * value so far. [Unlock requests may also be delivered ! 445: * out of order.] ! 446: * ! 447: * 2) To prevent another thread from racing us down the ! 448: * shadow chain and entering a new page in the top ! 449: * object before we do, we must keep a busy page in ! 450: * the top object while following the shadow chain. ! 451: * ! 452: * 3) We must increment paging_in_progress on any object ! 453: * for which we have a busy page, to prevent ! 454: * vm_object_collapse from removing the busy page ! 455: * without our noticing. ! 456: * ! 457: * 4) We leave busy pages on the pageout queues. ! 458: * If the pageout daemon comes across a busy page, ! 459: * it will remove the page from the pageout queues. ! 460: */ ! 461: ! 462: /* ! 463: * Search for the page at object/offset. ! 464: */ ! 465: ! 466: object = first_object; ! 467: offset = first_offset; ! 468: first_m = VM_PAGE_NULL; ! 469: access_required = fault_type; ! 470: ! 471: XPR(XPR_VM_FAULT, ! 472: "vm_f_page: obj 0x%X, offset 0x%X, type %d, prot %d\n", ! 473: (integer_t)object, offset, fault_type, *protection, 0); ! 474: ! 475: /* ! 476: * See whether this page is resident ! 477: */ ! 478: ! 479: while (TRUE) { ! 480: #if TRACEFAULTPAGE ! 481: dbgTrace(0xBEEF0003, (unsigned int) 0, (unsigned int) 0); /* (TEST/DEBUG) */ ! 482: #endif ! 483: if (!object->alive) { ! 484: vm_fault_cleanup(object, first_m); ! 485: return(VM_FAULT_MEMORY_ERROR); ! 486: } ! 487: m = vm_page_lookup(object, offset); ! 488: #if TRACEFAULTPAGE ! 489: dbgTrace(0xBEEF0004, (unsigned int) m, (unsigned int) object); /* (TEST/DEBUG) */ ! 490: #endif ! 491: if (m != VM_PAGE_NULL) { ! 492: /* ! 493: * If the page was pre-paged as part of a ! 494: * cluster, record the fact. ! 495: */ ! 496: if (m->clustered) { ! 497: vm_pagein_cluster_used++; ! 498: m->clustered = FALSE; ! 499: } ! 500: ! 501: /* ! 502: * If the page is being brought in, ! 503: * wait for it and then retry. ! 504: * ! 505: * A possible optimization: if the page ! 506: * is known to be resident, we can ignore ! 507: * pages that are absent (regardless of ! 508: * whether they're busy). ! 509: */ ! 510: ! 511: if (m->busy) { ! 512: kern_return_t wait_result; ! 513: ! 514: #if TRACEFAULTPAGE ! 515: dbgTrace(0xBEEF0005, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */ ! 516: #endif ! 517: PAGE_ASSERT_WAIT(m, interruptible); ! 518: vm_object_unlock(object); ! 519: XPR(XPR_VM_FAULT, ! 520: "vm_f_page: block busy obj 0x%X, offset 0x%X, page 0x%X\n", ! 521: (integer_t)object, offset, ! 522: (integer_t)m, 0, 0); ! 523: counter(c_vm_fault_page_block_busy_kernel++); ! 524: thread_block((void (*)(void))0); ! 525: ! 526: wait_result = thread->wait_result; ! 527: vm_object_lock(object); ! 528: if (wait_result != THREAD_AWAKENED) { ! 529: vm_fault_cleanup(object, first_m); ! 530: if (wait_result == THREAD_RESTART) ! 531: { ! 532: return(VM_FAULT_RETRY); ! 533: } ! 534: else ! 535: { ! 536: return(VM_FAULT_INTERRUPTED); ! 537: } ! 538: } ! 539: continue; ! 540: } ! 541: ! 542: /* ! 543: * If the page is in error, give up now. ! 544: */ ! 545: ! 546: if (m->error) { ! 547: #if TRACEFAULTPAGE ! 548: dbgTrace(0xBEEF0006, (unsigned int) m, (unsigned int) error_code); /* (TEST/DEBUG) */ ! 549: #endif ! 550: if (error_code) ! 551: *error_code = m->page_error; ! 552: VM_PAGE_FREE(m); ! 553: vm_fault_cleanup(object, first_m); ! 554: ! 555: return(VM_FAULT_MEMORY_ERROR); ! 556: } ! 557: ! 558: /* ! 559: * If the pager wants us to restart ! 560: * at the top of the chain, ! 561: * typically because it has moved the ! 562: * page to another pager, then do so. ! 563: */ ! 564: ! 565: if (m->restart) { ! 566: #if TRACEFAULTPAGE ! 567: dbgTrace(0xBEEF0007, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */ ! 568: #endif ! 569: VM_PAGE_FREE(m); ! 570: vm_fault_cleanup(object, first_m); ! 571: ! 572: return(VM_FAULT_RETRY); ! 573: } ! 574: ! 575: /* ! 576: * If the page isn't busy, but is absent, ! 577: * then it was deemed "unavailable". ! 578: */ ! 579: ! 580: if (m->absent) { ! 581: /* ! 582: * Remove the non-existent page (unless it's ! 583: * in the top object) and move on down to the ! 584: * next object (if there is one). ! 585: */ ! 586: #if TRACEFAULTPAGE ! 587: dbgTrace(0xBEEF0008, (unsigned int) m, (unsigned int) object->shadow); /* (TEST/DEBUG) */ ! 588: #endif ! 589: ! 590: next_object = object->shadow; ! 591: if (next_object == VM_OBJECT_NULL) { ! 592: vm_page_t real_m; ! 593: ! 594: assert(!must_be_resident); ! 595: ! 596: /* ! 597: * Absent page at bottom of shadow ! 598: * chain; zero fill the page we left ! 599: * busy in the first object, and flush ! 600: * the absent page. But first we ! 601: * need to allocate a real page. ! 602: */ ! 603: if ((vm_page_free_target - ! 604: ((vm_page_free_target ! 605: -vm_page_free_min)>>2)) ! 606: > vm_page_free_count) { ! 607: assert(object->ref_count > 0); ! 608: vm_fault_cleanup( ! 609: object, first_m); ! 610: /* kick off pageout daemon */ ! 611: vm_page_wait(); ! 612: if ((m = vm_page_grab()) ! 613: != VM_PAGE_NULL) { ! 614: /* need to kick off */ ! 615: /* other parties */ ! 616: /* waiting on free */ ! 617: /* pages */ ! 618: VM_PAGE_FREE(m); ! 619: } ! 620: return VM_FAULT_RETRY; ! 621: } ! 622: ! 623: real_m = vm_page_grab(); ! 624: if (real_m == VM_PAGE_NULL) { ! 625: vm_fault_cleanup(object, first_m); ! 626: return(VM_FAULT_MEMORY_SHORTAGE); ! 627: } ! 628: ! 629: XPR(XPR_VM_FAULT, ! 630: "vm_f_page: zero obj 0x%X, off 0x%X, page 0x%X, first_obj 0x%X\n", ! 631: (integer_t)object, offset, ! 632: (integer_t)m, ! 633: (integer_t)first_object, 0); ! 634: if (object != first_object) { ! 635: VM_PAGE_FREE(m); ! 636: vm_object_paging_end(object); ! 637: vm_object_unlock(object); ! 638: object = first_object; ! 639: offset = first_offset; ! 640: m = first_m; ! 641: first_m = VM_PAGE_NULL; ! 642: vm_object_lock(object); ! 643: } ! 644: ! 645: VM_PAGE_FREE(m); ! 646: assert(real_m->busy); ! 647: vm_page_insert(real_m, object, offset); ! 648: m = real_m; ! 649: ! 650: /* ! 651: * Drop the lock while zero filling ! 652: * page. Then break because this ! 653: * is the page we wanted. Checking ! 654: * the page lock is a waste of time; ! 655: * this page was either absent or ! 656: * newly allocated -- in both cases ! 657: * it can't be page locked by a pager. ! 658: */ ! 659: if (!no_zero_fill) { ! 660: vm_object_unlock(object); ! 661: vm_page_zero_fill(m); ! 662: if (type_of_fault) ! 663: *type_of_fault = DBG_ZERO_FILL_FAULT; ! 664: VM_STAT(zero_fill_count++); ! 665: vm_object_lock(object); ! 666: } ! 667: pmap_clear_modify(m->phys_addr); ! 668: vm_page_lock_queues(); ! 669: VM_PAGE_QUEUES_REMOVE(m); ! 670: queue_enter(&vm_page_queue_inactive, ! 671: m, vm_page_t, pageq); ! 672: m->inactive = TRUE; ! 673: vm_page_inactive_count++; ! 674: vm_page_unlock_queues(); ! 675: break; ! 676: } else { ! 677: if (must_be_resident) { ! 678: vm_object_paging_end(object); ! 679: } else if (object != first_object) { ! 680: vm_object_paging_end(object); ! 681: VM_PAGE_FREE(m); ! 682: } else { ! 683: first_m = m; ! 684: m->absent = FALSE; ! 685: m->unusual = FALSE; ! 686: vm_object_absent_release(object); ! 687: m->busy = TRUE; ! 688: ! 689: vm_page_lock_queues(); ! 690: VM_PAGE_QUEUES_REMOVE(m); ! 691: vm_page_unlock_queues(); ! 692: } ! 693: XPR(XPR_VM_FAULT, ! 694: "vm_f_page: unavail obj 0x%X, off 0x%X, next_obj 0x%X, newoff 0x%X\n", ! 695: (integer_t)object, offset, ! 696: (integer_t)next_object, ! 697: offset+object->shadow_offset,0); ! 698: offset += object->shadow_offset; ! 699: hi_offset += object->shadow_offset; ! 700: lo_offset += object->shadow_offset; ! 701: access_required = VM_PROT_READ; ! 702: vm_object_lock(next_object); ! 703: vm_object_unlock(object); ! 704: object = next_object; ! 705: vm_object_paging_begin(object); ! 706: continue; ! 707: } ! 708: } ! 709: ! 710: if ((m->cleaning) ! 711: && ((object != first_object) || ! 712: (object->copy != VM_OBJECT_NULL)) ! 713: && (fault_type & VM_PROT_WRITE)) { ! 714: /* ! 715: * This is a copy-on-write fault that will ! 716: * cause us to revoke access to this page, but ! 717: * this page is in the process of being cleaned ! 718: * in a clustered pageout. We must wait until ! 719: * the cleaning operation completes before ! 720: * revoking access to the original page, ! 721: * otherwise we might attempt to remove a ! 722: * wired mapping. ! 723: */ ! 724: #if TRACEFAULTPAGE ! 725: dbgTrace(0xBEEF0009, (unsigned int) m, (unsigned int) offset); /* (TEST/DEBUG) */ ! 726: #endif ! 727: XPR(XPR_VM_FAULT, ! 728: "vm_f_page: cleaning obj 0x%X, offset 0x%X, page 0x%X\n", ! 729: (integer_t)object, offset, ! 730: (integer_t)m, 0, 0); ! 731: /* take an extra ref so that object won't die */ ! 732: assert(object->ref_count > 0); ! 733: object->ref_count++; ! 734: vm_object_res_reference(object); ! 735: vm_fault_cleanup(object, first_m); ! 736: counter(c_vm_fault_page_block_backoff_kernel++); ! 737: vm_object_lock(object); ! 738: assert(object->ref_count > 0); ! 739: m = vm_page_lookup(object, offset); ! 740: if (m != VM_PAGE_NULL && m->cleaning) { ! 741: PAGE_ASSERT_WAIT(m, interruptible); ! 742: vm_object_unlock(object); ! 743: thread_block((void (*)(void)) 0); ! 744: vm_object_deallocate(object); ! 745: goto backoff; ! 746: } else { ! 747: vm_object_unlock(object); ! 748: vm_object_deallocate(object); ! 749: return VM_FAULT_RETRY; ! 750: } ! 751: } ! 752: ! 753: /* ! 754: * If the desired access to this page has ! 755: * been locked out, request that it be unlocked. ! 756: */ ! 757: ! 758: if (access_required & m->page_lock) { ! 759: if ((access_required & m->unlock_request) != access_required) { ! 760: vm_prot_t new_unlock_request; ! 761: kern_return_t rc; ! 762: ! 763: #if TRACEFAULTPAGE ! 764: dbgTrace(0xBEEF000A, (unsigned int) m, (unsigned int) object->pager_ready); /* (TEST/DEBUG) */ ! 765: #endif ! 766: if (!object->pager_ready) { ! 767: XPR(XPR_VM_FAULT, ! 768: "vm_f_page: ready wait acc_req %d, obj 0x%X, offset 0x%X, page 0x%X\n", ! 769: access_required, ! 770: (integer_t)object, offset, ! 771: (integer_t)m, 0); ! 772: /* take an extra ref */ ! 773: assert(object->ref_count > 0); ! 774: object->ref_count++; ! 775: vm_object_res_reference(object); ! 776: vm_fault_cleanup(object, ! 777: first_m); ! 778: counter(c_vm_fault_page_block_backoff_kernel++); ! 779: vm_object_lock(object); ! 780: assert(object->ref_count > 0); ! 781: if (!object->pager_ready) { ! 782: vm_object_assert_wait( ! 783: object, ! 784: VM_OBJECT_EVENT_PAGER_READY, ! 785: interruptible); ! 786: vm_object_unlock(object); ! 787: thread_block((void (*)(void))0); ! 788: vm_object_deallocate(object); ! 789: goto backoff; ! 790: } else { ! 791: vm_object_unlock(object); ! 792: vm_object_deallocate(object); ! 793: return VM_FAULT_RETRY; ! 794: } ! 795: } ! 796: ! 797: new_unlock_request = m->unlock_request = ! 798: (access_required | m->unlock_request); ! 799: vm_object_unlock(object); ! 800: XPR(XPR_VM_FAULT, ! 801: "vm_f_page: unlock obj 0x%X, offset 0x%X, page 0x%X, unl_req %d\n", ! 802: (integer_t)object, offset, ! 803: (integer_t)m, new_unlock_request, 0); ! 804: if ((rc = memory_object_data_unlock( ! 805: object->pager, ! 806: object->pager_request, ! 807: offset + object->paging_offset, ! 808: PAGE_SIZE, ! 809: new_unlock_request)) ! 810: != KERN_SUCCESS) { ! 811: if (vm_fault_debug) ! 812: printf("vm_fault: memory_object_data_unlock failed\n"); ! 813: vm_object_lock(object); ! 814: vm_fault_cleanup(object, first_m); ! 815: ! 816: return((rc == MACH_SEND_INTERRUPTED) ? ! 817: VM_FAULT_INTERRUPTED : ! 818: VM_FAULT_MEMORY_ERROR); ! 819: } ! 820: vm_object_lock(object); ! 821: continue; ! 822: } ! 823: ! 824: XPR(XPR_VM_FAULT, ! 825: "vm_f_page: access wait acc_req %d, obj 0x%X, offset 0x%X, page 0x%X\n", ! 826: access_required, (integer_t)object, ! 827: offset, (integer_t)m, 0); ! 828: /* take an extra ref so object won't die */ ! 829: assert(object->ref_count > 0); ! 830: object->ref_count++; ! 831: vm_object_res_reference(object); ! 832: vm_fault_cleanup(object, first_m); ! 833: counter(c_vm_fault_page_block_backoff_kernel++); ! 834: vm_object_lock(object); ! 835: assert(object->ref_count > 0); ! 836: m = vm_page_lookup(object, offset); ! 837: if (m != VM_PAGE_NULL && ! 838: (access_required & m->page_lock) && ! 839: !((access_required & m->unlock_request) != access_required)) { ! 840: PAGE_ASSERT_WAIT(m, interruptible); ! 841: vm_object_unlock(object); ! 842: thread_block((void (*)(void)) 0); ! 843: vm_object_deallocate(object); ! 844: goto backoff; ! 845: } else { ! 846: vm_object_unlock(object); ! 847: vm_object_deallocate(object); ! 848: return VM_FAULT_RETRY; ! 849: } ! 850: } ! 851: /* ! 852: * We mark the page busy and leave it on ! 853: * the pageout queues. If the pageout ! 854: * deamon comes across it, then it will ! 855: * remove the page. ! 856: */ ! 857: ! 858: #if TRACEFAULTPAGE ! 859: dbgTrace(0xBEEF000B, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */ ! 860: #endif ! 861: ! 862: #if !VM_FAULT_STATIC_CONFIG ! 863: if (!software_reference_bits) { ! 864: vm_page_lock_queues(); ! 865: if (m->inactive) ! 866: vm_stat.reactivations++; ! 867: ! 868: VM_PAGE_QUEUES_REMOVE(m); ! 869: vm_page_unlock_queues(); ! 870: } ! 871: #endif ! 872: XPR(XPR_VM_FAULT, ! 873: "vm_f_page: found page obj 0x%X, offset 0x%X, page 0x%X\n", ! 874: (integer_t)object, offset, (integer_t)m, 0, 0); ! 875: assert(!m->busy); ! 876: m->busy = TRUE; ! 877: assert(!m->absent); ! 878: break; ! 879: } ! 880: ! 881: look_for_page = ! 882: (object->pager_created) && ! 883: LOOK_FOR(object, offset) && ! 884: (!data_supply); ! 885: ! 886: #if TRACEFAULTPAGE ! 887: dbgTrace(0xBEEF000C, (unsigned int) look_for_page, (unsigned int) object); /* (TEST/DEBUG) */ ! 888: #endif ! 889: if ((look_for_page || (object == first_object)) ! 890: && !must_be_resident) { ! 891: /* ! 892: * Allocate a new page for this object/offset ! 893: * pair. ! 894: */ ! 895: ! 896: m = vm_page_grab_fictitious(); ! 897: #if TRACEFAULTPAGE ! 898: dbgTrace(0xBEEF000D, (unsigned int) m, (unsigned int) object); /* (TEST/DEBUG) */ ! 899: #endif ! 900: if (m == VM_PAGE_NULL) { ! 901: vm_fault_cleanup(object, first_m); ! 902: return(VM_FAULT_FICTITIOUS_SHORTAGE); ! 903: } ! 904: vm_page_insert(m, object, offset); ! 905: } ! 906: ! 907: if (look_for_page && !must_be_resident) { ! 908: kern_return_t rc; ! 909: ! 910: /* ! 911: * If the memory manager is not ready, we ! 912: * cannot make requests. ! 913: */ ! 914: if (!object->pager_ready) { ! 915: #if TRACEFAULTPAGE ! 916: dbgTrace(0xBEEF000E, (unsigned int) 0, (unsigned int) 0); /* (TEST/DEBUG) */ ! 917: #endif ! 918: VM_PAGE_FREE(m); ! 919: XPR(XPR_VM_FAULT, ! 920: "vm_f_page: ready wait obj 0x%X, offset 0x%X\n", ! 921: (integer_t)object, offset, 0, 0, 0); ! 922: /* take an extra ref so object won't die */ ! 923: assert(object->ref_count > 0); ! 924: object->ref_count++; ! 925: vm_object_res_reference(object); ! 926: vm_fault_cleanup(object, first_m); ! 927: counter(c_vm_fault_page_block_backoff_kernel++); ! 928: vm_object_lock(object); ! 929: assert(object->ref_count > 0); ! 930: if (!object->pager_ready) { ! 931: vm_object_assert_wait(object, ! 932: VM_OBJECT_EVENT_PAGER_READY, ! 933: interruptible); ! 934: vm_object_unlock(object); ! 935: thread_block((void (*)(void))0); ! 936: vm_object_deallocate(object); ! 937: goto backoff; ! 938: } else { ! 939: vm_object_unlock(object); ! 940: vm_object_deallocate(object); ! 941: return VM_FAULT_RETRY; ! 942: } ! 943: } ! 944: ! 945: if (object->internal) { ! 946: /* ! 947: * Requests to the default pager ! 948: * must reserve a real page in advance, ! 949: * because the pager's data-provided ! 950: * won't block for pages. IMPORTANT: ! 951: * this acts as a throttling mechanism ! 952: * for data_requests to the default ! 953: * pager. ! 954: */ ! 955: ! 956: #if TRACEFAULTPAGE ! 957: dbgTrace(0xBEEF000F, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */ ! 958: #endif ! 959: if (m->fictitious && !vm_page_convert(m)) { ! 960: VM_PAGE_FREE(m); ! 961: vm_fault_cleanup(object, first_m); ! 962: return(VM_FAULT_MEMORY_SHORTAGE); ! 963: } ! 964: } else if (object->absent_count > ! 965: vm_object_absent_max) { ! 966: /* ! 967: * If there are too many outstanding page ! 968: * requests pending on this object, we ! 969: * wait for them to be resolved now. ! 970: */ ! 971: ! 972: #if TRACEFAULTPAGE ! 973: dbgTrace(0xBEEF0010, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */ ! 974: #endif ! 975: VM_PAGE_FREE(m); ! 976: /* take an extra ref so object won't die */ ! 977: assert(object->ref_count > 0); ! 978: object->ref_count++; ! 979: vm_object_res_reference(object); ! 980: vm_fault_cleanup(object, first_m); ! 981: counter(c_vm_fault_page_block_backoff_kernel++); ! 982: vm_object_lock(object); ! 983: assert(object->ref_count > 0); ! 984: if (object->absent_count > vm_object_absent_max) { ! 985: vm_object_absent_assert_wait(object, ! 986: interruptible); ! 987: vm_object_unlock(object); ! 988: thread_block((void (*)(void))0); ! 989: vm_object_deallocate(object); ! 990: goto backoff; ! 991: } else { ! 992: vm_object_unlock(object); ! 993: vm_object_deallocate(object); ! 994: return VM_FAULT_RETRY; ! 995: } ! 996: } ! 997: ! 998: /* ! 999: * Indicate that the page is waiting for data ! 1000: * from the memory manager. ! 1001: */ ! 1002: ! 1003: m->list_req_pending = TRUE; ! 1004: m->absent = TRUE; ! 1005: m->unusual = TRUE; ! 1006: object->absent_count++; ! 1007: ! 1008: cluster_start = offset; ! 1009: length = PAGE_SIZE; ! 1010: cluster_size = object->cluster_size; ! 1011: ! 1012: /* ! 1013: * Skip clustered pagein if it is globally disabled ! 1014: * or random page reference behavior is expected ! 1015: * for the address range containing the faulting ! 1016: * address or the object paging block size is ! 1017: * equal to the page size. ! 1018: */ ! 1019: if (!vm_allow_clustered_pagein || ! 1020: behavior == VM_BEHAVIOR_RANDOM || ! 1021: cluster_size == PAGE_SIZE) ! 1022: goto no_clustering; ! 1023: ! 1024: assert(offset >= lo_offset); ! 1025: assert(offset < hi_offset); ! 1026: assert(ALIGNED(object->paging_offset)); ! 1027: assert(cluster_size >= PAGE_SIZE); ! 1028: ! 1029: #if TRACEFAULTPAGE ! 1030: dbgTrace(0xBEEF0011, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */ ! 1031: #endif ! 1032: /* ! 1033: * Decide whether to scan ahead or behind for ! 1034: * additional pages contiguous to the faulted ! 1035: * page in the same paging block. The decision ! 1036: * is based on system wide globals and the ! 1037: * expected page reference behavior of the ! 1038: * address range contained the faulting address. ! 1039: * First calculate some constants. ! 1040: */ ! 1041: paging_offset = offset + object->paging_offset; ! 1042: cluster_offset = paging_offset & (cluster_size - 1); ! 1043: align_offset = paging_offset&(PAGE_SIZE-1); ! 1044: if (align_offset != 0) { ! 1045: cluster_offset = trunc_page(cluster_offset); ! 1046: } ! 1047: ! 1048: #define SPANS_CLUSTER(x) ((((x) - align_offset) & (cluster_size - 1)) == 0) ! 1049: ! 1050: /* ! 1051: * Backward scan only if reverse sequential ! 1052: * behavior has been specified ! 1053: */ ! 1054: CLUSTER_STAT(pages_at_lower_offsets = 0;) ! 1055: if (((vm_default_behind != 0 && ! 1056: behavior == VM_BEHAVIOR_DEFAULT) || ! 1057: behavior == VM_BEHAVIOR_RSEQNTL) && offset) { ! 1058: vm_offset_t cluster_bot; ! 1059: ! 1060: /* ! 1061: * Calculate lower search boundary. ! 1062: * Exclude pages that span a cluster boundary. ! 1063: * Clip to start of map entry. ! 1064: * For default page reference behavior, scan ! 1065: * default pages behind. ! 1066: */ ! 1067: cluster_bot = (offset > cluster_offset) ? ! 1068: offset - cluster_offset : offset; ! 1069: if (align_offset != 0) { ! 1070: if ((cluster_bot < offset) && ! 1071: SPANS_CLUSTER(cluster_bot)) { ! 1072: cluster_bot += PAGE_SIZE; ! 1073: } ! 1074: } ! 1075: if (behavior == VM_BEHAVIOR_DEFAULT) { ! 1076: vm_offset_t bot = vm_default_behind*PAGE_SIZE; ! 1077: ! 1078: if (cluster_bot < (offset - bot)) ! 1079: cluster_bot = offset - bot; ! 1080: } ! 1081: if (lo_offset > cluster_bot) ! 1082: cluster_bot = lo_offset; ! 1083: ! 1084: for ( cluster_start = offset - PAGE_SIZE; ! 1085: (cluster_start >= cluster_bot) && ! 1086: (cluster_start != (align_offset - PAGE_SIZE)); ! 1087: cluster_start -= PAGE_SIZE) { ! 1088: assert(cluster_size > PAGE_SIZE); ! 1089: retry_cluster_backw: ! 1090: if (!LOOK_FOR(object, cluster_start) || ! 1091: vm_page_lookup(object, cluster_start) ! 1092: != VM_PAGE_NULL) { ! 1093: break; ! 1094: } ! 1095: if (object->internal) { ! 1096: /* ! 1097: * need to acquire a real page in ! 1098: * advance because this acts as ! 1099: * a throttling mechanism for ! 1100: * data_requests to the default ! 1101: * pager. If this fails, give up ! 1102: * trying to find any more pages ! 1103: * in the cluster and send off the ! 1104: * request for what we already have. ! 1105: */ ! 1106: if ((m = vm_page_grab()) ! 1107: == VM_PAGE_NULL) { ! 1108: cluster_start += PAGE_SIZE; ! 1109: cluster_end = offset + PAGE_SIZE; ! 1110: goto give_up; ! 1111: } ! 1112: } else if ((m = vm_page_grab_fictitious()) ! 1113: == VM_PAGE_NULL) { ! 1114: vm_object_unlock(object); ! 1115: vm_page_more_fictitious(); ! 1116: vm_object_lock(object); ! 1117: goto retry_cluster_backw; ! 1118: } ! 1119: m->absent = TRUE; ! 1120: m->unusual = TRUE; ! 1121: m->clustered = TRUE; ! 1122: m->list_req_pending = TRUE; ! 1123: ! 1124: vm_page_insert(m, object, cluster_start); ! 1125: CLUSTER_STAT(pages_at_lower_offsets++;) ! 1126: object->absent_count++; ! 1127: } ! 1128: cluster_start += PAGE_SIZE; ! 1129: assert(cluster_start >= cluster_bot); ! 1130: } ! 1131: assert(cluster_start <= offset); ! 1132: ! 1133: /* ! 1134: * Forward scan if default or sequential behavior ! 1135: * specified ! 1136: */ ! 1137: CLUSTER_STAT(pages_at_higher_offsets = 0;) ! 1138: if ((behavior == VM_BEHAVIOR_DEFAULT && ! 1139: vm_default_ahead != 0) || ! 1140: behavior == VM_BEHAVIOR_SEQUENTIAL) { ! 1141: vm_offset_t cluster_top; ! 1142: ! 1143: /* ! 1144: * Calculate upper search boundary. ! 1145: * Exclude pages that span a cluster boundary. ! 1146: * Clip to end of map entry. ! 1147: * For default page reference behavior, scan ! 1148: * default pages ahead. ! 1149: */ ! 1150: cluster_top = (offset + cluster_size) - ! 1151: cluster_offset; ! 1152: if (align_offset != 0) { ! 1153: if ((cluster_top > (offset + PAGE_SIZE)) && ! 1154: SPANS_CLUSTER(cluster_top)) { ! 1155: cluster_top -= PAGE_SIZE; ! 1156: } ! 1157: } ! 1158: if (behavior == VM_BEHAVIOR_DEFAULT) { ! 1159: vm_offset_t top = (vm_default_ahead*PAGE_SIZE)+ ! 1160: PAGE_SIZE; ! 1161: ! 1162: if (cluster_top > (offset + top)) ! 1163: cluster_top = offset + top; ! 1164: } ! 1165: if (cluster_top > hi_offset) ! 1166: cluster_top = hi_offset; ! 1167: ! 1168: for (cluster_end = offset + PAGE_SIZE; ! 1169: cluster_end < cluster_top; ! 1170: cluster_end += PAGE_SIZE) { ! 1171: assert(cluster_size > PAGE_SIZE); ! 1172: retry_cluster_forw: ! 1173: if (!LOOK_FOR(object, cluster_end) || ! 1174: vm_page_lookup(object, cluster_end) ! 1175: != VM_PAGE_NULL) { ! 1176: break; ! 1177: } ! 1178: if (object->internal) { ! 1179: /* ! 1180: * need to acquire a real page in ! 1181: * advance because this acts as ! 1182: * a throttling mechanism for ! 1183: * data_requests to the default ! 1184: * pager. If this fails, give up ! 1185: * trying to find any more pages ! 1186: * in the cluster and send off the ! 1187: * request for what we already have. ! 1188: */ ! 1189: if ((m = vm_page_grab()) ! 1190: == VM_PAGE_NULL) { ! 1191: break; ! 1192: } ! 1193: } else if ((m = vm_page_grab_fictitious()) ! 1194: == VM_PAGE_NULL) { ! 1195: vm_object_unlock(object); ! 1196: vm_page_more_fictitious(); ! 1197: vm_object_lock(object); ! 1198: goto retry_cluster_forw; ! 1199: } ! 1200: m->absent = TRUE; ! 1201: m->unusual = TRUE; ! 1202: m->clustered = TRUE; ! 1203: m->list_req_pending = TRUE; ! 1204: ! 1205: vm_page_insert(m, object, cluster_end); ! 1206: CLUSTER_STAT(pages_at_higher_offsets++;) ! 1207: object->absent_count++; ! 1208: } ! 1209: assert(cluster_end <= cluster_top); ! 1210: } ! 1211: else { ! 1212: cluster_end = offset + PAGE_SIZE; ! 1213: } ! 1214: give_up: ! 1215: assert(cluster_end >= offset + PAGE_SIZE); ! 1216: length = cluster_end - cluster_start; ! 1217: ! 1218: #if MACH_CLUSTER_STATS ! 1219: CLUSTER_STAT_HIGHER(pages_at_higher_offsets); ! 1220: CLUSTER_STAT_LOWER(pages_at_lower_offsets); ! 1221: CLUSTER_STAT_CLUSTER(length/PAGE_SIZE); ! 1222: #endif /* MACH_CLUSTER_STATS */ ! 1223: ! 1224: no_clustering: ! 1225: #if TRACEFAULTPAGE ! 1226: dbgTrace(0xBEEF0012, (unsigned int) object, (unsigned int) 0); /* (TEST/DEBUG) */ ! 1227: #endif ! 1228: /* ! 1229: * We have a busy page, so we can ! 1230: * release the object lock. ! 1231: */ ! 1232: vm_object_unlock(object); ! 1233: ! 1234: /* ! 1235: * Call the memory manager to retrieve the data. ! 1236: */ ! 1237: ! 1238: if (type_of_fault) ! 1239: *type_of_fault = DBG_PAGEIN_FAULT; ! 1240: VM_STAT(pageins++); ! 1241: current_task()->pageins++; ! 1242: ! 1243: /* ! 1244: * If this object uses a copy_call strategy, ! 1245: * and we are interested in a copy of this object ! 1246: * (having gotten here only by following a ! 1247: * shadow chain), then tell the memory manager ! 1248: * via a flag added to the desired_access ! 1249: * parameter, so that it can detect a race ! 1250: * between our walking down the shadow chain ! 1251: * and its pushing pages up into a copy of ! 1252: * the object that it manages. ! 1253: */ ! 1254: ! 1255: if (object->copy_strategy == MEMORY_OBJECT_COPY_CALL && ! 1256: object != first_object) { ! 1257: wants_copy_flag = VM_PROT_WANTS_COPY; ! 1258: } else { ! 1259: wants_copy_flag = VM_PROT_NONE; ! 1260: } ! 1261: ! 1262: XPR(XPR_VM_FAULT, ! 1263: "vm_f_page: data_req obj 0x%X, offset 0x%X, page 0x%X, acc %d\n", ! 1264: (integer_t)object, offset, (integer_t)m, ! 1265: access_required | wants_copy_flag, 0); ! 1266: ! 1267: #ifdef MACH_BSD ! 1268: if (((rpc_subsystem_t)pager_mux_hash_lookup(object->pager)) == ! 1269: ((rpc_subsystem_t) &vnode_pager_workaround)) { ! 1270: rc = vnode_pager_data_request(object->pager, ! 1271: object->pager_request, ! 1272: cluster_start + object->paging_offset, ! 1273: length, ! 1274: access_required | wants_copy_flag); ! 1275: } else { ! 1276: rc = memory_object_data_request(object->pager, ! 1277: object->pager_request, ! 1278: cluster_start + object->paging_offset, ! 1279: length, ! 1280: access_required | wants_copy_flag); ! 1281: } ! 1282: #else ! 1283: rc = memory_object_data_request(object->pager, ! 1284: object->pager_request, ! 1285: cluster_start + object->paging_offset, ! 1286: length, ! 1287: access_required | wants_copy_flag); ! 1288: ! 1289: #endif ! 1290: ! 1291: #if TRACEFAULTPAGE ! 1292: dbgTrace(0xBEEF0013, (unsigned int) object, (unsigned int) rc); /* (TEST/DEBUG) */ ! 1293: #endif ! 1294: if (rc != KERN_SUCCESS) { ! 1295: if (rc != MACH_SEND_INTERRUPTED ! 1296: && vm_fault_debug) ! 1297: printf("%s(0x%x, 0x%x, 0x%x, 0x%x, 0x%x) failed, rc=%d, object=0x%x\n", ! 1298: "memory_object_data_request", ! 1299: object->pager, ! 1300: object->pager_request, ! 1301: cluster_start + object->paging_offset, ! 1302: length, access_required, ! 1303: rc, object); ! 1304: /* ! 1305: * Don't want to leave a busy page around, ! 1306: * but the data request may have blocked, ! 1307: * so check if it's still there and busy. ! 1308: */ ! 1309: vm_object_lock(object); ! 1310: for (; length; ! 1311: length -= PAGE_SIZE, ! 1312: cluster_start += PAGE_SIZE) { ! 1313: vm_page_t p; ! 1314: if ((p = vm_page_lookup(object, ! 1315: cluster_start)) ! 1316: && p->absent && p->busy ! 1317: && p != first_m) { ! 1318: VM_PAGE_FREE(m); ! 1319: } ! 1320: } ! 1321: vm_fault_cleanup(object, first_m); ! 1322: ! 1323: return((rc == MACH_SEND_INTERRUPTED) ? ! 1324: VM_FAULT_INTERRUPTED : ! 1325: VM_FAULT_MEMORY_ERROR); ! 1326: } ! 1327: ! 1328: /* ! 1329: * Retry with same object/offset, since new data may ! 1330: * be in a different page (i.e., m is meaningless at ! 1331: * this point). ! 1332: */ ! 1333: vm_object_lock(object); ! 1334: continue; ! 1335: } ! 1336: ! 1337: /* ! 1338: * The only case in which we get here is if ! 1339: * object has no pager (or unwiring). If the pager doesn't ! 1340: * have the page this is handled in the m->absent case above ! 1341: * (and if you change things here you should look above). ! 1342: */ ! 1343: #if TRACEFAULTPAGE ! 1344: dbgTrace(0xBEEF0014, (unsigned int) object, (unsigned int) m); /* (TEST/DEBUG) */ ! 1345: #endif ! 1346: if (object == first_object) ! 1347: first_m = m; ! 1348: else ! 1349: assert(m == VM_PAGE_NULL); ! 1350: ! 1351: XPR(XPR_VM_FAULT, ! 1352: "vm_f_page: no pager obj 0x%X, offset 0x%X, page 0x%X, next_obj 0x%X\n", ! 1353: (integer_t)object, offset, (integer_t)m, ! 1354: (integer_t)object->shadow, 0); ! 1355: /* ! 1356: * Move on to the next object. Lock the next ! 1357: * object before unlocking the current one. ! 1358: */ ! 1359: next_object = object->shadow; ! 1360: if (next_object == VM_OBJECT_NULL) { ! 1361: assert(!must_be_resident); ! 1362: ! 1363: /* ! 1364: * If there's no object left, fill the page ! 1365: * in the top object with zeros. But first we ! 1366: * need to allocate a real page. ! 1367: */ ! 1368: ! 1369: if (object != first_object) { ! 1370: vm_object_paging_end(object); ! 1371: vm_object_unlock(object); ! 1372: ! 1373: object = first_object; ! 1374: offset = first_offset; ! 1375: vm_object_lock(object); ! 1376: } ! 1377: ! 1378: m = first_m; ! 1379: assert(m->object == object); ! 1380: first_m = VM_PAGE_NULL; ! 1381: ! 1382: ! 1383: if ((vm_page_free_target - ! 1384: ((vm_page_free_target-vm_page_free_min)>>2)) ! 1385: > vm_page_free_count) { ! 1386: VM_PAGE_FREE(m); ! 1387: /* take an extra ref so object won't die */ ! 1388: assert(object->ref_count > 0); ! 1389: vm_fault_cleanup(object, first_m); ! 1390: vm_page_wait(); /* kick off pageout daemon */ ! 1391: if ((m = vm_page_grab()) != VM_PAGE_NULL) { ! 1392: /* need to kick off other parties */ ! 1393: /* waiting on free pages */ ! 1394: VM_PAGE_FREE(m); ! 1395: } ! 1396: return VM_FAULT_RETRY; ! 1397: } ! 1398: ! 1399: if (m->fictitious && !vm_page_convert(m)) { ! 1400: VM_PAGE_FREE(m); ! 1401: vm_fault_cleanup(object, VM_PAGE_NULL); ! 1402: return(VM_FAULT_MEMORY_SHORTAGE); ! 1403: } ! 1404: ! 1405: if (!no_zero_fill) { ! 1406: vm_object_unlock(object); ! 1407: vm_page_zero_fill(m); ! 1408: if (type_of_fault) ! 1409: *type_of_fault = DBG_ZERO_FILL_FAULT; ! 1410: VM_STAT(zero_fill_count++); ! 1411: vm_object_lock(object); ! 1412: } ! 1413: vm_page_lock_queues(); ! 1414: VM_PAGE_QUEUES_REMOVE(m); ! 1415: queue_enter(&vm_page_queue_inactive, ! 1416: m, vm_page_t, pageq); ! 1417: m->inactive = TRUE; ! 1418: vm_page_inactive_count++; ! 1419: vm_page_unlock_queues(); ! 1420: pmap_clear_modify(m->phys_addr); ! 1421: break; ! 1422: } ! 1423: else { ! 1424: if ((object != first_object) || must_be_resident) ! 1425: vm_object_paging_end(object); ! 1426: offset += object->shadow_offset; ! 1427: hi_offset += object->shadow_offset; ! 1428: lo_offset += object->shadow_offset; ! 1429: access_required = VM_PROT_READ; ! 1430: vm_object_lock(next_object); ! 1431: vm_object_unlock(object); ! 1432: object = next_object; ! 1433: vm_object_paging_begin(object); ! 1434: } ! 1435: } ! 1436: ! 1437: /* ! 1438: * PAGE HAS BEEN FOUND. ! 1439: * ! 1440: * This page (m) is: ! 1441: * busy, so that we can play with it; ! 1442: * not absent, so that nobody else will fill it; ! 1443: * possibly eligible for pageout; ! 1444: * ! 1445: * The top-level page (first_m) is: ! 1446: * VM_PAGE_NULL if the page was found in the ! 1447: * top-level object; ! 1448: * busy, not absent, and ineligible for pageout. ! 1449: * ! 1450: * The current object (object) is locked. A paging ! 1451: * reference is held for the current and top-level ! 1452: * objects. ! 1453: */ ! 1454: ! 1455: #if TRACEFAULTPAGE ! 1456: dbgTrace(0xBEEF0015, (unsigned int) object, (unsigned int) m); /* (TEST/DEBUG) */ ! 1457: #endif ! 1458: #if EXTRA_ASSERTIONS ! 1459: assert(m->busy && !m->absent); ! 1460: assert((first_m == VM_PAGE_NULL) || ! 1461: (first_m->busy && !first_m->absent && ! 1462: !first_m->active && !first_m->inactive)); ! 1463: #endif /* EXTRA_ASSERTIONS */ ! 1464: ! 1465: XPR(XPR_VM_FAULT, ! 1466: "vm_f_page: FOUND obj 0x%X, off 0x%X, page 0x%X, 1_obj 0x%X, 1_m 0x%X\n", ! 1467: (integer_t)object, offset, (integer_t)m, ! 1468: (integer_t)first_object, (integer_t)first_m); ! 1469: /* ! 1470: * If the page is being written, but isn't ! 1471: * already owned by the top-level object, ! 1472: * we have to copy it into a new page owned ! 1473: * by the top-level object. ! 1474: */ ! 1475: ! 1476: if (object != first_object) { ! 1477: /* ! 1478: * We only really need to copy if we ! 1479: * want to write it. ! 1480: */ ! 1481: ! 1482: #if TRACEFAULTPAGE ! 1483: dbgTrace(0xBEEF0016, (unsigned int) object, (unsigned int) fault_type); /* (TEST/DEBUG) */ ! 1484: #endif ! 1485: if (fault_type & VM_PROT_WRITE) { ! 1486: vm_page_t copy_m; ! 1487: ! 1488: assert(!must_be_resident); ! 1489: ! 1490: /* ! 1491: * If we try to collapse first_object at this ! 1492: * point, we may deadlock when we try to get ! 1493: * the lock on an intermediate object (since we ! 1494: * have the bottom object locked). We can't ! 1495: * unlock the bottom object, because the page ! 1496: * we found may move (by collapse) if we do. ! 1497: * ! 1498: * Instead, we first copy the page. Then, when ! 1499: * we have no more use for the bottom object, ! 1500: * we unlock it and try to collapse. ! 1501: * ! 1502: * Note that we copy the page even if we didn't ! 1503: * need to... that's the breaks. ! 1504: */ ! 1505: ! 1506: /* ! 1507: * Allocate a page for the copy ! 1508: */ ! 1509: copy_m = vm_page_grab(); ! 1510: if (copy_m == VM_PAGE_NULL) { ! 1511: RELEASE_PAGE(m); ! 1512: vm_fault_cleanup(object, first_m); ! 1513: return(VM_FAULT_MEMORY_SHORTAGE); ! 1514: } ! 1515: ! 1516: vm_object_unlock(object); ! 1517: ! 1518: XPR(XPR_VM_FAULT, ! 1519: "vm_f_page: page_copy obj 0x%X, offset 0x%X, m 0x%X, copy_m 0x%X\n", ! 1520: (integer_t)object, offset, ! 1521: (integer_t)m, (integer_t)copy_m, 0); ! 1522: vm_page_copy(m, copy_m); ! 1523: vm_object_lock(object); ! 1524: ! 1525: /* ! 1526: * If another map is truly sharing this ! 1527: * page with us, we have to flush all ! 1528: * uses of the original page, since we ! 1529: * can't distinguish those which want the ! 1530: * original from those which need the ! 1531: * new copy. ! 1532: * ! 1533: * XXXO If we know that only one map has ! 1534: * access to this page, then we could ! 1535: * avoid the pmap_page_protect() call. ! 1536: */ ! 1537: ! 1538: vm_page_lock_queues(); ! 1539: assert(!m->cleaning); ! 1540: pmap_page_protect(m->phys_addr, VM_PROT_NONE); ! 1541: vm_page_deactivate(m); ! 1542: copy_m->dirty = TRUE; ! 1543: /* ! 1544: * Setting reference here prevents this fault from ! 1545: * being counted as a (per-thread) reactivate as well ! 1546: * as a copy-on-write. ! 1547: */ ! 1548: first_m->reference = TRUE; ! 1549: vm_page_unlock_queues(); ! 1550: ! 1551: /* ! 1552: * We no longer need the old page or object. ! 1553: */ ! 1554: ! 1555: PAGE_WAKEUP_DONE(m); ! 1556: vm_object_paging_end(object); ! 1557: vm_object_unlock(object); ! 1558: ! 1559: if (type_of_fault) ! 1560: *type_of_fault = DBG_COW_FAULT; ! 1561: VM_STAT(cow_faults++); ! 1562: current_task()->cow_faults++; ! 1563: object = first_object; ! 1564: offset = first_offset; ! 1565: ! 1566: vm_object_lock(object); ! 1567: VM_PAGE_FREE(first_m); ! 1568: first_m = VM_PAGE_NULL; ! 1569: assert(copy_m->busy); ! 1570: vm_page_insert(copy_m, object, offset); ! 1571: m = copy_m; ! 1572: ! 1573: /* ! 1574: * Now that we've gotten the copy out of the ! 1575: * way, let's try to collapse the top object. ! 1576: * But we have to play ugly games with ! 1577: * paging_in_progress to do that... ! 1578: */ ! 1579: ! 1580: vm_object_paging_end(object); ! 1581: vm_object_collapse(object); ! 1582: vm_object_paging_begin(object); ! 1583: } ! 1584: else { ! 1585: *protection &= (~VM_PROT_WRITE); ! 1586: } ! 1587: } ! 1588: ! 1589: /* ! 1590: * Now check whether the page needs to be pushed into the ! 1591: * copy object. The use of asymmetric copy on write for ! 1592: * shared temporary objects means that we may do two copies to ! 1593: * satisfy the fault; one above to get the page from a ! 1594: * shadowed object, and one here to push it into the copy. ! 1595: */ ! 1596: ! 1597: while (first_object->copy_strategy == MEMORY_OBJECT_COPY_DELAY && ! 1598: (copy_object = first_object->copy) != VM_OBJECT_NULL) { ! 1599: vm_offset_t copy_offset; ! 1600: vm_page_t copy_m; ! 1601: ! 1602: #if TRACEFAULTPAGE ! 1603: dbgTrace(0xBEEF0017, (unsigned int) copy_object, (unsigned int) fault_type); /* (TEST/DEBUG) */ ! 1604: #endif ! 1605: /* ! 1606: * If the page is being written, but hasn't been ! 1607: * copied to the copy-object, we have to copy it there. ! 1608: */ ! 1609: ! 1610: if ((fault_type & VM_PROT_WRITE) == 0) { ! 1611: *protection &= ~VM_PROT_WRITE; ! 1612: break; ! 1613: } ! 1614: ! 1615: /* ! 1616: * If the page was guaranteed to be resident, ! 1617: * we must have already performed the copy. ! 1618: */ ! 1619: ! 1620: if (must_be_resident) ! 1621: break; ! 1622: ! 1623: /* ! 1624: * Try to get the lock on the copy_object. ! 1625: */ ! 1626: if (!vm_object_lock_try(copy_object)) { ! 1627: vm_object_unlock(object); ! 1628: ! 1629: mutex_pause(); /* wait a bit */ ! 1630: ! 1631: vm_object_lock(object); ! 1632: continue; ! 1633: } ! 1634: ! 1635: /* ! 1636: * Make another reference to the copy-object, ! 1637: * to keep it from disappearing during the ! 1638: * copy. ! 1639: */ ! 1640: assert(copy_object->ref_count > 0); ! 1641: copy_object->ref_count++; ! 1642: VM_OBJ_RES_INCR(copy_object); ! 1643: ! 1644: /* ! 1645: * Does the page exist in the copy? ! 1646: */ ! 1647: copy_offset = first_offset - copy_object->shadow_offset; ! 1648: if (copy_object->size <= copy_offset) ! 1649: /* ! 1650: * Copy object doesn't cover this page -- do nothing. ! 1651: */ ! 1652: ; ! 1653: else if ((copy_m = ! 1654: vm_page_lookup(copy_object, copy_offset)) != VM_PAGE_NULL) { ! 1655: /* Page currently exists in the copy object */ ! 1656: if (copy_m->busy) { ! 1657: /* ! 1658: * If the page is being brought ! 1659: * in, wait for it and then retry. ! 1660: */ ! 1661: RELEASE_PAGE(m); ! 1662: /* take an extra ref so object won't die */ ! 1663: assert(copy_object->ref_count > 0); ! 1664: copy_object->ref_count++; ! 1665: vm_object_res_reference(copy_object); ! 1666: vm_object_unlock(copy_object); ! 1667: vm_fault_cleanup(object, first_m); ! 1668: counter(c_vm_fault_page_block_backoff_kernel++); ! 1669: vm_object_lock(copy_object); ! 1670: assert(copy_object->ref_count > 0); ! 1671: VM_OBJ_RES_DECR(copy_object); ! 1672: copy_object->ref_count--; ! 1673: assert(copy_object->ref_count > 0); ! 1674: copy_m = vm_page_lookup(copy_object, copy_offset); ! 1675: if (copy_m != VM_PAGE_NULL && ! 1676: copy_m->busy) { ! 1677: PAGE_ASSERT_WAIT(copy_m, interruptible); ! 1678: vm_object_unlock(copy_object); ! 1679: thread_block((void (*)(void))0); ! 1680: vm_object_deallocate(copy_object); ! 1681: goto backoff; ! 1682: } else { ! 1683: vm_object_unlock(copy_object); ! 1684: vm_object_deallocate(copy_object); ! 1685: return VM_FAULT_RETRY; ! 1686: } ! 1687: } ! 1688: } ! 1689: else if (!PAGED_OUT(copy_object, copy_offset)) { ! 1690: /* ! 1691: * If PAGED_OUT is TRUE, then the page used to exist ! 1692: * in the copy-object, and has already been paged out. ! 1693: * We don't need to repeat this. If PAGED_OUT is ! 1694: * FALSE, then either we don't know (!pager_created, ! 1695: * for example) or it hasn't been paged out. ! 1696: * (VM_EXTERNAL_STATE_UNKNOWN||VM_EXTERNAL_STATE_ABSENT) ! 1697: * We must copy the page to the copy object. ! 1698: */ ! 1699: ! 1700: /* ! 1701: * Allocate a page for the copy ! 1702: */ ! 1703: copy_m = vm_page_alloc(copy_object, copy_offset); ! 1704: if (copy_m == VM_PAGE_NULL) { ! 1705: RELEASE_PAGE(m); ! 1706: VM_OBJ_RES_DECR(copy_object); ! 1707: copy_object->ref_count--; ! 1708: assert(copy_object->ref_count > 0); ! 1709: vm_object_unlock(copy_object); ! 1710: vm_fault_cleanup(object, first_m); ! 1711: return(VM_FAULT_MEMORY_SHORTAGE); ! 1712: } ! 1713: ! 1714: /* ! 1715: * Must copy page into copy-object. ! 1716: */ ! 1717: ! 1718: vm_page_copy(m, copy_m); ! 1719: ! 1720: /* ! 1721: * If the old page was in use by any users ! 1722: * of the copy-object, it must be removed ! 1723: * from all pmaps. (We can't know which ! 1724: * pmaps use it.) ! 1725: */ ! 1726: ! 1727: vm_page_lock_queues(); ! 1728: assert(!m->cleaning); ! 1729: pmap_page_protect(m->phys_addr, VM_PROT_NONE); ! 1730: copy_m->dirty = TRUE; ! 1731: vm_page_unlock_queues(); ! 1732: ! 1733: /* ! 1734: * If there's a pager, then immediately ! 1735: * page out this page, using the "initialize" ! 1736: * option. Else, we use the copy. ! 1737: */ ! 1738: ! 1739: if ! 1740: #if MACH_PAGEMAP ! 1741: ((!copy_object->pager_created) || ! 1742: vm_external_state_get( ! 1743: copy_object->existence_map, copy_offset) ! 1744: == VM_EXTERNAL_STATE_ABSENT) ! 1745: #else ! 1746: (!copy_object->pager_created) ! 1747: #endif ! 1748: { ! 1749: vm_page_lock_queues(); ! 1750: vm_page_activate(copy_m); ! 1751: vm_page_unlock_queues(); ! 1752: PAGE_WAKEUP_DONE(copy_m); ! 1753: } ! 1754: else { ! 1755: assert(copy_m->busy == TRUE); ! 1756: ! 1757: /* ! 1758: * The page is already ready for pageout: ! 1759: * not on pageout queues and busy. ! 1760: * Unlock everything except the ! 1761: * copy_object itself. ! 1762: */ ! 1763: ! 1764: vm_object_unlock(object); ! 1765: ! 1766: /* ! 1767: * Write the page to the copy-object, ! 1768: * flushing it from the kernel. ! 1769: */ ! 1770: ! 1771: vm_pageout_initialize_page(copy_m); ! 1772: ! 1773: /* ! 1774: * Since the pageout may have ! 1775: * temporarily dropped the ! 1776: * copy_object's lock, we ! 1777: * check whether we'll have ! 1778: * to deallocate the hard way. ! 1779: */ ! 1780: ! 1781: if ((copy_object->shadow != object) || ! 1782: (copy_object->ref_count == 1)) { ! 1783: vm_object_unlock(copy_object); ! 1784: vm_object_deallocate(copy_object); ! 1785: vm_object_lock(object); ! 1786: continue; ! 1787: } ! 1788: ! 1789: /* ! 1790: * Pick back up the old object's ! 1791: * lock. [It is safe to do so, ! 1792: * since it must be deeper in the ! 1793: * object tree.] ! 1794: */ ! 1795: ! 1796: vm_object_lock(object); ! 1797: } ! 1798: ! 1799: /* ! 1800: * Because we're pushing a page upward ! 1801: * in the object tree, we must restart ! 1802: * any faults that are waiting here. ! 1803: * [Note that this is an expansion of ! 1804: * PAGE_WAKEUP that uses the THREAD_RESTART ! 1805: * wait result]. Can't turn off the page's ! 1806: * busy bit because we're not done with it. ! 1807: */ ! 1808: ! 1809: if (m->wanted) { ! 1810: m->wanted = FALSE; ! 1811: thread_wakeup_with_result((event_t) m, ! 1812: THREAD_RESTART); ! 1813: } ! 1814: } ! 1815: ! 1816: /* ! 1817: * The reference count on copy_object must be ! 1818: * at least 2: one for our extra reference, ! 1819: * and at least one from the outside world ! 1820: * (we checked that when we last locked ! 1821: * copy_object). ! 1822: */ ! 1823: copy_object->ref_count--; ! 1824: assert(copy_object->ref_count > 0); ! 1825: VM_OBJ_RES_DECR(copy_object); ! 1826: vm_object_unlock(copy_object); ! 1827: ! 1828: break; ! 1829: } ! 1830: ! 1831: *result_page = m; ! 1832: *top_page = first_m; ! 1833: ! 1834: XPR(XPR_VM_FAULT, ! 1835: "vm_f_page: DONE obj 0x%X, offset 0x%X, m 0x%X, first_m 0x%X\n", ! 1836: (integer_t)object, offset, (integer_t)m, (integer_t)first_m, 0); ! 1837: /* ! 1838: * If the page can be written, assume that it will be. ! 1839: * [Earlier, we restrict the permission to allow write ! 1840: * access only if the fault so required, so we don't ! 1841: * mark read-only data as dirty.] ! 1842: */ ! 1843: ! 1844: #if !VM_FAULT_STATIC_CONFIG ! 1845: if (vm_fault_dirty_handling && (*protection & VM_PROT_WRITE)) ! 1846: m->dirty = TRUE; ! 1847: #endif ! 1848: #if TRACEFAULTPAGE ! 1849: dbgTrace(0xBEEF0018, (unsigned int) object, (unsigned int) vm_page_deactivate_behind); /* (TEST/DEBUG) */ ! 1850: #endif ! 1851: if (vm_page_deactivate_behind) { ! 1852: if (offset && /* don't underflow */ ! 1853: (object->last_alloc == (offset - PAGE_SIZE))) { ! 1854: m = vm_page_lookup(object, object->last_alloc); ! 1855: if ((m != VM_PAGE_NULL) && !m->busy) { ! 1856: vm_page_lock_queues(); ! 1857: vm_page_deactivate(m); ! 1858: vm_page_unlock_queues(); ! 1859: } ! 1860: #if TRACEFAULTPAGE ! 1861: dbgTrace(0xBEEF0019, (unsigned int) object, (unsigned int) m); /* (TEST/DEBUG) */ ! 1862: #endif ! 1863: } ! 1864: object->last_alloc = offset; ! 1865: } ! 1866: #if TRACEFAULTPAGE ! 1867: dbgTrace(0xBEEF001A, (unsigned int) VM_FAULT_SUCCESS, 0); /* (TEST/DEBUG) */ ! 1868: #endif ! 1869: return(VM_FAULT_SUCCESS); ! 1870: ! 1871: #if 0 ! 1872: block_and_backoff: ! 1873: vm_fault_cleanup(object, first_m); ! 1874: ! 1875: counter(c_vm_fault_page_block_backoff_kernel++); ! 1876: thread_block((void (*)(void))0); ! 1877: #endif ! 1878: ! 1879: backoff: ! 1880: if (thread->wait_result == THREAD_AWAKENED) ! 1881: { ! 1882: return VM_FAULT_RETRY; ! 1883: } ! 1884: else ! 1885: { ! 1886: return VM_FAULT_INTERRUPTED; ! 1887: } ! 1888: ! 1889: #undef RELEASE_PAGE ! 1890: } ! 1891: ! 1892: /* ! 1893: * Routine: vm_fault ! 1894: * Purpose: ! 1895: * Handle page faults, including pseudo-faults ! 1896: * used to change the wiring status of pages. ! 1897: * Returns: ! 1898: * Explicit continuations have been removed. ! 1899: * Implementation: ! 1900: * vm_fault and vm_fault_page save mucho state ! 1901: * in the moral equivalent of a closure. The state ! 1902: * structure is allocated when first entering vm_fault ! 1903: * and deallocated when leaving vm_fault. ! 1904: */ ! 1905: ! 1906: kern_return_t ! 1907: vm_fault( ! 1908: vm_map_t map, ! 1909: vm_offset_t vaddr, ! 1910: vm_prot_t fault_type, ! 1911: boolean_t change_wiring) ! 1912: { ! 1913: vm_map_version_t version; /* Map version for verificiation */ ! 1914: boolean_t wired; /* Should mapping be wired down? */ ! 1915: vm_object_t object; /* Top-level object */ ! 1916: vm_offset_t offset; /* Top-level offset */ ! 1917: vm_prot_t prot; /* Protection for mapping */ ! 1918: vm_behavior_t behavior; /* Expected paging behavior */ ! 1919: vm_offset_t lo_offset, hi_offset; ! 1920: vm_object_t old_copy_object; /* Saved copy object */ ! 1921: vm_page_t result_page; /* Result of vm_fault_page */ ! 1922: vm_page_t top_page; /* Placeholder page */ ! 1923: kern_return_t kr; ! 1924: ! 1925: register ! 1926: vm_page_t m; /* Fast access to result_page */ ! 1927: kern_return_t error_code; /* page error reasons */ ! 1928: register ! 1929: vm_object_t cur_object; ! 1930: register ! 1931: vm_offset_t cur_offset; ! 1932: vm_page_t cur_m; ! 1933: vm_object_t new_object; ! 1934: int type_of_fault; ! 1935: ! 1936: ! 1937: KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 0)) | DBG_FUNC_START, ! 1938: vaddr, ! 1939: 0, ! 1940: 0, ! 1941: 0, ! 1942: 0); ! 1943: /* ! 1944: * assume we will hit a page in the cache ! 1945: * otherwise, explicitly override with ! 1946: * the real fault type once we determine it ! 1947: */ ! 1948: type_of_fault = DBG_CACHE_HIT_FAULT; ! 1949: ! 1950: VM_STAT(faults++); ! 1951: current_task()->faults++; ! 1952: ! 1953: RetryFault: ; ! 1954: ! 1955: /* ! 1956: * Find the backing store object and offset into ! 1957: * it to begin the search. ! 1958: */ ! 1959: vm_map_lock_read(map); ! 1960: kr = vm_map_lookup_locked(&map, vaddr, fault_type, &version, ! 1961: &object, &offset, ! 1962: &prot, &wired, ! 1963: &behavior, &lo_offset, &hi_offset); ! 1964: ! 1965: if (kr != KERN_SUCCESS) { ! 1966: vm_map_unlock_read(map); ! 1967: goto done; ! 1968: } ! 1969: ! 1970: /* ! 1971: * If the page is wired, we must fault for the current protection ! 1972: * value, to avoid further faults. ! 1973: */ ! 1974: ! 1975: if (wired) ! 1976: fault_type = prot | VM_PROT_WRITE; ! 1977: ! 1978: #if VM_FAULT_CLASSIFY ! 1979: /* ! 1980: * Temporary data gathering code ! 1981: */ ! 1982: vm_fault_classify(object, offset, fault_type); ! 1983: #endif ! 1984: /* ! 1985: * Fast fault code. The basic idea is to do as much as ! 1986: * possible while holding the map lock and object locks. ! 1987: * Busy pages are not used until the object lock has to ! 1988: * be dropped to do something (copy, zero fill, pmap enter). ! 1989: * Similarly, paging references aren't acquired until that ! 1990: * point, and object references aren't used. ! 1991: * ! 1992: * If we can figure out what to do ! 1993: * (zero fill, copy on write, pmap enter) while holding ! 1994: * the locks, then it gets done. Otherwise, we give up, ! 1995: * and use the original fault path (which doesn't hold ! 1996: * the map lock, and relies on busy pages). ! 1997: * The give up cases include: ! 1998: * - Have to talk to pager. ! 1999: * - Page is busy, absent or in error. ! 2000: * - Pager has locked out desired access. ! 2001: * - Fault needs to be restarted. ! 2002: * - Have to push page into copy object. ! 2003: * ! 2004: * The code is an infinite loop that moves one level down ! 2005: * the shadow chain each time. cur_object and cur_offset ! 2006: * refer to the current object being examined. object and offset ! 2007: * are the original object from the map. The loop is at the ! 2008: * top level if and only if object and cur_object are the same. ! 2009: * ! 2010: * Invariants: Map lock is held throughout. Lock is held on ! 2011: * original object and cur_object (if different) when ! 2012: * continuing or exiting loop. ! 2013: * ! 2014: */ ! 2015: ! 2016: ! 2017: /* ! 2018: * If this page is to be inserted in a copy delay object ! 2019: * for writing, and if the object has a copy, then the ! 2020: * copy delay strategy is implemented in the slow fault page. ! 2021: */ ! 2022: if (object->copy_strategy != MEMORY_OBJECT_COPY_DELAY || ! 2023: object->copy == VM_OBJECT_NULL || ! 2024: (fault_type & VM_PROT_WRITE) == 0) { ! 2025: cur_object = object; ! 2026: cur_offset = offset; ! 2027: ! 2028: while (TRUE) { ! 2029: m = vm_page_lookup(cur_object, cur_offset); ! 2030: if (m != VM_PAGE_NULL) { ! 2031: if (m->busy) ! 2032: break; ! 2033: ! 2034: if (m->unusual && (m->error || m->restart || ! 2035: m->absent || (fault_type & m->page_lock))) { ! 2036: ! 2037: /* ! 2038: * Unusual case. Give up. ! 2039: */ ! 2040: break; ! 2041: } ! 2042: ! 2043: /* ! 2044: * Two cases of map in faults: ! 2045: * - At top level w/o copy object. ! 2046: * - Read fault anywhere. ! 2047: * --> must disallow write. ! 2048: */ ! 2049: ! 2050: if (object == cur_object && ! 2051: object->copy == VM_OBJECT_NULL) ! 2052: goto FastMapInFault; ! 2053: ! 2054: if ((fault_type & VM_PROT_WRITE) == 0) { ! 2055: ! 2056: prot &= ~VM_PROT_WRITE; ! 2057: ! 2058: /* ! 2059: * Set up to map the page ... ! 2060: * mark the page busy, drop ! 2061: * locks and take a paging reference ! 2062: * on the object with the page. ! 2063: */ ! 2064: ! 2065: if (object != cur_object) { ! 2066: vm_object_unlock(object); ! 2067: object = cur_object; ! 2068: } ! 2069: FastMapInFault: ! 2070: m->busy = TRUE; ! 2071: ! 2072: vm_object_paging_begin(object); ! 2073: vm_object_unlock(object); ! 2074: ! 2075: FastPmapEnter: ! 2076: /* ! 2077: * Check a couple of global reasons to ! 2078: * be conservative about write access. ! 2079: * Then do the pmap_enter. ! 2080: */ ! 2081: #if !VM_FAULT_STATIC_CONFIG ! 2082: if (vm_fault_dirty_handling ! 2083: #if MACH_KDB ! 2084: || db_watchpoint_list ! 2085: #endif ! 2086: && (fault_type & VM_PROT_WRITE) == 0) ! 2087: prot &= ~VM_PROT_WRITE; ! 2088: #else /* STATIC_CONFIG */ ! 2089: #if MACH_KDB ! 2090: if (db_watchpoint_list ! 2091: && (fault_type & VM_PROT_WRITE) == 0) ! 2092: prot &= ~VM_PROT_WRITE; ! 2093: #endif /* MACH_KDB */ ! 2094: #endif /* STATIC_CONFIG */ ! 2095: PMAP_ENTER(vm_map_pmap(map), vaddr, m, ! 2096: prot, wired); ! 2097: ! 2098: if (m->clustered) { ! 2099: vm_pagein_cluster_used++; ! 2100: m->clustered = FALSE; ! 2101: ! 2102: pmap_attribute(vm_map_pmap(map), ! 2103: vaddr, ! 2104: PAGE_SIZE, ! 2105: MATTR_CACHE, ! 2106: &mv_cache_sync); ! 2107: } ! 2108: /* ! 2109: * Grab the object lock to manipulate ! 2110: * the page queues. Change wiring ! 2111: * case is obvious. In soft ref bits ! 2112: * case activate page only if it fell ! 2113: * off paging queues, otherwise just ! 2114: * activate it if it's inactive. ! 2115: * ! 2116: * NOTE: original vm_fault code will ! 2117: * move active page to back of active ! 2118: * queue. This code doesn't. ! 2119: */ ! 2120: vm_object_lock(object); ! 2121: vm_page_lock_queues(); ! 2122: ! 2123: m->reference = TRUE; ! 2124: ! 2125: if (change_wiring) { ! 2126: if (wired) ! 2127: vm_page_wire(m); ! 2128: else ! 2129: vm_page_unwire(m); ! 2130: } ! 2131: #if VM_FAULT_STATIC_CONFIG ! 2132: else { ! 2133: if (!m->active && !m->inactive) ! 2134: vm_page_activate(m); ! 2135: } ! 2136: #else ! 2137: else if (software_reference_bits) { ! 2138: if (!m->active && !m->inactive) ! 2139: vm_page_activate(m); ! 2140: } ! 2141: else if (!m->active) { ! 2142: vm_page_activate(m); ! 2143: } ! 2144: #endif ! 2145: vm_page_unlock_queues(); ! 2146: ! 2147: /* ! 2148: * That's it, clean up and return. ! 2149: */ ! 2150: PAGE_WAKEUP_DONE(m); ! 2151: vm_object_paging_end(object); ! 2152: vm_object_unlock(object); ! 2153: vm_map_unlock_read(map); ! 2154: ! 2155: KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 0)) | DBG_FUNC_END, ! 2156: vaddr, ! 2157: type_of_fault, ! 2158: KERN_SUCCESS, ! 2159: 0, ! 2160: 0); ! 2161: return KERN_SUCCESS; ! 2162: } ! 2163: ! 2164: /* ! 2165: * Copy on write fault. If objects match, then ! 2166: * object->copy must not be NULL (else control ! 2167: * would be in previous code block), and we ! 2168: * have a potential push into the copy object ! 2169: * with which we won't cope here. ! 2170: */ ! 2171: ! 2172: if (cur_object == object) ! 2173: break; ! 2174: ! 2175: /* ! 2176: * This is now a shadow based copy on write ! 2177: * fault -- it requires a copy up the shadow ! 2178: * chain. ! 2179: * ! 2180: * Allocate a page in the original top level ! 2181: * object. Give up if allocate fails. Also ! 2182: * need to remember current page, as it's the ! 2183: * source of the copy. ! 2184: */ ! 2185: cur_m = m; ! 2186: m = vm_page_alloc(object, offset); ! 2187: if (m == VM_PAGE_NULL) { ! 2188: break; ! 2189: } ! 2190: ! 2191: /* ! 2192: * Now do the copy. Mark the source busy ! 2193: * and take out paging references on both ! 2194: * objects. ! 2195: * ! 2196: * NOTE: This code holds the map lock across ! 2197: * the page copy. ! 2198: */ ! 2199: ! 2200: cur_m->busy = TRUE; ! 2201: ! 2202: vm_object_paging_begin(cur_object); ! 2203: vm_object_unlock(cur_object); ! 2204: vm_object_paging_begin(object); ! 2205: vm_object_unlock(object); ! 2206: ! 2207: vm_page_copy(cur_m, m); ! 2208: type_of_fault = DBG_COW_FAULT; ! 2209: VM_STAT(cow_faults++); ! 2210: current_task()->cow_faults++; ! 2211: ! 2212: /* ! 2213: * Now cope with the source page and object ! 2214: * If the top object has a ref count of 1 ! 2215: * then no other map can access it, and hence ! 2216: * it's not necessary to do the pmap_page_protect. ! 2217: */ ! 2218: ! 2219: vm_object_lock(object); ! 2220: vm_object_lock(cur_object); ! 2221: ! 2222: vm_page_lock_queues(); ! 2223: vm_page_deactivate(cur_m); ! 2224: m->dirty = TRUE; ! 2225: if (object->ref_count != 1) ! 2226: pmap_page_protect(cur_m->phys_addr, ! 2227: VM_PROT_NONE); ! 2228: vm_page_unlock_queues(); ! 2229: ! 2230: PAGE_WAKEUP_DONE(cur_m); ! 2231: vm_object_paging_end(cur_object); ! 2232: vm_object_unlock(cur_object); ! 2233: ! 2234: /* ! 2235: * Slight hack to call vm_object collapse ! 2236: * and then reuse common map in code. ! 2237: * note that the object lock was taken above. ! 2238: */ ! 2239: ! 2240: vm_object_paging_end(object); ! 2241: vm_object_collapse(object); ! 2242: vm_object_paging_begin(object); ! 2243: vm_object_unlock(object); ! 2244: ! 2245: goto FastPmapEnter; ! 2246: } ! 2247: else { ! 2248: ! 2249: /* ! 2250: * No page at cur_object, cur_offset ! 2251: */ ! 2252: ! 2253: if (cur_object->pager_created) { ! 2254: ! 2255: /* ! 2256: * Have to talk to the pager. Give up. ! 2257: */ ! 2258: ! 2259: break; ! 2260: } ! 2261: ! 2262: ! 2263: if (cur_object->shadow == VM_OBJECT_NULL) { ! 2264: ! 2265: /* ! 2266: * Zero fill fault. Page gets ! 2267: * filled in top object. Insert ! 2268: * page, then drop any lower lock. ! 2269: * Give up if no page. ! 2270: */ ! 2271: if ((vm_page_free_target - ! 2272: ((vm_page_free_target-vm_page_free_min)>>2)) ! 2273: > vm_page_free_count) { ! 2274: break; ! 2275: } ! 2276: m = vm_page_alloc(object, offset); ! 2277: if (m == VM_PAGE_NULL) { ! 2278: break; ! 2279: } ! 2280: ! 2281: if (cur_object != object) ! 2282: vm_object_unlock(cur_object); ! 2283: ! 2284: vm_object_paging_begin(object); ! 2285: vm_object_unlock(object); ! 2286: ! 2287: /* ! 2288: * Now zero fill page and map it. ! 2289: * the page is probably going to ! 2290: * be written soon, so don't bother ! 2291: * to clear the modified bit ! 2292: * ! 2293: * NOTE: This code holds the map ! 2294: * lock across the zero fill. ! 2295: */ ! 2296: ! 2297: if (!map->no_zero_fill) { ! 2298: vm_page_zero_fill(m); ! 2299: type_of_fault = DBG_ZERO_FILL_FAULT; ! 2300: VM_STAT(zero_fill_count++); ! 2301: } ! 2302: vm_page_lock_queues(); ! 2303: VM_PAGE_QUEUES_REMOVE(m); ! 2304: queue_enter(&vm_page_queue_inactive, ! 2305: m, vm_page_t, pageq); ! 2306: m->inactive = TRUE; ! 2307: vm_page_inactive_count++; ! 2308: vm_page_unlock_queues(); ! 2309: goto FastPmapEnter; ! 2310: } ! 2311: ! 2312: /* ! 2313: * On to the next level ! 2314: */ ! 2315: ! 2316: cur_offset += cur_object->shadow_offset; ! 2317: new_object = cur_object->shadow; ! 2318: vm_object_lock(new_object); ! 2319: if (cur_object != object) ! 2320: vm_object_unlock(cur_object); ! 2321: cur_object = new_object; ! 2322: ! 2323: continue; ! 2324: } ! 2325: } ! 2326: ! 2327: /* ! 2328: * Cleanup from fast fault failure. Drop any object ! 2329: * lock other than original and drop map lock. ! 2330: */ ! 2331: ! 2332: if (object != cur_object) ! 2333: vm_object_unlock(cur_object); ! 2334: } ! 2335: vm_map_unlock_read(map); ! 2336: ! 2337: /* ! 2338: * Make a reference to this object to ! 2339: * prevent its disposal while we are messing with ! 2340: * it. Once we have the reference, the map is free ! 2341: * to be diddled. Since objects reference their ! 2342: * shadows (and copies), they will stay around as well. ! 2343: */ ! 2344: ! 2345: assert(object->ref_count > 0); ! 2346: object->ref_count++; ! 2347: vm_object_res_reference(object); ! 2348: vm_object_paging_begin(object); ! 2349: ! 2350: XPR(XPR_VM_FAULT,"vm_fault -> vm_fault_page\n",0,0,0,0,0); ! 2351: kr = vm_fault_page(object, offset, fault_type, ! 2352: (change_wiring && !wired), ! 2353: ((!change_wiring) ? THREAD_ABORTSAFE : THREAD_UNINT), ! 2354: lo_offset, hi_offset, behavior, ! 2355: &prot, &result_page, &top_page, ! 2356: &type_of_fault, ! 2357: &error_code, map->no_zero_fill, FALSE); ! 2358: ! 2359: /* ! 2360: * If we didn't succeed, lose the object reference immediately. ! 2361: */ ! 2362: ! 2363: if (kr != VM_FAULT_SUCCESS) ! 2364: vm_object_deallocate(object); ! 2365: ! 2366: /* ! 2367: * See why we failed, and take corrective action. ! 2368: */ ! 2369: ! 2370: switch (kr) { ! 2371: case VM_FAULT_SUCCESS: ! 2372: break; ! 2373: case VM_FAULT_RETRY: ! 2374: goto RetryFault; ! 2375: case VM_FAULT_INTERRUPTED: ! 2376: kr = KERN_SUCCESS; ! 2377: goto done; ! 2378: case VM_FAULT_MEMORY_SHORTAGE: ! 2379: VM_PAGE_WAIT(); ! 2380: goto RetryFault; ! 2381: case VM_FAULT_FICTITIOUS_SHORTAGE: ! 2382: vm_page_more_fictitious(); ! 2383: goto RetryFault; ! 2384: case VM_FAULT_MEMORY_ERROR: ! 2385: if (error_code) ! 2386: kr = error_code; ! 2387: else ! 2388: kr = KERN_MEMORY_ERROR; ! 2389: goto done; ! 2390: } ! 2391: ! 2392: m = result_page; ! 2393: ! 2394: assert((change_wiring && !wired) ? ! 2395: (top_page == VM_PAGE_NULL) : ! 2396: ((top_page == VM_PAGE_NULL) == (m->object == object))); ! 2397: ! 2398: /* ! 2399: * How to clean up the result of vm_fault_page. This ! 2400: * happens whether the mapping is entered or not. ! 2401: */ ! 2402: ! 2403: #define UNLOCK_AND_DEALLOCATE \ ! 2404: MACRO_BEGIN \ ! 2405: vm_fault_cleanup(m->object, top_page); \ ! 2406: vm_object_deallocate(object); \ ! 2407: MACRO_END ! 2408: ! 2409: /* ! 2410: * What to do with the resulting page from vm_fault_page ! 2411: * if it doesn't get entered into the physical map: ! 2412: */ ! 2413: ! 2414: #define RELEASE_PAGE(m) \ ! 2415: MACRO_BEGIN \ ! 2416: PAGE_WAKEUP_DONE(m); \ ! 2417: vm_page_lock_queues(); \ ! 2418: if (!m->active && !m->inactive) \ ! 2419: vm_page_activate(m); \ ! 2420: vm_page_unlock_queues(); \ ! 2421: MACRO_END ! 2422: ! 2423: /* ! 2424: * We must verify that the maps have not changed ! 2425: * since our last lookup. ! 2426: */ ! 2427: ! 2428: old_copy_object = m->object->copy; ! 2429: ! 2430: vm_object_unlock(m->object); ! 2431: while (!vm_map_verify(map, &version)) { ! 2432: vm_object_t retry_object; ! 2433: vm_offset_t retry_offset; ! 2434: vm_prot_t retry_prot; ! 2435: ! 2436: /* ! 2437: * To avoid trying to write_lock the map while another ! 2438: * thread has it read_locked (in vm_map_pageable), we ! 2439: * do not try for write permission. If the page is ! 2440: * still writable, we will get write permission. If it ! 2441: * is not, or has been marked needs_copy, we enter the ! 2442: * mapping without write permission, and will merely ! 2443: * take another fault. ! 2444: */ ! 2445: vm_map_lock_read(map); ! 2446: kr = vm_map_lookup_locked(&map, vaddr, ! 2447: fault_type & ~VM_PROT_WRITE, &version, ! 2448: &retry_object, &retry_offset, &retry_prot, ! 2449: &wired, &behavior, &lo_offset, &hi_offset); ! 2450: vm_map_unlock_read(map); ! 2451: ! 2452: if (kr != KERN_SUCCESS) { ! 2453: vm_object_lock(m->object); ! 2454: RELEASE_PAGE(m); ! 2455: UNLOCK_AND_DEALLOCATE; ! 2456: goto done; ! 2457: } ! 2458: ! 2459: vm_object_unlock(retry_object); ! 2460: vm_object_lock(m->object); ! 2461: ! 2462: if ((retry_object != object) || ! 2463: (retry_offset != offset)) { ! 2464: RELEASE_PAGE(m); ! 2465: UNLOCK_AND_DEALLOCATE; ! 2466: goto RetryFault; ! 2467: } ! 2468: ! 2469: /* ! 2470: * Check whether the protection has changed or the object ! 2471: * has been copied while we left the map unlocked. ! 2472: */ ! 2473: prot &= retry_prot; ! 2474: vm_object_unlock(m->object); ! 2475: } ! 2476: vm_object_lock(m->object); ! 2477: ! 2478: /* ! 2479: * If the copy object changed while the top-level object ! 2480: * was unlocked, then we must take away write permission. ! 2481: */ ! 2482: ! 2483: if (m->object->copy != old_copy_object) ! 2484: prot &= ~VM_PROT_WRITE; ! 2485: ! 2486: /* ! 2487: * If we want to wire down this page, but no longer have ! 2488: * adequate permissions, we must start all over. ! 2489: */ ! 2490: ! 2491: if (wired && (fault_type != (prot|VM_PROT_WRITE))) { ! 2492: vm_map_verify_done(map, &version); ! 2493: RELEASE_PAGE(m); ! 2494: UNLOCK_AND_DEALLOCATE; ! 2495: goto RetryFault; ! 2496: } ! 2497: ! 2498: /* ! 2499: * It's critically important that a wired-down page be faulted ! 2500: * only once in each map for which it is wired. ! 2501: */ ! 2502: vm_object_unlock(m->object); ! 2503: ! 2504: /* ! 2505: * Put this page into the physical map. ! 2506: * We had to do the unlock above because pmap_enter ! 2507: * may cause other faults. The page may be on ! 2508: * the pageout queues. If the pageout daemon comes ! 2509: * across the page, it will remove it from the queues. ! 2510: */ ! 2511: PMAP_ENTER(map->pmap, vaddr, m, prot, wired); ! 2512: ! 2513: /* Sync I & D caches for new mapping*/ ! 2514: pmap_attribute(map->pmap, ! 2515: vaddr, ! 2516: PAGE_SIZE, ! 2517: MATTR_CACHE, ! 2518: &mv_cache_sync); ! 2519: ! 2520: /* ! 2521: * If the page is not wired down and isn't already ! 2522: * on a pageout queue, then put it where the ! 2523: * pageout daemon can find it. ! 2524: */ ! 2525: vm_object_lock(m->object); ! 2526: vm_page_lock_queues(); ! 2527: if (change_wiring) { ! 2528: if (wired) ! 2529: vm_page_wire(m); ! 2530: else ! 2531: vm_page_unwire(m); ! 2532: } ! 2533: #if VM_FAULT_STATIC_CONFIG ! 2534: else { ! 2535: if (!m->active && !m->inactive) ! 2536: vm_page_activate(m); ! 2537: m->reference = TRUE; ! 2538: } ! 2539: #else ! 2540: else if (software_reference_bits) { ! 2541: if (!m->active && !m->inactive) ! 2542: vm_page_activate(m); ! 2543: m->reference = TRUE; ! 2544: } else { ! 2545: vm_page_activate(m); ! 2546: } ! 2547: #endif ! 2548: vm_page_unlock_queues(); ! 2549: ! 2550: /* ! 2551: * Unlock everything, and return ! 2552: */ ! 2553: ! 2554: vm_map_verify_done(map, &version); ! 2555: PAGE_WAKEUP_DONE(m); ! 2556: kr = KERN_SUCCESS; ! 2557: UNLOCK_AND_DEALLOCATE; ! 2558: ! 2559: #undef UNLOCK_AND_DEALLOCATE ! 2560: #undef RELEASE_PAGE ! 2561: ! 2562: done: ! 2563: KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 0)) | DBG_FUNC_END, ! 2564: vaddr, ! 2565: type_of_fault, ! 2566: kr, ! 2567: 0, ! 2568: 0); ! 2569: return(kr); ! 2570: } ! 2571: ! 2572: /* ! 2573: * vm_fault_wire: ! 2574: * ! 2575: * Wire down a range of virtual addresses in a map. ! 2576: */ ! 2577: kern_return_t ! 2578: vm_fault_wire( ! 2579: vm_map_t map, ! 2580: vm_map_entry_t entry) ! 2581: { ! 2582: ! 2583: register vm_offset_t va; ! 2584: register pmap_t pmap; ! 2585: register vm_offset_t end_addr = entry->vme_end; ! 2586: register kern_return_t rc; ! 2587: ! 2588: assert(entry->in_transition); ! 2589: pmap = vm_map_pmap(map); ! 2590: ! 2591: /* ! 2592: * Inform the physical mapping system that the ! 2593: * range of addresses may not fault, so that ! 2594: * page tables and such can be locked down as well. ! 2595: */ ! 2596: ! 2597: pmap_pageable(pmap, entry->vme_start, end_addr, FALSE); ! 2598: ! 2599: /* ! 2600: * We simulate a fault to get the page and enter it ! 2601: * in the physical map. ! 2602: */ ! 2603: ! 2604: for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) { ! 2605: if ((rc = vm_fault_wire_fast(map, va, entry)) != KERN_SUCCESS) { ! 2606: rc = vm_fault(map, va, VM_PROT_NONE, TRUE); ! 2607: } ! 2608: ! 2609: if (rc != KERN_SUCCESS) { ! 2610: struct vm_map_entry tmp_entry = *entry; ! 2611: ! 2612: /* unwire wired pages */ ! 2613: tmp_entry.vme_end = va; ! 2614: vm_fault_unwire(map, &tmp_entry, FALSE); ! 2615: ! 2616: return rc; ! 2617: } ! 2618: } ! 2619: return KERN_SUCCESS; ! 2620: } ! 2621: ! 2622: /* ! 2623: * vm_fault_unwire: ! 2624: * ! 2625: * Unwire a range of virtual addresses in a map. ! 2626: */ ! 2627: void ! 2628: vm_fault_unwire( ! 2629: vm_map_t map, ! 2630: vm_map_entry_t entry, ! 2631: boolean_t deallocate) ! 2632: { ! 2633: register vm_offset_t va; ! 2634: register pmap_t pmap; ! 2635: register vm_offset_t end_addr = entry->vme_end; ! 2636: vm_object_t object; ! 2637: ! 2638: pmap = vm_map_pmap(map); ! 2639: ! 2640: object = (entry->is_sub_map) ! 2641: ? VM_OBJECT_NULL : entry->object.vm_object; ! 2642: ! 2643: /* ! 2644: * Since the pages are wired down, we must be able to ! 2645: * get their mappings from the physical map system. ! 2646: */ ! 2647: ! 2648: for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) { ! 2649: pmap_change_wiring(pmap, va, FALSE); ! 2650: ! 2651: if (object == VM_OBJECT_NULL) { ! 2652: (void) vm_fault(map, va, VM_PROT_NONE, TRUE); ! 2653: } else { ! 2654: vm_prot_t prot; ! 2655: vm_page_t result_page; ! 2656: vm_page_t top_page; ! 2657: vm_object_t result_object; ! 2658: vm_fault_return_t result; ! 2659: ! 2660: do { ! 2661: prot = VM_PROT_NONE; ! 2662: ! 2663: vm_object_lock(object); ! 2664: vm_object_paging_begin(object); ! 2665: XPR(XPR_VM_FAULT, ! 2666: "vm_fault_unwire -> vm_fault_page\n", ! 2667: 0,0,0,0,0); ! 2668: result = vm_fault_page(object, ! 2669: entry->offset + ! 2670: (va - entry->vme_start), ! 2671: VM_PROT_NONE, TRUE, ! 2672: THREAD_UNINT, ! 2673: entry->offset, ! 2674: entry->offset + ! 2675: (entry->vme_end ! 2676: - entry->vme_start), ! 2677: entry->behavior, ! 2678: &prot, ! 2679: &result_page, ! 2680: &top_page, ! 2681: (int *)0, ! 2682: 0, map->no_zero_fill, ! 2683: FALSE); ! 2684: } while (result == VM_FAULT_RETRY); ! 2685: ! 2686: if (result != VM_FAULT_SUCCESS) ! 2687: panic("vm_fault_unwire: failure"); ! 2688: ! 2689: result_object = result_page->object; ! 2690: if (deallocate) { ! 2691: assert(!result_page->fictitious); ! 2692: pmap_page_protect(result_page->phys_addr, ! 2693: VM_PROT_NONE); ! 2694: VM_PAGE_FREE(result_page); ! 2695: } else { ! 2696: vm_page_lock_queues(); ! 2697: vm_page_unwire(result_page); ! 2698: vm_page_unlock_queues(); ! 2699: PAGE_WAKEUP_DONE(result_page); ! 2700: } ! 2701: ! 2702: vm_fault_cleanup(result_object, top_page); ! 2703: } ! 2704: } ! 2705: ! 2706: /* ! 2707: * Inform the physical mapping system that the range ! 2708: * of addresses may fault, so that page tables and ! 2709: * such may be unwired themselves. ! 2710: */ ! 2711: ! 2712: pmap_pageable(pmap, entry->vme_start, end_addr, TRUE); ! 2713: ! 2714: } ! 2715: ! 2716: /* ! 2717: * vm_fault_wire_fast: ! 2718: * ! 2719: * Handle common case of a wire down page fault at the given address. ! 2720: * If successful, the page is inserted into the associated physical map. ! 2721: * The map entry is passed in to avoid the overhead of a map lookup. ! 2722: * ! 2723: * NOTE: the given address should be truncated to the ! 2724: * proper page address. ! 2725: * ! 2726: * KERN_SUCCESS is returned if the page fault is handled; otherwise, ! 2727: * a standard error specifying why the fault is fatal is returned. ! 2728: * ! 2729: * The map in question must be referenced, and remains so. ! 2730: * Caller has a read lock on the map. ! 2731: * ! 2732: * This is a stripped version of vm_fault() for wiring pages. Anything ! 2733: * other than the common case will return KERN_FAILURE, and the caller ! 2734: * is expected to call vm_fault(). ! 2735: */ ! 2736: kern_return_t ! 2737: vm_fault_wire_fast( ! 2738: vm_map_t map, ! 2739: vm_offset_t va, ! 2740: vm_map_entry_t entry) ! 2741: { ! 2742: vm_object_t object; ! 2743: vm_offset_t offset; ! 2744: register vm_page_t m; ! 2745: vm_prot_t prot; ! 2746: thread_act_t thr_act; ! 2747: ! 2748: VM_STAT(faults++); ! 2749: ! 2750: if((thr_act=current_act()) && (thr_act->task != TASK_NULL)) ! 2751: thr_act->task->faults++; ! 2752: ! 2753: /* ! 2754: * Recovery actions ! 2755: */ ! 2756: ! 2757: #undef RELEASE_PAGE ! 2758: #define RELEASE_PAGE(m) { \ ! 2759: PAGE_WAKEUP_DONE(m); \ ! 2760: vm_page_lock_queues(); \ ! 2761: vm_page_unwire(m); \ ! 2762: vm_page_unlock_queues(); \ ! 2763: } ! 2764: ! 2765: ! 2766: #undef UNLOCK_THINGS ! 2767: #define UNLOCK_THINGS { \ ! 2768: object->paging_in_progress--; \ ! 2769: vm_object_unlock(object); \ ! 2770: } ! 2771: ! 2772: #undef UNLOCK_AND_DEALLOCATE ! 2773: #define UNLOCK_AND_DEALLOCATE { \ ! 2774: UNLOCK_THINGS; \ ! 2775: vm_object_deallocate(object); \ ! 2776: } ! 2777: /* ! 2778: * Give up and have caller do things the hard way. ! 2779: */ ! 2780: ! 2781: #define GIVE_UP { \ ! 2782: UNLOCK_AND_DEALLOCATE; \ ! 2783: return(KERN_FAILURE); \ ! 2784: } ! 2785: ! 2786: ! 2787: /* ! 2788: * If this entry is not directly to a vm_object, bail out. ! 2789: */ ! 2790: if (entry->is_sub_map) ! 2791: return(KERN_FAILURE); ! 2792: ! 2793: /* ! 2794: * Find the backing store object and offset into it. ! 2795: */ ! 2796: ! 2797: object = entry->object.vm_object; ! 2798: offset = (va - entry->vme_start) + entry->offset; ! 2799: prot = entry->protection; ! 2800: ! 2801: /* ! 2802: * Make a reference to this object to prevent its ! 2803: * disposal while we are messing with it. ! 2804: */ ! 2805: ! 2806: vm_object_lock(object); ! 2807: assert(object->ref_count > 0); ! 2808: object->ref_count++; ! 2809: vm_object_res_reference(object); ! 2810: object->paging_in_progress++; ! 2811: ! 2812: /* ! 2813: * INVARIANTS (through entire routine): ! 2814: * ! 2815: * 1) At all times, we must either have the object ! 2816: * lock or a busy page in some object to prevent ! 2817: * some other thread from trying to bring in ! 2818: * the same page. ! 2819: * ! 2820: * 2) Once we have a busy page, we must remove it from ! 2821: * the pageout queues, so that the pageout daemon ! 2822: * will not grab it away. ! 2823: * ! 2824: */ ! 2825: ! 2826: /* ! 2827: * Look for page in top-level object. If it's not there or ! 2828: * there's something going on, give up. ! 2829: */ ! 2830: m = vm_page_lookup(object, offset); ! 2831: if ((m == VM_PAGE_NULL) || (m->busy) || ! 2832: (m->unusual && ( m->error || m->restart || m->absent || ! 2833: prot & m->page_lock))) { ! 2834: ! 2835: GIVE_UP; ! 2836: } ! 2837: ! 2838: /* ! 2839: * Wire the page down now. All bail outs beyond this ! 2840: * point must unwire the page. ! 2841: */ ! 2842: ! 2843: vm_page_lock_queues(); ! 2844: vm_page_wire(m); ! 2845: vm_page_unlock_queues(); ! 2846: ! 2847: /* ! 2848: * Mark page busy for other threads. ! 2849: */ ! 2850: assert(!m->busy); ! 2851: m->busy = TRUE; ! 2852: assert(!m->absent); ! 2853: ! 2854: /* ! 2855: * Give up if the page is being written and there's a copy object ! 2856: */ ! 2857: if ((object->copy != VM_OBJECT_NULL) && (prot & VM_PROT_WRITE)) { ! 2858: RELEASE_PAGE(m); ! 2859: GIVE_UP; ! 2860: } ! 2861: ! 2862: /* ! 2863: * Put this page into the physical map. ! 2864: * We have to unlock the object because pmap_enter ! 2865: * may cause other faults. ! 2866: */ ! 2867: vm_object_unlock(object); ! 2868: ! 2869: PMAP_ENTER(map->pmap, va, m, prot, TRUE); ! 2870: /* Sync I & D caches for new mapping */ ! 2871: pmap_attribute(map->pmap, ! 2872: va, ! 2873: PAGE_SIZE, ! 2874: MATTR_CACHE, ! 2875: &mv_cache_sync); ! 2876: ! 2877: /* ! 2878: * Must relock object so that paging_in_progress can be cleared. ! 2879: */ ! 2880: vm_object_lock(object); ! 2881: ! 2882: /* ! 2883: * Unlock everything, and return ! 2884: */ ! 2885: ! 2886: PAGE_WAKEUP_DONE(m); ! 2887: UNLOCK_AND_DEALLOCATE; ! 2888: ! 2889: return(KERN_SUCCESS); ! 2890: ! 2891: } ! 2892: ! 2893: /* ! 2894: * Routine: vm_fault_copy_cleanup ! 2895: * Purpose: ! 2896: * Release a page used by vm_fault_copy. ! 2897: */ ! 2898: ! 2899: void ! 2900: vm_fault_copy_cleanup( ! 2901: vm_page_t page, ! 2902: vm_page_t top_page) ! 2903: { ! 2904: vm_object_t object = page->object; ! 2905: ! 2906: vm_object_lock(object); ! 2907: PAGE_WAKEUP_DONE(page); ! 2908: vm_page_lock_queues(); ! 2909: if (!page->active && !page->inactive) ! 2910: vm_page_activate(page); ! 2911: vm_page_unlock_queues(); ! 2912: vm_fault_cleanup(object, top_page); ! 2913: } ! 2914: ! 2915: void ! 2916: vm_fault_copy_dst_cleanup( ! 2917: vm_page_t page) ! 2918: { ! 2919: vm_object_t object; ! 2920: ! 2921: if (page != VM_PAGE_NULL) { ! 2922: object = page->object; ! 2923: vm_object_lock(object); ! 2924: vm_page_lock_queues(); ! 2925: vm_page_unwire(page); ! 2926: vm_page_unlock_queues(); ! 2927: vm_object_paging_end(object); ! 2928: vm_object_unlock(object); ! 2929: } ! 2930: } ! 2931: ! 2932: /* ! 2933: * Routine: vm_fault_copy ! 2934: * ! 2935: * Purpose: ! 2936: * Copy pages from one virtual memory object to another -- ! 2937: * neither the source nor destination pages need be resident. ! 2938: * ! 2939: * Before actually copying a page, the version associated with ! 2940: * the destination address map wil be verified. ! 2941: * ! 2942: * In/out conditions: ! 2943: * The caller must hold a reference, but not a lock, to ! 2944: * each of the source and destination objects and to the ! 2945: * destination map. ! 2946: * ! 2947: * Results: ! 2948: * Returns KERN_SUCCESS if no errors were encountered in ! 2949: * reading or writing the data. Returns KERN_INTERRUPTED if ! 2950: * the operation was interrupted (only possible if the ! 2951: * "interruptible" argument is asserted). Other return values ! 2952: * indicate a permanent error in copying the data. ! 2953: * ! 2954: * The actual amount of data copied will be returned in the ! 2955: * "copy_size" argument. In the event that the destination map ! 2956: * verification failed, this amount may be less than the amount ! 2957: * requested. ! 2958: */ ! 2959: kern_return_t ! 2960: vm_fault_copy( ! 2961: vm_object_t src_object, ! 2962: vm_offset_t src_offset, ! 2963: vm_size_t *src_size, /* INOUT */ ! 2964: vm_object_t dst_object, ! 2965: vm_offset_t dst_offset, ! 2966: vm_map_t dst_map, ! 2967: vm_map_version_t *dst_version, ! 2968: int interruptible) ! 2969: { ! 2970: vm_page_t result_page; ! 2971: ! 2972: vm_page_t src_page; ! 2973: vm_page_t src_top_page; ! 2974: vm_prot_t src_prot; ! 2975: ! 2976: vm_page_t dst_page; ! 2977: vm_page_t dst_top_page; ! 2978: vm_prot_t dst_prot; ! 2979: ! 2980: vm_size_t amount_left; ! 2981: vm_object_t old_copy_object; ! 2982: kern_return_t error = 0; ! 2983: ! 2984: vm_size_t part_size; ! 2985: ! 2986: /* ! 2987: * In order not to confuse the clustered pageins, align ! 2988: * the different offsets on a page boundary. ! 2989: */ ! 2990: vm_offset_t src_lo_offset = trunc_page(src_offset); ! 2991: vm_offset_t dst_lo_offset = trunc_page(dst_offset); ! 2992: vm_offset_t src_hi_offset = round_page(src_offset + *src_size); ! 2993: vm_offset_t dst_hi_offset = round_page(dst_offset + *src_size); ! 2994: ! 2995: #define RETURN(x) \ ! 2996: MACRO_BEGIN \ ! 2997: *src_size -= amount_left; \ ! 2998: MACRO_RETURN(x); \ ! 2999: MACRO_END ! 3000: ! 3001: amount_left = *src_size; ! 3002: do { /* while (amount_left > 0) */ ! 3003: /* ! 3004: * There may be a deadlock if both source and destination ! 3005: * pages are the same. To avoid this deadlock, the copy must ! 3006: * start by getting the destination page in order to apply ! 3007: * COW semantics if any. ! 3008: */ ! 3009: ! 3010: RetryDestinationFault: ; ! 3011: ! 3012: dst_prot = VM_PROT_WRITE|VM_PROT_READ; ! 3013: ! 3014: vm_object_lock(dst_object); ! 3015: vm_object_paging_begin(dst_object); ! 3016: ! 3017: XPR(XPR_VM_FAULT,"vm_fault_copy -> vm_fault_page\n",0,0,0,0,0); ! 3018: switch (vm_fault_page(dst_object, ! 3019: trunc_page(dst_offset), ! 3020: VM_PROT_WRITE|VM_PROT_READ, ! 3021: FALSE, ! 3022: THREAD_UNINT, ! 3023: dst_lo_offset, ! 3024: dst_hi_offset, ! 3025: VM_BEHAVIOR_SEQUENTIAL, ! 3026: &dst_prot, ! 3027: &dst_page, ! 3028: &dst_top_page, ! 3029: (int *)0, ! 3030: &error, ! 3031: dst_map->no_zero_fill, ! 3032: FALSE)) { ! 3033: case VM_FAULT_SUCCESS: ! 3034: break; ! 3035: case VM_FAULT_RETRY: ! 3036: goto RetryDestinationFault; ! 3037: case VM_FAULT_INTERRUPTED: ! 3038: RETURN(MACH_SEND_INTERRUPTED); ! 3039: case VM_FAULT_MEMORY_SHORTAGE: ! 3040: VM_PAGE_WAIT(); ! 3041: goto RetryDestinationFault; ! 3042: case VM_FAULT_FICTITIOUS_SHORTAGE: ! 3043: vm_page_more_fictitious(); ! 3044: goto RetryDestinationFault; ! 3045: case VM_FAULT_MEMORY_ERROR: ! 3046: if (error) ! 3047: return (error); ! 3048: else ! 3049: return(KERN_MEMORY_ERROR); ! 3050: } ! 3051: assert ((dst_prot & VM_PROT_WRITE) != VM_PROT_NONE); ! 3052: ! 3053: old_copy_object = dst_page->object->copy; ! 3054: ! 3055: /* ! 3056: * There exists the possiblity that the source and ! 3057: * destination page are the same. But we can't ! 3058: * easily determine that now. If they are the ! 3059: * same, the call to vm_fault_page() for the ! 3060: * destination page will deadlock. To prevent this we ! 3061: * wire the page so we can drop busy without having ! 3062: * the page daemon steal the page. We clean up the ! 3063: * top page but keep the paging reference on the object ! 3064: * holding the dest page so it doesn't go away. ! 3065: */ ! 3066: ! 3067: vm_page_lock_queues(); ! 3068: vm_page_wire(dst_page); ! 3069: vm_page_unlock_queues(); ! 3070: PAGE_WAKEUP_DONE(dst_page); ! 3071: vm_object_unlock(dst_page->object); ! 3072: ! 3073: if (dst_top_page != VM_PAGE_NULL) { ! 3074: vm_object_lock(dst_object); ! 3075: VM_PAGE_FREE(dst_top_page); ! 3076: vm_object_paging_end(dst_object); ! 3077: vm_object_unlock(dst_object); ! 3078: } ! 3079: ! 3080: RetrySourceFault: ; ! 3081: ! 3082: if (src_object == VM_OBJECT_NULL) { ! 3083: /* ! 3084: * No source object. We will just ! 3085: * zero-fill the page in dst_object. ! 3086: */ ! 3087: src_page = VM_PAGE_NULL; ! 3088: } else { ! 3089: vm_object_lock(src_object); ! 3090: src_page = vm_page_lookup(src_object, ! 3091: trunc_page(src_offset)); ! 3092: if (src_page == dst_page) ! 3093: src_prot = dst_prot; ! 3094: else { ! 3095: src_prot = VM_PROT_READ; ! 3096: vm_object_paging_begin(src_object); ! 3097: ! 3098: XPR(XPR_VM_FAULT, ! 3099: "vm_fault_copy(2) -> vm_fault_page\n", ! 3100: 0,0,0,0,0); ! 3101: switch (vm_fault_page(src_object, ! 3102: trunc_page(src_offset), ! 3103: VM_PROT_READ, ! 3104: FALSE, ! 3105: interruptible, ! 3106: src_lo_offset, ! 3107: src_hi_offset, ! 3108: VM_BEHAVIOR_SEQUENTIAL, ! 3109: &src_prot, ! 3110: &result_page, ! 3111: &src_top_page, ! 3112: (int *)0, ! 3113: &error, ! 3114: FALSE, ! 3115: FALSE)) { ! 3116: ! 3117: case VM_FAULT_SUCCESS: ! 3118: break; ! 3119: case VM_FAULT_RETRY: ! 3120: goto RetrySourceFault; ! 3121: case VM_FAULT_INTERRUPTED: ! 3122: vm_fault_copy_dst_cleanup(dst_page); ! 3123: RETURN(MACH_SEND_INTERRUPTED); ! 3124: case VM_FAULT_MEMORY_SHORTAGE: ! 3125: VM_PAGE_WAIT(); ! 3126: goto RetrySourceFault; ! 3127: case VM_FAULT_FICTITIOUS_SHORTAGE: ! 3128: vm_page_more_fictitious(); ! 3129: goto RetrySourceFault; ! 3130: case VM_FAULT_MEMORY_ERROR: ! 3131: vm_fault_copy_dst_cleanup(dst_page); ! 3132: if (error) ! 3133: return (error); ! 3134: else ! 3135: return(KERN_MEMORY_ERROR); ! 3136: } ! 3137: ! 3138: src_page = result_page; ! 3139: ! 3140: assert((src_top_page == VM_PAGE_NULL) == ! 3141: (src_page->object == src_object)); ! 3142: } ! 3143: assert ((src_prot & VM_PROT_READ) != VM_PROT_NONE); ! 3144: vm_object_unlock(src_page->object); ! 3145: } ! 3146: ! 3147: if (!vm_map_verify(dst_map, dst_version)) { ! 3148: if (src_page != VM_PAGE_NULL && src_page != dst_page) ! 3149: vm_fault_copy_cleanup(src_page, src_top_page); ! 3150: vm_fault_copy_dst_cleanup(dst_page); ! 3151: break; ! 3152: } ! 3153: ! 3154: vm_object_lock(dst_page->object); ! 3155: ! 3156: if (dst_page->object->copy != old_copy_object) { ! 3157: vm_object_unlock(dst_page->object); ! 3158: vm_map_verify_done(dst_map, dst_version); ! 3159: if (src_page != VM_PAGE_NULL && src_page != dst_page) ! 3160: vm_fault_copy_cleanup(src_page, src_top_page); ! 3161: vm_fault_copy_dst_cleanup(dst_page); ! 3162: break; ! 3163: } ! 3164: vm_object_unlock(dst_page->object); ! 3165: ! 3166: /* ! 3167: * Copy the page, and note that it is dirty ! 3168: * immediately. ! 3169: */ ! 3170: ! 3171: if (!page_aligned(src_offset) || ! 3172: !page_aligned(dst_offset) || ! 3173: !page_aligned(amount_left)) { ! 3174: ! 3175: vm_offset_t src_po, ! 3176: dst_po; ! 3177: ! 3178: src_po = src_offset - trunc_page(src_offset); ! 3179: dst_po = dst_offset - trunc_page(dst_offset); ! 3180: ! 3181: if (dst_po > src_po) { ! 3182: part_size = PAGE_SIZE - dst_po; ! 3183: } else { ! 3184: part_size = PAGE_SIZE - src_po; ! 3185: } ! 3186: if (part_size > (amount_left)){ ! 3187: part_size = amount_left; ! 3188: } ! 3189: ! 3190: if (src_page == VM_PAGE_NULL) { ! 3191: vm_page_part_zero_fill(dst_page, ! 3192: dst_po, part_size); ! 3193: } else { ! 3194: vm_page_part_copy(src_page, src_po, ! 3195: dst_page, dst_po, part_size); ! 3196: if(!dst_page->dirty){ ! 3197: vm_object_lock(dst_object); ! 3198: dst_page->dirty = TRUE; ! 3199: vm_object_unlock(dst_page->object); ! 3200: } ! 3201: ! 3202: } ! 3203: } else { ! 3204: part_size = PAGE_SIZE; ! 3205: ! 3206: if (src_page == VM_PAGE_NULL) ! 3207: vm_page_zero_fill(dst_page); ! 3208: else{ ! 3209: vm_page_copy(src_page, dst_page); ! 3210: if(!dst_page->dirty){ ! 3211: vm_object_lock(dst_object); ! 3212: dst_page->dirty = TRUE; ! 3213: vm_object_unlock(dst_page->object); ! 3214: } ! 3215: } ! 3216: ! 3217: } ! 3218: ! 3219: /* ! 3220: * Unlock everything, and return ! 3221: */ ! 3222: ! 3223: vm_map_verify_done(dst_map, dst_version); ! 3224: ! 3225: if (src_page != VM_PAGE_NULL && src_page != dst_page) ! 3226: vm_fault_copy_cleanup(src_page, src_top_page); ! 3227: vm_fault_copy_dst_cleanup(dst_page); ! 3228: ! 3229: amount_left -= part_size; ! 3230: src_offset += part_size; ! 3231: dst_offset += part_size; ! 3232: } while (amount_left > 0); ! 3233: ! 3234: RETURN(KERN_SUCCESS); ! 3235: #undef RETURN ! 3236: ! 3237: /*NOTREACHED*/ ! 3238: } ! 3239: ! 3240: #ifdef notdef ! 3241: ! 3242: /* ! 3243: * Routine: vm_fault_page_overwrite ! 3244: * ! 3245: * Description: ! 3246: * A form of vm_fault_page that assumes that the ! 3247: * resulting page will be overwritten in its entirety, ! 3248: * making it unnecessary to obtain the correct *contents* ! 3249: * of the page. ! 3250: * ! 3251: * Implementation: ! 3252: * XXX Untested. Also unused. Eventually, this technology ! 3253: * could be used in vm_fault_copy() to advantage. ! 3254: */ ! 3255: vm_fault_return_t ! 3256: vm_fault_page_overwrite( ! 3257: register ! 3258: vm_object_t dst_object, ! 3259: vm_offset_t dst_offset, ! 3260: vm_page_t *result_page) /* OUT */ ! 3261: { ! 3262: register ! 3263: vm_page_t dst_page; ! 3264: thread_t thread = current_thread(); ! 3265: ! 3266: #define interruptible THREAD_UNINT /* XXX */ ! 3267: ! 3268: while (TRUE) { ! 3269: /* ! 3270: * Look for a page at this offset ! 3271: */ ! 3272: ! 3273: while ((dst_page = vm_page_lookup(dst_object, dst_offset)) ! 3274: == VM_PAGE_NULL) { ! 3275: /* ! 3276: * No page, no problem... just allocate one. ! 3277: */ ! 3278: ! 3279: dst_page = vm_page_alloc(dst_object, dst_offset); ! 3280: if (dst_page == VM_PAGE_NULL) { ! 3281: vm_object_unlock(dst_object); ! 3282: VM_PAGE_WAIT(); ! 3283: vm_object_lock(dst_object); ! 3284: continue; ! 3285: } ! 3286: ! 3287: /* ! 3288: * Pretend that the memory manager ! 3289: * write-protected the page. ! 3290: * ! 3291: * Note that we will be asking for write ! 3292: * permission without asking for the data ! 3293: * first. ! 3294: */ ! 3295: ! 3296: dst_page->overwriting = TRUE; ! 3297: dst_page->page_lock = VM_PROT_WRITE; ! 3298: dst_page->absent = TRUE; ! 3299: dst_page->unusual = TRUE; ! 3300: dst_object->absent_count++; ! 3301: ! 3302: break; ! 3303: ! 3304: /* ! 3305: * When we bail out, we might have to throw ! 3306: * away the page created here. ! 3307: */ ! 3308: ! 3309: #define DISCARD_PAGE \ ! 3310: MACRO_BEGIN \ ! 3311: vm_object_lock(dst_object); \ ! 3312: dst_page = vm_page_lookup(dst_object, dst_offset); \ ! 3313: if ((dst_page != VM_PAGE_NULL) && dst_page->overwriting) \ ! 3314: VM_PAGE_FREE(dst_page); \ ! 3315: vm_object_unlock(dst_object); \ ! 3316: MACRO_END ! 3317: } ! 3318: ! 3319: /* ! 3320: * If the page is write-protected... ! 3321: */ ! 3322: ! 3323: if (dst_page->page_lock & VM_PROT_WRITE) { ! 3324: /* ! 3325: * ... and an unlock request hasn't been sent ! 3326: */ ! 3327: ! 3328: if ( ! (dst_page->unlock_request & VM_PROT_WRITE)) { ! 3329: vm_prot_t u; ! 3330: kern_return_t rc; ! 3331: ! 3332: /* ! 3333: * ... then send one now. ! 3334: */ ! 3335: ! 3336: if (!dst_object->pager_ready) { ! 3337: vm_object_assert_wait(dst_object, ! 3338: VM_OBJECT_EVENT_PAGER_READY, ! 3339: interruptible); ! 3340: vm_object_unlock(dst_object); ! 3341: thread_block((void (*)(void))0); ! 3342: if (thread->wait_result != ! 3343: THREAD_AWAKENED) { ! 3344: DISCARD_PAGE; ! 3345: return(VM_FAULT_INTERRUPTED); ! 3346: } ! 3347: continue; ! 3348: } ! 3349: ! 3350: u = dst_page->unlock_request |= VM_PROT_WRITE; ! 3351: vm_object_unlock(dst_object); ! 3352: ! 3353: if ((rc = memory_object_data_unlock( ! 3354: dst_object->pager, ! 3355: dst_object->pager_request, ! 3356: dst_offset + dst_object->paging_offset, ! 3357: PAGE_SIZE, ! 3358: u)) != KERN_SUCCESS) { ! 3359: if (vm_fault_debug) ! 3360: printf("vm_object_overwrite: memory_object_data_unlock failed\n"); ! 3361: DISCARD_PAGE; ! 3362: return((rc == MACH_SEND_INTERRUPTED) ? ! 3363: VM_FAULT_INTERRUPTED : ! 3364: VM_FAULT_MEMORY_ERROR); ! 3365: } ! 3366: vm_object_lock(dst_object); ! 3367: continue; ! 3368: } ! 3369: ! 3370: /* ... fall through to wait below */ ! 3371: } else { ! 3372: /* ! 3373: * If the page isn't being used for other ! 3374: * purposes, then we're done. ! 3375: */ ! 3376: if ( ! (dst_page->busy || dst_page->absent || ! 3377: dst_page->error || dst_page->restart) ) ! 3378: break; ! 3379: } ! 3380: ! 3381: PAGE_ASSERT_WAIT(dst_page, interruptible); ! 3382: vm_object_unlock(dst_object); ! 3383: thread_block((void (*)(void))0); ! 3384: if (thread->wait_result != THREAD_AWAKENED) { ! 3385: DISCARD_PAGE; ! 3386: return(VM_FAULT_INTERRUPTED); ! 3387: } ! 3388: } ! 3389: ! 3390: *result_page = dst_page; ! 3391: return(VM_FAULT_SUCCESS); ! 3392: ! 3393: #undef interruptible ! 3394: #undef DISCARD_PAGE ! 3395: } ! 3396: ! 3397: #endif /* notdef */ ! 3398: ! 3399: #if VM_FAULT_CLASSIFY ! 3400: /* ! 3401: * Temporary statistics gathering support. ! 3402: */ ! 3403: ! 3404: /* ! 3405: * Statistics arrays: ! 3406: */ ! 3407: #define VM_FAULT_TYPES_MAX 5 ! 3408: #define VM_FAULT_LEVEL_MAX 8 ! 3409: ! 3410: int vm_fault_stats[VM_FAULT_TYPES_MAX][VM_FAULT_LEVEL_MAX]; ! 3411: ! 3412: #define VM_FAULT_TYPE_ZERO_FILL 0 ! 3413: #define VM_FAULT_TYPE_MAP_IN 1 ! 3414: #define VM_FAULT_TYPE_PAGER 2 ! 3415: #define VM_FAULT_TYPE_COPY 3 ! 3416: #define VM_FAULT_TYPE_OTHER 4 ! 3417: ! 3418: ! 3419: void ! 3420: vm_fault_classify(vm_object_t object, ! 3421: vm_offset_t offset, ! 3422: vm_prot_t fault_type) ! 3423: { ! 3424: int type, level = 0; ! 3425: vm_page_t m; ! 3426: ! 3427: while (TRUE) { ! 3428: m = vm_page_lookup(object, offset); ! 3429: if (m != VM_PAGE_NULL) { ! 3430: if (m->busy || m->error || m->restart || m->absent || ! 3431: fault_type & m->page_lock) { ! 3432: type = VM_FAULT_TYPE_OTHER; ! 3433: break; ! 3434: } ! 3435: if (((fault_type & VM_PROT_WRITE) == 0) || ! 3436: ((level == 0) && object->copy == VM_OBJECT_NULL)) { ! 3437: type = VM_FAULT_TYPE_MAP_IN; ! 3438: break; ! 3439: } ! 3440: type = VM_FAULT_TYPE_COPY; ! 3441: break; ! 3442: } ! 3443: else { ! 3444: if (object->pager_created) { ! 3445: type = VM_FAULT_TYPE_PAGER; ! 3446: break; ! 3447: } ! 3448: if (object->shadow == VM_OBJECT_NULL) { ! 3449: type = VM_FAULT_TYPE_ZERO_FILL; ! 3450: break; ! 3451: } ! 3452: ! 3453: offset += object->shadow_offset; ! 3454: object = object->shadow; ! 3455: level++; ! 3456: continue; ! 3457: } ! 3458: } ! 3459: ! 3460: if (level > VM_FAULT_LEVEL_MAX) ! 3461: level = VM_FAULT_LEVEL_MAX; ! 3462: ! 3463: vm_fault_stats[type][level] += 1; ! 3464: ! 3465: return; ! 3466: } ! 3467: ! 3468: /* cleanup routine to call from debugger */ ! 3469: ! 3470: void ! 3471: vm_fault_classify_init(void) ! 3472: { ! 3473: int type, level; ! 3474: ! 3475: for (type = 0; type < VM_FAULT_TYPES_MAX; type++) { ! 3476: for (level = 0; level < VM_FAULT_LEVEL_MAX; level++) { ! 3477: vm_fault_stats[type][level] = 0; ! 3478: } ! 3479: } ! 3480: ! 3481: return; ! 3482: } ! 3483: #endif /* VM_FAULT_CLASSIFY */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.