![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to memory_object.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple XNU] / XNU / osfmk / vm|
Return to memory_object.c CVS log | Up to [Apple XNU] / XNU / osfmk / vm |
1.1 ! root 1: /* ! 2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. ! 3: * ! 4: * @APPLE_LICENSE_HEADER_START@ ! 5: * ! 6: * The contents of this file constitute Original Code as defined in and ! 7: * are subject to the Apple Public Source License Version 1.1 (the ! 8: * "License"). You may not use this file except in compliance with the ! 9: * License. Please obtain a copy of the License at ! 10: * http://www.apple.com/publicsource and read it before using this file. ! 11: * ! 12: * This Original Code and all software distributed under the License are ! 13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER ! 14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, ! 15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, ! 16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the ! 17: * License for the specific language governing rights and limitations ! 18: * under the License. ! 19: * ! 20: * @APPLE_LICENSE_HEADER_END@ ! 21: */ ! 22: /* ! 23: * @OSF_COPYRIGHT@ ! 24: */ ! 25: /* ! 26: * Mach Operating System ! 27: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University ! 28: * All Rights Reserved. ! 29: * ! 30: * Permission to use, copy, modify and distribute this software and its ! 31: * documentation is hereby granted, provided that both the copyright ! 32: * notice and this permission notice appear in all copies of the ! 33: * software, derivative works or modified versions, and any portions ! 34: * thereof, and that both notices appear in supporting documentation. ! 35: * ! 36: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" ! 37: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR ! 38: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. ! 39: * ! 40: * Carnegie Mellon requests users of this software to return to ! 41: * ! 42: * Software Distribution Coordinator or [email protected] ! 43: * School of Computer Science ! 44: * Carnegie Mellon University ! 45: * Pittsburgh PA 15213-3890 ! 46: * ! 47: * any improvements or extensions that they make and grant Carnegie Mellon ! 48: * the rights to redistribute these changes. ! 49: */ ! 50: /* ! 51: */ ! 52: /* ! 53: * File: vm/memory_object.c ! 54: * Author: Michael Wayne Young ! 55: * ! 56: * External memory management interface control functions. ! 57: */ ! 58: ! 59: #ifdef MACH_BSD ! 60: /* THIS code should be removed when the component merge is completed */ ! 61: extern int vnode_pager_workaround; ! 62: #endif ! 63: ! 64: #include <advisory_pageout.h> ! 65: ! 66: /* ! 67: * Interface dependencies: ! 68: */ ! 69: ! 70: #include <mach/std_types.h> /* For pointer_t */ ! 71: #include <mach/mach_types.h> ! 72: ! 73: #include <mach/kern_return.h> ! 74: #include <mach/memory_object.h> ! 75: #include <mach/memory_object_default.h> ! 76: #include <mach/memory_object_control_server.h> ! 77: #include <mach/mach_host_server.h> ! 78: #include <mach/boolean.h> ! 79: #include <mach/vm_prot.h> ! 80: #include <mach/message.h> ! 81: ! 82: #include <vm/vm_object.h> ! 83: #include <vm/vm_fault.h> ! 84: /* ! 85: * Implementation dependencies: ! 86: */ ! 87: #include <string.h> /* For memcpy() */ ! 88: ! 89: #include <vm/memory_object.h> ! 90: #include <vm/vm_page.h> ! 91: #include <vm/vm_pageout.h> ! 92: #include <vm/pmap.h> /* For pmap_clear_modify */ ! 93: #include <kern/xpr.h> ! 94: #include <kern/thread.h> /* For current_thread() */ ! 95: #include <kern/host.h> ! 96: #include <vm/vm_kern.h> /* For kernel_map, vm_move */ ! 97: #include <vm/vm_map.h> /* For vm_map_pageable */ ! 98: #include <ipc/ipc_port.h> ! 99: #include <ipc/ipc_space.h> ! 100: ! 101: ! 102: ! 103: #include <kern/misc_protos.h> ! 104: ! 105: #if MACH_PAGEMAP ! 106: #include <vm/vm_external.h> ! 107: #endif /* MACH_PAGEMAP */ ! 108: ! 109: ipc_port_t memory_manager_default = IP_NULL; ! 110: vm_size_t memory_manager_default_cluster = 0; ! 111: decl_mutex_data(,memory_manager_default_lock) ! 112: ! 113: /* ! 114: * Forward ref to file-local function: ! 115: */ ! 116: boolean_t ! 117: memory_object_update(vm_object_t, ! 118: vm_offset_t, vm_size_t, memory_object_return_t, boolean_t, vm_prot_t); ! 119: ! 120: /* ! 121: * Important note: ! 122: * All of the external (user interface) routines gain a reference ! 123: * to the object (first argument) as part of the automatic ! 124: * argument conversion. Explicit deallocation is necessary. ! 125: */ ! 126: ! 127: kern_return_t ! 128: memory_object_data_supply( ! 129: register ! 130: vm_object_t object, ! 131: register ! 132: vm_offset_t offset, ! 133: vm_offset_t data, ! 134: mach_msg_type_number_t data_cnt, ! 135: vm_prot_t lock_value, ! 136: boolean_t precious, ! 137: ipc_port_t reply_to, ! 138: mach_msg_type_name_t reply_to_type) ! 139: { ! 140: vm_map_copy_t data_copy = (vm_map_copy_t)data; ! 141: kern_return_t result = KERN_SUCCESS; ! 142: vm_offset_t error_offset = 0; ! 143: register ! 144: vm_page_t m; ! 145: register ! 146: vm_page_t data_m; ! 147: vm_size_t original_length; ! 148: vm_offset_t original_offset; ! 149: vm_page_t *page_list; ! 150: boolean_t was_absent; ! 151: vm_map_copy_t orig_copy = data_copy; ! 152: boolean_t was_clustered; ! 153: ! 154: ! 155: XPR(XPR_MEMORY_OBJECT, ! 156: "m_o_data_supply, object 0x%X, offset 0x%X data_cnt 0x%X\n", ! 157: (integer_t)object, offset, data_cnt, 0, 0); ! 158: ! 159: /* ! 160: * Look for bogus arguments ! 161: */ ! 162: ! 163: if (object == VM_OBJECT_NULL) { ! 164: return(KERN_INVALID_ARGUMENT); ! 165: } ! 166: ! 167: if (lock_value & ~VM_PROT_ALL) { ! 168: vm_object_deallocate(object); ! 169: return(KERN_INVALID_ARGUMENT); ! 170: } ! 171: ! 172: if (! page_aligned(data_cnt)) { ! 173: vm_object_deallocate(object); ! 174: return(KERN_INVALID_ARGUMENT); ! 175: } ! 176: ! 177: /* ! 178: * Adjust the offset from the memory object to the offset ! 179: * within the vm_object. ! 180: */ ! 181: ! 182: original_length = data_cnt; ! 183: original_offset = offset; ! 184: ! 185: assert(data_copy->type == VM_MAP_COPY_PAGE_LIST); ! 186: page_list = &data_copy->cpy_page_list[0]; ! 187: ! 188: vm_object_lock(object); ! 189: vm_object_paging_begin(object); ! 190: offset -= object->paging_offset; ! 191: ! 192: /* ! 193: * Loop over copy stealing pages for pagein. ! 194: */ ! 195: ! 196: for (; data_cnt > 0 ; data_cnt -= PAGE_SIZE, offset += PAGE_SIZE) { ! 197: ! 198: assert(data_copy != VM_MAP_COPY_NULL); ! 199: assert(data_copy->type == VM_MAP_COPY_PAGE_LIST); ! 200: assert(data_copy->cpy_npages > 0); ! 201: data_m = *page_list; ! 202: ! 203: assert( !(data_m == VM_PAGE_NULL || data_m->tabled || ! 204: data_m->error || data_m->absent || data_m->fictitious || ! 205: data_m->restart)); ! 206: ! 207: /* ! 208: * Look up target page and check its state. ! 209: */ ! 210: ! 211: retry_lookup: ! 212: m = vm_page_lookup(object,offset); ! 213: was_clustered = FALSE; ! 214: if (m == VM_PAGE_NULL) { ! 215: was_absent = FALSE; ! 216: } ! 217: else { ! 218: if (m->absent) { ! 219: ! 220: /* ! 221: * Page was requested. Free the busy ! 222: * page waiting for it. Insertion ! 223: * of new page happens below. ! 224: */ ! 225: m->list_req_pending = FALSE; ! 226: if (m->clustered) { ! 227: was_clustered = TRUE; ! 228: m->clustered = FALSE; ! 229: } ! 230: if (m->busy) { ! 231: VM_PAGE_FREE(m); ! 232: } ! 233: was_absent = TRUE; ! 234: } ! 235: else { ! 236: ! 237: /* ! 238: * Have to wait for page that is busy and ! 239: * not absent. This is probably going to ! 240: * be an error, but go back and check. ! 241: */ ! 242: if (m->busy) { ! 243: PAGE_ASSERT_WAIT(m, THREAD_UNINT); ! 244: vm_object_unlock(object); ! 245: thread_block((void (*)(void))0); ! 246: vm_object_lock(object); ! 247: goto retry_lookup; ! 248: } ! 249: ! 250: /* ! 251: * Page already present; error. ! 252: * This is an error if data is precious. ! 253: */ ! 254: result = KERN_MEMORY_PRESENT; ! 255: error_offset = offset + object->paging_offset; ! 256: ! 257: break; ! 258: } ! 259: } ! 260: ! 261: /* ! 262: * Ok to pagein page. Target object now has no page ! 263: * at offset. Set the page parameters, then drop ! 264: * in new page and set up pageout state. Object is ! 265: * still locked here. ! 266: * ! 267: * Must clear busy bit in page before inserting it. ! 268: * Ok to skip wakeup logic because nobody else ! 269: * can possibly know about this page. ! 270: */ ! 271: ! 272: data_m->lock_supplied = TRUE; ! 273: data_m->busy = FALSE; ! 274: data_m->dirty = FALSE; ! 275: pmap_clear_modify(data_m->phys_addr); ! 276: ! 277: data_m->page_lock = lock_value; ! 278: if (lock_value != VM_PROT_NONE) ! 279: data_m->unusual = TRUE; ! 280: else ! 281: data_m->unusual = FALSE; ! 282: ! 283: data_m->unlock_request = VM_PROT_NONE; ! 284: data_m->precious = precious; ! 285: ! 286: data_m->clustered = was_clustered; ! 287: ! 288: vm_page_insert(data_m, object, offset); ! 289: ! 290: vm_page_lock_queues(); ! 291: if (was_absent) ! 292: vm_page_activate(data_m); ! 293: else ! 294: vm_page_deactivate(data_m); ! 295: vm_page_unlock_queues(); ! 296: ! 297: /* ! 298: * Null out this page list entry, and advance to next ! 299: * page. ! 300: */ ! 301: ! 302: *page_list++ = VM_PAGE_NULL; ! 303: ! 304: if (--(data_copy->cpy_npages) == 0 && ! 305: vm_map_copy_has_cont(data_copy)) { ! 306: vm_map_copy_t new_copy; ! 307: ! 308: vm_object_unlock(object); ! 309: vm_map_copy_invoke_cont(data_copy, &new_copy, &result); ! 310: vm_object_lock(object); ! 311: ! 312: if (result == KERN_SUCCESS) { ! 313: ! 314: /* ! 315: * Consume on success requires that ! 316: * we keep the original vm_map_copy ! 317: * around in case something fails. ! 318: * Free the old copy if it's not the original ! 319: */ ! 320: if (data_copy != orig_copy) { ! 321: vm_map_copy_discard(data_copy); ! 322: } ! 323: ! 324: if ((data_copy = new_copy) != VM_MAP_COPY_NULL) ! 325: page_list = &data_copy->cpy_page_list[0]; ! 326: } ! 327: else { ! 328: error_offset = offset + object->paging_offset + ! 329: PAGE_SIZE; ! 330: break; ! 331: } ! 332: } ! 333: } ! 334: ! 335: vm_object_paging_end(object); ! 336: ! 337: /* ! 338: * If there is a continuation that has not been invoked, ! 339: * there must have been an error. Abort the continuation. ! 340: */ ! 341: ! 342: if (data_copy != VM_MAP_COPY_NULL && vm_map_copy_has_cont(data_copy)) { ! 343: vm_object_unlock(object); ! 344: vm_map_copy_abort_cont(data_copy); ! 345: vm_object_lock(object); ! 346: } ! 347: ! 348: /* ! 349: * If the pager wants a reply, send it one. ! 350: */ ! 351: ! 352: if (IP_VALID(reply_to)) { ! 353: vm_object_unlock(object); ! 354: memory_object_supply_completed( ! 355: reply_to, reply_to_type, ! 356: object->pager_request, ! 357: original_offset, ! 358: original_length, ! 359: result, ! 360: error_offset); ! 361: } ! 362: else ! 363: vm_object_unlock(object); ! 364: ! 365: vm_object_deallocate(object); ! 366: ! 367: /* ! 368: * Consume on success: The final data copy must be ! 369: * discarded if it is not the original. The original ! 370: * gets discarded only if this routine succeeds. ! 371: */ ! 372: if (data_copy != VM_MAP_COPY_NULL && ! 373: data_copy != orig_copy) ! 374: vm_map_copy_discard(data_copy); ! 375: if (result == KERN_SUCCESS) ! 376: vm_map_copy_discard(orig_copy); ! 377: ! 378: return(result); ! 379: } ! 380: ! 381: kern_return_t ! 382: memory_object_data_error( ! 383: vm_object_t object, ! 384: vm_offset_t offset, ! 385: vm_size_t size, ! 386: kern_return_t error_value) ! 387: { ! 388: XPR(XPR_MEMORY_OBJECT, ! 389: "m_o_data_error, object 0x%X, offset 0x%X size 0x%X\n", ! 390: (integer_t)object, offset, size, 0, 0); ! 391: ! 392: if (object == VM_OBJECT_NULL) ! 393: return(KERN_INVALID_ARGUMENT); ! 394: ! 395: if (size != round_page(size)) { ! 396: vm_object_deallocate(object); ! 397: return(KERN_INVALID_ARGUMENT); ! 398: } ! 399: ! 400: vm_object_lock(object); ! 401: offset -= object->paging_offset; ! 402: ! 403: while (size != 0) { ! 404: register vm_page_t m; ! 405: ! 406: m = vm_page_lookup(object, offset); ! 407: if ((m != VM_PAGE_NULL) && m->busy && m->absent) { ! 408: if (error_value == KERN_MEMORY_DATA_MOVED) { ! 409: m->restart = TRUE; ! 410: } else { ! 411: if (error_value > KERN_RETURN_MAX) { ! 412: m->page_error = error_value; ! 413: } else { ! 414: m->page_error = KERN_MEMORY_ERROR; ! 415: } ! 416: m->error = TRUE; ! 417: } ! 418: ! 419: /* ! 420: * m->unusual was true because page was absent. ! 421: * It remains true because page is either error ! 422: * or restart. ! 423: */ ! 424: m->absent = FALSE; ! 425: vm_object_absent_release(object); ! 426: ! 427: PAGE_WAKEUP_DONE(m); ! 428: ! 429: /* ! 430: * If this page was not requested (clustered is set), ! 431: * then throw it away since there is no one to which ! 432: * an error can be reported. ! 433: */ ! 434: vm_page_lock_queues(); ! 435: if (m->clustered) ! 436: vm_page_free(m); ! 437: else ! 438: vm_page_activate(m); ! 439: vm_page_unlock_queues(); ! 440: } ! 441: ! 442: size -= PAGE_SIZE; ! 443: offset += PAGE_SIZE; ! 444: } ! 445: vm_object_unlock(object); ! 446: ! 447: vm_object_deallocate(object); ! 448: return(KERN_SUCCESS); ! 449: } ! 450: ! 451: kern_return_t ! 452: memory_object_data_unavailable( ! 453: vm_object_t object, ! 454: vm_offset_t offset, ! 455: vm_size_t size) ! 456: { ! 457: XPR(XPR_MEMORY_OBJECT, ! 458: "m_o_data_unavailable, object 0x%X, offset 0x%X size 0x%X\n", ! 459: (integer_t)object, offset, size, 0, 0); ! 460: ! 461: if (object == VM_OBJECT_NULL) ! 462: return(KERN_INVALID_ARGUMENT); ! 463: ! 464: if (size != round_page(size)) { ! 465: vm_object_deallocate(object); ! 466: return(KERN_INVALID_ARGUMENT); ! 467: } ! 468: ! 469: vm_object_lock(object); ! 470: offset -= object->paging_offset; ! 471: ! 472: while (size != 0) { ! 473: register vm_page_t m; ! 474: ! 475: /* ! 476: * We're looking for pages that are both busy and ! 477: * absent (waiting to be filled), converting them ! 478: * to just absent. ! 479: * ! 480: * Pages that are just busy can be ignored entirely. ! 481: */ ! 482: ! 483: m = vm_page_lookup(object, offset); ! 484: if (m == VM_PAGE_NULL) { ! 485: if (object->absent_count < vm_object_absent_max) ! 486: m = vm_page_grab_fictitious(); ! 487: if (m != VM_PAGE_NULL) { ! 488: m->absent = TRUE; ! 489: m->unusual = TRUE; ! 490: m->clustered = FALSE; ! 491: object->absent_count++; ! 492: vm_page_lock_queues(); ! 493: vm_page_insert(m, object, offset); ! 494: vm_page_activate(m); ! 495: vm_page_unlock_queues(); ! 496: PAGE_WAKEUP_DONE(m); ! 497: } ! 498: } else if (m->busy && m->absent) { ! 499: /* only consider real pageins in clustering stats */ ! 500: m->clustered = FALSE; ! 501: ! 502: PAGE_WAKEUP_DONE(m); ! 503: ! 504: vm_page_lock_queues(); ! 505: vm_page_activate(m); ! 506: vm_page_unlock_queues(); ! 507: } ! 508: size -= PAGE_SIZE; ! 509: offset += PAGE_SIZE; ! 510: } ! 511: ! 512: vm_object_unlock(object); ! 513: ! 514: vm_object_deallocate(object); ! 515: return(KERN_SUCCESS); ! 516: } ! 517: ! 518: /* ! 519: * Routine: memory_object_should_return_page ! 520: * ! 521: * Description: ! 522: * Determine whether the given page should be returned, ! 523: * based on the page's state and on the given return policy. ! 524: * ! 525: * We should return the page if one of the following is true: ! 526: * ! 527: * 1. Page is dirty and should_return is not RETURN_NONE. ! 528: * 2. Page is precious and should_return is RETURN_ALL. ! 529: * 3. Should_return is RETURN_ANYTHING. ! 530: * ! 531: * As a side effect, m->dirty will be made consistent ! 532: * with pmap_is_modified(m), if should_return is not ! 533: * MEMORY_OBJECT_RETURN_NONE. ! 534: */ ! 535: ! 536: #define memory_object_should_return_page(m, should_return) \ ! 537: (should_return != MEMORY_OBJECT_RETURN_NONE && \ ! 538: (((m)->dirty || ((m)->dirty = pmap_is_modified((m)->phys_addr))) || \ ! 539: ((m)->precious && (should_return) == MEMORY_OBJECT_RETURN_ALL) || \ ! 540: (should_return) == MEMORY_OBJECT_RETURN_ANYTHING)) ! 541: ! 542: typedef int memory_object_lock_result_t; ! 543: ! 544: #define MEMORY_OBJECT_LOCK_RESULT_DONE 0 ! 545: #define MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK 1 ! 546: #define MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN 2 ! 547: #define MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN 3 ! 548: ! 549: memory_object_lock_result_t memory_object_lock_page( ! 550: vm_page_t m, ! 551: memory_object_return_t should_return, ! 552: boolean_t should_flush, ! 553: vm_prot_t prot); ! 554: ! 555: /* ! 556: * Routine: memory_object_lock_page ! 557: * ! 558: * Description: ! 559: * Perform the appropriate lock operations on the ! 560: * given page. See the description of ! 561: * "memory_object_lock_request" for the meanings ! 562: * of the arguments. ! 563: * ! 564: * Returns an indication that the operation ! 565: * completed, blocked, or that the page must ! 566: * be cleaned. ! 567: */ ! 568: memory_object_lock_result_t ! 569: memory_object_lock_page( ! 570: vm_page_t m, ! 571: memory_object_return_t should_return, ! 572: boolean_t should_flush, ! 573: vm_prot_t prot) ! 574: { ! 575: XPR(XPR_MEMORY_OBJECT, ! 576: "m_o_lock_page, page 0x%X rtn %d flush %d prot %d\n", ! 577: (integer_t)m, should_return, should_flush, prot, 0); ! 578: ! 579: /* ! 580: * Don't worry about pages for which the kernel ! 581: * does not have any data. ! 582: */ ! 583: ! 584: if (m->absent || m->error || m->restart) ! 585: return(MEMORY_OBJECT_LOCK_RESULT_DONE); ! 586: ! 587: /* ! 588: * If we cannot change access to the page, ! 589: * either because a mapping is in progress ! 590: * (busy page) or because a mapping has been ! 591: * wired, then give up. ! 592: */ ! 593: ! 594: if (m->busy) ! 595: return(MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK); ! 596: ! 597: assert(!m->fictitious); ! 598: ! 599: if (m->wire_count != 0) { ! 600: /* ! 601: * If no change would take place ! 602: * anyway, return successfully. ! 603: * ! 604: * No change means: ! 605: * Not flushing AND ! 606: * No change to page lock [2 checks] AND ! 607: * Should not return page ! 608: * ! 609: * XXX This doesn't handle sending a copy of a wired ! 610: * XXX page to the pager, but that will require some ! 611: * XXX significant surgery. ! 612: */ ! 613: if (!should_flush && ! 614: (m->page_lock == prot || prot == VM_PROT_NO_CHANGE) && ! 615: ! memory_object_should_return_page(m, should_return)) { ! 616: ! 617: /* ! 618: * Restart page unlock requests, ! 619: * even though no change took place. ! 620: * [Memory managers may be expecting ! 621: * to see new requests.] ! 622: */ ! 623: m->unlock_request = VM_PROT_NONE; ! 624: PAGE_WAKEUP(m); ! 625: ! 626: return(MEMORY_OBJECT_LOCK_RESULT_DONE); ! 627: } ! 628: ! 629: return(MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK); ! 630: } ! 631: ! 632: /* ! 633: * If the page is to be flushed, allow ! 634: * that to be done as part of the protection. ! 635: */ ! 636: ! 637: if (should_flush) ! 638: prot = VM_PROT_ALL; ! 639: ! 640: /* ! 641: * Set the page lock. ! 642: * ! 643: * If we are decreasing permission, do it now; ! 644: * let the fault handler take care of increases ! 645: * (pmap_page_protect may not increase protection). ! 646: */ ! 647: ! 648: if (prot != VM_PROT_NO_CHANGE) { ! 649: if ((m->page_lock ^ prot) & prot) { ! 650: pmap_page_protect(m->phys_addr, VM_PROT_ALL & ~prot); ! 651: } ! 652: m->page_lock = prot; ! 653: m->lock_supplied = TRUE; ! 654: if (prot != VM_PROT_NONE) ! 655: m->unusual = TRUE; ! 656: else ! 657: m->unusual = FALSE; ! 658: ! 659: /* ! 660: * Restart any past unlock requests, even if no ! 661: * change resulted. If the manager explicitly ! 662: * requested no protection change, then it is assumed ! 663: * to be remembering past requests. ! 664: */ ! 665: ! 666: m->unlock_request = VM_PROT_NONE; ! 667: PAGE_WAKEUP(m); ! 668: } ! 669: ! 670: /* ! 671: * Handle page returning. ! 672: */ ! 673: ! 674: if (memory_object_should_return_page(m, should_return)) { ! 675: ! 676: /* ! 677: * If we weren't planning ! 678: * to flush the page anyway, ! 679: * we may need to remove the ! 680: * page from the pageout ! 681: * system and from physical ! 682: * maps now. ! 683: */ ! 684: ! 685: vm_page_lock_queues(); ! 686: VM_PAGE_QUEUES_REMOVE(m); ! 687: vm_page_unlock_queues(); ! 688: ! 689: if (!should_flush) ! 690: pmap_page_protect(m->phys_addr, VM_PROT_NONE); ! 691: ! 692: if (m->dirty) ! 693: return(MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN); ! 694: else ! 695: return(MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN); ! 696: } ! 697: ! 698: /* ! 699: * Handle flushing ! 700: */ ! 701: ! 702: if (should_flush) { ! 703: VM_PAGE_FREE(m); ! 704: } else { ! 705: extern boolean_t vm_page_deactivate_hint; ! 706: ! 707: /* ! 708: * XXX Make clean but not flush a paging hint, ! 709: * and deactivate the pages. This is a hack ! 710: * because it overloads flush/clean with ! 711: * implementation-dependent meaning. This only ! 712: * happens to pages that are already clean. ! 713: */ ! 714: ! 715: if (vm_page_deactivate_hint && ! 716: (should_return != MEMORY_OBJECT_RETURN_NONE)) { ! 717: vm_page_lock_queues(); ! 718: vm_page_deactivate(m); ! 719: vm_page_unlock_queues(); ! 720: } ! 721: } ! 722: ! 723: return(MEMORY_OBJECT_LOCK_RESULT_DONE); ! 724: } ! 725: #define LIST_REQ_PAGEOUT_PAGES(object, new_object, new_offset, action, po) \ ! 726: MACRO_BEGIN \ ! 727: \ ! 728: register int i; \ ! 729: register vm_page_t hp; \ ! 730: \ ! 731: vm_object_unlock(object); \ ! 732: \ ! 733: if(((rpc_subsystem_t)pager_mux_hash_lookup(object->pager)) == \ ! 734: ((rpc_subsystem_t) &vnode_pager_workaround)) { \ ! 735: (void) vnode_pager_data_return(object->pager, \ ! 736: object->pager_request, \ ! 737: po, \ ! 738: POINTER_T(0), \ ! 739: new_offset, \ ! 740: (action == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN), \ ! 741: !should_flush); \ ! 742: } else { \ ! 743: (void) memory_object_data_return(object->pager, \ ! 744: object->pager_request, \ ! 745: po, \ ! 746: POINTER_T(0), \ ! 747: new_offset, \ ! 748: (action == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN), \ ! 749: !should_flush); \ ! 750: } \ ! 751: \ ! 752: vm_object_lock(object); \ ! 753: \ ! 754: new_object = VM_OBJECT_NULL; \ ! 755: MACRO_END ! 756: ! 757: #ifdef MACH_BSD ! 758: #define PAGEOUT_PAGES(object, new_object, new_offset, action, po) \ ! 759: MACRO_BEGIN \ ! 760: \ ! 761: vm_map_copy_t copy; \ ! 762: register int i; \ ! 763: register vm_page_t hp; \ ! 764: \ ! 765: vm_object_unlock(object); \ ! 766: \ ! 767: (void) vm_map_copyin_object(new_object, 0, new_offset, ©); \ ! 768: \ ! 769: if(((rpc_subsystem_t)pager_mux_hash_lookup(object->pager)) == \ ! 770: ((rpc_subsystem_t) &vnode_pager_workaround)) { \ ! 771: (void) vnode_pager_data_return(object->pager, \ ! 772: object->pager_request, \ ! 773: po, \ ! 774: POINTER_T(copy), \ ! 775: new_offset, \ ! 776: (action == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN), \ ! 777: !should_flush); \ ! 778: } else { \ ! 779: (void) memory_object_data_return(object->pager, \ ! 780: object->pager_request, \ ! 781: po, \ ! 782: POINTER_T(copy), \ ! 783: new_offset, \ ! 784: (action == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN), \ ! 785: !should_flush); \ ! 786: } \ ! 787: \ ! 788: vm_object_lock(object); \ ! 789: \ ! 790: for (i = 0; i < atop(new_offset); i++) { \ ! 791: hp = holding_pages[i]; \ ! 792: if (hp != VM_PAGE_NULL) { \ ! 793: vm_object_paging_end(object); \ ! 794: VM_PAGE_FREE(hp); \ ! 795: } \ ! 796: } \ ! 797: \ ! 798: new_object = VM_OBJECT_NULL; \ ! 799: MACRO_END ! 800: #else ! 801: #define PAGEOUT_PAGES(object, new_object, new_offset, action, po) \ ! 802: MACRO_BEGIN \ ! 803: \ ! 804: vm_map_copy_t copy; \ ! 805: register int i; \ ! 806: register vm_page_t hp; \ ! 807: \ ! 808: vm_object_unlock(object); \ ! 809: \ ! 810: (void) vm_map_copyin_object(new_object, 0, new_offset, ©); \ ! 811: \ ! 812: (void) memory_object_data_return( \ ! 813: object->pager, \ ! 814: object->pager_request, \ ! 815: po, \ ! 816: POINTER_T(copy), \ ! 817: new_offset, \ ! 818: (action == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN), \ ! 819: !should_flush); \ ! 820: \ ! 821: vm_object_lock(object); \ ! 822: \ ! 823: for (i = 0; i < atop(new_offset); i++) { \ ! 824: hp = holding_pages[i]; \ ! 825: if (hp != VM_PAGE_NULL) { \ ! 826: vm_object_paging_end(object); \ ! 827: VM_PAGE_FREE(hp); \ ! 828: } \ ! 829: } \ ! 830: \ ! 831: new_object = VM_OBJECT_NULL; \ ! 832: MACRO_END ! 833: #endif ! 834: ! 835: /* ! 836: * Routine: memory_object_lock_request [user interface] ! 837: * ! 838: * Description: ! 839: * Control use of the data associated with the given ! 840: * memory object. For each page in the given range, ! 841: * perform the following operations, in order: ! 842: * 1) restrict access to the page (disallow ! 843: * forms specified by "prot"); ! 844: * 2) return data to the manager (if "should_return" ! 845: * is RETURN_DIRTY and the page is dirty, or ! 846: * "should_return" is RETURN_ALL and the page ! 847: * is either dirty or precious); and, ! 848: * 3) flush the cached copy (if "should_flush" ! 849: * is asserted). ! 850: * The set of pages is defined by a starting offset ! 851: * ("offset") and size ("size"). Only pages with the ! 852: * same page alignment as the starting offset are ! 853: * considered. ! 854: * ! 855: * A single acknowledgement is sent (to the "reply_to" ! 856: * port) when these actions are complete. If successful, ! 857: * the naked send right for reply_to is consumed. ! 858: */ ! 859: ! 860: kern_return_t ! 861: memory_object_lock_request( ! 862: register vm_object_t object, ! 863: register vm_offset_t offset, ! 864: register vm_size_t size, ! 865: memory_object_return_t should_return, ! 866: boolean_t should_flush, ! 867: vm_prot_t prot, ! 868: ipc_port_t reply_to, ! 869: mach_msg_type_name_t reply_to_type) ! 870: { ! 871: vm_offset_t original_offset = offset; ! 872: ! 873: XPR(XPR_MEMORY_OBJECT, ! 874: "m_o_lock_request, obj 0x%X off 0x%X size 0x%X flags %X prot %X\n", ! 875: (integer_t)object, offset, size, ! 876: (((should_return&1)<<1)|should_flush), prot); ! 877: ! 878: /* ! 879: * Check for bogus arguments. ! 880: */ ! 881: if (object == VM_OBJECT_NULL) ! 882: return (KERN_INVALID_ARGUMENT); ! 883: ! 884: if ((prot & ~VM_PROT_ALL) != 0 && prot != VM_PROT_NO_CHANGE) { ! 885: vm_object_deallocate(object); ! 886: return (KERN_INVALID_ARGUMENT); ! 887: } ! 888: ! 889: size = round_page(size); ! 890: ! 891: /* ! 892: * Lock the object, and acquire a paging reference to ! 893: * prevent the memory_object and control ports from ! 894: * being destroyed. ! 895: */ ! 896: ! 897: vm_object_lock(object); ! 898: vm_object_paging_begin(object); ! 899: offset -= object->paging_offset; ! 900: ! 901: (void)memory_object_update(object, ! 902: offset, size, should_return, should_flush, prot); ! 903: ! 904: if (IP_VALID(reply_to)) { ! 905: vm_object_unlock(object); ! 906: ! 907: /* consumes our naked send-once/send right for reply_to */ ! 908: (void) memory_object_lock_completed(reply_to, reply_to_type, ! 909: object->pager_request, original_offset, size); ! 910: ! 911: vm_object_lock(object); ! 912: } ! 913: ! 914: vm_object_paging_end(object); ! 915: vm_object_unlock(object); ! 916: vm_object_deallocate(object); ! 917: ! 918: return (KERN_SUCCESS); ! 919: } ! 920: ! 921: /* ! 922: * Routine: memory_object_sync ! 923: * ! 924: * Kernel internal function to synch out pages in a given ! 925: * range within an object to its memory manager. Much the ! 926: * same as memory_object_lock_request but page protection ! 927: * is not changed. ! 928: * ! 929: * If the should_flush and should_return flags are true pages ! 930: * are flushed, that is dirty & precious pages are written to ! 931: * the memory manager and then discarded. If should_return ! 932: * is false, only precious pages are returned to the memory ! 933: * manager. ! 934: * ! 935: * If should flush is false and should_return true, the memory ! 936: * manager's copy of the pages is updated. If should_return ! 937: * is also false, only the precious pages are updated. This ! 938: * last option is of limited utility. ! 939: * ! 940: * Returns: ! 941: * FALSE if no pages were returned to the pager ! 942: * TRUE otherwise. ! 943: */ ! 944: ! 945: boolean_t ! 946: memory_object_sync( ! 947: vm_object_t object, ! 948: vm_offset_t offset, ! 949: vm_size_t size, ! 950: boolean_t should_flush, ! 951: boolean_t should_return) ! 952: { ! 953: boolean_t rv; ! 954: ! 955: XPR(XPR_MEMORY_OBJECT, ! 956: "m_o_sync, object 0x%X, offset 0x%X size 0x%x flush %d rtn %d\n", ! 957: (integer_t)object, offset, size, should_flush, should_return); ! 958: ! 959: /* ! 960: * Lock the object, and acquire a paging reference to ! 961: * prevent the memory_object and control ports from ! 962: * being destroyed. ! 963: */ ! 964: vm_object_lock(object); ! 965: vm_object_paging_begin(object); ! 966: ! 967: rv = memory_object_update(object, offset, size, ! 968: (should_return) ? ! 969: MEMORY_OBJECT_RETURN_ALL : ! 970: MEMORY_OBJECT_RETURN_NONE, ! 971: should_flush, ! 972: VM_PROT_NO_CHANGE); ! 973: ! 974: ! 975: vm_object_paging_end(object); ! 976: vm_object_unlock(object); ! 977: return rv; ! 978: } ! 979: ! 980: /* ! 981: * Routine: memory_object_update ! 982: * Description: ! 983: * Work function for m_o_lock_request(), m_o_sync(). ! 984: * ! 985: * Called with object locked and paging ref taken. ! 986: */ ! 987: kern_return_t ! 988: memory_object_update( ! 989: register vm_object_t object, ! 990: register vm_offset_t offset, ! 991: register vm_size_t size, ! 992: memory_object_return_t should_return, ! 993: boolean_t should_flush, ! 994: vm_prot_t prot) ! 995: { ! 996: register vm_page_t m; ! 997: vm_page_t holding_page; ! 998: vm_size_t original_size = size; ! 999: vm_offset_t paging_offset = 0; ! 1000: vm_object_t new_object = VM_OBJECT_NULL; ! 1001: vm_object_t copy_object; ! 1002: vm_offset_t new_offset = 0; ! 1003: vm_offset_t last_offset = offset; ! 1004: memory_object_lock_result_t page_lock_result; ! 1005: memory_object_lock_result_t pageout_action; ! 1006: vm_page_t holding_pages[DATA_WRITE_MAX]; ! 1007: boolean_t data_returned = FALSE; ! 1008: #ifndef NOT_LIST_REQ ! 1009: boolean_t pending_pageout = FALSE; ! 1010: #endif ! 1011: ! 1012: /* ! 1013: * To avoid blocking while scanning for pages, save ! 1014: * dirty pages to be cleaned all at once. ! 1015: * ! 1016: * XXXO A similar strategy could be used to limit the ! 1017: * number of times that a scan must be restarted for ! 1018: * other reasons. Those pages that would require blocking ! 1019: * could be temporarily collected in another list, or ! 1020: * their offsets could be recorded in a small array. ! 1021: */ ! 1022: ! 1023: /* ! 1024: * XXX NOTE: May want to consider converting this to a page list ! 1025: * XXX vm_map_copy interface. Need to understand object ! 1026: * XXX coalescing implications before doing so. ! 1027: */ ! 1028: ! 1029: ! 1030: if(((copy_object = object->copy) != NULL) && should_flush) { ! 1031: vm_size_t i; ! 1032: vm_size_t copy_size; ! 1033: vm_offset_t copy_offset; ! 1034: vm_prot_t prot; ! 1035: vm_page_t page; ! 1036: vm_page_t top_page; ! 1037: kern_return_t error = 0; ! 1038: ! 1039: ! 1040: ! 1041: /* translate offset with respect to shadow's offset */ ! 1042: copy_offset = (offset >= copy_object->shadow_offset)? ! 1043: offset - copy_object->shadow_offset : ! 1044: (vm_offset_t)0; ! 1045: if(copy_offset > copy_object->size) ! 1046: copy_offset = copy_object->size; ! 1047: ! 1048: /* clip size with respect to shadow offset */ ! 1049: copy_size = (offset >= copy_object->shadow_offset) ? ! 1050: size : size - (copy_object->shadow_offset - offset); ! 1051: ! 1052: if(copy_size <= 0) { ! 1053: copy_size = 0; ! 1054: } else { ! 1055: copy_size = ((copy_offset + copy_size) ! 1056: <= copy_object->size) ? ! 1057: copy_size : copy_object->size - copy_offset; ! 1058: } ! 1059: /* check for a copy_offset which is beyond the end of */ ! 1060: /* the copy_object */ ! 1061: if(copy_size < 0) ! 1062: copy_size = 0; ! 1063: ! 1064: vm_object_unlock(object); ! 1065: vm_object_lock(copy_object); ! 1066: vm_object_paging_begin(copy_object); ! 1067: for (i=copy_offset; i<copy_size; i+=PAGE_SIZE) { ! 1068: RETRY_COW_OF_LOCK_REQUEST: ! 1069: prot = VM_PROT_WRITE|VM_PROT_READ; ! 1070: switch (vm_fault_page(copy_object, i, ! 1071: VM_PROT_WRITE|VM_PROT_READ, ! 1072: FALSE, ! 1073: THREAD_UNINT, ! 1074: copy_offset, ! 1075: copy_offset+copy_size, ! 1076: VM_BEHAVIOR_SEQUENTIAL, ! 1077: &prot, ! 1078: &page, ! 1079: &top_page, ! 1080: (int *)0, ! 1081: &error, ! 1082: FALSE, ! 1083: FALSE)) { ! 1084: ! 1085: case VM_FAULT_SUCCESS: ! 1086: if(top_page) { ! 1087: vm_fault_cleanup( ! 1088: page->object, top_page); ! 1089: PAGE_WAKEUP_DONE(page); ! 1090: vm_page_lock_queues(); ! 1091: if (!page->active && !page->inactive) ! 1092: vm_page_activate(page); ! 1093: vm_page_unlock_queues(); ! 1094: vm_object_lock(copy_object); ! 1095: vm_object_paging_begin(copy_object); ! 1096: } else { ! 1097: PAGE_WAKEUP_DONE(page); ! 1098: vm_page_lock_queues(); ! 1099: if (!page->active && !page->inactive) ! 1100: vm_page_activate(page); ! 1101: vm_page_unlock_queues(); ! 1102: } ! 1103: break; ! 1104: case VM_FAULT_RETRY: ! 1105: prot = VM_PROT_WRITE|VM_PROT_READ; ! 1106: vm_object_lock(copy_object); ! 1107: vm_object_paging_begin(copy_object); ! 1108: goto RETRY_COW_OF_LOCK_REQUEST; ! 1109: case VM_FAULT_INTERRUPTED: ! 1110: prot = VM_PROT_WRITE|VM_PROT_READ; ! 1111: vm_object_lock(copy_object); ! 1112: vm_object_paging_begin(copy_object); ! 1113: goto RETRY_COW_OF_LOCK_REQUEST; ! 1114: case VM_FAULT_MEMORY_SHORTAGE: ! 1115: VM_PAGE_WAIT(); ! 1116: prot = VM_PROT_WRITE|VM_PROT_READ; ! 1117: vm_object_lock(copy_object); ! 1118: vm_object_paging_begin(copy_object); ! 1119: goto RETRY_COW_OF_LOCK_REQUEST; ! 1120: case VM_FAULT_FICTITIOUS_SHORTAGE: ! 1121: vm_page_more_fictitious(); ! 1122: prot = VM_PROT_WRITE|VM_PROT_READ; ! 1123: vm_object_lock(copy_object); ! 1124: vm_object_paging_begin(copy_object); ! 1125: goto RETRY_COW_OF_LOCK_REQUEST; ! 1126: case VM_FAULT_MEMORY_ERROR: ! 1127: vm_object_lock(object); ! 1128: goto BYPASS_COW_COPYIN; ! 1129: } ! 1130: ! 1131: ! 1132: } ! 1133: vm_object_paging_end(copy_object); ! 1134: vm_object_unlock(copy_object); ! 1135: vm_object_lock(object); ! 1136: } ! 1137: BYPASS_COW_COPYIN: ! 1138: ! 1139: for (; ! 1140: size != 0; ! 1141: size -= PAGE_SIZE, offset += PAGE_SIZE) ! 1142: { ! 1143: /* ! 1144: * Limit the number of pages to be cleaned at once. ! 1145: */ ! 1146: #ifdef NOT_LIST_REQ ! 1147: if (new_object != VM_OBJECT_NULL && ! 1148: new_offset >= PAGE_SIZE * DATA_WRITE_MAX) ! 1149: { ! 1150: PAGEOUT_PAGES(object, new_object, new_offset, pageout_action, ! 1151: paging_offset); ! 1152: } ! 1153: #else ! 1154: /* ! 1155: if(((rpc_subsystem_t)pager_mux_hash_lookup(object->pager)) != ! 1156: ((rpc_subsystem_t) &vnode_pager_workaround)) { ! 1157: if (new_object != VM_OBJECT_NULL && ! 1158: new_offset >= PAGE_SIZE * DATA_WRITE_MAX) ! 1159: { ! 1160: PAGEOUT_PAGES(object, new_object, new_offset, pageout_action, ! 1161: paging_offset); ! 1162: } ! 1163: } else { ! 1164: */ ! 1165: if (pending_pageout && ! 1166: new_offset >= PAGE_SIZE * DATA_WRITE_MAX) ! 1167: { ! 1168: LIST_REQ_PAGEOUT_PAGES(object, new_object, ! 1169: new_offset, ! 1170: pageout_action, paging_offset); ! 1171: pending_pageout = FALSE; ! 1172: } ! 1173: /* ! 1174: } ! 1175: */ ! 1176: #endif ! 1177: ! 1178: while ((m = vm_page_lookup(object, offset)) != VM_PAGE_NULL) { ! 1179: page_lock_result = memory_object_lock_page(m, should_return, ! 1180: should_flush, prot); ! 1181: ! 1182: XPR(XPR_MEMORY_OBJECT, ! 1183: "m_o_update: lock_page, obj 0x%X offset 0x%X result %d\n", ! 1184: (integer_t)object, offset, page_lock_result, 0, 0); ! 1185: ! 1186: switch (page_lock_result) ! 1187: { ! 1188: case MEMORY_OBJECT_LOCK_RESULT_DONE: ! 1189: /* ! 1190: * End of a cluster of dirty pages. ! 1191: */ ! 1192: #ifdef NOT_LIST_REQ ! 1193: if (new_object != VM_OBJECT_NULL) { ! 1194: PAGEOUT_PAGES(object, new_object, new_offset, ! 1195: pageout_action, paging_offset); ! 1196: continue; ! 1197: } ! 1198: #else ! 1199: /* ! 1200: if(((rpc_subsystem_t) pager_mux_hash_lookup(object->pager)) != ! 1201: ((rpc_subsystem_t) &vnode_pager_workaround)) { ! 1202: if (new_object != VM_OBJECT_NULL) { ! 1203: PAGEOUT_PAGES(object, new_object, new_offset, ! 1204: pageout_action, paging_offset); ! 1205: continue; ! 1206: } ! 1207: } else { ! 1208: */ ! 1209: if(pending_pageout) { ! 1210: LIST_REQ_PAGEOUT_PAGES(object, new_object, ! 1211: new_offset, pageout_action, ! 1212: paging_offset); ! 1213: pending_pageout = FALSE; ! 1214: continue; ! 1215: } ! 1216: /* ! 1217: } ! 1218: */ ! 1219: #endif ! 1220: break; ! 1221: ! 1222: case MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK: ! 1223: /* ! 1224: * Since it is necessary to block, ! 1225: * clean any dirty pages now. ! 1226: */ ! 1227: #ifdef NOT_LIST_REQ ! 1228: if (new_object != VM_OBJECT_NULL) { ! 1229: PAGEOUT_PAGES(object, new_object, new_offset, ! 1230: pageout_action, paging_offset); ! 1231: continue; ! 1232: } ! 1233: #else ! 1234: /* ! 1235: if(((rpc_subsystem_t) pager_mux_hash_lookup(object->pager)) != ! 1236: ((rpc_subsystem_t) &vnode_pager_workaround)) { ! 1237: if (new_object != VM_OBJECT_NULL) { ! 1238: PAGEOUT_PAGES(object, new_object, new_offset, ! 1239: pageout_action, paging_offset); ! 1240: continue; ! 1241: } ! 1242: } else { ! 1243: */ ! 1244: if(pending_pageout) { ! 1245: LIST_REQ_PAGEOUT_PAGES(object, new_object, ! 1246: new_offset, pageout_action, ! 1247: paging_offset); ! 1248: pending_pageout = FALSE; ! 1249: continue; ! 1250: } ! 1251: /* ! 1252: } ! 1253: */ ! 1254: #endif ! 1255: ! 1256: PAGE_ASSERT_WAIT(m, THREAD_UNINT); ! 1257: vm_object_unlock(object); ! 1258: thread_block((void (*)(void))0); ! 1259: vm_object_lock(object); ! 1260: continue; ! 1261: ! 1262: case MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN: ! 1263: case MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN: ! 1264: /* ! 1265: * The clean and return cases are similar. ! 1266: * ! 1267: */ ! 1268: ! 1269: /* ! 1270: * if this would form a discontiguous block, ! 1271: * clean the old pages and start anew. ! 1272: * ! 1273: * NOTE: The first time through here, new_object ! 1274: * is null, hiding the fact that pageout_action ! 1275: * is not initialized. ! 1276: */ ! 1277: #ifdef NOT_LIST_REQ ! 1278: /* ! 1279: * Mark the page busy since we unlock the ! 1280: * object below. ! 1281: */ ! 1282: m->busy = TRUE; ! 1283: if (new_object != VM_OBJECT_NULL && ! 1284: (last_offset != offset || ! 1285: pageout_action != page_lock_result)) { ! 1286: PAGEOUT_PAGES(object, new_object, new_offset, ! 1287: pageout_action, paging_offset); ! 1288: } ! 1289: vm_object_unlock(object); ! 1290: ! 1291: /* ! 1292: * If we have not already allocated an object ! 1293: * for a range of pages to be written, do so ! 1294: * now. ! 1295: */ ! 1296: if (new_object == VM_OBJECT_NULL) { ! 1297: if (should_flush) { ! 1298: /* use clean-in-place if possible */ ! 1299: new_object = vm_pageout_object_allocate( ! 1300: m, ! 1301: original_size, ! 1302: m->offset); ! 1303: } else { ! 1304: /* do not use clean-in-place */ ! 1305: new_object = ! 1306: vm_object_allocate(original_size); ! 1307: } ! 1308: new_offset = 0; ! 1309: paging_offset = m->offset + ! 1310: object->paging_offset; ! 1311: pageout_action = page_lock_result; ! 1312: } ! 1313: ! 1314: if (should_flush) { ! 1315: /* ! 1316: * Move or shadow the dirty page into the ! 1317: * new object. ! 1318: * This applies to either trusted or ! 1319: * untrusted pagers. ! 1320: * Get a paging ref to donate to ! 1321: * vm_pageout_setup(). ! 1322: */ ! 1323: vm_object_lock(object); ! 1324: vm_object_paging_begin(object); ! 1325: vm_object_unlock(object); ! 1326: holding_page = vm_pageout_setup(m, ! 1327: new_object, ! 1328: new_offset); ! 1329: } else { ! 1330: /* ! 1331: * Clean but do not flush, always ! 1332: * copying the page. Allowing clean-in-place ! 1333: * is potentially confusing to pagers and XMM ! 1334: * that ask for clean pages, since the page ! 1335: * could become dirty before the pager ! 1336: * looks at it. ! 1337: */ ! 1338: vm_page_t new_m; ! 1339: ! 1340: while ((new_m=vm_page_grab()) == VM_PAGE_NULL) { ! 1341: VM_PAGE_WAIT(); ! 1342: } ! 1343: assert(new_m != VM_PAGE_NULL); ! 1344: ! 1345: vm_object_lock(m->object); ! 1346: m->busy = FALSE; ! 1347: vm_page_lock_queues(); ! 1348: vm_pageclean_copy(m, new_m, new_object, ! 1349: new_offset); ! 1350: vm_page_unlock_queues(); ! 1351: vm_object_unlock(m->object); ! 1352: holding_page = VM_PAGE_NULL; ! 1353: } ! 1354: /* ! 1355: * Save the holding page if there is one. ! 1356: */ ! 1357: holding_pages[atop(new_offset)] = holding_page; ! 1358: #else ! 1359: #if 0 ! 1360: if(((rpc_subsystem_t) pager_mux_hash_lookup(object->pager)) != ! 1361: ((rpc_subsystem_t) &vnode_pager_workaround)) { ! 1362: /* ! 1363: * Mark the page busy since we unlock the ! 1364: * object below. ! 1365: */ ! 1366: m->busy = TRUE; ! 1367: if (new_object != VM_OBJECT_NULL && ! 1368: (last_offset != offset || ! 1369: pageout_action != page_lock_result)) { ! 1370: PAGEOUT_PAGES(object, new_object, new_offset, ! 1371: pageout_action, paging_offset); ! 1372: } ! 1373: vm_object_unlock(object); ! 1374: ! 1375: /* ! 1376: * If we have not already allocated an object ! 1377: * for a range of pages to be written, do so ! 1378: * now. ! 1379: */ ! 1380: if (new_object == VM_OBJECT_NULL) { ! 1381: if (should_flush) { ! 1382: /* use clean-in-place if possible */ ! 1383: new_object = vm_pageout_object_allocate( ! 1384: m, ! 1385: original_size, ! 1386: m->offset); ! 1387: } else { ! 1388: /* do not use clean-in-place */ ! 1389: new_object = ! 1390: vm_object_allocate(original_size); ! 1391: } ! 1392: new_offset = 0; ! 1393: paging_offset = m->offset + ! 1394: object->paging_offset; ! 1395: pageout_action = page_lock_result; ! 1396: } ! 1397: ! 1398: if (should_flush) { ! 1399: /* ! 1400: * Move or shadow the dirty page into the ! 1401: * new object. ! 1402: * This applies to either trusted or ! 1403: * untrusted pagers. ! 1404: * Get a paging ref to donate to ! 1405: * vm_pageout_setup(). ! 1406: */ ! 1407: vm_object_lock(object); ! 1408: vm_object_paging_begin(object); ! 1409: vm_object_unlock(object); ! 1410: holding_page = vm_pageout_setup(m, ! 1411: new_object, ! 1412: new_offset); ! 1413: } else { ! 1414: /* ! 1415: * Clean but do not flush, always ! 1416: * copying the page. Allowing clean-in-place ! 1417: * is potentially confusing to pagers and XMM ! 1418: * that ask for clean pages, since the page ! 1419: * could become dirty before the pager ! 1420: * looks at it. ! 1421: */ ! 1422: vm_page_t new_m; ! 1423: ! 1424: while ((new_m=vm_page_grab()) == VM_PAGE_NULL) { ! 1425: VM_PAGE_WAIT(); ! 1426: } ! 1427: assert(new_m != VM_PAGE_NULL); ! 1428: ! 1429: vm_object_lock(m->object); ! 1430: m->busy = FALSE; ! 1431: vm_page_lock_queues(); ! 1432: vm_pageclean_copy(m, new_m, new_object, ! 1433: new_offset); ! 1434: vm_page_unlock_queues(); ! 1435: vm_object_unlock(m->object); ! 1436: holding_page = VM_PAGE_NULL; ! 1437: } ! 1438: /* ! 1439: * Save the holding page if there is one. ! 1440: */ ! 1441: holding_pages[atop(new_offset)] = holding_page; ! 1442: } else { ! 1443: #endif ! 1444: if (pending_pageout && ! 1445: (last_offset != offset || ! 1446: pageout_action != page_lock_result)) { ! 1447: LIST_REQ_PAGEOUT_PAGES(object, new_object, ! 1448: new_offset, pageout_action, ! 1449: paging_offset); ! 1450: pending_pageout = FALSE; ! 1451: } ! 1452: holding_page = VM_PAGE_NULL; ! 1453: if(!pending_pageout) { ! 1454: pending_pageout = TRUE; ! 1455: pageout_action = page_lock_result; ! 1456: paging_offset = offset; ! 1457: } ! 1458: m->list_req_pending = TRUE; ! 1459: m->cleaning = TRUE; ! 1460: if (should_flush) { ! 1461: m->busy = TRUE; ! 1462: m->pageout = TRUE; ! 1463: } ! 1464: vm_object_unlock(object); ! 1465: /* ! 1466: } ! 1467: */ ! 1468: ! 1469: #endif ! 1470: ! 1471: new_offset += PAGE_SIZE; ! 1472: last_offset = offset + PAGE_SIZE; ! 1473: data_returned = TRUE; ! 1474: ! 1475: vm_object_lock(object); ! 1476: break; ! 1477: } ! 1478: break; ! 1479: } ! 1480: } ! 1481: ! 1482: /* ! 1483: * We have completed the scan for applicable pages. ! 1484: * Clean any pages that have been saved. ! 1485: */ ! 1486: #ifdef NOT_LIST_REQ ! 1487: if (new_object != VM_OBJECT_NULL) { ! 1488: PAGEOUT_PAGES(object, new_object, new_offset, pageout_action, ! 1489: paging_offset); ! 1490: } ! 1491: #else ! 1492: /* ! 1493: if(((rpc_subsystem_t) pager_mux_hash_lookup(object->pager)) != ! 1494: ((rpc_subsystem_t) &vnode_pager_workaround)) { ! 1495: if (new_object != VM_OBJECT_NULL) { ! 1496: PAGEOUT_PAGES(object, new_object, new_offset, pageout_action, ! 1497: paging_offset); ! 1498: } ! 1499: } else { ! 1500: */ ! 1501: if (pending_pageout) { ! 1502: LIST_REQ_PAGEOUT_PAGES(object, new_object, ! 1503: new_offset, pageout_action, paging_offset); ! 1504: } ! 1505: /* ! 1506: } ! 1507: */ ! 1508: #endif ! 1509: return (data_returned); ! 1510: } ! 1511: ! 1512: /* ! 1513: * Routine: memory_object_synchronize_completed [user interface] ! 1514: * ! 1515: * Tell kernel that previously synchronized data ! 1516: * (memory_object_synchronize) has been queue or placed on the ! 1517: * backing storage. ! 1518: * ! 1519: * Note: there may be multiple synchronize requests for a given ! 1520: * memory object outstanding but they will not overlap. ! 1521: */ ! 1522: ! 1523: kern_return_t ! 1524: memory_object_synchronize_completed( ! 1525: vm_object_t object, ! 1526: vm_offset_t offset, ! 1527: vm_offset_t length) ! 1528: { ! 1529: msync_req_t msr; ! 1530: ! 1531: XPR(XPR_MEMORY_OBJECT, ! 1532: "m_o_sync_completed, object 0x%X, offset 0x%X length 0x%X\n", ! 1533: (integer_t)object, offset, length, 0, 0); ! 1534: ! 1535: /* ! 1536: * Look for bogus arguments ! 1537: */ ! 1538: ! 1539: if (object == VM_OBJECT_NULL) { ! 1540: return KERN_INVALID_ARGUMENT; ! 1541: } ! 1542: ! 1543: vm_object_lock(object); ! 1544: ! 1545: /* ! 1546: * search for sync request structure ! 1547: */ ! 1548: queue_iterate(&object->msr_q, msr, msync_req_t, msr_q) { ! 1549: if (msr->offset == offset && msr->length == length) { ! 1550: queue_remove(&object->msr_q, msr, msync_req_t, msr_q); ! 1551: break; ! 1552: } ! 1553: }/* queue_iterate */ ! 1554: ! 1555: if (queue_end(&object->msr_q, (queue_entry_t)msr)) { ! 1556: vm_object_unlock(object); ! 1557: vm_object_deallocate(object); ! 1558: return KERN_INVALID_ARGUMENT; ! 1559: } ! 1560: ! 1561: msr_lock(msr); ! 1562: vm_object_unlock(object); ! 1563: msr->flag = VM_MSYNC_DONE; ! 1564: msr_unlock(msr); ! 1565: thread_wakeup((event_t) msr); ! 1566: vm_object_deallocate(object); ! 1567: ! 1568: return KERN_SUCCESS; ! 1569: }/* memory_object_synchronize_completed */ ! 1570: ! 1571: kern_return_t ! 1572: memory_object_set_attributes_common( ! 1573: vm_object_t object, ! 1574: boolean_t may_cache, ! 1575: memory_object_copy_strategy_t copy_strategy, ! 1576: boolean_t temporary, ! 1577: vm_size_t cluster_size, ! 1578: boolean_t silent_overwrite, ! 1579: boolean_t advisory_pageout) ! 1580: { ! 1581: boolean_t object_became_ready; ! 1582: ! 1583: XPR(XPR_MEMORY_OBJECT, ! 1584: "m_o_set_attr_com, object 0x%X flg %x strat %d\n", ! 1585: (integer_t)object, (may_cache&1)|((temporary&1)<1), copy_strategy, 0, 0); ! 1586: ! 1587: if (object == VM_OBJECT_NULL) ! 1588: return(KERN_INVALID_ARGUMENT); ! 1589: ! 1590: /* ! 1591: * Verify the attributes of importance ! 1592: */ ! 1593: ! 1594: switch(copy_strategy) { ! 1595: case MEMORY_OBJECT_COPY_NONE: ! 1596: case MEMORY_OBJECT_COPY_DELAY: ! 1597: break; ! 1598: default: ! 1599: vm_object_deallocate(object); ! 1600: return(KERN_INVALID_ARGUMENT); ! 1601: } ! 1602: ! 1603: #if !ADVISORY_PAGEOUT ! 1604: if (silent_overwrite || advisory_pageout) { ! 1605: vm_object_deallocate(object); ! 1606: return(KERN_INVALID_ARGUMENT); ! 1607: } ! 1608: #endif /* !ADVISORY_PAGEOUT */ ! 1609: if (may_cache) ! 1610: may_cache = TRUE; ! 1611: if (temporary) ! 1612: temporary = TRUE; ! 1613: if (cluster_size != 0) { ! 1614: int pages_per_cluster; ! 1615: pages_per_cluster = atop(cluster_size); ! 1616: /* ! 1617: * Cluster size must be integral multiple of page size, ! 1618: * and be a power of 2 number of pages. ! 1619: */ ! 1620: if ((cluster_size & (PAGE_SIZE-1)) || ! 1621: ((pages_per_cluster-1) & pages_per_cluster)) { ! 1622: vm_object_deallocate(object); ! 1623: return KERN_INVALID_ARGUMENT; ! 1624: } ! 1625: } ! 1626: ! 1627: vm_object_lock(object); ! 1628: ! 1629: /* ! 1630: * Copy the attributes ! 1631: */ ! 1632: assert(!object->internal); ! 1633: object_became_ready = !object->pager_ready; ! 1634: object->copy_strategy = copy_strategy; ! 1635: object->can_persist = may_cache; ! 1636: object->temporary = temporary; ! 1637: object->silent_overwrite = silent_overwrite; ! 1638: object->advisory_pageout = advisory_pageout; ! 1639: if (cluster_size == 0) ! 1640: cluster_size = PAGE_SIZE; ! 1641: object->cluster_size = cluster_size; ! 1642: ! 1643: assert(cluster_size >= PAGE_SIZE && ! 1644: cluster_size % PAGE_SIZE == 0); ! 1645: ! 1646: /* ! 1647: * Wake up anyone waiting for the ready attribute ! 1648: * to become asserted. ! 1649: */ ! 1650: ! 1651: if (object_became_ready) { ! 1652: object->pager_ready = TRUE; ! 1653: vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY); ! 1654: } ! 1655: ! 1656: vm_object_unlock(object); ! 1657: ! 1658: vm_object_deallocate(object); ! 1659: ! 1660: return(KERN_SUCCESS); ! 1661: } ! 1662: ! 1663: /* ! 1664: * Set the memory object attribute as provided. ! 1665: * ! 1666: * XXX This routine cannot be completed until the vm_msync, clean ! 1667: * in place, and cluster work is completed. See ifdef notyet ! 1668: * below and note that memory_object_set_attributes_common() ! 1669: * may have to be expanded. ! 1670: */ ! 1671: kern_return_t ! 1672: memory_object_change_attributes( ! 1673: vm_object_t object, ! 1674: memory_object_flavor_t flavor, ! 1675: memory_object_info_t attributes, ! 1676: mach_msg_type_number_t count, ! 1677: ipc_port_t reply_to, ! 1678: mach_msg_type_name_t reply_to_type) ! 1679: { ! 1680: kern_return_t result = KERN_SUCCESS; ! 1681: boolean_t temporary; ! 1682: boolean_t may_cache; ! 1683: boolean_t invalidate; ! 1684: vm_size_t cluster_size; ! 1685: memory_object_copy_strategy_t copy_strategy; ! 1686: boolean_t silent_overwrite; ! 1687: boolean_t advisory_pageout; ! 1688: ! 1689: if (object == VM_OBJECT_NULL) ! 1690: return(KERN_INVALID_ARGUMENT); ! 1691: ! 1692: vm_object_lock(object); ! 1693: temporary = object->temporary; ! 1694: may_cache = object->can_persist; ! 1695: copy_strategy = object->copy_strategy; ! 1696: silent_overwrite = object->silent_overwrite; ! 1697: advisory_pageout = object->advisory_pageout; ! 1698: #if notyet ! 1699: invalidate = object->invalidate; ! 1700: #endif ! 1701: cluster_size = object->cluster_size; ! 1702: vm_object_unlock(object); ! 1703: ! 1704: switch (flavor) { ! 1705: case OLD_MEMORY_OBJECT_BEHAVIOR_INFO: ! 1706: { ! 1707: old_memory_object_behave_info_t behave; ! 1708: ! 1709: if (count != OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) { ! 1710: result = KERN_INVALID_ARGUMENT; ! 1711: break; ! 1712: } ! 1713: ! 1714: behave = (old_memory_object_behave_info_t) attributes; ! 1715: ! 1716: temporary = behave->temporary; ! 1717: invalidate = behave->invalidate; ! 1718: copy_strategy = behave->copy_strategy; ! 1719: ! 1720: break; ! 1721: } ! 1722: ! 1723: case MEMORY_OBJECT_BEHAVIOR_INFO: ! 1724: { ! 1725: memory_object_behave_info_t behave; ! 1726: ! 1727: if (count != MEMORY_OBJECT_BEHAVE_INFO_COUNT) { ! 1728: result = KERN_INVALID_ARGUMENT; ! 1729: break; ! 1730: } ! 1731: ! 1732: behave = (memory_object_behave_info_t) attributes; ! 1733: ! 1734: temporary = behave->temporary; ! 1735: invalidate = behave->invalidate; ! 1736: copy_strategy = behave->copy_strategy; ! 1737: silent_overwrite = behave->silent_overwrite; ! 1738: advisory_pageout = behave->advisory_pageout; ! 1739: break; ! 1740: } ! 1741: ! 1742: case MEMORY_OBJECT_PERFORMANCE_INFO: ! 1743: { ! 1744: memory_object_perf_info_t perf; ! 1745: ! 1746: if (count != MEMORY_OBJECT_PERF_INFO_COUNT) { ! 1747: result = KERN_INVALID_ARGUMENT; ! 1748: break; ! 1749: } ! 1750: ! 1751: perf = (memory_object_perf_info_t) attributes; ! 1752: ! 1753: may_cache = perf->may_cache; ! 1754: cluster_size = round_page(perf->cluster_size); ! 1755: ! 1756: break; ! 1757: } ! 1758: ! 1759: case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO: ! 1760: { ! 1761: old_memory_object_attr_info_t attr; ! 1762: ! 1763: if (count != OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) { ! 1764: result = KERN_INVALID_ARGUMENT; ! 1765: break; ! 1766: } ! 1767: ! 1768: attr = (old_memory_object_attr_info_t) attributes; ! 1769: ! 1770: may_cache = attr->may_cache; ! 1771: copy_strategy = attr->copy_strategy; ! 1772: cluster_size = page_size; ! 1773: ! 1774: break; ! 1775: } ! 1776: ! 1777: case MEMORY_OBJECT_ATTRIBUTE_INFO: ! 1778: { ! 1779: memory_object_attr_info_t attr; ! 1780: ! 1781: if (count != MEMORY_OBJECT_ATTR_INFO_COUNT) { ! 1782: result = KERN_INVALID_ARGUMENT; ! 1783: break; ! 1784: } ! 1785: ! 1786: attr = (memory_object_attr_info_t) attributes; ! 1787: ! 1788: copy_strategy = attr->copy_strategy; ! 1789: may_cache = attr->may_cache_object; ! 1790: cluster_size = attr->cluster_size; ! 1791: temporary = attr->temporary; ! 1792: ! 1793: break; ! 1794: } ! 1795: ! 1796: default: ! 1797: result = KERN_INVALID_ARGUMENT; ! 1798: break; ! 1799: } ! 1800: ! 1801: if (result != KERN_SUCCESS) { ! 1802: vm_object_deallocate(object); ! 1803: return(result); ! 1804: } ! 1805: ! 1806: if (copy_strategy == MEMORY_OBJECT_COPY_TEMPORARY) { ! 1807: copy_strategy = MEMORY_OBJECT_COPY_DELAY; ! 1808: temporary = TRUE; ! 1809: } else { ! 1810: temporary = FALSE; ! 1811: } ! 1812: ! 1813: /* ! 1814: * Do the work and throw away our object reference. It ! 1815: * is important that the object reference be deallocated ! 1816: * BEFORE sending the reply. The whole point of the reply ! 1817: * is that it shows up after the terminate message that ! 1818: * may be generated by setting the object uncacheable. ! 1819: * ! 1820: * XXX may_cache may become a tri-valued variable to handle ! 1821: * XXX uncache if not in use. ! 1822: */ ! 1823: result = memory_object_set_attributes_common(object, ! 1824: may_cache, ! 1825: copy_strategy, ! 1826: temporary, ! 1827: cluster_size, ! 1828: silent_overwrite, ! 1829: advisory_pageout); ! 1830: ! 1831: if (IP_VALID(reply_to)) { ! 1832: /* consumes our naked send-once/send right for reply_to */ ! 1833: (void) memory_object_change_completed(reply_to, reply_to_type, ! 1834: object->alive ? ! 1835: object->pager_request : PAGER_REQUEST_NULL, ! 1836: flavor); ! 1837: } ! 1838: ! 1839: return(result); ! 1840: } ! 1841: ! 1842: kern_return_t ! 1843: memory_object_get_attributes( ! 1844: vm_object_t object, ! 1845: memory_object_flavor_t flavor, ! 1846: memory_object_info_t attributes, /* pointer to OUT array */ ! 1847: mach_msg_type_number_t *count) /* IN/OUT */ ! 1848: { ! 1849: kern_return_t ret = KERN_SUCCESS; ! 1850: ! 1851: if (object == VM_OBJECT_NULL) ! 1852: return(KERN_INVALID_ARGUMENT); ! 1853: ! 1854: vm_object_lock(object); ! 1855: ! 1856: switch (flavor) { ! 1857: case OLD_MEMORY_OBJECT_BEHAVIOR_INFO: ! 1858: { ! 1859: old_memory_object_behave_info_t behave; ! 1860: ! 1861: if (*count < OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) { ! 1862: ret = KERN_INVALID_ARGUMENT; ! 1863: break; ! 1864: } ! 1865: ! 1866: behave = (old_memory_object_behave_info_t) attributes; ! 1867: behave->copy_strategy = object->copy_strategy; ! 1868: behave->temporary = object->temporary; ! 1869: #if notyet /* remove when vm_msync complies and clean in place fini */ ! 1870: behave->invalidate = object->invalidate; ! 1871: #else ! 1872: behave->invalidate = FALSE; ! 1873: #endif ! 1874: ! 1875: *count = OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT; ! 1876: break; ! 1877: } ! 1878: ! 1879: case MEMORY_OBJECT_BEHAVIOR_INFO: ! 1880: { ! 1881: memory_object_behave_info_t behave; ! 1882: ! 1883: if (*count < MEMORY_OBJECT_BEHAVE_INFO_COUNT) { ! 1884: ret = KERN_INVALID_ARGUMENT; ! 1885: break; ! 1886: } ! 1887: ! 1888: behave = (memory_object_behave_info_t) attributes; ! 1889: behave->copy_strategy = object->copy_strategy; ! 1890: behave->temporary = object->temporary; ! 1891: #if notyet /* remove when vm_msync complies and clean in place fini */ ! 1892: behave->invalidate = object->invalidate; ! 1893: #else ! 1894: behave->invalidate = FALSE; ! 1895: #endif ! 1896: behave->advisory_pageout = object->advisory_pageout; ! 1897: behave->silent_overwrite = object->silent_overwrite; ! 1898: *count = MEMORY_OBJECT_BEHAVE_INFO_COUNT; ! 1899: break; ! 1900: } ! 1901: ! 1902: case MEMORY_OBJECT_PERFORMANCE_INFO: ! 1903: { ! 1904: memory_object_perf_info_t perf; ! 1905: ! 1906: if (*count < MEMORY_OBJECT_PERF_INFO_COUNT) { ! 1907: ret = KERN_INVALID_ARGUMENT; ! 1908: break; ! 1909: } ! 1910: ! 1911: perf = (memory_object_perf_info_t) attributes; ! 1912: perf->cluster_size = object->cluster_size; ! 1913: perf->may_cache = object->can_persist; ! 1914: ! 1915: *count = MEMORY_OBJECT_PERF_INFO_COUNT; ! 1916: break; ! 1917: } ! 1918: ! 1919: case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO: ! 1920: { ! 1921: old_memory_object_attr_info_t attr; ! 1922: ! 1923: if (*count < OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) { ! 1924: ret = KERN_INVALID_ARGUMENT; ! 1925: break; ! 1926: } ! 1927: ! 1928: attr = (old_memory_object_attr_info_t) attributes; ! 1929: attr->may_cache = object->can_persist; ! 1930: attr->copy_strategy = object->copy_strategy; ! 1931: ! 1932: *count = OLD_MEMORY_OBJECT_ATTR_INFO_COUNT; ! 1933: break; ! 1934: } ! 1935: ! 1936: case MEMORY_OBJECT_ATTRIBUTE_INFO: ! 1937: { ! 1938: memory_object_attr_info_t attr; ! 1939: ! 1940: if (*count < MEMORY_OBJECT_ATTR_INFO_COUNT) { ! 1941: ret = KERN_INVALID_ARGUMENT; ! 1942: break; ! 1943: } ! 1944: ! 1945: attr = (memory_object_attr_info_t) attributes; ! 1946: attr->copy_strategy = object->copy_strategy; ! 1947: attr->cluster_size = object->cluster_size; ! 1948: attr->may_cache_object = object->can_persist; ! 1949: attr->temporary = object->temporary; ! 1950: ! 1951: *count = MEMORY_OBJECT_ATTR_INFO_COUNT; ! 1952: break; ! 1953: } ! 1954: ! 1955: default: ! 1956: ret = KERN_INVALID_ARGUMENT; ! 1957: break; ! 1958: } ! 1959: ! 1960: vm_object_unlock(object); ! 1961: ! 1962: vm_object_deallocate(object); ! 1963: ! 1964: return(ret); ! 1965: } ! 1966: ! 1967: int vm_stat_discard_cleared_reply = 0; ! 1968: int vm_stat_discard_cleared_unset = 0; ! 1969: int vm_stat_discard_cleared_too_late = 0; ! 1970: ! 1971: #if ADVISORY_PAGEOUT ! 1972: kern_return_t ! 1973: memory_object_discard_reply( ! 1974: vm_object_t object, ! 1975: vm_offset_t requested_offset, ! 1976: vm_size_t requested_size, ! 1977: vm_offset_t discard_offset, ! 1978: vm_size_t discard_size, ! 1979: memory_object_return_t should_return, ! 1980: ipc_port_t reply_to, ! 1981: mach_msg_type_name_t reply_to_type) ! 1982: { ! 1983: vm_offset_t original_offset = discard_offset; ! 1984: vm_size_t original_size = discard_size; ! 1985: ! 1986: XPR(XPR_MEMORY_OBJECT, ! 1987: "m_o_discard_reply, obj 0x%X req_off 0x%X req_size 0x%X " ! 1988: "discard_off 0x%X discard_size 0x%X\n", ! 1989: (integer_t) object, requested_offset, requested_size, ! 1990: discard_offset, discard_size); ! 1991: ! 1992: /* ! 1993: * Check for bogus arguments. ! 1994: */ ! 1995: if (object == VM_OBJECT_NULL) ! 1996: return KERN_INVALID_ARGUMENT; ! 1997: ! 1998: requested_size = round_page(requested_size); ! 1999: discard_size = round_page(discard_size); ! 2000: ! 2001: /* ! 2002: * Lock the object, and acquire a paging reference to ! 2003: * prevent the memory_object and control ports from ! 2004: * being destroyed. ! 2005: */ ! 2006: ! 2007: vm_object_lock(object); ! 2008: vm_object_paging_begin(object); ! 2009: discard_offset -= object->paging_offset; ! 2010: requested_offset -= object->paging_offset; ! 2011: ! 2012: if (discard_size != 0) { ! 2013: (void) memory_object_update(object, ! 2014: discard_offset, discard_size, ! 2015: should_return, TRUE, VM_PROT_ALL); ! 2016: } ! 2017: ! 2018: if (original_offset != requested_offset || ! 2019: discard_size != requested_size) { ! 2020: /* ! 2021: * We're not discarding the requested pages, so reset ! 2022: * the "discard_request" flag on the requested pages. ! 2023: * But don't rehabilitate more pages than we discard. ! 2024: */ ! 2025: while (requested_size != 0) { ! 2026: vm_page_t m; ! 2027: ! 2028: m = vm_page_lookup(object, requested_offset); ! 2029: if (m != VM_PAGE_NULL) { ! 2030: if (discard_size != 0) { ! 2031: if (m->discard_request) { ! 2032: m->discard_request = FALSE; ! 2033: vm_stat_discard_cleared_reply++; ! 2034: } else { ! 2035: vm_stat_discard_cleared_unset++; ! 2036: } ! 2037: } ! 2038: } else { ! 2039: vm_stat_discard_cleared_too_late++; ! 2040: } ! 2041: ! 2042: requested_offset += PAGE_SIZE; ! 2043: requested_size -= PAGE_SIZE; ! 2044: discard_size -= PAGE_SIZE; ! 2045: } ! 2046: } ! 2047: ! 2048: if (IP_VALID(reply_to)) { ! 2049: vm_object_unlock(object); ! 2050: ! 2051: /* consumes our naked send-once/send right for reply_to */ ! 2052: (void) memory_object_lock_completed(reply_to, reply_to_type, ! 2053: object->pager_request, ! 2054: original_offset, ! 2055: original_size); ! 2056: ! 2057: vm_object_lock(object); ! 2058: } ! 2059: ! 2060: vm_object_paging_end(object); ! 2061: vm_object_unlock(object); ! 2062: vm_object_deallocate(object); ! 2063: ! 2064: return KERN_SUCCESS; ! 2065: } ! 2066: #endif /* ADVISORY_PAGEOUT */ ! 2067: ! 2068: /* ! 2069: * vm_set_default_memory_manager(): ! 2070: * [Obsolete] ! 2071: */ ! 2072: kern_return_t ! 2073: vm_set_default_memory_manager( ! 2074: host_t host, ! 2075: ipc_port_t *default_manager) ! 2076: { ! 2077: return(host_default_memory_manager(host, default_manager, 4*PAGE_SIZE)); ! 2078: } ! 2079: ! 2080: /* ! 2081: * Routine: host_default_memory_manager ! 2082: * Purpose: ! 2083: * set/get the default memory manager port and default cluster ! 2084: * size. ! 2085: * ! 2086: * If successful, consumes the supplied naked send right. ! 2087: */ ! 2088: kern_return_t ! 2089: host_default_memory_manager( ! 2090: host_t host, ! 2091: ipc_port_t *default_manager, ! 2092: vm_size_t cluster_size) ! 2093: { ! 2094: ipc_port_t current_manager; ! 2095: ipc_port_t new_manager; ! 2096: ipc_port_t returned_manager; ! 2097: ! 2098: if (host == HOST_NULL) ! 2099: return(KERN_INVALID_HOST); ! 2100: ! 2101: new_manager = *default_manager; ! 2102: mutex_lock(&memory_manager_default_lock); ! 2103: current_manager = memory_manager_default; ! 2104: ! 2105: if (new_manager == IP_NULL) { ! 2106: /* ! 2107: * Retrieve the current value. ! 2108: */ ! 2109: ! 2110: returned_manager = ipc_port_copy_send(current_manager); ! 2111: } else { ! 2112: /* ! 2113: * Retrieve the current value, ! 2114: * and replace it with the supplied value. ! 2115: * We consume the supplied naked send right. ! 2116: */ ! 2117: ! 2118: returned_manager = current_manager; ! 2119: memory_manager_default = new_manager; ! 2120: if (cluster_size % PAGE_SIZE != 0) { ! 2121: #if 0 ! 2122: mutex_unlock(&memory_manager_default_lock); ! 2123: return KERN_INVALID_ARGUMENT; ! 2124: #else ! 2125: cluster_size = round_page(cluster_size); ! 2126: #endif ! 2127: } ! 2128: memory_manager_default_cluster = cluster_size; ! 2129: ! 2130: /* ! 2131: * In case anyone's been waiting for a memory ! 2132: * manager to be established, wake them up. ! 2133: */ ! 2134: ! 2135: thread_wakeup((event_t) &memory_manager_default); ! 2136: } ! 2137: ! 2138: mutex_unlock(&memory_manager_default_lock); ! 2139: ! 2140: *default_manager = returned_manager; ! 2141: return(KERN_SUCCESS); ! 2142: } ! 2143: ! 2144: /* ! 2145: * Routine: memory_manager_default_reference ! 2146: * Purpose: ! 2147: * Returns a naked send right for the default ! 2148: * memory manager. The returned right is always ! 2149: * valid (not IP_NULL or IP_DEAD). ! 2150: */ ! 2151: ! 2152: ipc_port_t ! 2153: memory_manager_default_reference( ! 2154: vm_size_t *cluster_size) ! 2155: { ! 2156: ipc_port_t current_manager; ! 2157: ! 2158: mutex_lock(&memory_manager_default_lock); ! 2159: ! 2160: while (current_manager = ipc_port_copy_send(memory_manager_default), ! 2161: !IP_VALID(current_manager)) { ! 2162: thread_sleep_mutex((event_t) &memory_manager_default, ! 2163: &memory_manager_default_lock, THREAD_UNINT); ! 2164: mutex_lock(&memory_manager_default_lock); ! 2165: } ! 2166: *cluster_size = memory_manager_default_cluster; ! 2167: ! 2168: mutex_unlock(&memory_manager_default_lock); ! 2169: ! 2170: return current_manager; ! 2171: } ! 2172: ! 2173: /* ! 2174: * Routine: memory_manager_default_port ! 2175: * Purpose: ! 2176: * Returns true if the receiver for the port ! 2177: * is the default memory manager. ! 2178: * ! 2179: * This is a hack to let ds_read_done ! 2180: * know when it should keep memory wired. ! 2181: */ ! 2182: ! 2183: boolean_t ! 2184: memory_manager_default_port( ! 2185: ipc_port_t port) ! 2186: { ! 2187: ipc_port_t current; ! 2188: boolean_t result; ! 2189: ! 2190: mutex_lock(&memory_manager_default_lock); ! 2191: current = memory_manager_default; ! 2192: if (IP_VALID(current)) { ! 2193: /* ! 2194: * There is no point in bothering to lock ! 2195: * both ports, which would be painful to do. ! 2196: * If the receive rights are moving around, ! 2197: * we might be inaccurate. ! 2198: */ ! 2199: ! 2200: result = port->ip_receiver == current->ip_receiver; ! 2201: } else ! 2202: result = FALSE; ! 2203: mutex_unlock(&memory_manager_default_lock); ! 2204: ! 2205: return result; ! 2206: } ! 2207: ! 2208: /* ! 2209: * Routine: memory_manager_default_check ! 2210: * ! 2211: * Purpose: ! 2212: * Check whether a default memory manager has been set ! 2213: * up yet, or not. Returns KERN_SUCCESS if dmm exists, ! 2214: * and KERN_FAILURE if dmm does not exist. ! 2215: * ! 2216: * If there is no default memory manager, log an error, ! 2217: * but only the first time. ! 2218: * ! 2219: */ ! 2220: kern_return_t ! 2221: memory_manager_default_check(void) ! 2222: { ! 2223: ipc_port_t current; ! 2224: ! 2225: mutex_lock(&memory_manager_default_lock); ! 2226: current = memory_manager_default; ! 2227: if (!IP_VALID(current)) { ! 2228: static boolean_t logged; /* initialized to 0 */ ! 2229: boolean_t complain = !logged; ! 2230: logged = TRUE; ! 2231: mutex_unlock(&memory_manager_default_lock); ! 2232: if (complain) ! 2233: printf("Warning: No default memory manager\n"); ! 2234: return(KERN_FAILURE); ! 2235: } else { ! 2236: mutex_unlock(&memory_manager_default_lock); ! 2237: return(KERN_SUCCESS); ! 2238: } ! 2239: } ! 2240: ! 2241: void ! 2242: memory_manager_default_init(void) ! 2243: { ! 2244: memory_manager_default = IP_NULL; ! 2245: mutex_init(&memory_manager_default_lock, ETAP_VM_MEMMAN); ! 2246: } ! 2247: ! 2248: ! 2249: void ! 2250: memory_object_kill_pages( ! 2251: vm_object_t object, ! 2252: vm_offset_t offset, ! 2253: vm_size_t size) ! 2254: { ! 2255: register vm_page_t m; ! 2256: ! 2257: /* ! 2258: * entered with object lock held, acquire a paging reference to ! 2259: * prevent the memory_object and control ports from ! 2260: * being destroyed. ! 2261: */ ! 2262: vm_object_paging_begin(object); ! 2263: ! 2264: for (; ! 2265: size != 0; ! 2266: size -= PAGE_SIZE, offset += PAGE_SIZE) ! 2267: { ! 2268: if ((m = vm_page_lookup(object, offset)) != VM_PAGE_NULL) { ! 2269: if (memory_object_lock_page(m, MEMORY_OBJECT_RETURN_NONE, TRUE, VM_PROT_NO_CHANGE) == ! 2270: MEMORY_OBJECT_LOCK_RESULT_DONE) ! 2271: vm_external_state_clr(object->existence_map, offset); ! 2272: } ! 2273: } ! 2274: vm_object_paging_end(object); ! 2275: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.