![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to dp_memory_object.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple XNU] / XNU / osfmk / default_pager|
Return to dp_memory_object.c CVS log | Up to [Apple XNU] / XNU / osfmk / default_pager |
1.1 ! root 1: /* ! 2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. ! 3: * ! 4: * @APPLE_LICENSE_HEADER_START@ ! 5: * ! 6: * The contents of this file constitute Original Code as defined in and ! 7: * are subject to the Apple Public Source License Version 1.1 (the ! 8: * "License"). You may not use this file except in compliance with the ! 9: * License. Please obtain a copy of the License at ! 10: * http://www.apple.com/publicsource and read it before using this file. ! 11: * ! 12: * This Original Code and all software distributed under the License are ! 13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER ! 14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, ! 15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, ! 16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the ! 17: * License for the specific language governing rights and limitations ! 18: * under the License. ! 19: * ! 20: * @APPLE_LICENSE_HEADER_END@ ! 21: */ ! 22: /* ! 23: * @OSF_COPYRIGHT@ ! 24: */ ! 25: /* ! 26: * Mach Operating System ! 27: * Copyright (c) 1991,1990,1989 Carnegie Mellon University ! 28: * All Rights Reserved. ! 29: * ! 30: * Permission to use, copy, modify and distribute this software and its ! 31: * documentation is hereby granted, provided that both the copyright ! 32: * notice and this permission notice appear in all copies of the ! 33: * software, derivative works or modified versions, and any portions ! 34: * thereof, and that both notices appear in supporting documentation. ! 35: * ! 36: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" ! 37: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR ! 38: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. ! 39: * ! 40: * Carnegie Mellon requests users of this software to return to ! 41: * ! 42: * Software Distribution Coordinator or [email protected] ! 43: * School of Computer Science ! 44: * Carnegie Mellon University ! 45: * Pittsburgh PA 15213-3890 ! 46: * ! 47: * any improvements or extensions that they make and grant Carnegie Mellon ! 48: * the rights to redistribute these changes. ! 49: */ ! 50: ! 51: /* ! 52: * Default Pager. ! 53: * Memory Object Management. ! 54: */ ! 55: ! 56: #include "default_pager_internal.h" ! 57: #include <mach/memory_object_server.h> ! 58: #include <vm/vm_pageout.h> /* include for upl_t */ ! 59: ! 60: ! 61: /* ! 62: * List of all vstructs. A specific vstruct is ! 63: * found directly via its port, this list is ! 64: * only used for monitoring purposes by the ! 65: * default_pager_object* calls and by ps_delete ! 66: * when abstract memory objects must be scanned ! 67: * to remove any live storage on a segment which ! 68: * is to be removed. ! 69: */ ! 70: struct vstruct_list_head vstruct_list; ! 71: ! 72: void vstruct_list_insert(vstruct_t vs); /* forward */ ! 73: ! 74: void ! 75: vstruct_list_insert( ! 76: vstruct_t vs) ! 77: { ! 78: VSL_LOCK(); ! 79: queue_enter(&vstruct_list.vsl_queue, vs, vstruct_t, vs_links); ! 80: vstruct_list.vsl_count++; ! 81: VSL_UNLOCK(); ! 82: } ! 83: ! 84: void vstruct_list_delete(vstruct_t vs); /* forward */ ! 85: ! 86: void ! 87: vstruct_list_delete( ! 88: vstruct_t vs) ! 89: { ! 90: queue_remove(&vstruct_list.vsl_queue, vs, vstruct_t, vs_links); ! 91: vstruct_list.vsl_count--; ! 92: } ! 93: ! 94: /* ! 95: * We use the sequence numbers on requests to regulate ! 96: * our parallelism. In general, we allow multiple reads and writes ! 97: * to proceed in parallel, with the exception that reads must ! 98: * wait for previous writes to finish. (Because the kernel might ! 99: * generate a data-request for a page on the heels of a data-write ! 100: * for the same page, and we must avoid returning stale data.) ! 101: * terminate requests wait for proceeding reads and writes to finish. ! 102: */ ! 103: ! 104: unsigned int default_pager_total = 0; /* debugging */ ! 105: unsigned int default_pager_wait_seqno = 0; /* debugging */ ! 106: unsigned int default_pager_wait_read = 0; /* debugging */ ! 107: unsigned int default_pager_wait_write = 0; /* debugging */ ! 108: unsigned int default_pager_wait_refs = 0; /* debugging */ ! 109: ! 110: void vs_async_wait(vstruct_t); /* forward */ ! 111: ! 112: void ! 113: vs_async_wait( ! 114: vstruct_t vs) ! 115: { ! 116: static char here[] = "vs_async_wait"; ! 117: ! 118: ASSERT(vs->vs_async_pending >= 0); ! 119: while (vs->vs_async_pending > 0) { ! 120: vs->vs_waiting_async = TRUE; ! 121: assert_wait(&vs->vs_waiting_async, THREAD_UNINT); ! 122: VS_UNLOCK(vs); ! 123: thread_block((void (*)(void))0); ! 124: VS_LOCK(vs); ! 125: } ! 126: ASSERT(vs->vs_async_pending == 0); ! 127: } ! 128: ! 129: #if PARALLEL ! 130: void vs_lock(vstruct_t, mach_port_seqno_t); ! 131: void vs_unlock(vstruct_t); ! 132: void vs_start_read(vstruct_t); ! 133: void vs_wait_for_readers(vstruct_t); ! 134: void vs_finish_read(vstruct_t); ! 135: void vs_start_write(vstruct_t); ! 136: void vs_wait_for_writers(vstruct_t); ! 137: void vs_finish_write(vstruct_t); ! 138: void vs_wait_for_refs(vstruct_t); ! 139: void vs_finish_refs(vstruct_t); ! 140: ! 141: /* ! 142: * Waits for correct sequence number. Leaves pager locked. ! 143: * JMM - Sequence numbers guarantee ordering, but in a preemptible ! 144: * kernel, they are generated without locks, and so their ! 145: * generation order is undefined (and therefore unreliable). ! 146: * Since we ned to fix this anyway, and I needed to get rid ! 147: * rid of asymmetry in the interface definitions, I have ! 148: * punted this to here. ! 149: */ ! 150: void ! 151: vs_lock( ! 152: vstruct_t vs, ! 153: mach_port_seqno_t seqno) ! 154: { ! 155: default_pager_total++; ! 156: VS_LOCK(vs); ! 157: ! 158: seqno = vs->vs_next_seqno++; ! 159: ! 160: while (vs->vs_seqno != seqno) { ! 161: default_pager_wait_seqno++; ! 162: vs->vs_waiting_seqno = TRUE; ! 163: assert_wait(&vs->vs_waiting_seqno, THREAD_UNINT); ! 164: VS_UNLOCK(vs); ! 165: thread_block((void (*)(void))0); ! 166: VS_LOCK(vs); ! 167: } ! 168: } ! 169: ! 170: /* ! 171: * Increments sequence number and unlocks pager. ! 172: */ ! 173: void ! 174: vs_unlock(vstruct_t vs) ! 175: { ! 176: boolean_t need_wakeups = vs->vs_waiting_seqno; ! 177: ! 178: vs->vs_waiting_seqno = FALSE; ! 179: vs->vs_seqno++; ! 180: VS_UNLOCK(vs); ! 181: if (need_wakeups) ! 182: thread_wakeup(&vs->vs_waiting_seqno); ! 183: } ! 184: ! 185: /* ! 186: * Start a read - one more reader. Pager must be locked. ! 187: */ ! 188: void ! 189: vs_start_read( ! 190: vstruct_t vs) ! 191: { ! 192: vs->vs_readers++; ! 193: } ! 194: ! 195: /* ! 196: * Wait for readers. Unlocks and relocks pager if wait needed. ! 197: */ ! 198: void ! 199: vs_wait_for_readers( ! 200: vstruct_t vs) ! 201: { ! 202: while (vs->vs_readers != 0) { ! 203: default_pager_wait_read++; ! 204: vs->vs_waiting_read = TRUE; ! 205: assert_wait(&vs->vs_waiting_read, THREAD_UNINT); ! 206: VS_UNLOCK(vs); ! 207: thread_block((void (*)(void))0); ! 208: VS_LOCK(vs); ! 209: } ! 210: } ! 211: ! 212: /* ! 213: * Finish a read. Pager is unlocked and returns unlocked. ! 214: */ ! 215: void ! 216: vs_finish_read( ! 217: vstruct_t vs) ! 218: { ! 219: VS_LOCK(vs); ! 220: if (--vs->vs_readers == 0) { ! 221: boolean_t need_wakeups = vs->vs_waiting_read; ! 222: ! 223: vs->vs_waiting_read = FALSE; ! 224: VS_UNLOCK(vs); ! 225: if (need_wakeups) ! 226: thread_wakeup(&vs->vs_waiting_read); ! 227: } else ! 228: VS_UNLOCK(vs); ! 229: } ! 230: ! 231: /* ! 232: * Start a write - one more writer. Pager must be locked. ! 233: */ ! 234: void ! 235: vs_start_write( ! 236: vstruct_t vs) ! 237: { ! 238: vs->vs_writers++; ! 239: } ! 240: ! 241: /* ! 242: * Wait for writers. Unlocks and relocks pager if wait needed. ! 243: */ ! 244: void ! 245: vs_wait_for_writers( ! 246: vstruct_t vs) ! 247: { ! 248: while (vs->vs_writers != 0) { ! 249: default_pager_wait_write++; ! 250: vs->vs_waiting_write = TRUE; ! 251: assert_wait(&vs->vs_waiting_write, THREAD_UNINT); ! 252: VS_UNLOCK(vs); ! 253: thread_block((void (*)(void))0); ! 254: VS_LOCK(vs); ! 255: } ! 256: vs_async_wait(vs); ! 257: } ! 258: ! 259: /* This is to be used for the transfer from segment code ONLY */ ! 260: /* The transfer code holds off vs destruction by keeping the */ ! 261: /* vs_async_wait count non-zero. It will not ocnflict with */ ! 262: /* other writers on an async basis because it only writes on */ ! 263: /* a cluster basis into fresh (as of sync time) cluster locations */ ! 264: void ! 265: vs_wait_for_sync_writers( ! 266: vstruct_t vs) ! 267: { ! 268: while (vs->vs_writers != 0) { ! 269: default_pager_wait_write++; ! 270: vs->vs_waiting_write = TRUE; ! 271: assert_wait(&vs->vs_waiting_write, THREAD_UNINT); ! 272: VS_UNLOCK(vs); ! 273: thread_block((void (*)(void))0); ! 274: VS_LOCK(vs); ! 275: } ! 276: } ! 277: ! 278: ! 279: /* ! 280: * Finish a write. Pager is unlocked and returns unlocked. ! 281: */ ! 282: void ! 283: vs_finish_write( ! 284: vstruct_t vs) ! 285: { ! 286: VS_LOCK(vs); ! 287: if (--vs->vs_writers == 0) { ! 288: boolean_t need_wakeups = vs->vs_waiting_write; ! 289: ! 290: vs->vs_waiting_write = FALSE; ! 291: VS_UNLOCK(vs); ! 292: if (need_wakeups) ! 293: thread_wakeup(&vs->vs_waiting_write); ! 294: } else ! 295: VS_UNLOCK(vs); ! 296: } ! 297: ! 298: /* ! 299: * Wait for concurrent default_pager_objects. ! 300: * Unlocks and relocks pager if wait needed. ! 301: */ ! 302: void ! 303: vs_wait_for_refs( ! 304: vstruct_t vs) ! 305: { ! 306: while (vs->vs_name_refs == 0) { ! 307: default_pager_wait_refs++; ! 308: vs->vs_waiting_refs = TRUE; ! 309: assert_wait(&vs->vs_waiting_refs, THREAD_UNINT); ! 310: VS_UNLOCK(vs); ! 311: thread_block((void (*)(void))0); ! 312: VS_LOCK(vs); ! 313: } ! 314: } ! 315: ! 316: /* ! 317: * Finished creating name refs - wake up waiters. ! 318: */ ! 319: void ! 320: vs_finish_refs( ! 321: vstruct_t vs) ! 322: { ! 323: boolean_t need_wakeups = vs->vs_waiting_refs; ! 324: vs->vs_waiting_refs = FALSE; ! 325: if (need_wakeups) ! 326: thread_wakeup(&vs->vs_waiting_refs); ! 327: } ! 328: ! 329: #else /* PARALLEL */ ! 330: ! 331: #define vs_lock(vs,seqno) ! 332: #define vs_unlock(vs) ! 333: #define vs_start_read(vs) ! 334: #define vs_wait_for_readers(vs) ! 335: #define vs_finish_read(vs) ! 336: #define vs_start_write(vs) ! 337: #define vs_wait_for_writers(vs) ! 338: #define vs_wait_for_sync_writers(vs) ! 339: #define vs_finish_write(vs) ! 340: #define vs_wait_for_refs(vs) ! 341: #define vs_finish_refs(vs) ! 342: ! 343: #endif /* PARALLEL */ ! 344: ! 345: vstruct_t vs_object_create(vm_size_t); /* forward */ ! 346: ! 347: vstruct_t ! 348: vs_object_create( ! 349: vm_size_t size) ! 350: { ! 351: vstruct_t vs; ! 352: static char here[] = "vs_object_create"; ! 353: ! 354: /* ! 355: * Allocate a vstruct. If there are any problems, then report them ! 356: * to the console. ! 357: */ ! 358: vs = ps_vstruct_create(size); ! 359: if (vs == VSTRUCT_NULL) { ! 360: dprintf(("vs_object_create: unable to allocate %s\n", ! 361: "-- either run swapon command or reboot")); ! 362: return VSTRUCT_NULL; ! 363: } ! 364: ! 365: return vs; ! 366: } ! 367: ! 368: mach_port_urefs_t default_pager_max_urefs = 10000; ! 369: ! 370: /* ! 371: * Check user reference count on memory object control port. ! 372: * Vstruct must be locked. ! 373: * Unlocks and re-locks vstruct if needs to call kernel. ! 374: */ ! 375: void vs_check_request(vstruct_t, MACH_PORT_FACE); /* forward */ ! 376: ! 377: void ! 378: vs_check_request( ! 379: vstruct_t vs, ! 380: MACH_PORT_FACE control_port) ! 381: { ! 382: mach_port_delta_t delta; ! 383: kern_return_t kr; ! 384: static char here[] = "vs_check_request"; ! 385: ! 386: if (++vs->vs_control_refs > default_pager_max_urefs) { ! 387: delta = 1 - vs->vs_control_refs; ! 388: vs->vs_control_refs = 1; ! 389: ! 390: VS_UNLOCK(vs); ! 391: ! 392: /* ! 393: * Deallocate excess user references. ! 394: */ ! 395: ! 396: { ! 397: /* find a better interface for this, what will we use as a component */ ! 398: int i; ! 399: delta = -delta; ! 400: for(i=0; i<delta; i++) ! 401: ipc_port_release_send(control_port); ! 402: } ! 403: ! 404: VS_LOCK(vs); ! 405: } ! 406: } ! 407: ! 408: void default_pager_add(vstruct_t, boolean_t); /* forward */ ! 409: ! 410: void ! 411: default_pager_add( ! 412: vstruct_t vs, ! 413: boolean_t internal) ! 414: { ! 415: MACH_PORT_FACE mem_obj = vs->vs_mem_obj_port; ! 416: MACH_PORT_FACE pset; ! 417: mach_port_mscount_t sync; ! 418: MACH_PORT_FACE previous; ! 419: kern_return_t kr; ! 420: static char here[] = "default_pager_add"; ! 421: ! 422: /* ! 423: * The port currently has a make-send count of zero, ! 424: * because either we just created the port or we just ! 425: * received the port in a memory_object_create request. ! 426: */ ! 427: ! 428: if (internal) { ! 429: /* possibly generate an immediate no-senders notification */ ! 430: sync = 0; ! 431: pset = default_pager_internal_set; ! 432: } else { ! 433: /* delay notification till send right is created */ ! 434: sync = 1; ! 435: pset = default_pager_external_set; ! 436: } ! 437: ! 438: ipc_port_make_sonce(mem_obj); ! 439: ip_lock(mem_obj); /* unlocked in nsrequest below */ ! 440: ipc_port_nsrequest(mem_obj, sync, mem_obj, &previous); ! 441: } ! 442: ! 443: ! 444: /* ! 445: * Routine: dp_memory_object_create ! 446: * Purpose: ! 447: * Handle requests for memory objects from the ! 448: * kernel. ! 449: * Notes: ! 450: * Because we only give out the default memory ! 451: * manager port to the kernel, we don't have to ! 452: * be so paranoid about the contents. ! 453: */ ! 454: kern_return_t ! 455: dp_memory_object_create( ! 456: MACH_PORT_FACE dmm, ! 457: MACH_PORT_FACE *new_mem_obj, ! 458: vm_size_t new_size) ! 459: { ! 460: mach_port_seqno_t seqno; ! 461: vstruct_t vs; ! 462: MACH_PORT_FACE pager; ! 463: static char here[] = "memory_object_create"; ! 464: ! 465: assert(dmm == default_pager_default_port); ! 466: ! 467: vs = vs_object_create(new_size); ! 468: if (vs == VSTRUCT_NULL) ! 469: return KERN_RESOURCE_SHORTAGE; ! 470: ! 471: pager = *new_mem_obj = ipc_port_alloc_kernel(); ! 472: assert (pager != IP_NULL); ! 473: (void) ipc_port_make_send(pager); ! 474: ! 475: { ! 476: struct vstruct_alias *alias_struct; ! 477: ! 478: alias_struct = (struct vstruct_alias *) ! 479: kalloc(sizeof(struct vstruct_alias)); ! 480: if(alias_struct != NULL) { ! 481: alias_struct->vs = vs; ! 482: alias_struct->name = ISVS; ! 483: pager->alias = (int) alias_struct; ! 484: } ! 485: else Panic("Out of kernel memory"); ! 486: ! 487: /* JMM - Add binding to this pager under components */ ! 488: pager_mux_hash_insert(pager, &dp_memory_object_subsystem); ! 489: vs->vs_next_seqno = 0; ! 490: pager->ip_receiver = ipc_space_kernel; ! 491: } ! 492: ! 493: /* ! 494: * Set up associations between this port ! 495: * and this default_pager structure ! 496: */ ! 497: ! 498: vs->vs_mem_obj_port = pager; ! 499: ! 500: /* ! 501: * After this, other threads might receive requests ! 502: * for this memory object or find it in the port list. ! 503: */ ! 504: ! 505: vstruct_list_insert(vs); ! 506: default_pager_add(vs, TRUE); ! 507: ! 508: return KERN_SUCCESS; ! 509: } ! 510: ! 511: kern_return_t ! 512: dp_memory_object_init( ! 513: MACH_PORT_FACE mem_obj, ! 514: MACH_PORT_FACE control_port, ! 515: vm_size_t pager_page_size) ! 516: { ! 517: mach_port_seqno_t seqno; ! 518: vstruct_t vs; ! 519: static char here[] = "memory_object_init"; ! 520: ! 521: assert(pager_page_size == vm_page_size); ! 522: ! 523: vs_lookup(mem_obj, vs); ! 524: vs_lock(vs, seqno); ! 525: ! 526: if (vs->vs_control_port != MACH_PORT_NULL) ! 527: Panic("bad request"); ! 528: ! 529: vs->vs_control_port = control_port; ! 530: vs->vs_control_refs = 1; ! 531: vs->vs_object_name = MACH_PORT_NULL; ! 532: vs->vs_name_refs = 1; ! 533: ! 534: vs_unlock(vs); ! 535: ! 536: return KERN_SUCCESS; ! 537: } ! 538: ! 539: kern_return_t ! 540: dp_memory_object_synchronize( ! 541: MACH_PORT_FACE mem_obj, ! 542: MACH_PORT_FACE control_port, ! 543: vm_offset_t offset, ! 544: vm_offset_t length, ! 545: vm_sync_t flags) ! 546: { ! 547: mach_port_seqno_t seqno; ! 548: vstruct_t vs; ! 549: static char here[] = "memory_object_synchronize"; ! 550: ! 551: vs_lookup(mem_obj, vs); ! 552: vs_lock(vs, seqno); ! 553: vs_check_request(vs, control_port); ! 554: vs_unlock(vs); ! 555: ! 556: memory_object_synchronize_completed( ! 557: vm_object_lookup(control_port), ! 558: offset, length); ! 559: ! 560: return KERN_SUCCESS; ! 561: } ! 562: ! 563: kern_return_t ! 564: dp_memory_object_terminate( ! 565: MACH_PORT_FACE mem_obj, ! 566: MACH_PORT_FACE control_port) ! 567: { ! 568: mach_port_seqno_t seqno; ! 569: vstruct_t vs; ! 570: mach_port_urefs_t request_refs; ! 571: kern_return_t kr; ! 572: static char here[] = "memory_object_terminate"; ! 573: ! 574: /* ! 575: * control port is a receive right, not a send right. ! 576: */ ! 577: ! 578: vs_lookup(mem_obj, vs); ! 579: vs_lock(vs, seqno); ! 580: ! 581: /* ! 582: * Wait for read and write requests to terminate. ! 583: */ ! 584: ! 585: vs_wait_for_readers(vs); ! 586: vs_wait_for_writers(vs); ! 587: ! 588: /* ! 589: * After memory_object_terminate both memory_object_init ! 590: * and a no-senders notification are possible, so we need ! 591: * to clean up the request and name ports but leave ! 592: * the mem_obj port. ! 593: * ! 594: * A concurrent default_pager_objects might be allocating ! 595: * more references for the name port. In this case, ! 596: * we must first wait for it to finish. ! 597: */ ! 598: ! 599: vs_wait_for_refs(vs); ! 600: ! 601: vs->vs_control_port = MACH_PORT_NULL; ! 602: ! 603: /* a bit of special case ugliness here. Wakeup any waiting reads */ ! 604: /* these data requests had to be removed from the seqno traffic */ ! 605: /* based on a performance bottleneck with large memory objects */ ! 606: /* the problem will right itself with the new component based */ ! 607: /* synchronous interface. The new async will be able to return */ ! 608: /* failure during its sync phase. In the mean time ... */ ! 609: ! 610: thread_wakeup(&vs->vs_waiting_write); ! 611: thread_wakeup(&vs->vs_waiting_async); ! 612: ! 613: request_refs = vs->vs_control_refs; ! 614: vs->vs_control_refs = 0; ! 615: ! 616: vs->vs_object_name = MACH_PORT_NULL; ! 617: ! 618: assert(vs->vs_name_refs != 0); ! 619: vs->vs_name_refs = 0; ! 620: ! 621: vs_unlock(vs); ! 622: ! 623: /* ! 624: * Now we deallocate our various port rights. ! 625: */ ! 626: ! 627: { ! 628: int i; ! 629: for(i=0; i<request_refs; i++) ! 630: ipc_port_release_send(control_port); ! 631: } ! 632: if(control_port->alias != (int)NULL) ! 633: kfree((vm_offset_t) (control_port->alias), ! 634: sizeof(struct vstruct_alias)); ! 635: ipc_port_release_receive(control_port); ! 636: return KERN_SUCCESS; ! 637: } ! 638: ! 639: void ! 640: default_pager_no_senders( ! 641: MACH_PORT_FACE mem_obj, ! 642: mach_port_seqno_t seqno, ! 643: mach_port_mscount_t mscount) ! 644: { ! 645: vstruct_t vs; ! 646: static char here[] = "default_pager_no_senders"; ! 647: ! 648: /* ! 649: * Because we don't give out multiple send rights ! 650: * for a memory object, there can't be a race ! 651: * between getting a no-senders notification ! 652: * and creating a new send right for the object. ! 653: * Hence we don't keep track of mscount. ! 654: */ ! 655: ! 656: vs_lookup(mem_obj, vs); ! 657: vs_lock(vs, seqno); ! 658: vs_async_wait(vs); /* wait for pending async IO */ ! 659: ! 660: /* do not delete the vs structure until the referencing pointers */ ! 661: /* in the vstruct list have been expunged */ ! 662: ! 663: /* get VSL_LOCK out of order by using TRY mechanism */ ! 664: while(!VSL_LOCK_TRY()) { ! 665: VS_UNLOCK(vs); ! 666: VSL_LOCK(); ! 667: VSL_UNLOCK(); ! 668: VS_LOCK(vs); ! 669: vs_async_wait(vs); /* wait for pending async IO */ ! 670: } ! 671: /* ! 672: * We shouldn't get a no-senders notification ! 673: * when the kernel has the object cached. ! 674: */ ! 675: if (vs->vs_control_port != MACH_PORT_NULL) ! 676: Panic("bad request"); ! 677: ! 678: /* ! 679: * Unlock the pager (though there should be no one ! 680: * waiting for it). ! 681: */ ! 682: VS_UNLOCK(vs); ! 683: ! 684: /* ! 685: * Remove the memory object port association, and then ! 686: * the destroy the port itself. We must remove the object ! 687: * from the port list before deallocating the pager, ! 688: * because of default_pager_objects. ! 689: */ ! 690: vstruct_list_delete(vs); ! 691: ps_vstruct_dealloc(vs); ! 692: ! 693: /* ! 694: * Recover memory that we might have wasted because ! 695: * of name conflicts ! 696: */ ! 697: while (!queue_empty(&vstruct_list.vsl_leak_queue)) { ! 698: vs = (vstruct_t) queue_first(&vstruct_list.vsl_leak_queue); ! 699: queue_remove_first(&vstruct_list.vsl_leak_queue, vs, vstruct_t, ! 700: vs_links); ! 701: kfree((vm_offset_t) vs, sizeof *vs); ! 702: } ! 703: VSL_UNLOCK(); ! 704: } ! 705: ! 706: kern_return_t ! 707: dp_memory_object_data_request( ! 708: MACH_PORT_FACE mem_obj, ! 709: MACH_PORT_FACE reply_to, ! 710: vm_offset_t offset, ! 711: vm_size_t length, ! 712: vm_prot_t protection_required) ! 713: { ! 714: mach_port_seqno_t seqno; ! 715: vstruct_t vs; ! 716: static char here[] = "memory_object_data_request"; ! 717: ! 718: GSTAT(global_stats.gs_pagein_calls++); ! 719: ! 720: ! 721: /* CDY at this moment vs_lookup panics when presented with the wrong */ ! 722: /* port. As we are expanding this pager to support user interfaces */ ! 723: /* this should be changed to return kern_failure */ ! 724: vs_lookup(mem_obj, vs); ! 725: vs_lock(vs, seqno); ! 726: vs_check_request(vs, reply_to); ! 727: ! 728: /* We are going to relax the strict sequencing here for performance */ ! 729: /* reasons. We can do this because we know that the read and */ ! 730: /* write threads are different and we rely on synchronization */ ! 731: /* of read and write requests at the cache memory_object level */ ! 732: /* break out wait_for_writers, all of this goes away when */ ! 733: /* we get real control of seqno with the new component interface */ ! 734: if (vs->vs_writers != 0) { ! 735: /* you can't hold on to the seqno and go */ ! 736: /* to sleep like that */ ! 737: vs_unlock(vs); /* bump internal count of seqno */ ! 738: VS_LOCK(vs); ! 739: while (vs->vs_writers != 0) { ! 740: default_pager_wait_write++; ! 741: vs->vs_waiting_write = TRUE; ! 742: assert_wait(&vs->vs_waiting_write, THREAD_UNINT); ! 743: VS_UNLOCK(vs); ! 744: thread_block((void (*)(void))0); ! 745: VS_LOCK(vs); ! 746: vs_async_wait(vs); ! 747: } ! 748: if(vs->vs_control_port == MACH_PORT_NULL) { ! 749: VS_UNLOCK(vs); ! 750: return KERN_FAILURE; ! 751: } ! 752: vs_start_read(vs); ! 753: VS_UNLOCK(vs); ! 754: } else { ! 755: vs_start_read(vs); ! 756: vs_unlock(vs); ! 757: } ! 758: ! 759: /* ! 760: * Request must be on a page boundary and a multiple of pages. ! 761: */ ! 762: if ((offset & vm_page_mask) != 0 || (length & vm_page_mask) != 0) ! 763: Panic("bad alignment"); ! 764: ! 765: pvs_cluster_read(vs, offset, length); ! 766: ! 767: vs_finish_read(vs); ! 768: ! 769: return KERN_SUCCESS; ! 770: } ! 771: ! 772: /* ! 773: * memory_object_data_initialize: check whether we already have each page, and ! 774: * write it if we do not. The implementation is far from optimized, and ! 775: * also assumes that the default_pager is single-threaded. ! 776: */ ! 777: /* It is questionable whether or not a pager should decide what is relevant */ ! 778: /* and what is not in data sent from the kernel. Data initialize has been */ ! 779: /* changed to copy back all data sent to it in preparation for its eventual */ ! 780: /* merge with data return. It is the kernel that should decide what pages */ ! 781: /* to write back. As of the writing of this note, this is indeed the case */ ! 782: /* the kernel writes back one page at a time through this interface */ ! 783: ! 784: kern_return_t ! 785: dp_memory_object_data_initialize( ! 786: MACH_PORT_FACE mem_obj, ! 787: MACH_PORT_FACE control_port, ! 788: vm_offset_t offset, ! 789: pointer_t addr, ! 790: vm_size_t data_cnt) ! 791: { ! 792: mach_port_seqno_t seqno; ! 793: vstruct_t vs; ! 794: static char here[] = "memory_object_data_initialize"; ! 795: ! 796: #ifdef lint ! 797: control_port++; ! 798: #endif /* lint */ ! 799: ! 800: DEBUG(DEBUG_MO_EXTERNAL, ! 801: ("mem_obj=0x%x,offset=0x%x,cnt=0x%x\n", ! 802: (int)mem_obj, (int)offset, (int)data_cnt)); ! 803: GSTAT(global_stats.gs_pages_init += atop(data_cnt)); ! 804: ! 805: vs_lookup(mem_obj, vs); ! 806: vs_lock(vs, seqno); ! 807: vs_check_request(vs, control_port); ! 808: vs_start_write(vs); ! 809: vs_unlock(vs); ! 810: ! 811: /* ! 812: * Write the data via clustered writes. vs_cluster_write will ! 813: * loop if the address range specified crosses cluster ! 814: * boundaries. ! 815: */ ! 816: vs_cluster_write(vs, offset, addr, data_cnt, FALSE); ! 817: ! 818: vs_finish_write(vs); ! 819: ! 820: return KERN_SUCCESS; ! 821: } ! 822: ! 823: kern_return_t ! 824: dp_memory_object_lock_completed( ! 825: memory_object_t mem_obj, ! 826: MACH_PORT_FACE control_port, ! 827: vm_offset_t offset, ! 828: vm_size_t length) ! 829: { ! 830: mach_port_seqno_t seqno; ! 831: static char here[] = "memory_object_lock_completed"; ! 832: ! 833: #ifdef lint ! 834: mem_obj++; ! 835: seqno++; ! 836: control_port++; ! 837: offset++; ! 838: length++; ! 839: #endif /* lint */ ! 840: ! 841: Panic("illegal"); ! 842: return KERN_FAILURE; ! 843: } ! 844: ! 845: kern_return_t ! 846: dp_memory_object_data_unlock( ! 847: memory_object_t mem_obj, ! 848: MACH_PORT_FACE control_port, ! 849: vm_offset_t offset, ! 850: vm_size_t data_cnt, ! 851: vm_prot_t desired_access) ! 852: { ! 853: static char here[] = "memory_object_data_unlock"; ! 854: ! 855: Panic("illegal"); ! 856: return KERN_FAILURE; ! 857: } ! 858: ! 859: ! 860: kern_return_t ! 861: dp_memory_object_supply_completed( ! 862: memory_object_t mem_obj, ! 863: MACH_PORT_FACE control_port, ! 864: vm_offset_t offset, ! 865: vm_size_t length, ! 866: kern_return_t result, ! 867: vm_offset_t error_offset) ! 868: { ! 869: static char here[] = "memory_object_supply_completed"; ! 870: ! 871: Panic("illegal"); ! 872: return KERN_FAILURE; ! 873: } ! 874: ! 875: kern_return_t ! 876: dp_memory_object_data_return( ! 877: MACH_PORT_FACE mem_obj, ! 878: MACH_PORT_FACE control_port, ! 879: vm_offset_t offset, ! 880: pointer_t addr, ! 881: vm_size_t data_cnt, ! 882: boolean_t dirty, ! 883: boolean_t kernel_copy) ! 884: { ! 885: mach_port_seqno_t seqno; ! 886: vstruct_t vs; ! 887: vm_offset_t pageout_address; ! 888: static char here[] = "memory_object_data_return"; ! 889: ! 890: #ifdef lint ! 891: control_port++; ! 892: dirty++; ! 893: kernel_copy++; ! 894: #endif /* lint */ ! 895: ! 896: DEBUG(DEBUG_MO_EXTERNAL, ! 897: ("mem_obj=0x%x,offset=0x%x,addr=0x%xcnt=0x%x\n", ! 898: (int)mem_obj, (int)offset, (int)addr, (int)data_cnt)); ! 899: GSTAT(global_stats.gs_pageout_calls++); ! 900: ! 901: /* This routine is called by the pageout thread. The pageout thread */ ! 902: /* cannot be blocked by read activities unless the read activities */ ! 903: /* Therefore the grant of vs lock must be done on a try versus a */ ! 904: /* blocking basis. The code below relies on the fact that the */ ! 905: /* interface is synchronous. Should this interface be again async */ ! 906: /* for some type of pager in the future the pages will have to be */ ! 907: /* returned through a separate, asynchronous path. */ ! 908: ! 909: vs_lookup(mem_obj, vs); ! 910: ! 911: default_pager_total++; ! 912: if(!VS_TRY_LOCK(vs)) { ! 913: /* the call below will not be done by caller when we have */ ! 914: /* a synchronous interface */ ! 915: /* return KERN_LOCK_OWNED; */ ! 916: upl_t page_list; ! 917: upl_system_list_request((vm_object_t) ! 918: vs->vs_control_port->ip_kobject, ! 919: offset, data_cnt, data_cnt, &page_list, NULL, 0, ! 920: UPL_NOBLOCK | UPL_CLEAN_IN_PLACE | ! 921: UPL_NO_SYNC | UPL_COPYOUT_FROM); ! 922: ! 923: upl_abort(page_list,0); ! 924: return KERN_SUCCESS; ! 925: } ! 926: ! 927: ! 928: ! 929: if ((vs->vs_seqno != vs->vs_next_seqno++) || (vs->vs_xfer_pending)) { ! 930: upl_t page_list; ! 931: vs->vs_next_seqno--; ! 932: VS_UNLOCK(vs); ! 933: /* the call below will not be done by caller when we have */ ! 934: /* a synchronous interface */ ! 935: /* return KERN_LOCK_OWNED; */ ! 936: upl_system_list_request((vm_object_t) ! 937: vs->vs_control_port->ip_kobject, ! 938: offset, data_cnt, data_cnt, &page_list, NULL, 0, ! 939: UPL_NOBLOCK | UPL_CLEAN_IN_PLACE | ! 940: UPL_NO_SYNC | UPL_COPYOUT_FROM); ! 941: upl_abort(page_list,0); ! 942: return KERN_SUCCESS; ! 943: } ! 944: ! 945: if ((data_cnt % vm_page_size) != 0) ! 946: Panic("bad alignment"); ! 947: ! 948: /* ! 949: vs_check_request(vs, control_port); ! 950: */ ! 951: vs_start_write(vs); ! 952: vs_unlock(vs); ! 953: ! 954: /* ! 955: * Write the data via clustered writes. vs_cluster_write will ! 956: * loop if the address range specified crosses cluster ! 957: * boundaries. ! 958: */ ! 959: vs_cluster_write(vs, offset, addr, data_cnt, FALSE); ! 960: ! 961: vs_finish_write(vs); ! 962: ! 963: return KERN_SUCCESS; ! 964: } ! 965: ! 966: kern_return_t ! 967: dp_memory_object_discard_request( ! 968: memory_object_t mem_obj, ! 969: memory_object_control_t memory_control, ! 970: vm_offset_t offset, ! 971: vm_size_t length) ! 972: { ! 973: panic("illegal"); ! 974: return KERN_FAILURE; ! 975: } ! 976: ! 977: kern_return_t ! 978: dp_memory_object_change_completed( ! 979: memory_object_t mem_obj, ! 980: memory_object_control_t memory_control, ! 981: memory_object_flavor_t flavor) ! 982: { ! 983: static char here[] = "memory_object_change_completed"; ! 984: ! 985: Panic("illegal"); ! 986: return KERN_FAILURE; ! 987: } ! 988: ! 989: /* ! 990: * Create an external object. ! 991: */ ! 992: kern_return_t ! 993: default_pager_object_create( ! 994: MACH_PORT_FACE pager, ! 995: MACH_PORT_FACE *mem_obj, ! 996: vm_size_t size) ! 997: { ! 998: vstruct_t vs; ! 999: MACH_PORT_FACE port; ! 1000: kern_return_t result; ! 1001: struct vstruct_alias *alias_struct; ! 1002: static char here[] = "default_pager_object_create"; ! 1003: ! 1004: ! 1005: if (pager != default_pager_default_port) ! 1006: return KERN_INVALID_ARGUMENT; ! 1007: ! 1008: vs = vs_object_create(size); ! 1009: ! 1010: port = ipc_port_alloc_kernel(); ! 1011: ipc_port_make_send(port); ! 1012: /* register abstract memory object port with pager mux routine */ ! 1013: /* (directs kernel internal calls to the right pager). */ ! 1014: alias_struct = (struct vstruct_alias *) ! 1015: kalloc(sizeof(struct vstruct_alias)); ! 1016: if(alias_struct != NULL) { ! 1017: alias_struct->vs = vs; ! 1018: alias_struct->name = ISVS; ! 1019: port->alias = (int) alias_struct; ! 1020: } ! 1021: else Panic("Out of kernel memory"); ! 1022: ! 1023: /* ! 1024: * Set up associations between these ports ! 1025: * and this vstruct structure ! 1026: */ ! 1027: ! 1028: vs->vs_mem_obj_port = port; ! 1029: vstruct_list_insert(vs); ! 1030: default_pager_add(vs, FALSE); ! 1031: ! 1032: *mem_obj = port; ! 1033: ! 1034: return KERN_SUCCESS; ! 1035: } ! 1036: ! 1037: kern_return_t ! 1038: default_pager_objects( ! 1039: MACH_PORT_FACE pager, ! 1040: default_pager_object_array_t *objectsp, ! 1041: mach_msg_type_number_t *ocountp, ! 1042: mach_port_array_t *portsp, ! 1043: mach_msg_type_number_t *pcountp) ! 1044: { ! 1045: vm_offset_t oaddr = 0; /* memory for objects */ ! 1046: vm_size_t osize = 0; /* current size */ ! 1047: default_pager_object_t * objects; ! 1048: unsigned int opotential; ! 1049: ! 1050: vm_offset_t paddr = 0; /* memory for ports */ ! 1051: vm_size_t psize = 0; /* current size */ ! 1052: MACH_PORT_FACE * ports; ! 1053: unsigned int ppotential; ! 1054: ! 1055: unsigned int actual; ! 1056: unsigned int num_objects; ! 1057: kern_return_t kr; ! 1058: vstruct_t entry; ! 1059: static char here[] = "default_pager_objects"; ! 1060: /* ! 1061: if (pager != default_pager_default_port) ! 1062: return KERN_INVALID_ARGUMENT; ! 1063: */ ! 1064: ! 1065: /* start with the inline memory */ ! 1066: ! 1067: kr = vm_map_copyout(ipc_kernel_map, (vm_offset_t *)&objects, ! 1068: (vm_map_copy_t) *objectsp); ! 1069: ! 1070: if (kr != KERN_SUCCESS) ! 1071: return kr; ! 1072: ! 1073: osize = round_page(*ocountp * sizeof * objects); ! 1074: kr = vm_map_wire(ipc_kernel_map, ! 1075: trunc_page((vm_offset_t)objects), ! 1076: round_page(((vm_offset_t)objects) + osize), ! 1077: VM_PROT_READ|VM_PROT_WRITE, FALSE); ! 1078: osize=0; ! 1079: ! 1080: *objectsp = objects; ! 1081: /* we start with the inline space */ ! 1082: ! 1083: ! 1084: num_objects = 0; ! 1085: opotential = *ocountp; ! 1086: ! 1087: ports = (MACH_PORT_FACE *) *portsp; ! 1088: ppotential = *pcountp; ! 1089: ! 1090: VSL_LOCK(); ! 1091: ! 1092: /* ! 1093: * We will send no more than this many ! 1094: */ ! 1095: actual = vstruct_list.vsl_count; ! 1096: VSL_UNLOCK(); ! 1097: ! 1098: if (opotential < actual) { ! 1099: vm_offset_t newaddr; ! 1100: vm_size_t newsize; ! 1101: ! 1102: newsize = 2 * round_page(actual * sizeof * objects); ! 1103: ! 1104: kr = vm_allocate(kernel_map, &newaddr, newsize, TRUE); ! 1105: if (kr != KERN_SUCCESS) ! 1106: goto nomemory; ! 1107: ! 1108: oaddr = newaddr; ! 1109: osize = newsize; ! 1110: opotential = osize / sizeof * objects; ! 1111: objects = (default_pager_object_t *)oaddr; ! 1112: } ! 1113: ! 1114: if (ppotential < actual) { ! 1115: vm_offset_t newaddr; ! 1116: vm_size_t newsize; ! 1117: ! 1118: newsize = 2 * round_page(actual * sizeof * ports); ! 1119: ! 1120: kr = vm_allocate(kernel_map, &newaddr, newsize, TRUE); ! 1121: if (kr != KERN_SUCCESS) ! 1122: goto nomemory; ! 1123: ! 1124: paddr = newaddr; ! 1125: psize = newsize; ! 1126: ppotential = psize / sizeof * ports; ! 1127: ports = (MACH_PORT_FACE *)paddr; ! 1128: } ! 1129: ! 1130: /* ! 1131: * Now scan the list. ! 1132: */ ! 1133: ! 1134: VSL_LOCK(); ! 1135: ! 1136: num_objects = 0; ! 1137: queue_iterate(&vstruct_list.vsl_queue, entry, vstruct_t, vs_links) { ! 1138: ! 1139: MACH_PORT_FACE port; ! 1140: vm_size_t size; ! 1141: ! 1142: if ((num_objects >= opotential) || ! 1143: (num_objects >= ppotential)) { ! 1144: ! 1145: /* ! 1146: * This should be rare. In any case, ! 1147: * we will only miss recent objects, ! 1148: * because they are added at the end. ! 1149: */ ! 1150: break; ! 1151: } ! 1152: ! 1153: /* ! 1154: * Avoid interfering with normal operations ! 1155: */ ! 1156: if (!VS_MAP_TRY_LOCK(entry)) ! 1157: goto not_this_one; ! 1158: size = ps_vstruct_allocated_size(entry); ! 1159: VS_MAP_UNLOCK(entry); ! 1160: ! 1161: VS_LOCK(entry); ! 1162: ! 1163: port = entry->vs_object_name; ! 1164: if (port == MACH_PORT_NULL) { ! 1165: ! 1166: /* ! 1167: * The object is waiting for no-senders ! 1168: * or memory_object_init. ! 1169: */ ! 1170: VS_UNLOCK(entry); ! 1171: goto not_this_one; ! 1172: } ! 1173: ! 1174: /* ! 1175: * We need a reference for the reply message. ! 1176: * While we are unlocked, the bucket queue ! 1177: * can change and the object might be terminated. ! 1178: * memory_object_terminate will wait for us, ! 1179: * preventing deallocation of the entry. ! 1180: */ ! 1181: ! 1182: if (--entry->vs_name_refs == 0) { ! 1183: VS_UNLOCK(entry); ! 1184: ! 1185: /* keep the list locked, wont take long */ ! 1186: ! 1187: { ! 1188: int i; ! 1189: for(i=0; i<default_pager_max_urefs; i++) ! 1190: ipc_port_make_send(port); ! 1191: } ! 1192: VS_LOCK(entry); ! 1193: ! 1194: entry->vs_name_refs += default_pager_max_urefs; ! 1195: vs_finish_refs(entry); ! 1196: } ! 1197: VS_UNLOCK(entry); ! 1198: ! 1199: /* the arrays are wired, so no deadlock worries */ ! 1200: ! 1201: objects[num_objects].dpo_object = (vm_offset_t) entry; ! 1202: objects[num_objects].dpo_size = size; ! 1203: ports [num_objects++] = port; ! 1204: continue; ! 1205: ! 1206: not_this_one: ! 1207: /* ! 1208: * Do not return garbage ! 1209: */ ! 1210: objects[num_objects].dpo_object = (vm_offset_t) 0; ! 1211: objects[num_objects].dpo_size = 0; ! 1212: ports [num_objects++] = MACH_PORT_NULL; ! 1213: ! 1214: } ! 1215: ! 1216: VSL_UNLOCK(); ! 1217: ! 1218: /* ! 1219: * Deallocate and clear unused memory. ! 1220: * (Returned memory will automagically become pageable.) ! 1221: */ ! 1222: ! 1223: if (objects == *objectsp) { ! 1224: ! 1225: /* ! 1226: * Our returned information fit inline. ! 1227: * Nothing to deallocate. ! 1228: */ ! 1229: *ocountp = num_objects; ! 1230: } else if (actual == 0) { ! 1231: (void) vm_deallocate(kernel_map, oaddr, osize); ! 1232: ! 1233: /* return zero items inline */ ! 1234: *ocountp = 0; ! 1235: } else { ! 1236: vm_offset_t used; ! 1237: ! 1238: used = round_page(actual * sizeof * objects); ! 1239: ! 1240: if (used != osize) ! 1241: (void) vm_deallocate(kernel_map, ! 1242: oaddr + used, osize - used); ! 1243: ! 1244: *objectsp = objects; ! 1245: *ocountp = num_objects; ! 1246: } ! 1247: ! 1248: if (ports == (MACH_PORT_FACE *)*portsp) { ! 1249: ! 1250: /* ! 1251: * Our returned information fit inline. ! 1252: * Nothing to deallocate. ! 1253: */ ! 1254: ! 1255: *pcountp = num_objects; ! 1256: } else if (actual == 0) { ! 1257: (void) vm_deallocate(kernel_map, paddr, psize); ! 1258: ! 1259: /* return zero items inline */ ! 1260: *pcountp = 0; ! 1261: } else { ! 1262: vm_offset_t used; ! 1263: ! 1264: used = round_page(actual * sizeof * ports); ! 1265: ! 1266: if (used != psize) ! 1267: (void) vm_deallocate(kernel_map, ! 1268: paddr + used, psize - used); ! 1269: ! 1270: *portsp = (mach_port_array_t)ports; ! 1271: *pcountp = num_objects; ! 1272: } ! 1273: (void) vm_map_unwire(kernel_map, (vm_offset_t)objects, ! 1274: *ocountp + (vm_offset_t)objects, FALSE); ! 1275: (void) vm_map_copyin(kernel_map, (vm_offset_t)objects, ! 1276: *ocountp, TRUE, (vm_map_copy_t *)objectsp); ! 1277: ! 1278: return KERN_SUCCESS; ! 1279: ! 1280: nomemory: ! 1281: { ! 1282: register int i; ! 1283: for (i = 0; i < num_objects; i++) ! 1284: ipc_port_dealloc_kernel(ports[i]); ! 1285: } ! 1286: ! 1287: if (objects != *objectsp) ! 1288: (void) vm_deallocate(kernel_map, oaddr, osize); ! 1289: ! 1290: if (ports != (MACH_PORT_FACE *)*portsp) ! 1291: (void) vm_deallocate(kernel_map, paddr, psize); ! 1292: ! 1293: return KERN_RESOURCE_SHORTAGE; ! 1294: } ! 1295: ! 1296: kern_return_t ! 1297: default_pager_object_pages( ! 1298: MACH_PORT_FACE pager, ! 1299: MACH_PORT_FACE object, ! 1300: default_pager_page_array_t *pagesp, ! 1301: mach_msg_type_number_t *countp) ! 1302: { ! 1303: vm_offset_t addr; /* memory for page offsets */ ! 1304: vm_size_t size = 0; /* current memory size */ ! 1305: default_pager_page_t * pages; ! 1306: unsigned int potential, actual; ! 1307: kern_return_t kr; ! 1308: ! 1309: /* ! 1310: if (pager != default_pager_default_port) ! 1311: return KERN_INVALID_ARGUMENT; ! 1312: */ ! 1313: kr = vm_map_copyout(ipc_kernel_map, (vm_offset_t *)&pages, ! 1314: (vm_map_copy_t) *pagesp); ! 1315: ! 1316: if (kr != KERN_SUCCESS) ! 1317: return kr; ! 1318: ! 1319: size = round_page(*countp * sizeof * pages); ! 1320: kr = vm_map_wire(ipc_kernel_map, ! 1321: trunc_page((vm_offset_t)pages), ! 1322: round_page(((vm_offset_t)pages) + size), ! 1323: VM_PROT_READ|VM_PROT_WRITE, FALSE); ! 1324: size=0; ! 1325: ! 1326: *pagesp = pages; ! 1327: /* we start with the inline space */ ! 1328: ! 1329: addr = (vm_offset_t)pages; ! 1330: potential = *countp; ! 1331: ! 1332: for (;;) { ! 1333: vstruct_t entry; ! 1334: ! 1335: VSL_LOCK(); ! 1336: queue_iterate(&vstruct_list.vsl_queue, entry, vstruct_t, ! 1337: vs_links) { ! 1338: VS_LOCK(entry); ! 1339: if (entry->vs_object_name == object) { ! 1340: VSL_UNLOCK(); ! 1341: goto found_object; ! 1342: } ! 1343: VS_UNLOCK(entry); ! 1344: } ! 1345: VSL_UNLOCK(); ! 1346: ! 1347: /* did not find the object */ ! 1348: ! 1349: if (pages != *pagesp) ! 1350: (void) vm_deallocate(kernel_map, addr, size); ! 1351: return KERN_INVALID_ARGUMENT; ! 1352: ! 1353: found_object: ! 1354: ! 1355: if (!VS_MAP_TRY_LOCK(entry)) { ! 1356: /* oh well bad luck */ ! 1357: ! 1358: VS_UNLOCK(entry); ! 1359: ! 1360: assert_wait_timeout( 1, THREAD_INTERRUPTIBLE); ! 1361: thread_block((void (*)(void)) 0); ! 1362: thread_cancel_timer(); ! 1363: continue; ! 1364: } ! 1365: ! 1366: actual = ps_vstruct_allocated_pages(entry, pages, potential); ! 1367: VS_MAP_UNLOCK(entry); ! 1368: VS_UNLOCK(entry); ! 1369: ! 1370: if (actual <= potential) ! 1371: break; ! 1372: ! 1373: /* allocate more memory */ ! 1374: ! 1375: if (pages != *pagesp) ! 1376: (void) vm_deallocate(kernel_map, addr, size); ! 1377: size = round_page(actual * sizeof * pages); ! 1378: kr = vm_allocate(kernel_map, &addr, size, TRUE); ! 1379: if (kr != KERN_SUCCESS) ! 1380: return kr; ! 1381: pages = (default_pager_page_t *)addr; ! 1382: potential = size / sizeof * pages; ! 1383: } ! 1384: ! 1385: /* ! 1386: * Deallocate and clear unused memory. ! 1387: * (Returned memory will automagically become pageable.) ! 1388: */ ! 1389: ! 1390: if (pages == *pagesp) { ! 1391: ! 1392: /* ! 1393: * Our returned information fit inline. ! 1394: * Nothing to deallocate. ! 1395: */ ! 1396: ! 1397: *countp = actual; ! 1398: } else if (actual == 0) { ! 1399: (void) vm_deallocate(kernel_map, addr, size); ! 1400: ! 1401: /* return zero items inline */ ! 1402: *countp = 0; ! 1403: } else { ! 1404: vm_offset_t used; ! 1405: ! 1406: used = round_page(actual * sizeof * pages); ! 1407: ! 1408: if (used != size) ! 1409: (void) vm_deallocate(kernel_map, ! 1410: addr + used, size - used); ! 1411: ! 1412: *pagesp = pages; ! 1413: *countp = actual; ! 1414: } ! 1415: (void) vm_map_unwire(kernel_map, (vm_offset_t)pages, ! 1416: *countp + (vm_offset_t)pages, FALSE); ! 1417: (void) vm_map_copyin(kernel_map, (vm_offset_t)pages, ! 1418: *countp, TRUE, (vm_map_copy_t *)pagesp); ! 1419: return KERN_SUCCESS; ! 1420: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.