![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_map.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / OSKit-Mach / vm|
Return to vm_map.c CVS log | Up to [GnuMach] / OSKit-Mach / vm |
1.1 ! root 1: /* ! 2: * Mach Operating System ! 3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University. ! 4: * Copyright (c) 1993,1994 The University of Utah and ! 5: * the Computer Systems Laboratory (CSL). ! 6: * All rights reserved. ! 7: * ! 8: * Permission to use, copy, modify and distribute this software and its ! 9: * documentation is hereby granted, provided that both the copyright ! 10: * notice and this permission notice appear in all copies of the ! 11: * software, derivative works or modified versions, and any portions ! 12: * thereof, and that both notices appear in supporting documentation. ! 13: * ! 14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF ! 15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY ! 16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF ! 17: * THIS SOFTWARE. ! 18: * ! 19: * Carnegie Mellon requests users of this software to return to ! 20: * ! 21: * Software Distribution Coordinator or [email protected] ! 22: * School of Computer Science ! 23: * Carnegie Mellon University ! 24: * Pittsburgh PA 15213-3890 ! 25: * ! 26: * any improvements or extensions that they make and grant Carnegie Mellon ! 27: * the rights to redistribute these changes. ! 28: */ ! 29: /* ! 30: * File: vm/vm_map.c ! 31: * Author: Avadis Tevanian, Jr., Michael Wayne Young ! 32: * Date: 1985 ! 33: * ! 34: * Virtual memory mapping module. ! 35: */ ! 36: ! 37: #include <norma_ipc.h> ! 38: ! 39: #include <mach/kern_return.h> ! 40: #include <mach/port.h> ! 41: #include <mach/vm_attributes.h> ! 42: #include <mach/vm_param.h> ! 43: #include <kern/assert.h> ! 44: #include <kern/zalloc.h> ! 45: #include <vm/vm_fault.h> ! 46: #include <vm/vm_map.h> ! 47: #include <vm/vm_object.h> ! 48: #include <vm/vm_page.h> ! 49: #include <vm/vm_kern.h> ! 50: #include <ipc/ipc_port.h> ! 51: ! 52: /* ! 53: * Macros to copy a vm_map_entry. We must be careful to correctly ! 54: * manage the wired page count. vm_map_entry_copy() creates a new ! 55: * map entry to the same memory - the wired count in the new entry ! 56: * must be set to zero. vm_map_entry_copy_full() creates a new ! 57: * entry that is identical to the old entry. This preserves the ! 58: * wire count; it's used for map splitting and zone changing in ! 59: * vm_map_copyout. ! 60: */ ! 61: #define vm_map_entry_copy(NEW,OLD) \ ! 62: MACRO_BEGIN \ ! 63: *(NEW) = *(OLD); \ ! 64: (NEW)->is_shared = FALSE; \ ! 65: (NEW)->needs_wakeup = FALSE; \ ! 66: (NEW)->in_transition = FALSE; \ ! 67: (NEW)->wired_count = 0; \ ! 68: (NEW)->user_wired_count = 0; \ ! 69: MACRO_END ! 70: ! 71: #define vm_map_entry_copy_full(NEW,OLD) (*(NEW) = *(OLD)) ! 72: ! 73: /* ! 74: * Virtual memory maps provide for the mapping, protection, ! 75: * and sharing of virtual memory objects. In addition, ! 76: * this module provides for an efficient virtual copy of ! 77: * memory from one map to another. ! 78: * ! 79: * Synchronization is required prior to most operations. ! 80: * ! 81: * Maps consist of an ordered doubly-linked list of simple ! 82: * entries; a single hint is used to speed up lookups. ! 83: * ! 84: * Sharing maps have been deleted from this version of Mach. ! 85: * All shared objects are now mapped directly into the respective ! 86: * maps. This requires a change in the copy on write strategy; ! 87: * the asymmetric (delayed) strategy is used for shared temporary ! 88: * objects instead of the symmetric (shadow) strategy. This is ! 89: * selected by the (new) use_shared_copy bit in the object. See ! 90: * vm_object_copy_temporary in vm_object.c for details. All maps ! 91: * are now "top level" maps (either task map, kernel map or submap ! 92: * of the kernel map). ! 93: * ! 94: * Since portions of maps are specified by start/end addreses, ! 95: * which may not align with existing map entries, all ! 96: * routines merely "clip" entries to these start/end values. ! 97: * [That is, an entry is split into two, bordering at a ! 98: * start or end value.] Note that these clippings may not ! 99: * always be necessary (as the two resulting entries are then ! 100: * not changed); however, the clipping is done for convenience. ! 101: * No attempt is currently made to "glue back together" two ! 102: * abutting entries. ! 103: * ! 104: * The symmetric (shadow) copy strategy implements virtual copy ! 105: * by copying VM object references from one map to ! 106: * another, and then marking both regions as copy-on-write. ! 107: * It is important to note that only one writeable reference ! 108: * to a VM object region exists in any map when this strategy ! 109: * is used -- this means that shadow object creation can be ! 110: * delayed until a write operation occurs. The asymmetric (delayed) ! 111: * strategy allows multiple maps to have writeable references to ! 112: * the same region of a vm object, and hence cannot delay creating ! 113: * its copy objects. See vm_object_copy_temporary() in vm_object.c. ! 114: * Copying of permanent objects is completely different; see ! 115: * vm_object_copy_strategically() in vm_object.c. ! 116: */ ! 117: ! 118: zone_t vm_map_zone; /* zone for vm_map structures */ ! 119: zone_t vm_map_entry_zone; /* zone for vm_map_entry structures */ ! 120: zone_t vm_map_kentry_zone; /* zone for kernel entry structures */ ! 121: zone_t vm_map_copy_zone; /* zone for vm_map_copy structures */ ! 122: ! 123: boolean_t vm_map_lookup_entry(); /* forward declaration */ ! 124: ! 125: /* ! 126: * Placeholder object for submap operations. This object is dropped ! 127: * into the range by a call to vm_map_find, and removed when ! 128: * vm_map_submap creates the submap. ! 129: */ ! 130: ! 131: vm_object_t vm_submap_object; ! 132: ! 133: /* ! 134: * vm_map_init: ! 135: * ! 136: * Initialize the vm_map module. Must be called before ! 137: * any other vm_map routines. ! 138: * ! 139: * Map and entry structures are allocated from zones -- we must ! 140: * initialize those zones. ! 141: * ! 142: * There are three zones of interest: ! 143: * ! 144: * vm_map_zone: used to allocate maps. ! 145: * vm_map_entry_zone: used to allocate map entries. ! 146: * vm_map_kentry_zone: used to allocate map entries for the kernel. ! 147: * ! 148: * The kernel allocates map entries from a special zone that is initially ! 149: * "crammed" with memory. It would be difficult (perhaps impossible) for ! 150: * the kernel to allocate more memory to a entry zone when it became ! 151: * empty since the very act of allocating memory implies the creation ! 152: * of a new entry. ! 153: */ ! 154: ! 155: vm_offset_t kentry_data; ! 156: vm_size_t kentry_data_size; ! 157: int kentry_count = 256; /* to init kentry_data_size */ ! 158: ! 159: void vm_map_init() ! 160: { ! 161: vm_map_zone = zinit((vm_size_t) sizeof(struct vm_map), 40*1024, ! 162: PAGE_SIZE, 0, "maps"); ! 163: vm_map_entry_zone = zinit((vm_size_t) sizeof(struct vm_map_entry), ! 164: 1024*1024, PAGE_SIZE*5, ! 165: 0, "non-kernel map entries"); ! 166: vm_map_kentry_zone = zinit((vm_size_t) sizeof(struct vm_map_entry), ! 167: kentry_data_size, kentry_data_size, ! 168: ZONE_FIXED /* XXX */, "kernel map entries"); ! 169: ! 170: vm_map_copy_zone = zinit((vm_size_t) sizeof(struct vm_map_copy), ! 171: 16*1024, PAGE_SIZE, 0, ! 172: "map copies"); ! 173: ! 174: /* ! 175: * Cram the kentry zone with initial data. ! 176: */ ! 177: zcram(vm_map_kentry_zone, kentry_data, kentry_data_size); ! 178: ! 179: /* ! 180: * Submap object is initialized by vm_object_init. ! 181: */ ! 182: } ! 183: ! 184: /* ! 185: * vm_map_create: ! 186: * ! 187: * Creates and returns a new empty VM map with ! 188: * the given physical map structure, and having ! 189: * the given lower and upper address bounds. ! 190: */ ! 191: vm_map_t vm_map_create(pmap, min, max, pageable) ! 192: pmap_t pmap; ! 193: vm_offset_t min, max; ! 194: boolean_t pageable; ! 195: { ! 196: register vm_map_t result; ! 197: ! 198: result = (vm_map_t) zalloc(vm_map_zone); ! 199: if (result == VM_MAP_NULL) ! 200: panic("vm_map_create"); ! 201: ! 202: vm_map_first_entry(result) = vm_map_to_entry(result); ! 203: vm_map_last_entry(result) = vm_map_to_entry(result); ! 204: result->hdr.nentries = 0; ! 205: result->hdr.entries_pageable = pageable; ! 206: ! 207: result->size = 0; ! 208: result->ref_count = 1; ! 209: result->pmap = pmap; ! 210: result->min_offset = min; ! 211: result->max_offset = max; ! 212: result->wiring_required = FALSE; ! 213: result->wait_for_space = FALSE; ! 214: result->first_free = vm_map_to_entry(result); ! 215: result->hint = vm_map_to_entry(result); ! 216: vm_map_lock_init(result); ! 217: simple_lock_init(&result->ref_lock); ! 218: simple_lock_init(&result->hint_lock); ! 219: ! 220: return(result); ! 221: } ! 222: ! 223: /* ! 224: * vm_map_entry_create: [ internal use only ] ! 225: * ! 226: * Allocates a VM map entry for insertion in the ! 227: * given map (or map copy). No fields are filled. ! 228: */ ! 229: #define vm_map_entry_create(map) \ ! 230: _vm_map_entry_create(&(map)->hdr) ! 231: ! 232: #define vm_map_copy_entry_create(copy) \ ! 233: _vm_map_entry_create(&(copy)->cpy_hdr) ! 234: ! 235: vm_map_entry_t _vm_map_entry_create(map_header) ! 236: register struct vm_map_header *map_header; ! 237: { ! 238: register zone_t zone; ! 239: register vm_map_entry_t entry; ! 240: ! 241: if (map_header->entries_pageable) ! 242: zone = vm_map_entry_zone; ! 243: else ! 244: zone = vm_map_kentry_zone; ! 245: ! 246: entry = (vm_map_entry_t) zalloc(zone); ! 247: if (entry == VM_MAP_ENTRY_NULL) ! 248: panic("vm_map_entry_create"); ! 249: ! 250: return(entry); ! 251: } ! 252: ! 253: /* ! 254: * vm_map_entry_dispose: [ internal use only ] ! 255: * ! 256: * Inverse of vm_map_entry_create. ! 257: */ ! 258: #define vm_map_entry_dispose(map, entry) \ ! 259: _vm_map_entry_dispose(&(map)->hdr, (entry)) ! 260: ! 261: #define vm_map_copy_entry_dispose(map, entry) \ ! 262: _vm_map_entry_dispose(&(copy)->cpy_hdr, (entry)) ! 263: ! 264: void _vm_map_entry_dispose(map_header, entry) ! 265: register struct vm_map_header *map_header; ! 266: register vm_map_entry_t entry; ! 267: { ! 268: register zone_t zone; ! 269: ! 270: if (map_header->entries_pageable) ! 271: zone = vm_map_entry_zone; ! 272: else ! 273: zone = vm_map_kentry_zone; ! 274: ! 275: zfree(zone, (vm_offset_t) entry); ! 276: } ! 277: ! 278: /* ! 279: * vm_map_entry_{un,}link: ! 280: * ! 281: * Insert/remove entries from maps (or map copies). ! 282: */ ! 283: #define vm_map_entry_link(map, after_where, entry) \ ! 284: _vm_map_entry_link(&(map)->hdr, after_where, entry) ! 285: ! 286: #define vm_map_copy_entry_link(copy, after_where, entry) \ ! 287: _vm_map_entry_link(&(copy)->cpy_hdr, after_where, entry) ! 288: ! 289: #define _vm_map_entry_link(hdr, after_where, entry) \ ! 290: MACRO_BEGIN \ ! 291: (hdr)->nentries++; \ ! 292: (entry)->vme_prev = (after_where); \ ! 293: (entry)->vme_next = (after_where)->vme_next; \ ! 294: (entry)->vme_prev->vme_next = \ ! 295: (entry)->vme_next->vme_prev = (entry); \ ! 296: MACRO_END ! 297: ! 298: #define vm_map_entry_unlink(map, entry) \ ! 299: _vm_map_entry_unlink(&(map)->hdr, entry) ! 300: ! 301: #define vm_map_copy_entry_unlink(copy, entry) \ ! 302: _vm_map_entry_unlink(&(copy)->cpy_hdr, entry) ! 303: ! 304: #define _vm_map_entry_unlink(hdr, entry) \ ! 305: MACRO_BEGIN \ ! 306: (hdr)->nentries--; \ ! 307: (entry)->vme_next->vme_prev = (entry)->vme_prev; \ ! 308: (entry)->vme_prev->vme_next = (entry)->vme_next; \ ! 309: MACRO_END ! 310: ! 311: /* ! 312: * vm_map_reference: ! 313: * ! 314: * Creates another valid reference to the given map. ! 315: * ! 316: */ ! 317: void vm_map_reference(map) ! 318: register vm_map_t map; ! 319: { ! 320: if (map == VM_MAP_NULL) ! 321: return; ! 322: ! 323: simple_lock(&map->ref_lock); ! 324: map->ref_count++; ! 325: simple_unlock(&map->ref_lock); ! 326: } ! 327: ! 328: /* ! 329: * vm_map_deallocate: ! 330: * ! 331: * Removes a reference from the specified map, ! 332: * destroying it if no references remain. ! 333: * The map should not be locked. ! 334: */ ! 335: void vm_map_deallocate(map) ! 336: register vm_map_t map; ! 337: { ! 338: register int c; ! 339: ! 340: if (map == VM_MAP_NULL) ! 341: return; ! 342: ! 343: simple_lock(&map->ref_lock); ! 344: c = --map->ref_count; ! 345: simple_unlock(&map->ref_lock); ! 346: ! 347: if (c > 0) { ! 348: return; ! 349: } ! 350: ! 351: projected_buffer_collect(map); ! 352: (void) vm_map_delete(map, map->min_offset, map->max_offset); ! 353: ! 354: pmap_destroy(map->pmap); ! 355: ! 356: zfree(vm_map_zone, (vm_offset_t) map); ! 357: } ! 358: ! 359: /* ! 360: * SAVE_HINT: ! 361: * ! 362: * Saves the specified entry as the hint for ! 363: * future lookups. Performs necessary interlocks. ! 364: */ ! 365: #define SAVE_HINT(map,value) \ ! 366: simple_lock(&(map)->hint_lock); \ ! 367: (map)->hint = (value); \ ! 368: simple_unlock(&(map)->hint_lock); ! 369: ! 370: /* ! 371: * vm_map_lookup_entry: [ internal use only ] ! 372: * ! 373: * Finds the map entry containing (or ! 374: * immediately preceding) the specified address ! 375: * in the given map; the entry is returned ! 376: * in the "entry" parameter. The boolean ! 377: * result indicates whether the address is ! 378: * actually contained in the map. ! 379: */ ! 380: boolean_t vm_map_lookup_entry(map, address, entry) ! 381: register vm_map_t map; ! 382: register vm_offset_t address; ! 383: vm_map_entry_t *entry; /* OUT */ ! 384: { ! 385: register vm_map_entry_t cur; ! 386: register vm_map_entry_t last; ! 387: ! 388: /* ! 389: * Start looking either from the head of the ! 390: * list, or from the hint. ! 391: */ ! 392: ! 393: simple_lock(&map->hint_lock); ! 394: cur = map->hint; ! 395: simple_unlock(&map->hint_lock); ! 396: ! 397: if (cur == vm_map_to_entry(map)) ! 398: cur = cur->vme_next; ! 399: ! 400: if (address >= cur->vme_start) { ! 401: /* ! 402: * Go from hint to end of list. ! 403: * ! 404: * But first, make a quick check to see if ! 405: * we are already looking at the entry we ! 406: * want (which is usually the case). ! 407: * Note also that we don't need to save the hint ! 408: * here... it is the same hint (unless we are ! 409: * at the header, in which case the hint didn't ! 410: * buy us anything anyway). ! 411: */ ! 412: last = vm_map_to_entry(map); ! 413: if ((cur != last) && (cur->vme_end > address)) { ! 414: *entry = cur; ! 415: return(TRUE); ! 416: } ! 417: } ! 418: else { ! 419: /* ! 420: * Go from start to hint, *inclusively* ! 421: */ ! 422: last = cur->vme_next; ! 423: cur = vm_map_first_entry(map); ! 424: } ! 425: ! 426: /* ! 427: * Search linearly ! 428: */ ! 429: ! 430: while (cur != last) { ! 431: if (cur->vme_end > address) { ! 432: if (address >= cur->vme_start) { ! 433: /* ! 434: * Save this lookup for future ! 435: * hints, and return ! 436: */ ! 437: ! 438: *entry = cur; ! 439: SAVE_HINT(map, cur); ! 440: return(TRUE); ! 441: } ! 442: break; ! 443: } ! 444: cur = cur->vme_next; ! 445: } ! 446: *entry = cur->vme_prev; ! 447: SAVE_HINT(map, *entry); ! 448: return(FALSE); ! 449: } ! 450: ! 451: /* ! 452: * Routine: invalid_user_access ! 453: * ! 454: * Verifies whether user access is valid. ! 455: */ ! 456: ! 457: boolean_t ! 458: invalid_user_access(map, start, end, prot) ! 459: vm_map_t map; ! 460: vm_offset_t start, end; ! 461: vm_prot_t prot; ! 462: { ! 463: vm_map_entry_t entry; ! 464: ! 465: return (map == VM_MAP_NULL || map == kernel_map || ! 466: !vm_map_lookup_entry(map, start, &entry) || ! 467: entry->vme_end < end || ! 468: (prot & ~(entry->protection))); ! 469: } ! 470: ! 471: ! 472: /* ! 473: * Routine: vm_map_find_entry ! 474: * Purpose: ! 475: * Allocate a range in the specified virtual address map, ! 476: * returning the entry allocated for that range. ! 477: * Used by kmem_alloc, etc. Returns wired entries. ! 478: * ! 479: * The map must be locked. ! 480: * ! 481: * If an entry is allocated, the object/offset fields ! 482: * are initialized to zero. If an object is supplied, ! 483: * then an existing entry may be extended. ! 484: */ ! 485: kern_return_t vm_map_find_entry(map, address, size, mask, object, o_entry) ! 486: register vm_map_t map; ! 487: vm_offset_t *address; /* OUT */ ! 488: vm_size_t size; ! 489: vm_offset_t mask; ! 490: vm_object_t object; ! 491: vm_map_entry_t *o_entry; /* OUT */ ! 492: { ! 493: register vm_map_entry_t entry, new_entry; ! 494: register vm_offset_t start; ! 495: register vm_offset_t end; ! 496: ! 497: /* ! 498: * Look for the first possible address; ! 499: * if there's already something at this ! 500: * address, we have to start after it. ! 501: */ ! 502: ! 503: if ((entry = map->first_free) == vm_map_to_entry(map)) ! 504: start = map->min_offset; ! 505: else ! 506: start = entry->vme_end; ! 507: ! 508: /* ! 509: * In any case, the "entry" always precedes ! 510: * the proposed new region throughout the loop: ! 511: */ ! 512: ! 513: while (TRUE) { ! 514: register vm_map_entry_t next; ! 515: ! 516: /* ! 517: * Find the end of the proposed new region. ! 518: * Be sure we didn't go beyond the end, or ! 519: * wrap around the address. ! 520: */ ! 521: ! 522: if (((start + mask) & ~mask) < start) ! 523: return(KERN_NO_SPACE); ! 524: start = ((start + mask) & ~mask); ! 525: end = start + size; ! 526: ! 527: if ((end > map->max_offset) || (end < start)) ! 528: return(KERN_NO_SPACE); ! 529: ! 530: /* ! 531: * If there are no more entries, we must win. ! 532: */ ! 533: ! 534: next = entry->vme_next; ! 535: if (next == vm_map_to_entry(map)) ! 536: break; ! 537: ! 538: /* ! 539: * If there is another entry, it must be ! 540: * after the end of the potential new region. ! 541: */ ! 542: ! 543: if (next->vme_start >= end) ! 544: break; ! 545: ! 546: /* ! 547: * Didn't fit -- move to the next entry. ! 548: */ ! 549: ! 550: entry = next; ! 551: start = entry->vme_end; ! 552: } ! 553: ! 554: /* ! 555: * At this point, ! 556: * "start" and "end" should define the endpoints of the ! 557: * available new range, and ! 558: * "entry" should refer to the region before the new ! 559: * range, and ! 560: * ! 561: * the map should be locked. ! 562: */ ! 563: ! 564: *address = start; ! 565: ! 566: /* ! 567: * See whether we can avoid creating a new entry by ! 568: * extending one of our neighbors. [So far, we only attempt to ! 569: * extend from below.] ! 570: */ ! 571: ! 572: if ((object != VM_OBJECT_NULL) && ! 573: (entry != vm_map_to_entry(map)) && ! 574: (entry->vme_end == start) && ! 575: (!entry->is_shared) && ! 576: (!entry->is_sub_map) && ! 577: (entry->object.vm_object == object) && ! 578: (entry->needs_copy == FALSE) && ! 579: (entry->inheritance == VM_INHERIT_DEFAULT) && ! 580: (entry->protection == VM_PROT_DEFAULT) && ! 581: (entry->max_protection == VM_PROT_ALL) && ! 582: (entry->wired_count == 1) && ! 583: (entry->user_wired_count == 0) && ! 584: (entry->projected_on == 0)) { ! 585: /* ! 586: * Because this is a special case, ! 587: * we don't need to use vm_object_coalesce. ! 588: */ ! 589: ! 590: entry->vme_end = end; ! 591: new_entry = entry; ! 592: } else { ! 593: new_entry = vm_map_entry_create(map); ! 594: ! 595: new_entry->vme_start = start; ! 596: new_entry->vme_end = end; ! 597: ! 598: new_entry->is_shared = FALSE; ! 599: new_entry->is_sub_map = FALSE; ! 600: new_entry->object.vm_object = VM_OBJECT_NULL; ! 601: new_entry->offset = (vm_offset_t) 0; ! 602: ! 603: new_entry->needs_copy = FALSE; ! 604: ! 605: new_entry->inheritance = VM_INHERIT_DEFAULT; ! 606: new_entry->protection = VM_PROT_DEFAULT; ! 607: new_entry->max_protection = VM_PROT_ALL; ! 608: new_entry->wired_count = 1; ! 609: new_entry->user_wired_count = 0; ! 610: ! 611: new_entry->in_transition = FALSE; ! 612: new_entry->needs_wakeup = FALSE; ! 613: new_entry->projected_on = 0; ! 614: ! 615: /* ! 616: * Insert the new entry into the list ! 617: */ ! 618: ! 619: vm_map_entry_link(map, entry, new_entry); ! 620: } ! 621: ! 622: map->size += size; ! 623: ! 624: /* ! 625: * Update the free space hint and the lookup hint ! 626: */ ! 627: ! 628: map->first_free = new_entry; ! 629: SAVE_HINT(map, new_entry); ! 630: ! 631: *o_entry = new_entry; ! 632: return(KERN_SUCCESS); ! 633: } ! 634: ! 635: int vm_map_pmap_enter_print = FALSE; ! 636: int vm_map_pmap_enter_enable = FALSE; ! 637: ! 638: /* ! 639: * Routine: vm_map_pmap_enter ! 640: * ! 641: * Description: ! 642: * Force pages from the specified object to be entered into ! 643: * the pmap at the specified address if they are present. ! 644: * As soon as a page not found in the object the scan ends. ! 645: * ! 646: * Returns: ! 647: * Nothing. ! 648: * ! 649: * In/out conditions: ! 650: * The source map should not be locked on entry. ! 651: */ ! 652: void ! 653: vm_map_pmap_enter(map, addr, end_addr, object, offset, protection) ! 654: vm_map_t map; ! 655: register ! 656: vm_offset_t addr; ! 657: register ! 658: vm_offset_t end_addr; ! 659: register ! 660: vm_object_t object; ! 661: vm_offset_t offset; ! 662: vm_prot_t protection; ! 663: { ! 664: while (addr < end_addr) { ! 665: register vm_page_t m; ! 666: ! 667: vm_object_lock(object); ! 668: vm_object_paging_begin(object); ! 669: ! 670: m = vm_page_lookup(object, offset); ! 671: if (m == VM_PAGE_NULL || m->absent) { ! 672: vm_object_paging_end(object); ! 673: vm_object_unlock(object); ! 674: return; ! 675: } ! 676: ! 677: if (vm_map_pmap_enter_print) { ! 678: printf("vm_map_pmap_enter:"); ! 679: printf("map: %x, addr: %x, object: %x, offset: %x\n", ! 680: map, addr, object, offset); ! 681: } ! 682: ! 683: m->busy = TRUE; ! 684: vm_object_unlock(object); ! 685: ! 686: PMAP_ENTER(map->pmap, addr, m, ! 687: protection, FALSE); ! 688: ! 689: vm_object_lock(object); ! 690: PAGE_WAKEUP_DONE(m); ! 691: vm_page_lock_queues(); ! 692: if (!m->active && !m->inactive) ! 693: vm_page_activate(m); ! 694: vm_page_unlock_queues(); ! 695: vm_object_paging_end(object); ! 696: vm_object_unlock(object); ! 697: ! 698: offset += PAGE_SIZE; ! 699: addr += PAGE_SIZE; ! 700: } ! 701: } ! 702: ! 703: /* ! 704: * Routine: vm_map_enter ! 705: * ! 706: * Description: ! 707: * Allocate a range in the specified virtual address map. ! 708: * The resulting range will refer to memory defined by ! 709: * the given memory object and offset into that object. ! 710: * ! 711: * Arguments are as defined in the vm_map call. ! 712: */ ! 713: kern_return_t vm_map_enter( ! 714: map, ! 715: address, size, mask, anywhere, ! 716: object, offset, needs_copy, ! 717: cur_protection, max_protection, inheritance) ! 718: register ! 719: vm_map_t map; ! 720: vm_offset_t *address; /* IN/OUT */ ! 721: vm_size_t size; ! 722: vm_offset_t mask; ! 723: boolean_t anywhere; ! 724: vm_object_t object; ! 725: vm_offset_t offset; ! 726: boolean_t needs_copy; ! 727: vm_prot_t cur_protection; ! 728: vm_prot_t max_protection; ! 729: vm_inherit_t inheritance; ! 730: { ! 731: register vm_map_entry_t entry; ! 732: register vm_offset_t start; ! 733: register vm_offset_t end; ! 734: kern_return_t result = KERN_SUCCESS; ! 735: ! 736: #define RETURN(value) { result = value; goto BailOut; } ! 737: ! 738: StartAgain: ; ! 739: ! 740: start = *address; ! 741: ! 742: if (anywhere) { ! 743: vm_map_lock(map); ! 744: ! 745: /* ! 746: * Calculate the first possible address. ! 747: */ ! 748: ! 749: if (start < map->min_offset) ! 750: start = map->min_offset; ! 751: if (start > map->max_offset) ! 752: RETURN(KERN_NO_SPACE); ! 753: ! 754: /* ! 755: * Look for the first possible address; ! 756: * if there's already something at this ! 757: * address, we have to start after it. ! 758: */ ! 759: ! 760: if (start == map->min_offset) { ! 761: if ((entry = map->first_free) != vm_map_to_entry(map)) ! 762: start = entry->vme_end; ! 763: } else { ! 764: vm_map_entry_t tmp_entry; ! 765: if (vm_map_lookup_entry(map, start, &tmp_entry)) ! 766: start = tmp_entry->vme_end; ! 767: entry = tmp_entry; ! 768: } ! 769: ! 770: /* ! 771: * In any case, the "entry" always precedes ! 772: * the proposed new region throughout the ! 773: * loop: ! 774: */ ! 775: ! 776: while (TRUE) { ! 777: register vm_map_entry_t next; ! 778: ! 779: /* ! 780: * Find the end of the proposed new region. ! 781: * Be sure we didn't go beyond the end, or ! 782: * wrap around the address. ! 783: */ ! 784: ! 785: if (((start + mask) & ~mask) < start) ! 786: return(KERN_NO_SPACE); ! 787: start = ((start + mask) & ~mask); ! 788: end = start + size; ! 789: ! 790: if ((end > map->max_offset) || (end < start)) { ! 791: if (map->wait_for_space) { ! 792: if (size <= (map->max_offset - ! 793: map->min_offset)) { ! 794: assert_wait((event_t) map, TRUE); ! 795: vm_map_unlock(map); ! 796: thread_block((void (*)()) 0); ! 797: goto StartAgain; ! 798: } ! 799: } ! 800: ! 801: RETURN(KERN_NO_SPACE); ! 802: } ! 803: ! 804: /* ! 805: * If there are no more entries, we must win. ! 806: */ ! 807: ! 808: next = entry->vme_next; ! 809: if (next == vm_map_to_entry(map)) ! 810: break; ! 811: ! 812: /* ! 813: * If there is another entry, it must be ! 814: * after the end of the potential new region. ! 815: */ ! 816: ! 817: if (next->vme_start >= end) ! 818: break; ! 819: ! 820: /* ! 821: * Didn't fit -- move to the next entry. ! 822: */ ! 823: ! 824: entry = next; ! 825: start = entry->vme_end; ! 826: } ! 827: *address = start; ! 828: } else { ! 829: vm_map_entry_t temp_entry; ! 830: ! 831: /* ! 832: * Verify that: ! 833: * the address doesn't itself violate ! 834: * the mask requirement. ! 835: */ ! 836: ! 837: if ((start & mask) != 0) ! 838: return(KERN_NO_SPACE); ! 839: ! 840: vm_map_lock(map); ! 841: ! 842: /* ! 843: * ... the address is within bounds ! 844: */ ! 845: ! 846: end = start + size; ! 847: ! 848: if ((start < map->min_offset) || ! 849: (end > map->max_offset) || ! 850: (start >= end)) { ! 851: RETURN(KERN_INVALID_ADDRESS); ! 852: } ! 853: ! 854: /* ! 855: * ... the starting address isn't allocated ! 856: */ ! 857: ! 858: if (vm_map_lookup_entry(map, start, &temp_entry)) ! 859: RETURN(KERN_NO_SPACE); ! 860: ! 861: entry = temp_entry; ! 862: ! 863: /* ! 864: * ... the next region doesn't overlap the ! 865: * end point. ! 866: */ ! 867: ! 868: if ((entry->vme_next != vm_map_to_entry(map)) && ! 869: (entry->vme_next->vme_start < end)) ! 870: RETURN(KERN_NO_SPACE); ! 871: } ! 872: ! 873: /* ! 874: * At this point, ! 875: * "start" and "end" should define the endpoints of the ! 876: * available new range, and ! 877: * "entry" should refer to the region before the new ! 878: * range, and ! 879: * ! 880: * the map should be locked. ! 881: */ ! 882: ! 883: /* ! 884: * See whether we can avoid creating a new entry (and object) by ! 885: * extending one of our neighbors. [So far, we only attempt to ! 886: * extend from below.] ! 887: */ ! 888: ! 889: if ((object == VM_OBJECT_NULL) && ! 890: (entry != vm_map_to_entry(map)) && ! 891: (entry->vme_end == start) && ! 892: (!entry->is_shared) && ! 893: (!entry->is_sub_map) && ! 894: (entry->inheritance == inheritance) && ! 895: (entry->protection == cur_protection) && ! 896: (entry->max_protection == max_protection) && ! 897: (entry->wired_count == 0) && /* implies user_wired_count == 0 */ ! 898: (entry->projected_on == 0)) { ! 899: if (vm_object_coalesce(entry->object.vm_object, ! 900: VM_OBJECT_NULL, ! 901: entry->offset, ! 902: (vm_offset_t) 0, ! 903: (vm_size_t)(entry->vme_end - entry->vme_start), ! 904: (vm_size_t)(end - entry->vme_end))) { ! 905: ! 906: /* ! 907: * Coalesced the two objects - can extend ! 908: * the previous map entry to include the ! 909: * new range. ! 910: */ ! 911: map->size += (end - entry->vme_end); ! 912: entry->vme_end = end; ! 913: RETURN(KERN_SUCCESS); ! 914: } ! 915: } ! 916: ! 917: /* ! 918: * Create a new entry ! 919: */ ! 920: ! 921: /**/ { ! 922: register vm_map_entry_t new_entry; ! 923: ! 924: new_entry = vm_map_entry_create(map); ! 925: ! 926: new_entry->vme_start = start; ! 927: new_entry->vme_end = end; ! 928: ! 929: new_entry->is_shared = FALSE; ! 930: new_entry->is_sub_map = FALSE; ! 931: new_entry->object.vm_object = object; ! 932: new_entry->offset = offset; ! 933: ! 934: new_entry->needs_copy = needs_copy; ! 935: ! 936: new_entry->inheritance = inheritance; ! 937: new_entry->protection = cur_protection; ! 938: new_entry->max_protection = max_protection; ! 939: new_entry->wired_count = 0; ! 940: new_entry->user_wired_count = 0; ! 941: ! 942: new_entry->in_transition = FALSE; ! 943: new_entry->needs_wakeup = FALSE; ! 944: new_entry->projected_on = 0; ! 945: ! 946: /* ! 947: * Insert the new entry into the list ! 948: */ ! 949: ! 950: vm_map_entry_link(map, entry, new_entry); ! 951: map->size += size; ! 952: ! 953: /* ! 954: * Update the free space hint and the lookup hint ! 955: */ ! 956: ! 957: if ((map->first_free == entry) && ! 958: ((entry == vm_map_to_entry(map) ? map->min_offset : entry->vme_end) ! 959: >= new_entry->vme_start)) ! 960: map->first_free = new_entry; ! 961: ! 962: SAVE_HINT(map, new_entry); ! 963: ! 964: vm_map_unlock(map); ! 965: ! 966: if ((object != VM_OBJECT_NULL) && ! 967: (vm_map_pmap_enter_enable) && ! 968: (!anywhere) && ! 969: (!needs_copy) && ! 970: (size < (128*1024))) { ! 971: vm_map_pmap_enter(map, start, end, ! 972: object, offset, cur_protection); ! 973: } ! 974: ! 975: return(result); ! 976: /**/ } ! 977: ! 978: BailOut: ; ! 979: ! 980: vm_map_unlock(map); ! 981: return(result); ! 982: ! 983: #undef RETURN ! 984: } ! 985: ! 986: /* ! 987: * vm_map_clip_start: [ internal use only ] ! 988: * ! 989: * Asserts that the given entry begins at or after ! 990: * the specified address; if necessary, ! 991: * it splits the entry into two. ! 992: */ ! 993: void _vm_map_clip_start(); ! 994: #define vm_map_clip_start(map, entry, startaddr) \ ! 995: MACRO_BEGIN \ ! 996: if ((startaddr) > (entry)->vme_start) \ ! 997: _vm_map_clip_start(&(map)->hdr,(entry),(startaddr)); \ ! 998: MACRO_END ! 999: ! 1000: void _vm_map_copy_clip_start(); ! 1001: #define vm_map_copy_clip_start(copy, entry, startaddr) \ ! 1002: MACRO_BEGIN \ ! 1003: if ((startaddr) > (entry)->vme_start) \ ! 1004: _vm_map_clip_start(&(copy)->cpy_hdr,(entry),(startaddr)); \ ! 1005: MACRO_END ! 1006: ! 1007: /* ! 1008: * This routine is called only when it is known that ! 1009: * the entry must be split. ! 1010: */ ! 1011: void _vm_map_clip_start(map_header, entry, start) ! 1012: register struct vm_map_header *map_header; ! 1013: register vm_map_entry_t entry; ! 1014: register vm_offset_t start; ! 1015: { ! 1016: register vm_map_entry_t new_entry; ! 1017: ! 1018: /* ! 1019: * Split off the front portion -- ! 1020: * note that we must insert the new ! 1021: * entry BEFORE this one, so that ! 1022: * this entry has the specified starting ! 1023: * address. ! 1024: */ ! 1025: ! 1026: new_entry = _vm_map_entry_create(map_header); ! 1027: vm_map_entry_copy_full(new_entry, entry); ! 1028: ! 1029: new_entry->vme_end = start; ! 1030: entry->offset += (start - entry->vme_start); ! 1031: entry->vme_start = start; ! 1032: ! 1033: _vm_map_entry_link(map_header, entry->vme_prev, new_entry); ! 1034: ! 1035: if (entry->is_sub_map) ! 1036: vm_map_reference(new_entry->object.sub_map); ! 1037: else ! 1038: vm_object_reference(new_entry->object.vm_object); ! 1039: } ! 1040: ! 1041: /* ! 1042: * vm_map_clip_end: [ internal use only ] ! 1043: * ! 1044: * Asserts that the given entry ends at or before ! 1045: * the specified address; if necessary, ! 1046: * it splits the entry into two. ! 1047: */ ! 1048: void _vm_map_clip_end(); ! 1049: #define vm_map_clip_end(map, entry, endaddr) \ ! 1050: MACRO_BEGIN \ ! 1051: if ((endaddr) < (entry)->vme_end) \ ! 1052: _vm_map_clip_end(&(map)->hdr,(entry),(endaddr)); \ ! 1053: MACRO_END ! 1054: ! 1055: void _vm_map_copy_clip_end(); ! 1056: #define vm_map_copy_clip_end(copy, entry, endaddr) \ ! 1057: MACRO_BEGIN \ ! 1058: if ((endaddr) < (entry)->vme_end) \ ! 1059: _vm_map_clip_end(&(copy)->cpy_hdr,(entry),(endaddr)); \ ! 1060: MACRO_END ! 1061: ! 1062: /* ! 1063: * This routine is called only when it is known that ! 1064: * the entry must be split. ! 1065: */ ! 1066: void _vm_map_clip_end(map_header, entry, end) ! 1067: register struct vm_map_header *map_header; ! 1068: register vm_map_entry_t entry; ! 1069: register vm_offset_t end; ! 1070: { ! 1071: register vm_map_entry_t new_entry; ! 1072: ! 1073: /* ! 1074: * Create a new entry and insert it ! 1075: * AFTER the specified entry ! 1076: */ ! 1077: ! 1078: new_entry = _vm_map_entry_create(map_header); ! 1079: vm_map_entry_copy_full(new_entry, entry); ! 1080: ! 1081: new_entry->vme_start = entry->vme_end = end; ! 1082: new_entry->offset += (end - entry->vme_start); ! 1083: ! 1084: _vm_map_entry_link(map_header, entry, new_entry); ! 1085: ! 1086: if (entry->is_sub_map) ! 1087: vm_map_reference(new_entry->object.sub_map); ! 1088: else ! 1089: vm_object_reference(new_entry->object.vm_object); ! 1090: } ! 1091: ! 1092: /* ! 1093: * VM_MAP_RANGE_CHECK: [ internal use only ] ! 1094: * ! 1095: * Asserts that the starting and ending region ! 1096: * addresses fall within the valid range of the map. ! 1097: */ ! 1098: #define VM_MAP_RANGE_CHECK(map, start, end) \ ! 1099: { \ ! 1100: if (start < vm_map_min(map)) \ ! 1101: start = vm_map_min(map); \ ! 1102: if (end > vm_map_max(map)) \ ! 1103: end = vm_map_max(map); \ ! 1104: if (start > end) \ ! 1105: start = end; \ ! 1106: } ! 1107: ! 1108: /* ! 1109: * vm_map_submap: [ kernel use only ] ! 1110: * ! 1111: * Mark the given range as handled by a subordinate map. ! 1112: * ! 1113: * This range must have been created with vm_map_find using ! 1114: * the vm_submap_object, and no other operations may have been ! 1115: * performed on this range prior to calling vm_map_submap. ! 1116: * ! 1117: * Only a limited number of operations can be performed ! 1118: * within this rage after calling vm_map_submap: ! 1119: * vm_fault ! 1120: * [Don't try vm_map_copyin!] ! 1121: * ! 1122: * To remove a submapping, one must first remove the ! 1123: * range from the superior map, and then destroy the ! 1124: * submap (if desired). [Better yet, don't try it.] ! 1125: */ ! 1126: kern_return_t vm_map_submap(map, start, end, submap) ! 1127: register vm_map_t map; ! 1128: register vm_offset_t start; ! 1129: register vm_offset_t end; ! 1130: vm_map_t submap; ! 1131: { ! 1132: vm_map_entry_t entry; ! 1133: register kern_return_t result = KERN_INVALID_ARGUMENT; ! 1134: register vm_object_t object; ! 1135: ! 1136: vm_map_lock(map); ! 1137: ! 1138: VM_MAP_RANGE_CHECK(map, start, end); ! 1139: ! 1140: if (vm_map_lookup_entry(map, start, &entry)) { ! 1141: vm_map_clip_start(map, entry, start); ! 1142: } ! 1143: else ! 1144: entry = entry->vme_next; ! 1145: ! 1146: vm_map_clip_end(map, entry, end); ! 1147: ! 1148: if ((entry->vme_start == start) && (entry->vme_end == end) && ! 1149: (!entry->is_sub_map) && ! 1150: ((object = entry->object.vm_object) == vm_submap_object) && ! 1151: (object->resident_page_count == 0) && ! 1152: (object->copy == VM_OBJECT_NULL) && ! 1153: (object->shadow == VM_OBJECT_NULL) && ! 1154: (!object->pager_created)) { ! 1155: entry->object.vm_object = VM_OBJECT_NULL; ! 1156: vm_object_deallocate(object); ! 1157: entry->is_sub_map = TRUE; ! 1158: vm_map_reference(entry->object.sub_map = submap); ! 1159: result = KERN_SUCCESS; ! 1160: } ! 1161: vm_map_unlock(map); ! 1162: ! 1163: return(result); ! 1164: } ! 1165: ! 1166: /* ! 1167: * vm_map_protect: ! 1168: * ! 1169: * Sets the protection of the specified address ! 1170: * region in the target map. If "set_max" is ! 1171: * specified, the maximum protection is to be set; ! 1172: * otherwise, only the current protection is affected. ! 1173: */ ! 1174: kern_return_t vm_map_protect(map, start, end, new_prot, set_max) ! 1175: register vm_map_t map; ! 1176: register vm_offset_t start; ! 1177: register vm_offset_t end; ! 1178: register vm_prot_t new_prot; ! 1179: register boolean_t set_max; ! 1180: { ! 1181: register vm_map_entry_t current; ! 1182: vm_map_entry_t entry; ! 1183: ! 1184: vm_map_lock(map); ! 1185: ! 1186: VM_MAP_RANGE_CHECK(map, start, end); ! 1187: ! 1188: if (vm_map_lookup_entry(map, start, &entry)) { ! 1189: vm_map_clip_start(map, entry, start); ! 1190: } ! 1191: else ! 1192: entry = entry->vme_next; ! 1193: ! 1194: /* ! 1195: * Make a first pass to check for protection ! 1196: * violations. ! 1197: */ ! 1198: ! 1199: current = entry; ! 1200: while ((current != vm_map_to_entry(map)) && ! 1201: (current->vme_start < end)) { ! 1202: ! 1203: if (current->is_sub_map) { ! 1204: vm_map_unlock(map); ! 1205: return(KERN_INVALID_ARGUMENT); ! 1206: } ! 1207: if ((new_prot & (VM_PROT_NOTIFY | current->max_protection)) ! 1208: != new_prot) { ! 1209: vm_map_unlock(map); ! 1210: return(KERN_PROTECTION_FAILURE); ! 1211: } ! 1212: ! 1213: current = current->vme_next; ! 1214: } ! 1215: ! 1216: /* ! 1217: * Go back and fix up protections. ! 1218: * [Note that clipping is not necessary the second time.] ! 1219: */ ! 1220: ! 1221: current = entry; ! 1222: ! 1223: while ((current != vm_map_to_entry(map)) && ! 1224: (current->vme_start < end)) { ! 1225: ! 1226: vm_prot_t old_prot; ! 1227: ! 1228: vm_map_clip_end(map, current, end); ! 1229: ! 1230: old_prot = current->protection; ! 1231: if (set_max) ! 1232: current->protection = ! 1233: (current->max_protection = new_prot) & ! 1234: old_prot; ! 1235: else ! 1236: current->protection = new_prot; ! 1237: ! 1238: /* ! 1239: * Update physical map if necessary. ! 1240: */ ! 1241: ! 1242: if (current->protection != old_prot) { ! 1243: pmap_protect(map->pmap, current->vme_start, ! 1244: current->vme_end, ! 1245: current->protection); ! 1246: } ! 1247: current = current->vme_next; ! 1248: } ! 1249: ! 1250: vm_map_unlock(map); ! 1251: return(KERN_SUCCESS); ! 1252: } ! 1253: ! 1254: /* ! 1255: * vm_map_inherit: ! 1256: * ! 1257: * Sets the inheritance of the specified address ! 1258: * range in the target map. Inheritance ! 1259: * affects how the map will be shared with ! 1260: * child maps at the time of vm_map_fork. ! 1261: */ ! 1262: kern_return_t vm_map_inherit(map, start, end, new_inheritance) ! 1263: register vm_map_t map; ! 1264: register vm_offset_t start; ! 1265: register vm_offset_t end; ! 1266: register vm_inherit_t new_inheritance; ! 1267: { ! 1268: register vm_map_entry_t entry; ! 1269: vm_map_entry_t temp_entry; ! 1270: ! 1271: vm_map_lock(map); ! 1272: ! 1273: VM_MAP_RANGE_CHECK(map, start, end); ! 1274: ! 1275: if (vm_map_lookup_entry(map, start, &temp_entry)) { ! 1276: entry = temp_entry; ! 1277: vm_map_clip_start(map, entry, start); ! 1278: } ! 1279: else ! 1280: entry = temp_entry->vme_next; ! 1281: ! 1282: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) { ! 1283: vm_map_clip_end(map, entry, end); ! 1284: ! 1285: entry->inheritance = new_inheritance; ! 1286: ! 1287: entry = entry->vme_next; ! 1288: } ! 1289: ! 1290: vm_map_unlock(map); ! 1291: return(KERN_SUCCESS); ! 1292: } ! 1293: ! 1294: /* ! 1295: * vm_map_pageable_common: ! 1296: * ! 1297: * Sets the pageability of the specified address ! 1298: * range in the target map. Regions specified ! 1299: * as not pageable require locked-down physical ! 1300: * memory and physical page maps. access_type indicates ! 1301: * types of accesses that must not generate page faults. ! 1302: * This is checked against protection of memory being locked-down. ! 1303: * access_type of VM_PROT_NONE makes memory pageable. ! 1304: * ! 1305: * The map must not be locked, but a reference ! 1306: * must remain to the map throughout the call. ! 1307: * ! 1308: * Callers should use macros in vm/vm_map.h (i.e. vm_map_pageable, ! 1309: * or vm_map_pageable_user); don't call vm_map_pageable directly. ! 1310: */ ! 1311: kern_return_t vm_map_pageable_common(map, start, end, access_type, user_wire) ! 1312: register vm_map_t map; ! 1313: register vm_offset_t start; ! 1314: register vm_offset_t end; ! 1315: register vm_prot_t access_type; ! 1316: boolean_t user_wire; ! 1317: { ! 1318: register vm_map_entry_t entry; ! 1319: vm_map_entry_t start_entry; ! 1320: ! 1321: vm_map_lock(map); ! 1322: ! 1323: VM_MAP_RANGE_CHECK(map, start, end); ! 1324: ! 1325: if (vm_map_lookup_entry(map, start, &start_entry)) { ! 1326: entry = start_entry; ! 1327: /* ! 1328: * vm_map_clip_start will be done later. ! 1329: */ ! 1330: } ! 1331: else { ! 1332: /* ! 1333: * Start address is not in map; this is fatal. ! 1334: */ ! 1335: vm_map_unlock(map); ! 1336: return(KERN_FAILURE); ! 1337: } ! 1338: ! 1339: /* ! 1340: * Actions are rather different for wiring and unwiring, ! 1341: * so we have two separate cases. ! 1342: */ ! 1343: ! 1344: if (access_type == VM_PROT_NONE) { ! 1345: ! 1346: vm_map_clip_start(map, entry, start); ! 1347: ! 1348: /* ! 1349: * Unwiring. First ensure that the range to be ! 1350: * unwired is really wired down. ! 1351: */ ! 1352: while ((entry != vm_map_to_entry(map)) && ! 1353: (entry->vme_start < end)) { ! 1354: ! 1355: if ((entry->wired_count == 0) || ! 1356: ((entry->vme_end < end) && ! 1357: ((entry->vme_next == vm_map_to_entry(map)) || ! 1358: (entry->vme_next->vme_start > entry->vme_end))) || ! 1359: (user_wire && (entry->user_wired_count == 0))) { ! 1360: vm_map_unlock(map); ! 1361: return(KERN_INVALID_ARGUMENT); ! 1362: } ! 1363: entry = entry->vme_next; ! 1364: } ! 1365: ! 1366: /* ! 1367: * Now decrement the wiring count for each region. ! 1368: * If a region becomes completely unwired, ! 1369: * unwire its physical pages and mappings. ! 1370: */ ! 1371: entry = start_entry; ! 1372: while ((entry != vm_map_to_entry(map)) && ! 1373: (entry->vme_start < end)) { ! 1374: vm_map_clip_end(map, entry, end); ! 1375: ! 1376: if (user_wire) { ! 1377: if (--(entry->user_wired_count) == 0) ! 1378: entry->wired_count--; ! 1379: } ! 1380: else { ! 1381: entry->wired_count--; ! 1382: } ! 1383: ! 1384: if (entry->wired_count == 0) ! 1385: vm_fault_unwire(map, entry); ! 1386: ! 1387: entry = entry->vme_next; ! 1388: } ! 1389: } ! 1390: ! 1391: else { ! 1392: /* ! 1393: * Wiring. We must do this in two passes: ! 1394: * ! 1395: * 1. Holding the write lock, we create any shadow ! 1396: * or zero-fill objects that need to be created. ! 1397: * Then we clip each map entry to the region to be ! 1398: * wired and increment its wiring count. We ! 1399: * create objects before clipping the map entries ! 1400: * to avoid object proliferation. ! 1401: * ! 1402: * 2. We downgrade to a read lock, and call ! 1403: * vm_fault_wire to fault in the pages for any ! 1404: * newly wired area (wired_count is 1). ! 1405: * ! 1406: * Downgrading to a read lock for vm_fault_wire avoids ! 1407: * a possible deadlock with another thread that may have ! 1408: * faulted on one of the pages to be wired (it would mark ! 1409: * the page busy, blocking us, then in turn block on the ! 1410: * map lock that we hold). Because of problems in the ! 1411: * recursive lock package, we cannot upgrade to a write ! 1412: * lock in vm_map_lookup. Thus, any actions that require ! 1413: * the write lock must be done beforehand. Because we ! 1414: * keep the read lock on the map, the copy-on-write ! 1415: * status of the entries we modify here cannot change. ! 1416: */ ! 1417: ! 1418: /* ! 1419: * Pass 1. ! 1420: */ ! 1421: while ((entry != vm_map_to_entry(map)) && ! 1422: (entry->vme_start < end)) { ! 1423: vm_map_clip_end(map, entry, end); ! 1424: ! 1425: if (entry->wired_count == 0) { ! 1426: ! 1427: /* ! 1428: * Perform actions of vm_map_lookup that need ! 1429: * the write lock on the map: create a shadow ! 1430: * object for a copy-on-write region, or an ! 1431: * object for a zero-fill region. ! 1432: */ ! 1433: if (entry->needs_copy && ! 1434: ((entry->protection & VM_PROT_WRITE) != 0)) { ! 1435: ! 1436: vm_object_shadow(&entry->object.vm_object, ! 1437: &entry->offset, ! 1438: (vm_size_t)(entry->vme_end ! 1439: - entry->vme_start)); ! 1440: entry->needs_copy = FALSE; ! 1441: } ! 1442: if (entry->object.vm_object == VM_OBJECT_NULL) { ! 1443: entry->object.vm_object = ! 1444: vm_object_allocate( ! 1445: (vm_size_t)(entry->vme_end ! 1446: - entry->vme_start)); ! 1447: entry->offset = (vm_offset_t)0; ! 1448: } ! 1449: } ! 1450: vm_map_clip_start(map, entry, start); ! 1451: vm_map_clip_end(map, entry, end); ! 1452: ! 1453: if (user_wire) { ! 1454: if ((entry->user_wired_count)++ == 0) ! 1455: entry->wired_count++; ! 1456: } ! 1457: else { ! 1458: entry->wired_count++; ! 1459: } ! 1460: ! 1461: /* ! 1462: * Check for holes and protection mismatch. ! 1463: * Holes: Next entry should be contiguous unless ! 1464: * this is the end of the region. ! 1465: * Protection: Access requested must be allowed. ! 1466: */ ! 1467: if (((entry->vme_end < end) && ! 1468: ((entry->vme_next == vm_map_to_entry(map)) || ! 1469: (entry->vme_next->vme_start > entry->vme_end))) || ! 1470: ((entry->protection & access_type) != access_type)) { ! 1471: /* ! 1472: * Found a hole or protection problem. ! 1473: * Object creation actions ! 1474: * do not need to be undone, but the ! 1475: * wired counts need to be restored. ! 1476: */ ! 1477: while ((entry != vm_map_to_entry(map)) && ! 1478: (entry->vme_end > start)) { ! 1479: if (user_wire) { ! 1480: if (--(entry->user_wired_count) == 0) ! 1481: entry->wired_count--; ! 1482: } ! 1483: else { ! 1484: entry->wired_count--; ! 1485: } ! 1486: ! 1487: entry = entry->vme_prev; ! 1488: } ! 1489: ! 1490: vm_map_unlock(map); ! 1491: return(KERN_FAILURE); ! 1492: } ! 1493: entry = entry->vme_next; ! 1494: } ! 1495: ! 1496: /* ! 1497: * Pass 2. ! 1498: */ ! 1499: ! 1500: /* ! 1501: * HACK HACK HACK HACK ! 1502: * ! 1503: * If we are wiring in the kernel map or a submap of it, ! 1504: * unlock the map to avoid deadlocks. We trust that the ! 1505: * kernel threads are well-behaved, and therefore will ! 1506: * not do anything destructive to this region of the map ! 1507: * while we have it unlocked. We cannot trust user threads ! 1508: * to do the same. ! 1509: * ! 1510: * HACK HACK HACK HACK ! 1511: */ ! 1512: if (vm_map_pmap(map) == kernel_pmap) { ! 1513: vm_map_unlock(map); /* trust me ... */ ! 1514: } ! 1515: else { ! 1516: vm_map_lock_set_recursive(map); ! 1517: vm_map_lock_write_to_read(map); ! 1518: } ! 1519: ! 1520: entry = start_entry; ! 1521: while (entry != vm_map_to_entry(map) && ! 1522: entry->vme_start < end) { ! 1523: /* ! 1524: * Wiring cases: ! 1525: * Kernel: wired == 1 && user_wired == 0 ! 1526: * User: wired == 1 && user_wired == 1 ! 1527: * ! 1528: * Don't need to wire if either is > 1. wired = 0 && ! 1529: * user_wired == 1 can't happen. ! 1530: */ ! 1531: ! 1532: /* ! 1533: * XXX This assumes that the faults always succeed. ! 1534: */ ! 1535: if ((entry->wired_count == 1) && ! 1536: (entry->user_wired_count <= 1)) { ! 1537: vm_fault_wire(map, entry); ! 1538: } ! 1539: entry = entry->vme_next; ! 1540: } ! 1541: ! 1542: if (vm_map_pmap(map) == kernel_pmap) { ! 1543: vm_map_lock(map); ! 1544: } ! 1545: else { ! 1546: vm_map_lock_clear_recursive(map); ! 1547: } ! 1548: } ! 1549: ! 1550: vm_map_unlock(map); ! 1551: ! 1552: return(KERN_SUCCESS); ! 1553: } ! 1554: ! 1555: /* ! 1556: * vm_map_entry_delete: [ internal use only ] ! 1557: * ! 1558: * Deallocate the given entry from the target map. ! 1559: */ ! 1560: void vm_map_entry_delete(map, entry) ! 1561: register vm_map_t map; ! 1562: register vm_map_entry_t entry; ! 1563: { ! 1564: register vm_offset_t s, e; ! 1565: register vm_object_t object; ! 1566: extern vm_object_t kernel_object; ! 1567: ! 1568: s = entry->vme_start; ! 1569: e = entry->vme_end; ! 1570: ! 1571: /*Check if projected buffer*/ ! 1572: if (map != kernel_map && entry->projected_on != 0) { ! 1573: /*Check if projected kernel entry is persistent; ! 1574: may only manipulate directly if it is*/ ! 1575: if (entry->projected_on->projected_on == 0) ! 1576: entry->wired_count = 0; /*Avoid unwire fault*/ ! 1577: else ! 1578: return; ! 1579: } ! 1580: ! 1581: /* ! 1582: * Get the object. Null objects cannot have pmap entries. ! 1583: */ ! 1584: ! 1585: if ((object = entry->object.vm_object) != VM_OBJECT_NULL) { ! 1586: ! 1587: /* ! 1588: * Unwire before removing addresses from the pmap; ! 1589: * otherwise, unwiring will put the entries back in ! 1590: * the pmap. ! 1591: */ ! 1592: ! 1593: if (entry->wired_count != 0) { ! 1594: vm_fault_unwire(map, entry); ! 1595: entry->wired_count = 0; ! 1596: entry->user_wired_count = 0; ! 1597: } ! 1598: ! 1599: /* ! 1600: * If the object is shared, we must remove ! 1601: * *all* references to this data, since we can't ! 1602: * find all of the physical maps which are sharing ! 1603: * it. ! 1604: */ ! 1605: ! 1606: if (object == kernel_object) { ! 1607: vm_object_lock(object); ! 1608: vm_object_page_remove(object, entry->offset, ! 1609: entry->offset + (e - s)); ! 1610: vm_object_unlock(object); ! 1611: } else if (entry->is_shared) { ! 1612: vm_object_pmap_remove(object, ! 1613: entry->offset, ! 1614: entry->offset + (e - s)); ! 1615: } ! 1616: else { ! 1617: pmap_remove(map->pmap, s, e); ! 1618: } ! 1619: } ! 1620: ! 1621: /* ! 1622: * Deallocate the object only after removing all ! 1623: * pmap entries pointing to its pages. ! 1624: */ ! 1625: ! 1626: if (entry->is_sub_map) ! 1627: vm_map_deallocate(entry->object.sub_map); ! 1628: else ! 1629: vm_object_deallocate(entry->object.vm_object); ! 1630: ! 1631: vm_map_entry_unlink(map, entry); ! 1632: map->size -= e - s; ! 1633: ! 1634: vm_map_entry_dispose(map, entry); ! 1635: } ! 1636: ! 1637: /* ! 1638: * vm_map_delete: [ internal use only ] ! 1639: * ! 1640: * Deallocates the given address range from the target ! 1641: * map. ! 1642: */ ! 1643: ! 1644: kern_return_t vm_map_delete(map, start, end) ! 1645: register vm_map_t map; ! 1646: register vm_offset_t start; ! 1647: register vm_offset_t end; ! 1648: { ! 1649: vm_map_entry_t entry; ! 1650: vm_map_entry_t first_entry; ! 1651: ! 1652: /* ! 1653: * Find the start of the region, and clip it ! 1654: */ ! 1655: ! 1656: if (!vm_map_lookup_entry(map, start, &first_entry)) ! 1657: entry = first_entry->vme_next; ! 1658: else { ! 1659: entry = first_entry; ! 1660: #if NORMA_IPC_xxx ! 1661: /* ! 1662: * XXX Had to disable this code because: ! 1663: ! 1664: _vm_map_delete(c0804b78,c2198000,c219a000,0,c219a000)+df ! 1665: [vm/vm_map.c:2007] ! 1666: _vm_map_remove(c0804b78,c2198000,c219a000,c0817834, ! 1667: c081786c)+42 [vm/vm_map.c:2094] ! 1668: _kmem_io_map_deallocate(c0804b78,c2198000,2000,c0817834, ! 1669: c081786c)+43 [vm/vm_kern.c:818] ! 1670: _device_write_dealloc(c081786c)+117 [device/ds_routines.c:814] ! 1671: _ds_write_done(c081786c,0)+2e [device/ds_routines.c:848] ! 1672: _io_done_thread_continue(c08150c0,c21d4e14,c21d4e30,c08150c0, ! 1673: c080c114)+14 [device/ds_routines.c:1350] ! 1674: ! 1675: */ ! 1676: if (start > entry->vme_start ! 1677: && end == entry->vme_end ! 1678: && ! entry->wired_count /* XXX ??? */ ! 1679: && ! entry->is_shared ! 1680: && ! entry->projected_on ! 1681: && ! entry->is_sub_map) { ! 1682: extern vm_object_t kernel_object; ! 1683: register vm_object_t object = entry->object.vm_object; ! 1684: ! 1685: /* ! 1686: * The region to be deleted lives at the end ! 1687: * of this entry, and thus all we have to do is ! 1688: * truncate the entry. ! 1689: * ! 1690: * This special case is necessary if we want ! 1691: * coalescing to do us any good. ! 1692: * ! 1693: * XXX Do we have to adjust object size? ! 1694: */ ! 1695: if (object == kernel_object) { ! 1696: vm_object_lock(object); ! 1697: vm_object_page_remove(object, ! 1698: entry->offset + start, ! 1699: entry->offset + ! 1700: (end - start)); ! 1701: vm_object_unlock(object); ! 1702: } else if (entry->is_shared) { ! 1703: vm_object_pmap_remove(object, ! 1704: entry->offset + start, ! 1705: entry->offset + ! 1706: (end - start)); ! 1707: } else { ! 1708: pmap_remove(map->pmap, start, end); ! 1709: } ! 1710: object->size -= (end - start); /* XXX */ ! 1711: ! 1712: entry->vme_end = start; ! 1713: map->size -= (end - start); ! 1714: ! 1715: if (map->wait_for_space) { ! 1716: thread_wakeup((event_t) map); ! 1717: } ! 1718: return KERN_SUCCESS; ! 1719: } ! 1720: #endif /* NORMA_IPC */ ! 1721: vm_map_clip_start(map, entry, start); ! 1722: ! 1723: /* ! 1724: * Fix the lookup hint now, rather than each ! 1725: * time though the loop. ! 1726: */ ! 1727: ! 1728: SAVE_HINT(map, entry->vme_prev); ! 1729: } ! 1730: ! 1731: /* ! 1732: * Save the free space hint ! 1733: */ ! 1734: ! 1735: if (map->first_free->vme_start >= start) ! 1736: map->first_free = entry->vme_prev; ! 1737: ! 1738: /* ! 1739: * Step through all entries in this region ! 1740: */ ! 1741: ! 1742: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) { ! 1743: vm_map_entry_t next; ! 1744: ! 1745: vm_map_clip_end(map, entry, end); ! 1746: ! 1747: /* ! 1748: * If the entry is in transition, we must wait ! 1749: * for it to exit that state. It could be clipped ! 1750: * while we leave the map unlocked. ! 1751: */ ! 1752: if(entry->in_transition) { ! 1753: /* ! 1754: * Say that we are waiting, and wait for entry. ! 1755: */ ! 1756: entry->needs_wakeup = TRUE; ! 1757: vm_map_entry_wait(map, FALSE); ! 1758: vm_map_lock(map); ! 1759: ! 1760: /* ! 1761: * The entry could have been clipped or it ! 1762: * may not exist anymore. look it up again. ! 1763: */ ! 1764: if(!vm_map_lookup_entry(map, start, &entry)) { ! 1765: entry = entry->vme_next; ! 1766: } ! 1767: continue; ! 1768: } ! 1769: ! 1770: next = entry->vme_next; ! 1771: ! 1772: vm_map_entry_delete(map, entry); ! 1773: entry = next; ! 1774: } ! 1775: ! 1776: if (map->wait_for_space) ! 1777: thread_wakeup((event_t) map); ! 1778: ! 1779: return(KERN_SUCCESS); ! 1780: } ! 1781: ! 1782: /* ! 1783: * vm_map_remove: ! 1784: * ! 1785: * Remove the given address range from the target map. ! 1786: * This is the exported form of vm_map_delete. ! 1787: */ ! 1788: kern_return_t vm_map_remove(map, start, end) ! 1789: register vm_map_t map; ! 1790: register vm_offset_t start; ! 1791: register vm_offset_t end; ! 1792: { ! 1793: register kern_return_t result; ! 1794: ! 1795: vm_map_lock(map); ! 1796: VM_MAP_RANGE_CHECK(map, start, end); ! 1797: result = vm_map_delete(map, start, end); ! 1798: vm_map_unlock(map); ! 1799: ! 1800: return(result); ! 1801: } ! 1802: ! 1803: ! 1804: /* ! 1805: * vm_map_copy_steal_pages: ! 1806: * ! 1807: * Steal all the pages from a vm_map_copy page_list by copying ones ! 1808: * that have not already been stolen. ! 1809: */ ! 1810: void ! 1811: vm_map_copy_steal_pages(copy) ! 1812: vm_map_copy_t copy; ! 1813: { ! 1814: register vm_page_t m, new_m; ! 1815: register int i; ! 1816: vm_object_t object; ! 1817: ! 1818: for (i = 0; i < copy->cpy_npages; i++) { ! 1819: ! 1820: /* ! 1821: * If the page is not tabled, then it's already stolen. ! 1822: */ ! 1823: m = copy->cpy_page_list[i]; ! 1824: if (!m->tabled) ! 1825: continue; ! 1826: ! 1827: /* ! 1828: * Page was not stolen, get a new ! 1829: * one and do the copy now. ! 1830: */ ! 1831: while ((new_m = vm_page_grab(FALSE)) == VM_PAGE_NULL) { ! 1832: VM_PAGE_WAIT((void(*)()) 0); ! 1833: } ! 1834: ! 1835: vm_page_copy(m, new_m); ! 1836: ! 1837: object = m->object; ! 1838: vm_object_lock(object); ! 1839: vm_page_lock_queues(); ! 1840: if (!m->active && !m->inactive) ! 1841: vm_page_activate(m); ! 1842: vm_page_unlock_queues(); ! 1843: PAGE_WAKEUP_DONE(m); ! 1844: vm_object_paging_end(object); ! 1845: vm_object_unlock(object); ! 1846: ! 1847: copy->cpy_page_list[i] = new_m; ! 1848: } ! 1849: } ! 1850: ! 1851: /* ! 1852: * vm_map_copy_page_discard: ! 1853: * ! 1854: * Get rid of the pages in a page_list copy. If the pages are ! 1855: * stolen, they are freed. If the pages are not stolen, they ! 1856: * are unbusied, and associated state is cleaned up. ! 1857: */ ! 1858: void vm_map_copy_page_discard(copy) ! 1859: vm_map_copy_t copy; ! 1860: { ! 1861: while (copy->cpy_npages > 0) { ! 1862: vm_page_t m; ! 1863: ! 1864: if((m = copy->cpy_page_list[--(copy->cpy_npages)]) != ! 1865: VM_PAGE_NULL) { ! 1866: ! 1867: /* ! 1868: * If it's not in the table, then it's ! 1869: * a stolen page that goes back ! 1870: * to the free list. Else it belongs ! 1871: * to some object, and we hold a ! 1872: * paging reference on that object. ! 1873: */ ! 1874: if (!m->tabled) { ! 1875: VM_PAGE_FREE(m); ! 1876: } ! 1877: else { ! 1878: vm_object_t object; ! 1879: ! 1880: object = m->object; ! 1881: ! 1882: vm_object_lock(object); ! 1883: vm_page_lock_queues(); ! 1884: if (!m->active && !m->inactive) ! 1885: vm_page_activate(m); ! 1886: vm_page_unlock_queues(); ! 1887: ! 1888: PAGE_WAKEUP_DONE(m); ! 1889: vm_object_paging_end(object); ! 1890: vm_object_unlock(object); ! 1891: } ! 1892: } ! 1893: } ! 1894: } ! 1895: ! 1896: /* ! 1897: * Routine: vm_map_copy_discard ! 1898: * ! 1899: * Description: ! 1900: * Dispose of a map copy object (returned by ! 1901: * vm_map_copyin). ! 1902: */ ! 1903: void ! 1904: vm_map_copy_discard(copy) ! 1905: vm_map_copy_t copy; ! 1906: { ! 1907: free_next_copy: ! 1908: if (copy == VM_MAP_COPY_NULL) ! 1909: return; ! 1910: ! 1911: switch (copy->type) { ! 1912: case VM_MAP_COPY_ENTRY_LIST: ! 1913: while (vm_map_copy_first_entry(copy) != ! 1914: vm_map_copy_to_entry(copy)) { ! 1915: vm_map_entry_t entry = vm_map_copy_first_entry(copy); ! 1916: ! 1917: vm_map_copy_entry_unlink(copy, entry); ! 1918: vm_object_deallocate(entry->object.vm_object); ! 1919: vm_map_copy_entry_dispose(copy, entry); ! 1920: } ! 1921: break; ! 1922: case VM_MAP_COPY_OBJECT: ! 1923: vm_object_deallocate(copy->cpy_object); ! 1924: break; ! 1925: case VM_MAP_COPY_PAGE_LIST: ! 1926: ! 1927: /* ! 1928: * To clean this up, we have to unbusy all the pages ! 1929: * and release the paging references in their objects. ! 1930: */ ! 1931: if (copy->cpy_npages > 0) ! 1932: vm_map_copy_page_discard(copy); ! 1933: ! 1934: /* ! 1935: * If there's a continuation, abort it. The ! 1936: * abort routine releases any storage. ! 1937: */ ! 1938: if (vm_map_copy_has_cont(copy)) { ! 1939: ! 1940: /* ! 1941: * Special case: recognize ! 1942: * vm_map_copy_discard_cont and optimize ! 1943: * here to avoid tail recursion. ! 1944: */ ! 1945: if (copy->cpy_cont == vm_map_copy_discard_cont) { ! 1946: register vm_map_copy_t new_copy; ! 1947: ! 1948: new_copy = (vm_map_copy_t) copy->cpy_cont_args; ! 1949: zfree(vm_map_copy_zone, (vm_offset_t) copy); ! 1950: copy = new_copy; ! 1951: goto free_next_copy; ! 1952: } ! 1953: else { ! 1954: vm_map_copy_abort_cont(copy); ! 1955: } ! 1956: } ! 1957: ! 1958: break; ! 1959: } ! 1960: zfree(vm_map_copy_zone, (vm_offset_t) copy); ! 1961: } ! 1962: ! 1963: /* ! 1964: * Routine: vm_map_copy_copy ! 1965: * ! 1966: * Description: ! 1967: * Move the information in a map copy object to ! 1968: * a new map copy object, leaving the old one ! 1969: * empty. ! 1970: * ! 1971: * This is used by kernel routines that need ! 1972: * to look at out-of-line data (in copyin form) ! 1973: * before deciding whether to return SUCCESS. ! 1974: * If the routine returns FAILURE, the original ! 1975: * copy object will be deallocated; therefore, ! 1976: * these routines must make a copy of the copy ! 1977: * object and leave the original empty so that ! 1978: * deallocation will not fail. ! 1979: */ ! 1980: vm_map_copy_t ! 1981: vm_map_copy_copy(copy) ! 1982: vm_map_copy_t copy; ! 1983: { ! 1984: vm_map_copy_t new_copy; ! 1985: ! 1986: if (copy == VM_MAP_COPY_NULL) ! 1987: return VM_MAP_COPY_NULL; ! 1988: ! 1989: /* ! 1990: * Allocate a new copy object, and copy the information ! 1991: * from the old one into it. ! 1992: */ ! 1993: ! 1994: new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); ! 1995: *new_copy = *copy; ! 1996: ! 1997: if (copy->type == VM_MAP_COPY_ENTRY_LIST) { ! 1998: /* ! 1999: * The links in the entry chain must be ! 2000: * changed to point to the new copy object. ! 2001: */ ! 2002: vm_map_copy_first_entry(copy)->vme_prev ! 2003: = vm_map_copy_to_entry(new_copy); ! 2004: vm_map_copy_last_entry(copy)->vme_next ! 2005: = vm_map_copy_to_entry(new_copy); ! 2006: } ! 2007: ! 2008: /* ! 2009: * Change the old copy object into one that contains ! 2010: * nothing to be deallocated. ! 2011: */ ! 2012: copy->type = VM_MAP_COPY_OBJECT; ! 2013: copy->cpy_object = VM_OBJECT_NULL; ! 2014: ! 2015: /* ! 2016: * Return the new object. ! 2017: */ ! 2018: return new_copy; ! 2019: } ! 2020: ! 2021: /* ! 2022: * Routine: vm_map_copy_discard_cont ! 2023: * ! 2024: * Description: ! 2025: * A version of vm_map_copy_discard that can be called ! 2026: * as a continuation from a vm_map_copy page list. ! 2027: */ ! 2028: kern_return_t vm_map_copy_discard_cont(cont_args, copy_result) ! 2029: vm_map_copyin_args_t cont_args; ! 2030: vm_map_copy_t *copy_result; /* OUT */ ! 2031: { ! 2032: vm_map_copy_discard((vm_map_copy_t) cont_args); ! 2033: if (copy_result != (vm_map_copy_t *)0) ! 2034: *copy_result = VM_MAP_COPY_NULL; ! 2035: return(KERN_SUCCESS); ! 2036: } ! 2037: ! 2038: /* ! 2039: * Routine: vm_map_copy_overwrite ! 2040: * ! 2041: * Description: ! 2042: * Copy the memory described by the map copy ! 2043: * object (copy; returned by vm_map_copyin) onto ! 2044: * the specified destination region (dst_map, dst_addr). ! 2045: * The destination must be writeable. ! 2046: * ! 2047: * Unlike vm_map_copyout, this routine actually ! 2048: * writes over previously-mapped memory. If the ! 2049: * previous mapping was to a permanent (user-supplied) ! 2050: * memory object, it is preserved. ! 2051: * ! 2052: * The attributes (protection and inheritance) of the ! 2053: * destination region are preserved. ! 2054: * ! 2055: * If successful, consumes the copy object. ! 2056: * Otherwise, the caller is responsible for it. ! 2057: * ! 2058: * Implementation notes: ! 2059: * To overwrite temporary virtual memory, it is ! 2060: * sufficient to remove the previous mapping and insert ! 2061: * the new copy. This replacement is done either on ! 2062: * the whole region (if no permanent virtual memory ! 2063: * objects are embedded in the destination region) or ! 2064: * in individual map entries. ! 2065: * ! 2066: * To overwrite permanent virtual memory, it is ! 2067: * necessary to copy each page, as the external ! 2068: * memory management interface currently does not ! 2069: * provide any optimizations. ! 2070: * ! 2071: * Once a page of permanent memory has been overwritten, ! 2072: * it is impossible to interrupt this function; otherwise, ! 2073: * the call would be neither atomic nor location-independent. ! 2074: * The kernel-state portion of a user thread must be ! 2075: * interruptible. ! 2076: * ! 2077: * It may be expensive to forward all requests that might ! 2078: * overwrite permanent memory (vm_write, vm_copy) to ! 2079: * uninterruptible kernel threads. This routine may be ! 2080: * called by interruptible threads; however, success is ! 2081: * not guaranteed -- if the request cannot be performed ! 2082: * atomically and interruptibly, an error indication is ! 2083: * returned. ! 2084: */ ! 2085: kern_return_t vm_map_copy_overwrite(dst_map, dst_addr, copy, interruptible) ! 2086: vm_map_t dst_map; ! 2087: vm_offset_t dst_addr; ! 2088: vm_map_copy_t copy; ! 2089: boolean_t interruptible; ! 2090: { ! 2091: vm_size_t size; ! 2092: vm_offset_t start; ! 2093: vm_map_entry_t tmp_entry; ! 2094: vm_map_entry_t entry; ! 2095: ! 2096: boolean_t contains_permanent_objects = FALSE; ! 2097: ! 2098: interruptible = FALSE; /* XXX */ ! 2099: ! 2100: /* ! 2101: * Check for null copy object. ! 2102: */ ! 2103: ! 2104: if (copy == VM_MAP_COPY_NULL) ! 2105: return(KERN_SUCCESS); ! 2106: ! 2107: /* ! 2108: * Only works for entry lists at the moment. Will ! 2109: * support page lists LATER. ! 2110: */ ! 2111: ! 2112: #if NORMA_IPC ! 2113: vm_map_convert_from_page_list(copy); ! 2114: #else ! 2115: assert(copy->type == VM_MAP_COPY_ENTRY_LIST); ! 2116: #endif ! 2117: ! 2118: /* ! 2119: * Currently this routine only handles page-aligned ! 2120: * regions. Eventually, it should handle misalignments ! 2121: * by actually copying pages. ! 2122: */ ! 2123: ! 2124: if (!page_aligned(copy->offset) || ! 2125: !page_aligned(copy->size) || ! 2126: !page_aligned(dst_addr)) ! 2127: return(KERN_INVALID_ARGUMENT); ! 2128: ! 2129: size = copy->size; ! 2130: ! 2131: if (size == 0) { ! 2132: vm_map_copy_discard(copy); ! 2133: return(KERN_SUCCESS); ! 2134: } ! 2135: ! 2136: /* ! 2137: * Verify that the destination is all writeable ! 2138: * initially. ! 2139: */ ! 2140: start_pass_1: ! 2141: vm_map_lock(dst_map); ! 2142: if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) { ! 2143: vm_map_unlock(dst_map); ! 2144: return(KERN_INVALID_ADDRESS); ! 2145: } ! 2146: vm_map_clip_start(dst_map, tmp_entry, dst_addr); ! 2147: for (entry = tmp_entry;;) { ! 2148: vm_size_t sub_size = (entry->vme_end - entry->vme_start); ! 2149: vm_map_entry_t next = entry->vme_next; ! 2150: ! 2151: if ( ! (entry->protection & VM_PROT_WRITE)) { ! 2152: vm_map_unlock(dst_map); ! 2153: return(KERN_PROTECTION_FAILURE); ! 2154: } ! 2155: ! 2156: /* ! 2157: * If the entry is in transition, we must wait ! 2158: * for it to exit that state. Anything could happen ! 2159: * when we unlock the map, so start over. ! 2160: */ ! 2161: if (entry->in_transition) { ! 2162: ! 2163: /* ! 2164: * Say that we are waiting, and wait for entry. ! 2165: */ ! 2166: entry->needs_wakeup = TRUE; ! 2167: vm_map_entry_wait(dst_map, FALSE); ! 2168: ! 2169: goto start_pass_1; ! 2170: } ! 2171: ! 2172: if (size <= sub_size) ! 2173: break; ! 2174: ! 2175: if ((next == vm_map_to_entry(dst_map)) || ! 2176: (next->vme_start != entry->vme_end)) { ! 2177: vm_map_unlock(dst_map); ! 2178: return(KERN_INVALID_ADDRESS); ! 2179: } ! 2180: ! 2181: ! 2182: /* ! 2183: * Check for permanent objects in the destination. ! 2184: */ ! 2185: ! 2186: if ((entry->object.vm_object != VM_OBJECT_NULL) && ! 2187: !entry->object.vm_object->temporary) ! 2188: contains_permanent_objects = TRUE; ! 2189: ! 2190: size -= sub_size; ! 2191: entry = next; ! 2192: } ! 2193: ! 2194: /* ! 2195: * If there are permanent objects in the destination, then ! 2196: * the copy cannot be interrupted. ! 2197: */ ! 2198: ! 2199: if (interruptible && contains_permanent_objects) ! 2200: return(KERN_FAILURE); /* XXX */ ! 2201: ! 2202: /* ! 2203: * XXXO If there are no permanent objects in the destination, ! 2204: * XXXO and the source and destination map entry zones match, ! 2205: * XXXO and the destination map entry is not shared, ! 2206: * XXXO then the map entries can be deleted and replaced ! 2207: * XXXO with those from the copy. The following code is the ! 2208: * XXXO basic idea of what to do, but there are lots of annoying ! 2209: * XXXO little details about getting protection and inheritance ! 2210: * XXXO right. Should add protection, inheritance, and sharing checks ! 2211: * XXXO to the above pass and make sure that no wiring is involved. ! 2212: */ ! 2213: /* ! 2214: * if (!contains_permanent_objects && ! 2215: * copy->cpy_hdr.entries_pageable == dst_map->hdr.entries_pageable) { ! 2216: * ! 2217: * * ! 2218: * * Run over copy and adjust entries. Steal code ! 2219: * * from vm_map_copyout() to do this. ! 2220: * * ! 2221: * ! 2222: * tmp_entry = tmp_entry->vme_prev; ! 2223: * vm_map_delete(dst_map, dst_addr, dst_addr + copy->size); ! 2224: * vm_map_copy_insert(dst_map, tmp_entry, copy); ! 2225: * ! 2226: * vm_map_unlock(dst_map); ! 2227: * vm_map_copy_discard(copy); ! 2228: * } ! 2229: */ ! 2230: /* ! 2231: * ! 2232: * Make a second pass, overwriting the data ! 2233: * At the beginning of each loop iteration, ! 2234: * the next entry to be overwritten is "tmp_entry" ! 2235: * (initially, the value returned from the lookup above), ! 2236: * and the starting address expected in that entry ! 2237: * is "start". ! 2238: */ ! 2239: ! 2240: start = dst_addr; ! 2241: ! 2242: while (vm_map_copy_first_entry(copy) != vm_map_copy_to_entry(copy)) { ! 2243: vm_map_entry_t copy_entry = vm_map_copy_first_entry(copy); ! 2244: vm_size_t copy_size = (copy_entry->vme_end - copy_entry->vme_start); ! 2245: vm_object_t object; ! 2246: ! 2247: entry = tmp_entry; ! 2248: size = (entry->vme_end - entry->vme_start); ! 2249: /* ! 2250: * Make sure that no holes popped up in the ! 2251: * address map, and that the protection is ! 2252: * still valid, in case the map was unlocked ! 2253: * earlier. ! 2254: */ ! 2255: ! 2256: if (entry->vme_start != start) { ! 2257: vm_map_unlock(dst_map); ! 2258: return(KERN_INVALID_ADDRESS); ! 2259: } ! 2260: assert(entry != vm_map_to_entry(dst_map)); ! 2261: ! 2262: /* ! 2263: * Check protection again ! 2264: */ ! 2265: ! 2266: if ( ! (entry->protection & VM_PROT_WRITE)) { ! 2267: vm_map_unlock(dst_map); ! 2268: return(KERN_PROTECTION_FAILURE); ! 2269: } ! 2270: ! 2271: /* ! 2272: * Adjust to source size first ! 2273: */ ! 2274: ! 2275: if (copy_size < size) { ! 2276: vm_map_clip_end(dst_map, entry, entry->vme_start + copy_size); ! 2277: size = copy_size; ! 2278: } ! 2279: ! 2280: /* ! 2281: * Adjust to destination size ! 2282: */ ! 2283: ! 2284: if (size < copy_size) { ! 2285: vm_map_copy_clip_end(copy, copy_entry, ! 2286: copy_entry->vme_start + size); ! 2287: copy_size = size; ! 2288: } ! 2289: ! 2290: assert((entry->vme_end - entry->vme_start) == size); ! 2291: assert((tmp_entry->vme_end - tmp_entry->vme_start) == size); ! 2292: assert((copy_entry->vme_end - copy_entry->vme_start) == size); ! 2293: ! 2294: /* ! 2295: * If the destination contains temporary unshared memory, ! 2296: * we can perform the copy by throwing it away and ! 2297: * installing the source data. ! 2298: */ ! 2299: ! 2300: object = entry->object.vm_object; ! 2301: if (!entry->is_shared && ! 2302: ((object == VM_OBJECT_NULL) || object->temporary)) { ! 2303: vm_object_t old_object = entry->object.vm_object; ! 2304: vm_offset_t old_offset = entry->offset; ! 2305: ! 2306: entry->object = copy_entry->object; ! 2307: entry->offset = copy_entry->offset; ! 2308: entry->needs_copy = copy_entry->needs_copy; ! 2309: entry->wired_count = 0; ! 2310: entry->user_wired_count = 0; ! 2311: ! 2312: vm_map_copy_entry_unlink(copy, copy_entry); ! 2313: vm_map_copy_entry_dispose(copy, copy_entry); ! 2314: ! 2315: vm_object_pmap_protect( ! 2316: old_object, ! 2317: old_offset, ! 2318: size, ! 2319: dst_map->pmap, ! 2320: tmp_entry->vme_start, ! 2321: VM_PROT_NONE); ! 2322: ! 2323: vm_object_deallocate(old_object); ! 2324: ! 2325: /* ! 2326: * Set up for the next iteration. The map ! 2327: * has not been unlocked, so the next ! 2328: * address should be at the end of this ! 2329: * entry, and the next map entry should be ! 2330: * the one following it. ! 2331: */ ! 2332: ! 2333: start = tmp_entry->vme_end; ! 2334: tmp_entry = tmp_entry->vme_next; ! 2335: } else { ! 2336: vm_map_version_t version; ! 2337: vm_object_t dst_object = entry->object.vm_object; ! 2338: vm_offset_t dst_offset = entry->offset; ! 2339: kern_return_t r; ! 2340: ! 2341: /* ! 2342: * Take an object reference, and record ! 2343: * the map version information so that the ! 2344: * map can be safely unlocked. ! 2345: */ ! 2346: ! 2347: vm_object_reference(dst_object); ! 2348: ! 2349: version.main_timestamp = dst_map->timestamp; ! 2350: ! 2351: vm_map_unlock(dst_map); ! 2352: ! 2353: /* ! 2354: * Copy as much as possible in one pass ! 2355: */ ! 2356: ! 2357: copy_size = size; ! 2358: r = vm_fault_copy( ! 2359: copy_entry->object.vm_object, ! 2360: copy_entry->offset, ! 2361: ©_size, ! 2362: dst_object, ! 2363: dst_offset, ! 2364: dst_map, ! 2365: &version, ! 2366: FALSE /* XXX interruptible */ ); ! 2367: ! 2368: /* ! 2369: * Release the object reference ! 2370: */ ! 2371: ! 2372: vm_object_deallocate(dst_object); ! 2373: ! 2374: /* ! 2375: * If a hard error occurred, return it now ! 2376: */ ! 2377: ! 2378: if (r != KERN_SUCCESS) ! 2379: return(r); ! 2380: ! 2381: if (copy_size != 0) { ! 2382: /* ! 2383: * Dispose of the copied region ! 2384: */ ! 2385: ! 2386: vm_map_copy_clip_end(copy, copy_entry, ! 2387: copy_entry->vme_start + copy_size); ! 2388: vm_map_copy_entry_unlink(copy, copy_entry); ! 2389: vm_object_deallocate(copy_entry->object.vm_object); ! 2390: vm_map_copy_entry_dispose(copy, copy_entry); ! 2391: } ! 2392: ! 2393: /* ! 2394: * Pick up in the destination map where we left off. ! 2395: * ! 2396: * Use the version information to avoid a lookup ! 2397: * in the normal case. ! 2398: */ ! 2399: ! 2400: start += copy_size; ! 2401: vm_map_lock(dst_map); ! 2402: if ((version.main_timestamp + 1) == dst_map->timestamp) { ! 2403: /* We can safely use saved tmp_entry value */ ! 2404: ! 2405: vm_map_clip_end(dst_map, tmp_entry, start); ! 2406: tmp_entry = tmp_entry->vme_next; ! 2407: } else { ! 2408: /* Must do lookup of tmp_entry */ ! 2409: ! 2410: if (!vm_map_lookup_entry(dst_map, start, &tmp_entry)) { ! 2411: vm_map_unlock(dst_map); ! 2412: return(KERN_INVALID_ADDRESS); ! 2413: } ! 2414: vm_map_clip_start(dst_map, tmp_entry, start); ! 2415: } ! 2416: } ! 2417: ! 2418: } ! 2419: vm_map_unlock(dst_map); ! 2420: ! 2421: /* ! 2422: * Throw away the vm_map_copy object ! 2423: */ ! 2424: vm_map_copy_discard(copy); ! 2425: ! 2426: return(KERN_SUCCESS); ! 2427: } ! 2428: ! 2429: /* ! 2430: * Macro: vm_map_copy_insert ! 2431: * ! 2432: * Description: ! 2433: * Link a copy chain ("copy") into a map at the ! 2434: * specified location (after "where"). ! 2435: * Side effects: ! 2436: * The copy chain is destroyed. ! 2437: * Warning: ! 2438: * The arguments are evaluated multiple times. ! 2439: */ ! 2440: #define vm_map_copy_insert(map, where, copy) \ ! 2441: MACRO_BEGIN \ ! 2442: (((where)->vme_next)->vme_prev = vm_map_copy_last_entry(copy)) \ ! 2443: ->vme_next = ((where)->vme_next); \ ! 2444: ((where)->vme_next = vm_map_copy_first_entry(copy)) \ ! 2445: ->vme_prev = (where); \ ! 2446: (map)->hdr.nentries += (copy)->cpy_hdr.nentries; \ ! 2447: zfree(vm_map_copy_zone, (vm_offset_t) copy); \ ! 2448: MACRO_END ! 2449: ! 2450: /* ! 2451: * Routine: vm_map_copyout ! 2452: * ! 2453: * Description: ! 2454: * Copy out a copy chain ("copy") into newly-allocated ! 2455: * space in the destination map. ! 2456: * ! 2457: * If successful, consumes the copy object. ! 2458: * Otherwise, the caller is responsible for it. ! 2459: */ ! 2460: kern_return_t vm_map_copyout(dst_map, dst_addr, copy) ! 2461: register ! 2462: vm_map_t dst_map; ! 2463: vm_offset_t *dst_addr; /* OUT */ ! 2464: register ! 2465: vm_map_copy_t copy; ! 2466: { ! 2467: vm_size_t size; ! 2468: vm_size_t adjustment; ! 2469: vm_offset_t start; ! 2470: vm_offset_t vm_copy_start; ! 2471: vm_map_entry_t last; ! 2472: register ! 2473: vm_map_entry_t entry; ! 2474: ! 2475: /* ! 2476: * Check for null copy object. ! 2477: */ ! 2478: ! 2479: if (copy == VM_MAP_COPY_NULL) { ! 2480: *dst_addr = 0; ! 2481: return(KERN_SUCCESS); ! 2482: } ! 2483: ! 2484: /* ! 2485: * Check for special copy object, created ! 2486: * by vm_map_copyin_object. ! 2487: */ ! 2488: ! 2489: if (copy->type == VM_MAP_COPY_OBJECT) { ! 2490: vm_object_t object = copy->cpy_object; ! 2491: vm_size_t offset = copy->offset; ! 2492: vm_size_t tmp_size = copy->size; ! 2493: kern_return_t kr; ! 2494: ! 2495: *dst_addr = 0; ! 2496: kr = vm_map_enter(dst_map, dst_addr, tmp_size, ! 2497: (vm_offset_t) 0, TRUE, ! 2498: object, offset, FALSE, ! 2499: VM_PROT_DEFAULT, VM_PROT_ALL, ! 2500: VM_INHERIT_DEFAULT); ! 2501: if (kr != KERN_SUCCESS) ! 2502: return(kr); ! 2503: zfree(vm_map_copy_zone, (vm_offset_t) copy); ! 2504: return(KERN_SUCCESS); ! 2505: } ! 2506: ! 2507: if (copy->type == VM_MAP_COPY_PAGE_LIST) ! 2508: return(vm_map_copyout_page_list(dst_map, dst_addr, copy)); ! 2509: ! 2510: /* ! 2511: * Find space for the data ! 2512: */ ! 2513: ! 2514: vm_copy_start = trunc_page(copy->offset); ! 2515: size = round_page(copy->offset + copy->size) - vm_copy_start; ! 2516: ! 2517: StartAgain: ; ! 2518: ! 2519: vm_map_lock(dst_map); ! 2520: start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)) ? ! 2521: vm_map_min(dst_map) : last->vme_end; ! 2522: ! 2523: while (TRUE) { ! 2524: vm_map_entry_t next = last->vme_next; ! 2525: vm_offset_t end = start + size; ! 2526: ! 2527: if ((end > dst_map->max_offset) || (end < start)) { ! 2528: if (dst_map->wait_for_space) { ! 2529: if (size <= (dst_map->max_offset - dst_map->min_offset)) { ! 2530: assert_wait((event_t) dst_map, TRUE); ! 2531: vm_map_unlock(dst_map); ! 2532: thread_block((void (*)()) 0); ! 2533: goto StartAgain; ! 2534: } ! 2535: } ! 2536: vm_map_unlock(dst_map); ! 2537: return(KERN_NO_SPACE); ! 2538: } ! 2539: ! 2540: if ((next == vm_map_to_entry(dst_map)) || ! 2541: (next->vme_start >= end)) ! 2542: break; ! 2543: ! 2544: last = next; ! 2545: start = last->vme_end; ! 2546: } ! 2547: ! 2548: /* ! 2549: * Since we're going to just drop the map ! 2550: * entries from the copy into the destination ! 2551: * map, they must come from the same pool. ! 2552: */ ! 2553: ! 2554: if (copy->cpy_hdr.entries_pageable != dst_map->hdr.entries_pageable) { ! 2555: /* ! 2556: * Mismatches occur when dealing with the default ! 2557: * pager. ! 2558: */ ! 2559: zone_t old_zone; ! 2560: vm_map_entry_t next, new; ! 2561: ! 2562: /* ! 2563: * Find the zone that the copies were allocated from ! 2564: */ ! 2565: old_zone = (copy->cpy_hdr.entries_pageable) ! 2566: ? vm_map_entry_zone ! 2567: : vm_map_kentry_zone; ! 2568: entry = vm_map_copy_first_entry(copy); ! 2569: ! 2570: /* ! 2571: * Reinitialize the copy so that vm_map_copy_entry_link ! 2572: * will work. ! 2573: */ ! 2574: copy->cpy_hdr.nentries = 0; ! 2575: copy->cpy_hdr.entries_pageable = dst_map->hdr.entries_pageable; ! 2576: vm_map_copy_first_entry(copy) = ! 2577: vm_map_copy_last_entry(copy) = ! 2578: vm_map_copy_to_entry(copy); ! 2579: ! 2580: /* ! 2581: * Copy each entry. ! 2582: */ ! 2583: while (entry != vm_map_copy_to_entry(copy)) { ! 2584: new = vm_map_copy_entry_create(copy); ! 2585: vm_map_entry_copy_full(new, entry); ! 2586: vm_map_copy_entry_link(copy, ! 2587: vm_map_copy_last_entry(copy), ! 2588: new); ! 2589: next = entry->vme_next; ! 2590: zfree(old_zone, (vm_offset_t) entry); ! 2591: entry = next; ! 2592: } ! 2593: } ! 2594: ! 2595: /* ! 2596: * Adjust the addresses in the copy chain, and ! 2597: * reset the region attributes. ! 2598: */ ! 2599: ! 2600: adjustment = start - vm_copy_start; ! 2601: for (entry = vm_map_copy_first_entry(copy); ! 2602: entry != vm_map_copy_to_entry(copy); ! 2603: entry = entry->vme_next) { ! 2604: entry->vme_start += adjustment; ! 2605: entry->vme_end += adjustment; ! 2606: ! 2607: entry->inheritance = VM_INHERIT_DEFAULT; ! 2608: entry->protection = VM_PROT_DEFAULT; ! 2609: entry->max_protection = VM_PROT_ALL; ! 2610: entry->projected_on = 0; ! 2611: ! 2612: /* ! 2613: * If the entry is now wired, ! 2614: * map the pages into the destination map. ! 2615: */ ! 2616: if (entry->wired_count != 0) { ! 2617: register vm_offset_t va; ! 2618: vm_offset_t offset; ! 2619: register vm_object_t object; ! 2620: ! 2621: object = entry->object.vm_object; ! 2622: offset = entry->offset; ! 2623: va = entry->vme_start; ! 2624: ! 2625: pmap_pageable(dst_map->pmap, ! 2626: entry->vme_start, ! 2627: entry->vme_end, ! 2628: TRUE); ! 2629: ! 2630: while (va < entry->vme_end) { ! 2631: register vm_page_t m; ! 2632: ! 2633: /* ! 2634: * Look up the page in the object. ! 2635: * Assert that the page will be found in the ! 2636: * top object: ! 2637: * either ! 2638: * the object was newly created by ! 2639: * vm_object_copy_slowly, and has ! 2640: * copies of all of the pages from ! 2641: * the source object ! 2642: * or ! 2643: * the object was moved from the old ! 2644: * map entry; because the old map ! 2645: * entry was wired, all of the pages ! 2646: * were in the top-level object. ! 2647: * (XXX not true if we wire pages for ! 2648: * reading) ! 2649: */ ! 2650: vm_object_lock(object); ! 2651: vm_object_paging_begin(object); ! 2652: ! 2653: m = vm_page_lookup(object, offset); ! 2654: if (m == VM_PAGE_NULL || m->wire_count == 0 || ! 2655: m->absent) ! 2656: panic("vm_map_copyout: wiring 0x%x", m); ! 2657: ! 2658: m->busy = TRUE; ! 2659: vm_object_unlock(object); ! 2660: ! 2661: PMAP_ENTER(dst_map->pmap, va, m, ! 2662: entry->protection, TRUE); ! 2663: ! 2664: vm_object_lock(object); ! 2665: PAGE_WAKEUP_DONE(m); ! 2666: /* the page is wired, so we don't have to activate */ ! 2667: vm_object_paging_end(object); ! 2668: vm_object_unlock(object); ! 2669: ! 2670: offset += PAGE_SIZE; ! 2671: va += PAGE_SIZE; ! 2672: } ! 2673: } ! 2674: ! 2675: ! 2676: } ! 2677: ! 2678: /* ! 2679: * Correct the page alignment for the result ! 2680: */ ! 2681: ! 2682: *dst_addr = start + (copy->offset - vm_copy_start); ! 2683: ! 2684: /* ! 2685: * Update the hints and the map size ! 2686: */ ! 2687: ! 2688: if (dst_map->first_free == last) ! 2689: dst_map->first_free = vm_map_copy_last_entry(copy); ! 2690: SAVE_HINT(dst_map, vm_map_copy_last_entry(copy)); ! 2691: ! 2692: dst_map->size += size; ! 2693: ! 2694: /* ! 2695: * Link in the copy ! 2696: */ ! 2697: ! 2698: vm_map_copy_insert(dst_map, last, copy); ! 2699: ! 2700: vm_map_unlock(dst_map); ! 2701: ! 2702: /* ! 2703: * XXX If wiring_required, call vm_map_pageable ! 2704: */ ! 2705: ! 2706: return(KERN_SUCCESS); ! 2707: } ! 2708: ! 2709: /* ! 2710: * ! 2711: * vm_map_copyout_page_list: ! 2712: * ! 2713: * Version of vm_map_copyout() for page list vm map copies. ! 2714: * ! 2715: */ ! 2716: kern_return_t vm_map_copyout_page_list(dst_map, dst_addr, copy) ! 2717: register ! 2718: vm_map_t dst_map; ! 2719: vm_offset_t *dst_addr; /* OUT */ ! 2720: register ! 2721: vm_map_copy_t copy; ! 2722: { ! 2723: vm_size_t size; ! 2724: vm_offset_t start; ! 2725: vm_offset_t end; ! 2726: vm_offset_t offset; ! 2727: vm_map_entry_t last; ! 2728: register ! 2729: vm_object_t object; ! 2730: vm_page_t *page_list, m; ! 2731: vm_map_entry_t entry; ! 2732: vm_offset_t old_last_offset; ! 2733: boolean_t cont_invoked, needs_wakeup = FALSE; ! 2734: kern_return_t result = KERN_SUCCESS; ! 2735: vm_map_copy_t orig_copy; ! 2736: vm_offset_t dst_offset; ! 2737: boolean_t must_wire; ! 2738: ! 2739: /* ! 2740: * Make sure the pages are stolen, because we are ! 2741: * going to put them in a new object. Assume that ! 2742: * all pages are identical to first in this regard. ! 2743: */ ! 2744: ! 2745: page_list = ©->cpy_page_list[0]; ! 2746: if ((*page_list)->tabled) ! 2747: vm_map_copy_steal_pages(copy); ! 2748: ! 2749: /* ! 2750: * Find space for the data ! 2751: */ ! 2752: ! 2753: size = round_page(copy->offset + copy->size) - ! 2754: trunc_page(copy->offset); ! 2755: StartAgain: ! 2756: vm_map_lock(dst_map); ! 2757: must_wire = dst_map->wiring_required; ! 2758: ! 2759: last = dst_map->first_free; ! 2760: if (last == vm_map_to_entry(dst_map)) { ! 2761: start = vm_map_min(dst_map); ! 2762: } else { ! 2763: start = last->vme_end; ! 2764: } ! 2765: ! 2766: while (TRUE) { ! 2767: vm_map_entry_t next = last->vme_next; ! 2768: end = start + size; ! 2769: ! 2770: if ((end > dst_map->max_offset) || (end < start)) { ! 2771: if (dst_map->wait_for_space) { ! 2772: if (size <= (dst_map->max_offset - ! 2773: dst_map->min_offset)) { ! 2774: assert_wait((event_t) dst_map, TRUE); ! 2775: vm_map_unlock(dst_map); ! 2776: thread_block((void (*)()) 0); ! 2777: goto StartAgain; ! 2778: } ! 2779: } ! 2780: vm_map_unlock(dst_map); ! 2781: return(KERN_NO_SPACE); ! 2782: } ! 2783: ! 2784: if ((next == vm_map_to_entry(dst_map)) || ! 2785: (next->vme_start >= end)) { ! 2786: break; ! 2787: } ! 2788: ! 2789: last = next; ! 2790: start = last->vme_end; ! 2791: } ! 2792: ! 2793: /* ! 2794: * See whether we can avoid creating a new entry (and object) by ! 2795: * extending one of our neighbors. [So far, we only attempt to ! 2796: * extend from below.] ! 2797: * ! 2798: * The code path below here is a bit twisted. If any of the ! 2799: * extension checks fails, we branch to create_object. If ! 2800: * it all works, we fall out the bottom and goto insert_pages. ! 2801: */ ! 2802: if (last == vm_map_to_entry(dst_map) || ! 2803: last->vme_end != start || ! 2804: last->is_shared != FALSE || ! 2805: last->is_sub_map != FALSE || ! 2806: last->inheritance != VM_INHERIT_DEFAULT || ! 2807: last->protection != VM_PROT_DEFAULT || ! 2808: last->max_protection != VM_PROT_ALL || ! 2809: (must_wire ? (last->wired_count != 1 || ! 2810: last->user_wired_count != 1) : ! 2811: (last->wired_count != 0))) { ! 2812: goto create_object; ! 2813: } ! 2814: ! 2815: /* ! 2816: * If this entry needs an object, make one. ! 2817: */ ! 2818: if (last->object.vm_object == VM_OBJECT_NULL) { ! 2819: object = vm_object_allocate( ! 2820: (vm_size_t)(last->vme_end - last->vme_start + size)); ! 2821: last->object.vm_object = object; ! 2822: last->offset = 0; ! 2823: vm_object_lock(object); ! 2824: } ! 2825: else { ! 2826: vm_offset_t prev_offset = last->offset; ! 2827: vm_size_t prev_size = start - last->vme_start; ! 2828: vm_size_t new_size; ! 2829: ! 2830: /* ! 2831: * This is basically vm_object_coalesce. ! 2832: */ ! 2833: ! 2834: object = last->object.vm_object; ! 2835: vm_object_lock(object); ! 2836: ! 2837: /* ! 2838: * Try to collapse the object first ! 2839: */ ! 2840: vm_object_collapse(object); ! 2841: ! 2842: /* ! 2843: * Can't coalesce if pages not mapped to ! 2844: * last may be in use anyway: ! 2845: * . more than one reference ! 2846: * . paged out ! 2847: * . shadows another object ! 2848: * . has a copy elsewhere ! 2849: * . paging references (pages might be in page-list) ! 2850: */ ! 2851: ! 2852: if ((object->ref_count > 1) || ! 2853: object->pager_created || ! 2854: (object->shadow != VM_OBJECT_NULL) || ! 2855: (object->copy != VM_OBJECT_NULL) || ! 2856: (object->paging_in_progress != 0)) { ! 2857: vm_object_unlock(object); ! 2858: goto create_object; ! 2859: } ! 2860: ! 2861: /* ! 2862: * Extend the object if necessary. Don't have to call ! 2863: * vm_object_page_remove because the pages aren't mapped, ! 2864: * and vm_page_replace will free up any old ones it encounters. ! 2865: */ ! 2866: new_size = prev_offset + prev_size + size; ! 2867: if (new_size > object->size) ! 2868: object->size = new_size; ! 2869: } ! 2870: ! 2871: /* ! 2872: * Coalesced the two objects - can extend ! 2873: * the previous map entry to include the ! 2874: * new range. ! 2875: */ ! 2876: dst_map->size += size; ! 2877: last->vme_end = end; ! 2878: ! 2879: SAVE_HINT(dst_map, last); ! 2880: ! 2881: goto insert_pages; ! 2882: ! 2883: create_object: ! 2884: ! 2885: /* ! 2886: * Create object ! 2887: */ ! 2888: object = vm_object_allocate(size); ! 2889: ! 2890: /* ! 2891: * Create entry ! 2892: */ ! 2893: ! 2894: entry = vm_map_entry_create(dst_map); ! 2895: ! 2896: entry->object.vm_object = object; ! 2897: entry->offset = 0; ! 2898: ! 2899: entry->is_shared = FALSE; ! 2900: entry->is_sub_map = FALSE; ! 2901: entry->needs_copy = FALSE; ! 2902: ! 2903: if (must_wire) { ! 2904: entry->wired_count = 1; ! 2905: entry->user_wired_count = 1; ! 2906: } else { ! 2907: entry->wired_count = 0; ! 2908: entry->user_wired_count = 0; ! 2909: } ! 2910: ! 2911: entry->in_transition = TRUE; ! 2912: entry->needs_wakeup = FALSE; ! 2913: ! 2914: entry->vme_start = start; ! 2915: entry->vme_end = start + size; ! 2916: ! 2917: entry->inheritance = VM_INHERIT_DEFAULT; ! 2918: entry->protection = VM_PROT_DEFAULT; ! 2919: entry->max_protection = VM_PROT_ALL; ! 2920: entry->projected_on = 0; ! 2921: ! 2922: vm_object_lock(object); ! 2923: ! 2924: /* ! 2925: * Update the hints and the map size ! 2926: */ ! 2927: if (dst_map->first_free == last) { ! 2928: dst_map->first_free = entry; ! 2929: } ! 2930: SAVE_HINT(dst_map, entry); ! 2931: dst_map->size += size; ! 2932: ! 2933: /* ! 2934: * Link in the entry ! 2935: */ ! 2936: vm_map_entry_link(dst_map, last, entry); ! 2937: last = entry; ! 2938: ! 2939: /* ! 2940: * Transfer pages into new object. ! 2941: * Scan page list in vm_map_copy. ! 2942: */ ! 2943: insert_pages: ! 2944: dst_offset = copy->offset & PAGE_MASK; ! 2945: cont_invoked = FALSE; ! 2946: orig_copy = copy; ! 2947: last->in_transition = TRUE; ! 2948: old_last_offset = last->offset ! 2949: + (start - last->vme_start); ! 2950: ! 2951: vm_page_lock_queues(); ! 2952: ! 2953: for (offset = 0; offset < size; offset += PAGE_SIZE) { ! 2954: m = *page_list; ! 2955: assert(m && !m->tabled); ! 2956: ! 2957: /* ! 2958: * Must clear busy bit in page before inserting it. ! 2959: * Ok to skip wakeup logic because nobody else ! 2960: * can possibly know about this page. ! 2961: * The page is dirty in its new object. ! 2962: */ ! 2963: ! 2964: assert(!m->wanted); ! 2965: ! 2966: m->busy = FALSE; ! 2967: m->dirty = TRUE; ! 2968: vm_page_replace(m, object, old_last_offset + offset); ! 2969: if (must_wire) { ! 2970: vm_page_wire(m); ! 2971: PMAP_ENTER(dst_map->pmap, ! 2972: last->vme_start + m->offset - last->offset, ! 2973: m, last->protection, TRUE); ! 2974: } else { ! 2975: vm_page_activate(m); ! 2976: } ! 2977: ! 2978: *page_list++ = VM_PAGE_NULL; ! 2979: if (--(copy->cpy_npages) == 0 && ! 2980: vm_map_copy_has_cont(copy)) { ! 2981: vm_map_copy_t new_copy; ! 2982: ! 2983: /* ! 2984: * Ok to unlock map because entry is ! 2985: * marked in_transition. ! 2986: */ ! 2987: cont_invoked = TRUE; ! 2988: vm_page_unlock_queues(); ! 2989: vm_object_unlock(object); ! 2990: vm_map_unlock(dst_map); ! 2991: vm_map_copy_invoke_cont(copy, &new_copy, &result); ! 2992: ! 2993: if (result == KERN_SUCCESS) { ! 2994: ! 2995: /* ! 2996: * If we got back a copy with real pages, ! 2997: * steal them now. Either all of the ! 2998: * pages in the list are tabled or none ! 2999: * of them are; mixtures are not possible. ! 3000: * ! 3001: * Save original copy for consume on ! 3002: * success logic at end of routine. ! 3003: */ ! 3004: if (copy != orig_copy) ! 3005: vm_map_copy_discard(copy); ! 3006: ! 3007: if ((copy = new_copy) != VM_MAP_COPY_NULL) { ! 3008: page_list = ©->cpy_page_list[0]; ! 3009: if ((*page_list)->tabled) ! 3010: vm_map_copy_steal_pages(copy); ! 3011: } ! 3012: } ! 3013: else { ! 3014: /* ! 3015: * Continuation failed. ! 3016: */ ! 3017: vm_map_lock(dst_map); ! 3018: goto error; ! 3019: } ! 3020: ! 3021: vm_map_lock(dst_map); ! 3022: vm_object_lock(object); ! 3023: vm_page_lock_queues(); ! 3024: } ! 3025: } ! 3026: ! 3027: vm_page_unlock_queues(); ! 3028: vm_object_unlock(object); ! 3029: ! 3030: *dst_addr = start + dst_offset; ! 3031: ! 3032: /* ! 3033: * Clear the in transition bits. This is easy if we ! 3034: * didn't have a continuation. ! 3035: */ ! 3036: error: ! 3037: if (!cont_invoked) { ! 3038: /* ! 3039: * We didn't unlock the map, so nobody could ! 3040: * be waiting. ! 3041: */ ! 3042: last->in_transition = FALSE; ! 3043: assert(!last->needs_wakeup); ! 3044: needs_wakeup = FALSE; ! 3045: } ! 3046: else { ! 3047: if (!vm_map_lookup_entry(dst_map, start, &entry)) ! 3048: panic("vm_map_copyout_page_list: missing entry"); ! 3049: ! 3050: /* ! 3051: * Clear transition bit for all constituent entries that ! 3052: * were in the original entry. Also check for waiters. ! 3053: */ ! 3054: while((entry != vm_map_to_entry(dst_map)) && ! 3055: (entry->vme_start < end)) { ! 3056: assert(entry->in_transition); ! 3057: entry->in_transition = FALSE; ! 3058: if(entry->needs_wakeup) { ! 3059: entry->needs_wakeup = FALSE; ! 3060: needs_wakeup = TRUE; ! 3061: } ! 3062: entry = entry->vme_next; ! 3063: } ! 3064: } ! 3065: ! 3066: if (result != KERN_SUCCESS) ! 3067: vm_map_delete(dst_map, start, end); ! 3068: ! 3069: vm_map_unlock(dst_map); ! 3070: ! 3071: if (needs_wakeup) ! 3072: vm_map_entry_wakeup(dst_map); ! 3073: ! 3074: /* ! 3075: * Consume on success logic. ! 3076: */ ! 3077: if (copy != orig_copy) { ! 3078: zfree(vm_map_copy_zone, (vm_offset_t) copy); ! 3079: } ! 3080: if (result == KERN_SUCCESS) { ! 3081: zfree(vm_map_copy_zone, (vm_offset_t) orig_copy); ! 3082: } ! 3083: ! 3084: return(result); ! 3085: } ! 3086: ! 3087: /* ! 3088: * Routine: vm_map_copyin ! 3089: * ! 3090: * Description: ! 3091: * Copy the specified region (src_addr, len) from the ! 3092: * source address space (src_map), possibly removing ! 3093: * the region from the source address space (src_destroy). ! 3094: * ! 3095: * Returns: ! 3096: * A vm_map_copy_t object (copy_result), suitable for ! 3097: * insertion into another address space (using vm_map_copyout), ! 3098: * copying over another address space region (using ! 3099: * vm_map_copy_overwrite). If the copy is unused, it ! 3100: * should be destroyed (using vm_map_copy_discard). ! 3101: * ! 3102: * In/out conditions: ! 3103: * The source map should not be locked on entry. ! 3104: */ ! 3105: kern_return_t vm_map_copyin(src_map, src_addr, len, src_destroy, copy_result) ! 3106: vm_map_t src_map; ! 3107: vm_offset_t src_addr; ! 3108: vm_size_t len; ! 3109: boolean_t src_destroy; ! 3110: vm_map_copy_t *copy_result; /* OUT */ ! 3111: { ! 3112: vm_map_entry_t tmp_entry; /* Result of last map lookup -- ! 3113: * in multi-level lookup, this ! 3114: * entry contains the actual ! 3115: * vm_object/offset. ! 3116: */ ! 3117: ! 3118: vm_offset_t src_start; /* Start of current entry -- ! 3119: * where copy is taking place now ! 3120: */ ! 3121: vm_offset_t src_end; /* End of entire region to be ! 3122: * copied */ ! 3123: ! 3124: register ! 3125: vm_map_copy_t copy; /* Resulting copy */ ! 3126: ! 3127: /* ! 3128: * Check for copies of zero bytes. ! 3129: */ ! 3130: ! 3131: if (len == 0) { ! 3132: *copy_result = VM_MAP_COPY_NULL; ! 3133: return(KERN_SUCCESS); ! 3134: } ! 3135: ! 3136: /* ! 3137: * Compute start and end of region ! 3138: */ ! 3139: ! 3140: src_start = trunc_page(src_addr); ! 3141: src_end = round_page(src_addr + len); ! 3142: ! 3143: /* ! 3144: * Check that the end address doesn't overflow ! 3145: */ ! 3146: ! 3147: if (src_end <= src_start) ! 3148: if ((src_end < src_start) || (src_start != 0)) ! 3149: return(KERN_INVALID_ADDRESS); ! 3150: ! 3151: /* ! 3152: * Allocate a header element for the list. ! 3153: * ! 3154: * Use the start and end in the header to ! 3155: * remember the endpoints prior to rounding. ! 3156: */ ! 3157: ! 3158: copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); ! 3159: vm_map_copy_first_entry(copy) = ! 3160: vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy); ! 3161: copy->type = VM_MAP_COPY_ENTRY_LIST; ! 3162: copy->cpy_hdr.nentries = 0; ! 3163: copy->cpy_hdr.entries_pageable = TRUE; ! 3164: ! 3165: copy->offset = src_addr; ! 3166: copy->size = len; ! 3167: ! 3168: #define RETURN(x) \ ! 3169: MACRO_BEGIN \ ! 3170: vm_map_unlock(src_map); \ ! 3171: vm_map_copy_discard(copy); \ ! 3172: MACRO_RETURN(x); \ ! 3173: MACRO_END ! 3174: ! 3175: /* ! 3176: * Find the beginning of the region. ! 3177: */ ! 3178: ! 3179: vm_map_lock(src_map); ! 3180: ! 3181: if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry)) ! 3182: RETURN(KERN_INVALID_ADDRESS); ! 3183: vm_map_clip_start(src_map, tmp_entry, src_start); ! 3184: ! 3185: /* ! 3186: * Go through entries until we get to the end. ! 3187: */ ! 3188: ! 3189: while (TRUE) { ! 3190: register ! 3191: vm_map_entry_t src_entry = tmp_entry; /* Top-level entry */ ! 3192: vm_size_t src_size; /* Size of source ! 3193: * map entry (in both ! 3194: * maps) ! 3195: */ ! 3196: ! 3197: register ! 3198: vm_object_t src_object; /* Object to copy */ ! 3199: vm_offset_t src_offset; ! 3200: ! 3201: boolean_t src_needs_copy; /* Should source map ! 3202: * be made read-only ! 3203: * for copy-on-write? ! 3204: */ ! 3205: ! 3206: register ! 3207: vm_map_entry_t new_entry; /* Map entry for copy */ ! 3208: boolean_t new_entry_needs_copy; /* Will new entry be COW? */ ! 3209: ! 3210: boolean_t was_wired; /* Was source wired? */ ! 3211: vm_map_version_t version; /* Version before locks ! 3212: * dropped to make copy ! 3213: */ ! 3214: ! 3215: /* ! 3216: * Verify that the region can be read. ! 3217: */ ! 3218: ! 3219: if (! (src_entry->protection & VM_PROT_READ)) ! 3220: RETURN(KERN_PROTECTION_FAILURE); ! 3221: ! 3222: /* ! 3223: * Clip against the endpoints of the entire region. ! 3224: */ ! 3225: ! 3226: vm_map_clip_end(src_map, src_entry, src_end); ! 3227: ! 3228: src_size = src_entry->vme_end - src_start; ! 3229: src_object = src_entry->object.vm_object; ! 3230: src_offset = src_entry->offset; ! 3231: was_wired = (src_entry->wired_count != 0); ! 3232: ! 3233: /* ! 3234: * Create a new address map entry to ! 3235: * hold the result. Fill in the fields from ! 3236: * the appropriate source entries. ! 3237: */ ! 3238: ! 3239: new_entry = vm_map_copy_entry_create(copy); ! 3240: vm_map_entry_copy(new_entry, src_entry); ! 3241: ! 3242: /* ! 3243: * Attempt non-blocking copy-on-write optimizations. ! 3244: */ ! 3245: ! 3246: if (src_destroy && ! 3247: (src_object == VM_OBJECT_NULL || ! 3248: (src_object->temporary && !src_object->use_shared_copy))) ! 3249: { ! 3250: /* ! 3251: * If we are destroying the source, and the object ! 3252: * is temporary, and not shared writable, ! 3253: * we can move the object reference ! 3254: * from the source to the copy. The copy is ! 3255: * copy-on-write only if the source is. ! 3256: * We make another reference to the object, because ! 3257: * destroying the source entry will deallocate it. ! 3258: */ ! 3259: vm_object_reference(src_object); ! 3260: ! 3261: /* ! 3262: * Copy is always unwired. vm_map_copy_entry ! 3263: * set its wired count to zero. ! 3264: */ ! 3265: ! 3266: goto CopySuccessful; ! 3267: } ! 3268: ! 3269: if (!was_wired && ! 3270: vm_object_copy_temporary( ! 3271: &new_entry->object.vm_object, ! 3272: &new_entry->offset, ! 3273: &src_needs_copy, ! 3274: &new_entry_needs_copy)) { ! 3275: ! 3276: new_entry->needs_copy = new_entry_needs_copy; ! 3277: ! 3278: /* ! 3279: * Handle copy-on-write obligations ! 3280: */ ! 3281: ! 3282: if (src_needs_copy && !tmp_entry->needs_copy) { ! 3283: vm_object_pmap_protect( ! 3284: src_object, ! 3285: src_offset, ! 3286: src_size, ! 3287: (src_entry->is_shared ? PMAP_NULL ! 3288: : src_map->pmap), ! 3289: src_entry->vme_start, ! 3290: src_entry->protection & ! 3291: ~VM_PROT_WRITE); ! 3292: ! 3293: tmp_entry->needs_copy = TRUE; ! 3294: } ! 3295: ! 3296: /* ! 3297: * The map has never been unlocked, so it's safe to ! 3298: * move to the next entry rather than doing another ! 3299: * lookup. ! 3300: */ ! 3301: ! 3302: goto CopySuccessful; ! 3303: } ! 3304: ! 3305: new_entry->needs_copy = FALSE; ! 3306: ! 3307: /* ! 3308: * Take an object reference, so that we may ! 3309: * release the map lock(s). ! 3310: */ ! 3311: ! 3312: assert(src_object != VM_OBJECT_NULL); ! 3313: vm_object_reference(src_object); ! 3314: ! 3315: /* ! 3316: * Record the timestamp for later verification. ! 3317: * Unlock the map. ! 3318: */ ! 3319: ! 3320: version.main_timestamp = src_map->timestamp; ! 3321: vm_map_unlock(src_map); ! 3322: ! 3323: /* ! 3324: * Perform the copy ! 3325: */ ! 3326: ! 3327: if (was_wired) { ! 3328: vm_object_lock(src_object); ! 3329: (void) vm_object_copy_slowly( ! 3330: src_object, ! 3331: src_offset, ! 3332: src_size, ! 3333: FALSE, ! 3334: &new_entry->object.vm_object); ! 3335: new_entry->offset = 0; ! 3336: new_entry->needs_copy = FALSE; ! 3337: } else { ! 3338: kern_return_t result; ! 3339: ! 3340: result = vm_object_copy_strategically(src_object, ! 3341: src_offset, ! 3342: src_size, ! 3343: &new_entry->object.vm_object, ! 3344: &new_entry->offset, ! 3345: &new_entry_needs_copy); ! 3346: ! 3347: new_entry->needs_copy = new_entry_needs_copy; ! 3348: ! 3349: ! 3350: if (result != KERN_SUCCESS) { ! 3351: vm_map_copy_entry_dispose(copy, new_entry); ! 3352: ! 3353: vm_map_lock(src_map); ! 3354: RETURN(result); ! 3355: } ! 3356: ! 3357: } ! 3358: ! 3359: /* ! 3360: * Throw away the extra reference ! 3361: */ ! 3362: ! 3363: vm_object_deallocate(src_object); ! 3364: ! 3365: /* ! 3366: * Verify that the map has not substantially ! 3367: * changed while the copy was being made. ! 3368: */ ! 3369: ! 3370: vm_map_lock(src_map); /* Increments timestamp once! */ ! 3371: ! 3372: if ((version.main_timestamp + 1) == src_map->timestamp) ! 3373: goto CopySuccessful; ! 3374: ! 3375: /* ! 3376: * Simple version comparison failed. ! 3377: * ! 3378: * Retry the lookup and verify that the ! 3379: * same object/offset are still present. ! 3380: * ! 3381: * [Note: a memory manager that colludes with ! 3382: * the calling task can detect that we have ! 3383: * cheated. While the map was unlocked, the ! 3384: * mapping could have been changed and restored.] ! 3385: */ ! 3386: ! 3387: if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry)) { ! 3388: vm_map_copy_entry_dispose(copy, new_entry); ! 3389: RETURN(KERN_INVALID_ADDRESS); ! 3390: } ! 3391: ! 3392: src_entry = tmp_entry; ! 3393: vm_map_clip_start(src_map, src_entry, src_start); ! 3394: ! 3395: if ((src_entry->protection & VM_PROT_READ) == VM_PROT_NONE) ! 3396: goto VerificationFailed; ! 3397: ! 3398: if (src_entry->vme_end < new_entry->vme_end) ! 3399: src_size = (new_entry->vme_end = src_entry->vme_end) - src_start; ! 3400: ! 3401: if ((src_entry->object.vm_object != src_object) || ! 3402: (src_entry->offset != src_offset) ) { ! 3403: ! 3404: /* ! 3405: * Verification failed. ! 3406: * ! 3407: * Start over with this top-level entry. ! 3408: */ ! 3409: ! 3410: VerificationFailed: ; ! 3411: ! 3412: vm_object_deallocate(new_entry->object.vm_object); ! 3413: vm_map_copy_entry_dispose(copy, new_entry); ! 3414: tmp_entry = src_entry; ! 3415: continue; ! 3416: } ! 3417: ! 3418: /* ! 3419: * Verification succeeded. ! 3420: */ ! 3421: ! 3422: CopySuccessful: ; ! 3423: ! 3424: /* ! 3425: * Link in the new copy entry. ! 3426: */ ! 3427: ! 3428: vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy), ! 3429: new_entry); ! 3430: ! 3431: /* ! 3432: * Determine whether the entire region ! 3433: * has been copied. ! 3434: */ ! 3435: src_start = new_entry->vme_end; ! 3436: if ((src_start >= src_end) && (src_end != 0)) ! 3437: break; ! 3438: ! 3439: /* ! 3440: * Verify that there are no gaps in the region ! 3441: */ ! 3442: ! 3443: tmp_entry = src_entry->vme_next; ! 3444: if (tmp_entry->vme_start != src_start) ! 3445: RETURN(KERN_INVALID_ADDRESS); ! 3446: } ! 3447: ! 3448: /* ! 3449: * If the source should be destroyed, do it now, since the ! 3450: * copy was successful. ! 3451: */ ! 3452: if (src_destroy) ! 3453: (void) vm_map_delete(src_map, trunc_page(src_addr), src_end); ! 3454: ! 3455: vm_map_unlock(src_map); ! 3456: ! 3457: *copy_result = copy; ! 3458: return(KERN_SUCCESS); ! 3459: ! 3460: #undef RETURN ! 3461: } ! 3462: ! 3463: /* ! 3464: * vm_map_copyin_object: ! 3465: * ! 3466: * Create a copy object from an object. ! 3467: * Our caller donates an object reference. ! 3468: */ ! 3469: ! 3470: kern_return_t vm_map_copyin_object(object, offset, size, copy_result) ! 3471: vm_object_t object; ! 3472: vm_offset_t offset; /* offset of region in object */ ! 3473: vm_size_t size; /* size of region in object */ ! 3474: vm_map_copy_t *copy_result; /* OUT */ ! 3475: { ! 3476: vm_map_copy_t copy; /* Resulting copy */ ! 3477: ! 3478: /* ! 3479: * We drop the object into a special copy object ! 3480: * that contains the object directly. These copy objects ! 3481: * are distinguished by entries_pageable == FALSE ! 3482: * and null links. ! 3483: */ ! 3484: ! 3485: copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); ! 3486: vm_map_copy_first_entry(copy) = ! 3487: vm_map_copy_last_entry(copy) = VM_MAP_ENTRY_NULL; ! 3488: copy->type = VM_MAP_COPY_OBJECT; ! 3489: copy->cpy_object = object; ! 3490: copy->offset = offset; ! 3491: copy->size = size; ! 3492: ! 3493: *copy_result = copy; ! 3494: return(KERN_SUCCESS); ! 3495: } ! 3496: ! 3497: /* ! 3498: * vm_map_copyin_page_list_cont: ! 3499: * ! 3500: * Continuation routine for vm_map_copyin_page_list. ! 3501: * ! 3502: * If vm_map_copyin_page_list can't fit the entire vm range ! 3503: * into a single page list object, it creates a continuation. ! 3504: * When the target of the operation has used the pages in the ! 3505: * initial page list, it invokes the continuation, which calls ! 3506: * this routine. If an error happens, the continuation is aborted ! 3507: * (abort arg to this routine is TRUE). To avoid deadlocks, the ! 3508: * pages are discarded from the initial page list before invoking ! 3509: * the continuation. ! 3510: * ! 3511: * NOTE: This is not the same sort of continuation used by ! 3512: * the scheduler. ! 3513: */ ! 3514: ! 3515: kern_return_t vm_map_copyin_page_list_cont(cont_args, copy_result) ! 3516: vm_map_copyin_args_t cont_args; ! 3517: vm_map_copy_t *copy_result; /* OUT */ ! 3518: { ! 3519: kern_return_t result = 0; /* '=0' to quiet gcc warnings */ ! 3520: register boolean_t do_abort, src_destroy, src_destroy_only; ! 3521: ! 3522: /* ! 3523: * Check for cases that only require memory destruction. ! 3524: */ ! 3525: do_abort = (copy_result == (vm_map_copy_t *) 0); ! 3526: src_destroy = (cont_args->destroy_len != (vm_size_t) 0); ! 3527: src_destroy_only = (cont_args->src_len == (vm_size_t) 0); ! 3528: ! 3529: if (do_abort || src_destroy_only) { ! 3530: if (src_destroy) ! 3531: result = vm_map_remove(cont_args->map, ! 3532: cont_args->destroy_addr, ! 3533: cont_args->destroy_addr + cont_args->destroy_len); ! 3534: if (!do_abort) ! 3535: *copy_result = VM_MAP_COPY_NULL; ! 3536: } ! 3537: else { ! 3538: result = vm_map_copyin_page_list(cont_args->map, ! 3539: cont_args->src_addr, cont_args->src_len, src_destroy, ! 3540: cont_args->steal_pages, copy_result, TRUE); ! 3541: ! 3542: if (src_destroy && !cont_args->steal_pages && ! 3543: vm_map_copy_has_cont(*copy_result)) { ! 3544: vm_map_copyin_args_t new_args; ! 3545: /* ! 3546: * Transfer old destroy info. ! 3547: */ ! 3548: new_args = (vm_map_copyin_args_t) ! 3549: (*copy_result)->cpy_cont_args; ! 3550: new_args->destroy_addr = cont_args->destroy_addr; ! 3551: new_args->destroy_len = cont_args->destroy_len; ! 3552: } ! 3553: } ! 3554: ! 3555: vm_map_deallocate(cont_args->map); ! 3556: kfree((vm_offset_t)cont_args, sizeof(vm_map_copyin_args_data_t)); ! 3557: ! 3558: return(result); ! 3559: } ! 3560: ! 3561: /* ! 3562: * vm_map_copyin_page_list: ! 3563: * ! 3564: * This is a variant of vm_map_copyin that copies in a list of pages. ! 3565: * If steal_pages is TRUE, the pages are only in the returned list. ! 3566: * If steal_pages is FALSE, the pages are busy and still in their ! 3567: * objects. A continuation may be returned if not all the pages fit: ! 3568: * the recipient of this copy_result must be prepared to deal with it. ! 3569: */ ! 3570: ! 3571: kern_return_t vm_map_copyin_page_list(src_map, src_addr, len, src_destroy, ! 3572: steal_pages, copy_result, is_cont) ! 3573: vm_map_t src_map; ! 3574: vm_offset_t src_addr; ! 3575: vm_size_t len; ! 3576: boolean_t src_destroy; ! 3577: boolean_t steal_pages; ! 3578: vm_map_copy_t *copy_result; /* OUT */ ! 3579: boolean_t is_cont; ! 3580: { ! 3581: vm_map_entry_t src_entry; ! 3582: vm_page_t m; ! 3583: vm_offset_t src_start; ! 3584: vm_offset_t src_end; ! 3585: vm_size_t src_size; ! 3586: register ! 3587: vm_object_t src_object; ! 3588: register ! 3589: vm_offset_t src_offset; ! 3590: vm_offset_t src_last_offset; ! 3591: register ! 3592: vm_map_copy_t copy; /* Resulting copy */ ! 3593: kern_return_t result = KERN_SUCCESS; ! 3594: boolean_t need_map_lookup; ! 3595: vm_map_copyin_args_t cont_args; ! 3596: ! 3597: /* ! 3598: * If steal_pages is FALSE, this leaves busy pages in ! 3599: * the object. A continuation must be used if src_destroy ! 3600: * is true in this case (!steal_pages && src_destroy). ! 3601: * ! 3602: * XXX Still have a more general problem of what happens ! 3603: * XXX if the same page occurs twice in a list. Deadlock ! 3604: * XXX can happen if vm_fault_page was called. A ! 3605: * XXX possible solution is to use a continuation if vm_fault_page ! 3606: * XXX is called and we cross a map entry boundary. ! 3607: */ ! 3608: ! 3609: /* ! 3610: * Check for copies of zero bytes. ! 3611: */ ! 3612: ! 3613: if (len == 0) { ! 3614: *copy_result = VM_MAP_COPY_NULL; ! 3615: return(KERN_SUCCESS); ! 3616: } ! 3617: ! 3618: /* ! 3619: * Compute start and end of region ! 3620: */ ! 3621: ! 3622: src_start = trunc_page(src_addr); ! 3623: src_end = round_page(src_addr + len); ! 3624: ! 3625: /* ! 3626: * Check that the end address doesn't overflow ! 3627: */ ! 3628: ! 3629: if (src_end <= src_start && (src_end < src_start || src_start != 0)) { ! 3630: return KERN_INVALID_ADDRESS; ! 3631: } ! 3632: ! 3633: /* ! 3634: * Allocate a header element for the page list. ! 3635: * ! 3636: * Record original offset and size, as caller may not ! 3637: * be page-aligned. ! 3638: */ ! 3639: ! 3640: copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); ! 3641: copy->type = VM_MAP_COPY_PAGE_LIST; ! 3642: copy->cpy_npages = 0; ! 3643: copy->offset = src_addr; ! 3644: copy->size = len; ! 3645: copy->cpy_cont = ((kern_return_t (*)()) 0); ! 3646: copy->cpy_cont_args = (char *) VM_MAP_COPYIN_ARGS_NULL; ! 3647: ! 3648: /* ! 3649: * Find the beginning of the region. ! 3650: */ ! 3651: ! 3652: do_map_lookup: ! 3653: ! 3654: vm_map_lock(src_map); ! 3655: ! 3656: if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) { ! 3657: result = KERN_INVALID_ADDRESS; ! 3658: goto error; ! 3659: } ! 3660: need_map_lookup = FALSE; ! 3661: ! 3662: /* ! 3663: * Go through entries until we get to the end. ! 3664: */ ! 3665: ! 3666: while (TRUE) { ! 3667: ! 3668: if (! (src_entry->protection & VM_PROT_READ)) { ! 3669: result = KERN_PROTECTION_FAILURE; ! 3670: goto error; ! 3671: } ! 3672: ! 3673: if (src_end > src_entry->vme_end) ! 3674: src_size = src_entry->vme_end - src_start; ! 3675: else ! 3676: src_size = src_end - src_start; ! 3677: ! 3678: src_object = src_entry->object.vm_object; ! 3679: src_offset = src_entry->offset + ! 3680: (src_start - src_entry->vme_start); ! 3681: ! 3682: /* ! 3683: * If src_object is NULL, allocate it now; ! 3684: * we're going to fault on it shortly. ! 3685: */ ! 3686: if (src_object == VM_OBJECT_NULL) { ! 3687: src_object = vm_object_allocate((vm_size_t) ! 3688: src_entry->vme_end - ! 3689: src_entry->vme_start); ! 3690: src_entry->object.vm_object = src_object; ! 3691: } ! 3692: ! 3693: /* ! 3694: * Iterate over pages. Fault in ones that aren't present. ! 3695: */ ! 3696: src_last_offset = src_offset + src_size; ! 3697: for (; (src_offset < src_last_offset && !need_map_lookup); ! 3698: src_offset += PAGE_SIZE, src_start += PAGE_SIZE) { ! 3699: ! 3700: if (copy->cpy_npages == VM_MAP_COPY_PAGE_LIST_MAX) { ! 3701: make_continuation: ! 3702: /* ! 3703: * At this point we have the max number of ! 3704: * pages busy for this thread that we're ! 3705: * willing to allow. Stop here and record ! 3706: * arguments for the remainder. Note: ! 3707: * this means that this routine isn't atomic, ! 3708: * but that's the breaks. Note that only ! 3709: * the first vm_map_copy_t that comes back ! 3710: * from this routine has the right offset ! 3711: * and size; those from continuations are ! 3712: * page rounded, and short by the amount ! 3713: * already done. ! 3714: * ! 3715: * Reset src_end so the src_destroy ! 3716: * code at the bottom doesn't do ! 3717: * something stupid. ! 3718: */ ! 3719: ! 3720: cont_args = (vm_map_copyin_args_t) ! 3721: kalloc(sizeof(vm_map_copyin_args_data_t)); ! 3722: cont_args->map = src_map; ! 3723: vm_map_reference(src_map); ! 3724: cont_args->src_addr = src_start; ! 3725: cont_args->src_len = len - (src_start - src_addr); ! 3726: if (src_destroy) { ! 3727: cont_args->destroy_addr = cont_args->src_addr; ! 3728: cont_args->destroy_len = cont_args->src_len; ! 3729: } ! 3730: else { ! 3731: cont_args->destroy_addr = (vm_offset_t) 0; ! 3732: cont_args->destroy_len = (vm_offset_t) 0; ! 3733: } ! 3734: cont_args->steal_pages = steal_pages; ! 3735: ! 3736: copy->cpy_cont_args = (char *) cont_args; ! 3737: copy->cpy_cont = vm_map_copyin_page_list_cont; ! 3738: ! 3739: src_end = src_start; ! 3740: vm_map_clip_end(src_map, src_entry, src_end); ! 3741: break; ! 3742: } ! 3743: ! 3744: /* ! 3745: * Try to find the page of data. ! 3746: */ ! 3747: vm_object_lock(src_object); ! 3748: vm_object_paging_begin(src_object); ! 3749: if (((m = vm_page_lookup(src_object, src_offset)) != ! 3750: VM_PAGE_NULL) && !m->busy && !m->fictitious && ! 3751: !m->absent && !m->error) { ! 3752: ! 3753: /* ! 3754: * This is the page. Mark it busy ! 3755: * and keep the paging reference on ! 3756: * the object whilst we do our thing. ! 3757: */ ! 3758: m->busy = TRUE; ! 3759: ! 3760: /* ! 3761: * Also write-protect the page, so ! 3762: * that the map`s owner cannot change ! 3763: * the data. The busy bit will prevent ! 3764: * faults on the page from succeeding ! 3765: * until the copy is released; after ! 3766: * that, the page can be re-entered ! 3767: * as writable, since we didn`t alter ! 3768: * the map entry. This scheme is a ! 3769: * cheap copy-on-write. ! 3770: * ! 3771: * Don`t forget the protection and ! 3772: * the page_lock value! ! 3773: * ! 3774: * If the source is being destroyed ! 3775: * AND not shared writable, we don`t ! 3776: * have to protect the page, since ! 3777: * we will destroy the (only) ! 3778: * writable mapping later. ! 3779: */ ! 3780: if (!src_destroy || ! 3781: src_object->use_shared_copy) ! 3782: { ! 3783: pmap_page_protect(m->phys_addr, ! 3784: src_entry->protection ! 3785: & ~m->page_lock ! 3786: & ~VM_PROT_WRITE); ! 3787: } ! 3788: ! 3789: } ! 3790: else { ! 3791: vm_prot_t result_prot; ! 3792: vm_page_t top_page; ! 3793: kern_return_t kr; ! 3794: ! 3795: /* ! 3796: * Have to fault the page in; must ! 3797: * unlock the map to do so. While ! 3798: * the map is unlocked, anything ! 3799: * can happen, we must lookup the ! 3800: * map entry before continuing. ! 3801: */ ! 3802: vm_map_unlock(src_map); ! 3803: need_map_lookup = TRUE; ! 3804: retry: ! 3805: result_prot = VM_PROT_READ; ! 3806: ! 3807: kr = vm_fault_page(src_object, src_offset, ! 3808: VM_PROT_READ, FALSE, FALSE, ! 3809: &result_prot, &m, &top_page, ! 3810: FALSE, (void (*)()) 0); ! 3811: /* ! 3812: * Cope with what happened. ! 3813: */ ! 3814: switch (kr) { ! 3815: case VM_FAULT_SUCCESS: ! 3816: break; ! 3817: case VM_FAULT_INTERRUPTED: /* ??? */ ! 3818: case VM_FAULT_RETRY: ! 3819: vm_object_lock(src_object); ! 3820: vm_object_paging_begin(src_object); ! 3821: goto retry; ! 3822: case VM_FAULT_MEMORY_SHORTAGE: ! 3823: VM_PAGE_WAIT((void (*)()) 0); ! 3824: vm_object_lock(src_object); ! 3825: vm_object_paging_begin(src_object); ! 3826: goto retry; ! 3827: case VM_FAULT_FICTITIOUS_SHORTAGE: ! 3828: vm_page_more_fictitious(); ! 3829: vm_object_lock(src_object); ! 3830: vm_object_paging_begin(src_object); ! 3831: goto retry; ! 3832: case VM_FAULT_MEMORY_ERROR: ! 3833: /* ! 3834: * Something broke. If this ! 3835: * is a continuation, return ! 3836: * a partial result if possible, ! 3837: * else fail the whole thing. ! 3838: * In the continuation case, the ! 3839: * next continuation call will ! 3840: * get this error if it persists. ! 3841: */ ! 3842: vm_map_lock(src_map); ! 3843: if (is_cont && ! 3844: copy->cpy_npages != 0) ! 3845: goto make_continuation; ! 3846: ! 3847: result = KERN_MEMORY_ERROR; ! 3848: goto error; ! 3849: } ! 3850: ! 3851: if (top_page != VM_PAGE_NULL) { ! 3852: vm_object_lock(src_object); ! 3853: VM_PAGE_FREE(top_page); ! 3854: vm_object_paging_end(src_object); ! 3855: vm_object_unlock(src_object); ! 3856: } ! 3857: ! 3858: /* ! 3859: * We do not need to write-protect ! 3860: * the page, since it cannot have ! 3861: * been in the pmap (and we did not ! 3862: * enter it above). The busy bit ! 3863: * will protect the page from being ! 3864: * entered as writable until it is ! 3865: * unlocked. ! 3866: */ ! 3867: ! 3868: } ! 3869: ! 3870: /* ! 3871: * The page is busy, its object is locked, and ! 3872: * we have a paging reference on it. Either ! 3873: * the map is locked, or need_map_lookup is ! 3874: * TRUE. ! 3875: * ! 3876: * Put the page in the page list. ! 3877: */ ! 3878: copy->cpy_page_list[copy->cpy_npages++] = m; ! 3879: vm_object_unlock(m->object); ! 3880: } ! 3881: ! 3882: /* ! 3883: * DETERMINE whether the entire region ! 3884: * has been copied. ! 3885: */ ! 3886: if (src_start >= src_end && src_end != 0) { ! 3887: if (need_map_lookup) ! 3888: vm_map_lock(src_map); ! 3889: break; ! 3890: } ! 3891: ! 3892: /* ! 3893: * If need_map_lookup is TRUE, have to start over with ! 3894: * another map lookup. Note that we dropped the map ! 3895: * lock (to call vm_fault_page) above only in this case. ! 3896: */ ! 3897: if (need_map_lookup) ! 3898: goto do_map_lookup; ! 3899: ! 3900: /* ! 3901: * Verify that there are no gaps in the region ! 3902: */ ! 3903: ! 3904: src_start = src_entry->vme_end; ! 3905: src_entry = src_entry->vme_next; ! 3906: if (src_entry->vme_start != src_start) { ! 3907: result = KERN_INVALID_ADDRESS; ! 3908: goto error; ! 3909: } ! 3910: } ! 3911: ! 3912: /* ! 3913: * If steal_pages is true, make sure all ! 3914: * pages in the copy are not in any object ! 3915: * We try to remove them from the original ! 3916: * object, but we may have to copy them. ! 3917: * ! 3918: * At this point every page in the list is busy ! 3919: * and holds a paging reference to its object. ! 3920: * When we're done stealing, every page is busy, ! 3921: * and in no object (m->tabled == FALSE). ! 3922: */ ! 3923: src_start = trunc_page(src_addr); ! 3924: if (steal_pages) { ! 3925: register int i; ! 3926: vm_offset_t unwire_end; ! 3927: ! 3928: unwire_end = src_start; ! 3929: for (i = 0; i < copy->cpy_npages; i++) { ! 3930: ! 3931: /* ! 3932: * Remove the page from its object if it ! 3933: * can be stolen. It can be stolen if: ! 3934: * ! 3935: * (1) The source is being destroyed, ! 3936: * the object is temporary, and ! 3937: * not shared. ! 3938: * (2) The page is not precious. ! 3939: * ! 3940: * The not shared check consists of two ! 3941: * parts: (a) there are no objects that ! 3942: * shadow this object. (b) it is not the ! 3943: * object in any shared map entries (i.e., ! 3944: * use_shared_copy is not set). ! 3945: * ! 3946: * The first check (a) means that we can't ! 3947: * steal pages from objects that are not ! 3948: * at the top of their shadow chains. This ! 3949: * should not be a frequent occurrence. ! 3950: * ! 3951: * Stealing wired pages requires telling the ! 3952: * pmap module to let go of them. ! 3953: * ! 3954: * NOTE: stealing clean pages from objects ! 3955: * whose mappings survive requires a call to ! 3956: * the pmap module. Maybe later. ! 3957: */ ! 3958: m = copy->cpy_page_list[i]; ! 3959: src_object = m->object; ! 3960: vm_object_lock(src_object); ! 3961: ! 3962: if (src_destroy && ! 3963: src_object->temporary && ! 3964: (!src_object->shadowed) && ! 3965: (!src_object->use_shared_copy) && ! 3966: !m->precious) { ! 3967: vm_offset_t page_vaddr; ! 3968: ! 3969: page_vaddr = src_start + (i * PAGE_SIZE); ! 3970: if (m->wire_count > 0) { ! 3971: ! 3972: assert(m->wire_count == 1); ! 3973: /* ! 3974: * In order to steal a wired ! 3975: * page, we have to unwire it ! 3976: * first. We do this inline ! 3977: * here because we have the page. ! 3978: * ! 3979: * Step 1: Unwire the map entry. ! 3980: * Also tell the pmap module ! 3981: * that this piece of the ! 3982: * pmap is pageable. ! 3983: */ ! 3984: vm_object_unlock(src_object); ! 3985: if (page_vaddr >= unwire_end) { ! 3986: if (!vm_map_lookup_entry(src_map, ! 3987: page_vaddr, &src_entry)) ! 3988: panic("vm_map_copyin_page_list: missing wired map entry"); ! 3989: ! 3990: vm_map_clip_start(src_map, src_entry, ! 3991: page_vaddr); ! 3992: vm_map_clip_end(src_map, src_entry, ! 3993: src_start + src_size); ! 3994: ! 3995: assert(src_entry->wired_count > 0); ! 3996: src_entry->wired_count = 0; ! 3997: src_entry->user_wired_count = 0; ! 3998: unwire_end = src_entry->vme_end; ! 3999: pmap_pageable(vm_map_pmap(src_map), ! 4000: page_vaddr, unwire_end, TRUE); ! 4001: } ! 4002: ! 4003: /* ! 4004: * Step 2: Unwire the page. ! 4005: * pmap_remove handles this for us. ! 4006: */ ! 4007: vm_object_lock(src_object); ! 4008: } ! 4009: ! 4010: /* ! 4011: * Don't need to remove the mapping; ! 4012: * vm_map_delete will handle it. ! 4013: * ! 4014: * Steal the page. Setting the wire count ! 4015: * to zero is vm_page_unwire without ! 4016: * activating the page. ! 4017: */ ! 4018: vm_page_lock_queues(); ! 4019: vm_page_remove(m); ! 4020: if (m->wire_count > 0) { ! 4021: m->wire_count = 0; ! 4022: vm_page_wire_count--; ! 4023: } else { ! 4024: VM_PAGE_QUEUES_REMOVE(m); ! 4025: } ! 4026: vm_page_unlock_queues(); ! 4027: } ! 4028: else { ! 4029: /* ! 4030: * Have to copy this page. Have to ! 4031: * unlock the map while copying, ! 4032: * hence no further page stealing. ! 4033: * Hence just copy all the pages. ! 4034: * Unlock the map while copying; ! 4035: * This means no further page stealing. ! 4036: */ ! 4037: vm_object_unlock(src_object); ! 4038: vm_map_unlock(src_map); ! 4039: ! 4040: vm_map_copy_steal_pages(copy); ! 4041: ! 4042: vm_map_lock(src_map); ! 4043: break; ! 4044: } ! 4045: ! 4046: vm_object_paging_end(src_object); ! 4047: vm_object_unlock(src_object); ! 4048: } ! 4049: ! 4050: /* ! 4051: * If the source should be destroyed, do it now, since the ! 4052: * copy was successful. ! 4053: */ ! 4054: ! 4055: if (src_destroy) { ! 4056: (void) vm_map_delete(src_map, src_start, src_end); ! 4057: } ! 4058: } ! 4059: else { ! 4060: /* ! 4061: * !steal_pages leaves busy pages in the map. ! 4062: * This will cause src_destroy to hang. Use ! 4063: * a continuation to prevent this. ! 4064: */ ! 4065: if (src_destroy && !vm_map_copy_has_cont(copy)) { ! 4066: cont_args = (vm_map_copyin_args_t) ! 4067: kalloc(sizeof(vm_map_copyin_args_data_t)); ! 4068: vm_map_reference(src_map); ! 4069: cont_args->map = src_map; ! 4070: cont_args->src_addr = (vm_offset_t) 0; ! 4071: cont_args->src_len = (vm_size_t) 0; ! 4072: cont_args->destroy_addr = src_start; ! 4073: cont_args->destroy_len = src_end - src_start; ! 4074: cont_args->steal_pages = FALSE; ! 4075: ! 4076: copy->cpy_cont_args = (char *) cont_args; ! 4077: copy->cpy_cont = vm_map_copyin_page_list_cont; ! 4078: } ! 4079: ! 4080: } ! 4081: ! 4082: vm_map_unlock(src_map); ! 4083: ! 4084: *copy_result = copy; ! 4085: return(result); ! 4086: ! 4087: error: ! 4088: vm_map_unlock(src_map); ! 4089: vm_map_copy_discard(copy); ! 4090: return(result); ! 4091: } ! 4092: ! 4093: /* ! 4094: * vm_map_fork: ! 4095: * ! 4096: * Create and return a new map based on the old ! 4097: * map, according to the inheritance values on the ! 4098: * regions in that map. ! 4099: * ! 4100: * The source map must not be locked. ! 4101: */ ! 4102: vm_map_t vm_map_fork(old_map) ! 4103: vm_map_t old_map; ! 4104: { ! 4105: vm_map_t new_map; ! 4106: register ! 4107: vm_map_entry_t old_entry; ! 4108: register ! 4109: vm_map_entry_t new_entry; ! 4110: pmap_t new_pmap = pmap_create((vm_size_t) 0); ! 4111: vm_size_t new_size = 0; ! 4112: vm_size_t entry_size; ! 4113: register ! 4114: vm_object_t object; ! 4115: ! 4116: vm_map_lock(old_map); ! 4117: ! 4118: new_map = vm_map_create(new_pmap, ! 4119: old_map->min_offset, ! 4120: old_map->max_offset, ! 4121: old_map->hdr.entries_pageable); ! 4122: ! 4123: for ( ! 4124: old_entry = vm_map_first_entry(old_map); ! 4125: old_entry != vm_map_to_entry(old_map); ! 4126: ) { ! 4127: if (old_entry->is_sub_map) ! 4128: panic("vm_map_fork: encountered a submap"); ! 4129: ! 4130: entry_size = (old_entry->vme_end - old_entry->vme_start); ! 4131: ! 4132: switch (old_entry->inheritance) { ! 4133: case VM_INHERIT_NONE: ! 4134: break; ! 4135: ! 4136: case VM_INHERIT_SHARE: ! 4137: /* ! 4138: * New sharing code. New map entry ! 4139: * references original object. Temporary ! 4140: * objects use asynchronous copy algorithm for ! 4141: * future copies. First make sure we have ! 4142: * the right object. If we need a shadow, ! 4143: * or someone else already has one, then ! 4144: * make a new shadow and share it. ! 4145: */ ! 4146: ! 4147: object = old_entry->object.vm_object; ! 4148: if (object == VM_OBJECT_NULL) { ! 4149: object = vm_object_allocate( ! 4150: (vm_size_t)(old_entry->vme_end - ! 4151: old_entry->vme_start)); ! 4152: old_entry->offset = 0; ! 4153: old_entry->object.vm_object = object; ! 4154: assert(!old_entry->needs_copy); ! 4155: } ! 4156: else if (old_entry->needs_copy || object->shadowed || ! 4157: (object->temporary && !old_entry->is_shared && ! 4158: object->size > (vm_size_t)(old_entry->vme_end - ! 4159: old_entry->vme_start))) { ! 4160: ! 4161: assert(object->temporary); ! 4162: assert(!(object->shadowed && old_entry->is_shared)); ! 4163: vm_object_shadow( ! 4164: &old_entry->object.vm_object, ! 4165: &old_entry->offset, ! 4166: (vm_size_t) (old_entry->vme_end - ! 4167: old_entry->vme_start)); ! 4168: ! 4169: /* ! 4170: * If we're making a shadow for other than ! 4171: * copy on write reasons, then we have ! 4172: * to remove write permission. ! 4173: */ ! 4174: ! 4175: if (!old_entry->needs_copy && ! 4176: (old_entry->protection & VM_PROT_WRITE)) { ! 4177: pmap_protect(vm_map_pmap(old_map), ! 4178: old_entry->vme_start, ! 4179: old_entry->vme_end, ! 4180: old_entry->protection & ! 4181: ~VM_PROT_WRITE); ! 4182: } ! 4183: old_entry->needs_copy = FALSE; ! 4184: object = old_entry->object.vm_object; ! 4185: } ! 4186: ! 4187: /* ! 4188: * Set use_shared_copy to indicate that ! 4189: * object must use shared (delayed) copy-on ! 4190: * write. This is ignored for permanent objects. ! 4191: * Bump the reference count for the new entry ! 4192: */ ! 4193: ! 4194: vm_object_lock(object); ! 4195: object->use_shared_copy = TRUE; ! 4196: object->ref_count++; ! 4197: vm_object_unlock(object); ! 4198: ! 4199: if (old_entry->projected_on != 0) { ! 4200: /* ! 4201: * If entry is projected buffer, clone the ! 4202: * entry exactly. ! 4203: */ ! 4204: ! 4205: vm_map_entry_copy_full(new_entry, old_entry); ! 4206: ! 4207: } else { ! 4208: /* ! 4209: * Clone the entry, using object ref from above. ! 4210: * Mark both entries as shared. ! 4211: */ ! 4212: ! 4213: new_entry = vm_map_entry_create(new_map); ! 4214: vm_map_entry_copy(new_entry, old_entry); ! 4215: old_entry->is_shared = TRUE; ! 4216: new_entry->is_shared = TRUE; ! 4217: } ! 4218: ! 4219: /* ! 4220: * Insert the entry into the new map -- we ! 4221: * know we're inserting at the end of the new ! 4222: * map. ! 4223: */ ! 4224: ! 4225: vm_map_entry_link( ! 4226: new_map, ! 4227: vm_map_last_entry(new_map), ! 4228: new_entry); ! 4229: ! 4230: /* ! 4231: * Update the physical map ! 4232: */ ! 4233: ! 4234: pmap_copy(new_map->pmap, old_map->pmap, ! 4235: new_entry->vme_start, ! 4236: entry_size, ! 4237: old_entry->vme_start); ! 4238: ! 4239: new_size += entry_size; ! 4240: break; ! 4241: ! 4242: case VM_INHERIT_COPY: ! 4243: if (old_entry->wired_count == 0) { ! 4244: boolean_t src_needs_copy; ! 4245: boolean_t new_entry_needs_copy; ! 4246: ! 4247: new_entry = vm_map_entry_create(new_map); ! 4248: vm_map_entry_copy(new_entry, old_entry); ! 4249: ! 4250: if (vm_object_copy_temporary( ! 4251: &new_entry->object.vm_object, ! 4252: &new_entry->offset, ! 4253: &src_needs_copy, ! 4254: &new_entry_needs_copy)) { ! 4255: ! 4256: /* ! 4257: * Handle copy-on-write obligations ! 4258: */ ! 4259: ! 4260: if (src_needs_copy && !old_entry->needs_copy) { ! 4261: vm_object_pmap_protect( ! 4262: old_entry->object.vm_object, ! 4263: old_entry->offset, ! 4264: entry_size, ! 4265: (old_entry->is_shared ? ! 4266: PMAP_NULL : ! 4267: old_map->pmap), ! 4268: old_entry->vme_start, ! 4269: old_entry->protection & ! 4270: ~VM_PROT_WRITE); ! 4271: ! 4272: old_entry->needs_copy = TRUE; ! 4273: } ! 4274: ! 4275: new_entry->needs_copy = new_entry_needs_copy; ! 4276: ! 4277: /* ! 4278: * Insert the entry at the end ! 4279: * of the map. ! 4280: */ ! 4281: ! 4282: vm_map_entry_link(new_map, ! 4283: vm_map_last_entry(new_map), ! 4284: new_entry); ! 4285: ! 4286: ! 4287: new_size += entry_size; ! 4288: break; ! 4289: } ! 4290: ! 4291: vm_map_entry_dispose(new_map, new_entry); ! 4292: } ! 4293: ! 4294: /* INNER BLOCK (copy cannot be optimized) */ { ! 4295: ! 4296: vm_offset_t start = old_entry->vme_start; ! 4297: vm_map_copy_t copy; ! 4298: vm_map_entry_t last = vm_map_last_entry(new_map); ! 4299: ! 4300: vm_map_unlock(old_map); ! 4301: if (vm_map_copyin(old_map, ! 4302: start, ! 4303: entry_size, ! 4304: FALSE, ! 4305: ©) ! 4306: != KERN_SUCCESS) { ! 4307: vm_map_lock(old_map); ! 4308: if (!vm_map_lookup_entry(old_map, start, &last)) ! 4309: last = last->vme_next; ! 4310: old_entry = last; ! 4311: /* ! 4312: * For some error returns, want to ! 4313: * skip to the next element. ! 4314: */ ! 4315: ! 4316: continue; ! 4317: } ! 4318: ! 4319: /* ! 4320: * Insert the copy into the new map ! 4321: */ ! 4322: ! 4323: vm_map_copy_insert(new_map, last, copy); ! 4324: new_size += entry_size; ! 4325: ! 4326: /* ! 4327: * Pick up the traversal at the end of ! 4328: * the copied region. ! 4329: */ ! 4330: ! 4331: vm_map_lock(old_map); ! 4332: start += entry_size; ! 4333: if (!vm_map_lookup_entry(old_map, start, &last)) ! 4334: last = last->vme_next; ! 4335: else ! 4336: vm_map_clip_start(old_map, last, start); ! 4337: old_entry = last; ! 4338: ! 4339: continue; ! 4340: /* INNER BLOCK (copy cannot be optimized) */ } ! 4341: } ! 4342: old_entry = old_entry->vme_next; ! 4343: } ! 4344: ! 4345: new_map->size = new_size; ! 4346: vm_map_unlock(old_map); ! 4347: ! 4348: return(new_map); ! 4349: } ! 4350: ! 4351: /* ! 4352: * vm_map_lookup: ! 4353: * ! 4354: * Finds the VM object, offset, and ! 4355: * protection for a given virtual address in the ! 4356: * specified map, assuming a page fault of the ! 4357: * type specified. ! 4358: * ! 4359: * Returns the (object, offset, protection) for ! 4360: * this address, whether it is wired down, and whether ! 4361: * this map has the only reference to the data in question. ! 4362: * In order to later verify this lookup, a "version" ! 4363: * is returned. ! 4364: * ! 4365: * The map should not be locked; it will not be ! 4366: * locked on exit. In order to guarantee the ! 4367: * existence of the returned object, it is returned ! 4368: * locked. ! 4369: * ! 4370: * If a lookup is requested with "write protection" ! 4371: * specified, the map may be changed to perform virtual ! 4372: * copying operations, although the data referenced will ! 4373: * remain the same. ! 4374: */ ! 4375: kern_return_t vm_map_lookup(var_map, vaddr, fault_type, out_version, ! 4376: object, offset, out_prot, wired) ! 4377: vm_map_t *var_map; /* IN/OUT */ ! 4378: register vm_offset_t vaddr; ! 4379: register vm_prot_t fault_type; ! 4380: ! 4381: vm_map_version_t *out_version; /* OUT */ ! 4382: vm_object_t *object; /* OUT */ ! 4383: vm_offset_t *offset; /* OUT */ ! 4384: vm_prot_t *out_prot; /* OUT */ ! 4385: boolean_t *wired; /* OUT */ ! 4386: { ! 4387: register vm_map_entry_t entry; ! 4388: register vm_map_t map = *var_map; ! 4389: register vm_prot_t prot; ! 4390: ! 4391: RetryLookup: ; ! 4392: ! 4393: /* ! 4394: * Lookup the faulting address. ! 4395: */ ! 4396: ! 4397: vm_map_lock_read(map); ! 4398: ! 4399: #define RETURN(why) \ ! 4400: { \ ! 4401: vm_map_unlock_read(map); \ ! 4402: return(why); \ ! 4403: } ! 4404: ! 4405: /* ! 4406: * If the map has an interesting hint, try it before calling ! 4407: * full blown lookup routine. ! 4408: */ ! 4409: ! 4410: simple_lock(&map->hint_lock); ! 4411: entry = map->hint; ! 4412: simple_unlock(&map->hint_lock); ! 4413: ! 4414: if ((entry == vm_map_to_entry(map)) || ! 4415: (vaddr < entry->vme_start) || (vaddr >= entry->vme_end)) { ! 4416: vm_map_entry_t tmp_entry; ! 4417: ! 4418: /* ! 4419: * Entry was either not a valid hint, or the vaddr ! 4420: * was not contained in the entry, so do a full lookup. ! 4421: */ ! 4422: if (!vm_map_lookup_entry(map, vaddr, &tmp_entry)) ! 4423: RETURN(KERN_INVALID_ADDRESS); ! 4424: ! 4425: entry = tmp_entry; ! 4426: } ! 4427: ! 4428: /* ! 4429: * Handle submaps. ! 4430: */ ! 4431: ! 4432: if (entry->is_sub_map) { ! 4433: vm_map_t old_map = map; ! 4434: ! 4435: *var_map = map = entry->object.sub_map; ! 4436: vm_map_unlock_read(old_map); ! 4437: goto RetryLookup; ! 4438: } ! 4439: ! 4440: /* ! 4441: * Check whether this task is allowed to have ! 4442: * this page. ! 4443: */ ! 4444: ! 4445: prot = entry->protection; ! 4446: ! 4447: if ((fault_type & (prot)) != fault_type) ! 4448: if ((prot & VM_PROT_NOTIFY) && (fault_type & VM_PROT_WRITE)) { ! 4449: RETURN(KERN_WRITE_PROTECTION_FAILURE); ! 4450: } else { ! 4451: RETURN(KERN_PROTECTION_FAILURE); ! 4452: } ! 4453: ! 4454: /* ! 4455: * If this page is not pageable, we have to get ! 4456: * it for all possible accesses. ! 4457: */ ! 4458: ! 4459: if (*wired = (entry->wired_count != 0)) ! 4460: prot = fault_type = entry->protection; ! 4461: ! 4462: /* ! 4463: * If the entry was copy-on-write, we either ... ! 4464: */ ! 4465: ! 4466: if (entry->needs_copy) { ! 4467: /* ! 4468: * If we want to write the page, we may as well ! 4469: * handle that now since we've got the map locked. ! 4470: * ! 4471: * If we don't need to write the page, we just ! 4472: * demote the permissions allowed. ! 4473: */ ! 4474: ! 4475: if (fault_type & VM_PROT_WRITE) { ! 4476: /* ! 4477: * Make a new object, and place it in the ! 4478: * object chain. Note that no new references ! 4479: * have appeared -- one just moved from the ! 4480: * map to the new object. ! 4481: */ ! 4482: ! 4483: if (vm_map_lock_read_to_write(map)) { ! 4484: goto RetryLookup; ! 4485: } ! 4486: map->timestamp++; ! 4487: ! 4488: vm_object_shadow( ! 4489: &entry->object.vm_object, ! 4490: &entry->offset, ! 4491: (vm_size_t) (entry->vme_end - entry->vme_start)); ! 4492: ! 4493: entry->needs_copy = FALSE; ! 4494: ! 4495: vm_map_lock_write_to_read(map); ! 4496: } ! 4497: else { ! 4498: /* ! 4499: * We're attempting to read a copy-on-write ! 4500: * page -- don't allow writes. ! 4501: */ ! 4502: ! 4503: prot &= (~VM_PROT_WRITE); ! 4504: } ! 4505: } ! 4506: ! 4507: /* ! 4508: * Create an object if necessary. ! 4509: */ ! 4510: if (entry->object.vm_object == VM_OBJECT_NULL) { ! 4511: ! 4512: if (vm_map_lock_read_to_write(map)) { ! 4513: goto RetryLookup; ! 4514: } ! 4515: ! 4516: entry->object.vm_object = vm_object_allocate( ! 4517: (vm_size_t)(entry->vme_end - entry->vme_start)); ! 4518: entry->offset = 0; ! 4519: vm_map_lock_write_to_read(map); ! 4520: } ! 4521: ! 4522: /* ! 4523: * Return the object/offset from this entry. If the entry ! 4524: * was copy-on-write or empty, it has been fixed up. Also ! 4525: * return the protection. ! 4526: */ ! 4527: ! 4528: *offset = (vaddr - entry->vme_start) + entry->offset; ! 4529: *object = entry->object.vm_object; ! 4530: *out_prot = prot; ! 4531: ! 4532: /* ! 4533: * Lock the object to prevent it from disappearing ! 4534: */ ! 4535: ! 4536: vm_object_lock(*object); ! 4537: ! 4538: /* ! 4539: * Save the version number and unlock the map. ! 4540: */ ! 4541: ! 4542: out_version->main_timestamp = map->timestamp; ! 4543: ! 4544: RETURN(KERN_SUCCESS); ! 4545: ! 4546: #undef RETURN ! 4547: } ! 4548: ! 4549: /* ! 4550: * vm_map_verify: ! 4551: * ! 4552: * Verifies that the map in question has not changed ! 4553: * since the given version. If successful, the map ! 4554: * will not change until vm_map_verify_done() is called. ! 4555: */ ! 4556: boolean_t vm_map_verify(map, version) ! 4557: register ! 4558: vm_map_t map; ! 4559: register ! 4560: vm_map_version_t *version; /* REF */ ! 4561: { ! 4562: boolean_t result; ! 4563: ! 4564: vm_map_lock_read(map); ! 4565: result = (map->timestamp == version->main_timestamp); ! 4566: ! 4567: if (!result) ! 4568: vm_map_unlock_read(map); ! 4569: ! 4570: return(result); ! 4571: } ! 4572: ! 4573: /* ! 4574: * vm_map_verify_done: ! 4575: * ! 4576: * Releases locks acquired by a vm_map_verify. ! 4577: * ! 4578: * This is now a macro in vm/vm_map.h. It does a ! 4579: * vm_map_unlock_read on the map. ! 4580: */ ! 4581: ! 4582: /* ! 4583: * vm_region: ! 4584: * ! 4585: * User call to obtain information about a region in ! 4586: * a task's address map. ! 4587: */ ! 4588: ! 4589: kern_return_t vm_region(map, address, size, ! 4590: protection, max_protection, ! 4591: inheritance, is_shared, ! 4592: object_name, offset_in_object) ! 4593: vm_map_t map; ! 4594: vm_offset_t *address; /* IN/OUT */ ! 4595: vm_size_t *size; /* OUT */ ! 4596: vm_prot_t *protection; /* OUT */ ! 4597: vm_prot_t *max_protection; /* OUT */ ! 4598: vm_inherit_t *inheritance; /* OUT */ ! 4599: boolean_t *is_shared; /* OUT */ ! 4600: ipc_port_t *object_name; /* OUT */ ! 4601: vm_offset_t *offset_in_object; /* OUT */ ! 4602: { ! 4603: vm_map_entry_t tmp_entry; ! 4604: register ! 4605: vm_map_entry_t entry; ! 4606: register ! 4607: vm_offset_t tmp_offset; ! 4608: vm_offset_t start; ! 4609: ! 4610: if (map == VM_MAP_NULL) ! 4611: return(KERN_INVALID_ARGUMENT); ! 4612: ! 4613: start = *address; ! 4614: ! 4615: vm_map_lock_read(map); ! 4616: if (!vm_map_lookup_entry(map, start, &tmp_entry)) { ! 4617: if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) { ! 4618: vm_map_unlock_read(map); ! 4619: return(KERN_NO_SPACE); ! 4620: } ! 4621: } else { ! 4622: entry = tmp_entry; ! 4623: } ! 4624: ! 4625: start = entry->vme_start; ! 4626: *protection = entry->protection; ! 4627: *max_protection = entry->max_protection; ! 4628: *inheritance = entry->inheritance; ! 4629: *address = start; ! 4630: *size = (entry->vme_end - start); ! 4631: ! 4632: tmp_offset = entry->offset; ! 4633: ! 4634: ! 4635: if (entry->is_sub_map) { ! 4636: *is_shared = FALSE; ! 4637: *object_name = IP_NULL; ! 4638: *offset_in_object = tmp_offset; ! 4639: } else { ! 4640: *is_shared = entry->is_shared; ! 4641: *object_name = vm_object_name(entry->object.vm_object); ! 4642: *offset_in_object = tmp_offset; ! 4643: } ! 4644: ! 4645: vm_map_unlock_read(map); ! 4646: ! 4647: return(KERN_SUCCESS); ! 4648: } ! 4649: ! 4650: /* ! 4651: * Routine: vm_map_simplify ! 4652: * ! 4653: * Description: ! 4654: * Attempt to simplify the map representation in ! 4655: * the vicinity of the given starting address. ! 4656: * Note: ! 4657: * This routine is intended primarily to keep the ! 4658: * kernel maps more compact -- they generally don't ! 4659: * benefit from the "expand a map entry" technology ! 4660: * at allocation time because the adjacent entry ! 4661: * is often wired down. ! 4662: */ ! 4663: void vm_map_simplify(map, start) ! 4664: vm_map_t map; ! 4665: vm_offset_t start; ! 4666: { ! 4667: vm_map_entry_t this_entry; ! 4668: vm_map_entry_t prev_entry; ! 4669: ! 4670: vm_map_lock(map); ! 4671: if ( ! 4672: (vm_map_lookup_entry(map, start, &this_entry)) && ! 4673: ((prev_entry = this_entry->vme_prev) != vm_map_to_entry(map)) && ! 4674: ! 4675: (prev_entry->vme_end == start) && ! 4676: ! 4677: (prev_entry->is_shared == FALSE) && ! 4678: (prev_entry->is_sub_map == FALSE) && ! 4679: ! 4680: (this_entry->is_shared == FALSE) && ! 4681: (this_entry->is_sub_map == FALSE) && ! 4682: ! 4683: (prev_entry->inheritance == this_entry->inheritance) && ! 4684: (prev_entry->protection == this_entry->protection) && ! 4685: (prev_entry->max_protection == this_entry->max_protection) && ! 4686: (prev_entry->wired_count == this_entry->wired_count) && ! 4687: (prev_entry->user_wired_count == this_entry->user_wired_count) && ! 4688: ! 4689: (prev_entry->needs_copy == this_entry->needs_copy) && ! 4690: ! 4691: (prev_entry->object.vm_object == this_entry->object.vm_object) && ! 4692: ((prev_entry->offset + (prev_entry->vme_end - prev_entry->vme_start)) ! 4693: == this_entry->offset) && ! 4694: (prev_entry->projected_on == 0) && ! 4695: (this_entry->projected_on == 0) ! 4696: ) { ! 4697: if (map->first_free == this_entry) ! 4698: map->first_free = prev_entry; ! 4699: ! 4700: SAVE_HINT(map, prev_entry); ! 4701: vm_map_entry_unlink(map, this_entry); ! 4702: prev_entry->vme_end = this_entry->vme_end; ! 4703: vm_object_deallocate(this_entry->object.vm_object); ! 4704: vm_map_entry_dispose(map, this_entry); ! 4705: } ! 4706: vm_map_unlock(map); ! 4707: } ! 4708: ! 4709: ! 4710: /* ! 4711: * Routine: vm_map_machine_attribute ! 4712: * Purpose: ! 4713: * Provide machine-specific attributes to mappings, ! 4714: * such as cachability etc. for machines that provide ! 4715: * them. NUMA architectures and machines with big/strange ! 4716: * caches will use this. ! 4717: * Note: ! 4718: * Responsibilities for locking and checking are handled here, ! 4719: * everything else in the pmap module. If any non-volatile ! 4720: * information must be kept, the pmap module should handle ! 4721: * it itself. [This assumes that attributes do not ! 4722: * need to be inherited, which seems ok to me] ! 4723: */ ! 4724: kern_return_t vm_map_machine_attribute(map, address, size, attribute, value) ! 4725: vm_map_t map; ! 4726: vm_offset_t address; ! 4727: vm_size_t size; ! 4728: vm_machine_attribute_t attribute; ! 4729: vm_machine_attribute_val_t* value; /* IN/OUT */ ! 4730: { ! 4731: kern_return_t ret; ! 4732: ! 4733: if (address < vm_map_min(map) || ! 4734: (address + size) > vm_map_max(map)) ! 4735: return KERN_INVALID_ARGUMENT; ! 4736: ! 4737: vm_map_lock(map); ! 4738: ! 4739: ret = pmap_attribute(map->pmap, address, size, attribute, value); ! 4740: ! 4741: vm_map_unlock(map); ! 4742: ! 4743: return ret; ! 4744: } ! 4745: ! 4746: #include <mach_kdb.h> ! 4747: ! 4748: ! 4749: #if MACH_KDB ! 4750: ! 4751: #define printf kdbprintf ! 4752: ! 4753: /* ! 4754: * vm_map_print: [ debug ] ! 4755: */ ! 4756: void vm_map_print(map) ! 4757: register vm_map_t map; ! 4758: { ! 4759: register vm_map_entry_t entry; ! 4760: extern int indent; ! 4761: ! 4762: iprintf("Task map 0x%X: pmap=0x%X,", ! 4763: (vm_offset_t) map, (vm_offset_t) (map->pmap)); ! 4764: printf("ref=%d,nentries=%d,", map->ref_count, map->hdr.nentries); ! 4765: printf("version=%d\n", map->timestamp); ! 4766: indent += 2; ! 4767: for (entry = vm_map_first_entry(map); ! 4768: entry != vm_map_to_entry(map); ! 4769: entry = entry->vme_next) { ! 4770: static char *inheritance_name[3] = { "share", "copy", "none"}; ! 4771: ! 4772: iprintf("map entry 0x%X: ", (vm_offset_t) entry); ! 4773: printf("start=0x%X, end=0x%X, ", ! 4774: (vm_offset_t) entry->vme_start, (vm_offset_t) entry->vme_end); ! 4775: printf("prot=%X/%X/%s, ", ! 4776: entry->protection, ! 4777: entry->max_protection, ! 4778: inheritance_name[entry->inheritance]); ! 4779: if (entry->wired_count != 0) { ! 4780: printf("wired("); ! 4781: if (entry->user_wired_count != 0) ! 4782: printf("u"); ! 4783: if (entry->wired_count > ! 4784: ((entry->user_wired_count == 0) ? 0 : 1)) ! 4785: printf("k"); ! 4786: printf(") "); ! 4787: } ! 4788: if (entry->in_transition) { ! 4789: printf("in transition"); ! 4790: if (entry->needs_wakeup) ! 4791: printf("(wake request)"); ! 4792: printf(", "); ! 4793: } ! 4794: if (entry->is_sub_map) { ! 4795: printf("submap=0x%X, offset=0x%X\n", ! 4796: (vm_offset_t) entry->object.sub_map, ! 4797: (vm_offset_t) entry->offset); ! 4798: } else { ! 4799: printf("object=0x%X, offset=0x%X", ! 4800: (vm_offset_t) entry->object.vm_object, ! 4801: (vm_offset_t) entry->offset); ! 4802: if (entry->is_shared) ! 4803: printf(", shared"); ! 4804: if (entry->needs_copy) ! 4805: printf(", copy needed"); ! 4806: printf("\n"); ! 4807: ! 4808: if ((entry->vme_prev == vm_map_to_entry(map)) || ! 4809: (entry->vme_prev->object.vm_object != entry->object.vm_object)) { ! 4810: indent += 2; ! 4811: vm_object_print(entry->object.vm_object); ! 4812: indent -= 2; ! 4813: } ! 4814: } ! 4815: } ! 4816: indent -= 2; ! 4817: } ! 4818: ! 4819: /* ! 4820: * Routine: vm_map_copy_print ! 4821: * Purpose: ! 4822: * Pretty-print a copy object for ddb. ! 4823: */ ! 4824: ! 4825: void vm_map_copy_print(copy) ! 4826: vm_map_copy_t copy; ! 4827: { ! 4828: extern int indent; ! 4829: int i, npages; ! 4830: ! 4831: printf("copy object 0x%x\n", copy); ! 4832: ! 4833: indent += 2; ! 4834: ! 4835: iprintf("type=%d", copy->type); ! 4836: switch (copy->type) { ! 4837: case VM_MAP_COPY_ENTRY_LIST: ! 4838: printf("[entry_list]"); ! 4839: break; ! 4840: ! 4841: case VM_MAP_COPY_OBJECT: ! 4842: printf("[object]"); ! 4843: break; ! 4844: ! 4845: case VM_MAP_COPY_PAGE_LIST: ! 4846: printf("[page_list]"); ! 4847: break; ! 4848: ! 4849: default: ! 4850: printf("[bad type]"); ! 4851: break; ! 4852: } ! 4853: printf(", offset=0x%x", copy->offset); ! 4854: printf(", size=0x%x\n", copy->size); ! 4855: ! 4856: switch (copy->type) { ! 4857: case VM_MAP_COPY_ENTRY_LIST: ! 4858: /* XXX add stuff here */ ! 4859: break; ! 4860: ! 4861: case VM_MAP_COPY_OBJECT: ! 4862: iprintf("object=0x%x\n", copy->cpy_object); ! 4863: break; ! 4864: ! 4865: case VM_MAP_COPY_PAGE_LIST: ! 4866: iprintf("npages=%d", copy->cpy_npages); ! 4867: printf(", cont=%x", copy->cpy_cont); ! 4868: printf(", cont_args=%x\n", copy->cpy_cont_args); ! 4869: if (copy->cpy_npages < 0) { ! 4870: npages = 0; ! 4871: } else if (copy->cpy_npages > VM_MAP_COPY_PAGE_LIST_MAX) { ! 4872: npages = VM_MAP_COPY_PAGE_LIST_MAX; ! 4873: } else { ! 4874: npages = copy->cpy_npages; ! 4875: } ! 4876: iprintf("copy->cpy_page_list[0..%d] = {", npages); ! 4877: for (i = 0; i < npages - 1; i++) { ! 4878: printf("0x%x, ", copy->cpy_page_list[i]); ! 4879: } ! 4880: if (npages > 0) { ! 4881: printf("0x%x", copy->cpy_page_list[npages - 1]); ! 4882: } ! 4883: printf("}\n"); ! 4884: break; ! 4885: } ! 4886: ! 4887: indent -=2; ! 4888: } ! 4889: #endif /* MACH_KDB */ ! 4890: ! 4891: #if NORMA_IPC ! 4892: /* ! 4893: * This should one day be eliminated; ! 4894: * we should always construct the right flavor of copy object ! 4895: * the first time. Troublesome areas include vm_read, where vm_map_copyin ! 4896: * is called without knowing whom the copy object is for. ! 4897: * There are also situations where we do want a lazy data structure ! 4898: * even if we are sending to a remote port... ! 4899: */ ! 4900: ! 4901: /* ! 4902: * Convert a copy to a page list. The copy argument is in/out ! 4903: * because we probably have to allocate a new vm_map_copy structure. ! 4904: * We take responsibility for discarding the old structure and ! 4905: * use a continuation to do so. Postponing this discard ensures ! 4906: * that the objects containing the pages we've marked busy will stick ! 4907: * around. ! 4908: */ ! 4909: kern_return_t ! 4910: vm_map_convert_to_page_list(caller_copy) ! 4911: vm_map_copy_t *caller_copy; ! 4912: { ! 4913: vm_map_entry_t entry, next_entry; ! 4914: vm_offset_t va; ! 4915: vm_offset_t offset; ! 4916: vm_object_t object; ! 4917: kern_return_t result; ! 4918: vm_map_copy_t copy, new_copy; ! 4919: int i, num_pages = 0; ! 4920: ! 4921: zone_t entry_zone; ! 4922: ! 4923: copy = *caller_copy; ! 4924: ! 4925: /* ! 4926: * We may not have to do anything, ! 4927: * or may not be able to do anything. ! 4928: */ ! 4929: if (copy == VM_MAP_COPY_NULL || copy->type == VM_MAP_COPY_PAGE_LIST) { ! 4930: return KERN_SUCCESS; ! 4931: } ! 4932: if (copy->type == VM_MAP_COPY_OBJECT) { ! 4933: return vm_map_convert_to_page_list_from_object(caller_copy); ! 4934: } ! 4935: if (copy->type != VM_MAP_COPY_ENTRY_LIST) { ! 4936: panic("vm_map_convert_to_page_list: copy type %d!\n", ! 4937: copy->type); ! 4938: } ! 4939: ! 4940: /* ! 4941: * Allocate the new copy. Set its continuation to ! 4942: * discard the old one. ! 4943: */ ! 4944: new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); ! 4945: new_copy->type = VM_MAP_COPY_PAGE_LIST; ! 4946: new_copy->cpy_npages = 0; ! 4947: new_copy->offset = copy->offset; ! 4948: new_copy->size = copy->size; ! 4949: new_copy->cpy_cont = vm_map_copy_discard_cont; ! 4950: new_copy->cpy_cont_args = (char *) copy; ! 4951: ! 4952: /* ! 4953: * Iterate over entries. ! 4954: */ ! 4955: for (entry = vm_map_copy_first_entry(copy); ! 4956: entry != vm_map_copy_to_entry(copy); ! 4957: entry = entry->vme_next) { ! 4958: ! 4959: object = entry->object.vm_object; ! 4960: offset = entry->offset; ! 4961: /* ! 4962: * Iterate over pages. ! 4963: */ ! 4964: for (va = entry->vme_start; ! 4965: va < entry->vme_end; ! 4966: va += PAGE_SIZE, offset += PAGE_SIZE) { ! 4967: ! 4968: vm_page_t m; ! 4969: ! 4970: if (new_copy->cpy_npages == VM_MAP_COPY_PAGE_LIST_MAX) { ! 4971: /* ! 4972: * What a mess. We need a continuation ! 4973: * to do the page list, but also one ! 4974: * to discard the old copy. The right ! 4975: * thing to do is probably to copy ! 4976: * out the old copy into the kernel ! 4977: * map (or some temporary task holding ! 4978: * map if we're paranoid about large ! 4979: * copies), and then copyin the page ! 4980: * list that we really wanted with ! 4981: * src_destroy. LATER. ! 4982: */ ! 4983: panic("vm_map_convert_to_page_list: num\n"); ! 4984: } ! 4985: ! 4986: /* ! 4987: * Try to find the page of data. ! 4988: */ ! 4989: vm_object_lock(object); ! 4990: vm_object_paging_begin(object); ! 4991: if (((m = vm_page_lookup(object, offset)) != ! 4992: VM_PAGE_NULL) && !m->busy && !m->fictitious && ! 4993: !m->absent && !m->error) { ! 4994: ! 4995: /* ! 4996: * This is the page. Mark it busy ! 4997: * and keep the paging reference on ! 4998: * the object whilst we do our thing. ! 4999: */ ! 5000: m->busy = TRUE; ! 5001: ! 5002: /* ! 5003: * Also write-protect the page, so ! 5004: * that the map`s owner cannot change ! 5005: * the data. The busy bit will prevent ! 5006: * faults on the page from succeeding ! 5007: * until the copy is released; after ! 5008: * that, the page can be re-entered ! 5009: * as writable, since we didn`t alter ! 5010: * the map entry. This scheme is a ! 5011: * cheap copy-on-write. ! 5012: * ! 5013: * Don`t forget the protection and ! 5014: * the page_lock value! ! 5015: */ ! 5016: ! 5017: pmap_page_protect(m->phys_addr, ! 5018: entry->protection ! 5019: & ~m->page_lock ! 5020: & ~VM_PROT_WRITE); ! 5021: ! 5022: } ! 5023: else { ! 5024: vm_prot_t result_prot; ! 5025: vm_page_t top_page; ! 5026: kern_return_t kr; ! 5027: ! 5028: retry: ! 5029: result_prot = VM_PROT_READ; ! 5030: ! 5031: kr = vm_fault_page(object, offset, ! 5032: VM_PROT_READ, FALSE, FALSE, ! 5033: &result_prot, &m, &top_page, ! 5034: FALSE, (void (*)()) 0); ! 5035: if (kr == VM_FAULT_MEMORY_SHORTAGE) { ! 5036: VM_PAGE_WAIT((void (*)()) 0); ! 5037: vm_object_lock(object); ! 5038: vm_object_paging_begin(object); ! 5039: goto retry; ! 5040: } ! 5041: if (kr != VM_FAULT_SUCCESS) { ! 5042: /* XXX what about data_error? */ ! 5043: vm_object_lock(object); ! 5044: vm_object_paging_begin(object); ! 5045: goto retry; ! 5046: } ! 5047: if (top_page != VM_PAGE_NULL) { ! 5048: vm_object_lock(object); ! 5049: VM_PAGE_FREE(top_page); ! 5050: vm_object_paging_end(object); ! 5051: vm_object_unlock(object); ! 5052: } ! 5053: } ! 5054: assert(m); ! 5055: m->busy = TRUE; ! 5056: new_copy->cpy_page_list[new_copy->cpy_npages++] = m; ! 5057: vm_object_unlock(object); ! 5058: } ! 5059: } ! 5060: ! 5061: *caller_copy = new_copy; ! 5062: return KERN_SUCCESS; ! 5063: } ! 5064: ! 5065: kern_return_t ! 5066: vm_map_convert_to_page_list_from_object(caller_copy) ! 5067: vm_map_copy_t *caller_copy; ! 5068: { ! 5069: vm_object_t object; ! 5070: vm_offset_t offset; ! 5071: vm_map_copy_t copy, new_copy; ! 5072: ! 5073: copy = *caller_copy; ! 5074: assert(copy->type == VM_MAP_COPY_OBJECT); ! 5075: object = copy->cpy_object; ! 5076: assert(object->size == round_page(object->size)); ! 5077: ! 5078: /* ! 5079: * Allocate the new copy. Set its continuation to ! 5080: * discard the old one. ! 5081: */ ! 5082: new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone); ! 5083: new_copy->type = VM_MAP_COPY_PAGE_LIST; ! 5084: new_copy->cpy_npages = 0; ! 5085: new_copy->offset = copy->offset; ! 5086: new_copy->size = copy->size; ! 5087: new_copy->cpy_cont = vm_map_copy_discard_cont; ! 5088: new_copy->cpy_cont_args = (char *) copy; ! 5089: ! 5090: /* ! 5091: * XXX memory_object_lock_request can probably bust this ! 5092: * XXX See continuation comment in previous routine for solution. ! 5093: */ ! 5094: assert(object->size <= VM_MAP_COPY_PAGE_LIST_MAX * PAGE_SIZE); ! 5095: ! 5096: for (offset = 0; offset < object->size; offset += PAGE_SIZE) { ! 5097: vm_page_t m; ! 5098: ! 5099: /* ! 5100: * Try to find the page of data. ! 5101: */ ! 5102: vm_object_lock(object); ! 5103: vm_object_paging_begin(object); ! 5104: m = vm_page_lookup(object, offset); ! 5105: if ((m != VM_PAGE_NULL) && !m->busy && !m->fictitious && ! 5106: !m->absent && !m->error) { ! 5107: ! 5108: /* ! 5109: * This is the page. Mark it busy ! 5110: * and keep the paging reference on ! 5111: * the object whilst we do our thing. ! 5112: */ ! 5113: m->busy = TRUE; ! 5114: } ! 5115: else { ! 5116: vm_prot_t result_prot; ! 5117: vm_page_t top_page; ! 5118: kern_return_t kr; ! 5119: ! 5120: retry: ! 5121: result_prot = VM_PROT_READ; ! 5122: ! 5123: kr = vm_fault_page(object, offset, ! 5124: VM_PROT_READ, FALSE, FALSE, ! 5125: &result_prot, &m, &top_page, ! 5126: FALSE, (void (*)()) 0); ! 5127: if (kr == VM_FAULT_MEMORY_SHORTAGE) { ! 5128: VM_PAGE_WAIT((void (*)()) 0); ! 5129: vm_object_lock(object); ! 5130: vm_object_paging_begin(object); ! 5131: goto retry; ! 5132: } ! 5133: if (kr != VM_FAULT_SUCCESS) { ! 5134: /* XXX what about data_error? */ ! 5135: vm_object_lock(object); ! 5136: vm_object_paging_begin(object); ! 5137: goto retry; ! 5138: } ! 5139: ! 5140: if (top_page != VM_PAGE_NULL) { ! 5141: vm_object_lock(object); ! 5142: VM_PAGE_FREE(top_page); ! 5143: vm_object_paging_end(object); ! 5144: vm_object_unlock(object); ! 5145: } ! 5146: } ! 5147: assert(m); ! 5148: m->busy = TRUE; ! 5149: new_copy->cpy_page_list[new_copy->cpy_npages++] = m; ! 5150: vm_object_unlock(object); ! 5151: } ! 5152: ! 5153: *caller_copy = new_copy; ! 5154: return (KERN_SUCCESS); ! 5155: } ! 5156: ! 5157: kern_return_t ! 5158: vm_map_convert_from_page_list(copy) ! 5159: vm_map_copy_t copy; ! 5160: { ! 5161: vm_object_t object; ! 5162: int i; ! 5163: vm_map_entry_t new_entry; ! 5164: vm_page_t *page_list; ! 5165: ! 5166: /* ! 5167: * Check type of copy object. ! 5168: */ ! 5169: if (copy->type == VM_MAP_COPY_ENTRY_LIST) { ! 5170: return KERN_SUCCESS; ! 5171: } ! 5172: if (copy->type == VM_MAP_COPY_OBJECT) { ! 5173: printf("vm_map_convert_from_page_list: COPY_OBJECT?"); ! 5174: return KERN_SUCCESS; ! 5175: } ! 5176: if (copy->type != VM_MAP_COPY_PAGE_LIST) { ! 5177: panic("vm_map_convert_from_page_list 0x%x %d", ! 5178: copy, ! 5179: copy->type); ! 5180: } ! 5181: ! 5182: /* ! 5183: * Make sure the pages are loose. This may be ! 5184: * a "Can't Happen", but just to be safe ... ! 5185: */ ! 5186: page_list = ©->cpy_page_list[0]; ! 5187: if ((*page_list)->tabled) ! 5188: vm_map_copy_steal_pages(copy); ! 5189: ! 5190: /* ! 5191: * Create object, and stuff pages into it. ! 5192: */ ! 5193: object = vm_object_allocate(copy->cpy_npages); ! 5194: for (i = 0; i < copy->cpy_npages; i++) { ! 5195: register vm_page_t m = *page_list++; ! 5196: vm_page_insert(m, object, i * PAGE_SIZE); ! 5197: m->busy = FALSE; ! 5198: m->dirty = TRUE; ! 5199: vm_page_activate(m); ! 5200: } ! 5201: ! 5202: /* ! 5203: * XXX If this page list contained a continuation, then ! 5204: * XXX we're screwed. The right thing to do is probably do ! 5205: * XXX the copyout, and then copyin the entry list we really ! 5206: * XXX wanted. ! 5207: */ ! 5208: if (vm_map_copy_has_cont(copy)) ! 5209: panic("convert_from_page_list: continuation"); ! 5210: ! 5211: /* ! 5212: * Change type of copy object ! 5213: */ ! 5214: vm_map_copy_first_entry(copy) = ! 5215: vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy); ! 5216: copy->type = VM_MAP_COPY_ENTRY_LIST; ! 5217: copy->cpy_hdr.nentries = 0; ! 5218: copy->cpy_hdr.entries_pageable = TRUE; ! 5219: ! 5220: /* ! 5221: * Allocate and initialize an entry for object ! 5222: */ ! 5223: new_entry = vm_map_copy_entry_create(copy); ! 5224: new_entry->vme_start = trunc_page(copy->offset); ! 5225: new_entry->vme_end = round_page(copy->offset + copy->size); ! 5226: new_entry->object.vm_object = object; ! 5227: new_entry->offset = 0; ! 5228: new_entry->is_shared = FALSE; ! 5229: new_entry->is_sub_map = FALSE; ! 5230: new_entry->needs_copy = FALSE; ! 5231: new_entry->protection = VM_PROT_DEFAULT; ! 5232: new_entry->max_protection = VM_PROT_ALL; ! 5233: new_entry->inheritance = VM_INHERIT_DEFAULT; ! 5234: new_entry->wired_count = 0; ! 5235: new_entry->user_wired_count = 0; ! 5236: new_entry->projected_on = 0; ! 5237: ! 5238: /* ! 5239: * Insert entry into copy object, and return. ! 5240: */ ! 5241: vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy), new_entry); ! 5242: return(KERN_SUCCESS); ! 5243: } ! 5244: #endif /* NORMA_IPC */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.