![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_kern.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple XNU] / XNU / osfmk / vm|
Return to vm_kern.c CVS log | Up to [Apple XNU] / XNU / osfmk / vm |
1.1 ! root 1: /* ! 2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. ! 3: * ! 4: * @APPLE_LICENSE_HEADER_START@ ! 5: * ! 6: * The contents of this file constitute Original Code as defined in and ! 7: * are subject to the Apple Public Source License Version 1.1 (the ! 8: * "License"). You may not use this file except in compliance with the ! 9: * License. Please obtain a copy of the License at ! 10: * http://www.apple.com/publicsource and read it before using this file. ! 11: * ! 12: * This Original Code and all software distributed under the License are ! 13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER ! 14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, ! 15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, ! 16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the ! 17: * License for the specific language governing rights and limitations ! 18: * under the License. ! 19: * ! 20: * @APPLE_LICENSE_HEADER_END@ ! 21: */ ! 22: /* ! 23: * @OSF_COPYRIGHT@ ! 24: */ ! 25: /* ! 26: * Mach Operating System ! 27: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University ! 28: * All Rights Reserved. ! 29: * ! 30: * Permission to use, copy, modify and distribute this software and its ! 31: * documentation is hereby granted, provided that both the copyright ! 32: * notice and this permission notice appear in all copies of the ! 33: * software, derivative works or modified versions, and any portions ! 34: * thereof, and that both notices appear in supporting documentation. ! 35: * ! 36: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" ! 37: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR ! 38: * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. ! 39: * ! 40: * Carnegie Mellon requests users of this software to return to ! 41: * ! 42: * Software Distribution Coordinator or [email protected] ! 43: * School of Computer Science ! 44: * Carnegie Mellon University ! 45: * Pittsburgh PA 15213-3890 ! 46: * ! 47: * any improvements or extensions that they make and grant Carnegie Mellon ! 48: * the rights to redistribute these changes. ! 49: */ ! 50: /* ! 51: */ ! 52: /* ! 53: * File: vm/vm_kern.c ! 54: * Author: Avadis Tevanian, Jr., Michael Wayne Young ! 55: * Date: 1985 ! 56: * ! 57: * Kernel memory management. ! 58: */ ! 59: ! 60: #include <cpus.h> ! 61: #include <mach/kern_return.h> ! 62: #include <mach/vm_param.h> ! 63: #include <kern/assert.h> ! 64: #include <kern/lock.h> ! 65: #include <kern/thread.h> ! 66: #include <vm/vm_kern.h> ! 67: #include <vm/vm_map.h> ! 68: #include <vm/vm_object.h> ! 69: #include <vm/vm_page.h> ! 70: #include <vm/vm_pageout.h> ! 71: #include <kern/misc_protos.h> ! 72: #include <vm/cpm.h> ! 73: ! 74: #include <string.h> ! 75: /* ! 76: * Variables exported by this module. ! 77: */ ! 78: ! 79: vm_map_t kernel_map; ! 80: vm_map_t kernel_pageable_map; ! 81: ! 82: /* ! 83: * Forward declarations for internal functions. ! 84: */ ! 85: extern kern_return_t kmem_alloc_pages( ! 86: register vm_object_t object, ! 87: register vm_offset_t offset, ! 88: register vm_offset_t start, ! 89: register vm_offset_t end, ! 90: vm_prot_t protection); ! 91: ! 92: extern void kmem_remap_pages( ! 93: register vm_object_t object, ! 94: register vm_offset_t offset, ! 95: register vm_offset_t start, ! 96: register vm_offset_t end, ! 97: vm_prot_t protection); ! 98: ! 99: kern_return_t ! 100: kmem_alloc_contig( ! 101: vm_map_t map, ! 102: vm_offset_t *addrp, ! 103: vm_size_t size, ! 104: vm_offset_t mask, ! 105: int flags) ! 106: { ! 107: vm_object_t object; ! 108: vm_page_t m, pages; ! 109: kern_return_t kr; ! 110: vm_offset_t addr, i, offset; ! 111: vm_map_entry_t entry; ! 112: ! 113: if (map == VM_MAP_NULL || (flags && (flags ^ KMA_KOBJECT))) ! 114: return KERN_INVALID_ARGUMENT; ! 115: ! 116: if (size == 0) { ! 117: *addrp = 0; ! 118: return KERN_INVALID_ARGUMENT; ! 119: } ! 120: ! 121: size = round_page(size); ! 122: if ((flags & KMA_KOBJECT) == 0) { ! 123: object = vm_object_allocate(size); ! 124: kr = vm_map_find_space(map, &addr, size, mask, &entry); ! 125: } ! 126: else { ! 127: object = kernel_object; ! 128: kr = vm_map_find_space(map, &addr, size, mask, &entry); ! 129: } ! 130: ! 131: if ((flags & KMA_KOBJECT) == 0) { ! 132: entry->object.vm_object = object; ! 133: entry->offset = offset = 0; ! 134: } else { ! 135: offset = addr - VM_MIN_KERNEL_ADDRESS; ! 136: ! 137: if (entry->object.vm_object == VM_OBJECT_NULL) { ! 138: vm_object_reference(object); ! 139: entry->object.vm_object = object; ! 140: entry->offset = offset; ! 141: } ! 142: } ! 143: ! 144: if (kr != KERN_SUCCESS) { ! 145: if ((flags & KMA_KOBJECT) == 0) ! 146: vm_object_deallocate(object); ! 147: return kr; ! 148: } ! 149: ! 150: vm_map_unlock(map); ! 151: ! 152: kr = cpm_allocate(size, &pages, FALSE); ! 153: ! 154: if (kr != KERN_SUCCESS) { ! 155: vm_map_remove(map, addr, addr + size, 0); ! 156: *addrp = 0; ! 157: return kr; ! 158: } ! 159: ! 160: vm_object_lock(object); ! 161: for (i = 0; i < size; i += PAGE_SIZE) { ! 162: m = pages; ! 163: pages = NEXT_PAGE(m); ! 164: m->busy = FALSE; ! 165: vm_page_insert(m, object, offset + i); ! 166: } ! 167: vm_object_unlock(object); ! 168: ! 169: if ((kr = vm_map_wire(map, addr, addr + size, VM_PROT_DEFAULT, FALSE)) ! 170: != KERN_SUCCESS) { ! 171: if (object == kernel_object) { ! 172: vm_object_lock(object); ! 173: vm_object_page_remove(object, offset, offset + size); ! 174: vm_object_unlock(object); ! 175: } ! 176: vm_map_remove(map, addr, addr + size, 0); ! 177: return kr; ! 178: } ! 179: if (object == kernel_object) ! 180: vm_map_simplify(map, addr); ! 181: ! 182: *addrp = addr; ! 183: return KERN_SUCCESS; ! 184: } ! 185: ! 186: /* ! 187: * Master entry point for allocating kernel memory. ! 188: * NOTE: this routine is _never_ interrupt safe. ! 189: * ! 190: * map : map to allocate into ! 191: * addrp : pointer to start address of new memory ! 192: * size : size of memory requested ! 193: * flags : options ! 194: * KMA_HERE *addrp is base address, else "anywhere" ! 195: * KMA_NOPAGEWAIT don't wait for pages if unavailable ! 196: * KMA_KOBJECT use kernel_object ! 197: */ ! 198: ! 199: kern_return_t ! 200: kernel_memory_allocate( ! 201: register vm_map_t map, ! 202: register vm_offset_t *addrp, ! 203: register vm_size_t size, ! 204: register vm_offset_t mask, ! 205: int flags) ! 206: { ! 207: vm_object_t object = VM_OBJECT_NULL; ! 208: vm_map_entry_t entry; ! 209: vm_offset_t offset; ! 210: vm_offset_t addr; ! 211: vm_offset_t i; ! 212: kern_return_t kr; ! 213: ! 214: size = round_page(size); ! 215: if ((flags & KMA_KOBJECT) == 0) { ! 216: /* ! 217: * Allocate a new object. We must do this before locking ! 218: * the map, or risk deadlock with the default pager: ! 219: * device_read_alloc uses kmem_alloc, ! 220: * which tries to allocate an object, ! 221: * which uses kmem_alloc_wired to get memory, ! 222: * which blocks for pages. ! 223: * then the default pager needs to read a block ! 224: * to process a memory_object_data_write, ! 225: * and device_read_alloc calls kmem_alloc ! 226: * and deadlocks on the map lock. ! 227: */ ! 228: object = vm_object_allocate(size); ! 229: kr = vm_map_find_space(map, &addr, size, mask, &entry); ! 230: } ! 231: else { ! 232: object = kernel_object; ! 233: kr = vm_map_find_space(map, &addr, size, mask, &entry); ! 234: } ! 235: if (kr != KERN_SUCCESS) { ! 236: if ((flags & KMA_KOBJECT) == 0) ! 237: vm_object_deallocate(object); ! 238: return kr; ! 239: } ! 240: ! 241: if ((flags & KMA_KOBJECT) == 0) { ! 242: entry->object.vm_object = object; ! 243: entry->offset = offset = 0; ! 244: } else { ! 245: offset = addr - VM_MIN_KERNEL_ADDRESS; ! 246: ! 247: if (entry->object.vm_object == VM_OBJECT_NULL) { ! 248: vm_object_reference(object); ! 249: entry->object.vm_object = object; ! 250: entry->offset = offset; ! 251: } ! 252: } ! 253: ! 254: /* ! 255: * Since we have not given out this address yet, ! 256: * it is safe to unlock the map. ! 257: */ ! 258: vm_map_unlock(map); ! 259: ! 260: vm_object_lock(object); ! 261: for (i = 0; i < size; i += PAGE_SIZE) { ! 262: vm_page_t mem; ! 263: ! 264: while ((mem = vm_page_alloc(object, offset + i)) ! 265: == VM_PAGE_NULL) { ! 266: if (flags & KMA_NOPAGEWAIT) { ! 267: if (object == kernel_object) ! 268: vm_object_page_remove(object, offset, ! 269: offset + i); ! 270: vm_object_unlock(object); ! 271: vm_map_remove(map, addr, addr + size, 0); ! 272: return KERN_RESOURCE_SHORTAGE; ! 273: } ! 274: vm_object_unlock(object); ! 275: VM_PAGE_WAIT(); ! 276: vm_object_lock(object); ! 277: } ! 278: mem->busy = FALSE; ! 279: } ! 280: vm_object_unlock(object); ! 281: ! 282: if ((kr = vm_map_wire(map, addr, addr + size, VM_PROT_DEFAULT, FALSE)) ! 283: != KERN_SUCCESS) { ! 284: if (object == kernel_object) { ! 285: vm_object_lock(object); ! 286: vm_object_page_remove(object, offset, offset + size); ! 287: vm_object_unlock(object); ! 288: } ! 289: vm_map_remove(map, addr, addr + size, 0); ! 290: return (kr); ! 291: } ! 292: if (object == kernel_object) ! 293: vm_map_simplify(map, addr); ! 294: ! 295: /* ! 296: * Return the memory, not zeroed. ! 297: */ ! 298: #if (NCPUS > 1) && i860 ! 299: bzero( addr, size ); ! 300: #endif /* #if (NCPUS > 1) && i860 */ ! 301: *addrp = addr; ! 302: return KERN_SUCCESS; ! 303: } ! 304: ! 305: /* ! 306: * kmem_alloc: ! 307: * ! 308: * Allocate wired-down memory in the kernel's address map ! 309: * or a submap. The memory is not zero-filled. ! 310: */ ! 311: ! 312: kern_return_t ! 313: kmem_alloc( ! 314: vm_map_t map, ! 315: vm_offset_t *addrp, ! 316: vm_size_t size) ! 317: { ! 318: return kernel_memory_allocate(map, addrp, size, 0, 0); ! 319: } ! 320: ! 321: /* ! 322: * kmem_realloc: ! 323: * ! 324: * Reallocate wired-down memory in the kernel's address map ! 325: * or a submap. Newly allocated pages are not zeroed. ! 326: * This can only be used on regions allocated with kmem_alloc. ! 327: * ! 328: * If successful, the pages in the old region are mapped twice. ! 329: * The old region is unchanged. Use kmem_free to get rid of it. ! 330: */ ! 331: kern_return_t ! 332: kmem_realloc( ! 333: vm_map_t map, ! 334: vm_offset_t oldaddr, ! 335: vm_size_t oldsize, ! 336: vm_offset_t *newaddrp, ! 337: vm_size_t newsize) ! 338: { ! 339: vm_offset_t oldmin, oldmax; ! 340: vm_offset_t newaddr; ! 341: vm_object_t object; ! 342: vm_map_entry_t oldentry, newentry; ! 343: kern_return_t kr; ! 344: ! 345: oldmin = trunc_page(oldaddr); ! 346: oldmax = round_page(oldaddr + oldsize); ! 347: oldsize = oldmax - oldmin; ! 348: newsize = round_page(newsize); ! 349: ! 350: /* ! 351: * Find space for the new region. ! 352: */ ! 353: ! 354: kr = vm_map_find_space(map, &newaddr, newsize, (vm_offset_t) 0, ! 355: &newentry); ! 356: if (kr != KERN_SUCCESS) { ! 357: return kr; ! 358: } ! 359: ! 360: /* ! 361: * Find the VM object backing the old region. ! 362: */ ! 363: ! 364: if (!vm_map_lookup_entry(map, oldmin, &oldentry)) ! 365: panic("kmem_realloc"); ! 366: object = oldentry->object.vm_object; ! 367: ! 368: /* ! 369: * Increase the size of the object and ! 370: * fill in the new region. ! 371: */ ! 372: ! 373: vm_object_reference(object); ! 374: vm_object_lock(object); ! 375: if (object->size != oldsize) ! 376: panic("kmem_realloc"); ! 377: object->size = newsize; ! 378: vm_object_unlock(object); ! 379: ! 380: newentry->object.vm_object = object; ! 381: newentry->offset = 0; ! 382: assert (newentry->wired_count == 0); ! 383: newentry->wired_count = 1; ! 384: ! 385: /* ! 386: * Since we have not given out this address yet, ! 387: * it is safe to unlock the map. We are trusting ! 388: * that nobody will play with either region. ! 389: */ ! 390: ! 391: vm_map_unlock(map); ! 392: ! 393: /* ! 394: * Remap the pages in the old region and ! 395: * allocate more pages for the new region. ! 396: */ ! 397: ! 398: kmem_remap_pages(object, 0, ! 399: newaddr, newaddr + oldsize, ! 400: VM_PROT_DEFAULT); ! 401: kmem_alloc_pages(object, oldsize, ! 402: newaddr + oldsize, newaddr + newsize, ! 403: VM_PROT_DEFAULT); ! 404: ! 405: *newaddrp = newaddr; ! 406: return KERN_SUCCESS; ! 407: } ! 408: ! 409: /* ! 410: * kmem_alloc_wired: ! 411: * ! 412: * Allocate wired-down memory in the kernel's address map ! 413: * or a submap. The memory is not zero-filled. ! 414: * ! 415: * The memory is allocated in the kernel_object. ! 416: * It may not be copied with vm_map_copy, and ! 417: * it may not be reallocated with kmem_realloc. ! 418: */ ! 419: ! 420: kern_return_t ! 421: kmem_alloc_wired( ! 422: vm_map_t map, ! 423: vm_offset_t *addrp, ! 424: vm_size_t size) ! 425: { ! 426: return kernel_memory_allocate(map, addrp, size, 0, KMA_KOBJECT); ! 427: } ! 428: ! 429: /* ! 430: * kmem_alloc_aligned: ! 431: * ! 432: * Like kmem_alloc_wired, except that the memory is aligned. ! 433: * The size should be a power-of-2. ! 434: */ ! 435: ! 436: kern_return_t ! 437: kmem_alloc_aligned( ! 438: vm_map_t map, ! 439: vm_offset_t *addrp, ! 440: vm_size_t size) ! 441: { ! 442: if ((size & (size - 1)) != 0) ! 443: panic("kmem_alloc_aligned: size not aligned"); ! 444: return kernel_memory_allocate(map, addrp, size, size - 1, KMA_KOBJECT); ! 445: } ! 446: ! 447: /* ! 448: * kmem_alloc_pageable: ! 449: * ! 450: * Allocate pageable memory in the kernel's address map. ! 451: */ ! 452: ! 453: kern_return_t ! 454: kmem_alloc_pageable( ! 455: vm_map_t map, ! 456: vm_offset_t *addrp, ! 457: vm_size_t size) ! 458: { ! 459: vm_offset_t addr; ! 460: kern_return_t kr; ! 461: ! 462: addr = vm_map_min(map); ! 463: kr = vm_map_enter(map, &addr, round_page(size), ! 464: (vm_offset_t) 0, TRUE, ! 465: VM_OBJECT_NULL, (vm_offset_t) 0, FALSE, ! 466: VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); ! 467: if (kr != KERN_SUCCESS) ! 468: return kr; ! 469: ! 470: *addrp = addr; ! 471: return KERN_SUCCESS; ! 472: } ! 473: ! 474: /* ! 475: * kmem_free: ! 476: * ! 477: * Release a region of kernel virtual memory allocated ! 478: * with kmem_alloc, kmem_alloc_wired, or kmem_alloc_pageable, ! 479: * and return the physical pages associated with that region. ! 480: */ ! 481: ! 482: void ! 483: kmem_free( ! 484: vm_map_t map, ! 485: vm_offset_t addr, ! 486: vm_size_t size) ! 487: { ! 488: kern_return_t kr; ! 489: ! 490: kr = vm_map_remove(map, trunc_page(addr), ! 491: round_page(addr + size), VM_MAP_REMOVE_KUNWIRE); ! 492: if (kr != KERN_SUCCESS) ! 493: panic("kmem_free"); ! 494: } ! 495: ! 496: /* ! 497: * Allocate new wired pages in an object. ! 498: * The object is assumed to be mapped into the kernel map or ! 499: * a submap. ! 500: */ ! 501: ! 502: kern_return_t ! 503: kmem_alloc_pages( ! 504: register vm_object_t object, ! 505: register vm_offset_t offset, ! 506: register vm_offset_t start, ! 507: register vm_offset_t end, ! 508: vm_prot_t protection) ! 509: { ! 510: /* ! 511: * Mark the pmap region as not pageable. ! 512: */ ! 513: pmap_pageable(kernel_pmap, start, end, FALSE); ! 514: ! 515: while (start < end) { ! 516: register vm_page_t mem; ! 517: ! 518: vm_object_lock(object); ! 519: ! 520: /* ! 521: * Allocate a page ! 522: */ ! 523: while ((mem = vm_page_alloc(object, offset)) ! 524: == VM_PAGE_NULL) { ! 525: vm_object_unlock(object); ! 526: VM_PAGE_WAIT(); ! 527: vm_object_lock(object); ! 528: } ! 529: ! 530: /* ! 531: * Wire it down ! 532: */ ! 533: vm_page_lock_queues(); ! 534: vm_page_wire(mem); ! 535: vm_page_unlock_queues(); ! 536: vm_object_unlock(object); ! 537: ! 538: /* ! 539: * Enter it in the kernel pmap ! 540: */ ! 541: PMAP_ENTER(kernel_pmap, start, mem, ! 542: protection, TRUE); ! 543: ! 544: vm_object_lock(object); ! 545: PAGE_WAKEUP_DONE(mem); ! 546: vm_object_unlock(object); ! 547: ! 548: start += PAGE_SIZE; ! 549: offset += PAGE_SIZE; ! 550: } ! 551: return KERN_SUCCESS; ! 552: } ! 553: ! 554: /* ! 555: * Remap wired pages in an object into a new region. ! 556: * The object is assumed to be mapped into the kernel map or ! 557: * a submap. ! 558: */ ! 559: void ! 560: kmem_remap_pages( ! 561: register vm_object_t object, ! 562: register vm_offset_t offset, ! 563: register vm_offset_t start, ! 564: register vm_offset_t end, ! 565: vm_prot_t protection) ! 566: { ! 567: /* ! 568: * Mark the pmap region as not pageable. ! 569: */ ! 570: pmap_pageable(kernel_pmap, start, end, FALSE); ! 571: ! 572: while (start < end) { ! 573: register vm_page_t mem; ! 574: ! 575: vm_object_lock(object); ! 576: ! 577: /* ! 578: * Find a page ! 579: */ ! 580: if ((mem = vm_page_lookup(object, offset)) == VM_PAGE_NULL) ! 581: panic("kmem_remap_pages"); ! 582: ! 583: /* ! 584: * Wire it down (again) ! 585: */ ! 586: vm_page_lock_queues(); ! 587: vm_page_wire(mem); ! 588: vm_page_unlock_queues(); ! 589: vm_object_unlock(object); ! 590: ! 591: /* ! 592: * Enter it in the kernel pmap. The page isn't busy, ! 593: * but this shouldn't be a problem because it is wired. ! 594: */ ! 595: PMAP_ENTER(kernel_pmap, start, mem, ! 596: protection, TRUE); ! 597: ! 598: start += PAGE_SIZE; ! 599: offset += PAGE_SIZE; ! 600: } ! 601: } ! 602: ! 603: /* ! 604: * kmem_suballoc: ! 605: * ! 606: * Allocates a map to manage a subrange ! 607: * of the kernel virtual address space. ! 608: * ! 609: * Arguments are as follows: ! 610: * ! 611: * parent Map to take range from ! 612: * addr Address of start of range (IN/OUT) ! 613: * size Size of range to find ! 614: * pageable Can region be paged ! 615: * anywhere Can region be located anywhere in map ! 616: * new_map Pointer to new submap ! 617: */ ! 618: kern_return_t ! 619: kmem_suballoc( ! 620: vm_map_t parent, ! 621: vm_offset_t *addr, ! 622: vm_size_t size, ! 623: boolean_t pageable, ! 624: boolean_t anywhere, ! 625: vm_map_t *new_map) ! 626: { ! 627: vm_map_t map; ! 628: kern_return_t kr; ! 629: ! 630: size = round_page(size); ! 631: ! 632: /* ! 633: * Need reference on submap object because it is internal ! 634: * to the vm_system. vm_object_enter will never be called ! 635: * on it (usual source of reference for vm_map_enter). ! 636: */ ! 637: vm_object_reference(vm_submap_object); ! 638: ! 639: if (anywhere == TRUE) ! 640: *addr = (vm_offset_t)vm_map_min(parent); ! 641: kr = vm_map_enter(parent, addr, size, ! 642: (vm_offset_t) 0, anywhere, ! 643: vm_submap_object, (vm_offset_t) 0, FALSE, ! 644: VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); ! 645: if (kr != KERN_SUCCESS) { ! 646: vm_object_deallocate(vm_submap_object); ! 647: return (kr); ! 648: } ! 649: ! 650: pmap_reference(vm_map_pmap(parent)); ! 651: map = vm_map_create(vm_map_pmap(parent), *addr, *addr + size, pageable); ! 652: if (map == VM_MAP_NULL) ! 653: panic("kmem_suballoc: vm_map_create failed"); /* "can't happen" */ ! 654: ! 655: kr = vm_map_submap(parent, *addr, *addr + size, map, *addr); ! 656: if (kr != KERN_SUCCESS) { ! 657: /* ! 658: * See comment preceding vm_map_submap(). ! 659: */ ! 660: vm_map_remove(parent, *addr, *addr + size, VM_MAP_NO_FLAGS); ! 661: vm_map_deallocate(map); /* also removes ref to pmap */ ! 662: vm_object_deallocate(vm_submap_object); ! 663: return (kr); ! 664: } ! 665: ! 666: *new_map = map; ! 667: return (KERN_SUCCESS); ! 668: } ! 669: ! 670: /* ! 671: * kmem_init: ! 672: * ! 673: * Initialize the kernel's virtual memory map, taking ! 674: * into account all memory allocated up to this time. ! 675: */ ! 676: void ! 677: kmem_init( ! 678: vm_offset_t start, ! 679: vm_offset_t end) ! 680: { ! 681: kernel_map = vm_map_create(pmap_kernel(), ! 682: VM_MIN_KERNEL_ADDRESS, end, ! 683: FALSE); ! 684: ! 685: /* ! 686: * Reserve virtual memory allocated up to this time. ! 687: */ ! 688: ! 689: if (start != VM_MIN_KERNEL_ADDRESS) { ! 690: vm_offset_t addr = VM_MIN_KERNEL_ADDRESS; ! 691: (void) vm_map_enter(kernel_map, ! 692: &addr, start - VM_MIN_KERNEL_ADDRESS, ! 693: (vm_offset_t) 0, TRUE, ! 694: VM_OBJECT_NULL, (vm_offset_t) 0, FALSE, ! 695: VM_PROT_DEFAULT, VM_PROT_ALL, ! 696: VM_INHERIT_DEFAULT); ! 697: } ! 698: ! 699: /* ! 700: * Account for kernel memory (text, data, bss, vm shenanigans). ! 701: * This may include inaccessible "holes" as determined by what ! 702: * the machine-dependent init code includes in mem_size. ! 703: */ ! 704: vm_page_wire_count = (atop(mem_size) - (vm_page_free_count ! 705: + vm_page_active_count ! 706: + vm_page_inactive_count)); ! 707: } ! 708: ! 709: /* ! 710: * kmem_io_map_copyout: ! 711: * ! 712: * Establish temporary mapping in designated map for the memory ! 713: * passed in. Memory format must be a page_list vm_map_copy. ! 714: */ ! 715: ! 716: kern_return_t ! 717: kmem_io_map_copyout( ! 718: vm_map_t map, ! 719: vm_offset_t *addr, /* actual addr of data */ ! 720: vm_size_t *alloc_size, /* size allocated */ ! 721: vm_map_copy_t copy, ! 722: vm_size_t min_size, /* Do at least this much */ ! 723: vm_prot_t prot) /* Protection of mapping */ ! 724: { ! 725: vm_offset_t myaddr, offset; ! 726: vm_size_t mysize, copy_size; ! 727: kern_return_t ret; ! 728: register ! 729: vm_page_t *page_list; ! 730: vm_map_copy_t new_copy; ! 731: register ! 732: int i; ! 733: ! 734: assert(copy->type == VM_MAP_COPY_PAGE_LIST); ! 735: assert(min_size != 0); ! 736: ! 737: /* ! 738: * Figure out the size in vm pages. ! 739: */ ! 740: min_size += copy->offset - trunc_page(copy->offset); ! 741: min_size = round_page(min_size); ! 742: mysize = round_page(copy->offset + copy->size) - ! 743: trunc_page(copy->offset); ! 744: ! 745: /* ! 746: * If total size is larger than one page list and ! 747: * we don't have to do more than one page list, then ! 748: * only do one page list. ! 749: * ! 750: * XXX Could be much smarter about this ... like trimming length ! 751: * XXX if we need more than one page list but not all of them. ! 752: */ ! 753: ! 754: copy_size = ptoa(copy->cpy_npages); ! 755: if (mysize > copy_size && copy_size > min_size) ! 756: mysize = copy_size; ! 757: ! 758: /* ! 759: * Allocate some address space in the map (must be kernel ! 760: * space). ! 761: */ ! 762: myaddr = vm_map_min(map); ! 763: ret = vm_map_enter(map, &myaddr, mysize, ! 764: (vm_offset_t) 0, TRUE, ! 765: VM_OBJECT_NULL, (vm_offset_t) 0, FALSE, ! 766: prot, prot, VM_INHERIT_DEFAULT); ! 767: ! 768: if (ret != KERN_SUCCESS) ! 769: return(ret); ! 770: ! 771: /* ! 772: * Tell the pmap module that this will be wired, and ! 773: * enter the mappings. ! 774: */ ! 775: pmap_pageable(vm_map_pmap(map), myaddr, myaddr + mysize, TRUE); ! 776: ! 777: *addr = myaddr + (copy->offset - trunc_page(copy->offset)); ! 778: *alloc_size = mysize; ! 779: ! 780: offset = myaddr; ! 781: page_list = ©->cpy_page_list[0]; ! 782: while (TRUE) { ! 783: for ( i = 0; i < copy->cpy_npages; i++, offset += PAGE_SIZE) { ! 784: PMAP_ENTER(vm_map_pmap(map), offset, *page_list, ! 785: prot, TRUE); ! 786: page_list++; ! 787: } ! 788: ! 789: if (offset == (myaddr + mysize)) ! 790: break; ! 791: ! 792: /* ! 793: * Onward to the next page_list. The extend_cont ! 794: * leaves the current page list's pages alone; ! 795: * they'll be cleaned up at discard. Reset this ! 796: * copy's continuation to discard the next one. ! 797: */ ! 798: vm_map_copy_invoke_extend_cont(copy, &new_copy, &ret); ! 799: ! 800: if (ret != KERN_SUCCESS) { ! 801: kmem_io_map_deallocate(map, myaddr, mysize); ! 802: return(ret); ! 803: } ! 804: copy->cpy_cont = vm_map_copy_discard_cont; ! 805: copy->cpy_cont_args = (vm_map_copyin_args_t) new_copy; ! 806: assert(new_copy != VM_MAP_COPY_NULL); ! 807: assert(new_copy->type == VM_MAP_COPY_PAGE_LIST); ! 808: copy = new_copy; ! 809: page_list = ©->cpy_page_list[0]; ! 810: } ! 811: ! 812: return(ret); ! 813: } ! 814: ! 815: /* ! 816: * kmem_io_map_deallocate: ! 817: * ! 818: * Get rid of the mapping established by kmem_io_map_copyout. ! 819: * Assumes that addr and size have been rounded to page boundaries. ! 820: */ ! 821: ! 822: void ! 823: kmem_io_map_deallocate( ! 824: vm_map_t map, ! 825: vm_offset_t addr, ! 826: vm_size_t size) ! 827: { ! 828: ! 829: register vm_offset_t va, end; ! 830: ! 831: end = round_page(addr + size); ! 832: for (va = trunc_page(addr); va < end; va += PAGE_SIZE) ! 833: pmap_change_wiring(vm_map_pmap(map), va, FALSE); ! 834: ! 835: /* ! 836: * Remove the mappings. The pmap_remove is needed. ! 837: */ ! 838: ! 839: pmap_remove(vm_map_pmap(map), addr, addr + size); ! 840: vm_map_remove(map, addr, addr + size, VM_MAP_REMOVE_KUNWIRE); ! 841: } ! 842: ! 843: ! 844: /* ! 845: * kmem_io_object_trunc: ! 846: * ! 847: * Truncate an object vm_map_copy_t. ! 848: * Called by the scatter/gather list network code to remove pages from ! 849: * the tail end of a packet. Also unwires the objects pages. ! 850: */ ! 851: ! 852: kern_return_t ! 853: kmem_io_object_trunc(copy, new_size) ! 854: vm_map_copy_t copy; /* IN/OUT copy object */ ! 855: register vm_size_t new_size; /* IN new object size */ ! 856: { ! 857: register vm_size_t offset, old_size; ! 858: ! 859: assert(copy->type == VM_MAP_COPY_OBJECT); ! 860: ! 861: old_size = round_page(copy->size); ! 862: copy->size = new_size; ! 863: new_size = round_page(new_size); ! 864: ! 865: vm_object_lock(copy->cpy_object); ! 866: vm_object_page_remove(copy->cpy_object, ! 867: (vm_offset_t)new_size, (vm_offset_t)old_size); ! 868: for (offset = 0; offset < new_size; offset += PAGE_SIZE) { ! 869: register vm_page_t mem; ! 870: ! 871: if ((mem = vm_page_lookup(copy->cpy_object, offset)) == VM_PAGE_NULL) ! 872: panic("kmem_io_object_trunc: unable to find object page"); ! 873: ! 874: /* ! 875: * Make sure these pages are marked dirty ! 876: */ ! 877: mem->dirty = TRUE; ! 878: vm_page_lock_queues(); ! 879: vm_page_unwire(mem); ! 880: vm_page_unlock_queues(); ! 881: } ! 882: copy->cpy_object->size = new_size; /* adjust size of object */ ! 883: vm_object_unlock(copy->cpy_object); ! 884: return(KERN_SUCCESS); ! 885: } ! 886: ! 887: /* ! 888: * kmem_io_object_deallocate: ! 889: * ! 890: * Free an vm_map_copy_t. ! 891: * Called by the scatter/gather list network code to free a packet. ! 892: */ ! 893: ! 894: void ! 895: kmem_io_object_deallocate( ! 896: vm_map_copy_t copy) /* IN/OUT copy object */ ! 897: { ! 898: kern_return_t ret; ! 899: ! 900: /* ! 901: * Clear out all the object pages (this will leave an empty object). ! 902: */ ! 903: ret = kmem_io_object_trunc(copy, 0); ! 904: if (ret != KERN_SUCCESS) ! 905: panic("kmem_io_object_deallocate: unable to truncate object"); ! 906: /* ! 907: * ...and discard the copy object. ! 908: */ ! 909: vm_map_copy_discard(copy); ! 910: } ! 911: ! 912: /* ! 913: * Routine: copyinmap ! 914: * Purpose: ! 915: * Like copyin, except that fromaddr is an address ! 916: * in the specified VM map. This implementation ! 917: * is incomplete; it handles the current user map ! 918: * and the kernel map/submaps. ! 919: */ ! 920: boolean_t ! 921: copyinmap( ! 922: vm_map_t map, ! 923: vm_offset_t fromaddr, ! 924: vm_offset_t toaddr, ! 925: vm_size_t length) ! 926: { ! 927: if (vm_map_pmap(map) == pmap_kernel()) { ! 928: /* assume a correct copy */ ! 929: memcpy((void *)toaddr, (void *)fromaddr, length); ! 930: return FALSE; ! 931: } ! 932: ! 933: if (current_map() == map) ! 934: return copyin((char *)fromaddr, (char *)toaddr, length); ! 935: ! 936: return TRUE; ! 937: } ! 938: ! 939: /* ! 940: * Routine: copyoutmap ! 941: * Purpose: ! 942: * Like copyout, except that toaddr is an address ! 943: * in the specified VM map. This implementation ! 944: * is incomplete; it handles the current user map ! 945: * and the kernel map/submaps. ! 946: */ ! 947: boolean_t ! 948: copyoutmap( ! 949: vm_map_t map, ! 950: vm_offset_t fromaddr, ! 951: vm_offset_t toaddr, ! 952: vm_size_t length) ! 953: { ! 954: if (vm_map_pmap(map) == pmap_kernel()) { ! 955: /* assume a correct copy */ ! 956: memcpy((void *)toaddr, (void *)fromaddr, length); ! 957: return FALSE; ! 958: } ! 959: ! 960: if (current_map() == map) ! 961: return copyout((char *)fromaddr, (char *)toaddr, length); ! 962: ! 963: return TRUE; ! 964: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.