![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vm_map.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [GnuMach] / Gnu-Mach / vm|
Return to vm_map.c CVS log | Up to [GnuMach] / Gnu-Mach / vm |
1.1 root 1: /*
2: * Mach Operating System
3: * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4: * Copyright (c) 1993,1994 The University of Utah and
5: * the Computer Systems Laboratory (CSL).
6: * All rights reserved.
7: *
8: * Permission to use, copy, modify and distribute this software and its
9: * documentation is hereby granted, provided that both the copyright
10: * notice and this permission notice appear in all copies of the
11: * software, derivative works or modified versions, and any portions
12: * thereof, and that both notices appear in supporting documentation.
13: *
14: * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15: * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16: * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17: * THIS SOFTWARE.
18: *
19: * Carnegie Mellon requests users of this software to return to
20: *
21: * Software Distribution Coordinator or [email protected]
22: * School of Computer Science
23: * Carnegie Mellon University
24: * Pittsburgh PA 15213-3890
25: *
26: * any improvements or extensions that they make and grant Carnegie Mellon
27: * the rights to redistribute these changes.
28: */
29: /*
30: * File: vm/vm_map.c
31: * Author: Avadis Tevanian, Jr., Michael Wayne Young
32: * Date: 1985
33: *
34: * Virtual memory mapping module.
35: */
36:
1.1.1.3 ! root 37: #include <kern/printf.h>
1.1 root 38: #include <mach/kern_return.h>
39: #include <mach/port.h>
40: #include <mach/vm_attributes.h>
41: #include <mach/vm_param.h>
42: #include <kern/assert.h>
1.1.1.3 ! root 43: #include <kern/debug.h>
! 44: #include <kern/kalloc.h>
! 45: #include <kern/rbtree.h>
! 46: #include <kern/slab.h>
! 47: #include <vm/pmap.h>
1.1 root 48: #include <vm/vm_fault.h>
49: #include <vm/vm_map.h>
50: #include <vm/vm_object.h>
51: #include <vm/vm_page.h>
1.1.1.3 ! root 52: #include <vm/vm_resident.h>
1.1 root 53: #include <vm/vm_kern.h>
54: #include <ipc/ipc_port.h>
55:
1.1.1.3 ! root 56: #if MACH_KDB
! 57: #include <ddb/db_output.h>
! 58: #include <vm/vm_print.h>
! 59: #endif /* MACH_KDB */
! 60:
! 61:
! 62: /* Forward declarations */
! 63: kern_return_t vm_map_delete(
! 64: vm_map_t map,
! 65: vm_offset_t start,
! 66: vm_offset_t end);
! 67:
! 68: kern_return_t vm_map_copyout_page_list(
! 69: vm_map_t dst_map,
! 70: vm_offset_t *dst_addr, /* OUT */
! 71: vm_map_copy_t copy);
! 72:
! 73: void vm_map_copy_page_discard (vm_map_copy_t copy);
! 74:
1.1 root 75: /*
76: * Macros to copy a vm_map_entry. We must be careful to correctly
77: * manage the wired page count. vm_map_entry_copy() creates a new
78: * map entry to the same memory - the wired count in the new entry
79: * must be set to zero. vm_map_entry_copy_full() creates a new
80: * entry that is identical to the old entry. This preserves the
1.1.1.3 ! root 81: * wire count; it's used for map splitting and cache changing in
1.1 root 82: * vm_map_copyout.
83: */
84: #define vm_map_entry_copy(NEW,OLD) \
85: MACRO_BEGIN \
86: *(NEW) = *(OLD); \
87: (NEW)->is_shared = FALSE; \
88: (NEW)->needs_wakeup = FALSE; \
89: (NEW)->in_transition = FALSE; \
90: (NEW)->wired_count = 0; \
91: (NEW)->user_wired_count = 0; \
92: MACRO_END
93:
94: #define vm_map_entry_copy_full(NEW,OLD) (*(NEW) = *(OLD))
95:
96: /*
97: * Virtual memory maps provide for the mapping, protection,
98: * and sharing of virtual memory objects. In addition,
99: * this module provides for an efficient virtual copy of
100: * memory from one map to another.
101: *
102: * Synchronization is required prior to most operations.
103: *
104: * Maps consist of an ordered doubly-linked list of simple
1.1.1.3 ! root 105: * entries; a hint and a red-black tree are used to speed up lookups.
1.1 root 106: *
107: * Sharing maps have been deleted from this version of Mach.
108: * All shared objects are now mapped directly into the respective
109: * maps. This requires a change in the copy on write strategy;
110: * the asymmetric (delayed) strategy is used for shared temporary
111: * objects instead of the symmetric (shadow) strategy. This is
112: * selected by the (new) use_shared_copy bit in the object. See
113: * vm_object_copy_temporary in vm_object.c for details. All maps
114: * are now "top level" maps (either task map, kernel map or submap
1.1.1.2 root 115: * of the kernel map).
1.1 root 116: *
117: * Since portions of maps are specified by start/end addreses,
118: * which may not align with existing map entries, all
119: * routines merely "clip" entries to these start/end values.
120: * [That is, an entry is split into two, bordering at a
121: * start or end value.] Note that these clippings may not
122: * always be necessary (as the two resulting entries are then
123: * not changed); however, the clipping is done for convenience.
124: * No attempt is currently made to "glue back together" two
125: * abutting entries.
126: *
127: * The symmetric (shadow) copy strategy implements virtual copy
128: * by copying VM object references from one map to
129: * another, and then marking both regions as copy-on-write.
130: * It is important to note that only one writeable reference
131: * to a VM object region exists in any map when this strategy
132: * is used -- this means that shadow object creation can be
133: * delayed until a write operation occurs. The asymmetric (delayed)
134: * strategy allows multiple maps to have writeable references to
135: * the same region of a vm object, and hence cannot delay creating
136: * its copy objects. See vm_object_copy_temporary() in vm_object.c.
137: * Copying of permanent objects is completely different; see
138: * vm_object_copy_strategically() in vm_object.c.
139: */
140:
1.1.1.3 ! root 141: struct kmem_cache vm_map_cache; /* cache for vm_map structures */
! 142: struct kmem_cache vm_map_entry_cache; /* cache for vm_map_entry structures */
! 143: struct kmem_cache vm_map_kentry_cache; /* cache for kernel entry structures */
! 144: struct kmem_cache vm_map_copy_cache; /* cache for vm_map_copy structures */
1.1 root 145:
146: boolean_t vm_map_lookup_entry(); /* forward declaration */
147:
148: /*
149: * Placeholder object for submap operations. This object is dropped
150: * into the range by a call to vm_map_find, and removed when
151: * vm_map_submap creates the submap.
152: */
153:
1.1.1.3 ! root 154: static struct vm_object vm_submap_object_store;
! 155: vm_object_t vm_submap_object = &vm_submap_object_store;
1.1 root 156:
157: /*
158: * vm_map_init:
159: *
160: * Initialize the vm_map module. Must be called before
161: * any other vm_map routines.
162: *
1.1.1.3 ! root 163: * Map and entry structures are allocated from caches -- we must
! 164: * initialize those caches.
1.1 root 165: *
1.1.1.3 ! root 166: * There are three caches of interest:
1.1 root 167: *
1.1.1.3 ! root 168: * vm_map_cache: used to allocate maps.
! 169: * vm_map_entry_cache: used to allocate map entries.
! 170: * vm_map_kentry_cache: used to allocate map entries for the kernel.
! 171: *
! 172: * Kernel map entries are allocated from a special cache, using a custom
! 173: * page allocation function to avoid recursion. It would be difficult
! 174: * (perhaps impossible) for the kernel to allocate more memory to an entry
! 175: * cache when it became empty since the very act of allocating memory
! 176: * implies the creation of a new entry.
1.1 root 177: */
178:
179: vm_offset_t kentry_data;
1.1.1.3 ! root 180: vm_size_t kentry_data_size = KENTRY_DATA_SIZE;
1.1 root 181:
1.1.1.3 ! root 182: static vm_offset_t kentry_pagealloc(vm_size_t size)
1.1 root 183: {
1.1.1.3 ! root 184: vm_offset_t result;
1.1 root 185:
1.1.1.3 ! root 186: if (size > kentry_data_size)
! 187: panic("vm_map: kentry memory exhausted");
1.1 root 188:
1.1.1.3 ! root 189: result = kentry_data;
! 190: kentry_data += size;
! 191: kentry_data_size -= size;
! 192: return result;
! 193: }
! 194:
! 195: void vm_map_init(void)
! 196: {
! 197: kmem_cache_init(&vm_map_cache, "vm_map", sizeof(struct vm_map), 0,
! 198: NULL, NULL, NULL, 0);
! 199: kmem_cache_init(&vm_map_entry_cache, "vm_map_entry",
! 200: sizeof(struct vm_map_entry), 0, NULL, NULL, NULL, 0);
! 201: kmem_cache_init(&vm_map_kentry_cache, "vm_map_kentry",
! 202: sizeof(struct vm_map_entry), 0, NULL, kentry_pagealloc,
! 203: NULL, KMEM_CACHE_NOCPUPOOL | KMEM_CACHE_NOOFFSLAB
! 204: | KMEM_CACHE_NORECLAIM);
! 205: kmem_cache_init(&vm_map_copy_cache, "vm_map_copy",
! 206: sizeof(struct vm_map_copy), 0, NULL, NULL, NULL, 0);
1.1 root 207:
208: /*
209: * Submap object is initialized by vm_object_init.
210: */
211: }
212:
1.1.1.3 ! root 213: void vm_map_setup(map, pmap, min, max, pageable)
! 214: vm_map_t map;
! 215: pmap_t pmap;
! 216: vm_offset_t min, max;
! 217: boolean_t pageable;
! 218: {
! 219: vm_map_first_entry(map) = vm_map_to_entry(map);
! 220: vm_map_last_entry(map) = vm_map_to_entry(map);
! 221: map->hdr.nentries = 0;
! 222: map->hdr.entries_pageable = pageable;
! 223: rbtree_init(&map->hdr.tree);
! 224:
! 225: map->size = 0;
! 226: map->ref_count = 1;
! 227: map->pmap = pmap;
! 228: map->min_offset = min;
! 229: map->max_offset = max;
! 230: map->wiring_required = FALSE;
! 231: map->wait_for_space = FALSE;
! 232: map->first_free = vm_map_to_entry(map);
! 233: map->hint = vm_map_to_entry(map);
! 234: vm_map_lock_init(map);
! 235: simple_lock_init(&map->ref_lock);
! 236: simple_lock_init(&map->hint_lock);
! 237: }
! 238:
1.1 root 239: /*
240: * vm_map_create:
241: *
242: * Creates and returns a new empty VM map with
243: * the given physical map structure, and having
244: * the given lower and upper address bounds.
245: */
246: vm_map_t vm_map_create(pmap, min, max, pageable)
247: pmap_t pmap;
248: vm_offset_t min, max;
249: boolean_t pageable;
250: {
251: register vm_map_t result;
252:
1.1.1.3 ! root 253: result = (vm_map_t) kmem_cache_alloc(&vm_map_cache);
1.1 root 254: if (result == VM_MAP_NULL)
255: panic("vm_map_create");
256:
1.1.1.3 ! root 257: vm_map_setup(result, pmap, min, max, pageable);
1.1 root 258:
259: return(result);
260: }
261:
262: /*
263: * vm_map_entry_create: [ internal use only ]
264: *
265: * Allocates a VM map entry for insertion in the
266: * given map (or map copy). No fields are filled.
267: */
268: #define vm_map_entry_create(map) \
269: _vm_map_entry_create(&(map)->hdr)
270:
271: #define vm_map_copy_entry_create(copy) \
272: _vm_map_entry_create(&(copy)->cpy_hdr)
273:
274: vm_map_entry_t _vm_map_entry_create(map_header)
275: register struct vm_map_header *map_header;
276: {
1.1.1.3 ! root 277: register kmem_cache_t cache;
1.1 root 278: register vm_map_entry_t entry;
279:
280: if (map_header->entries_pageable)
1.1.1.3 ! root 281: cache = &vm_map_entry_cache;
1.1 root 282: else
1.1.1.3 ! root 283: cache = &vm_map_kentry_cache;
1.1 root 284:
1.1.1.3 ! root 285: entry = (vm_map_entry_t) kmem_cache_alloc(cache);
1.1 root 286: if (entry == VM_MAP_ENTRY_NULL)
287: panic("vm_map_entry_create");
288:
289: return(entry);
290: }
291:
292: /*
293: * vm_map_entry_dispose: [ internal use only ]
294: *
295: * Inverse of vm_map_entry_create.
296: */
297: #define vm_map_entry_dispose(map, entry) \
298: _vm_map_entry_dispose(&(map)->hdr, (entry))
299:
300: #define vm_map_copy_entry_dispose(map, entry) \
301: _vm_map_entry_dispose(&(copy)->cpy_hdr, (entry))
302:
303: void _vm_map_entry_dispose(map_header, entry)
304: register struct vm_map_header *map_header;
305: register vm_map_entry_t entry;
306: {
1.1.1.3 ! root 307: register kmem_cache_t cache;
1.1 root 308:
309: if (map_header->entries_pageable)
1.1.1.3 ! root 310: cache = &vm_map_entry_cache;
! 311: else
! 312: cache = &vm_map_kentry_cache;
! 313:
! 314: kmem_cache_free(cache, (vm_offset_t) entry);
! 315: }
! 316:
! 317: /*
! 318: * Red-black tree lookup/insert comparison functions
! 319: */
! 320: static inline int vm_map_entry_cmp_lookup(vm_offset_t addr,
! 321: const struct rbtree_node *node)
! 322: {
! 323: struct vm_map_entry *entry;
! 324:
! 325: entry = rbtree_entry(node, struct vm_map_entry, tree_node);
! 326:
! 327: if (addr < entry->vme_start)
! 328: return -1;
! 329: else if (addr < entry->vme_end)
! 330: return 0;
1.1 root 331: else
1.1.1.3 ! root 332: return 1;
! 333: }
! 334:
! 335: static inline int vm_map_entry_cmp_insert(const struct rbtree_node *a,
! 336: const struct rbtree_node *b)
! 337: {
! 338: struct vm_map_entry *entry;
1.1 root 339:
1.1.1.3 ! root 340: entry = rbtree_entry(a, struct vm_map_entry, tree_node);
! 341: return vm_map_entry_cmp_lookup(entry->vme_start, b);
1.1 root 342: }
343:
344: /*
345: * vm_map_entry_{un,}link:
346: *
347: * Insert/remove entries from maps (or map copies).
1.1.1.3 ! root 348: *
! 349: * The start and end addresses of the entries must be properly set
! 350: * before using these macros.
1.1 root 351: */
352: #define vm_map_entry_link(map, after_where, entry) \
353: _vm_map_entry_link(&(map)->hdr, after_where, entry)
354:
355: #define vm_map_copy_entry_link(copy, after_where, entry) \
356: _vm_map_entry_link(&(copy)->cpy_hdr, after_where, entry)
357:
358: #define _vm_map_entry_link(hdr, after_where, entry) \
359: MACRO_BEGIN \
360: (hdr)->nentries++; \
361: (entry)->vme_prev = (after_where); \
362: (entry)->vme_next = (after_where)->vme_next; \
363: (entry)->vme_prev->vme_next = \
364: (entry)->vme_next->vme_prev = (entry); \
1.1.1.3 ! root 365: rbtree_insert(&(hdr)->tree, &(entry)->tree_node, \
! 366: vm_map_entry_cmp_insert); \
1.1 root 367: MACRO_END
368:
369: #define vm_map_entry_unlink(map, entry) \
370: _vm_map_entry_unlink(&(map)->hdr, entry)
371:
372: #define vm_map_copy_entry_unlink(copy, entry) \
373: _vm_map_entry_unlink(&(copy)->cpy_hdr, entry)
374:
375: #define _vm_map_entry_unlink(hdr, entry) \
376: MACRO_BEGIN \
377: (hdr)->nentries--; \
378: (entry)->vme_next->vme_prev = (entry)->vme_prev; \
379: (entry)->vme_prev->vme_next = (entry)->vme_next; \
1.1.1.3 ! root 380: rbtree_remove(&(hdr)->tree, &(entry)->tree_node); \
1.1 root 381: MACRO_END
382:
383: /*
384: * vm_map_reference:
385: *
386: * Creates another valid reference to the given map.
387: *
388: */
389: void vm_map_reference(map)
390: register vm_map_t map;
391: {
392: if (map == VM_MAP_NULL)
393: return;
394:
395: simple_lock(&map->ref_lock);
396: map->ref_count++;
397: simple_unlock(&map->ref_lock);
398: }
399:
400: /*
401: * vm_map_deallocate:
402: *
403: * Removes a reference from the specified map,
404: * destroying it if no references remain.
405: * The map should not be locked.
406: */
407: void vm_map_deallocate(map)
408: register vm_map_t map;
409: {
410: register int c;
411:
412: if (map == VM_MAP_NULL)
413: return;
414:
415: simple_lock(&map->ref_lock);
416: c = --map->ref_count;
417: simple_unlock(&map->ref_lock);
418:
419: if (c > 0) {
420: return;
421: }
422:
423: projected_buffer_collect(map);
424: (void) vm_map_delete(map, map->min_offset, map->max_offset);
425:
426: pmap_destroy(map->pmap);
427:
1.1.1.3 ! root 428: kmem_cache_free(&vm_map_cache, (vm_offset_t) map);
1.1 root 429: }
430:
431: /*
432: * SAVE_HINT:
433: *
434: * Saves the specified entry as the hint for
435: * future lookups. Performs necessary interlocks.
436: */
437: #define SAVE_HINT(map,value) \
438: simple_lock(&(map)->hint_lock); \
439: (map)->hint = (value); \
440: simple_unlock(&(map)->hint_lock);
441:
442: /*
443: * vm_map_lookup_entry: [ internal use only ]
444: *
445: * Finds the map entry containing (or
446: * immediately preceding) the specified address
447: * in the given map; the entry is returned
448: * in the "entry" parameter. The boolean
449: * result indicates whether the address is
450: * actually contained in the map.
451: */
452: boolean_t vm_map_lookup_entry(map, address, entry)
453: register vm_map_t map;
454: register vm_offset_t address;
455: vm_map_entry_t *entry; /* OUT */
456: {
1.1.1.3 ! root 457: register struct rbtree_node *node;
! 458: register vm_map_entry_t hint;
1.1 root 459:
460: /*
1.1.1.3 ! root 461: * First, make a quick check to see if we are already
! 462: * looking at the entry we want (which is often the case).
1.1 root 463: */
464:
465: simple_lock(&map->hint_lock);
1.1.1.3 ! root 466: hint = map->hint;
1.1 root 467: simple_unlock(&map->hint_lock);
468:
1.1.1.3 ! root 469: if ((hint != vm_map_to_entry(map)) && (address >= hint->vme_start)) {
! 470: if (address < hint->vme_end) {
! 471: *entry = hint;
1.1 root 472: return(TRUE);
1.1.1.3 ! root 473: } else {
! 474: vm_map_entry_t next = hint->vme_next;
! 475:
! 476: if ((next == vm_map_to_entry(map))
! 477: || (address < next->vme_start)) {
! 478: *entry = hint;
! 479: return(FALSE);
! 480: }
1.1 root 481: }
482: }
483:
484: /*
1.1.1.3 ! root 485: * If the hint didn't help, use the red-black tree.
1.1 root 486: */
487:
1.1.1.3 ! root 488: node = rbtree_lookup_nearest(&map->hdr.tree, address,
! 489: vm_map_entry_cmp_lookup, RBTREE_LEFT);
1.1 root 490:
1.1.1.3 ! root 491: if (node == NULL) {
! 492: *entry = vm_map_to_entry(map);
! 493: SAVE_HINT(map, *entry);
! 494: return(FALSE);
! 495: } else {
! 496: *entry = rbtree_entry(node, struct vm_map_entry, tree_node);
! 497: SAVE_HINT(map, *entry);
! 498: return((address < (*entry)->vme_end) ? TRUE : FALSE);
1.1 root 499: }
500: }
501:
502: /*
503: * Routine: invalid_user_access
504: *
505: * Verifies whether user access is valid.
506: */
507:
508: boolean_t
509: invalid_user_access(map, start, end, prot)
510: vm_map_t map;
511: vm_offset_t start, end;
512: vm_prot_t prot;
513: {
514: vm_map_entry_t entry;
515:
516: return (map == VM_MAP_NULL || map == kernel_map ||
517: !vm_map_lookup_entry(map, start, &entry) ||
518: entry->vme_end < end ||
519: (prot & ~(entry->protection)));
520: }
521:
522:
523: /*
524: * Routine: vm_map_find_entry
525: * Purpose:
526: * Allocate a range in the specified virtual address map,
527: * returning the entry allocated for that range.
528: * Used by kmem_alloc, etc. Returns wired entries.
529: *
530: * The map must be locked.
531: *
532: * If an entry is allocated, the object/offset fields
533: * are initialized to zero. If an object is supplied,
534: * then an existing entry may be extended.
535: */
536: kern_return_t vm_map_find_entry(map, address, size, mask, object, o_entry)
537: register vm_map_t map;
538: vm_offset_t *address; /* OUT */
539: vm_size_t size;
540: vm_offset_t mask;
541: vm_object_t object;
542: vm_map_entry_t *o_entry; /* OUT */
543: {
544: register vm_map_entry_t entry, new_entry;
545: register vm_offset_t start;
546: register vm_offset_t end;
547:
548: /*
549: * Look for the first possible address;
550: * if there's already something at this
551: * address, we have to start after it.
552: */
553:
554: if ((entry = map->first_free) == vm_map_to_entry(map))
555: start = map->min_offset;
556: else
557: start = entry->vme_end;
558:
559: /*
560: * In any case, the "entry" always precedes
561: * the proposed new region throughout the loop:
562: */
563:
564: while (TRUE) {
565: register vm_map_entry_t next;
566:
567: /*
568: * Find the end of the proposed new region.
569: * Be sure we didn't go beyond the end, or
570: * wrap around the address.
571: */
572:
1.1.1.3 ! root 573: if (((start + mask) & ~mask) < start) {
! 574: printf_once("no more room for vm_map_find_entry in %p\n", map);
1.1 root 575: return(KERN_NO_SPACE);
1.1.1.3 ! root 576: }
1.1 root 577: start = ((start + mask) & ~mask);
578: end = start + size;
579:
1.1.1.3 ! root 580: if ((end > map->max_offset) || (end < start)) {
! 581: printf_once("no more room for vm_map_find_entry in %p\n", map);
1.1 root 582: return(KERN_NO_SPACE);
1.1.1.3 ! root 583: }
1.1 root 584:
585: /*
586: * If there are no more entries, we must win.
587: */
588:
589: next = entry->vme_next;
590: if (next == vm_map_to_entry(map))
591: break;
592:
593: /*
594: * If there is another entry, it must be
595: * after the end of the potential new region.
596: */
597:
598: if (next->vme_start >= end)
599: break;
600:
601: /*
602: * Didn't fit -- move to the next entry.
603: */
604:
605: entry = next;
606: start = entry->vme_end;
607: }
608:
609: /*
610: * At this point,
611: * "start" and "end" should define the endpoints of the
612: * available new range, and
613: * "entry" should refer to the region before the new
614: * range, and
615: *
616: * the map should be locked.
617: */
618:
619: *address = start;
620:
621: /*
622: * See whether we can avoid creating a new entry by
623: * extending one of our neighbors. [So far, we only attempt to
624: * extend from below.]
625: */
626:
627: if ((object != VM_OBJECT_NULL) &&
628: (entry != vm_map_to_entry(map)) &&
629: (entry->vme_end == start) &&
630: (!entry->is_shared) &&
631: (!entry->is_sub_map) &&
632: (entry->object.vm_object == object) &&
633: (entry->needs_copy == FALSE) &&
634: (entry->inheritance == VM_INHERIT_DEFAULT) &&
635: (entry->protection == VM_PROT_DEFAULT) &&
636: (entry->max_protection == VM_PROT_ALL) &&
637: (entry->wired_count == 1) &&
638: (entry->user_wired_count == 0) &&
639: (entry->projected_on == 0)) {
640: /*
641: * Because this is a special case,
642: * we don't need to use vm_object_coalesce.
643: */
644:
645: entry->vme_end = end;
646: new_entry = entry;
647: } else {
648: new_entry = vm_map_entry_create(map);
649:
650: new_entry->vme_start = start;
651: new_entry->vme_end = end;
652:
653: new_entry->is_shared = FALSE;
654: new_entry->is_sub_map = FALSE;
655: new_entry->object.vm_object = VM_OBJECT_NULL;
656: new_entry->offset = (vm_offset_t) 0;
657:
658: new_entry->needs_copy = FALSE;
659:
660: new_entry->inheritance = VM_INHERIT_DEFAULT;
661: new_entry->protection = VM_PROT_DEFAULT;
662: new_entry->max_protection = VM_PROT_ALL;
663: new_entry->wired_count = 1;
664: new_entry->user_wired_count = 0;
665:
666: new_entry->in_transition = FALSE;
667: new_entry->needs_wakeup = FALSE;
668: new_entry->projected_on = 0;
669:
670: /*
671: * Insert the new entry into the list
672: */
673:
674: vm_map_entry_link(map, entry, new_entry);
675: }
676:
677: map->size += size;
678:
679: /*
680: * Update the free space hint and the lookup hint
681: */
682:
683: map->first_free = new_entry;
684: SAVE_HINT(map, new_entry);
685:
686: *o_entry = new_entry;
687: return(KERN_SUCCESS);
688: }
689:
690: int vm_map_pmap_enter_print = FALSE;
691: int vm_map_pmap_enter_enable = FALSE;
692:
693: /*
694: * Routine: vm_map_pmap_enter
695: *
696: * Description:
697: * Force pages from the specified object to be entered into
698: * the pmap at the specified address if they are present.
699: * As soon as a page not found in the object the scan ends.
700: *
701: * Returns:
1.1.1.2 root 702: * Nothing.
1.1 root 703: *
704: * In/out conditions:
705: * The source map should not be locked on entry.
706: */
707: void
708: vm_map_pmap_enter(map, addr, end_addr, object, offset, protection)
709: vm_map_t map;
710: register
711: vm_offset_t addr;
712: register
713: vm_offset_t end_addr;
714: register
715: vm_object_t object;
716: vm_offset_t offset;
717: vm_prot_t protection;
718: {
719: while (addr < end_addr) {
720: register vm_page_t m;
721:
722: vm_object_lock(object);
723: vm_object_paging_begin(object);
724:
725: m = vm_page_lookup(object, offset);
726: if (m == VM_PAGE_NULL || m->absent) {
727: vm_object_paging_end(object);
728: vm_object_unlock(object);
729: return;
730: }
731:
732: if (vm_map_pmap_enter_print) {
733: printf("vm_map_pmap_enter:");
1.1.1.3 ! root 734: printf("map: %p, addr: %lx, object: %p, offset: %lx\n",
1.1 root 735: map, addr, object, offset);
736: }
737:
738: m->busy = TRUE;
739: vm_object_unlock(object);
740:
741: PMAP_ENTER(map->pmap, addr, m,
742: protection, FALSE);
743:
744: vm_object_lock(object);
745: PAGE_WAKEUP_DONE(m);
746: vm_page_lock_queues();
747: if (!m->active && !m->inactive)
748: vm_page_activate(m);
749: vm_page_unlock_queues();
750: vm_object_paging_end(object);
751: vm_object_unlock(object);
752:
753: offset += PAGE_SIZE;
754: addr += PAGE_SIZE;
755: }
756: }
757:
758: /*
759: * Routine: vm_map_enter
760: *
761: * Description:
762: * Allocate a range in the specified virtual address map.
763: * The resulting range will refer to memory defined by
764: * the given memory object and offset into that object.
765: *
766: * Arguments are as defined in the vm_map call.
767: */
768: kern_return_t vm_map_enter(
769: map,
770: address, size, mask, anywhere,
771: object, offset, needs_copy,
772: cur_protection, max_protection, inheritance)
773: register
774: vm_map_t map;
775: vm_offset_t *address; /* IN/OUT */
776: vm_size_t size;
777: vm_offset_t mask;
778: boolean_t anywhere;
779: vm_object_t object;
780: vm_offset_t offset;
781: boolean_t needs_copy;
782: vm_prot_t cur_protection;
783: vm_prot_t max_protection;
784: vm_inherit_t inheritance;
785: {
786: register vm_map_entry_t entry;
787: register vm_offset_t start;
788: register vm_offset_t end;
789: kern_return_t result = KERN_SUCCESS;
790:
791: #define RETURN(value) { result = value; goto BailOut; }
792:
1.1.1.3 ! root 793: if (size == 0)
! 794: return KERN_INVALID_ARGUMENT;
! 795:
1.1 root 796: StartAgain: ;
797:
798: start = *address;
799:
800: if (anywhere) {
801: vm_map_lock(map);
802:
803: /*
804: * Calculate the first possible address.
805: */
806:
807: if (start < map->min_offset)
808: start = map->min_offset;
809: if (start > map->max_offset)
810: RETURN(KERN_NO_SPACE);
811:
812: /*
813: * Look for the first possible address;
814: * if there's already something at this
815: * address, we have to start after it.
816: */
817:
818: if (start == map->min_offset) {
819: if ((entry = map->first_free) != vm_map_to_entry(map))
820: start = entry->vme_end;
821: } else {
822: vm_map_entry_t tmp_entry;
823: if (vm_map_lookup_entry(map, start, &tmp_entry))
824: start = tmp_entry->vme_end;
825: entry = tmp_entry;
826: }
827:
828: /*
829: * In any case, the "entry" always precedes
830: * the proposed new region throughout the
831: * loop:
832: */
833:
834: while (TRUE) {
835: register vm_map_entry_t next;
836:
837: /*
838: * Find the end of the proposed new region.
839: * Be sure we didn't go beyond the end, or
840: * wrap around the address.
841: */
842:
1.1.1.3 ! root 843: if (((start + mask) & ~mask) < start) {
! 844: printf_once("no more room for vm_map_enter in %p\n", map);
! 845: RETURN(KERN_NO_SPACE);
! 846: }
1.1 root 847: start = ((start + mask) & ~mask);
848: end = start + size;
849:
850: if ((end > map->max_offset) || (end < start)) {
851: if (map->wait_for_space) {
852: if (size <= (map->max_offset -
853: map->min_offset)) {
854: assert_wait((event_t) map, TRUE);
855: vm_map_unlock(map);
856: thread_block((void (*)()) 0);
857: goto StartAgain;
858: }
859: }
1.1.1.2 root 860:
1.1.1.3 ! root 861: printf_once("no more room for vm_map_enter in %p\n", map);
1.1 root 862: RETURN(KERN_NO_SPACE);
863: }
864:
865: /*
866: * If there are no more entries, we must win.
867: */
868:
869: next = entry->vme_next;
870: if (next == vm_map_to_entry(map))
871: break;
872:
873: /*
874: * If there is another entry, it must be
875: * after the end of the potential new region.
876: */
877:
878: if (next->vme_start >= end)
879: break;
880:
881: /*
882: * Didn't fit -- move to the next entry.
883: */
884:
885: entry = next;
886: start = entry->vme_end;
887: }
888: *address = start;
889: } else {
890: vm_map_entry_t temp_entry;
891:
892: /*
893: * Verify that:
894: * the address doesn't itself violate
895: * the mask requirement.
896: */
897:
898: if ((start & mask) != 0)
899: return(KERN_NO_SPACE);
900:
901: vm_map_lock(map);
902:
903: /*
904: * ... the address is within bounds
905: */
906:
907: end = start + size;
908:
909: if ((start < map->min_offset) ||
910: (end > map->max_offset) ||
911: (start >= end)) {
912: RETURN(KERN_INVALID_ADDRESS);
913: }
914:
915: /*
916: * ... the starting address isn't allocated
917: */
918:
919: if (vm_map_lookup_entry(map, start, &temp_entry))
920: RETURN(KERN_NO_SPACE);
921:
922: entry = temp_entry;
923:
924: /*
925: * ... the next region doesn't overlap the
926: * end point.
927: */
928:
929: if ((entry->vme_next != vm_map_to_entry(map)) &&
930: (entry->vme_next->vme_start < end))
931: RETURN(KERN_NO_SPACE);
932: }
933:
934: /*
935: * At this point,
936: * "start" and "end" should define the endpoints of the
937: * available new range, and
938: * "entry" should refer to the region before the new
939: * range, and
940: *
941: * the map should be locked.
942: */
943:
944: /*
945: * See whether we can avoid creating a new entry (and object) by
946: * extending one of our neighbors. [So far, we only attempt to
947: * extend from below.]
948: */
949:
950: if ((object == VM_OBJECT_NULL) &&
951: (entry != vm_map_to_entry(map)) &&
952: (entry->vme_end == start) &&
953: (!entry->is_shared) &&
954: (!entry->is_sub_map) &&
955: (entry->inheritance == inheritance) &&
956: (entry->protection == cur_protection) &&
957: (entry->max_protection == max_protection) &&
958: (entry->wired_count == 0) && /* implies user_wired_count == 0 */
1.1.1.2 root 959: (entry->projected_on == 0)) {
1.1 root 960: if (vm_object_coalesce(entry->object.vm_object,
961: VM_OBJECT_NULL,
962: entry->offset,
963: (vm_offset_t) 0,
964: (vm_size_t)(entry->vme_end - entry->vme_start),
965: (vm_size_t)(end - entry->vme_end))) {
966:
967: /*
968: * Coalesced the two objects - can extend
969: * the previous map entry to include the
970: * new range.
971: */
972: map->size += (end - entry->vme_end);
973: entry->vme_end = end;
974: RETURN(KERN_SUCCESS);
975: }
976: }
977:
978: /*
979: * Create a new entry
980: */
981:
982: /**/ {
983: register vm_map_entry_t new_entry;
984:
985: new_entry = vm_map_entry_create(map);
986:
987: new_entry->vme_start = start;
988: new_entry->vme_end = end;
989:
990: new_entry->is_shared = FALSE;
991: new_entry->is_sub_map = FALSE;
992: new_entry->object.vm_object = object;
993: new_entry->offset = offset;
994:
995: new_entry->needs_copy = needs_copy;
996:
997: new_entry->inheritance = inheritance;
998: new_entry->protection = cur_protection;
999: new_entry->max_protection = max_protection;
1000: new_entry->wired_count = 0;
1001: new_entry->user_wired_count = 0;
1002:
1003: new_entry->in_transition = FALSE;
1004: new_entry->needs_wakeup = FALSE;
1005: new_entry->projected_on = 0;
1006:
1007: /*
1008: * Insert the new entry into the list
1009: */
1010:
1011: vm_map_entry_link(map, entry, new_entry);
1012: map->size += size;
1013:
1014: /*
1015: * Update the free space hint and the lookup hint
1016: */
1017:
1018: if ((map->first_free == entry) &&
1019: ((entry == vm_map_to_entry(map) ? map->min_offset : entry->vme_end)
1020: >= new_entry->vme_start))
1021: map->first_free = new_entry;
1022:
1023: SAVE_HINT(map, new_entry);
1024:
1025: vm_map_unlock(map);
1026:
1027: if ((object != VM_OBJECT_NULL) &&
1028: (vm_map_pmap_enter_enable) &&
1029: (!anywhere) &&
1.1.1.2 root 1030: (!needs_copy) &&
1.1 root 1031: (size < (128*1024))) {
1.1.1.2 root 1032: vm_map_pmap_enter(map, start, end,
1.1 root 1033: object, offset, cur_protection);
1034: }
1035:
1036: return(result);
1037: /**/ }
1038:
1039: BailOut: ;
1040:
1041: vm_map_unlock(map);
1042: return(result);
1043:
1044: #undef RETURN
1045: }
1046:
1047: /*
1048: * vm_map_clip_start: [ internal use only ]
1049: *
1050: * Asserts that the given entry begins at or after
1051: * the specified address; if necessary,
1052: * it splits the entry into two.
1053: */
1054: void _vm_map_clip_start();
1055: #define vm_map_clip_start(map, entry, startaddr) \
1056: MACRO_BEGIN \
1057: if ((startaddr) > (entry)->vme_start) \
1058: _vm_map_clip_start(&(map)->hdr,(entry),(startaddr)); \
1059: MACRO_END
1060:
1061: void _vm_map_copy_clip_start();
1062: #define vm_map_copy_clip_start(copy, entry, startaddr) \
1063: MACRO_BEGIN \
1064: if ((startaddr) > (entry)->vme_start) \
1065: _vm_map_clip_start(&(copy)->cpy_hdr,(entry),(startaddr)); \
1066: MACRO_END
1067:
1068: /*
1069: * This routine is called only when it is known that
1070: * the entry must be split.
1071: */
1072: void _vm_map_clip_start(map_header, entry, start)
1073: register struct vm_map_header *map_header;
1074: register vm_map_entry_t entry;
1075: register vm_offset_t start;
1076: {
1077: register vm_map_entry_t new_entry;
1078:
1079: /*
1080: * Split off the front portion --
1081: * note that we must insert the new
1082: * entry BEFORE this one, so that
1083: * this entry has the specified starting
1084: * address.
1085: */
1086:
1087: new_entry = _vm_map_entry_create(map_header);
1088: vm_map_entry_copy_full(new_entry, entry);
1089:
1090: new_entry->vme_end = start;
1091: entry->offset += (start - entry->vme_start);
1092: entry->vme_start = start;
1093:
1094: _vm_map_entry_link(map_header, entry->vme_prev, new_entry);
1095:
1096: if (entry->is_sub_map)
1097: vm_map_reference(new_entry->object.sub_map);
1098: else
1099: vm_object_reference(new_entry->object.vm_object);
1100: }
1101:
1102: /*
1103: * vm_map_clip_end: [ internal use only ]
1104: *
1105: * Asserts that the given entry ends at or before
1106: * the specified address; if necessary,
1107: * it splits the entry into two.
1108: */
1109: void _vm_map_clip_end();
1110: #define vm_map_clip_end(map, entry, endaddr) \
1111: MACRO_BEGIN \
1112: if ((endaddr) < (entry)->vme_end) \
1113: _vm_map_clip_end(&(map)->hdr,(entry),(endaddr)); \
1114: MACRO_END
1115:
1116: void _vm_map_copy_clip_end();
1117: #define vm_map_copy_clip_end(copy, entry, endaddr) \
1118: MACRO_BEGIN \
1119: if ((endaddr) < (entry)->vme_end) \
1120: _vm_map_clip_end(&(copy)->cpy_hdr,(entry),(endaddr)); \
1121: MACRO_END
1122:
1123: /*
1124: * This routine is called only when it is known that
1125: * the entry must be split.
1126: */
1127: void _vm_map_clip_end(map_header, entry, end)
1128: register struct vm_map_header *map_header;
1129: register vm_map_entry_t entry;
1130: register vm_offset_t end;
1131: {
1132: register vm_map_entry_t new_entry;
1133:
1134: /*
1135: * Create a new entry and insert it
1136: * AFTER the specified entry
1137: */
1138:
1139: new_entry = _vm_map_entry_create(map_header);
1140: vm_map_entry_copy_full(new_entry, entry);
1141:
1142: new_entry->vme_start = entry->vme_end = end;
1143: new_entry->offset += (end - entry->vme_start);
1144:
1145: _vm_map_entry_link(map_header, entry, new_entry);
1146:
1147: if (entry->is_sub_map)
1148: vm_map_reference(new_entry->object.sub_map);
1149: else
1150: vm_object_reference(new_entry->object.vm_object);
1151: }
1152:
1153: /*
1154: * VM_MAP_RANGE_CHECK: [ internal use only ]
1155: *
1156: * Asserts that the starting and ending region
1157: * addresses fall within the valid range of the map.
1158: */
1159: #define VM_MAP_RANGE_CHECK(map, start, end) \
1160: { \
1161: if (start < vm_map_min(map)) \
1162: start = vm_map_min(map); \
1163: if (end > vm_map_max(map)) \
1164: end = vm_map_max(map); \
1165: if (start > end) \
1166: start = end; \
1167: }
1168:
1169: /*
1170: * vm_map_submap: [ kernel use only ]
1171: *
1172: * Mark the given range as handled by a subordinate map.
1173: *
1174: * This range must have been created with vm_map_find using
1175: * the vm_submap_object, and no other operations may have been
1176: * performed on this range prior to calling vm_map_submap.
1177: *
1178: * Only a limited number of operations can be performed
1179: * within this rage after calling vm_map_submap:
1180: * vm_fault
1181: * [Don't try vm_map_copyin!]
1182: *
1183: * To remove a submapping, one must first remove the
1184: * range from the superior map, and then destroy the
1185: * submap (if desired). [Better yet, don't try it.]
1186: */
1187: kern_return_t vm_map_submap(map, start, end, submap)
1188: register vm_map_t map;
1189: register vm_offset_t start;
1190: register vm_offset_t end;
1191: vm_map_t submap;
1192: {
1193: vm_map_entry_t entry;
1194: register kern_return_t result = KERN_INVALID_ARGUMENT;
1195: register vm_object_t object;
1196:
1197: vm_map_lock(map);
1198:
1199: VM_MAP_RANGE_CHECK(map, start, end);
1200:
1201: if (vm_map_lookup_entry(map, start, &entry)) {
1202: vm_map_clip_start(map, entry, start);
1203: }
1204: else
1205: entry = entry->vme_next;
1206:
1207: vm_map_clip_end(map, entry, end);
1208:
1209: if ((entry->vme_start == start) && (entry->vme_end == end) &&
1210: (!entry->is_sub_map) &&
1211: ((object = entry->object.vm_object) == vm_submap_object) &&
1212: (object->resident_page_count == 0) &&
1213: (object->copy == VM_OBJECT_NULL) &&
1214: (object->shadow == VM_OBJECT_NULL) &&
1215: (!object->pager_created)) {
1216: entry->object.vm_object = VM_OBJECT_NULL;
1217: vm_object_deallocate(object);
1218: entry->is_sub_map = TRUE;
1219: vm_map_reference(entry->object.sub_map = submap);
1220: result = KERN_SUCCESS;
1221: }
1222: vm_map_unlock(map);
1223:
1224: return(result);
1225: }
1226:
1227: /*
1228: * vm_map_protect:
1229: *
1230: * Sets the protection of the specified address
1231: * region in the target map. If "set_max" is
1232: * specified, the maximum protection is to be set;
1233: * otherwise, only the current protection is affected.
1234: */
1235: kern_return_t vm_map_protect(map, start, end, new_prot, set_max)
1236: register vm_map_t map;
1237: register vm_offset_t start;
1238: register vm_offset_t end;
1239: register vm_prot_t new_prot;
1240: register boolean_t set_max;
1241: {
1242: register vm_map_entry_t current;
1243: vm_map_entry_t entry;
1244:
1245: vm_map_lock(map);
1246:
1247: VM_MAP_RANGE_CHECK(map, start, end);
1248:
1249: if (vm_map_lookup_entry(map, start, &entry)) {
1250: vm_map_clip_start(map, entry, start);
1251: }
1252: else
1253: entry = entry->vme_next;
1254:
1255: /*
1256: * Make a first pass to check for protection
1257: * violations.
1258: */
1259:
1260: current = entry;
1261: while ((current != vm_map_to_entry(map)) &&
1262: (current->vme_start < end)) {
1263:
1264: if (current->is_sub_map) {
1265: vm_map_unlock(map);
1266: return(KERN_INVALID_ARGUMENT);
1267: }
1268: if ((new_prot & (VM_PROT_NOTIFY | current->max_protection))
1269: != new_prot) {
1270: vm_map_unlock(map);
1271: return(KERN_PROTECTION_FAILURE);
1272: }
1273:
1274: current = current->vme_next;
1275: }
1276:
1277: /*
1278: * Go back and fix up protections.
1279: * [Note that clipping is not necessary the second time.]
1280: */
1281:
1282: current = entry;
1283:
1284: while ((current != vm_map_to_entry(map)) &&
1285: (current->vme_start < end)) {
1286:
1287: vm_prot_t old_prot;
1288:
1289: vm_map_clip_end(map, current, end);
1290:
1291: old_prot = current->protection;
1292: if (set_max)
1293: current->protection =
1294: (current->max_protection = new_prot) &
1295: old_prot;
1296: else
1297: current->protection = new_prot;
1298:
1299: /*
1300: * Update physical map if necessary.
1301: */
1302:
1303: if (current->protection != old_prot) {
1304: pmap_protect(map->pmap, current->vme_start,
1305: current->vme_end,
1306: current->protection);
1307: }
1308: current = current->vme_next;
1309: }
1310:
1311: vm_map_unlock(map);
1312: return(KERN_SUCCESS);
1313: }
1314:
1315: /*
1316: * vm_map_inherit:
1317: *
1318: * Sets the inheritance of the specified address
1319: * range in the target map. Inheritance
1320: * affects how the map will be shared with
1321: * child maps at the time of vm_map_fork.
1322: */
1323: kern_return_t vm_map_inherit(map, start, end, new_inheritance)
1324: register vm_map_t map;
1325: register vm_offset_t start;
1326: register vm_offset_t end;
1327: register vm_inherit_t new_inheritance;
1328: {
1329: register vm_map_entry_t entry;
1330: vm_map_entry_t temp_entry;
1331:
1332: vm_map_lock(map);
1333:
1334: VM_MAP_RANGE_CHECK(map, start, end);
1335:
1336: if (vm_map_lookup_entry(map, start, &temp_entry)) {
1337: entry = temp_entry;
1338: vm_map_clip_start(map, entry, start);
1339: }
1340: else
1341: entry = temp_entry->vme_next;
1342:
1343: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
1344: vm_map_clip_end(map, entry, end);
1345:
1346: entry->inheritance = new_inheritance;
1347:
1348: entry = entry->vme_next;
1349: }
1350:
1351: vm_map_unlock(map);
1352: return(KERN_SUCCESS);
1353: }
1354:
1355: /*
1356: * vm_map_pageable_common:
1357: *
1358: * Sets the pageability of the specified address
1359: * range in the target map. Regions specified
1360: * as not pageable require locked-down physical
1361: * memory and physical page maps. access_type indicates
1362: * types of accesses that must not generate page faults.
1363: * This is checked against protection of memory being locked-down.
1364: * access_type of VM_PROT_NONE makes memory pageable.
1365: *
1366: * The map must not be locked, but a reference
1367: * must remain to the map throughout the call.
1368: *
1369: * Callers should use macros in vm/vm_map.h (i.e. vm_map_pageable,
1370: * or vm_map_pageable_user); don't call vm_map_pageable directly.
1371: */
1372: kern_return_t vm_map_pageable_common(map, start, end, access_type, user_wire)
1373: register vm_map_t map;
1374: register vm_offset_t start;
1375: register vm_offset_t end;
1376: register vm_prot_t access_type;
1377: boolean_t user_wire;
1378: {
1379: register vm_map_entry_t entry;
1380: vm_map_entry_t start_entry;
1381:
1382: vm_map_lock(map);
1383:
1384: VM_MAP_RANGE_CHECK(map, start, end);
1385:
1386: if (vm_map_lookup_entry(map, start, &start_entry)) {
1387: entry = start_entry;
1388: /*
1389: * vm_map_clip_start will be done later.
1390: */
1391: }
1392: else {
1393: /*
1394: * Start address is not in map; this is fatal.
1395: */
1396: vm_map_unlock(map);
1397: return(KERN_FAILURE);
1398: }
1399:
1400: /*
1401: * Actions are rather different for wiring and unwiring,
1402: * so we have two separate cases.
1403: */
1404:
1405: if (access_type == VM_PROT_NONE) {
1406:
1407: vm_map_clip_start(map, entry, start);
1408:
1409: /*
1410: * Unwiring. First ensure that the range to be
1411: * unwired is really wired down.
1412: */
1413: while ((entry != vm_map_to_entry(map)) &&
1414: (entry->vme_start < end)) {
1415:
1416: if ((entry->wired_count == 0) ||
1.1.1.2 root 1417: ((entry->vme_end < end) &&
1.1 root 1418: ((entry->vme_next == vm_map_to_entry(map)) ||
1419: (entry->vme_next->vme_start > entry->vme_end))) ||
1420: (user_wire && (entry->user_wired_count == 0))) {
1421: vm_map_unlock(map);
1422: return(KERN_INVALID_ARGUMENT);
1423: }
1424: entry = entry->vme_next;
1425: }
1426:
1427: /*
1428: * Now decrement the wiring count for each region.
1429: * If a region becomes completely unwired,
1430: * unwire its physical pages and mappings.
1431: */
1432: entry = start_entry;
1433: while ((entry != vm_map_to_entry(map)) &&
1434: (entry->vme_start < end)) {
1435: vm_map_clip_end(map, entry, end);
1436:
1437: if (user_wire) {
1438: if (--(entry->user_wired_count) == 0)
1439: entry->wired_count--;
1440: }
1441: else {
1442: entry->wired_count--;
1443: }
1.1.1.2 root 1444:
1.1 root 1445: if (entry->wired_count == 0)
1446: vm_fault_unwire(map, entry);
1447:
1448: entry = entry->vme_next;
1449: }
1450: }
1451:
1452: else {
1453: /*
1454: * Wiring. We must do this in two passes:
1455: *
1456: * 1. Holding the write lock, we create any shadow
1457: * or zero-fill objects that need to be created.
1458: * Then we clip each map entry to the region to be
1459: * wired and increment its wiring count. We
1460: * create objects before clipping the map entries
1461: * to avoid object proliferation.
1462: *
1463: * 2. We downgrade to a read lock, and call
1464: * vm_fault_wire to fault in the pages for any
1465: * newly wired area (wired_count is 1).
1466: *
1467: * Downgrading to a read lock for vm_fault_wire avoids
1468: * a possible deadlock with another thread that may have
1469: * faulted on one of the pages to be wired (it would mark
1470: * the page busy, blocking us, then in turn block on the
1471: * map lock that we hold). Because of problems in the
1472: * recursive lock package, we cannot upgrade to a write
1473: * lock in vm_map_lookup. Thus, any actions that require
1474: * the write lock must be done beforehand. Because we
1475: * keep the read lock on the map, the copy-on-write
1476: * status of the entries we modify here cannot change.
1477: */
1478:
1479: /*
1480: * Pass 1.
1481: */
1482: while ((entry != vm_map_to_entry(map)) &&
1483: (entry->vme_start < end)) {
1484: vm_map_clip_end(map, entry, end);
1485:
1486: if (entry->wired_count == 0) {
1487:
1488: /*
1489: * Perform actions of vm_map_lookup that need
1490: * the write lock on the map: create a shadow
1491: * object for a copy-on-write region, or an
1492: * object for a zero-fill region.
1493: */
1494: if (entry->needs_copy &&
1495: ((entry->protection & VM_PROT_WRITE) != 0)) {
1496:
1497: vm_object_shadow(&entry->object.vm_object,
1498: &entry->offset,
1499: (vm_size_t)(entry->vme_end
1500: - entry->vme_start));
1501: entry->needs_copy = FALSE;
1502: }
1503: if (entry->object.vm_object == VM_OBJECT_NULL) {
1504: entry->object.vm_object =
1505: vm_object_allocate(
1.1.1.2 root 1506: (vm_size_t)(entry->vme_end
1.1 root 1507: - entry->vme_start));
1508: entry->offset = (vm_offset_t)0;
1509: }
1510: }
1511: vm_map_clip_start(map, entry, start);
1512: vm_map_clip_end(map, entry, end);
1513:
1514: if (user_wire) {
1515: if ((entry->user_wired_count)++ == 0)
1516: entry->wired_count++;
1517: }
1518: else {
1519: entry->wired_count++;
1520: }
1521:
1522: /*
1523: * Check for holes and protection mismatch.
1524: * Holes: Next entry should be contiguous unless
1525: * this is the end of the region.
1526: * Protection: Access requested must be allowed.
1527: */
1.1.1.2 root 1528: if (((entry->vme_end < end) &&
1.1 root 1529: ((entry->vme_next == vm_map_to_entry(map)) ||
1530: (entry->vme_next->vme_start > entry->vme_end))) ||
1531: ((entry->protection & access_type) != access_type)) {
1532: /*
1533: * Found a hole or protection problem.
1534: * Object creation actions
1535: * do not need to be undone, but the
1536: * wired counts need to be restored.
1537: */
1538: while ((entry != vm_map_to_entry(map)) &&
1539: (entry->vme_end > start)) {
1540: if (user_wire) {
1541: if (--(entry->user_wired_count) == 0)
1542: entry->wired_count--;
1543: }
1544: else {
1545: entry->wired_count--;
1546: }
1547:
1548: entry = entry->vme_prev;
1549: }
1550:
1551: vm_map_unlock(map);
1552: return(KERN_FAILURE);
1553: }
1554: entry = entry->vme_next;
1555: }
1556:
1557: /*
1558: * Pass 2.
1559: */
1560:
1561: /*
1562: * HACK HACK HACK HACK
1563: *
1564: * If we are wiring in the kernel map or a submap of it,
1565: * unlock the map to avoid deadlocks. We trust that the
1566: * kernel threads are well-behaved, and therefore will
1567: * not do anything destructive to this region of the map
1568: * while we have it unlocked. We cannot trust user threads
1569: * to do the same.
1570: *
1571: * HACK HACK HACK HACK
1572: */
1573: if (vm_map_pmap(map) == kernel_pmap) {
1574: vm_map_unlock(map); /* trust me ... */
1575: }
1576: else {
1577: vm_map_lock_set_recursive(map);
1578: vm_map_lock_write_to_read(map);
1579: }
1580:
1581: entry = start_entry;
1582: while (entry != vm_map_to_entry(map) &&
1583: entry->vme_start < end) {
1584: /*
1585: * Wiring cases:
1586: * Kernel: wired == 1 && user_wired == 0
1587: * User: wired == 1 && user_wired == 1
1588: *
1589: * Don't need to wire if either is > 1. wired = 0 &&
1590: * user_wired == 1 can't happen.
1591: */
1592:
1593: /*
1594: * XXX This assumes that the faults always succeed.
1595: */
1596: if ((entry->wired_count == 1) &&
1597: (entry->user_wired_count <= 1)) {
1598: vm_fault_wire(map, entry);
1599: }
1600: entry = entry->vme_next;
1601: }
1602:
1603: if (vm_map_pmap(map) == kernel_pmap) {
1604: vm_map_lock(map);
1605: }
1606: else {
1607: vm_map_lock_clear_recursive(map);
1608: }
1609: }
1610:
1611: vm_map_unlock(map);
1612:
1613: return(KERN_SUCCESS);
1614: }
1615:
1616: /*
1617: * vm_map_entry_delete: [ internal use only ]
1618: *
1619: * Deallocate the given entry from the target map.
1.1.1.2 root 1620: */
1.1 root 1621: void vm_map_entry_delete(map, entry)
1622: register vm_map_t map;
1623: register vm_map_entry_t entry;
1624: {
1625: register vm_offset_t s, e;
1626: register vm_object_t object;
1627: extern vm_object_t kernel_object;
1628:
1629: s = entry->vme_start;
1630: e = entry->vme_end;
1631:
1632: /*Check if projected buffer*/
1633: if (map != kernel_map && entry->projected_on != 0) {
1634: /*Check if projected kernel entry is persistent;
1635: may only manipulate directly if it is*/
1636: if (entry->projected_on->projected_on == 0)
1637: entry->wired_count = 0; /*Avoid unwire fault*/
1638: else
1639: return;
1640: }
1641:
1642: /*
1643: * Get the object. Null objects cannot have pmap entries.
1644: */
1645:
1646: if ((object = entry->object.vm_object) != VM_OBJECT_NULL) {
1647:
1648: /*
1649: * Unwire before removing addresses from the pmap;
1650: * otherwise, unwiring will put the entries back in
1651: * the pmap.
1652: */
1653:
1654: if (entry->wired_count != 0) {
1655: vm_fault_unwire(map, entry);
1656: entry->wired_count = 0;
1657: entry->user_wired_count = 0;
1658: }
1659:
1660: /*
1661: * If the object is shared, we must remove
1662: * *all* references to this data, since we can't
1663: * find all of the physical maps which are sharing
1664: * it.
1665: */
1666:
1667: if (object == kernel_object) {
1668: vm_object_lock(object);
1669: vm_object_page_remove(object, entry->offset,
1670: entry->offset + (e - s));
1671: vm_object_unlock(object);
1672: } else if (entry->is_shared) {
1673: vm_object_pmap_remove(object,
1674: entry->offset,
1675: entry->offset + (e - s));
1676: }
1677: else {
1678: pmap_remove(map->pmap, s, e);
1679: }
1680: }
1681:
1682: /*
1683: * Deallocate the object only after removing all
1684: * pmap entries pointing to its pages.
1685: */
1686:
1687: if (entry->is_sub_map)
1688: vm_map_deallocate(entry->object.sub_map);
1689: else
1690: vm_object_deallocate(entry->object.vm_object);
1691:
1692: vm_map_entry_unlink(map, entry);
1693: map->size -= e - s;
1694:
1695: vm_map_entry_dispose(map, entry);
1696: }
1697:
1698: /*
1699: * vm_map_delete: [ internal use only ]
1700: *
1701: * Deallocates the given address range from the target
1702: * map.
1703: */
1704:
1705: kern_return_t vm_map_delete(map, start, end)
1706: register vm_map_t map;
1707: register vm_offset_t start;
1708: register vm_offset_t end;
1709: {
1710: vm_map_entry_t entry;
1711: vm_map_entry_t first_entry;
1712:
1713: /*
1714: * Find the start of the region, and clip it
1715: */
1716:
1717: if (!vm_map_lookup_entry(map, start, &first_entry))
1718: entry = first_entry->vme_next;
1719: else {
1720: entry = first_entry;
1721: vm_map_clip_start(map, entry, start);
1722:
1723: /*
1724: * Fix the lookup hint now, rather than each
1725: * time though the loop.
1726: */
1727:
1728: SAVE_HINT(map, entry->vme_prev);
1729: }
1730:
1731: /*
1732: * Save the free space hint
1733: */
1734:
1735: if (map->first_free->vme_start >= start)
1736: map->first_free = entry->vme_prev;
1737:
1738: /*
1739: * Step through all entries in this region
1740: */
1741:
1742: while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
1743: vm_map_entry_t next;
1744:
1745: vm_map_clip_end(map, entry, end);
1746:
1747: /*
1748: * If the entry is in transition, we must wait
1749: * for it to exit that state. It could be clipped
1750: * while we leave the map unlocked.
1751: */
1752: if(entry->in_transition) {
1753: /*
1754: * Say that we are waiting, and wait for entry.
1755: */
1756: entry->needs_wakeup = TRUE;
1757: vm_map_entry_wait(map, FALSE);
1758: vm_map_lock(map);
1759:
1760: /*
1761: * The entry could have been clipped or it
1762: * may not exist anymore. look it up again.
1763: */
1764: if(!vm_map_lookup_entry(map, start, &entry)) {
1765: entry = entry->vme_next;
1766: }
1767: continue;
1768: }
1769:
1770: next = entry->vme_next;
1771:
1772: vm_map_entry_delete(map, entry);
1773: entry = next;
1774: }
1775:
1776: if (map->wait_for_space)
1777: thread_wakeup((event_t) map);
1778:
1779: return(KERN_SUCCESS);
1780: }
1781:
1782: /*
1783: * vm_map_remove:
1784: *
1785: * Remove the given address range from the target map.
1786: * This is the exported form of vm_map_delete.
1787: */
1788: kern_return_t vm_map_remove(map, start, end)
1789: register vm_map_t map;
1790: register vm_offset_t start;
1791: register vm_offset_t end;
1792: {
1793: register kern_return_t result;
1794:
1795: vm_map_lock(map);
1796: VM_MAP_RANGE_CHECK(map, start, end);
1797: result = vm_map_delete(map, start, end);
1798: vm_map_unlock(map);
1799:
1800: return(result);
1801: }
1802:
1803:
1804: /*
1805: * vm_map_copy_steal_pages:
1806: *
1807: * Steal all the pages from a vm_map_copy page_list by copying ones
1808: * that have not already been stolen.
1809: */
1810: void
1811: vm_map_copy_steal_pages(copy)
1812: vm_map_copy_t copy;
1813: {
1814: register vm_page_t m, new_m;
1815: register int i;
1816: vm_object_t object;
1817:
1818: for (i = 0; i < copy->cpy_npages; i++) {
1819:
1820: /*
1821: * If the page is not tabled, then it's already stolen.
1822: */
1823: m = copy->cpy_page_list[i];
1824: if (!m->tabled)
1825: continue;
1826:
1827: /*
1828: * Page was not stolen, get a new
1829: * one and do the copy now.
1830: */
1.1.1.2 root 1831: while ((new_m = vm_page_grab(FALSE)) == VM_PAGE_NULL) {
1.1 root 1832: VM_PAGE_WAIT((void(*)()) 0);
1833: }
1834:
1835: vm_page_copy(m, new_m);
1836:
1837: object = m->object;
1838: vm_object_lock(object);
1839: vm_page_lock_queues();
1840: if (!m->active && !m->inactive)
1841: vm_page_activate(m);
1842: vm_page_unlock_queues();
1843: PAGE_WAKEUP_DONE(m);
1844: vm_object_paging_end(object);
1845: vm_object_unlock(object);
1846:
1847: copy->cpy_page_list[i] = new_m;
1848: }
1849: }
1850:
1851: /*
1852: * vm_map_copy_page_discard:
1853: *
1854: * Get rid of the pages in a page_list copy. If the pages are
1855: * stolen, they are freed. If the pages are not stolen, they
1856: * are unbusied, and associated state is cleaned up.
1857: */
1858: void vm_map_copy_page_discard(copy)
1859: vm_map_copy_t copy;
1860: {
1861: while (copy->cpy_npages > 0) {
1862: vm_page_t m;
1863:
1864: if((m = copy->cpy_page_list[--(copy->cpy_npages)]) !=
1865: VM_PAGE_NULL) {
1866:
1867: /*
1868: * If it's not in the table, then it's
1869: * a stolen page that goes back
1870: * to the free list. Else it belongs
1871: * to some object, and we hold a
1872: * paging reference on that object.
1873: */
1874: if (!m->tabled) {
1875: VM_PAGE_FREE(m);
1876: }
1877: else {
1878: vm_object_t object;
1879:
1880: object = m->object;
1881:
1882: vm_object_lock(object);
1883: vm_page_lock_queues();
1884: if (!m->active && !m->inactive)
1885: vm_page_activate(m);
1886: vm_page_unlock_queues();
1887:
1888: PAGE_WAKEUP_DONE(m);
1889: vm_object_paging_end(object);
1890: vm_object_unlock(object);
1891: }
1892: }
1893: }
1894: }
1895:
1896: /*
1897: * Routine: vm_map_copy_discard
1898: *
1899: * Description:
1900: * Dispose of a map copy object (returned by
1901: * vm_map_copyin).
1902: */
1903: void
1904: vm_map_copy_discard(copy)
1905: vm_map_copy_t copy;
1906: {
1907: free_next_copy:
1908: if (copy == VM_MAP_COPY_NULL)
1909: return;
1910:
1911: switch (copy->type) {
1912: case VM_MAP_COPY_ENTRY_LIST:
1913: while (vm_map_copy_first_entry(copy) !=
1914: vm_map_copy_to_entry(copy)) {
1915: vm_map_entry_t entry = vm_map_copy_first_entry(copy);
1916:
1917: vm_map_copy_entry_unlink(copy, entry);
1918: vm_object_deallocate(entry->object.vm_object);
1919: vm_map_copy_entry_dispose(copy, entry);
1920: }
1921: break;
1922: case VM_MAP_COPY_OBJECT:
1923: vm_object_deallocate(copy->cpy_object);
1924: break;
1925: case VM_MAP_COPY_PAGE_LIST:
1926:
1927: /*
1928: * To clean this up, we have to unbusy all the pages
1929: * and release the paging references in their objects.
1930: */
1931: if (copy->cpy_npages > 0)
1932: vm_map_copy_page_discard(copy);
1933:
1934: /*
1935: * If there's a continuation, abort it. The
1936: * abort routine releases any storage.
1937: */
1938: if (vm_map_copy_has_cont(copy)) {
1939:
1940: /*
1941: * Special case: recognize
1942: * vm_map_copy_discard_cont and optimize
1943: * here to avoid tail recursion.
1944: */
1945: if (copy->cpy_cont == vm_map_copy_discard_cont) {
1946: register vm_map_copy_t new_copy;
1947:
1948: new_copy = (vm_map_copy_t) copy->cpy_cont_args;
1.1.1.3 ! root 1949: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 1950: copy = new_copy;
1951: goto free_next_copy;
1952: }
1953: else {
1954: vm_map_copy_abort_cont(copy);
1955: }
1956: }
1957:
1958: break;
1959: }
1.1.1.3 ! root 1960: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 1961: }
1962:
1963: /*
1964: * Routine: vm_map_copy_copy
1965: *
1966: * Description:
1967: * Move the information in a map copy object to
1968: * a new map copy object, leaving the old one
1969: * empty.
1970: *
1971: * This is used by kernel routines that need
1972: * to look at out-of-line data (in copyin form)
1973: * before deciding whether to return SUCCESS.
1974: * If the routine returns FAILURE, the original
1975: * copy object will be deallocated; therefore,
1976: * these routines must make a copy of the copy
1977: * object and leave the original empty so that
1978: * deallocation will not fail.
1979: */
1980: vm_map_copy_t
1981: vm_map_copy_copy(copy)
1982: vm_map_copy_t copy;
1983: {
1984: vm_map_copy_t new_copy;
1985:
1986: if (copy == VM_MAP_COPY_NULL)
1987: return VM_MAP_COPY_NULL;
1988:
1989: /*
1990: * Allocate a new copy object, and copy the information
1991: * from the old one into it.
1992: */
1993:
1.1.1.3 ! root 1994: new_copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 1995: *new_copy = *copy;
1996:
1997: if (copy->type == VM_MAP_COPY_ENTRY_LIST) {
1998: /*
1999: * The links in the entry chain must be
2000: * changed to point to the new copy object.
2001: */
2002: vm_map_copy_first_entry(copy)->vme_prev
2003: = vm_map_copy_to_entry(new_copy);
2004: vm_map_copy_last_entry(copy)->vme_next
2005: = vm_map_copy_to_entry(new_copy);
2006: }
2007:
2008: /*
2009: * Change the old copy object into one that contains
2010: * nothing to be deallocated.
2011: */
2012: copy->type = VM_MAP_COPY_OBJECT;
2013: copy->cpy_object = VM_OBJECT_NULL;
2014:
2015: /*
2016: * Return the new object.
2017: */
2018: return new_copy;
2019: }
2020:
2021: /*
2022: * Routine: vm_map_copy_discard_cont
2023: *
2024: * Description:
2025: * A version of vm_map_copy_discard that can be called
2026: * as a continuation from a vm_map_copy page list.
2027: */
2028: kern_return_t vm_map_copy_discard_cont(cont_args, copy_result)
2029: vm_map_copyin_args_t cont_args;
2030: vm_map_copy_t *copy_result; /* OUT */
2031: {
2032: vm_map_copy_discard((vm_map_copy_t) cont_args);
2033: if (copy_result != (vm_map_copy_t *)0)
2034: *copy_result = VM_MAP_COPY_NULL;
2035: return(KERN_SUCCESS);
2036: }
2037:
2038: /*
2039: * Routine: vm_map_copy_overwrite
2040: *
2041: * Description:
2042: * Copy the memory described by the map copy
2043: * object (copy; returned by vm_map_copyin) onto
2044: * the specified destination region (dst_map, dst_addr).
2045: * The destination must be writeable.
2046: *
2047: * Unlike vm_map_copyout, this routine actually
2048: * writes over previously-mapped memory. If the
2049: * previous mapping was to a permanent (user-supplied)
2050: * memory object, it is preserved.
2051: *
2052: * The attributes (protection and inheritance) of the
2053: * destination region are preserved.
2054: *
2055: * If successful, consumes the copy object.
2056: * Otherwise, the caller is responsible for it.
2057: *
2058: * Implementation notes:
2059: * To overwrite temporary virtual memory, it is
2060: * sufficient to remove the previous mapping and insert
2061: * the new copy. This replacement is done either on
2062: * the whole region (if no permanent virtual memory
2063: * objects are embedded in the destination region) or
2064: * in individual map entries.
2065: *
2066: * To overwrite permanent virtual memory, it is
2067: * necessary to copy each page, as the external
2068: * memory management interface currently does not
2069: * provide any optimizations.
2070: *
2071: * Once a page of permanent memory has been overwritten,
2072: * it is impossible to interrupt this function; otherwise,
2073: * the call would be neither atomic nor location-independent.
2074: * The kernel-state portion of a user thread must be
2075: * interruptible.
2076: *
2077: * It may be expensive to forward all requests that might
2078: * overwrite permanent memory (vm_write, vm_copy) to
2079: * uninterruptible kernel threads. This routine may be
2080: * called by interruptible threads; however, success is
2081: * not guaranteed -- if the request cannot be performed
2082: * atomically and interruptibly, an error indication is
2083: * returned.
2084: */
2085: kern_return_t vm_map_copy_overwrite(dst_map, dst_addr, copy, interruptible)
2086: vm_map_t dst_map;
2087: vm_offset_t dst_addr;
2088: vm_map_copy_t copy;
2089: boolean_t interruptible;
2090: {
2091: vm_size_t size;
2092: vm_offset_t start;
2093: vm_map_entry_t tmp_entry;
2094: vm_map_entry_t entry;
2095:
2096: boolean_t contains_permanent_objects = FALSE;
2097:
2098: interruptible = FALSE; /* XXX */
2099:
2100: /*
2101: * Check for null copy object.
2102: */
2103:
2104: if (copy == VM_MAP_COPY_NULL)
2105: return(KERN_SUCCESS);
2106:
2107: /*
2108: * Only works for entry lists at the moment. Will
2109: * support page lists LATER.
2110: */
2111:
2112: assert(copy->type == VM_MAP_COPY_ENTRY_LIST);
2113:
2114: /*
2115: * Currently this routine only handles page-aligned
2116: * regions. Eventually, it should handle misalignments
2117: * by actually copying pages.
2118: */
2119:
2120: if (!page_aligned(copy->offset) ||
2121: !page_aligned(copy->size) ||
2122: !page_aligned(dst_addr))
2123: return(KERN_INVALID_ARGUMENT);
2124:
2125: size = copy->size;
2126:
2127: if (size == 0) {
2128: vm_map_copy_discard(copy);
2129: return(KERN_SUCCESS);
2130: }
2131:
2132: /*
2133: * Verify that the destination is all writeable
2134: * initially.
2135: */
2136: start_pass_1:
2137: vm_map_lock(dst_map);
2138: if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {
2139: vm_map_unlock(dst_map);
2140: return(KERN_INVALID_ADDRESS);
2141: }
2142: vm_map_clip_start(dst_map, tmp_entry, dst_addr);
2143: for (entry = tmp_entry;;) {
2144: vm_size_t sub_size = (entry->vme_end - entry->vme_start);
2145: vm_map_entry_t next = entry->vme_next;
2146:
2147: if ( ! (entry->protection & VM_PROT_WRITE)) {
2148: vm_map_unlock(dst_map);
2149: return(KERN_PROTECTION_FAILURE);
2150: }
2151:
2152: /*
2153: * If the entry is in transition, we must wait
2154: * for it to exit that state. Anything could happen
2155: * when we unlock the map, so start over.
2156: */
2157: if (entry->in_transition) {
2158:
2159: /*
2160: * Say that we are waiting, and wait for entry.
2161: */
2162: entry->needs_wakeup = TRUE;
2163: vm_map_entry_wait(dst_map, FALSE);
2164:
2165: goto start_pass_1;
2166: }
2167:
2168: if (size <= sub_size)
2169: break;
2170:
2171: if ((next == vm_map_to_entry(dst_map)) ||
2172: (next->vme_start != entry->vme_end)) {
2173: vm_map_unlock(dst_map);
2174: return(KERN_INVALID_ADDRESS);
2175: }
2176:
2177:
2178: /*
2179: * Check for permanent objects in the destination.
2180: */
2181:
2182: if ((entry->object.vm_object != VM_OBJECT_NULL) &&
2183: !entry->object.vm_object->temporary)
2184: contains_permanent_objects = TRUE;
2185:
2186: size -= sub_size;
2187: entry = next;
2188: }
2189:
2190: /*
2191: * If there are permanent objects in the destination, then
2192: * the copy cannot be interrupted.
2193: */
2194:
1.1.1.3 ! root 2195: if (interruptible && contains_permanent_objects) {
! 2196: vm_map_unlock(dst_map);
1.1 root 2197: return(KERN_FAILURE); /* XXX */
1.1.1.3 ! root 2198: }
1.1 root 2199:
2200: /*
2201: * XXXO If there are no permanent objects in the destination,
1.1.1.3 ! root 2202: * XXXO and the source and destination map entry caches match,
1.1.1.2 root 2203: * XXXO and the destination map entry is not shared,
1.1 root 2204: * XXXO then the map entries can be deleted and replaced
2205: * XXXO with those from the copy. The following code is the
2206: * XXXO basic idea of what to do, but there are lots of annoying
2207: * XXXO little details about getting protection and inheritance
2208: * XXXO right. Should add protection, inheritance, and sharing checks
2209: * XXXO to the above pass and make sure that no wiring is involved.
2210: */
2211: /*
2212: * if (!contains_permanent_objects &&
2213: * copy->cpy_hdr.entries_pageable == dst_map->hdr.entries_pageable) {
2214: *
2215: * *
2216: * * Run over copy and adjust entries. Steal code
2217: * * from vm_map_copyout() to do this.
2218: * *
2219: *
2220: * tmp_entry = tmp_entry->vme_prev;
2221: * vm_map_delete(dst_map, dst_addr, dst_addr + copy->size);
2222: * vm_map_copy_insert(dst_map, tmp_entry, copy);
2223: *
2224: * vm_map_unlock(dst_map);
2225: * vm_map_copy_discard(copy);
2226: * }
2227: */
2228: /*
2229: *
2230: * Make a second pass, overwriting the data
2231: * At the beginning of each loop iteration,
2232: * the next entry to be overwritten is "tmp_entry"
2233: * (initially, the value returned from the lookup above),
2234: * and the starting address expected in that entry
2235: * is "start".
2236: */
2237:
2238: start = dst_addr;
2239:
2240: while (vm_map_copy_first_entry(copy) != vm_map_copy_to_entry(copy)) {
2241: vm_map_entry_t copy_entry = vm_map_copy_first_entry(copy);
2242: vm_size_t copy_size = (copy_entry->vme_end - copy_entry->vme_start);
2243: vm_object_t object;
1.1.1.2 root 2244:
1.1 root 2245: entry = tmp_entry;
2246: size = (entry->vme_end - entry->vme_start);
2247: /*
2248: * Make sure that no holes popped up in the
2249: * address map, and that the protection is
2250: * still valid, in case the map was unlocked
2251: * earlier.
2252: */
2253:
2254: if (entry->vme_start != start) {
2255: vm_map_unlock(dst_map);
2256: return(KERN_INVALID_ADDRESS);
2257: }
2258: assert(entry != vm_map_to_entry(dst_map));
2259:
2260: /*
2261: * Check protection again
2262: */
2263:
2264: if ( ! (entry->protection & VM_PROT_WRITE)) {
2265: vm_map_unlock(dst_map);
2266: return(KERN_PROTECTION_FAILURE);
2267: }
2268:
2269: /*
2270: * Adjust to source size first
2271: */
2272:
2273: if (copy_size < size) {
2274: vm_map_clip_end(dst_map, entry, entry->vme_start + copy_size);
2275: size = copy_size;
2276: }
2277:
2278: /*
2279: * Adjust to destination size
2280: */
2281:
2282: if (size < copy_size) {
2283: vm_map_copy_clip_end(copy, copy_entry,
2284: copy_entry->vme_start + size);
2285: copy_size = size;
2286: }
2287:
2288: assert((entry->vme_end - entry->vme_start) == size);
2289: assert((tmp_entry->vme_end - tmp_entry->vme_start) == size);
2290: assert((copy_entry->vme_end - copy_entry->vme_start) == size);
2291:
2292: /*
2293: * If the destination contains temporary unshared memory,
2294: * we can perform the copy by throwing it away and
2295: * installing the source data.
2296: */
2297:
2298: object = entry->object.vm_object;
2299: if (!entry->is_shared &&
2300: ((object == VM_OBJECT_NULL) || object->temporary)) {
2301: vm_object_t old_object = entry->object.vm_object;
2302: vm_offset_t old_offset = entry->offset;
2303:
2304: entry->object = copy_entry->object;
2305: entry->offset = copy_entry->offset;
2306: entry->needs_copy = copy_entry->needs_copy;
2307: entry->wired_count = 0;
2308: entry->user_wired_count = 0;
2309:
2310: vm_map_copy_entry_unlink(copy, copy_entry);
2311: vm_map_copy_entry_dispose(copy, copy_entry);
2312:
2313: vm_object_pmap_protect(
2314: old_object,
2315: old_offset,
2316: size,
2317: dst_map->pmap,
2318: tmp_entry->vme_start,
2319: VM_PROT_NONE);
2320:
2321: vm_object_deallocate(old_object);
2322:
2323: /*
2324: * Set up for the next iteration. The map
2325: * has not been unlocked, so the next
2326: * address should be at the end of this
2327: * entry, and the next map entry should be
2328: * the one following it.
2329: */
2330:
2331: start = tmp_entry->vme_end;
2332: tmp_entry = tmp_entry->vme_next;
2333: } else {
2334: vm_map_version_t version;
2335: vm_object_t dst_object = entry->object.vm_object;
2336: vm_offset_t dst_offset = entry->offset;
2337: kern_return_t r;
2338:
2339: /*
2340: * Take an object reference, and record
2341: * the map version information so that the
2342: * map can be safely unlocked.
2343: */
2344:
2345: vm_object_reference(dst_object);
2346:
2347: version.main_timestamp = dst_map->timestamp;
2348:
2349: vm_map_unlock(dst_map);
2350:
2351: /*
2352: * Copy as much as possible in one pass
2353: */
2354:
2355: copy_size = size;
2356: r = vm_fault_copy(
2357: copy_entry->object.vm_object,
2358: copy_entry->offset,
2359: ©_size,
2360: dst_object,
2361: dst_offset,
2362: dst_map,
2363: &version,
2364: FALSE /* XXX interruptible */ );
2365:
2366: /*
2367: * Release the object reference
2368: */
2369:
2370: vm_object_deallocate(dst_object);
2371:
2372: /*
2373: * If a hard error occurred, return it now
2374: */
2375:
2376: if (r != KERN_SUCCESS)
2377: return(r);
2378:
2379: if (copy_size != 0) {
2380: /*
2381: * Dispose of the copied region
2382: */
2383:
2384: vm_map_copy_clip_end(copy, copy_entry,
2385: copy_entry->vme_start + copy_size);
2386: vm_map_copy_entry_unlink(copy, copy_entry);
2387: vm_object_deallocate(copy_entry->object.vm_object);
2388: vm_map_copy_entry_dispose(copy, copy_entry);
2389: }
2390:
2391: /*
2392: * Pick up in the destination map where we left off.
2393: *
2394: * Use the version information to avoid a lookup
2395: * in the normal case.
2396: */
2397:
2398: start += copy_size;
2399: vm_map_lock(dst_map);
2400: if ((version.main_timestamp + 1) == dst_map->timestamp) {
2401: /* We can safely use saved tmp_entry value */
2402:
2403: vm_map_clip_end(dst_map, tmp_entry, start);
2404: tmp_entry = tmp_entry->vme_next;
2405: } else {
2406: /* Must do lookup of tmp_entry */
2407:
2408: if (!vm_map_lookup_entry(dst_map, start, &tmp_entry)) {
2409: vm_map_unlock(dst_map);
2410: return(KERN_INVALID_ADDRESS);
2411: }
2412: vm_map_clip_start(dst_map, tmp_entry, start);
2413: }
2414: }
2415:
2416: }
2417: vm_map_unlock(dst_map);
2418:
2419: /*
2420: * Throw away the vm_map_copy object
2421: */
2422: vm_map_copy_discard(copy);
2423:
2424: return(KERN_SUCCESS);
2425: }
2426:
2427: /*
2428: * Macro: vm_map_copy_insert
1.1.1.2 root 2429: *
1.1 root 2430: * Description:
2431: * Link a copy chain ("copy") into a map at the
2432: * specified location (after "where").
2433: * Side effects:
2434: * The copy chain is destroyed.
2435: * Warning:
2436: * The arguments are evaluated multiple times.
2437: */
2438: #define vm_map_copy_insert(map, where, copy) \
2439: MACRO_BEGIN \
1.1.1.3 ! root 2440: struct rbtree_node *node, *tmp; \
! 2441: rbtree_for_each_remove(&(copy)->cpy_hdr.tree, node, tmp) \
! 2442: rbtree_insert(&(map)->hdr.tree, node, \
! 2443: vm_map_entry_cmp_insert); \
1.1 root 2444: (((where)->vme_next)->vme_prev = vm_map_copy_last_entry(copy)) \
2445: ->vme_next = ((where)->vme_next); \
2446: ((where)->vme_next = vm_map_copy_first_entry(copy)) \
2447: ->vme_prev = (where); \
2448: (map)->hdr.nentries += (copy)->cpy_hdr.nentries; \
1.1.1.3 ! root 2449: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); \
1.1 root 2450: MACRO_END
2451:
2452: /*
2453: * Routine: vm_map_copyout
2454: *
2455: * Description:
2456: * Copy out a copy chain ("copy") into newly-allocated
2457: * space in the destination map.
2458: *
2459: * If successful, consumes the copy object.
2460: * Otherwise, the caller is responsible for it.
2461: */
2462: kern_return_t vm_map_copyout(dst_map, dst_addr, copy)
2463: register
2464: vm_map_t dst_map;
2465: vm_offset_t *dst_addr; /* OUT */
2466: register
2467: vm_map_copy_t copy;
2468: {
2469: vm_size_t size;
2470: vm_size_t adjustment;
2471: vm_offset_t start;
2472: vm_offset_t vm_copy_start;
2473: vm_map_entry_t last;
2474: register
2475: vm_map_entry_t entry;
2476:
2477: /*
2478: * Check for null copy object.
2479: */
2480:
2481: if (copy == VM_MAP_COPY_NULL) {
2482: *dst_addr = 0;
2483: return(KERN_SUCCESS);
2484: }
2485:
2486: /*
2487: * Check for special copy object, created
2488: * by vm_map_copyin_object.
2489: */
2490:
2491: if (copy->type == VM_MAP_COPY_OBJECT) {
2492: vm_object_t object = copy->cpy_object;
2493: vm_size_t offset = copy->offset;
2494: vm_size_t tmp_size = copy->size;
2495: kern_return_t kr;
2496:
2497: *dst_addr = 0;
2498: kr = vm_map_enter(dst_map, dst_addr, tmp_size,
2499: (vm_offset_t) 0, TRUE,
2500: object, offset, FALSE,
2501: VM_PROT_DEFAULT, VM_PROT_ALL,
2502: VM_INHERIT_DEFAULT);
2503: if (kr != KERN_SUCCESS)
2504: return(kr);
1.1.1.3 ! root 2505: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 2506: return(KERN_SUCCESS);
2507: }
2508:
2509: if (copy->type == VM_MAP_COPY_PAGE_LIST)
2510: return(vm_map_copyout_page_list(dst_map, dst_addr, copy));
2511:
2512: /*
2513: * Find space for the data
2514: */
2515:
2516: vm_copy_start = trunc_page(copy->offset);
2517: size = round_page(copy->offset + copy->size) - vm_copy_start;
2518:
2519: StartAgain: ;
2520:
2521: vm_map_lock(dst_map);
2522: start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)) ?
2523: vm_map_min(dst_map) : last->vme_end;
2524:
2525: while (TRUE) {
2526: vm_map_entry_t next = last->vme_next;
2527: vm_offset_t end = start + size;
2528:
2529: if ((end > dst_map->max_offset) || (end < start)) {
2530: if (dst_map->wait_for_space) {
2531: if (size <= (dst_map->max_offset - dst_map->min_offset)) {
2532: assert_wait((event_t) dst_map, TRUE);
2533: vm_map_unlock(dst_map);
2534: thread_block((void (*)()) 0);
2535: goto StartAgain;
2536: }
2537: }
2538: vm_map_unlock(dst_map);
1.1.1.3 ! root 2539: printf_once("no more room for vm_map_copyout in %p\n", dst_map);
1.1 root 2540: return(KERN_NO_SPACE);
2541: }
2542:
2543: if ((next == vm_map_to_entry(dst_map)) ||
2544: (next->vme_start >= end))
2545: break;
2546:
2547: last = next;
2548: start = last->vme_end;
2549: }
2550:
2551: /*
2552: * Since we're going to just drop the map
2553: * entries from the copy into the destination
2554: * map, they must come from the same pool.
2555: */
2556:
2557: if (copy->cpy_hdr.entries_pageable != dst_map->hdr.entries_pageable) {
2558: /*
2559: * Mismatches occur when dealing with the default
2560: * pager.
2561: */
1.1.1.3 ! root 2562: kmem_cache_t old_cache;
1.1 root 2563: vm_map_entry_t next, new;
2564:
2565: /*
1.1.1.3 ! root 2566: * Find the cache that the copies were allocated from
1.1 root 2567: */
1.1.1.3 ! root 2568: old_cache = (copy->cpy_hdr.entries_pageable)
! 2569: ? &vm_map_entry_cache
! 2570: : &vm_map_kentry_cache;
1.1 root 2571: entry = vm_map_copy_first_entry(copy);
2572:
2573: /*
2574: * Reinitialize the copy so that vm_map_copy_entry_link
2575: * will work.
2576: */
2577: copy->cpy_hdr.nentries = 0;
2578: copy->cpy_hdr.entries_pageable = dst_map->hdr.entries_pageable;
2579: vm_map_copy_first_entry(copy) =
2580: vm_map_copy_last_entry(copy) =
2581: vm_map_copy_to_entry(copy);
2582:
2583: /*
2584: * Copy each entry.
2585: */
2586: while (entry != vm_map_copy_to_entry(copy)) {
2587: new = vm_map_copy_entry_create(copy);
2588: vm_map_entry_copy_full(new, entry);
2589: vm_map_copy_entry_link(copy,
2590: vm_map_copy_last_entry(copy),
2591: new);
2592: next = entry->vme_next;
1.1.1.3 ! root 2593: kmem_cache_free(old_cache, (vm_offset_t) entry);
1.1 root 2594: entry = next;
2595: }
2596: }
2597:
2598: /*
2599: * Adjust the addresses in the copy chain, and
2600: * reset the region attributes.
2601: */
2602:
2603: adjustment = start - vm_copy_start;
2604: for (entry = vm_map_copy_first_entry(copy);
2605: entry != vm_map_copy_to_entry(copy);
2606: entry = entry->vme_next) {
2607: entry->vme_start += adjustment;
2608: entry->vme_end += adjustment;
2609:
2610: entry->inheritance = VM_INHERIT_DEFAULT;
2611: entry->protection = VM_PROT_DEFAULT;
2612: entry->max_protection = VM_PROT_ALL;
2613: entry->projected_on = 0;
2614:
2615: /*
2616: * If the entry is now wired,
2617: * map the pages into the destination map.
2618: */
2619: if (entry->wired_count != 0) {
2620: register vm_offset_t va;
2621: vm_offset_t offset;
2622: register vm_object_t object;
2623:
2624: object = entry->object.vm_object;
2625: offset = entry->offset;
2626: va = entry->vme_start;
2627:
2628: pmap_pageable(dst_map->pmap,
2629: entry->vme_start,
2630: entry->vme_end,
2631: TRUE);
2632:
2633: while (va < entry->vme_end) {
2634: register vm_page_t m;
2635:
2636: /*
2637: * Look up the page in the object.
2638: * Assert that the page will be found in the
2639: * top object:
2640: * either
2641: * the object was newly created by
2642: * vm_object_copy_slowly, and has
2643: * copies of all of the pages from
2644: * the source object
2645: * or
2646: * the object was moved from the old
2647: * map entry; because the old map
2648: * entry was wired, all of the pages
2649: * were in the top-level object.
2650: * (XXX not true if we wire pages for
2651: * reading)
2652: */
2653: vm_object_lock(object);
2654: vm_object_paging_begin(object);
2655:
2656: m = vm_page_lookup(object, offset);
2657: if (m == VM_PAGE_NULL || m->wire_count == 0 ||
2658: m->absent)
2659: panic("vm_map_copyout: wiring 0x%x", m);
2660:
2661: m->busy = TRUE;
2662: vm_object_unlock(object);
2663:
2664: PMAP_ENTER(dst_map->pmap, va, m,
2665: entry->protection, TRUE);
2666:
2667: vm_object_lock(object);
2668: PAGE_WAKEUP_DONE(m);
2669: /* the page is wired, so we don't have to activate */
2670: vm_object_paging_end(object);
2671: vm_object_unlock(object);
2672:
2673: offset += PAGE_SIZE;
2674: va += PAGE_SIZE;
2675: }
2676: }
2677:
2678:
2679: }
2680:
2681: /*
2682: * Correct the page alignment for the result
2683: */
2684:
2685: *dst_addr = start + (copy->offset - vm_copy_start);
2686:
2687: /*
2688: * Update the hints and the map size
2689: */
2690:
2691: if (dst_map->first_free == last)
2692: dst_map->first_free = vm_map_copy_last_entry(copy);
2693: SAVE_HINT(dst_map, vm_map_copy_last_entry(copy));
2694:
2695: dst_map->size += size;
2696:
2697: /*
2698: * Link in the copy
2699: */
2700:
2701: vm_map_copy_insert(dst_map, last, copy);
2702:
2703: vm_map_unlock(dst_map);
2704:
2705: /*
2706: * XXX If wiring_required, call vm_map_pageable
2707: */
2708:
2709: return(KERN_SUCCESS);
2710: }
2711:
2712: /*
2713: *
2714: * vm_map_copyout_page_list:
2715: *
2716: * Version of vm_map_copyout() for page list vm map copies.
2717: *
2718: */
2719: kern_return_t vm_map_copyout_page_list(dst_map, dst_addr, copy)
2720: register
2721: vm_map_t dst_map;
2722: vm_offset_t *dst_addr; /* OUT */
2723: register
2724: vm_map_copy_t copy;
2725: {
2726: vm_size_t size;
2727: vm_offset_t start;
2728: vm_offset_t end;
2729: vm_offset_t offset;
2730: vm_map_entry_t last;
2731: register
2732: vm_object_t object;
2733: vm_page_t *page_list, m;
2734: vm_map_entry_t entry;
2735: vm_offset_t old_last_offset;
2736: boolean_t cont_invoked, needs_wakeup = FALSE;
2737: kern_return_t result = KERN_SUCCESS;
2738: vm_map_copy_t orig_copy;
2739: vm_offset_t dst_offset;
2740: boolean_t must_wire;
2741:
2742: /*
2743: * Make sure the pages are stolen, because we are
2744: * going to put them in a new object. Assume that
2745: * all pages are identical to first in this regard.
2746: */
2747:
2748: page_list = ©->cpy_page_list[0];
2749: if ((*page_list)->tabled)
2750: vm_map_copy_steal_pages(copy);
2751:
2752: /*
2753: * Find space for the data
2754: */
2755:
2756: size = round_page(copy->offset + copy->size) -
2757: trunc_page(copy->offset);
2758: StartAgain:
2759: vm_map_lock(dst_map);
2760: must_wire = dst_map->wiring_required;
2761:
2762: last = dst_map->first_free;
2763: if (last == vm_map_to_entry(dst_map)) {
2764: start = vm_map_min(dst_map);
2765: } else {
2766: start = last->vme_end;
2767: }
2768:
2769: while (TRUE) {
2770: vm_map_entry_t next = last->vme_next;
2771: end = start + size;
2772:
2773: if ((end > dst_map->max_offset) || (end < start)) {
2774: if (dst_map->wait_for_space) {
2775: if (size <= (dst_map->max_offset -
2776: dst_map->min_offset)) {
2777: assert_wait((event_t) dst_map, TRUE);
2778: vm_map_unlock(dst_map);
2779: thread_block((void (*)()) 0);
2780: goto StartAgain;
2781: }
2782: }
2783: vm_map_unlock(dst_map);
1.1.1.3 ! root 2784: printf_once("no more room for vm_map_copyout_page_list in %p\n", dst_map);
1.1 root 2785: return(KERN_NO_SPACE);
2786: }
2787:
2788: if ((next == vm_map_to_entry(dst_map)) ||
2789: (next->vme_start >= end)) {
2790: break;
2791: }
2792:
2793: last = next;
2794: start = last->vme_end;
2795: }
2796:
2797: /*
2798: * See whether we can avoid creating a new entry (and object) by
2799: * extending one of our neighbors. [So far, we only attempt to
2800: * extend from below.]
2801: *
2802: * The code path below here is a bit twisted. If any of the
2803: * extension checks fails, we branch to create_object. If
2804: * it all works, we fall out the bottom and goto insert_pages.
2805: */
2806: if (last == vm_map_to_entry(dst_map) ||
2807: last->vme_end != start ||
2808: last->is_shared != FALSE ||
2809: last->is_sub_map != FALSE ||
2810: last->inheritance != VM_INHERIT_DEFAULT ||
2811: last->protection != VM_PROT_DEFAULT ||
2812: last->max_protection != VM_PROT_ALL ||
2813: (must_wire ? (last->wired_count != 1 ||
2814: last->user_wired_count != 1) :
2815: (last->wired_count != 0))) {
2816: goto create_object;
2817: }
1.1.1.2 root 2818:
1.1 root 2819: /*
2820: * If this entry needs an object, make one.
2821: */
2822: if (last->object.vm_object == VM_OBJECT_NULL) {
2823: object = vm_object_allocate(
2824: (vm_size_t)(last->vme_end - last->vme_start + size));
2825: last->object.vm_object = object;
2826: last->offset = 0;
2827: vm_object_lock(object);
2828: }
2829: else {
2830: vm_offset_t prev_offset = last->offset;
2831: vm_size_t prev_size = start - last->vme_start;
2832: vm_size_t new_size;
2833:
2834: /*
2835: * This is basically vm_object_coalesce.
2836: */
2837:
2838: object = last->object.vm_object;
2839: vm_object_lock(object);
2840:
2841: /*
2842: * Try to collapse the object first
2843: */
2844: vm_object_collapse(object);
2845:
2846: /*
2847: * Can't coalesce if pages not mapped to
2848: * last may be in use anyway:
2849: * . more than one reference
2850: * . paged out
2851: * . shadows another object
2852: * . has a copy elsewhere
2853: * . paging references (pages might be in page-list)
2854: */
2855:
2856: if ((object->ref_count > 1) ||
2857: object->pager_created ||
2858: (object->shadow != VM_OBJECT_NULL) ||
2859: (object->copy != VM_OBJECT_NULL) ||
2860: (object->paging_in_progress != 0)) {
2861: vm_object_unlock(object);
2862: goto create_object;
2863: }
2864:
2865: /*
2866: * Extend the object if necessary. Don't have to call
2867: * vm_object_page_remove because the pages aren't mapped,
2868: * and vm_page_replace will free up any old ones it encounters.
2869: */
2870: new_size = prev_offset + prev_size + size;
2871: if (new_size > object->size)
2872: object->size = new_size;
2873: }
2874:
2875: /*
2876: * Coalesced the two objects - can extend
2877: * the previous map entry to include the
2878: * new range.
2879: */
2880: dst_map->size += size;
2881: last->vme_end = end;
2882:
2883: SAVE_HINT(dst_map, last);
2884:
2885: goto insert_pages;
2886:
2887: create_object:
2888:
2889: /*
2890: * Create object
2891: */
2892: object = vm_object_allocate(size);
2893:
2894: /*
2895: * Create entry
2896: */
2897:
2898: entry = vm_map_entry_create(dst_map);
2899:
2900: entry->object.vm_object = object;
2901: entry->offset = 0;
2902:
2903: entry->is_shared = FALSE;
2904: entry->is_sub_map = FALSE;
2905: entry->needs_copy = FALSE;
2906:
2907: if (must_wire) {
2908: entry->wired_count = 1;
2909: entry->user_wired_count = 1;
2910: } else {
2911: entry->wired_count = 0;
2912: entry->user_wired_count = 0;
2913: }
2914:
2915: entry->in_transition = TRUE;
2916: entry->needs_wakeup = FALSE;
2917:
2918: entry->vme_start = start;
2919: entry->vme_end = start + size;
1.1.1.2 root 2920:
1.1 root 2921: entry->inheritance = VM_INHERIT_DEFAULT;
2922: entry->protection = VM_PROT_DEFAULT;
2923: entry->max_protection = VM_PROT_ALL;
2924: entry->projected_on = 0;
2925:
2926: vm_object_lock(object);
2927:
2928: /*
2929: * Update the hints and the map size
2930: */
2931: if (dst_map->first_free == last) {
2932: dst_map->first_free = entry;
2933: }
2934: SAVE_HINT(dst_map, entry);
2935: dst_map->size += size;
2936:
2937: /*
2938: * Link in the entry
2939: */
2940: vm_map_entry_link(dst_map, last, entry);
2941: last = entry;
2942:
2943: /*
1.1.1.2 root 2944: * Transfer pages into new object.
1.1 root 2945: * Scan page list in vm_map_copy.
2946: */
2947: insert_pages:
2948: dst_offset = copy->offset & PAGE_MASK;
2949: cont_invoked = FALSE;
2950: orig_copy = copy;
2951: last->in_transition = TRUE;
2952: old_last_offset = last->offset
2953: + (start - last->vme_start);
2954:
2955: vm_page_lock_queues();
2956:
2957: for (offset = 0; offset < size; offset += PAGE_SIZE) {
2958: m = *page_list;
2959: assert(m && !m->tabled);
2960:
2961: /*
2962: * Must clear busy bit in page before inserting it.
2963: * Ok to skip wakeup logic because nobody else
2964: * can possibly know about this page.
2965: * The page is dirty in its new object.
2966: */
2967:
2968: assert(!m->wanted);
2969:
2970: m->busy = FALSE;
2971: m->dirty = TRUE;
2972: vm_page_replace(m, object, old_last_offset + offset);
2973: if (must_wire) {
2974: vm_page_wire(m);
2975: PMAP_ENTER(dst_map->pmap,
2976: last->vme_start + m->offset - last->offset,
2977: m, last->protection, TRUE);
2978: } else {
2979: vm_page_activate(m);
2980: }
2981:
2982: *page_list++ = VM_PAGE_NULL;
2983: if (--(copy->cpy_npages) == 0 &&
2984: vm_map_copy_has_cont(copy)) {
2985: vm_map_copy_t new_copy;
2986:
2987: /*
2988: * Ok to unlock map because entry is
2989: * marked in_transition.
2990: */
2991: cont_invoked = TRUE;
2992: vm_page_unlock_queues();
2993: vm_object_unlock(object);
2994: vm_map_unlock(dst_map);
2995: vm_map_copy_invoke_cont(copy, &new_copy, &result);
2996:
2997: if (result == KERN_SUCCESS) {
2998:
2999: /*
3000: * If we got back a copy with real pages,
3001: * steal them now. Either all of the
3002: * pages in the list are tabled or none
3003: * of them are; mixtures are not possible.
3004: *
3005: * Save original copy for consume on
3006: * success logic at end of routine.
3007: */
3008: if (copy != orig_copy)
3009: vm_map_copy_discard(copy);
3010:
3011: if ((copy = new_copy) != VM_MAP_COPY_NULL) {
3012: page_list = ©->cpy_page_list[0];
3013: if ((*page_list)->tabled)
3014: vm_map_copy_steal_pages(copy);
3015: }
3016: }
3017: else {
3018: /*
3019: * Continuation failed.
3020: */
3021: vm_map_lock(dst_map);
3022: goto error;
3023: }
3024:
3025: vm_map_lock(dst_map);
3026: vm_object_lock(object);
3027: vm_page_lock_queues();
3028: }
3029: }
3030:
3031: vm_page_unlock_queues();
3032: vm_object_unlock(object);
3033:
3034: *dst_addr = start + dst_offset;
1.1.1.2 root 3035:
1.1 root 3036: /*
3037: * Clear the in transition bits. This is easy if we
3038: * didn't have a continuation.
3039: */
3040: error:
3041: if (!cont_invoked) {
3042: /*
3043: * We didn't unlock the map, so nobody could
3044: * be waiting.
3045: */
3046: last->in_transition = FALSE;
3047: assert(!last->needs_wakeup);
3048: needs_wakeup = FALSE;
3049: }
3050: else {
3051: if (!vm_map_lookup_entry(dst_map, start, &entry))
3052: panic("vm_map_copyout_page_list: missing entry");
3053:
3054: /*
3055: * Clear transition bit for all constituent entries that
3056: * were in the original entry. Also check for waiters.
3057: */
3058: while((entry != vm_map_to_entry(dst_map)) &&
3059: (entry->vme_start < end)) {
3060: assert(entry->in_transition);
3061: entry->in_transition = FALSE;
3062: if(entry->needs_wakeup) {
3063: entry->needs_wakeup = FALSE;
3064: needs_wakeup = TRUE;
3065: }
3066: entry = entry->vme_next;
3067: }
3068: }
1.1.1.2 root 3069:
1.1 root 3070: if (result != KERN_SUCCESS)
3071: vm_map_delete(dst_map, start, end);
3072:
3073: vm_map_unlock(dst_map);
3074:
3075: if (needs_wakeup)
3076: vm_map_entry_wakeup(dst_map);
3077:
3078: /*
3079: * Consume on success logic.
3080: */
3081: if (copy != orig_copy) {
1.1.1.3 ! root 3082: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1.1 root 3083: }
3084: if (result == KERN_SUCCESS) {
1.1.1.3 ! root 3085: kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) orig_copy);
1.1 root 3086: }
1.1.1.2 root 3087:
1.1 root 3088: return(result);
3089: }
3090:
3091: /*
3092: * Routine: vm_map_copyin
3093: *
3094: * Description:
3095: * Copy the specified region (src_addr, len) from the
3096: * source address space (src_map), possibly removing
3097: * the region from the source address space (src_destroy).
3098: *
3099: * Returns:
3100: * A vm_map_copy_t object (copy_result), suitable for
3101: * insertion into another address space (using vm_map_copyout),
3102: * copying over another address space region (using
3103: * vm_map_copy_overwrite). If the copy is unused, it
3104: * should be destroyed (using vm_map_copy_discard).
3105: *
3106: * In/out conditions:
3107: * The source map should not be locked on entry.
3108: */
3109: kern_return_t vm_map_copyin(src_map, src_addr, len, src_destroy, copy_result)
3110: vm_map_t src_map;
3111: vm_offset_t src_addr;
3112: vm_size_t len;
3113: boolean_t src_destroy;
3114: vm_map_copy_t *copy_result; /* OUT */
3115: {
3116: vm_map_entry_t tmp_entry; /* Result of last map lookup --
3117: * in multi-level lookup, this
3118: * entry contains the actual
3119: * vm_object/offset.
3120: */
3121:
3122: vm_offset_t src_start; /* Start of current entry --
3123: * where copy is taking place now
3124: */
3125: vm_offset_t src_end; /* End of entire region to be
3126: * copied */
3127:
3128: register
3129: vm_map_copy_t copy; /* Resulting copy */
3130:
3131: /*
3132: * Check for copies of zero bytes.
3133: */
3134:
3135: if (len == 0) {
3136: *copy_result = VM_MAP_COPY_NULL;
3137: return(KERN_SUCCESS);
3138: }
3139:
3140: /*
3141: * Compute start and end of region
3142: */
3143:
3144: src_start = trunc_page(src_addr);
3145: src_end = round_page(src_addr + len);
3146:
3147: /*
3148: * Check that the end address doesn't overflow
3149: */
3150:
3151: if (src_end <= src_start)
3152: if ((src_end < src_start) || (src_start != 0))
3153: return(KERN_INVALID_ADDRESS);
3154:
3155: /*
3156: * Allocate a header element for the list.
3157: *
1.1.1.2 root 3158: * Use the start and end in the header to
1.1 root 3159: * remember the endpoints prior to rounding.
3160: */
3161:
1.1.1.3 ! root 3162: copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 3163: vm_map_copy_first_entry(copy) =
3164: vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy);
3165: copy->type = VM_MAP_COPY_ENTRY_LIST;
3166: copy->cpy_hdr.nentries = 0;
3167: copy->cpy_hdr.entries_pageable = TRUE;
1.1.1.3 ! root 3168: rbtree_init(©->cpy_hdr.tree);
1.1 root 3169:
3170: copy->offset = src_addr;
3171: copy->size = len;
1.1.1.2 root 3172:
1.1 root 3173: #define RETURN(x) \
3174: MACRO_BEGIN \
3175: vm_map_unlock(src_map); \
3176: vm_map_copy_discard(copy); \
3177: MACRO_RETURN(x); \
3178: MACRO_END
3179:
3180: /*
3181: * Find the beginning of the region.
3182: */
3183:
3184: vm_map_lock(src_map);
3185:
3186: if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry))
3187: RETURN(KERN_INVALID_ADDRESS);
3188: vm_map_clip_start(src_map, tmp_entry, src_start);
3189:
3190: /*
3191: * Go through entries until we get to the end.
3192: */
3193:
3194: while (TRUE) {
3195: register
3196: vm_map_entry_t src_entry = tmp_entry; /* Top-level entry */
3197: vm_size_t src_size; /* Size of source
3198: * map entry (in both
3199: * maps)
3200: */
3201:
3202: register
3203: vm_object_t src_object; /* Object to copy */
3204: vm_offset_t src_offset;
3205:
3206: boolean_t src_needs_copy; /* Should source map
3207: * be made read-only
3208: * for copy-on-write?
3209: */
3210:
3211: register
3212: vm_map_entry_t new_entry; /* Map entry for copy */
3213: boolean_t new_entry_needs_copy; /* Will new entry be COW? */
3214:
3215: boolean_t was_wired; /* Was source wired? */
3216: vm_map_version_t version; /* Version before locks
3217: * dropped to make copy
3218: */
3219:
3220: /*
3221: * Verify that the region can be read.
3222: */
3223:
3224: if (! (src_entry->protection & VM_PROT_READ))
3225: RETURN(KERN_PROTECTION_FAILURE);
3226:
3227: /*
3228: * Clip against the endpoints of the entire region.
3229: */
3230:
3231: vm_map_clip_end(src_map, src_entry, src_end);
3232:
3233: src_size = src_entry->vme_end - src_start;
3234: src_object = src_entry->object.vm_object;
3235: src_offset = src_entry->offset;
3236: was_wired = (src_entry->wired_count != 0);
3237:
3238: /*
3239: * Create a new address map entry to
3240: * hold the result. Fill in the fields from
3241: * the appropriate source entries.
3242: */
3243:
3244: new_entry = vm_map_copy_entry_create(copy);
3245: vm_map_entry_copy(new_entry, src_entry);
3246:
3247: /*
3248: * Attempt non-blocking copy-on-write optimizations.
3249: */
3250:
3251: if (src_destroy &&
3252: (src_object == VM_OBJECT_NULL ||
3253: (src_object->temporary && !src_object->use_shared_copy)))
3254: {
3255: /*
3256: * If we are destroying the source, and the object
3257: * is temporary, and not shared writable,
3258: * we can move the object reference
3259: * from the source to the copy. The copy is
3260: * copy-on-write only if the source is.
3261: * We make another reference to the object, because
3262: * destroying the source entry will deallocate it.
3263: */
3264: vm_object_reference(src_object);
3265:
3266: /*
3267: * Copy is always unwired. vm_map_copy_entry
3268: * set its wired count to zero.
3269: */
3270:
3271: goto CopySuccessful;
3272: }
3273:
3274: if (!was_wired &&
3275: vm_object_copy_temporary(
3276: &new_entry->object.vm_object,
3277: &new_entry->offset,
3278: &src_needs_copy,
3279: &new_entry_needs_copy)) {
3280:
3281: new_entry->needs_copy = new_entry_needs_copy;
3282:
3283: /*
3284: * Handle copy-on-write obligations
3285: */
3286:
3287: if (src_needs_copy && !tmp_entry->needs_copy) {
3288: vm_object_pmap_protect(
3289: src_object,
3290: src_offset,
3291: src_size,
3292: (src_entry->is_shared ? PMAP_NULL
3293: : src_map->pmap),
3294: src_entry->vme_start,
3295: src_entry->protection &
3296: ~VM_PROT_WRITE);
3297:
3298: tmp_entry->needs_copy = TRUE;
3299: }
3300:
3301: /*
3302: * The map has never been unlocked, so it's safe to
3303: * move to the next entry rather than doing another
3304: * lookup.
3305: */
3306:
3307: goto CopySuccessful;
3308: }
3309:
3310: new_entry->needs_copy = FALSE;
3311:
3312: /*
3313: * Take an object reference, so that we may
3314: * release the map lock(s).
3315: */
3316:
3317: assert(src_object != VM_OBJECT_NULL);
3318: vm_object_reference(src_object);
3319:
3320: /*
3321: * Record the timestamp for later verification.
3322: * Unlock the map.
3323: */
3324:
3325: version.main_timestamp = src_map->timestamp;
3326: vm_map_unlock(src_map);
3327:
3328: /*
3329: * Perform the copy
3330: */
3331:
3332: if (was_wired) {
3333: vm_object_lock(src_object);
3334: (void) vm_object_copy_slowly(
3335: src_object,
3336: src_offset,
3337: src_size,
3338: FALSE,
3339: &new_entry->object.vm_object);
3340: new_entry->offset = 0;
3341: new_entry->needs_copy = FALSE;
3342: } else {
3343: kern_return_t result;
3344:
3345: result = vm_object_copy_strategically(src_object,
3346: src_offset,
3347: src_size,
3348: &new_entry->object.vm_object,
3349: &new_entry->offset,
3350: &new_entry_needs_copy);
3351:
3352: new_entry->needs_copy = new_entry_needs_copy;
1.1.1.2 root 3353:
1.1 root 3354:
3355: if (result != KERN_SUCCESS) {
3356: vm_map_copy_entry_dispose(copy, new_entry);
3357:
3358: vm_map_lock(src_map);
3359: RETURN(result);
3360: }
3361:
3362: }
3363:
3364: /*
3365: * Throw away the extra reference
3366: */
3367:
3368: vm_object_deallocate(src_object);
3369:
3370: /*
3371: * Verify that the map has not substantially
3372: * changed while the copy was being made.
3373: */
3374:
3375: vm_map_lock(src_map); /* Increments timestamp once! */
3376:
3377: if ((version.main_timestamp + 1) == src_map->timestamp)
3378: goto CopySuccessful;
3379:
3380: /*
3381: * Simple version comparison failed.
3382: *
3383: * Retry the lookup and verify that the
3384: * same object/offset are still present.
3385: *
3386: * [Note: a memory manager that colludes with
3387: * the calling task can detect that we have
3388: * cheated. While the map was unlocked, the
3389: * mapping could have been changed and restored.]
3390: */
3391:
3392: if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry)) {
3393: vm_map_copy_entry_dispose(copy, new_entry);
3394: RETURN(KERN_INVALID_ADDRESS);
3395: }
3396:
3397: src_entry = tmp_entry;
3398: vm_map_clip_start(src_map, src_entry, src_start);
3399:
3400: if ((src_entry->protection & VM_PROT_READ) == VM_PROT_NONE)
3401: goto VerificationFailed;
3402:
3403: if (src_entry->vme_end < new_entry->vme_end)
3404: src_size = (new_entry->vme_end = src_entry->vme_end) - src_start;
3405:
3406: if ((src_entry->object.vm_object != src_object) ||
3407: (src_entry->offset != src_offset) ) {
3408:
3409: /*
3410: * Verification failed.
3411: *
3412: * Start over with this top-level entry.
3413: */
3414:
3415: VerificationFailed: ;
3416:
3417: vm_object_deallocate(new_entry->object.vm_object);
3418: vm_map_copy_entry_dispose(copy, new_entry);
3419: tmp_entry = src_entry;
3420: continue;
3421: }
3422:
3423: /*
3424: * Verification succeeded.
3425: */
3426:
3427: CopySuccessful: ;
3428:
3429: /*
3430: * Link in the new copy entry.
3431: */
3432:
3433: vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy),
3434: new_entry);
1.1.1.2 root 3435:
1.1 root 3436: /*
3437: * Determine whether the entire region
3438: * has been copied.
3439: */
3440: src_start = new_entry->vme_end;
3441: if ((src_start >= src_end) && (src_end != 0))
3442: break;
3443:
3444: /*
3445: * Verify that there are no gaps in the region
3446: */
3447:
3448: tmp_entry = src_entry->vme_next;
3449: if (tmp_entry->vme_start != src_start)
3450: RETURN(KERN_INVALID_ADDRESS);
3451: }
3452:
3453: /*
3454: * If the source should be destroyed, do it now, since the
1.1.1.2 root 3455: * copy was successful.
1.1 root 3456: */
3457: if (src_destroy)
3458: (void) vm_map_delete(src_map, trunc_page(src_addr), src_end);
3459:
3460: vm_map_unlock(src_map);
3461:
3462: *copy_result = copy;
3463: return(KERN_SUCCESS);
3464:
3465: #undef RETURN
3466: }
3467:
3468: /*
3469: * vm_map_copyin_object:
3470: *
3471: * Create a copy object from an object.
3472: * Our caller donates an object reference.
3473: */
3474:
3475: kern_return_t vm_map_copyin_object(object, offset, size, copy_result)
3476: vm_object_t object;
3477: vm_offset_t offset; /* offset of region in object */
3478: vm_size_t size; /* size of region in object */
3479: vm_map_copy_t *copy_result; /* OUT */
3480: {
3481: vm_map_copy_t copy; /* Resulting copy */
3482:
3483: /*
3484: * We drop the object into a special copy object
3485: * that contains the object directly. These copy objects
3486: * are distinguished by entries_pageable == FALSE
3487: * and null links.
3488: */
3489:
1.1.1.3 ! root 3490: copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 3491: vm_map_copy_first_entry(copy) =
3492: vm_map_copy_last_entry(copy) = VM_MAP_ENTRY_NULL;
3493: copy->type = VM_MAP_COPY_OBJECT;
3494: copy->cpy_object = object;
3495: copy->offset = offset;
3496: copy->size = size;
3497:
3498: *copy_result = copy;
3499: return(KERN_SUCCESS);
3500: }
3501:
3502: /*
3503: * vm_map_copyin_page_list_cont:
3504: *
3505: * Continuation routine for vm_map_copyin_page_list.
1.1.1.2 root 3506: *
1.1 root 3507: * If vm_map_copyin_page_list can't fit the entire vm range
3508: * into a single page list object, it creates a continuation.
3509: * When the target of the operation has used the pages in the
3510: * initial page list, it invokes the continuation, which calls
3511: * this routine. If an error happens, the continuation is aborted
3512: * (abort arg to this routine is TRUE). To avoid deadlocks, the
3513: * pages are discarded from the initial page list before invoking
3514: * the continuation.
3515: *
3516: * NOTE: This is not the same sort of continuation used by
3517: * the scheduler.
3518: */
3519:
3520: kern_return_t vm_map_copyin_page_list_cont(cont_args, copy_result)
3521: vm_map_copyin_args_t cont_args;
3522: vm_map_copy_t *copy_result; /* OUT */
3523: {
3524: kern_return_t result = 0; /* '=0' to quiet gcc warnings */
3525: register boolean_t do_abort, src_destroy, src_destroy_only;
3526:
3527: /*
3528: * Check for cases that only require memory destruction.
3529: */
3530: do_abort = (copy_result == (vm_map_copy_t *) 0);
3531: src_destroy = (cont_args->destroy_len != (vm_size_t) 0);
3532: src_destroy_only = (cont_args->src_len == (vm_size_t) 0);
3533:
3534: if (do_abort || src_destroy_only) {
1.1.1.2 root 3535: if (src_destroy)
1.1 root 3536: result = vm_map_remove(cont_args->map,
3537: cont_args->destroy_addr,
3538: cont_args->destroy_addr + cont_args->destroy_len);
3539: if (!do_abort)
3540: *copy_result = VM_MAP_COPY_NULL;
3541: }
3542: else {
3543: result = vm_map_copyin_page_list(cont_args->map,
3544: cont_args->src_addr, cont_args->src_len, src_destroy,
3545: cont_args->steal_pages, copy_result, TRUE);
3546:
3547: if (src_destroy && !cont_args->steal_pages &&
3548: vm_map_copy_has_cont(*copy_result)) {
3549: vm_map_copyin_args_t new_args;
3550: /*
3551: * Transfer old destroy info.
3552: */
3553: new_args = (vm_map_copyin_args_t)
3554: (*copy_result)->cpy_cont_args;
3555: new_args->destroy_addr = cont_args->destroy_addr;
3556: new_args->destroy_len = cont_args->destroy_len;
3557: }
3558: }
1.1.1.2 root 3559:
1.1 root 3560: vm_map_deallocate(cont_args->map);
3561: kfree((vm_offset_t)cont_args, sizeof(vm_map_copyin_args_data_t));
3562:
3563: return(result);
3564: }
3565:
3566: /*
3567: * vm_map_copyin_page_list:
3568: *
3569: * This is a variant of vm_map_copyin that copies in a list of pages.
3570: * If steal_pages is TRUE, the pages are only in the returned list.
3571: * If steal_pages is FALSE, the pages are busy and still in their
3572: * objects. A continuation may be returned if not all the pages fit:
3573: * the recipient of this copy_result must be prepared to deal with it.
3574: */
3575:
3576: kern_return_t vm_map_copyin_page_list(src_map, src_addr, len, src_destroy,
3577: steal_pages, copy_result, is_cont)
3578: vm_map_t src_map;
3579: vm_offset_t src_addr;
3580: vm_size_t len;
3581: boolean_t src_destroy;
3582: boolean_t steal_pages;
3583: vm_map_copy_t *copy_result; /* OUT */
3584: boolean_t is_cont;
3585: {
3586: vm_map_entry_t src_entry;
3587: vm_page_t m;
3588: vm_offset_t src_start;
3589: vm_offset_t src_end;
3590: vm_size_t src_size;
3591: register
3592: vm_object_t src_object;
3593: register
3594: vm_offset_t src_offset;
3595: vm_offset_t src_last_offset;
3596: register
3597: vm_map_copy_t copy; /* Resulting copy */
3598: kern_return_t result = KERN_SUCCESS;
3599: boolean_t need_map_lookup;
3600: vm_map_copyin_args_t cont_args;
3601:
3602: /*
3603: * If steal_pages is FALSE, this leaves busy pages in
3604: * the object. A continuation must be used if src_destroy
3605: * is true in this case (!steal_pages && src_destroy).
3606: *
3607: * XXX Still have a more general problem of what happens
3608: * XXX if the same page occurs twice in a list. Deadlock
3609: * XXX can happen if vm_fault_page was called. A
3610: * XXX possible solution is to use a continuation if vm_fault_page
3611: * XXX is called and we cross a map entry boundary.
3612: */
3613:
3614: /*
3615: * Check for copies of zero bytes.
3616: */
3617:
3618: if (len == 0) {
3619: *copy_result = VM_MAP_COPY_NULL;
3620: return(KERN_SUCCESS);
3621: }
3622:
3623: /*
3624: * Compute start and end of region
3625: */
3626:
3627: src_start = trunc_page(src_addr);
3628: src_end = round_page(src_addr + len);
3629:
3630: /*
3631: * Check that the end address doesn't overflow
3632: */
3633:
3634: if (src_end <= src_start && (src_end < src_start || src_start != 0)) {
3635: return KERN_INVALID_ADDRESS;
3636: }
3637:
3638: /*
3639: * Allocate a header element for the page list.
3640: *
3641: * Record original offset and size, as caller may not
3642: * be page-aligned.
3643: */
3644:
1.1.1.3 ! root 3645: copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1.1 root 3646: copy->type = VM_MAP_COPY_PAGE_LIST;
3647: copy->cpy_npages = 0;
3648: copy->offset = src_addr;
3649: copy->size = len;
3650: copy->cpy_cont = ((kern_return_t (*)()) 0);
3651: copy->cpy_cont_args = (char *) VM_MAP_COPYIN_ARGS_NULL;
1.1.1.2 root 3652:
1.1 root 3653: /*
3654: * Find the beginning of the region.
3655: */
3656:
3657: do_map_lookup:
3658:
3659: vm_map_lock(src_map);
3660:
3661: if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
3662: result = KERN_INVALID_ADDRESS;
3663: goto error;
3664: }
3665: need_map_lookup = FALSE;
3666:
3667: /*
3668: * Go through entries until we get to the end.
3669: */
3670:
3671: while (TRUE) {
3672:
3673: if (! (src_entry->protection & VM_PROT_READ)) {
3674: result = KERN_PROTECTION_FAILURE;
3675: goto error;
3676: }
3677:
3678: if (src_end > src_entry->vme_end)
3679: src_size = src_entry->vme_end - src_start;
3680: else
3681: src_size = src_end - src_start;
3682:
3683: src_object = src_entry->object.vm_object;
3684: src_offset = src_entry->offset +
3685: (src_start - src_entry->vme_start);
3686:
3687: /*
3688: * If src_object is NULL, allocate it now;
3689: * we're going to fault on it shortly.
3690: */
3691: if (src_object == VM_OBJECT_NULL) {
3692: src_object = vm_object_allocate((vm_size_t)
3693: src_entry->vme_end -
3694: src_entry->vme_start);
3695: src_entry->object.vm_object = src_object;
3696: }
3697:
3698: /*
3699: * Iterate over pages. Fault in ones that aren't present.
3700: */
3701: src_last_offset = src_offset + src_size;
3702: for (; (src_offset < src_last_offset && !need_map_lookup);
3703: src_offset += PAGE_SIZE, src_start += PAGE_SIZE) {
3704:
3705: if (copy->cpy_npages == VM_MAP_COPY_PAGE_LIST_MAX) {
3706: make_continuation:
3707: /*
3708: * At this point we have the max number of
3709: * pages busy for this thread that we're
3710: * willing to allow. Stop here and record
3711: * arguments for the remainder. Note:
3712: * this means that this routine isn't atomic,
3713: * but that's the breaks. Note that only
3714: * the first vm_map_copy_t that comes back
3715: * from this routine has the right offset
3716: * and size; those from continuations are
3717: * page rounded, and short by the amount
3718: * already done.
3719: *
3720: * Reset src_end so the src_destroy
3721: * code at the bottom doesn't do
3722: * something stupid.
3723: */
3724:
1.1.1.2 root 3725: cont_args = (vm_map_copyin_args_t)
1.1 root 3726: kalloc(sizeof(vm_map_copyin_args_data_t));
3727: cont_args->map = src_map;
3728: vm_map_reference(src_map);
3729: cont_args->src_addr = src_start;
3730: cont_args->src_len = len - (src_start - src_addr);
3731: if (src_destroy) {
3732: cont_args->destroy_addr = cont_args->src_addr;
3733: cont_args->destroy_len = cont_args->src_len;
3734: }
3735: else {
3736: cont_args->destroy_addr = (vm_offset_t) 0;
3737: cont_args->destroy_len = (vm_offset_t) 0;
3738: }
3739: cont_args->steal_pages = steal_pages;
3740:
3741: copy->cpy_cont_args = (char *) cont_args;
3742: copy->cpy_cont = vm_map_copyin_page_list_cont;
3743:
3744: src_end = src_start;
3745: vm_map_clip_end(src_map, src_entry, src_end);
3746: break;
3747: }
3748:
3749: /*
3750: * Try to find the page of data.
3751: */
3752: vm_object_lock(src_object);
3753: vm_object_paging_begin(src_object);
3754: if (((m = vm_page_lookup(src_object, src_offset)) !=
3755: VM_PAGE_NULL) && !m->busy && !m->fictitious &&
3756: !m->absent && !m->error) {
3757:
3758: /*
3759: * This is the page. Mark it busy
3760: * and keep the paging reference on
3761: * the object whilst we do our thing.
3762: */
3763: m->busy = TRUE;
3764:
3765: /*
3766: * Also write-protect the page, so
3767: * that the map`s owner cannot change
3768: * the data. The busy bit will prevent
3769: * faults on the page from succeeding
3770: * until the copy is released; after
3771: * that, the page can be re-entered
3772: * as writable, since we didn`t alter
3773: * the map entry. This scheme is a
3774: * cheap copy-on-write.
3775: *
3776: * Don`t forget the protection and
3777: * the page_lock value!
3778: *
3779: * If the source is being destroyed
3780: * AND not shared writable, we don`t
3781: * have to protect the page, since
3782: * we will destroy the (only)
3783: * writable mapping later.
3784: */
3785: if (!src_destroy ||
3786: src_object->use_shared_copy)
3787: {
3788: pmap_page_protect(m->phys_addr,
3789: src_entry->protection
3790: & ~m->page_lock
3791: & ~VM_PROT_WRITE);
3792: }
3793:
3794: }
3795: else {
3796: vm_prot_t result_prot;
3797: vm_page_t top_page;
3798: kern_return_t kr;
1.1.1.2 root 3799:
1.1 root 3800: /*
3801: * Have to fault the page in; must
3802: * unlock the map to do so. While
3803: * the map is unlocked, anything
3804: * can happen, we must lookup the
3805: * map entry before continuing.
3806: */
3807: vm_map_unlock(src_map);
3808: need_map_lookup = TRUE;
3809: retry:
3810: result_prot = VM_PROT_READ;
1.1.1.2 root 3811:
1.1 root 3812: kr = vm_fault_page(src_object, src_offset,
3813: VM_PROT_READ, FALSE, FALSE,
3814: &result_prot, &m, &top_page,
3815: FALSE, (void (*)()) 0);
3816: /*
3817: * Cope with what happened.
3818: */
3819: switch (kr) {
3820: case VM_FAULT_SUCCESS:
3821: break;
3822: case VM_FAULT_INTERRUPTED: /* ??? */
3823: case VM_FAULT_RETRY:
3824: vm_object_lock(src_object);
3825: vm_object_paging_begin(src_object);
3826: goto retry;
3827: case VM_FAULT_MEMORY_SHORTAGE:
3828: VM_PAGE_WAIT((void (*)()) 0);
3829: vm_object_lock(src_object);
3830: vm_object_paging_begin(src_object);
3831: goto retry;
3832: case VM_FAULT_FICTITIOUS_SHORTAGE:
3833: vm_page_more_fictitious();
3834: vm_object_lock(src_object);
3835: vm_object_paging_begin(src_object);
3836: goto retry;
3837: case VM_FAULT_MEMORY_ERROR:
3838: /*
3839: * Something broke. If this
3840: * is a continuation, return
3841: * a partial result if possible,
3842: * else fail the whole thing.
3843: * In the continuation case, the
3844: * next continuation call will
3845: * get this error if it persists.
3846: */
3847: vm_map_lock(src_map);
3848: if (is_cont &&
3849: copy->cpy_npages != 0)
3850: goto make_continuation;
3851:
3852: result = KERN_MEMORY_ERROR;
3853: goto error;
3854: }
1.1.1.2 root 3855:
1.1 root 3856: if (top_page != VM_PAGE_NULL) {
3857: vm_object_lock(src_object);
3858: VM_PAGE_FREE(top_page);
3859: vm_object_paging_end(src_object);
3860: vm_object_unlock(src_object);
3861: }
3862:
3863: /*
3864: * We do not need to write-protect
3865: * the page, since it cannot have
3866: * been in the pmap (and we did not
3867: * enter it above). The busy bit
3868: * will protect the page from being
3869: * entered as writable until it is
3870: * unlocked.
3871: */
3872:
3873: }
3874:
3875: /*
3876: * The page is busy, its object is locked, and
3877: * we have a paging reference on it. Either
3878: * the map is locked, or need_map_lookup is
3879: * TRUE.
1.1.1.2 root 3880: *
1.1 root 3881: * Put the page in the page list.
3882: */
3883: copy->cpy_page_list[copy->cpy_npages++] = m;
3884: vm_object_unlock(m->object);
3885: }
1.1.1.2 root 3886:
1.1 root 3887: /*
3888: * DETERMINE whether the entire region
3889: * has been copied.
3890: */
3891: if (src_start >= src_end && src_end != 0) {
3892: if (need_map_lookup)
3893: vm_map_lock(src_map);
3894: break;
3895: }
3896:
3897: /*
3898: * If need_map_lookup is TRUE, have to start over with
3899: * another map lookup. Note that we dropped the map
3900: * lock (to call vm_fault_page) above only in this case.
3901: */
3902: if (need_map_lookup)
3903: goto do_map_lookup;
3904:
3905: /*
3906: * Verify that there are no gaps in the region
3907: */
3908:
3909: src_start = src_entry->vme_end;
3910: src_entry = src_entry->vme_next;
3911: if (src_entry->vme_start != src_start) {
3912: result = KERN_INVALID_ADDRESS;
3913: goto error;
3914: }
3915: }
3916:
3917: /*
3918: * If steal_pages is true, make sure all
3919: * pages in the copy are not in any object
3920: * We try to remove them from the original
3921: * object, but we may have to copy them.
3922: *
3923: * At this point every page in the list is busy
3924: * and holds a paging reference to its object.
3925: * When we're done stealing, every page is busy,
3926: * and in no object (m->tabled == FALSE).
3927: */
3928: src_start = trunc_page(src_addr);
3929: if (steal_pages) {
3930: register int i;
3931: vm_offset_t unwire_end;
3932:
3933: unwire_end = src_start;
3934: for (i = 0; i < copy->cpy_npages; i++) {
3935:
3936: /*
3937: * Remove the page from its object if it
3938: * can be stolen. It can be stolen if:
3939: *
1.1.1.2 root 3940: * (1) The source is being destroyed,
1.1 root 3941: * the object is temporary, and
3942: * not shared.
3943: * (2) The page is not precious.
3944: *
3945: * The not shared check consists of two
3946: * parts: (a) there are no objects that
3947: * shadow this object. (b) it is not the
3948: * object in any shared map entries (i.e.,
3949: * use_shared_copy is not set).
3950: *
3951: * The first check (a) means that we can't
3952: * steal pages from objects that are not
3953: * at the top of their shadow chains. This
3954: * should not be a frequent occurrence.
3955: *
3956: * Stealing wired pages requires telling the
3957: * pmap module to let go of them.
1.1.1.2 root 3958: *
1.1 root 3959: * NOTE: stealing clean pages from objects
3960: * whose mappings survive requires a call to
3961: * the pmap module. Maybe later.
3962: */
3963: m = copy->cpy_page_list[i];
3964: src_object = m->object;
3965: vm_object_lock(src_object);
3966:
3967: if (src_destroy &&
3968: src_object->temporary &&
3969: (!src_object->shadowed) &&
3970: (!src_object->use_shared_copy) &&
3971: !m->precious) {
3972: vm_offset_t page_vaddr;
1.1.1.2 root 3973:
1.1 root 3974: page_vaddr = src_start + (i * PAGE_SIZE);
3975: if (m->wire_count > 0) {
3976:
3977: assert(m->wire_count == 1);
3978: /*
3979: * In order to steal a wired
3980: * page, we have to unwire it
3981: * first. We do this inline
3982: * here because we have the page.
3983: *
3984: * Step 1: Unwire the map entry.
3985: * Also tell the pmap module
3986: * that this piece of the
3987: * pmap is pageable.
3988: */
3989: vm_object_unlock(src_object);
3990: if (page_vaddr >= unwire_end) {
3991: if (!vm_map_lookup_entry(src_map,
3992: page_vaddr, &src_entry))
3993: panic("vm_map_copyin_page_list: missing wired map entry");
3994:
3995: vm_map_clip_start(src_map, src_entry,
3996: page_vaddr);
3997: vm_map_clip_end(src_map, src_entry,
3998: src_start + src_size);
3999:
4000: assert(src_entry->wired_count > 0);
4001: src_entry->wired_count = 0;
4002: src_entry->user_wired_count = 0;
4003: unwire_end = src_entry->vme_end;
4004: pmap_pageable(vm_map_pmap(src_map),
4005: page_vaddr, unwire_end, TRUE);
4006: }
4007:
4008: /*
4009: * Step 2: Unwire the page.
4010: * pmap_remove handles this for us.
4011: */
4012: vm_object_lock(src_object);
4013: }
4014:
4015: /*
4016: * Don't need to remove the mapping;
4017: * vm_map_delete will handle it.
4018: *
4019: * Steal the page. Setting the wire count
4020: * to zero is vm_page_unwire without
4021: * activating the page.
4022: */
4023: vm_page_lock_queues();
4024: vm_page_remove(m);
4025: if (m->wire_count > 0) {
4026: m->wire_count = 0;
4027: vm_page_wire_count--;
4028: } else {
4029: VM_PAGE_QUEUES_REMOVE(m);
4030: }
4031: vm_page_unlock_queues();
4032: }
4033: else {
4034: /*
4035: * Have to copy this page. Have to
4036: * unlock the map while copying,
4037: * hence no further page stealing.
4038: * Hence just copy all the pages.
4039: * Unlock the map while copying;
4040: * This means no further page stealing.
4041: */
4042: vm_object_unlock(src_object);
4043: vm_map_unlock(src_map);
4044:
4045: vm_map_copy_steal_pages(copy);
4046:
4047: vm_map_lock(src_map);
4048: break;
4049: }
4050:
4051: vm_object_paging_end(src_object);
4052: vm_object_unlock(src_object);
4053: }
4054:
4055: /*
4056: * If the source should be destroyed, do it now, since the
4057: * copy was successful.
4058: */
4059:
4060: if (src_destroy) {
4061: (void) vm_map_delete(src_map, src_start, src_end);
4062: }
4063: }
4064: else {
4065: /*
4066: * !steal_pages leaves busy pages in the map.
4067: * This will cause src_destroy to hang. Use
4068: * a continuation to prevent this.
4069: */
4070: if (src_destroy && !vm_map_copy_has_cont(copy)) {
1.1.1.2 root 4071: cont_args = (vm_map_copyin_args_t)
1.1 root 4072: kalloc(sizeof(vm_map_copyin_args_data_t));
4073: vm_map_reference(src_map);
4074: cont_args->map = src_map;
4075: cont_args->src_addr = (vm_offset_t) 0;
4076: cont_args->src_len = (vm_size_t) 0;
4077: cont_args->destroy_addr = src_start;
4078: cont_args->destroy_len = src_end - src_start;
4079: cont_args->steal_pages = FALSE;
4080:
4081: copy->cpy_cont_args = (char *) cont_args;
4082: copy->cpy_cont = vm_map_copyin_page_list_cont;
4083: }
1.1.1.2 root 4084:
1.1 root 4085: }
4086:
4087: vm_map_unlock(src_map);
4088:
4089: *copy_result = copy;
4090: return(result);
4091:
4092: error:
4093: vm_map_unlock(src_map);
4094: vm_map_copy_discard(copy);
4095: return(result);
4096: }
4097:
4098: /*
4099: * vm_map_fork:
4100: *
4101: * Create and return a new map based on the old
4102: * map, according to the inheritance values on the
4103: * regions in that map.
4104: *
4105: * The source map must not be locked.
4106: */
4107: vm_map_t vm_map_fork(old_map)
4108: vm_map_t old_map;
4109: {
4110: vm_map_t new_map;
4111: register
4112: vm_map_entry_t old_entry;
4113: register
4114: vm_map_entry_t new_entry;
4115: pmap_t new_pmap = pmap_create((vm_size_t) 0);
4116: vm_size_t new_size = 0;
4117: vm_size_t entry_size;
4118: register
4119: vm_object_t object;
4120:
4121: vm_map_lock(old_map);
4122:
4123: new_map = vm_map_create(new_pmap,
4124: old_map->min_offset,
4125: old_map->max_offset,
4126: old_map->hdr.entries_pageable);
4127:
4128: for (
4129: old_entry = vm_map_first_entry(old_map);
4130: old_entry != vm_map_to_entry(old_map);
4131: ) {
4132: if (old_entry->is_sub_map)
4133: panic("vm_map_fork: encountered a submap");
4134:
4135: entry_size = (old_entry->vme_end - old_entry->vme_start);
4136:
4137: switch (old_entry->inheritance) {
4138: case VM_INHERIT_NONE:
4139: break;
4140:
4141: case VM_INHERIT_SHARE:
4142: /*
4143: * New sharing code. New map entry
4144: * references original object. Temporary
4145: * objects use asynchronous copy algorithm for
4146: * future copies. First make sure we have
4147: * the right object. If we need a shadow,
4148: * or someone else already has one, then
4149: * make a new shadow and share it.
4150: */
4151:
4152: object = old_entry->object.vm_object;
4153: if (object == VM_OBJECT_NULL) {
4154: object = vm_object_allocate(
4155: (vm_size_t)(old_entry->vme_end -
4156: old_entry->vme_start));
4157: old_entry->offset = 0;
4158: old_entry->object.vm_object = object;
4159: assert(!old_entry->needs_copy);
4160: }
4161: else if (old_entry->needs_copy || object->shadowed ||
4162: (object->temporary && !old_entry->is_shared &&
4163: object->size > (vm_size_t)(old_entry->vme_end -
4164: old_entry->vme_start))) {
4165:
4166: assert(object->temporary);
4167: assert(!(object->shadowed && old_entry->is_shared));
4168: vm_object_shadow(
4169: &old_entry->object.vm_object,
4170: &old_entry->offset,
4171: (vm_size_t) (old_entry->vme_end -
4172: old_entry->vme_start));
1.1.1.2 root 4173:
1.1 root 4174: /*
4175: * If we're making a shadow for other than
4176: * copy on write reasons, then we have
4177: * to remove write permission.
4178: */
4179:
4180: if (!old_entry->needs_copy &&
4181: (old_entry->protection & VM_PROT_WRITE)) {
4182: pmap_protect(vm_map_pmap(old_map),
4183: old_entry->vme_start,
4184: old_entry->vme_end,
4185: old_entry->protection &
4186: ~VM_PROT_WRITE);
4187: }
4188: old_entry->needs_copy = FALSE;
4189: object = old_entry->object.vm_object;
4190: }
4191:
4192: /*
4193: * Set use_shared_copy to indicate that
4194: * object must use shared (delayed) copy-on
4195: * write. This is ignored for permanent objects.
4196: * Bump the reference count for the new entry
4197: */
4198:
4199: vm_object_lock(object);
4200: object->use_shared_copy = TRUE;
4201: object->ref_count++;
4202: vm_object_unlock(object);
4203:
1.1.1.3 ! root 4204: new_entry = vm_map_entry_create(new_map);
! 4205:
1.1 root 4206: if (old_entry->projected_on != 0) {
4207: /*
4208: * If entry is projected buffer, clone the
4209: * entry exactly.
4210: */
4211:
4212: vm_map_entry_copy_full(new_entry, old_entry);
4213:
4214: } else {
4215: /*
4216: * Clone the entry, using object ref from above.
4217: * Mark both entries as shared.
4218: */
4219:
4220: vm_map_entry_copy(new_entry, old_entry);
4221: old_entry->is_shared = TRUE;
4222: new_entry->is_shared = TRUE;
4223: }
4224:
4225: /*
4226: * Insert the entry into the new map -- we
4227: * know we're inserting at the end of the new
4228: * map.
4229: */
4230:
4231: vm_map_entry_link(
4232: new_map,
4233: vm_map_last_entry(new_map),
4234: new_entry);
4235:
4236: /*
4237: * Update the physical map
4238: */
4239:
4240: pmap_copy(new_map->pmap, old_map->pmap,
4241: new_entry->vme_start,
4242: entry_size,
4243: old_entry->vme_start);
4244:
4245: new_size += entry_size;
4246: break;
4247:
4248: case VM_INHERIT_COPY:
4249: if (old_entry->wired_count == 0) {
4250: boolean_t src_needs_copy;
4251: boolean_t new_entry_needs_copy;
4252:
4253: new_entry = vm_map_entry_create(new_map);
4254: vm_map_entry_copy(new_entry, old_entry);
4255:
4256: if (vm_object_copy_temporary(
4257: &new_entry->object.vm_object,
4258: &new_entry->offset,
4259: &src_needs_copy,
4260: &new_entry_needs_copy)) {
4261:
4262: /*
4263: * Handle copy-on-write obligations
4264: */
4265:
4266: if (src_needs_copy && !old_entry->needs_copy) {
4267: vm_object_pmap_protect(
4268: old_entry->object.vm_object,
4269: old_entry->offset,
4270: entry_size,
4271: (old_entry->is_shared ?
4272: PMAP_NULL :
4273: old_map->pmap),
4274: old_entry->vme_start,
4275: old_entry->protection &
4276: ~VM_PROT_WRITE);
4277:
4278: old_entry->needs_copy = TRUE;
4279: }
4280:
4281: new_entry->needs_copy = new_entry_needs_copy;
4282:
4283: /*
4284: * Insert the entry at the end
4285: * of the map.
4286: */
4287:
4288: vm_map_entry_link(new_map,
4289: vm_map_last_entry(new_map),
4290: new_entry);
4291:
4292:
4293: new_size += entry_size;
4294: break;
4295: }
4296:
4297: vm_map_entry_dispose(new_map, new_entry);
4298: }
4299:
4300: /* INNER BLOCK (copy cannot be optimized) */ {
4301:
4302: vm_offset_t start = old_entry->vme_start;
4303: vm_map_copy_t copy;
4304: vm_map_entry_t last = vm_map_last_entry(new_map);
4305:
4306: vm_map_unlock(old_map);
4307: if (vm_map_copyin(old_map,
4308: start,
4309: entry_size,
4310: FALSE,
1.1.1.2 root 4311: ©)
1.1 root 4312: != KERN_SUCCESS) {
4313: vm_map_lock(old_map);
4314: if (!vm_map_lookup_entry(old_map, start, &last))
4315: last = last->vme_next;
4316: old_entry = last;
4317: /*
4318: * For some error returns, want to
4319: * skip to the next element.
4320: */
4321:
4322: continue;
4323: }
4324:
4325: /*
4326: * Insert the copy into the new map
4327: */
4328:
4329: vm_map_copy_insert(new_map, last, copy);
4330: new_size += entry_size;
4331:
4332: /*
4333: * Pick up the traversal at the end of
4334: * the copied region.
4335: */
4336:
4337: vm_map_lock(old_map);
4338: start += entry_size;
4339: if (!vm_map_lookup_entry(old_map, start, &last))
4340: last = last->vme_next;
4341: else
4342: vm_map_clip_start(old_map, last, start);
4343: old_entry = last;
4344:
4345: continue;
4346: /* INNER BLOCK (copy cannot be optimized) */ }
4347: }
4348: old_entry = old_entry->vme_next;
4349: }
4350:
4351: new_map->size = new_size;
4352: vm_map_unlock(old_map);
4353:
4354: return(new_map);
4355: }
4356:
4357: /*
4358: * vm_map_lookup:
4359: *
4360: * Finds the VM object, offset, and
4361: * protection for a given virtual address in the
4362: * specified map, assuming a page fault of the
4363: * type specified.
4364: *
4365: * Returns the (object, offset, protection) for
4366: * this address, whether it is wired down, and whether
4367: * this map has the only reference to the data in question.
4368: * In order to later verify this lookup, a "version"
4369: * is returned.
4370: *
4371: * The map should not be locked; it will not be
4372: * locked on exit. In order to guarantee the
4373: * existence of the returned object, it is returned
4374: * locked.
4375: *
4376: * If a lookup is requested with "write protection"
4377: * specified, the map may be changed to perform virtual
4378: * copying operations, although the data referenced will
4379: * remain the same.
4380: */
4381: kern_return_t vm_map_lookup(var_map, vaddr, fault_type, out_version,
4382: object, offset, out_prot, wired)
4383: vm_map_t *var_map; /* IN/OUT */
4384: register vm_offset_t vaddr;
4385: register vm_prot_t fault_type;
4386:
4387: vm_map_version_t *out_version; /* OUT */
4388: vm_object_t *object; /* OUT */
4389: vm_offset_t *offset; /* OUT */
4390: vm_prot_t *out_prot; /* OUT */
4391: boolean_t *wired; /* OUT */
4392: {
4393: register vm_map_entry_t entry;
4394: register vm_map_t map = *var_map;
4395: register vm_prot_t prot;
4396:
4397: RetryLookup: ;
4398:
4399: /*
4400: * Lookup the faulting address.
4401: */
4402:
4403: vm_map_lock_read(map);
4404:
4405: #define RETURN(why) \
4406: { \
4407: vm_map_unlock_read(map); \
4408: return(why); \
4409: }
4410:
4411: /*
4412: * If the map has an interesting hint, try it before calling
4413: * full blown lookup routine.
4414: */
4415:
4416: simple_lock(&map->hint_lock);
4417: entry = map->hint;
4418: simple_unlock(&map->hint_lock);
4419:
4420: if ((entry == vm_map_to_entry(map)) ||
4421: (vaddr < entry->vme_start) || (vaddr >= entry->vme_end)) {
4422: vm_map_entry_t tmp_entry;
4423:
4424: /*
4425: * Entry was either not a valid hint, or the vaddr
4426: * was not contained in the entry, so do a full lookup.
4427: */
4428: if (!vm_map_lookup_entry(map, vaddr, &tmp_entry))
4429: RETURN(KERN_INVALID_ADDRESS);
4430:
4431: entry = tmp_entry;
4432: }
4433:
4434: /*
4435: * Handle submaps.
4436: */
4437:
4438: if (entry->is_sub_map) {
4439: vm_map_t old_map = map;
4440:
4441: *var_map = map = entry->object.sub_map;
4442: vm_map_unlock_read(old_map);
4443: goto RetryLookup;
4444: }
1.1.1.2 root 4445:
1.1 root 4446: /*
4447: * Check whether this task is allowed to have
4448: * this page.
4449: */
4450:
4451: prot = entry->protection;
4452:
1.1.1.3 ! root 4453: if ((fault_type & (prot)) != fault_type) {
1.1 root 4454: if ((prot & VM_PROT_NOTIFY) && (fault_type & VM_PROT_WRITE)) {
4455: RETURN(KERN_WRITE_PROTECTION_FAILURE);
4456: } else {
4457: RETURN(KERN_PROTECTION_FAILURE);
4458: }
1.1.1.3 ! root 4459: }
1.1 root 4460:
4461: /*
4462: * If this page is not pageable, we have to get
4463: * it for all possible accesses.
4464: */
4465:
1.1.1.3 ! root 4466: if ((*wired = (entry->wired_count != 0)))
1.1 root 4467: prot = fault_type = entry->protection;
4468:
4469: /*
4470: * If the entry was copy-on-write, we either ...
4471: */
4472:
4473: if (entry->needs_copy) {
4474: /*
4475: * If we want to write the page, we may as well
4476: * handle that now since we've got the map locked.
4477: *
4478: * If we don't need to write the page, we just
4479: * demote the permissions allowed.
4480: */
4481:
4482: if (fault_type & VM_PROT_WRITE) {
4483: /*
4484: * Make a new object, and place it in the
4485: * object chain. Note that no new references
4486: * have appeared -- one just moved from the
4487: * map to the new object.
4488: */
4489:
4490: if (vm_map_lock_read_to_write(map)) {
4491: goto RetryLookup;
4492: }
4493: map->timestamp++;
4494:
4495: vm_object_shadow(
4496: &entry->object.vm_object,
4497: &entry->offset,
4498: (vm_size_t) (entry->vme_end - entry->vme_start));
1.1.1.2 root 4499:
1.1 root 4500: entry->needs_copy = FALSE;
1.1.1.2 root 4501:
1.1 root 4502: vm_map_lock_write_to_read(map);
4503: }
4504: else {
4505: /*
4506: * We're attempting to read a copy-on-write
4507: * page -- don't allow writes.
4508: */
4509:
4510: prot &= (~VM_PROT_WRITE);
4511: }
4512: }
4513:
4514: /*
4515: * Create an object if necessary.
4516: */
4517: if (entry->object.vm_object == VM_OBJECT_NULL) {
4518:
4519: if (vm_map_lock_read_to_write(map)) {
4520: goto RetryLookup;
4521: }
4522:
4523: entry->object.vm_object = vm_object_allocate(
4524: (vm_size_t)(entry->vme_end - entry->vme_start));
4525: entry->offset = 0;
4526: vm_map_lock_write_to_read(map);
4527: }
4528:
4529: /*
4530: * Return the object/offset from this entry. If the entry
4531: * was copy-on-write or empty, it has been fixed up. Also
4532: * return the protection.
4533: */
4534:
4535: *offset = (vaddr - entry->vme_start) + entry->offset;
4536: *object = entry->object.vm_object;
4537: *out_prot = prot;
4538:
4539: /*
4540: * Lock the object to prevent it from disappearing
4541: */
4542:
4543: vm_object_lock(*object);
4544:
4545: /*
4546: * Save the version number and unlock the map.
4547: */
4548:
4549: out_version->main_timestamp = map->timestamp;
4550:
4551: RETURN(KERN_SUCCESS);
1.1.1.2 root 4552:
1.1 root 4553: #undef RETURN
4554: }
4555:
4556: /*
4557: * vm_map_verify:
4558: *
4559: * Verifies that the map in question has not changed
4560: * since the given version. If successful, the map
4561: * will not change until vm_map_verify_done() is called.
4562: */
4563: boolean_t vm_map_verify(map, version)
4564: register
4565: vm_map_t map;
4566: register
4567: vm_map_version_t *version; /* REF */
4568: {
4569: boolean_t result;
4570:
4571: vm_map_lock_read(map);
4572: result = (map->timestamp == version->main_timestamp);
4573:
4574: if (!result)
4575: vm_map_unlock_read(map);
4576:
4577: return(result);
4578: }
4579:
4580: /*
4581: * vm_map_verify_done:
4582: *
4583: * Releases locks acquired by a vm_map_verify.
4584: *
4585: * This is now a macro in vm/vm_map.h. It does a
4586: * vm_map_unlock_read on the map.
4587: */
4588:
4589: /*
4590: * vm_region:
4591: *
4592: * User call to obtain information about a region in
4593: * a task's address map.
4594: */
4595:
4596: kern_return_t vm_region(map, address, size,
4597: protection, max_protection,
4598: inheritance, is_shared,
4599: object_name, offset_in_object)
4600: vm_map_t map;
4601: vm_offset_t *address; /* IN/OUT */
4602: vm_size_t *size; /* OUT */
4603: vm_prot_t *protection; /* OUT */
4604: vm_prot_t *max_protection; /* OUT */
4605: vm_inherit_t *inheritance; /* OUT */
4606: boolean_t *is_shared; /* OUT */
4607: ipc_port_t *object_name; /* OUT */
4608: vm_offset_t *offset_in_object; /* OUT */
4609: {
4610: vm_map_entry_t tmp_entry;
4611: register
4612: vm_map_entry_t entry;
4613: register
4614: vm_offset_t tmp_offset;
4615: vm_offset_t start;
4616:
4617: if (map == VM_MAP_NULL)
4618: return(KERN_INVALID_ARGUMENT);
4619:
4620: start = *address;
4621:
4622: vm_map_lock_read(map);
4623: if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
4624: if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
4625: vm_map_unlock_read(map);
4626: return(KERN_NO_SPACE);
4627: }
4628: } else {
4629: entry = tmp_entry;
4630: }
4631:
4632: start = entry->vme_start;
4633: *protection = entry->protection;
4634: *max_protection = entry->max_protection;
4635: *inheritance = entry->inheritance;
4636: *address = start;
4637: *size = (entry->vme_end - start);
4638:
4639: tmp_offset = entry->offset;
4640:
4641:
4642: if (entry->is_sub_map) {
4643: *is_shared = FALSE;
4644: *object_name = IP_NULL;
4645: *offset_in_object = tmp_offset;
4646: } else {
4647: *is_shared = entry->is_shared;
4648: *object_name = vm_object_name(entry->object.vm_object);
4649: *offset_in_object = tmp_offset;
4650: }
4651:
4652: vm_map_unlock_read(map);
4653:
4654: return(KERN_SUCCESS);
4655: }
4656:
4657: /*
4658: * Routine: vm_map_simplify
4659: *
4660: * Description:
4661: * Attempt to simplify the map representation in
4662: * the vicinity of the given starting address.
4663: * Note:
4664: * This routine is intended primarily to keep the
4665: * kernel maps more compact -- they generally don't
4666: * benefit from the "expand a map entry" technology
4667: * at allocation time because the adjacent entry
4668: * is often wired down.
4669: */
4670: void vm_map_simplify(map, start)
4671: vm_map_t map;
4672: vm_offset_t start;
4673: {
4674: vm_map_entry_t this_entry;
4675: vm_map_entry_t prev_entry;
4676:
4677: vm_map_lock(map);
4678: if (
4679: (vm_map_lookup_entry(map, start, &this_entry)) &&
4680: ((prev_entry = this_entry->vme_prev) != vm_map_to_entry(map)) &&
4681:
4682: (prev_entry->vme_end == start) &&
4683:
4684: (prev_entry->is_shared == FALSE) &&
4685: (prev_entry->is_sub_map == FALSE) &&
4686:
4687: (this_entry->is_shared == FALSE) &&
4688: (this_entry->is_sub_map == FALSE) &&
4689:
4690: (prev_entry->inheritance == this_entry->inheritance) &&
4691: (prev_entry->protection == this_entry->protection) &&
4692: (prev_entry->max_protection == this_entry->max_protection) &&
4693: (prev_entry->wired_count == this_entry->wired_count) &&
4694: (prev_entry->user_wired_count == this_entry->user_wired_count) &&
4695:
4696: (prev_entry->needs_copy == this_entry->needs_copy) &&
4697:
4698: (prev_entry->object.vm_object == this_entry->object.vm_object) &&
4699: ((prev_entry->offset + (prev_entry->vme_end - prev_entry->vme_start))
4700: == this_entry->offset) &&
4701: (prev_entry->projected_on == 0) &&
1.1.1.2 root 4702: (this_entry->projected_on == 0)
1.1 root 4703: ) {
4704: if (map->first_free == this_entry)
4705: map->first_free = prev_entry;
4706:
4707: SAVE_HINT(map, prev_entry);
4708: vm_map_entry_unlink(map, this_entry);
4709: prev_entry->vme_end = this_entry->vme_end;
4710: vm_object_deallocate(this_entry->object.vm_object);
4711: vm_map_entry_dispose(map, this_entry);
4712: }
4713: vm_map_unlock(map);
4714: }
4715:
4716:
4717: /*
4718: * Routine: vm_map_machine_attribute
4719: * Purpose:
4720: * Provide machine-specific attributes to mappings,
4721: * such as cachability etc. for machines that provide
4722: * them. NUMA architectures and machines with big/strange
4723: * caches will use this.
4724: * Note:
4725: * Responsibilities for locking and checking are handled here,
4726: * everything else in the pmap module. If any non-volatile
4727: * information must be kept, the pmap module should handle
4728: * it itself. [This assumes that attributes do not
4729: * need to be inherited, which seems ok to me]
4730: */
4731: kern_return_t vm_map_machine_attribute(map, address, size, attribute, value)
4732: vm_map_t map;
4733: vm_offset_t address;
4734: vm_size_t size;
4735: vm_machine_attribute_t attribute;
4736: vm_machine_attribute_val_t* value; /* IN/OUT */
4737: {
4738: kern_return_t ret;
4739:
4740: if (address < vm_map_min(map) ||
4741: (address + size) > vm_map_max(map))
4742: return KERN_INVALID_ARGUMENT;
4743:
4744: vm_map_lock(map);
4745:
4746: ret = pmap_attribute(map->pmap, address, size, attribute, value);
4747:
4748: vm_map_unlock(map);
4749:
4750: return ret;
4751: }
4752:
4753:
4754: #if MACH_KDB
4755:
4756: #define printf kdbprintf
4757:
4758: /*
4759: * vm_map_print: [ debug ]
4760: */
4761: void vm_map_print(map)
4762: register vm_map_t map;
4763: {
4764: register vm_map_entry_t entry;
4765:
4766: iprintf("Task map 0x%X: pmap=0x%X,",
4767: (vm_offset_t) map, (vm_offset_t) (map->pmap));
4768: printf("ref=%d,nentries=%d,", map->ref_count, map->hdr.nentries);
4769: printf("version=%d\n", map->timestamp);
4770: indent += 2;
4771: for (entry = vm_map_first_entry(map);
4772: entry != vm_map_to_entry(map);
4773: entry = entry->vme_next) {
4774: static char *inheritance_name[3] = { "share", "copy", "none"};
4775:
4776: iprintf("map entry 0x%X: ", (vm_offset_t) entry);
4777: printf("start=0x%X, end=0x%X, ",
4778: (vm_offset_t) entry->vme_start, (vm_offset_t) entry->vme_end);
4779: printf("prot=%X/%X/%s, ",
4780: entry->protection,
4781: entry->max_protection,
4782: inheritance_name[entry->inheritance]);
4783: if (entry->wired_count != 0) {
4784: printf("wired(");
4785: if (entry->user_wired_count != 0)
4786: printf("u");
4787: if (entry->wired_count >
4788: ((entry->user_wired_count == 0) ? 0 : 1))
4789: printf("k");
4790: printf(") ");
4791: }
4792: if (entry->in_transition) {
4793: printf("in transition");
4794: if (entry->needs_wakeup)
4795: printf("(wake request)");
4796: printf(", ");
4797: }
4798: if (entry->is_sub_map) {
4799: printf("submap=0x%X, offset=0x%X\n",
4800: (vm_offset_t) entry->object.sub_map,
4801: (vm_offset_t) entry->offset);
4802: } else {
4803: printf("object=0x%X, offset=0x%X",
4804: (vm_offset_t) entry->object.vm_object,
4805: (vm_offset_t) entry->offset);
4806: if (entry->is_shared)
4807: printf(", shared");
4808: if (entry->needs_copy)
4809: printf(", copy needed");
4810: printf("\n");
4811:
4812: if ((entry->vme_prev == vm_map_to_entry(map)) ||
4813: (entry->vme_prev->object.vm_object != entry->object.vm_object)) {
4814: indent += 2;
4815: vm_object_print(entry->object.vm_object);
4816: indent -= 2;
4817: }
4818: }
4819: }
4820: indent -= 2;
4821: }
4822:
4823: /*
4824: * Routine: vm_map_copy_print
4825: * Purpose:
4826: * Pretty-print a copy object for ddb.
4827: */
4828:
4829: void vm_map_copy_print(copy)
4830: vm_map_copy_t copy;
4831: {
4832: int i, npages;
4833:
4834: printf("copy object 0x%x\n", copy);
4835:
4836: indent += 2;
4837:
4838: iprintf("type=%d", copy->type);
4839: switch (copy->type) {
4840: case VM_MAP_COPY_ENTRY_LIST:
4841: printf("[entry_list]");
4842: break;
1.1.1.2 root 4843:
1.1 root 4844: case VM_MAP_COPY_OBJECT:
4845: printf("[object]");
4846: break;
1.1.1.2 root 4847:
1.1 root 4848: case VM_MAP_COPY_PAGE_LIST:
4849: printf("[page_list]");
4850: break;
4851:
4852: default:
4853: printf("[bad type]");
4854: break;
4855: }
4856: printf(", offset=0x%x", copy->offset);
4857: printf(", size=0x%x\n", copy->size);
4858:
4859: switch (copy->type) {
4860: case VM_MAP_COPY_ENTRY_LIST:
4861: /* XXX add stuff here */
4862: break;
4863:
4864: case VM_MAP_COPY_OBJECT:
4865: iprintf("object=0x%x\n", copy->cpy_object);
4866: break;
4867:
4868: case VM_MAP_COPY_PAGE_LIST:
4869: iprintf("npages=%d", copy->cpy_npages);
4870: printf(", cont=%x", copy->cpy_cont);
4871: printf(", cont_args=%x\n", copy->cpy_cont_args);
4872: if (copy->cpy_npages < 0) {
4873: npages = 0;
4874: } else if (copy->cpy_npages > VM_MAP_COPY_PAGE_LIST_MAX) {
4875: npages = VM_MAP_COPY_PAGE_LIST_MAX;
4876: } else {
4877: npages = copy->cpy_npages;
4878: }
4879: iprintf("copy->cpy_page_list[0..%d] = {", npages);
4880: for (i = 0; i < npages - 1; i++) {
4881: printf("0x%x, ", copy->cpy_page_list[i]);
4882: }
4883: if (npages > 0) {
4884: printf("0x%x", copy->cpy_page_list[npages - 1]);
4885: }
4886: printf("}\n");
4887: break;
4888: }
4889:
4890: indent -=2;
4891: }
1.1.1.2 root 4892: #endif /* MACH_KDB */
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.